Sync eng/common directory with azure-sdk-tools repository (#377)

This commit is contained in:
Azure SDK Bot 2020-08-17 15:09:40 -07:00 committed by GitHub
parent 24959f30da
commit 9067d8ac5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 363 additions and 21 deletions

View File

@ -9,11 +9,14 @@ parameters:
TargetDocRepoName: ''
TargetDocRepoOwner: ''
PRBranchName: 'smoke-test-rdme'
SourceBranchName: 'smoke-test'
ArtifactName: ''
Language: ''
DocRepoDestinationPath: '' #usually docs-ref-services/
CIConfigs: '[]'
GHReviewersVariable: ''
GHTeamReviewersVariable: '' # externally set, as eng-common does not have the identity-resolver. Run as pre-step
OnboardingBranch: ''
steps:
- pwsh: |
@ -22,8 +25,8 @@ steps:
try {
Push-Location ${{ parameters.WorkingDirectory }}/repo
Write-Host "git checkout smoke-test"
git checkout smoke-test
Write-Host "git checkout ${{ parameters.SourceBranchName }}"
git checkout ${{ parameters.SourceBranchName }}
} finally {
Pop-Location
}
@ -48,15 +51,73 @@ steps:
env:
GH_TOKEN: $(azuresdk-github-pat)
- task: PowerShell@2
displayName: 'Update Docs.MS CI Targeted Packages'
condition: and(succeededOrFailed(), eq('${{ parameters.OnboardingBranch }}',''))
inputs:
targetType: filePath
filePath: ${{ parameters.ScriptDirectory }}/update-docs-ci.ps1
arguments: >
-ArtifactLocation ${{ parameters.ArtifactLocation }}
-WorkDirectory "${{ parameters.WorkingDirectory }}"
-RepoId ${{ parameters.RepoId }}
-Repository ${{ parameters.PackageRepository }}
-ReleaseSHA ${{ parameters.ReleaseSha }}
-CIRepository "${{ parameters.WorkingDirectory }}/repo"
-Configs "${{ parameters.CIConfigs }}"
pwsh: true
env:
GH_TOKEN: $(azuresdk-github-pat)
- template: /eng/common/pipelines/templates/steps/create-pull-request.yml
parameters:
RepoName: ${{ parameters.TargetDocRepoName }}
RepoOwner: ${{ parameters.TargetDocRepoOwner }}
PRBranchName: ${{ parameters.PRBranchName }}
CommitMsg: "Update readme content for ${{ parameters.ArtifactName }}"
PRTitle: "Docs.MS Readme Update."
BaseBranchName: smoke-test
CommitMsg: "Update docs metadata and targeting for release of ${{ parameters.ArtifactName }}"
PRTitle: "Docs.MS Release Updates for ${{ parameters.ArtifactName }}"
BaseBranchName: ${{ parameters.SourceBranchName }}
WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo
ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }}
GHReviewersVariable: ${{ parameters.GHReviewersVariable }}
GHTeamReviewersVariable: ${{ parameters.GHTeamReviewersVariable }}
- ${{if ne( parameters['OnboardingBranch'], '')}}:
- pwsh: |
Push-Location ${{ parameters.WorkingDirectory }}/repo
git reset --hard HEAD
git remote rm azure-sdk-fork
git checkout ${{ parameters.OnboardingBranch}}
displayName: Reset Docs Repo, Checkout Onboarding Branch
ignoreLASTEXITCODE: false
- task: PowerShell@2
displayName: 'Update Docs.MS CI Targeted Packages'
inputs:
targetType: filePath
filePath: ${{ parameters.ScriptDirectory }}/update-docs-ci.ps1
arguments: >
-ArtifactLocation ${{ parameters.ArtifactLocation }}
-WorkDirectory "${{ parameters.WorkingDirectory }}"
-RepoId ${{ parameters.RepoId }}
-Repository ${{ parameters.PackageRepository }}
-ReleaseSHA ${{ parameters.ReleaseSha }}
-CIRepository "${{ parameters.WorkingDirectory }}/repo"
-Configs "${{ parameters.CIConfigs }}"
pwsh: true
env:
GH_TOKEN: $(azuresdk-github-pat)
- template: /eng/common/pipelines/templates/steps/create-pull-request.yml
parameters:
RepoName: ${{ parameters.TargetDocRepoName }}
RepoOwner: ${{ parameters.TargetDocRepoOwner }}
PRBranchName: ${{ parameters.PRBranchName }}-ci
CommitMsg: "CI Update for release of ${{ parameters.ArtifactName }}"
PRTitle: "Docs.MS CI Updates for ${{ parameters.ArtifactName }}"
BaseBranchName: ${{ parameters.OnboardingBranch }}
WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo
ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }}
GHReviewersVariable: ${{ parameters.GHReviewersVariable }}
GHTeamReviewersVariable: ${{ parameters.GHTeamReviewersVariable }}

View File

@ -11,8 +11,8 @@ param (
[string] $baseUrl = "",
# path to the root of the site for resolving rooted relative links, defaults to host root for http and file directory for local files
[string] $rootUrl = "",
# list of http status codes count as broken links. Defaults to 404.
[array] $errorStatusCodes = @(404),
# list of http status codes count as broken links. Defaults to 400, 401, 404, SocketError.HostNotFound = 11001, SocketError.NoData = 11004
[array] $errorStatusCodes = @(400, 401, 404, 11001, 11004),
# flag to allow resolving relative paths or not
[bool] $resolveRelativeLinks = $true
)
@ -145,6 +145,11 @@ function CheckLink ([System.Uri]$linkUri)
catch {
$statusCode = $_.Exception.Response.StatusCode.value__
if(!$statusCode) {
# Try to pull the error code from any inner SocketException we might hit
$statusCode = $_.Exception.InnerException.ErrorCode
}
if ($statusCode -in $errorStatusCodes) {
LogWarning "[$statusCode] broken link $linkUri"
$script:badLinks += $linkUri

View File

@ -45,11 +45,11 @@ catch {
$userReviewers = @($resp.users | % { return $_.login })
$teamReviewers = @($resp.teams | % { return $_.slug })
if (!$usersReviewers) { $modifiedUserReviewers = @() } else { $modifiedUserReviewers = $usersReviewers.Clone() }
$modifiedUserReviewers += ($modifiedUserReviewers | ? { !$usersReviews.Contains($_) })
if (!$userReviewers) { $modifiedUserReviewers = @() } else { $modifiedUserReviewers = $userReviewers.Clone() }
$modifiedUserReviewers += ($userAdditions | ? { !$modifiedUserReviewers.Contains($_) })
if ($teamReviewers) { $modifiedTeamReviewers = @() } else { $modifiedTeamReviewers = $teamReviewers.Clone() }
$modifiedTeamReviewers += ($modifiedUserReviewers | ? { !$teamReviewers.Contains($_) })
$modifiedTeamReviewers += ($teamAdditions | ? { !$modifiedTeamReviewers.Contains($_) })
$detectedUserDiffs = Compare-Object -ReferenceObject $userReviewers -DifferenceObject $modifiedUserReviewers
$detectedTeamDiffs = Compare-Object -ReferenceObject $teamReviewers -DifferenceObject $modifiedTeamReviewers
@ -65,6 +65,7 @@ if ($detectedUserDiffs -or $detectedTeamDiffs) {
$postResp = $postResp | ConvertTo-Json
try {
Write-Host $postResp
$resp = Invoke-RestMethod -Method Post -Headers $headers -Body $postResp -Uri $uri -MaximumRetryCount 3
$resp | Write-Verbose
}

View File

@ -67,6 +67,7 @@ function ParseMavenPackage($pkg, $workingDirectory) {
return New-Object PSObject -Property @{
PackageId = $pkgId
GroupId = $groupId
PackageVersion = $pkgVersion
Deployable = $forceCreate -or !(IsMavenPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion -groupId $groupId.Replace(".", "/"))
ReleaseNotes = $releaseNotes
@ -219,7 +220,7 @@ function IsNugetPackageVersionPublished($pkgId, $pkgVersion) {
$nugetUri = "https://api.nuget.org/v3-flatcontainer/$($pkgId.ToLowerInvariant())/index.json"
try {
$nugetVersions = Invoke-RestMethod -MaximumRetryCount 3 -uri $nugetUri -Method "GET"
$nugetVersions = Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -uri $nugetUri -Method "GET"
return $nugetVersions.versions.Contains($pkgVersion)
}
@ -452,9 +453,11 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
$pkgList += New-Object PSObject -Property @{
PackageId = $parsedPackage.PackageId
PackageVersion = $parsedPackage.PackageVersion
GroupId = $parsedPackage.GroupId
Tag = $tag
ReleaseNotes = $parsedPackage.ReleaseNotes
ReadmeContent = $parsedPackage.ReadmeContent
IsPrerelease = [AzureEngSemanticVersion]::ParseVersionString($parsedPackage.PackageVersion).IsPrerelease
}
}
catch {

View File

@ -2,8 +2,9 @@ param (
$TargetDirectory, # should be in relative form from root of repo. EG: sdk/servicebus
$RootDirectory # ideally $(Build.SourcesDirectory)
)
$target = $TargetDirectory.ToLower().Trim("/")
$codeOwnersLocation = Join-Path $RootDirectory -ChildPath ".github/CODEOWNERS"
$ownedFolders = @{}
if (!(Test-Path $codeOwnersLocation)) {
Write-Host "Unable to find CODEOWNERS file in target directory $RootDirectory"
@ -12,29 +13,27 @@ if (!(Test-Path $codeOwnersLocation)) {
$codeOwnersContent = Get-Content $codeOwnersLocation
$ownedFolders = @{}
foreach ($contentLine in $codeOwnersContent) {
if (-not $contentLine.StartsWith("#") -and $contentLine){
$splitLine = $contentLine -split "\s+"
# CODEOWNERS file can also have labels present after the owner aliases
# gh aliases start with @ in codeowners. don't pass on to API calls
$ownedFolders[$splitLine[0].ToLower()] = ($splitLine[1..$($splitLine.Length)] `
$ownedFolders[$splitLine[0].ToLower().Trim("/")] = ($splitLine[1..$($splitLine.Length)] `
| ? { $_.StartsWith("@") } `
| % { return $_.substring(1) }) -join ","
}
}
$results = $ownedFolders[$TargetDirectory.ToLower()]
$results = $ownedFolders[$target]
if ($results) {
Write-Host "Discovered code owners for path $TargetDirectory are $results."
Write-Host "Found a folder $results to match $target"
return $results
}
else {
Write-Host "Unable to match path $TargetDirectory in CODEOWNERS file located at $codeOwnersLocation."
Write-Host $ownedFolders | ConvertTo-Json
Write-Host "Unable to match path $target in CODEOWNERS file located at $codeOwnersLocation."
Write-Host ($ownedFolders | ConvertTo-Json)
return ""
}

View File

@ -0,0 +1,275 @@
#Requires -Version 6.0
# This script is intended to update docs.ms CI configuration (currently supports Java, Python, C#, JS)
# as part of the azure-sdk release. For details on calling, check `archtype-<language>-release` in each azure-sdk
# repository.
# Where possible, this script adds as few changes as possible to the target config. We only
# specifically mark a version for Python Preview and Java. This script is intended to be invoked
# multiple times. Once for each moniker. Currently only supports "latest" and "preview" artifact selection however.
param (
[Parameter(Mandatory = $true)]
$ArtifactLocation, # the root of the artifact folder. DevOps $(System.ArtifactsDirectory)
[Parameter(Mandatory = $true)]
$WorkDirectory, # a clean folder that we can work in
[Parameter(Mandatory = $true)]
$ReleaseSHA, # the SHA for the artifacts. DevOps: $(Release.Artifacts.<artifactAlias>.SourceVersion) or $(Build.SourceVersion)
[Parameter(Mandatory = $true)]
$RepoId, # full repo id. EG azure/azure-sdk-for-net DevOps: $(Build.Repository.Id). Used as a part of VerifyPackages
[Parameter(Mandatory = $true)]
[ValidateSet("Nuget","NPM","PyPI","Maven")]
$Repository, # EG: "Maven", "PyPI", "NPM"
[Parameter(Mandatory = $true)]
$CIRepository,
[Parameter(Mandatory = $true)]
$Configs
)
# import artifact parsing and semver handling
. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1)
. (Join-Path $PSScriptRoot SemVer.ps1)
# Updates a python CI configuration json.
# For "latest", the version attribute is cleared, as default behavior is to pull latest "non-preview".
# For "preview", we update to >= the target releasing package version.
function UpdateParamsJsonPython($pkgs, $ciRepo, $locationInDocRepo){
$pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo)
if (-not (Test-Path $pkgJsonLoc)) {
Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting."
exit(1)
}
$allJson = Get-Content $pkgJsonLoc | ConvertFrom-Json
$visibleInCI = @{}
for ($i=0; $i -lt $allJson.packages.Length; $i++) {
$pkgDef = $allJson.packages[$i]
if ($pkgDef.package_info.name) {
$visibleInCI[$pkgDef.package_info.name] = $i
}
}
foreach ($releasingPkg in $pkgs) {
if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) {
$packagesIndex = $visibleInCI[$releasingPkg.PackageId]
$existingPackageDef = $allJson.packages[$packagesIndex]
if ($releasingPkg.IsPrerelease) {
if (-not $existingPackageDef.package_info.version) {
$existingPackageDef.package_info | Add-Member -NotePropertyName version -NotePropertyValue ""
}
$existingPackageDef.package_info.version = ">=$($releasingPkg.PackageVersion)"
}
else {
if ($def.version) {
$def.PSObject.Properties.Remove('version')
}
}
}
else {
$newItem = New-Object PSObject -Property @{
package_info = New-Object PSObject -Property @{
prefer_source_distribution = "true"
install_type = "pypi"
name=$releasingPkg.PackageId
}
excludePath = @("test*","example*","sample*","doc*")
}
$allJson.packages += $newItem
}
}
$jsonContent = $allJson | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " }
Set-Content -Path $pkgJsonLoc -Value $jsonContent
}
# Updates a js CI configuration json.
# For "latest", we simply set a target package name
# For "preview", we add @next to the target package name
function UpdateParamsJsonJS($pkgs, $ciRepo, $locationInDocRepo){
$pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo)
if (-not (Test-Path $pkgJsonLoc)) {
Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting."
exit(1)
}
$allJson = Get-Content $pkgJsonLoc | ConvertFrom-Json
$visibleInCI = @{}
for ($i=0; $i -lt $allJson.npm_package_sources.Length; $i++) {
$pkgDef = $allJson.npm_package_sources[$i]
$accessor = ($pkgDef.name).Replace("`@next", "")
$visibleInCI[$accessor] = $i
}
foreach ($releasingPkg in $pkgs) {
$name = $releasingPkg.PackageId
if ($releasingPkg.IsPrerelease) {
$name += "`@next"
}
if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) {
$packagesIndex = $visibleInCI[$releasingPkg.PackageId]
$existingPackageDef = $allJson.npm_package_sources[$packagesIndex]
$existingPackageDef.name = $name
}
else {
$newItem = New-Object PSObject -Property @{
name = $name
}
if ($newItem) { $allJson.npm_package_sources += $newItem }
}
}
$jsonContent = $allJson | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " }
Set-Content -Path $pkgJsonLoc -Value $jsonContent
}
# details on CSV schema can be found here
# https://review.docs.microsoft.com/en-us/help/onboard/admin/reference/dotnet/documenting-nuget?branch=master#set-up-the-ci-job
function UpdateCSVBasedCI($pkgs, $ciRepo, $locationInDocRepo){
$csvLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo)
if (-not (Test-Path $csvLoc)) {
Write-Error "Unable to locate package csv at location $csvLoc, exiting."
exit(1)
}
$allCSVRows = Get-Content $csvLoc
$visibleInCI = @{}
# first pull what's already available
for ($i=0; $i -lt $allCSVRows.Length; $i++) {
$pkgDef = $allCSVRows[$i]
# get rid of the modifiers to get just the package id
$id = $pkgDef.split(",")[1] -replace "\[.*?\]", ""
$visibleInCI[$id] = $i
}
foreach ($releasingPkg in $pkgs) {
$installModifiers = "tfm=netstandard2.0"
if ($releasingPkg.IsPrerelease) {
$installModifiers += ";isPrerelease=true"
}
$lineId = $releasingPkg.PackageId.Replace(".","").ToLower()
if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) {
$packagesIndex = $visibleInCI[$releasingPkg.PackageId]
$allCSVRows[$packagesIndex] = "$($lineId),[$installModifiers]$($releasingPkg.PackageId)"
}
else {
$newItem = "$($lineId),[$installModifiers]$($releasingPkg.PackageId)"
$allCSVRows += ($newItem)
}
}
Set-Content -Path $csvLoc -Value $allCSVRows
}
# a "package.json configures target packages for all the monikers in a Repository, it also has a slightly different
# schema than the moniker-specific json config that is seen in python and js
function UpdatePackageJson($pkgs, $ciRepo, $locationInDocRepo, $monikerId){
$pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo)
if (-not (Test-Path $pkgJsonLoc)) {
Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting."
exit(1)
}
$allJsonData = Get-Content $pkgJsonLoc | ConvertFrom-Json
$visibleInCI = @{}
for ($i=0; $i -lt $allJsonData[$monikerId].packages.Length; $i++) {
$pkgDef = $allJsonData[$monikerId].packages[$i]
$visibleInCI[$pkgDef.packageArtifactId] = $i
}
foreach ($releasingPkg in $pkgs) {
if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) {
$packagesIndex = $visibleInCI[$releasingPkg.PackageId]
$existingPackageDef = $allJsonData[$monikerId].packages[$packagesIndex]
$existingPackageDef.packageVersion = $releasingPkg.PackageVersion
}
else {
$newItem = New-Object PSObject -Property @{
packageDownloadUrl = "https://repo1.maven.org/maven2"
packageGroupId = $releasingPkg.GroupId
packageArtifactId = $releasingPkg.PackageId
packageVersion = $releasingPkg.PackageVersion
inputPath = @()
excludePath = @()
}
$allJsonData[$monikerId].packages += $newItem
}
}
$jsonContent = $allJsonData | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " }
Set-Content -Path $pkgJsonLoc -Value $jsonContent
}
$targets = ($Configs | ConvertFrom-Json).targets
#{
# path_to_config:
# mode:
# monikerid
#}
$apiUrl = "https://api.github.com/repos/$repoId"
$pkgs = VerifyPackages -pkgRepository $Repository `
-artifactLocation $ArtifactLocation `
-workingDirectory $WorkDirectory `
-apiUrl $apiUrl `
-continueOnError $True
foreach ($config in $targets) {
if ($config.mode -eq "Preview") { $includePreview = $true } else { $includePreview = $false }
$pkgsFiltered = $pkgs | ? { $_.IsPrerelease -eq $includePreview}
if ($pkgs) {
Write-Host "Given the visible artifacts, CI updates against $($config.path_to_config) will be processed for the following packages."
Write-Host ($pkgsFiltered | % { $_.PackageId + " " + $_.PackageVersion })
switch ($Repository) {
"Nuget" {
UpdateCSVBasedCI -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config
break
}
"NPM" {
UpdateParamsJsonJS -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config
break
}
"PyPI" {
UpdateParamsJsonPython -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config
break
}
"Maven" {
UpdatePackageJson -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config -monikerId $config.monikerid
break
}
default {
Write-Host "Unrecognized target: $Repository"
exit(1)
}
}
}
}

View File

@ -14,8 +14,6 @@ param (
$DocRepoContentLocation = "docs-ref-services/" # within the doc repo, where does our readme go?
)
# import artifact parsing and semver handling
. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1)
. (Join-Path $PSScriptRoot SemVer.ps1)