From 9067d8ac5d298ffafed33bc372032f4c7f2dd489 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Mon, 17 Aug 2020 15:09:40 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools repository (#377) --- .../templates/steps/docs-metadata-release.yml | 71 ++++- eng/common/scripts/Verify-Links.ps1 | 9 +- .../scripts/add-pullrequest-reviewers.ps1 | 7 +- .../scripts/artifact-metadata-parsing.ps1 | 5 +- eng/common/scripts/get-codeowners.ps1 | 15 +- eng/common/scripts/update-docs-ci.ps1 | 275 ++++++++++++++++++ eng/common/scripts/update-docs-metadata.ps1 | 2 - 7 files changed, 363 insertions(+), 21 deletions(-) create mode 100644 eng/common/scripts/update-docs-ci.ps1 diff --git a/eng/common/pipelines/templates/steps/docs-metadata-release.yml b/eng/common/pipelines/templates/steps/docs-metadata-release.yml index a9ff8c4e1..d75fed40a 100644 --- a/eng/common/pipelines/templates/steps/docs-metadata-release.yml +++ b/eng/common/pipelines/templates/steps/docs-metadata-release.yml @@ -9,11 +9,14 @@ parameters: TargetDocRepoName: '' TargetDocRepoOwner: '' PRBranchName: 'smoke-test-rdme' + SourceBranchName: 'smoke-test' ArtifactName: '' Language: '' DocRepoDestinationPath: '' #usually docs-ref-services/ + CIConfigs: '[]' GHReviewersVariable: '' GHTeamReviewersVariable: '' # externally set, as eng-common does not have the identity-resolver. Run as pre-step + OnboardingBranch: '' steps: - pwsh: | @@ -22,8 +25,8 @@ steps: try { Push-Location ${{ parameters.WorkingDirectory }}/repo - Write-Host "git checkout smoke-test" - git checkout smoke-test + Write-Host "git checkout ${{ parameters.SourceBranchName }}" + git checkout ${{ parameters.SourceBranchName }} } finally { Pop-Location } @@ -48,15 +51,73 @@ steps: env: GH_TOKEN: $(azuresdk-github-pat) +- task: PowerShell@2 + displayName: 'Update Docs.MS CI Targeted Packages' + condition: and(succeededOrFailed(), eq('${{ parameters.OnboardingBranch }}','')) + inputs: + targetType: filePath + filePath: ${{ parameters.ScriptDirectory }}/update-docs-ci.ps1 + arguments: > + -ArtifactLocation ${{ parameters.ArtifactLocation }} + -WorkDirectory "${{ parameters.WorkingDirectory }}" + -RepoId ${{ parameters.RepoId }} + -Repository ${{ parameters.PackageRepository }} + -ReleaseSHA ${{ parameters.ReleaseSha }} + -CIRepository "${{ parameters.WorkingDirectory }}/repo" + -Configs "${{ parameters.CIConfigs }}" + pwsh: true + env: + GH_TOKEN: $(azuresdk-github-pat) + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml parameters: RepoName: ${{ parameters.TargetDocRepoName }} RepoOwner: ${{ parameters.TargetDocRepoOwner }} PRBranchName: ${{ parameters.PRBranchName }} - CommitMsg: "Update readme content for ${{ parameters.ArtifactName }}" - PRTitle: "Docs.MS Readme Update." - BaseBranchName: smoke-test + CommitMsg: "Update docs metadata and targeting for release of ${{ parameters.ArtifactName }}" + PRTitle: "Docs.MS Release Updates for ${{ parameters.ArtifactName }}" + BaseBranchName: ${{ parameters.SourceBranchName }} WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }} GHReviewersVariable: ${{ parameters.GHReviewersVariable }} GHTeamReviewersVariable: ${{ parameters.GHTeamReviewersVariable }} + +- ${{if ne( parameters['OnboardingBranch'], '')}}: + - pwsh: | + Push-Location ${{ parameters.WorkingDirectory }}/repo + + git reset --hard HEAD + git remote rm azure-sdk-fork + git checkout ${{ parameters.OnboardingBranch}} + displayName: Reset Docs Repo, Checkout Onboarding Branch + ignoreLASTEXITCODE: false + + - task: PowerShell@2 + displayName: 'Update Docs.MS CI Targeted Packages' + inputs: + targetType: filePath + filePath: ${{ parameters.ScriptDirectory }}/update-docs-ci.ps1 + arguments: > + -ArtifactLocation ${{ parameters.ArtifactLocation }} + -WorkDirectory "${{ parameters.WorkingDirectory }}" + -RepoId ${{ parameters.RepoId }} + -Repository ${{ parameters.PackageRepository }} + -ReleaseSHA ${{ parameters.ReleaseSha }} + -CIRepository "${{ parameters.WorkingDirectory }}/repo" + -Configs "${{ parameters.CIConfigs }}" + pwsh: true + env: + GH_TOKEN: $(azuresdk-github-pat) + + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + RepoName: ${{ parameters.TargetDocRepoName }} + RepoOwner: ${{ parameters.TargetDocRepoOwner }} + PRBranchName: ${{ parameters.PRBranchName }}-ci + CommitMsg: "CI Update for release of ${{ parameters.ArtifactName }}" + PRTitle: "Docs.MS CI Updates for ${{ parameters.ArtifactName }}" + BaseBranchName: ${{ parameters.OnboardingBranch }} + WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo + ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }} + GHReviewersVariable: ${{ parameters.GHReviewersVariable }} + GHTeamReviewersVariable: ${{ parameters.GHTeamReviewersVariable }} \ No newline at end of file diff --git a/eng/common/scripts/Verify-Links.ps1 b/eng/common/scripts/Verify-Links.ps1 index 0dca2989a..5da025f9d 100644 --- a/eng/common/scripts/Verify-Links.ps1 +++ b/eng/common/scripts/Verify-Links.ps1 @@ -11,8 +11,8 @@ param ( [string] $baseUrl = "", # path to the root of the site for resolving rooted relative links, defaults to host root for http and file directory for local files [string] $rootUrl = "", - # list of http status codes count as broken links. Defaults to 404. - [array] $errorStatusCodes = @(404), + # list of http status codes count as broken links. Defaults to 400, 401, 404, SocketError.HostNotFound = 11001, SocketError.NoData = 11004 + [array] $errorStatusCodes = @(400, 401, 404, 11001, 11004), # flag to allow resolving relative paths or not [bool] $resolveRelativeLinks = $true ) @@ -145,6 +145,11 @@ function CheckLink ([System.Uri]$linkUri) catch { $statusCode = $_.Exception.Response.StatusCode.value__ + if(!$statusCode) { + # Try to pull the error code from any inner SocketException we might hit + $statusCode = $_.Exception.InnerException.ErrorCode + } + if ($statusCode -in $errorStatusCodes) { LogWarning "[$statusCode] broken link $linkUri" $script:badLinks += $linkUri diff --git a/eng/common/scripts/add-pullrequest-reviewers.ps1 b/eng/common/scripts/add-pullrequest-reviewers.ps1 index 00460ce70..a80d79485 100644 --- a/eng/common/scripts/add-pullrequest-reviewers.ps1 +++ b/eng/common/scripts/add-pullrequest-reviewers.ps1 @@ -45,11 +45,11 @@ catch { $userReviewers = @($resp.users | % { return $_.login }) $teamReviewers = @($resp.teams | % { return $_.slug }) -if (!$usersReviewers) { $modifiedUserReviewers = @() } else { $modifiedUserReviewers = $usersReviewers.Clone() } -$modifiedUserReviewers += ($modifiedUserReviewers | ? { !$usersReviews.Contains($_) }) +if (!$userReviewers) { $modifiedUserReviewers = @() } else { $modifiedUserReviewers = $userReviewers.Clone() } +$modifiedUserReviewers += ($userAdditions | ? { !$modifiedUserReviewers.Contains($_) }) if ($teamReviewers) { $modifiedTeamReviewers = @() } else { $modifiedTeamReviewers = $teamReviewers.Clone() } -$modifiedTeamReviewers += ($modifiedUserReviewers | ? { !$teamReviewers.Contains($_) }) +$modifiedTeamReviewers += ($teamAdditions | ? { !$modifiedTeamReviewers.Contains($_) }) $detectedUserDiffs = Compare-Object -ReferenceObject $userReviewers -DifferenceObject $modifiedUserReviewers $detectedTeamDiffs = Compare-Object -ReferenceObject $teamReviewers -DifferenceObject $modifiedTeamReviewers @@ -65,6 +65,7 @@ if ($detectedUserDiffs -or $detectedTeamDiffs) { $postResp = $postResp | ConvertTo-Json try { + Write-Host $postResp $resp = Invoke-RestMethod -Method Post -Headers $headers -Body $postResp -Uri $uri -MaximumRetryCount 3 $resp | Write-Verbose } diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index 92017cc7b..93bd6ac5f 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -67,6 +67,7 @@ function ParseMavenPackage($pkg, $workingDirectory) { return New-Object PSObject -Property @{ PackageId = $pkgId + GroupId = $groupId PackageVersion = $pkgVersion Deployable = $forceCreate -or !(IsMavenPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion -groupId $groupId.Replace(".", "/")) ReleaseNotes = $releaseNotes @@ -219,7 +220,7 @@ function IsNugetPackageVersionPublished($pkgId, $pkgVersion) { $nugetUri = "https://api.nuget.org/v3-flatcontainer/$($pkgId.ToLowerInvariant())/index.json" try { - $nugetVersions = Invoke-RestMethod -MaximumRetryCount 3 -uri $nugetUri -Method "GET" + $nugetVersions = Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -uri $nugetUri -Method "GET" return $nugetVersions.versions.Contains($pkgVersion) } @@ -452,9 +453,11 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a $pkgList += New-Object PSObject -Property @{ PackageId = $parsedPackage.PackageId PackageVersion = $parsedPackage.PackageVersion + GroupId = $parsedPackage.GroupId Tag = $tag ReleaseNotes = $parsedPackage.ReleaseNotes ReadmeContent = $parsedPackage.ReadmeContent + IsPrerelease = [AzureEngSemanticVersion]::ParseVersionString($parsedPackage.PackageVersion).IsPrerelease } } catch { diff --git a/eng/common/scripts/get-codeowners.ps1 b/eng/common/scripts/get-codeowners.ps1 index edbae4cb1..8452f9fd6 100644 --- a/eng/common/scripts/get-codeowners.ps1 +++ b/eng/common/scripts/get-codeowners.ps1 @@ -2,8 +2,9 @@ param ( $TargetDirectory, # should be in relative form from root of repo. EG: sdk/servicebus $RootDirectory # ideally $(Build.SourcesDirectory) ) - +$target = $TargetDirectory.ToLower().Trim("/") $codeOwnersLocation = Join-Path $RootDirectory -ChildPath ".github/CODEOWNERS" +$ownedFolders = @{} if (!(Test-Path $codeOwnersLocation)) { Write-Host "Unable to find CODEOWNERS file in target directory $RootDirectory" @@ -12,29 +13,27 @@ if (!(Test-Path $codeOwnersLocation)) { $codeOwnersContent = Get-Content $codeOwnersLocation -$ownedFolders = @{} - foreach ($contentLine in $codeOwnersContent) { if (-not $contentLine.StartsWith("#") -and $contentLine){ $splitLine = $contentLine -split "\s+" # CODEOWNERS file can also have labels present after the owner aliases # gh aliases start with @ in codeowners. don't pass on to API calls - $ownedFolders[$splitLine[0].ToLower()] = ($splitLine[1..$($splitLine.Length)] ` + $ownedFolders[$splitLine[0].ToLower().Trim("/")] = ($splitLine[1..$($splitLine.Length)] ` | ? { $_.StartsWith("@") } ` | % { return $_.substring(1) }) -join "," } } -$results = $ownedFolders[$TargetDirectory.ToLower()] +$results = $ownedFolders[$target] if ($results) { - Write-Host "Discovered code owners for path $TargetDirectory are $results." + Write-Host "Found a folder $results to match $target" return $results } else { - Write-Host "Unable to match path $TargetDirectory in CODEOWNERS file located at $codeOwnersLocation." - Write-Host $ownedFolders | ConvertTo-Json + Write-Host "Unable to match path $target in CODEOWNERS file located at $codeOwnersLocation." + Write-Host ($ownedFolders | ConvertTo-Json) return "" } diff --git a/eng/common/scripts/update-docs-ci.ps1 b/eng/common/scripts/update-docs-ci.ps1 new file mode 100644 index 000000000..466df8a92 --- /dev/null +++ b/eng/common/scripts/update-docs-ci.ps1 @@ -0,0 +1,275 @@ +#Requires -Version 6.0 +# This script is intended to update docs.ms CI configuration (currently supports Java, Python, C#, JS) +# as part of the azure-sdk release. For details on calling, check `archtype--release` in each azure-sdk +# repository. + +# Where possible, this script adds as few changes as possible to the target config. We only +# specifically mark a version for Python Preview and Java. This script is intended to be invoked +# multiple times. Once for each moniker. Currently only supports "latest" and "preview" artifact selection however. +param ( + [Parameter(Mandatory = $true)] + $ArtifactLocation, # the root of the artifact folder. DevOps $(System.ArtifactsDirectory) + + [Parameter(Mandatory = $true)] + $WorkDirectory, # a clean folder that we can work in + + [Parameter(Mandatory = $true)] + $ReleaseSHA, # the SHA for the artifacts. DevOps: $(Release.Artifacts..SourceVersion) or $(Build.SourceVersion) + + [Parameter(Mandatory = $true)] + $RepoId, # full repo id. EG azure/azure-sdk-for-net DevOps: $(Build.Repository.Id). Used as a part of VerifyPackages + + [Parameter(Mandatory = $true)] + [ValidateSet("Nuget","NPM","PyPI","Maven")] + $Repository, # EG: "Maven", "PyPI", "NPM" + + [Parameter(Mandatory = $true)] + $CIRepository, + + [Parameter(Mandatory = $true)] + $Configs +) + +# import artifact parsing and semver handling +. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) +. (Join-Path $PSScriptRoot SemVer.ps1) + +# Updates a python CI configuration json. +# For "latest", the version attribute is cleared, as default behavior is to pull latest "non-preview". +# For "preview", we update to >= the target releasing package version. +function UpdateParamsJsonPython($pkgs, $ciRepo, $locationInDocRepo){ + $pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) + + if (-not (Test-Path $pkgJsonLoc)) { + Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting." + exit(1) + } + + $allJson = Get-Content $pkgJsonLoc | ConvertFrom-Json + $visibleInCI = @{} + + for ($i=0; $i -lt $allJson.packages.Length; $i++) { + $pkgDef = $allJson.packages[$i] + + if ($pkgDef.package_info.name) { + $visibleInCI[$pkgDef.package_info.name] = $i + } + } + + foreach ($releasingPkg in $pkgs) { + if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { + $packagesIndex = $visibleInCI[$releasingPkg.PackageId] + $existingPackageDef = $allJson.packages[$packagesIndex] + + if ($releasingPkg.IsPrerelease) { + if (-not $existingPackageDef.package_info.version) { + $existingPackageDef.package_info | Add-Member -NotePropertyName version -NotePropertyValue "" + } + + $existingPackageDef.package_info.version = ">=$($releasingPkg.PackageVersion)" + } + else { + if ($def.version) { + $def.PSObject.Properties.Remove('version') + } + } + } + else { + $newItem = New-Object PSObject -Property @{ + package_info = New-Object PSObject -Property @{ + prefer_source_distribution = "true" + install_type = "pypi" + name=$releasingPkg.PackageId + } + excludePath = @("test*","example*","sample*","doc*") + } + $allJson.packages += $newItem + } + } + + $jsonContent = $allJson | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " } + + Set-Content -Path $pkgJsonLoc -Value $jsonContent +} + +# Updates a js CI configuration json. +# For "latest", we simply set a target package name +# For "preview", we add @next to the target package name +function UpdateParamsJsonJS($pkgs, $ciRepo, $locationInDocRepo){ + $pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) + + if (-not (Test-Path $pkgJsonLoc)) { + Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting." + exit(1) + } + + $allJson = Get-Content $pkgJsonLoc | ConvertFrom-Json + + $visibleInCI = @{} + + for ($i=0; $i -lt $allJson.npm_package_sources.Length; $i++) { + $pkgDef = $allJson.npm_package_sources[$i] + $accessor = ($pkgDef.name).Replace("`@next", "") + $visibleInCI[$accessor] = $i + } + + foreach ($releasingPkg in $pkgs) { + $name = $releasingPkg.PackageId + + if ($releasingPkg.IsPrerelease) { + $name += "`@next" + } + + if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { + $packagesIndex = $visibleInCI[$releasingPkg.PackageId] + $existingPackageDef = $allJson.npm_package_sources[$packagesIndex] + $existingPackageDef.name = $name + } + else { + $newItem = New-Object PSObject -Property @{ + name = $name + } + + if ($newItem) { $allJson.npm_package_sources += $newItem } + } + } + + $jsonContent = $allJson | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " } + + Set-Content -Path $pkgJsonLoc -Value $jsonContent +} + +# details on CSV schema can be found here +# https://review.docs.microsoft.com/en-us/help/onboard/admin/reference/dotnet/documenting-nuget?branch=master#set-up-the-ci-job +function UpdateCSVBasedCI($pkgs, $ciRepo, $locationInDocRepo){ + $csvLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) + + if (-not (Test-Path $csvLoc)) { + Write-Error "Unable to locate package csv at location $csvLoc, exiting." + exit(1) + } + + $allCSVRows = Get-Content $csvLoc + $visibleInCI = @{} + + # first pull what's already available + for ($i=0; $i -lt $allCSVRows.Length; $i++) { + $pkgDef = $allCSVRows[$i] + + # get rid of the modifiers to get just the package id + $id = $pkgDef.split(",")[1] -replace "\[.*?\]", "" + + $visibleInCI[$id] = $i + } + + foreach ($releasingPkg in $pkgs) { + $installModifiers = "tfm=netstandard2.0" + if ($releasingPkg.IsPrerelease) { + $installModifiers += ";isPrerelease=true" + } + $lineId = $releasingPkg.PackageId.Replace(".","").ToLower() + + if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { + $packagesIndex = $visibleInCI[$releasingPkg.PackageId] + $allCSVRows[$packagesIndex] = "$($lineId),[$installModifiers]$($releasingPkg.PackageId)" + } + else { + $newItem = "$($lineId),[$installModifiers]$($releasingPkg.PackageId)" + $allCSVRows += ($newItem) + } + } + + Set-Content -Path $csvLoc -Value $allCSVRows +} + +# a "package.json configures target packages for all the monikers in a Repository, it also has a slightly different +# schema than the moniker-specific json config that is seen in python and js +function UpdatePackageJson($pkgs, $ciRepo, $locationInDocRepo, $monikerId){ + $pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) + + if (-not (Test-Path $pkgJsonLoc)) { + Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting." + exit(1) + } + + $allJsonData = Get-Content $pkgJsonLoc | ConvertFrom-Json + + $visibleInCI = @{} + + for ($i=0; $i -lt $allJsonData[$monikerId].packages.Length; $i++) { + $pkgDef = $allJsonData[$monikerId].packages[$i] + $visibleInCI[$pkgDef.packageArtifactId] = $i + } + + foreach ($releasingPkg in $pkgs) { + if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { + $packagesIndex = $visibleInCI[$releasingPkg.PackageId] + $existingPackageDef = $allJsonData[$monikerId].packages[$packagesIndex] + $existingPackageDef.packageVersion = $releasingPkg.PackageVersion + } + else { + $newItem = New-Object PSObject -Property @{ + packageDownloadUrl = "https://repo1.maven.org/maven2" + packageGroupId = $releasingPkg.GroupId + packageArtifactId = $releasingPkg.PackageId + packageVersion = $releasingPkg.PackageVersion + inputPath = @() + excludePath = @() + } + + $allJsonData[$monikerId].packages += $newItem + } + } + + $jsonContent = $allJsonData | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " } + + Set-Content -Path $pkgJsonLoc -Value $jsonContent +} + +$targets = ($Configs | ConvertFrom-Json).targets + +#{ +# path_to_config: +# mode: +# monikerid +#} + +$apiUrl = "https://api.github.com/repos/$repoId" +$pkgs = VerifyPackages -pkgRepository $Repository ` + -artifactLocation $ArtifactLocation ` + -workingDirectory $WorkDirectory ` + -apiUrl $apiUrl ` + -continueOnError $True + +foreach ($config in $targets) { + if ($config.mode -eq "Preview") { $includePreview = $true } else { $includePreview = $false } + $pkgsFiltered = $pkgs | ? { $_.IsPrerelease -eq $includePreview} + + if ($pkgs) { + Write-Host "Given the visible artifacts, CI updates against $($config.path_to_config) will be processed for the following packages." + Write-Host ($pkgsFiltered | % { $_.PackageId + " " + $_.PackageVersion }) + + switch ($Repository) { + "Nuget" { + UpdateCSVBasedCI -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config + break + } + "NPM" { + UpdateParamsJsonJS -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config + break + } + "PyPI" { + UpdateParamsJsonPython -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config + break + } + "Maven" { + UpdatePackageJson -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config -monikerId $config.monikerid + break + } + default { + Write-Host "Unrecognized target: $Repository" + exit(1) + } + } + } +} \ No newline at end of file diff --git a/eng/common/scripts/update-docs-metadata.ps1 b/eng/common/scripts/update-docs-metadata.ps1 index b96ea284b..42ca30894 100644 --- a/eng/common/scripts/update-docs-metadata.ps1 +++ b/eng/common/scripts/update-docs-metadata.ps1 @@ -14,8 +14,6 @@ param ( $DocRepoContentLocation = "docs-ref-services/" # within the doc repo, where does our readme go? ) - -# import artifact parsing and semver handling . (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) . (Join-Path $PSScriptRoot SemVer.ps1)