Sync eng/common directory with azure-sdk-tools for PR 1203 (#999)
* Refactoring artifact-metadata-parsing.ps1, update-docs-metadata.ps1, and create-tags-and-git-release.ps1 * Clean up common imports * Refactor Update-docs-ci.ps1 Co-authored-by: Chidozie Ononiwu <chononiw@microsoft.com>
This commit is contained in:
Azure SDK Bot
GitHub
parent
91c6116ead
commit
e2bfa85d8b
@ -16,7 +16,7 @@ param(
|
||||
[string]$AuthToken
|
||||
)
|
||||
|
||||
. "${PSScriptRoot}\common.ps1"
|
||||
. (Join-Path $PSScriptRoot common.ps1)
|
||||
|
||||
try {
|
||||
Add-GithubIssueComment -RepoOwner $RepoOwner -RepoName $RepoName `
|
||||
|
||||
@ -16,7 +16,7 @@ param(
|
||||
[string]$AuthToken
|
||||
)
|
||||
|
||||
. "${PSScriptRoot}\common.ps1"
|
||||
. (Join-Path $PSScriptRoot common.ps1)
|
||||
|
||||
try {
|
||||
Add-GithubIssueLabels -RepoOwner $RepoOwner -RepoName $RepoName `
|
||||
|
||||
@ -5,7 +5,7 @@ param(
|
||||
$AuthToken
|
||||
)
|
||||
|
||||
. "${PSScriptRoot}\common.ps1"
|
||||
. (Join-Path $PSScriptRoot common.ps1)
|
||||
|
||||
LogDebug "Operating on Repo [ $RepoName ]"
|
||||
try{
|
||||
|
||||
@ -12,7 +12,7 @@ param (
|
||||
[string]$AuthToken
|
||||
)
|
||||
|
||||
. "${PSScriptRoot}\common.ps1"
|
||||
. (Join-Path $PSScriptRoot common.ps1)
|
||||
|
||||
try
|
||||
{
|
||||
|
||||
@ -93,7 +93,7 @@ function Get-PkgProperties
|
||||
{
|
||||
$pkgDirectoryPath = Join-Path $serviceDirectoryPath $directory.Name
|
||||
|
||||
if ((Get-ChildItem -Path Function: | ? { $_.Name -eq $GetPackageInfoFromRepoFn }).Count -gt 0)
|
||||
if ($GetPackageInfoFromRepoFn -and (Test-Path "Function:$GetPackageInfoFromRepoFn"))
|
||||
{
|
||||
$pkgProps = &$GetPackageInfoFromRepoFn -pkgPath $pkgDirectoryPath -serviceDirectory $ServiceDirectory -pkgName $PackageName
|
||||
}
|
||||
|
||||
@ -21,8 +21,7 @@ param(
|
||||
[string]$Base64EncodedAuthToken
|
||||
)
|
||||
|
||||
. "${PSScriptRoot}\logging.ps1"
|
||||
. "${PSScriptRoot}\Invoke-DevOpsAPI.ps1"
|
||||
. (Join-Path $PSScriptRoot common.ps1)
|
||||
|
||||
if ($CancelPreviousBuilds)
|
||||
{
|
||||
|
||||
@ -59,7 +59,7 @@ param(
|
||||
[boolean]$CloseAfterOpenForTesting=$false
|
||||
)
|
||||
|
||||
. "${PSScriptRoot}\common.ps1"
|
||||
. (Join-Path $PSScriptRoot common.ps1)
|
||||
|
||||
try {
|
||||
$resp = Get-GitHubPullRequests -RepoOwner $RepoOwner -RepoName $RepoName `
|
||||
|
||||
@ -22,7 +22,7 @@ if ($ReleaseDate -and ($Unreleased -eq $True)) {
|
||||
exit 1
|
||||
}
|
||||
|
||||
. "${PSScriptRoot}\common.ps1"
|
||||
. (Join-Path $PSScriptRoot common.ps1)
|
||||
|
||||
if ($ReleaseDate)
|
||||
{
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
Import-Module "${PSScriptRoot}/modules/ChangeLog-Operations.psm1"
|
||||
. (Join-Path $PSScriptRoot SemVer.ps1)
|
||||
. (Join-Path $EngCommonScriptsDir SemVer.ps1)
|
||||
|
||||
$SDIST_PACKAGE_REGEX = "^(?<package>.*)\-(?<versionstring>$([AzureEngSemanticVersion]::SEMVER_REGEX))"
|
||||
|
||||
@ -40,336 +39,6 @@ function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) {
|
||||
}
|
||||
}
|
||||
|
||||
# Parse out package publishing information given a maven POM file
|
||||
function ParseMavenPackage($pkg, $workingDirectory) {
|
||||
[xml]$contentXML = Get-Content $pkg
|
||||
|
||||
$pkgId = $contentXML.project.artifactId
|
||||
$pkgVersion = $contentXML.project.version
|
||||
$groupId = if ($contentXML.project.groupId -eq $null) { $contentXML.project.parent.groupId } else { $contentXML.project.groupId }
|
||||
$releaseNotes = ""
|
||||
$readmeContent = ""
|
||||
|
||||
# if it's a snapshot. return $null (as we don't want to create tags for this, but we also don't want to fail)
|
||||
if ($pkgVersion.Contains("SNAPSHOT")) {
|
||||
return $null
|
||||
}
|
||||
|
||||
$changeLogLoc = @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-changelog.md")[0]
|
||||
if ($changeLogLoc) {
|
||||
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
|
||||
}
|
||||
|
||||
$readmeContentLoc = @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-readme.md")[0]
|
||||
if ($readmeContentLoc) {
|
||||
$readmeContent = Get-Content -Raw $readmeContentLoc
|
||||
}
|
||||
|
||||
return New-Object PSObject -Property @{
|
||||
PackageId = $pkgId
|
||||
GroupId = $groupId
|
||||
PackageVersion = $pkgVersion
|
||||
ReleaseTag = "$($pkgId)_$($pkgVersion)"
|
||||
Deployable = $forceCreate -or !(IsMavenPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion -groupId $groupId.Replace(".", "/"))
|
||||
ReleaseNotes = $releaseNotes
|
||||
ReadmeContent = $readmeContent
|
||||
}
|
||||
}
|
||||
|
||||
# Returns the maven (really sonatype) publish status of a package id and version.
|
||||
function IsMavenPackageVersionPublished($pkgId, $pkgVersion, $groupId) {
|
||||
try {
|
||||
|
||||
$uri = "https://oss.sonatype.org/content/repositories/releases/$groupId/$pkgId/$pkgVersion/$pkgId-$pkgVersion.pom"
|
||||
$pomContent = Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -Method "GET" -uri $uri
|
||||
|
||||
if ($pomContent -ne $null -or $pomContent.Length -eq 0) {
|
||||
return $true
|
||||
}
|
||||
else {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
catch {
|
||||
$statusCode = $_.Exception.Response.StatusCode.value__
|
||||
$statusDescription = $_.Exception.Response.StatusDescription
|
||||
|
||||
# if this is 404ing, then this pkg has never been published before
|
||||
if ($statusCode -eq 404) {
|
||||
return $false
|
||||
}
|
||||
|
||||
Write-Host "VersionCheck to maven for packageId $pkgId failed with statuscode $statusCode"
|
||||
Write-Host $statusDescription
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
# make certain to always take the package json closest to the top
|
||||
function ResolvePkgJson($workFolder) {
|
||||
$pathsWithComplexity = @()
|
||||
foreach ($file in (Get-ChildItem -Path $workFolder -Recurse -Include "package.json")) {
|
||||
$complexity = ($file.FullName -Split { $_ -eq "/" -or $_ -eq "\" }).Length
|
||||
$pathsWithComplexity += New-Object PSObject -Property @{
|
||||
Path = $file
|
||||
Complexity = $complexity
|
||||
}
|
||||
}
|
||||
|
||||
return ($pathsWithComplexity | Sort-Object -Property Complexity)[0].Path
|
||||
}
|
||||
|
||||
# Parse out package publishing information given a .tgz npm artifact
|
||||
function ParseNPMPackage($pkg, $workingDirectory) {
|
||||
$workFolder = "$workingDirectory$($pkg.Basename)"
|
||||
$origFolder = Get-Location
|
||||
$releaseNotes = ""
|
||||
$readmeContent = ""
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $workFolder
|
||||
cd $workFolder
|
||||
|
||||
tar -xzf $pkg
|
||||
|
||||
$packageJSON = ResolvePkgJson -workFolder $workFolder | Get-Content | ConvertFrom-Json
|
||||
$pkgId = $packageJSON.name
|
||||
$pkgVersion = $packageJSON.version
|
||||
|
||||
$changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
|
||||
if ($changeLogLoc) {
|
||||
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
|
||||
}
|
||||
|
||||
$readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md") | Select-Object -Last 1
|
||||
if ($readmeContentLoc) {
|
||||
$readmeContent = Get-Content -Raw $readmeContentLoc
|
||||
}
|
||||
|
||||
cd $origFolder
|
||||
Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue
|
||||
|
||||
$resultObj = New-Object PSObject -Property @{
|
||||
PackageId = $pkgId
|
||||
PackageVersion = $pkgVersion
|
||||
ReleaseTag = "$($pkgId)_$($pkgVersion)"
|
||||
Deployable = $forceCreate -or !(IsNPMPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion)
|
||||
ReleaseNotes = $releaseNotes
|
||||
ReadmeContent = $readmeContent
|
||||
}
|
||||
|
||||
return $resultObj
|
||||
}
|
||||
|
||||
# Returns the npm publish status of a package id and version.
|
||||
function IsNPMPackageVersionPublished($pkgId, $pkgVersion) {
|
||||
$npmVersions = (npm show $pkgId versions)
|
||||
|
||||
if ($LastExitCode -ne 0) {
|
||||
npm ping
|
||||
|
||||
if ($LastExitCode -eq 0) {
|
||||
return $False
|
||||
}
|
||||
|
||||
Write-Host "Could not find a deployed version of $pkgId, and NPM connectivity check failed."
|
||||
exit(1)
|
||||
}
|
||||
|
||||
$npmVersionList = $npmVersions.split(",") | % { return $_.replace("[", "").replace("]", "").Trim() }
|
||||
return $npmVersionList.Contains($pkgVersion)
|
||||
}
|
||||
|
||||
# Parse out package publishing information given a nupkg ZIP format.
|
||||
function ParseNugetPackage($pkg, $workingDirectory) {
|
||||
$workFolder = "$workingDirectory$($pkg.Basename)"
|
||||
$origFolder = Get-Location
|
||||
$zipFileLocation = "$workFolder/$($pkg.Basename).zip"
|
||||
$releaseNotes = ""
|
||||
$readmeContent = ""
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $workFolder
|
||||
|
||||
Copy-Item -Path $pkg -Destination $zipFileLocation
|
||||
Expand-Archive -Path $zipFileLocation -DestinationPath $workFolder
|
||||
[xml] $packageXML = Get-ChildItem -Path "$workFolder/*.nuspec" | Get-Content
|
||||
$pkgId = $packageXML.package.metadata.id
|
||||
$pkgVersion = $packageXML.package.metadata.version
|
||||
|
||||
$changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
|
||||
if ($changeLogLoc) {
|
||||
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
|
||||
}
|
||||
|
||||
$readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0]
|
||||
if ($readmeContentLoc) {
|
||||
$readmeContent = Get-Content -Raw $readmeContentLoc
|
||||
}
|
||||
|
||||
Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue
|
||||
|
||||
return New-Object PSObject -Property @{
|
||||
PackageId = $pkgId
|
||||
PackageVersion = $pkgVersion
|
||||
ReleaseTag = "$($pkgId)_$($pkgVersion)"
|
||||
Deployable = $forceCreate -or !(IsNugetPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion)
|
||||
ReleaseNotes = $releaseNotes
|
||||
ReadmeContent = $readmeContent
|
||||
}
|
||||
}
|
||||
|
||||
# Returns the nuget publish status of a package id and version.
|
||||
function IsNugetPackageVersionPublished($pkgId, $pkgVersion) {
|
||||
|
||||
$nugetUri = "https://api.nuget.org/v3-flatcontainer/$($pkgId.ToLowerInvariant())/index.json"
|
||||
|
||||
try {
|
||||
$nugetVersions = Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -uri $nugetUri -Method "GET"
|
||||
|
||||
return $nugetVersions.versions.Contains($pkgVersion)
|
||||
}
|
||||
catch {
|
||||
$statusCode = $_.Exception.Response.StatusCode.value__
|
||||
$statusDescription = $_.Exception.Response.StatusDescription
|
||||
|
||||
# if this is 404ing, then this pkg has never been published before
|
||||
if ($statusCode -eq 404) {
|
||||
return $False
|
||||
}
|
||||
|
||||
Write-Host "Nuget Invocation failed:"
|
||||
Write-Host "StatusCode:" $statusCode
|
||||
Write-Host "StatusDescription:" $statusDescription
|
||||
exit(1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
# Parse out package publishing information given a python sdist of ZIP format.
|
||||
function ParsePyPIPackage($pkg, $workingDirectory) {
|
||||
$pkg.Basename -match $SDIST_PACKAGE_REGEX | Out-Null
|
||||
|
||||
$pkgId = $matches["package"]
|
||||
$pkgVersion = $matches["versionstring"]
|
||||
|
||||
$workFolder = "$workingDirectory$($pkg.Basename)"
|
||||
$origFolder = Get-Location
|
||||
$releaseNotes = ""
|
||||
$readmeContent = ""
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $workFolder
|
||||
Expand-Archive -Path $pkg -DestinationPath $workFolder
|
||||
|
||||
$changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
|
||||
if ($changeLogLoc) {
|
||||
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
|
||||
}
|
||||
|
||||
$readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md") | Select-Object -Last 1
|
||||
|
||||
if ($readmeContentLoc) {
|
||||
$readmeContent = Get-Content -Raw $readmeContentLoc
|
||||
}
|
||||
|
||||
Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue
|
||||
|
||||
return New-Object PSObject -Property @{
|
||||
PackageId = $pkgId
|
||||
PackageVersion = $pkgVersion
|
||||
ReleaseTag = "$($pkgId)_$($pkgVersion)"
|
||||
Deployable = $forceCreate -or !(IsPythonPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion)
|
||||
ReleaseNotes = $releaseNotes
|
||||
ReadmeContent = $readmeContent
|
||||
}
|
||||
}
|
||||
|
||||
function ParseCArtifact($pkg, $workingDirectory) {
|
||||
$packageInfo = Get-Content -Raw -Path $pkg | ConvertFrom-JSON
|
||||
$packageArtifactLocation = (Get-ItemProperty $pkg).Directory.FullName
|
||||
$releaseNotes = ""
|
||||
$readmeContent = ""
|
||||
|
||||
$pkgVersion = $packageInfo.version
|
||||
|
||||
$changeLogLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "CHANGELOG.md")[0]
|
||||
if ($changeLogLoc)
|
||||
{
|
||||
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
|
||||
}
|
||||
|
||||
$readmeContentLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "README.md")[0]
|
||||
if ($readmeContentLoc) {
|
||||
$readmeContent = Get-Content -Raw $readmeContentLoc
|
||||
}
|
||||
|
||||
return New-Object PSObject -Property @{
|
||||
PackageId = ''
|
||||
PackageVersion = $pkgVersion
|
||||
ReleaseTag = $pkgVersion
|
||||
# Artifact info is always considered deployable for C becasue it is not
|
||||
# deployed anywhere. Dealing with duplicate tags happens downstream in
|
||||
# CheckArtifactShaAgainstTagsList
|
||||
Deployable = $true
|
||||
ReleaseNotes = $releaseNotes
|
||||
}
|
||||
}
|
||||
|
||||
function ParseCppArtifact($pkg, $workingDirectory) {
|
||||
$packageInfo = Get-Content -Raw -Path $pkg | ConvertFrom-JSON
|
||||
$packageArtifactLocation = (Get-ItemProperty $pkg).Directory.FullName
|
||||
$releaseNotes = ""
|
||||
$readmeContent = ""
|
||||
|
||||
$pkgVersion = $packageInfo.version
|
||||
$pkgName = $packageInfo.name
|
||||
|
||||
$changeLogLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "CHANGELOG.md")[0]
|
||||
if ($changeLogLoc)
|
||||
{
|
||||
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
|
||||
}
|
||||
|
||||
$readmeContentLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "README.md")[0]
|
||||
if ($readmeContentLoc) {
|
||||
$readmeContent = Get-Content -Raw $readmeContentLoc
|
||||
}
|
||||
|
||||
return New-Object PSObject -Property @{
|
||||
PackageId = $pkgName
|
||||
PackageVersion = $pkgVersion
|
||||
ReleaseTag = "${pkgName}_${pkgVersion}"
|
||||
# Artifact info is always considered deployable for now becasue it is not
|
||||
# deployed anywhere. Dealing with duplicate tags happens downstream in
|
||||
# CheckArtifactShaAgainstTagsList
|
||||
Deployable = $true
|
||||
ReleaseNotes = $releaseNotes
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# Returns the pypi publish status of a package id and version.
|
||||
function IsPythonPackageVersionPublished($pkgId, $pkgVersion) {
|
||||
try {
|
||||
$existingVersion = (Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -Method "Get" -uri "https://pypi.org/pypi/$pkgId/$pkgVersion/json").info.version
|
||||
|
||||
# if existingVersion exists, then it's already been published
|
||||
return $True
|
||||
}
|
||||
catch {
|
||||
$statusCode = $_.Exception.Response.StatusCode.value__
|
||||
$statusDescription = $_.Exception.Response.StatusDescription
|
||||
|
||||
# if this is 404ing, then this pkg has never been published before
|
||||
if ($statusCode -eq 404) {
|
||||
return $False
|
||||
}
|
||||
|
||||
Write-Host "PyPI Invocation failed:"
|
||||
Write-Host "StatusCode:" $statusCode
|
||||
Write-Host "StatusDescription:" $statusDescription
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
# Retrieves the list of all tags that exist on the target repository
|
||||
function GetExistingTags($apiUrl) {
|
||||
try {
|
||||
@ -394,12 +63,12 @@ function GetExistingTags($apiUrl) {
|
||||
}
|
||||
|
||||
# Retrieve release tag for artiface package. If multiple packages, then output the first one.
|
||||
function RetrieveReleaseTag($pkgRepository, $artifactLocation, $continueOnError = $true) {
|
||||
function RetrieveReleaseTag($artifactLocation, $continueOnError = $true) {
|
||||
if (!$artifactLocation) {
|
||||
return ""
|
||||
}
|
||||
try {
|
||||
$pkgs, $parsePkgInfoFn = RetrievePackages -pkgRepository $pkgRepository -artifactLocation $artifactLocation
|
||||
$pkgs, $parsePkgInfoFn = RetrievePackages -artifactLocation $artifactLocation
|
||||
if (!$pkgs -or !$pkgs[0]) {
|
||||
Write-Host "No packages retrieved from artifact location."
|
||||
return ""
|
||||
@ -421,52 +90,23 @@ function RetrieveReleaseTag($pkgRepository, $artifactLocation, $continueOnError
|
||||
Write-Error "No release tag retrieved from $artifactLocation"
|
||||
}
|
||||
}
|
||||
function RetrievePackages($pkgRepository, $artifactLocation) {
|
||||
$parsePkgInfoFn = ""
|
||||
$packagePattern = ""
|
||||
$pkgRepository = $pkgRepository.Trim()
|
||||
switch ($pkgRepository) {
|
||||
"Maven" {
|
||||
$parsePkgInfoFn = "ParseMavenPackage"
|
||||
$packagePattern = "*.pom"
|
||||
break
|
||||
}
|
||||
"Nuget" {
|
||||
$parsePkgInfoFn = "ParseNugetPackage"
|
||||
$packagePattern = "*.nupkg"
|
||||
break
|
||||
}
|
||||
"NPM" {
|
||||
$parsePkgInfoFn = "ParseNPMPackage"
|
||||
$packagePattern = "*.tgz"
|
||||
break
|
||||
}
|
||||
"PyPI" {
|
||||
$parsePkgInfoFn = "ParsePyPIPackage"
|
||||
$packagePattern = "*.zip"
|
||||
break
|
||||
}
|
||||
"C" {
|
||||
$parsePkgInfoFn = "ParseCArtifact"
|
||||
$packagePattern = "*.json"
|
||||
}
|
||||
"CPP" {
|
||||
$parsePkgInfoFn = "ParseCppArtifact"
|
||||
$packagePattern = "*.json"
|
||||
}
|
||||
default {
|
||||
Write-Host "Unrecognized Language: $language"
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
function RetrievePackages($artifactLocation) {
|
||||
$pkgs = Get-ChildItem -Path $artifactLocation -Include $packagePattern -Recurse -File
|
||||
return $pkgs, $parsePkgInfoFn
|
||||
if ($GetPackageInfoFromPackageFileFn -and (Test-Path "Function:$GetPackageInfoFromPackageFileFn"))
|
||||
{
|
||||
return $pkgs, $GetPackageInfoFromPackageFileFn
|
||||
}
|
||||
else
|
||||
{
|
||||
LogError "The function '$GetPackageInfoFromPackageFileFn' was not found."
|
||||
}
|
||||
}
|
||||
|
||||
# Walk across all build artifacts, check them against the appropriate repository, return a list of tags/releases
|
||||
function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha, $continueOnError = $false) {
|
||||
function VerifyPackages($artifactLocation, $workingDirectory, $apiUrl, $releaseSha, $continueOnError = $false) {
|
||||
$pkgList = [array]@()
|
||||
$pkgs, $parsePkgInfoFn = RetrievePackages -pkgRepository $pkgRepository -artifactLocation $artifactLocation
|
||||
$pkgs, $parsePkgInfoFn = RetrievePackages -artifactLocation $artifactLocation
|
||||
|
||||
foreach ($pkg in $pkgs) {
|
||||
try {
|
||||
|
||||
@ -10,6 +10,7 @@ $EngScriptsDir = Join-Path $EngDir "scripts"
|
||||
. (Join-Path $EngCommonScriptsDir Package-Properties.ps1)
|
||||
. (Join-Path $EngCommonScriptsDir logging.ps1)
|
||||
. (Join-Path $EngCommonScriptsDir Invoke-GitHubAPI.ps1)
|
||||
. (Join-Path $EngCommonScriptsDir Invoke-DevOpsAPI.ps1)
|
||||
. (Join-Path $EngCommonScriptsDir artifact-metadata-parsing.ps1)
|
||||
|
||||
# Setting expected from common languages settings
|
||||
@ -31,4 +32,5 @@ if (-not $LanguageShort)
|
||||
# Transformed Functions
|
||||
$GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo"
|
||||
$GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile"
|
||||
$PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs"
|
||||
$PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs"
|
||||
$UpdateDocCIFn = "Update-${Language}-CIConfig"
|
||||
@ -231,7 +231,7 @@ function Upload-Blobs
|
||||
}
|
||||
|
||||
|
||||
if ((Get-ChildItem -Path Function: | ? { $_.Name -eq $PublishGithubIODocsFn }).Count -gt 0)
|
||||
if ($PublishGithubIODocsFn -and (Test-Path "Function:$PublishGithubIODocsFn"))
|
||||
{
|
||||
&$PublishGithubIODocsFn -DocLocation $DocLocation -PublicArtifactLocation $PublicArtifactLocation
|
||||
}
|
||||
|
||||
@ -18,13 +18,13 @@ param (
|
||||
[switch]$continueOnError = $false
|
||||
)
|
||||
|
||||
. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1)
|
||||
. (Join-Path $PSScriptRoot common.ps1)
|
||||
|
||||
$apiUrl = "https://api.github.com/repos/$repoId"
|
||||
Write-Host "Using API URL $apiUrl"
|
||||
|
||||
# VERIFY PACKAGES
|
||||
$pkgList = VerifyPackages -pkgRepository $packageRepository -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha -continueOnError $continueOnError
|
||||
$pkgList = VerifyPackages -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha -continueOnError $continueOnError
|
||||
|
||||
if ($pkgList) {
|
||||
Write-Host "Given the visible artifacts, github releases will be created for the following:"
|
||||
|
||||
@ -30,201 +30,7 @@ param (
|
||||
$Configs
|
||||
)
|
||||
|
||||
# import artifact parsing and semver handling
|
||||
. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1)
|
||||
. (Join-Path $PSScriptRoot SemVer.ps1)
|
||||
|
||||
# Updates a python CI configuration json.
|
||||
# For "latest", the version attribute is cleared, as default behavior is to pull latest "non-preview".
|
||||
# For "preview", we update to >= the target releasing package version.
|
||||
function UpdateParamsJsonPython($pkgs, $ciRepo, $locationInDocRepo){
|
||||
$pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo)
|
||||
|
||||
if (-not (Test-Path $pkgJsonLoc)) {
|
||||
Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting."
|
||||
exit(1)
|
||||
}
|
||||
|
||||
$allJson = Get-Content $pkgJsonLoc | ConvertFrom-Json
|
||||
$visibleInCI = @{}
|
||||
|
||||
for ($i=0; $i -lt $allJson.packages.Length; $i++) {
|
||||
$pkgDef = $allJson.packages[$i]
|
||||
|
||||
if ($pkgDef.package_info.name) {
|
||||
$visibleInCI[$pkgDef.package_info.name] = $i
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($releasingPkg in $pkgs) {
|
||||
if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) {
|
||||
$packagesIndex = $visibleInCI[$releasingPkg.PackageId]
|
||||
$existingPackageDef = $allJson.packages[$packagesIndex]
|
||||
|
||||
if ($releasingPkg.IsPrerelease) {
|
||||
if (-not $existingPackageDef.package_info.version) {
|
||||
$existingPackageDef.package_info | Add-Member -NotePropertyName version -NotePropertyValue ""
|
||||
}
|
||||
|
||||
$existingPackageDef.package_info.version = ">=$($releasingPkg.PackageVersion)"
|
||||
}
|
||||
else {
|
||||
if ($def.version) {
|
||||
$def.PSObject.Properties.Remove('version')
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
$newItem = New-Object PSObject -Property @{
|
||||
package_info = New-Object PSObject -Property @{
|
||||
prefer_source_distribution = "true"
|
||||
install_type = "pypi"
|
||||
name=$releasingPkg.PackageId
|
||||
}
|
||||
excludePath = @("test*","example*","sample*","doc*")
|
||||
}
|
||||
$allJson.packages += $newItem
|
||||
}
|
||||
}
|
||||
|
||||
$jsonContent = $allJson | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " }
|
||||
|
||||
Set-Content -Path $pkgJsonLoc -Value $jsonContent
|
||||
}
|
||||
|
||||
# Updates a js CI configuration json.
|
||||
# For "latest", we simply set a target package name
|
||||
# For "preview", we add @next to the target package name
|
||||
function UpdateParamsJsonJS($pkgs, $ciRepo, $locationInDocRepo){
|
||||
$pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo)
|
||||
|
||||
if (-not (Test-Path $pkgJsonLoc)) {
|
||||
Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting."
|
||||
exit(1)
|
||||
}
|
||||
|
||||
$allJson = Get-Content $pkgJsonLoc | ConvertFrom-Json
|
||||
|
||||
$visibleInCI = @{}
|
||||
|
||||
for ($i=0; $i -lt $allJson.npm_package_sources.Length; $i++) {
|
||||
$pkgDef = $allJson.npm_package_sources[$i]
|
||||
$accessor = ($pkgDef.name).Replace("`@next", "")
|
||||
$visibleInCI[$accessor] = $i
|
||||
}
|
||||
|
||||
foreach ($releasingPkg in $pkgs) {
|
||||
$name = $releasingPkg.PackageId
|
||||
|
||||
if ($releasingPkg.IsPrerelease) {
|
||||
$name += "`@next"
|
||||
}
|
||||
|
||||
if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) {
|
||||
$packagesIndex = $visibleInCI[$releasingPkg.PackageId]
|
||||
$existingPackageDef = $allJson.npm_package_sources[$packagesIndex]
|
||||
$existingPackageDef.name = $name
|
||||
}
|
||||
else {
|
||||
$newItem = New-Object PSObject -Property @{
|
||||
name = $name
|
||||
}
|
||||
|
||||
if ($newItem) { $allJson.npm_package_sources += $newItem }
|
||||
}
|
||||
}
|
||||
|
||||
$jsonContent = $allJson | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " }
|
||||
|
||||
Set-Content -Path $pkgJsonLoc -Value $jsonContent
|
||||
}
|
||||
|
||||
# details on CSV schema can be found here
|
||||
# https://review.docs.microsoft.com/en-us/help/onboard/admin/reference/dotnet/documenting-nuget?branch=master#set-up-the-ci-job
|
||||
function UpdateCSVBasedCI($pkgs, $ciRepo, $locationInDocRepo){
|
||||
$csvLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo)
|
||||
|
||||
if (-not (Test-Path $csvLoc)) {
|
||||
Write-Error "Unable to locate package csv at location $csvLoc, exiting."
|
||||
exit(1)
|
||||
}
|
||||
|
||||
$allCSVRows = Get-Content $csvLoc
|
||||
$visibleInCI = @{}
|
||||
|
||||
# first pull what's already available
|
||||
for ($i=0; $i -lt $allCSVRows.Length; $i++) {
|
||||
$pkgDef = $allCSVRows[$i]
|
||||
|
||||
# get rid of the modifiers to get just the package id
|
||||
$id = $pkgDef.split(",")[1] -replace "\[.*?\]", ""
|
||||
|
||||
$visibleInCI[$id] = $i
|
||||
}
|
||||
|
||||
foreach ($releasingPkg in $pkgs) {
|
||||
$installModifiers = "tfm=netstandard2.0"
|
||||
if ($releasingPkg.IsPrerelease) {
|
||||
$installModifiers += ";isPrerelease=true"
|
||||
}
|
||||
$lineId = $releasingPkg.PackageId.Replace(".","").ToLower()
|
||||
|
||||
if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) {
|
||||
$packagesIndex = $visibleInCI[$releasingPkg.PackageId]
|
||||
$allCSVRows[$packagesIndex] = "$($lineId),[$installModifiers]$($releasingPkg.PackageId)"
|
||||
}
|
||||
else {
|
||||
$newItem = "$($lineId),[$installModifiers]$($releasingPkg.PackageId)"
|
||||
$allCSVRows += ($newItem)
|
||||
}
|
||||
}
|
||||
|
||||
Set-Content -Path $csvLoc -Value $allCSVRows
|
||||
}
|
||||
|
||||
# a "package.json configures target packages for all the monikers in a Repository, it also has a slightly different
|
||||
# schema than the moniker-specific json config that is seen in python and js
|
||||
function UpdatePackageJson($pkgs, $ciRepo, $locationInDocRepo, $monikerId){
|
||||
$pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo)
|
||||
|
||||
if (-not (Test-Path $pkgJsonLoc)) {
|
||||
Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting."
|
||||
exit(1)
|
||||
}
|
||||
|
||||
$allJsonData = Get-Content $pkgJsonLoc | ConvertFrom-Json
|
||||
|
||||
$visibleInCI = @{}
|
||||
|
||||
for ($i=0; $i -lt $allJsonData[$monikerId].packages.Length; $i++) {
|
||||
$pkgDef = $allJsonData[$monikerId].packages[$i]
|
||||
$visibleInCI[$pkgDef.packageArtifactId] = $i
|
||||
}
|
||||
|
||||
foreach ($releasingPkg in $pkgs) {
|
||||
if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) {
|
||||
$packagesIndex = $visibleInCI[$releasingPkg.PackageId]
|
||||
$existingPackageDef = $allJsonData[$monikerId].packages[$packagesIndex]
|
||||
$existingPackageDef.packageVersion = $releasingPkg.PackageVersion
|
||||
}
|
||||
else {
|
||||
$newItem = New-Object PSObject -Property @{
|
||||
packageDownloadUrl = "https://repo1.maven.org/maven2"
|
||||
packageGroupId = $releasingPkg.GroupId
|
||||
packageArtifactId = $releasingPkg.PackageId
|
||||
packageVersion = $releasingPkg.PackageVersion
|
||||
inputPath = @()
|
||||
excludePath = @()
|
||||
}
|
||||
|
||||
$allJsonData[$monikerId].packages += $newItem
|
||||
}
|
||||
}
|
||||
|
||||
$jsonContent = $allJsonData | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " }
|
||||
|
||||
Set-Content -Path $pkgJsonLoc -Value $jsonContent
|
||||
}
|
||||
. (Join-Path $PSScriptRoot common.ps1)
|
||||
|
||||
$targets = ($Configs | ConvertFrom-Json).targets
|
||||
|
||||
@ -235,8 +41,7 @@ $targets = ($Configs | ConvertFrom-Json).targets
|
||||
#}
|
||||
|
||||
$apiUrl = "https://api.github.com/repos/$repoId"
|
||||
$pkgs = VerifyPackages -pkgRepository $Repository `
|
||||
-artifactLocation $ArtifactLocation `
|
||||
$pkgs = VerifyPackages -artifactLocation $ArtifactLocation `
|
||||
-workingDirectory $WorkDirectory `
|
||||
-apiUrl $apiUrl `
|
||||
-continueOnError $True
|
||||
@ -249,27 +54,13 @@ foreach ($config in $targets) {
|
||||
Write-Host "Given the visible artifacts, CI updates against $($config.path_to_config) will be processed for the following packages."
|
||||
Write-Host ($pkgsFiltered | % { $_.PackageId + " " + $_.PackageVersion })
|
||||
|
||||
switch ($Repository) {
|
||||
"Nuget" {
|
||||
UpdateCSVBasedCI -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config
|
||||
break
|
||||
}
|
||||
"NPM" {
|
||||
UpdateParamsJsonJS -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config
|
||||
break
|
||||
}
|
||||
"PyPI" {
|
||||
UpdateParamsJsonPython -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config
|
||||
break
|
||||
}
|
||||
"Maven" {
|
||||
UpdatePackageJson -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config -monikerId $config.monikerid
|
||||
break
|
||||
}
|
||||
default {
|
||||
Write-Host "Unrecognized target: $Repository"
|
||||
exit(1)
|
||||
}
|
||||
if ($UpdateDocCIFn -and (Test-Path "Function:$UpdateDocCIFn"))
|
||||
{
|
||||
&$UpdateDocCIFn -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config -monikerId $config.monikerid
|
||||
}
|
||||
else
|
||||
{
|
||||
LogWarning "The function '$UpdateDocCIFn' was not found."
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -14,41 +14,22 @@ param (
|
||||
$DocRepoContentLocation = "docs-ref-services/" # within the doc repo, where does our readme go?
|
||||
)
|
||||
|
||||
. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1)
|
||||
. (Join-Path $PSScriptRoot SemVer.ps1)
|
||||
. (Join-Path $PSScriptRoot common.ps1)
|
||||
|
||||
$releaseReplaceRegex = "(https://github.com/$RepoId/(?:blob|tree)/)master"
|
||||
|
||||
function GetMetaData($lang){
|
||||
switch ($lang) {
|
||||
"java" {
|
||||
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/java-packages.csv"
|
||||
break
|
||||
}
|
||||
".net" {
|
||||
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/dotnet-packages.csv"
|
||||
break
|
||||
}
|
||||
"python" {
|
||||
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/python-packages.csv"
|
||||
break
|
||||
}
|
||||
"javascript" {
|
||||
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/js-packages.csv"
|
||||
break
|
||||
}
|
||||
default {
|
||||
Write-Host "Unrecognized Language: $language"
|
||||
exit(1)
|
||||
}
|
||||
function GetMetaData {
|
||||
if (Test-Path Variable:MetadataUri) {
|
||||
$metadataResponse = Invoke-RestMethod -Uri $MetadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv
|
||||
}
|
||||
else {
|
||||
LogError "The variable '$MetadataUri' was not found."
|
||||
}
|
||||
|
||||
$metadataResponse = Invoke-RestMethod -Uri $metadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv
|
||||
|
||||
return $metadataResponse
|
||||
}
|
||||
|
||||
function GetAdjustedReadmeContent($pkgInfo, $lang){
|
||||
function GetAdjustedReadmeContent($pkgInfo){
|
||||
$date = Get-Date -Format "MM/dd/yyyy"
|
||||
$service = ""
|
||||
|
||||
@ -56,7 +37,7 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){
|
||||
$pkgId = $pkgInfo.PackageId.Replace("@azure/", "")
|
||||
|
||||
try {
|
||||
$metadata = GetMetaData -lang $lang
|
||||
$metadata = GetMetaData
|
||||
|
||||
$service = $metadata | ? { $_.Package -eq $pkgId }
|
||||
|
||||
@ -93,8 +74,7 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){
|
||||
}
|
||||
|
||||
$apiUrl = "https://api.github.com/repos/$repoId"
|
||||
$pkgs = VerifyPackages -pkgRepository $Repository `
|
||||
-artifactLocation $ArtifactLocation `
|
||||
$pkgs = VerifyPackages -artifactLocation $ArtifactLocation `
|
||||
-workingDirectory $WorkDirectory `
|
||||
-apiUrl $apiUrl `
|
||||
-releaseSha $ReleaseSHA `
|
||||
@ -116,7 +96,7 @@ if ($pkgs) {
|
||||
$readmeLocation = Join-Path $DocRepoLocation $DocRepoContentLocation $readmeName
|
||||
|
||||
if ($packageInfo.ReadmeContent) {
|
||||
$adjustedContent = GetAdjustedReadmeContent -pkgInfo $packageInfo -lang $Language
|
||||
$adjustedContent = GetAdjustedReadmeContent -pkgInfo $packageInfo
|
||||
}
|
||||
|
||||
if ($adjustedContent) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user