diff --git a/eng/common/pipelines/templates/steps/publish-blobs.yml b/eng/common/pipelines/templates/steps/publish-blobs.yml index 5888edff8..7446c1af6 100644 --- a/eng/common/pipelines/templates/steps/publish-blobs.yml +++ b/eng/common/pipelines/templates/steps/publish-blobs.yml @@ -21,7 +21,6 @@ steps: -AzCopy $(Resolve-Path "$(Build.BinariesDirectory)/azcopy/azcopy_windows_amd64_*/azcopy.exe")[0] -DocLocation "${{ parameters.FolderForUpload }}" -SASKey "${{ parameters.BlobSASKey }}" - -Language "${{ parameters.TargetLanguage }}" -BlobName "${{ parameters.BlobName }}" -PublicArtifactLocation "${{ parameters.ArtifactLocation }}" -RepoReplaceRegex "(https://github.com/${{ parameters.RepoId }}/(?:blob|tree)/)master" diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 2ca4aacdb..f17b244f3 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -90,13 +90,14 @@ function Get-PkgProperties foreach ($directory in $directoriesPresent) { $pkgDirectoryPath = Join-Path $serviceDirectoryPath $directory.Name - if ($GetPackageInfoFromRepoFn) + + if ((Get-ChildItem -Path Function: | ? { $_.Name -eq $GetPackageInfoFromRepoFn }).Count -gt 0) { $pkgProps = &$GetPackageInfoFromRepoFn -pkgPath $pkgDirectoryPath -serviceDirectory $ServiceDirectory -pkgName $PackageName } else { - Write-Error "The function 'Get-${Language}-PackageInfoFromRepo' was not found." + Write-Error "The function '$GetPackageInfoFromRepoFn' was not found." } if ($pkgProps -ne $null) diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 1f1228fce..891821529 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -10,6 +10,7 @@ $EngScriptsDir = Join-Path $EngDir "scripts" . (Join-Path $EngCommonScriptsDir Package-Properties.ps1) . (Join-Path $EngCommonScriptsDir logging.ps1) . (Join-Path $EngCommonScriptsDir Invoke-GitHubAPI.ps1) +. (Join-Path $EngCommonScriptsDir artifact-metadata-parsing.ps1) # Setting expected from common languages settings $Language = "Unknown" diff --git a/eng/common/scripts/copy-docs-to-blobstorage.ps1 b/eng/common/scripts/copy-docs-to-blobstorage.ps1 index c10f15549..a81fc485f 100644 --- a/eng/common/scripts/copy-docs-to-blobstorage.ps1 +++ b/eng/common/scripts/copy-docs-to-blobstorage.ps1 @@ -4,16 +4,14 @@ param ( $AzCopy, $DocLocation, $SASKey, - $Language, $BlobName, $ExitOnError=1, $UploadLatest=1, $PublicArtifactLocation = "", $RepoReplaceRegex = "(https://github.com/.*/(?:blob|tree)/)master" ) -. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) -$Language = $Language.ToLower() +. (Join-Path $PSScriptRoot common.ps1) # Regex inspired but simplified from https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string $SEMVER_REGEX = "^(?0|[1-9]\d*)\.(?0|[1-9]\d*)\.(?0|[1-9]\d*)(?:-?(?[a-zA-Z-]*)(?:\.?(?0|[1-9]\d*))?)?$" @@ -120,7 +118,7 @@ function Get-Existing-Versions [Parameter(Mandatory=$true)] [String]$PkgName ) $versionUri = "$($BlobName)/`$web/$($Language)/$($PkgName)/versioning/versions" - Write-Host "Heading to $versionUri to retrieve known versions" + LogDebug "Heading to $versionUri to retrieve known versions" try { return ((Invoke-RestMethod -Uri $versionUri -MaximumRetryCount 3 -RetryIntervalSec 5) -Split "\n" | % {$_.Trim()} | ? { return $_ }) @@ -128,12 +126,12 @@ function Get-Existing-Versions catch { # Handle 404. If it's 404, this is the first time we've published this package. if ($_.Exception.Response.StatusCode.value__ -eq 404){ - Write-Host "Version file does not exist. This is the first time we have published this package." + LogDebug "Version file does not exist. This is the first time we have published this package." } else { # If it's not a 404. exit. We don't know what's gone wrong. - Write-Host "Exception getting version file. Aborting" - Write-Host $_ + LogError "Exception getting version file. Aborting" + LogError $_ exit(1) } } @@ -148,18 +146,18 @@ function Update-Existing-Versions ) $existingVersions = @(Get-Existing-Versions -PkgName $PkgName) - Write-Host "Before I update anything, I am seeing $existingVersions" + LogDebug "Before I update anything, I am seeing $existingVersions" if (!$existingVersions) { $existingVersions = @() $existingVersions += $PkgVersion - Write-Host "No existing versions. Adding $PkgVersion." + LogDebug "No existing versions. Adding $PkgVersion." } else { $existingVersions += $pkgVersion - Write-Host "Already Existing Versions. Adding $PkgVersion." + LogDebug "Already Existing Versions. Adding $PkgVersion." } $existingVersions = $existingVersions | Select-Object -Unique @@ -167,9 +165,9 @@ function Update-Existing-Versions # newest first $sortedVersionObj = (Sort-Versions -VersionArray $existingVersions) - Write-Host $sortedVersionObj - Write-Host $sortedVersionObj.LatestGAPackage - Write-Host $sortedVersionObj.LatestPreviewPackage + LogDebug $sortedVersionObj + LogDebug $sortedVersionObj.LatestGAPackage + LogDebug $sortedVersionObj.LatestPreviewPackage # write to file. to get the correct performance with "actually empty" files, we gotta do the newline # join ourselves. This way we have absolute control over the trailing whitespace. @@ -194,12 +192,12 @@ function Upload-Blobs #eg : $BlobName = "https://azuresdkdocs.blob.core.windows.net" $DocDest = "$($BlobName)/`$web/$($Language)" - Write-Host "DocDest $($DocDest)" - Write-Host "PkgName $($PkgName)" - Write-Host "DocVersion $($DocVersion)" - Write-Host "DocDir $($DocDir)" - Write-Host "Final Dest $($DocDest)/$($PkgName)/$($DocVersion)" - Write-Host "Release Tag $($ReleaseTag)" + LogDebug "DocDest $($DocDest)" + LogDebug "PkgName $($PkgName)" + LogDebug "DocVersion $($DocVersion)" + LogDebug "DocDir $($DocDir)" + LogDebug "Final Dest $($DocDest)/$($PkgName)/$($DocVersion)" + LogDebug "Release Tag $($ReleaseTag)" # Use the step to replace master link to release tag link if ($ReleaseTag) { @@ -213,13 +211,13 @@ function Upload-Blobs } } else { - Write-Warning "Not able to do the master link replacement, since no release tag found for the release. Please manually check." + LogWarning "Not able to do the master link replacement, since no release tag found for the release. Please manually check." } - Write-Host "Uploading $($PkgName)/$($DocVersion) to $($DocDest)..." + LogDebug "Uploading $($PkgName)/$($DocVersion) to $($DocDest)..." & $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/$($DocVersion)$($SASKey)" --recursive=true --cache-control "max-age=300, must-revalidate" - Write-Host "Handling versioning files under $($DocDest)/$($PkgName)/versioning/" + LogDebug "Handling versioning files under $($DocDest)/$($PkgName)/versioning/" $versionsObj = (Update-Existing-Versions -PkgName $PkgName -PkgVersion $DocVersion -DocDest $DocDest) # we can safely assume we have AT LEAST one version here. Reason being we just completed Update-Existing-Versions @@ -227,156 +225,18 @@ function Upload-Blobs if ($UploadLatest -and ($latestVersion -eq $DocVersion)) { - Write-Host "Uploading $($PkgName) to latest folder in $($DocDest)..." + LogDebug "Uploading $($PkgName) to latest folder in $($DocDest)..." & $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/latest$($SASKey)" --recursive=true --cache-control "max-age=300, must-revalidate" } } -if ($Language -eq "javascript") + +if ((Get-ChildItem -Path Function: | ? { $_.Name -eq $PublishGithubIODocsFn }).Count -gt 0) { - $PublishedDocs = Get-ChildItem "$($DocLocation)/documentation" | Where-Object -FilterScript {$_.Name.EndsWith(".zip")} - - foreach ($Item in $PublishedDocs) { - $PkgName = "azure-$($Item.BaseName)" - Write-Host $PkgName - Expand-Archive -Force -Path "$($DocLocation)/documentation/$($Item.Name)" -DestinationPath "$($DocLocation)/documentation/$($Item.BaseName)" - $dirList = Get-ChildItem "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)" -Attributes Directory - - if($dirList.Length -eq 1){ - $DocVersion = $dirList[0].Name - Write-Host "Uploading Doc for $($PkgName) Version:- $($DocVersion)..." - $releaseTag = RetrieveReleaseTag "NPM" $PublicArtifactLocation - Upload-Blobs -DocDir "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)/$($DocVersion)" -PkgName $PkgName -DocVersion $DocVersion -ReleaseTag $releaseTag - } - else{ - Write-Host "found more than 1 folder under the documentation for package - $($Item.Name)" - } - } + &$PublishGithubIODocsFn -DocLocation $DocLocation -PublicArtifactLocation $PublicArtifactLocation +} +else +{ + LogWarning "The function '$PublishGithubIODocsFn' was not found." } -if ($Language -eq "dotnet") -{ - $PublishedPkgs = Get-ChildItem "$($DocLocation)" | Where-Object -FilterScript {$_.Name.EndsWith(".nupkg") -and -not $_.Name.EndsWith(".symbols.nupkg")} - $PublishedDocs = Get-ChildItem "$($DocLocation)" | Where-Object -FilterScript {$_.Name.EndsWith("docs.zip")} - - if (($PublishedPkgs.Count -gt 1) -or ($PublishedDoc.Count -gt 1)) - { - Write-Host "$($DocLocation) should contain only one (1) published package and docs" - Write-Host "No of Packages $($PublishedPkgs.Count)" - Write-Host "No of Docs $($PublishedDoc.Count)" - exit 1 - } - - $DocsStagingDir = "$WorkingDirectory/docstaging" - $TempDir = "$WorkingDirectory/temp" - - New-Item -ItemType directory -Path $DocsStagingDir - New-Item -ItemType directory -Path $TempDir - - Expand-Archive -LiteralPath $PublishedDocs[0].FullName -DestinationPath $DocsStagingDir - $pkgProperties = ParseNugetPackage -pkg $PublishedPkgs[0].FullName -workingDirectory $TempDir - - Write-Host "Start Upload for $($pkgProperties.ReleaseTag)" - Write-Host "DocDir $($DocsStagingDir)" - Write-Host "PkgName $($pkgProperties.PackageId)" - Write-Host "DocVersion $($pkgProperties.PackageVersion)" - Upload-Blobs -DocDir "$($DocsStagingDir)" -PkgName $pkgProperties.PackageId -DocVersion $pkgProperties.PackageVersion -ReleaseTag $pkgProperties.ReleaseTag -} - -if ($Language -eq "python") -{ - $PublishedDocs = Get-ChildItem "$DocLocation" | Where-Object -FilterScript {$_.Name.EndsWith(".zip")} - - foreach ($Item in $PublishedDocs) { - $PkgName = $Item.BaseName - $ZippedDocumentationPath = Join-Path -Path $DocLocation -ChildPath $Item.Name - $UnzippedDocumentationPath = Join-Path -Path $DocLocation -ChildPath $PkgName - $VersionFileLocation = Join-Path -Path $UnzippedDocumentationPath -ChildPath "version.txt" - - Expand-Archive -Force -Path $ZippedDocumentationPath -DestinationPath $UnzippedDocumentationPath - - $Version = $(Get-Content $VersionFileLocation).Trim() - - Write-Host "Discovered Package Name: $PkgName" - Write-Host "Discovered Package Version: $Version" - Write-Host "Directory for Upload: $UnzippedDocumentationPath" - $releaseTag = RetrieveReleaseTag "PyPI" $PublicArtifactLocation - Upload-Blobs -DocDir $UnzippedDocumentationPath -PkgName $PkgName -DocVersion $Version -ReleaseTag $releaseTag - } -} - -if ($Language -eq "java") -{ - $PublishedDocs = Get-ChildItem "$DocLocation" | Where-Object -FilterScript {$_.Name.EndsWith("-javadoc.jar")} - foreach ($Item in $PublishedDocs) { - $UnjarredDocumentationPath = "" - try { - $PkgName = $Item.BaseName - # The jar's unpacking command doesn't allow specifying a target directory - # and will unjar all of the files in whatever the current directory is. - # Create a subdirectory to unjar into, set the location, unjar and then - # set the location back to its original location. - $UnjarredDocumentationPath = Join-Path -Path $DocLocation -ChildPath $PkgName - New-Item -ItemType directory -Path "$UnjarredDocumentationPath" - $CurrentLocation = Get-Location - Set-Location $UnjarredDocumentationPath - jar -xf "$($Item.FullName)" - Set-Location $CurrentLocation - - # If javadocs are produced for a library with source, there will always be an - # index.html. If this file doesn't exist in the UnjarredDocumentationPath then - # this is a sourceless library which means there are no javadocs and nothing - # should be uploaded to blob storage. - $IndexHtml = Join-Path -Path $UnjarredDocumentationPath -ChildPath "index.html" - if (!(Test-Path -path $IndexHtml)) - { - Write-Host "$($PkgName) does not have an index.html file, skippping." - continue - } - - # Get the POM file for the artifact we're processing - $PomFile = $Item.FullName.Substring(0,$Item.FullName.LastIndexOf(("-javadoc.jar"))) + ".pom" - Write-Host "PomFile $($PomFile)" - - # Pull the version from the POM - [xml]$PomXml = Get-Content $PomFile - $Version = $PomXml.project.version - $ArtifactId = $PomXml.project.artifactId - - Write-Host "Start Upload for $($PkgName)/$($Version)" - Write-Host "DocDir $($UnjarredDocumentationPath)" - Write-Host "PkgName $($ArtifactId)" - Write-Host "DocVersion $($Version)" - $releaseTag = RetrieveReleaseTag "Maven" $PublicArtifactLocation - Upload-Blobs -DocDir $UnjarredDocumentationPath -PkgName $ArtifactId -DocVersion $Version -ReleaseTag $releaseTag - - } Finally { - if (![string]::IsNullOrEmpty($UnjarredDocumentationPath)) { - if (Test-Path -Path $UnjarredDocumentationPath) { - Write-Host "Cleaning up $UnjarredDocumentationPath" - Remove-Item -Recurse -Force $UnjarredDocumentationPath - } - } - } - } -} - -if ($Language -eq "c") -{ - # The documentation publishing process for C differs from the other - # langauges in this file because this script is invoked for the whole SDK - # publishing. It is not, for example, invoked once per service publishing. - # There is a similar situation for other langauge publishing steps above... - # Those loops are left over from previous versions of this script which were - # used to publish multiple docs packages in a single invocation. - $pkgInfo = Get-Content $DocLocation/package-info.json | ConvertFrom-Json - $releaseTag = RetrieveReleaseTag "C" $PublicArtifactLocation - Upload-Blobs -DocDir $DocLocation -PkgName 'docs' -DocVersion $pkgInfo.version -ReleaseTag $releaseTag -} - -if ($Language -eq "cpp") -{ - $packageInfo = (Get-Content (Join-Path $DocLocation 'package-info.json') | ConvertFrom-Json) - $releaseTag = RetrieveReleaseTag "CPP" $PublicArtifactLocation - Upload-Blobs -DocDir $DocLocation -PkgName $packageInfo.name -DocVersion $packageInfo.version -ReleaseTag $releaseTag -}