Sync eng/common directory with azure-sdk-tools for PR 968 (#808)
* Restructure Language Specific Code * Restructure Copy to Blob Storage Co-authored-by: Chidozie Ononiwu <chononiw@microsoft.com>
This commit is contained in:
Azure SDK Bot
GitHub
parent
43a13e4958
commit
d67bed6973
@ -21,7 +21,6 @@ steps:
|
||||
-AzCopy $(Resolve-Path "$(Build.BinariesDirectory)/azcopy/azcopy_windows_amd64_*/azcopy.exe")[0]
|
||||
-DocLocation "${{ parameters.FolderForUpload }}"
|
||||
-SASKey "${{ parameters.BlobSASKey }}"
|
||||
-Language "${{ parameters.TargetLanguage }}"
|
||||
-BlobName "${{ parameters.BlobName }}"
|
||||
-PublicArtifactLocation "${{ parameters.ArtifactLocation }}"
|
||||
-RepoReplaceRegex "(https://github.com/${{ parameters.RepoId }}/(?:blob|tree)/)master"
|
||||
|
||||
@ -90,13 +90,14 @@ function Get-PkgProperties
|
||||
foreach ($directory in $directoriesPresent)
|
||||
{
|
||||
$pkgDirectoryPath = Join-Path $serviceDirectoryPath $directory.Name
|
||||
if ($GetPackageInfoFromRepoFn)
|
||||
|
||||
if ((Get-ChildItem -Path Function: | ? { $_.Name -eq $GetPackageInfoFromRepoFn }).Count -gt 0)
|
||||
{
|
||||
$pkgProps = &$GetPackageInfoFromRepoFn -pkgPath $pkgDirectoryPath -serviceDirectory $ServiceDirectory -pkgName $PackageName
|
||||
}
|
||||
else
|
||||
{
|
||||
Write-Error "The function 'Get-${Language}-PackageInfoFromRepo' was not found."
|
||||
Write-Error "The function '$GetPackageInfoFromRepoFn' was not found."
|
||||
}
|
||||
|
||||
if ($pkgProps -ne $null)
|
||||
|
||||
@ -10,6 +10,7 @@ $EngScriptsDir = Join-Path $EngDir "scripts"
|
||||
. (Join-Path $EngCommonScriptsDir Package-Properties.ps1)
|
||||
. (Join-Path $EngCommonScriptsDir logging.ps1)
|
||||
. (Join-Path $EngCommonScriptsDir Invoke-GitHubAPI.ps1)
|
||||
. (Join-Path $EngCommonScriptsDir artifact-metadata-parsing.ps1)
|
||||
|
||||
# Setting expected from common languages settings
|
||||
$Language = "Unknown"
|
||||
|
||||
@ -4,16 +4,14 @@ param (
|
||||
$AzCopy,
|
||||
$DocLocation,
|
||||
$SASKey,
|
||||
$Language,
|
||||
$BlobName,
|
||||
$ExitOnError=1,
|
||||
$UploadLatest=1,
|
||||
$PublicArtifactLocation = "",
|
||||
$RepoReplaceRegex = "(https://github.com/.*/(?:blob|tree)/)master"
|
||||
)
|
||||
. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1)
|
||||
|
||||
$Language = $Language.ToLower()
|
||||
. (Join-Path $PSScriptRoot common.ps1)
|
||||
|
||||
# Regex inspired but simplified from https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
|
||||
$SEMVER_REGEX = "^(?<major>0|[1-9]\d*)\.(?<minor>0|[1-9]\d*)\.(?<patch>0|[1-9]\d*)(?:-?(?<prelabel>[a-zA-Z-]*)(?:\.?(?<prenumber>0|[1-9]\d*))?)?$"
|
||||
@ -120,7 +118,7 @@ function Get-Existing-Versions
|
||||
[Parameter(Mandatory=$true)] [String]$PkgName
|
||||
)
|
||||
$versionUri = "$($BlobName)/`$web/$($Language)/$($PkgName)/versioning/versions"
|
||||
Write-Host "Heading to $versionUri to retrieve known versions"
|
||||
LogDebug "Heading to $versionUri to retrieve known versions"
|
||||
|
||||
try {
|
||||
return ((Invoke-RestMethod -Uri $versionUri -MaximumRetryCount 3 -RetryIntervalSec 5) -Split "\n" | % {$_.Trim()} | ? { return $_ })
|
||||
@ -128,12 +126,12 @@ function Get-Existing-Versions
|
||||
catch {
|
||||
# Handle 404. If it's 404, this is the first time we've published this package.
|
||||
if ($_.Exception.Response.StatusCode.value__ -eq 404){
|
||||
Write-Host "Version file does not exist. This is the first time we have published this package."
|
||||
LogDebug "Version file does not exist. This is the first time we have published this package."
|
||||
}
|
||||
else {
|
||||
# If it's not a 404. exit. We don't know what's gone wrong.
|
||||
Write-Host "Exception getting version file. Aborting"
|
||||
Write-Host $_
|
||||
LogError "Exception getting version file. Aborting"
|
||||
LogError $_
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
@ -148,18 +146,18 @@ function Update-Existing-Versions
|
||||
)
|
||||
$existingVersions = @(Get-Existing-Versions -PkgName $PkgName)
|
||||
|
||||
Write-Host "Before I update anything, I am seeing $existingVersions"
|
||||
LogDebug "Before I update anything, I am seeing $existingVersions"
|
||||
|
||||
if (!$existingVersions)
|
||||
{
|
||||
$existingVersions = @()
|
||||
$existingVersions += $PkgVersion
|
||||
Write-Host "No existing versions. Adding $PkgVersion."
|
||||
LogDebug "No existing versions. Adding $PkgVersion."
|
||||
}
|
||||
else
|
||||
{
|
||||
$existingVersions += $pkgVersion
|
||||
Write-Host "Already Existing Versions. Adding $PkgVersion."
|
||||
LogDebug "Already Existing Versions. Adding $PkgVersion."
|
||||
}
|
||||
|
||||
$existingVersions = $existingVersions | Select-Object -Unique
|
||||
@ -167,9 +165,9 @@ function Update-Existing-Versions
|
||||
# newest first
|
||||
$sortedVersionObj = (Sort-Versions -VersionArray $existingVersions)
|
||||
|
||||
Write-Host $sortedVersionObj
|
||||
Write-Host $sortedVersionObj.LatestGAPackage
|
||||
Write-Host $sortedVersionObj.LatestPreviewPackage
|
||||
LogDebug $sortedVersionObj
|
||||
LogDebug $sortedVersionObj.LatestGAPackage
|
||||
LogDebug $sortedVersionObj.LatestPreviewPackage
|
||||
|
||||
# write to file. to get the correct performance with "actually empty" files, we gotta do the newline
|
||||
# join ourselves. This way we have absolute control over the trailing whitespace.
|
||||
@ -194,12 +192,12 @@ function Upload-Blobs
|
||||
#eg : $BlobName = "https://azuresdkdocs.blob.core.windows.net"
|
||||
$DocDest = "$($BlobName)/`$web/$($Language)"
|
||||
|
||||
Write-Host "DocDest $($DocDest)"
|
||||
Write-Host "PkgName $($PkgName)"
|
||||
Write-Host "DocVersion $($DocVersion)"
|
||||
Write-Host "DocDir $($DocDir)"
|
||||
Write-Host "Final Dest $($DocDest)/$($PkgName)/$($DocVersion)"
|
||||
Write-Host "Release Tag $($ReleaseTag)"
|
||||
LogDebug "DocDest $($DocDest)"
|
||||
LogDebug "PkgName $($PkgName)"
|
||||
LogDebug "DocVersion $($DocVersion)"
|
||||
LogDebug "DocDir $($DocDir)"
|
||||
LogDebug "Final Dest $($DocDest)/$($PkgName)/$($DocVersion)"
|
||||
LogDebug "Release Tag $($ReleaseTag)"
|
||||
|
||||
# Use the step to replace master link to release tag link
|
||||
if ($ReleaseTag) {
|
||||
@ -213,13 +211,13 @@ function Upload-Blobs
|
||||
}
|
||||
}
|
||||
else {
|
||||
Write-Warning "Not able to do the master link replacement, since no release tag found for the release. Please manually check."
|
||||
LogWarning "Not able to do the master link replacement, since no release tag found for the release. Please manually check."
|
||||
}
|
||||
|
||||
Write-Host "Uploading $($PkgName)/$($DocVersion) to $($DocDest)..."
|
||||
LogDebug "Uploading $($PkgName)/$($DocVersion) to $($DocDest)..."
|
||||
& $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/$($DocVersion)$($SASKey)" --recursive=true --cache-control "max-age=300, must-revalidate"
|
||||
|
||||
Write-Host "Handling versioning files under $($DocDest)/$($PkgName)/versioning/"
|
||||
LogDebug "Handling versioning files under $($DocDest)/$($PkgName)/versioning/"
|
||||
$versionsObj = (Update-Existing-Versions -PkgName $PkgName -PkgVersion $DocVersion -DocDest $DocDest)
|
||||
|
||||
# we can safely assume we have AT LEAST one version here. Reason being we just completed Update-Existing-Versions
|
||||
@ -227,156 +225,18 @@ function Upload-Blobs
|
||||
|
||||
if ($UploadLatest -and ($latestVersion -eq $DocVersion))
|
||||
{
|
||||
Write-Host "Uploading $($PkgName) to latest folder in $($DocDest)..."
|
||||
LogDebug "Uploading $($PkgName) to latest folder in $($DocDest)..."
|
||||
& $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/latest$($SASKey)" --recursive=true --cache-control "max-age=300, must-revalidate"
|
||||
}
|
||||
}
|
||||
|
||||
if ($Language -eq "javascript")
|
||||
|
||||
if ((Get-ChildItem -Path Function: | ? { $_.Name -eq $PublishGithubIODocsFn }).Count -gt 0)
|
||||
{
|
||||
$PublishedDocs = Get-ChildItem "$($DocLocation)/documentation" | Where-Object -FilterScript {$_.Name.EndsWith(".zip")}
|
||||
|
||||
foreach ($Item in $PublishedDocs) {
|
||||
$PkgName = "azure-$($Item.BaseName)"
|
||||
Write-Host $PkgName
|
||||
Expand-Archive -Force -Path "$($DocLocation)/documentation/$($Item.Name)" -DestinationPath "$($DocLocation)/documentation/$($Item.BaseName)"
|
||||
$dirList = Get-ChildItem "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)" -Attributes Directory
|
||||
|
||||
if($dirList.Length -eq 1){
|
||||
$DocVersion = $dirList[0].Name
|
||||
Write-Host "Uploading Doc for $($PkgName) Version:- $($DocVersion)..."
|
||||
$releaseTag = RetrieveReleaseTag "NPM" $PublicArtifactLocation
|
||||
Upload-Blobs -DocDir "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)/$($DocVersion)" -PkgName $PkgName -DocVersion $DocVersion -ReleaseTag $releaseTag
|
||||
}
|
||||
else{
|
||||
Write-Host "found more than 1 folder under the documentation for package - $($Item.Name)"
|
||||
}
|
||||
}
|
||||
&$PublishGithubIODocsFn -DocLocation $DocLocation -PublicArtifactLocation $PublicArtifactLocation
|
||||
}
|
||||
else
|
||||
{
|
||||
LogWarning "The function '$PublishGithubIODocsFn' was not found."
|
||||
}
|
||||
|
||||
if ($Language -eq "dotnet")
|
||||
{
|
||||
$PublishedPkgs = Get-ChildItem "$($DocLocation)" | Where-Object -FilterScript {$_.Name.EndsWith(".nupkg") -and -not $_.Name.EndsWith(".symbols.nupkg")}
|
||||
$PublishedDocs = Get-ChildItem "$($DocLocation)" | Where-Object -FilterScript {$_.Name.EndsWith("docs.zip")}
|
||||
|
||||
if (($PublishedPkgs.Count -gt 1) -or ($PublishedDoc.Count -gt 1))
|
||||
{
|
||||
Write-Host "$($DocLocation) should contain only one (1) published package and docs"
|
||||
Write-Host "No of Packages $($PublishedPkgs.Count)"
|
||||
Write-Host "No of Docs $($PublishedDoc.Count)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
$DocsStagingDir = "$WorkingDirectory/docstaging"
|
||||
$TempDir = "$WorkingDirectory/temp"
|
||||
|
||||
New-Item -ItemType directory -Path $DocsStagingDir
|
||||
New-Item -ItemType directory -Path $TempDir
|
||||
|
||||
Expand-Archive -LiteralPath $PublishedDocs[0].FullName -DestinationPath $DocsStagingDir
|
||||
$pkgProperties = ParseNugetPackage -pkg $PublishedPkgs[0].FullName -workingDirectory $TempDir
|
||||
|
||||
Write-Host "Start Upload for $($pkgProperties.ReleaseTag)"
|
||||
Write-Host "DocDir $($DocsStagingDir)"
|
||||
Write-Host "PkgName $($pkgProperties.PackageId)"
|
||||
Write-Host "DocVersion $($pkgProperties.PackageVersion)"
|
||||
Upload-Blobs -DocDir "$($DocsStagingDir)" -PkgName $pkgProperties.PackageId -DocVersion $pkgProperties.PackageVersion -ReleaseTag $pkgProperties.ReleaseTag
|
||||
}
|
||||
|
||||
if ($Language -eq "python")
|
||||
{
|
||||
$PublishedDocs = Get-ChildItem "$DocLocation" | Where-Object -FilterScript {$_.Name.EndsWith(".zip")}
|
||||
|
||||
foreach ($Item in $PublishedDocs) {
|
||||
$PkgName = $Item.BaseName
|
||||
$ZippedDocumentationPath = Join-Path -Path $DocLocation -ChildPath $Item.Name
|
||||
$UnzippedDocumentationPath = Join-Path -Path $DocLocation -ChildPath $PkgName
|
||||
$VersionFileLocation = Join-Path -Path $UnzippedDocumentationPath -ChildPath "version.txt"
|
||||
|
||||
Expand-Archive -Force -Path $ZippedDocumentationPath -DestinationPath $UnzippedDocumentationPath
|
||||
|
||||
$Version = $(Get-Content $VersionFileLocation).Trim()
|
||||
|
||||
Write-Host "Discovered Package Name: $PkgName"
|
||||
Write-Host "Discovered Package Version: $Version"
|
||||
Write-Host "Directory for Upload: $UnzippedDocumentationPath"
|
||||
$releaseTag = RetrieveReleaseTag "PyPI" $PublicArtifactLocation
|
||||
Upload-Blobs -DocDir $UnzippedDocumentationPath -PkgName $PkgName -DocVersion $Version -ReleaseTag $releaseTag
|
||||
}
|
||||
}
|
||||
|
||||
if ($Language -eq "java")
|
||||
{
|
||||
$PublishedDocs = Get-ChildItem "$DocLocation" | Where-Object -FilterScript {$_.Name.EndsWith("-javadoc.jar")}
|
||||
foreach ($Item in $PublishedDocs) {
|
||||
$UnjarredDocumentationPath = ""
|
||||
try {
|
||||
$PkgName = $Item.BaseName
|
||||
# The jar's unpacking command doesn't allow specifying a target directory
|
||||
# and will unjar all of the files in whatever the current directory is.
|
||||
# Create a subdirectory to unjar into, set the location, unjar and then
|
||||
# set the location back to its original location.
|
||||
$UnjarredDocumentationPath = Join-Path -Path $DocLocation -ChildPath $PkgName
|
||||
New-Item -ItemType directory -Path "$UnjarredDocumentationPath"
|
||||
$CurrentLocation = Get-Location
|
||||
Set-Location $UnjarredDocumentationPath
|
||||
jar -xf "$($Item.FullName)"
|
||||
Set-Location $CurrentLocation
|
||||
|
||||
# If javadocs are produced for a library with source, there will always be an
|
||||
# index.html. If this file doesn't exist in the UnjarredDocumentationPath then
|
||||
# this is a sourceless library which means there are no javadocs and nothing
|
||||
# should be uploaded to blob storage.
|
||||
$IndexHtml = Join-Path -Path $UnjarredDocumentationPath -ChildPath "index.html"
|
||||
if (!(Test-Path -path $IndexHtml))
|
||||
{
|
||||
Write-Host "$($PkgName) does not have an index.html file, skippping."
|
||||
continue
|
||||
}
|
||||
|
||||
# Get the POM file for the artifact we're processing
|
||||
$PomFile = $Item.FullName.Substring(0,$Item.FullName.LastIndexOf(("-javadoc.jar"))) + ".pom"
|
||||
Write-Host "PomFile $($PomFile)"
|
||||
|
||||
# Pull the version from the POM
|
||||
[xml]$PomXml = Get-Content $PomFile
|
||||
$Version = $PomXml.project.version
|
||||
$ArtifactId = $PomXml.project.artifactId
|
||||
|
||||
Write-Host "Start Upload for $($PkgName)/$($Version)"
|
||||
Write-Host "DocDir $($UnjarredDocumentationPath)"
|
||||
Write-Host "PkgName $($ArtifactId)"
|
||||
Write-Host "DocVersion $($Version)"
|
||||
$releaseTag = RetrieveReleaseTag "Maven" $PublicArtifactLocation
|
||||
Upload-Blobs -DocDir $UnjarredDocumentationPath -PkgName $ArtifactId -DocVersion $Version -ReleaseTag $releaseTag
|
||||
|
||||
} Finally {
|
||||
if (![string]::IsNullOrEmpty($UnjarredDocumentationPath)) {
|
||||
if (Test-Path -Path $UnjarredDocumentationPath) {
|
||||
Write-Host "Cleaning up $UnjarredDocumentationPath"
|
||||
Remove-Item -Recurse -Force $UnjarredDocumentationPath
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($Language -eq "c")
|
||||
{
|
||||
# The documentation publishing process for C differs from the other
|
||||
# langauges in this file because this script is invoked for the whole SDK
|
||||
# publishing. It is not, for example, invoked once per service publishing.
|
||||
# There is a similar situation for other langauge publishing steps above...
|
||||
# Those loops are left over from previous versions of this script which were
|
||||
# used to publish multiple docs packages in a single invocation.
|
||||
$pkgInfo = Get-Content $DocLocation/package-info.json | ConvertFrom-Json
|
||||
$releaseTag = RetrieveReleaseTag "C" $PublicArtifactLocation
|
||||
Upload-Blobs -DocDir $DocLocation -PkgName 'docs' -DocVersion $pkgInfo.version -ReleaseTag $releaseTag
|
||||
}
|
||||
|
||||
if ($Language -eq "cpp")
|
||||
{
|
||||
$packageInfo = (Get-Content (Join-Path $DocLocation 'package-info.json') | ConvertFrom-Json)
|
||||
$releaseTag = RetrieveReleaseTag "CPP" $PublicArtifactLocation
|
||||
Upload-Blobs -DocDir $DocLocation -PkgName $packageInfo.name -DocVersion $packageInfo.version -ReleaseTag $releaseTag
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user