From 97845b77eb600fb666b420ac64cf480e79bde7f8 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Wed, 23 Aug 2023 14:53:33 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 6632 (#4912) * Docs Onboarding 2 - Add eng/common changes * Dictionary Syntax * Remove $UpdatePackageMetadata, it can be handled in other places * Set exit code to 0 on success. The last executable to run (pip or docker or other) may be passed through * Add optional _DocsOnboardingOrdinal sorting to work around onboarding sequence problems * Update-DocsMsPackages2.ps1 -> Update-DocsMsPackages.ps1 --------- Co-authored-by: Daniel Jurek --- eng/common/scripts/Update-DocsMsPackages.ps1 | 141 ++++++++++--------- eng/common/scripts/common.ps1 | 6 + 2 files changed, 78 insertions(+), 69 deletions(-) diff --git a/eng/common/scripts/Update-DocsMsPackages.ps1 b/eng/common/scripts/Update-DocsMsPackages.ps1 index e84c1d566..00ab4b458 100644 --- a/eng/common/scripts/Update-DocsMsPackages.ps1 +++ b/eng/common/scripts/Update-DocsMsPackages.ps1 @@ -37,90 +37,93 @@ param ( ) . (Join-Path $PSScriptRoot common.ps1) +. "$PSScriptRoot/../../scripts/docs/Docs-Onboarding.ps1" -function GetDocsMetadataForMoniker($moniker) { - $searchPath = Join-Path $DocRepoLocation 'metadata' $moniker - if (!(Test-Path $searchPath)) { - return @() - } - $paths = Get-ChildItem -Path $searchPath -Filter *.json +Set-StrictMode -Version 3 +function GetMetadata($moniker) { + $jsonFiles = Get-ChildItem -Path (Join-Path $DocRepoLocation "metadata/$moniker") -Filter *.json $metadata = @() - foreach ($path in $paths) { - $fileContents = Get-Content $path -Raw - $fileObject = ConvertFrom-Json -InputObject $fileContents - $versionGa = '' - $versionPreview = '' - if ($moniker -eq 'latest') { - $versionGa = $fileObject.Version - } else { - $versionPreview = $fileObject.Version - } - - $entry = @{ - Package = $fileObject.Name; - VersionGA = $versionGa; - VersionPreview = $versionPreview; - RepoPath = $fileObject.ServiceDirectory; - Type = $fileObject.SdkType; - New = $fileObject.IsNewSdk; - } - if ($fileObject.PSObject.Members.Name -contains "Group") - { - $entry.Add("GroupId", $fileObject.Group) - } - $metadata += $entry + foreach ($jsonFile in $jsonFiles) { + # Converting to a hashtable gives more beneficial semantics for handling + # metadata (easier to check existence of property, easier to add new + # properties) + $metadata += Get-Content $jsonFile -Raw | ConvertFrom-Json -AsHashtable } return $metadata } -function GetDocsMetadata() { - # Read metadata from docs repo - $metadataByPackage = @{} - foreach ($package in GetDocsMetadataForMoniker 'latest') { - if ($metadataByPackage.ContainsKey($package.Package)) { - LogWarning "Duplicate package in latest metadata: $($package.Package)" - } - Write-Host "Adding latest package: $($package.Package)" - $metadataByPackage[$package.Package] = $package +function ValidatePackageForOnboarding2($package) { + if (!(Test-Path "Function:$ValidateDocsMsPackagesFn")) { + return $true } - foreach ($package in GetDocsMetadataForMoniker 'preview') { - if ($metadataByPackage.ContainsKey($package.Package)) { - # Merge VersionPreview of each object - Write-Host "Merging preview package version for $($package.Package))" - $metadataByPackage[$package.Package].VersionPreview = $package.VersionPreview - } else { - Write-Host "Adding preview package: $($package.Package)" - $metadataByPackage[$package.Package] = $package - } - } - - # TODO - Add a call to GetDocsMetadataForMoniker for 'legacy' when that is implemented - - return $metadataByPackage.Values + return &$ValidateDocsMsPackagesFn ` + -PackageInfo $package ` + -DocValidationImageId $ImageId ` + -DocRepoLocation $DocRepoLocation } -if ($UpdateDocsMsPackagesFn -and (Test-Path "Function:$UpdateDocsMsPackagesFn")) { - +$MONIKERS = @('latest', 'preview', 'legacy') +foreach ($moniker in $MONIKERS) { try { - $docsMetadata = GetDocsMetadata - &$UpdateDocsMsPackagesFn -DocsRepoLocation $DocRepoLocation -DocsMetadata $docsMetadata -PackageSourceOverride $PackageSourceOverride -DocValidationImageId $ImageId - } catch { - LogError "Exception while updating docs.ms packages" - LogError $_ - LogError $_.ScriptStackTrace + Write-Host "Onboarding packages for moniker: $moniker" + $metadata = GetMetadata $moniker + $alreadyOnboardedPackages = &$GetDocsPackagesAlreadyOnboarded $DocRepoLocation $moniker + + # Sort the metadata entries by package name so that the output is + # deterministic (more simple diffs) + $sortedMetadata = $metadata | Sort-Object -Property '_DocsOnboardingOrdinal', 'Name' + + $outputPackages = @() + foreach ($package in $sortedMetadata) { + $packageIdentity = $package.Name + if (Test-Path "Function:$GetPackageIdentity") { + $packageIdentity = &$GetPackageIdentity $package + } + + if (!($alreadyOnboardedPackages.ContainsKey($packageIdentity))) { + Write-Host "Evaluating package for onboarding: $($packageIdentity)" + if ($package.ContainsKey('_SkipDocsValidation') -and $true -eq $package['_SkipDocsValidation']) { + Write-Host "Skip validation for package: $($packageIdentity)" + } + elseif (!(ValidatePackageForOnboarding2 $package)) { + LogWarning "Skip adding package that did not pass validation: $($packageIdentity)" + continue + } + + Write-Host "Add new package: $($packageIdentity)@$($package.Version)" + $outputPackages += $package + continue + } + + $oldPackage = $alreadyOnboardedPackages[$packageIdentity] + + if ($oldPackage.Version -ne $package.Version) { + if (!(ValidatePackageForOnboarding2 $package)) { + LogWarning "Omitting package that failed validation: $($packageIdentity)@$($package.Version)" + continue + } + + Write-Host "Update package: $($packageIdentity)@$($oldPackage.Version) to $($packageIdentity)@$($package.Version)" + $outputPackages += $package + continue + } + + Write-Host "Unchanged package: $($packageIdentity)@$($package.Version)" + $outputPackages += $package + } + + &$SetDocsPackageOnboarding $moniker $outputPackages $DocRepoLocation $PackageSourceOverride + } + catch { + Write-Host "Error onboarding packages for moniker: $moniker" + Write-Host "Error: $_" + Write-Host "Stacktrace: $($_.ScriptStackTrace)" + Write-Error $_ exit 1 } - -} else { - LogError "The function for '$UpdateFn' was not found.` - Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.` - See https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/common_engsys.md#code-structure" - exit 1 } -# Exit 0 so DevOps doesn't fail the build when the last command called by the -# domain-specific function exited with a non-zero exit code. exit 0 diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 8f9c707ee..0770de667 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -40,6 +40,7 @@ if (!(Get-Variable -Name "LanguageDisplayName" -ValueOnly -ErrorAction "Ignore") } # Transformed Functions +# Expected to be set in eng/scripts/Language-Settings.ps1 $GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo" $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" $PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs" @@ -60,3 +61,8 @@ $GetRepositoryLinkFn = "Get-${Language}-RepositoryLink" $GetEmitterAdditionalOptionsFn = "Get-${Language}-EmitterAdditionalOptions" $GetEmitterNameFn = "Get-${Language}-EmitterName" $GetEmitterPackageJsonPathFn = "Get-${Language}-EmitterPackageJsonPath" + +# Expected to be set in eng/scripts/docs/Docs-Onboarding.ps1 +$SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding" +$GetDocsPackagesAlreadyOnboarded = "Get-${Language}-DocsPackagesAlreadyOnboarded" +$GetPackageIdentity = "Get-${Language}-PackageIdentity"