Sync eng/common directory with azure-sdk-tools for PR 12478 (#6793)

* Updated validate pkg template to use packageInfo

* Fixed typo

* Fixed the right variable to use

* output debug log

* Fixed errors in expression evaluation

* removed debug code

* Fixed an issue in pipeline

* Updated condition for variable setting step

* Join paths of the script path

* Use join-path

* return from the function rather than exit

---------

Co-authored-by: ray chen <raychen@microsoft.com>
This commit is contained in:
Azure SDK Bot 2025-10-20 09:49:40 -07:00 committed by GitHub
parent 397ae78b1e
commit 3587514b29
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 438 additions and 50 deletions

View File

@ -1,14 +1,23 @@
parameters:
ArtifactPath: $(Build.ArtifactStagingDirectory)
Artifacts: []
ConfigFileDir: $(Build.ArtifactStagingDirectory)/PackageInfo
- name: ArtifactPath
type: string
default: $(Build.ArtifactStagingDirectory)
- name: Artifacts
type: object
default: []
- name: ConfigFileDir
type: string
default: $(Build.ArtifactStagingDirectory)/PackageInfo
- name: PackageInfoFiles
type: object
default: []
steps:
- ${{ if and(ne(variables['Skip.PackageValidation'], 'true'), and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))) }}:
- pwsh: |
echo "##vso[task.setvariable variable=SetAsReleaseBuild]false"
displayName: "Set as release build"
condition: and(succeeded(), eq(variables['SetAsReleaseBuild'], ''))
condition: and(succeededOrFailed(), eq(variables['SetAsReleaseBuild'], ''))
- task: AzureCLI@2
inputs:
@ -24,7 +33,8 @@ steps:
-ConfigFileDir '${{ parameters.ConfigFileDir }}' `
-BuildDefinition $(System.CollectionUri)$(System.TeamProject)/_build?definitionId=$(System.DefinitionId) `
-PipelineUrl $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId) `
-IsReleaseBuild $$(SetAsReleaseBuild)
-IsReleaseBuild $$(SetAsReleaseBuild) `
-PackageInfoFiles ('${{ convertToJson(parameters.PackageInfoFiles) }}' | ConvertFrom-Json -NoEnumerate)
workingDirectory: $(Pipeline.Workspace)
displayName: Validate packages and update work items
continueOnError: true

View File

@ -1,8 +1,15 @@
. (Join-Path $PSScriptRoot .. SemVer.ps1)
$ReleaseDevOpsOrgParameters = @("--organization", "https://dev.azure.com/azure-sdk")
$ReleaseDevOpsCommonParameters = $ReleaseDevOpsOrgParameters + @("--output", "json")
$ReleaseDevOpsCommonParametersWithProject = $ReleaseDevOpsCommonParameters + @("--project", "Release")
# This is used to determine whether or not the az login and azure-devops extension
# install have already been completed.
$global:AzLoginAndDevOpsExtensionInstallComplete = $false
$global:HasDevOpsAccess = $false
function Get-DevOpsRestHeaders()
{
# Get a temp access token from the logged in az cli user for azure devops resource
@ -17,17 +24,44 @@ function Get-DevOpsRestHeaders()
return $headers
}
# Function was created from the same code being in Update-DevOps-Release-WorkItem.ps1
# and Validate-Package.ps1. The global variable is used to prevent az commands from
# being rerun multiple times
function CheckAzLoginAndDevOpsExtensionInstall()
{
if (-not $global:AzLoginAndDevOpsExtensionInstallComplete) {
az account show *> $null
if (!$?) {
Write-Host 'Running az login...'
az login *> $null
}
az extension show -n azure-devops *> $null
if (!$?){
az extension add --name azure-devops
} else {
# Force update the extension to the latest version if it was already installed
# this is needed to ensure we have the authentication issue fixed from earlier versions
az extension update -n azure-devops *> $null
}
$global:AzLoginAndDevOpsExtensionInstallComplete = $true
}
}
function CheckDevOpsAccess()
{
# Dummy test query to validate permissions
$query = "SELECT [System.ID] FROM WorkItems WHERE [Work Item Type] = 'Package' AND [Package] = 'azure-sdk-template'"
if (-not $global:HasDevOpsAccess) {
# Dummy test query to validate permissions
$query = "SELECT [System.ID] FROM WorkItems WHERE [Work Item Type] = 'Package' AND [Package] = 'azure-sdk-template'"
$response = Invoke-RestMethod -Method POST `
-Uri "https://dev.azure.com/azure-sdk/Release/_apis/wit/wiql/?api-version=6.0" `
-Headers (Get-DevOpsRestHeaders) -Body "{ ""query"": ""$query"" }" -ContentType "application/json" | ConvertTo-Json -Depth 10 | ConvertFrom-Json -AsHashTable
$response = Invoke-RestMethod -Method POST `
-Uri "https://dev.azure.com/azure-sdk/Release/_apis/wit/wiql/?api-version=6.0" `
-Headers (Get-DevOpsRestHeaders) -Body "{ ""query"": ""$query"" }" -ContentType "application/json" | ConvertTo-Json -Depth 10 | ConvertFrom-Json -AsHashTable
if ($response -isnot [HashTable] -or !$response.ContainsKey("workItems")) {
throw "Failed to run test query against Azure DevOps. Please ensure you are logged into the public azure cloud. Consider running 'az logout' and then 'az login'."
if ($response -isnot [HashTable] -or !$response.ContainsKey("workItems")) {
throw "Failed to run test query against Azure DevOps. Please ensure you are logged into the public azure cloud. Consider running 'az logout' and then 'az login'."
}
$global:HasDevOpsAccess = $true
}
}
@ -1018,7 +1052,7 @@ function UpdateValidationStatus($pkgvalidationDetails, $BuildDefinition, $Pipeli
function Get-LanguageDevOpsName($LanguageShort)
{
switch ($LanguageShort.ToLower())
switch ($LanguageShort.ToLower())
{
"net" { return "Dotnet" }
"js" { return "JavaScript" }
@ -1057,7 +1091,7 @@ function Get-ReleasePlanForPackage($packageName)
}
function Update-ReleaseStatusInReleasePlan($releasePlanWorkItemId, $status, $version)
{
{
$devopsFieldLanguage = Get-LanguageDevOpsName -LanguageShort $LanguageShort
if (!$devopsFieldLanguage)
{
@ -1170,7 +1204,7 @@ function Get-TriagesForCPEXAttestation()
$query += " AND [Custom.ProductType] IN ('Feature', 'Offering', 'Sku')"
$workItems = Invoke-Query $fields $query
return $workItems
return $workItems
}
function Update-AttestationStatusInWorkItem($workItemId, $fieldName, $status)
@ -1181,4 +1215,96 @@ function Update-AttestationStatusInWorkItem($workItemId, $fieldName, $status)
$workItem = UpdateWorkItem -id $workItemId -fields $fields
Write-Host "Updated attestation status for [$fieldName] in Work Item [$workItemId]"
return $true
}
}
# This function was originally the entirety of what was in Update-DevOps-Release-WorkItem.ps1
# and has been converted to a function.
function Update-DevOpsReleaseWorkItem {
param(
[Parameter(Mandatory=$true)]
[string]$language,
[Parameter(Mandatory=$true)]
[string]$packageName,
[Parameter(Mandatory=$true)]
[string]$version,
[string]$plannedDate,
[string]$serviceName = $null,
[string]$packageDisplayName = $null,
[string]$packageRepoPath = "NA",
[string]$packageType = "client",
[string]$packageNewLibrary = "true",
[string]$relatedWorkItemId = $null,
[string]$tag = $null,
[bool]$inRelease = $true
)
if (!(Get-Command az -ErrorAction SilentlyContinue)) {
Write-Error 'You must have the Azure CLI installed: https://aka.ms/azure-cli'
return $false
}
CheckAzLoginAndDevOpsExtensionInstall
CheckDevOpsAccess
$parsedNewVersion = [AzureEngSemanticVersion]::new($version)
$state = "In Release"
$releaseType = $parsedNewVersion.VersionType
$versionMajorMinor = "" + $parsedNewVersion.Major + "." + $parsedNewVersion.Minor
$packageInfo = [PSCustomObject][ordered]@{
Package = $packageName
DisplayName = $packageDisplayName
ServiceName = $serviceName
RepoPath = $packageRepoPath
Type = $packageType
New = $packageNewLibrary
};
if (!$plannedDate) {
$plannedDate = Get-Date -Format "MM/dd/yyyy"
}
$plannedVersions = @(
[PSCustomObject][ordered]@{
Type = $releaseType
Version = $version
Date = $plannedDate
}
)
$ignoreReleasePlannerTests = $true
if ($tag -and $tag.Contains("Release Planner App Test")) {
$ignoreReleasePlannerTests = $false
}
$workItem = FindOrCreateClonePackageWorkItem $language $packageInfo $versionMajorMinor -allowPrompt $true -outputCommand $false -relatedId $relatedWorkItemId -tag $tag -ignoreReleasePlannerTests $ignoreReleasePlannerTests
if (!$workItem) {
Write-Host "Something failed as we don't have a work-item so exiting."
return $false
}
Write-Host "Updated or created a release work item for a package release with the following properties:"
Write-Host " Language: $($workItem.fields['Custom.Language'])"
Write-Host " Version: $($workItem.fields['Custom.PackageVersionMajorMinor'])"
Write-Host " Package: $($workItem.fields['Custom.Package'])"
if ($workItem.fields['System.AssignedTo']) {
Write-Host " AssignedTo: $($workItem.fields['System.AssignedTo']["uniqueName"])"
}
else {
Write-Host " AssignedTo: unassigned"
}
Write-Host " PackageDisplayName: $($workItem.fields['Custom.PackageDisplayName'])"
Write-Host " ServiceName: $($workItem.fields['Custom.ServiceName'])"
Write-Host " PackageType: $($workItem.fields['Custom.PackageType'])"
Write-Host ""
if ($inRelease)
{
Write-Host "Marking item [$($workItem.id)]$($workItem.fields['System.Title']) as '$state' for '$releaseType'"
$updatedWI = UpdatePackageWorkItemReleaseState -id $workItem.id -state "In Release" -releaseType $releaseType -outputCommand $false
}
$updatedWI = UpdatePackageVersions $workItem -plannedVersions $plannedVersions
Write-Host "Release tracking item is at https://dev.azure.com/azure-sdk/Release/_workitems/edit/$($updatedWI.id)/"
return $true
}

View File

@ -55,6 +55,7 @@ Set-StrictMode -Version 3
. ${PSScriptRoot}\common.ps1
. ${PSScriptRoot}\Helpers\ApiView-Helpers.ps1
. ${PSScriptRoot}\Helpers\DevOps-WorkItem-Helpers.ps1
function Get-ReleaseDay($baseDate)
{
@ -141,18 +142,18 @@ if ($null -eq $newVersionParsed)
exit 1
}
&$EngCommonScriptsDir/Update-DevOps-Release-WorkItem.ps1 `
-language $LanguageDisplayName `
-packageName $packageProperties.Name `
-version $newVersion `
-plannedDate $releaseDateString `
-packageRepoPath $packageProperties.serviceDirectory `
-packageType $packageProperties.SDKType `
-packageNewLibrary $packageProperties.IsNewSDK
$result = Update-DevOpsReleaseWorkItem -language $LanguageDisplayName `
-packageName $packageProperties.Name `
-version $newVersion `
-plannedDate $releaseDateString `
-packageRepoPath $packageProperties.serviceDirectory `
-packageType $packageProperties.SDKType `
-packageNewLibrary $packageProperties.IsNewSDK
if ($LASTEXITCODE -ne 0) {
Write-Error "Updating of the Devops Release WorkItem failed."
exit 1
if (-not $result)
{
Write-Error "Update of the Devops Release WorkItem failed."
exit 1
}
# Check API status

View File

@ -1,6 +1,6 @@
[CmdletBinding()]
Param (
[Parameter(Mandatory=$True)]
[Parameter(Mandatory=$False)]
[array]$ArtifactList,
[Parameter(Mandatory=$True)]
[string]$ArtifactPath,
@ -12,39 +12,290 @@ Param (
[string]$BuildDefinition,
[string]$PipelineUrl,
[string]$APIViewUri = "https://apiview.dev/AutoReview/GetReviewStatus",
[bool] $IsReleaseBuild = $false
[bool] $IsReleaseBuild = $false,
[Parameter(Mandatory=$False)]
[array] $PackageInfoFiles
)
# Validate-All-Packages.ps1 folds in the code that was originally in Validate-Package.ps1
# since Validate-Package.ps1 was only called from Validate-All-Packages.ps1. This replaces
# script calls with function calls and also allows calling CheckAzLoginAndDevOpsExtensionInstall
# and CheckDevOpsAccess once for all of the PackageInfo files being processed instead of once
# per artifact in Validate-Package.ps1 and then again in Update-DevOps-Release-WorkItem.ps1
Set-StrictMode -Version 3
. (Join-Path $PSScriptRoot common.ps1)
. (Join-Path $PSScriptRoot Helpers\ApiView-Helpers.ps1)
. (Join-Path $PSScriptRoot Helpers\DevOps-WorkItem-Helpers.ps1)
function ProcessPackage($PackageName, $ConfigFileDir)
# Function to validate change log
function ValidateChangeLog($changeLogPath, $versionString, $validationStatus)
{
Write-Host "Artifact path: $($ArtifactPath)"
Write-Host "Package Name: $($PackageName)"
Write-Host "Config File directory: $($ConfigFileDir)"
&$EngCommonScriptsDir/Validate-Package.ps1 `
-PackageName $PackageName `
-ArtifactPath $ArtifactPath `
-RepoRoot $RepoRoot `
-APIViewUri $APIViewUri `
-APIKey $APIKey `
-BuildDefinition $BuildDefinition `
-PipelineUrl $PipelineUrl `
-ConfigFileDir $ConfigFileDir
if ($LASTEXITCODE -ne 0) {
Write-Error "Failed to validate package $PackageName"
exit 1
try
{
$ChangeLogStatus = [PSCustomObject]@{
IsValid = $false
Message = ""
}
$changeLogFullPath = Join-Path $RepoRoot $changeLogPath
Write-Host "Path to change log: [$changeLogFullPath]"
if (Test-Path $changeLogFullPath)
{
Confirm-ChangeLogEntry -ChangeLogLocation $changeLogFullPath -VersionString $versionString -ForRelease $true -ChangeLogStatus $ChangeLogStatus -SuppressErrors $true
$validationStatus.Status = if ($ChangeLogStatus.IsValid) { "Success" } else { "Failed" }
$validationStatus.Message = $ChangeLogStatus.Message
}
else {
$validationStatus.Status = "Failed"
$validationStatus.Message = "Change log is not found in [$changeLogPath]. Change log file must be present in package root directory."
}
}
catch
{
Write-Host "Current directory: $(Get-Location)"
$validationStatus.Status = "Failed"
$validationStatus.Message = $_.Exception.Message
}
}
# Function to verify API review status
function VerifyAPIReview($packageName, $packageVersion, $language)
{
$APIReviewValidation = [PSCustomObject]@{
Name = "API Review Approval"
Status = "Pending"
Message = ""
}
$PackageNameValidation = [PSCustomObject]@{
Name = "Package Name Approval"
Status = "Pending"
Message = ""
}
try
{
$apiStatus = [PSCustomObject]@{
IsApproved = $false
Details = ""
}
$packageNameStatus = [PSCustomObject]@{
IsApproved = $false
Details = ""
}
Write-Host "Checking API review status for package $packageName with version $packageVersion. language [$language]."
Check-ApiReviewStatus $packageName $packageVersion $language $APIViewUri $APIKey $apiStatus $packageNameStatus
Write-Host "API review approval details: $($apiStatus.Details)"
Write-Host "Package name approval details: $($packageNameStatus.Details)"
#API review approval status
$APIReviewValidation.Message = $apiStatus.Details
$APIReviewValidation.Status = if ($apiStatus.IsApproved) { "Approved" } else { "Pending" }
# Package name approval status
$PackageNameValidation.Status = if ($packageNameStatus.IsApproved) { "Approved" } else { "Pending" }
$PackageNameValidation.Message = $packageNameStatus.Details
}
catch
{
Write-Warning "Failed to get API review status. Error: $_"
$PackageNameValidation.Status = "Failed"
$PackageNameValidation.Message = $_.Exception.Message
$APIReviewValidation.Status = "Failed"
$APIReviewValidation.Message = $_.Exception.Message
}
return [PSCustomObject]@{
ApiviewApproval = $APIReviewValidation
PackageNameApproval = $PackageNameValidation
}
}
function IsVersionShipped($packageName, $packageVersion)
{
# This function will decide if a package version is already shipped or not
Write-Host "Checking if a version is already shipped for package $packageName with version $packageVersion."
$parsedNewVersion = [AzureEngSemanticVersion]::new($packageVersion)
$versionMajorMinor = "" + $parsedNewVersion.Major + "." + $parsedNewVersion.Minor
$workItem = FindPackageWorkItem -lang $LanguageDisplayName -packageName $packageName -version $versionMajorMinor -includeClosed $true -outputCommand $false
if ($workItem)
{
# Check if the package version is already shipped
$shippedVersionSet = ParseVersionSetFromMDField $workItem.fields["Custom.ShippedPackages"]
if ($shippedVersionSet.ContainsKey($packageVersion)) {
return $true
}
}
else {
Write-Host "No work item found for package [$packageName]. Creating new work item for package."
}
return $false
}
function CreateUpdatePackageWorkItem($pkgInfo)
{
# This function will create or update package work item in Azure DevOps
$versionString = $pkgInfo.Version
$packageName = $pkgInfo.Name
$plannedDate = $pkgInfo.ReleaseStatus
$setReleaseState = $true
if (!$plannedDate -or $plannedDate -eq "Unreleased")
{
$setReleaseState = $false
$plannedDate = "unknown"
}
# Create or update package work item
$result = Update-DevOpsReleaseWorkItem -language $LanguageDisplayName `
-packageName $packageName `
-version $versionString `
-plannedDate $plannedDate `
-packageRepoPath $pkgInfo.serviceDirectory `
-packageType $pkgInfo.SDKType `
-packageNewLibrary $pkgInfo.IsNewSDK `
-serviceName "unknown" `
-packageDisplayName "unknown" `
-inRelease $IsReleaseBuild
if (-not $result)
{
Write-Host "Update of the Devops Release WorkItem failed."
}
return [bool]$result
}
function ProcessPackage($packageInfo)
{
# Read package property file and identify all packages to process
# $packageInfo.Name is the package name published to package managers, e.g. @azure/azure-template
# $packageInfo.ArtifactName is the name can be used in path and file names, e.g. azure-template
Write-Host "Processing artifact: $($packageInfo.ArtifactName)"
Write-Host "Is Release Build: $IsReleaseBuild"
$pkgName = $packageInfo.Name
$changeLogPath = $packageInfo.ChangeLogPath
$versionString = $packageInfo.Version
Write-Host "Checking if we need to create or update work item for package $pkgName with version $versionString."
$isShipped = IsVersionShipped $pkgName $versionString
if ($isShipped) {
Write-Host "Package work item already exists for version [$versionString] that is marked as shipped. Skipping the update of package work item."
return
}
Write-Host "Validating package $pkgName with version $versionString."
# Change log validation
$changeLogStatus = [PSCustomObject]@{
Name = "Change Log Validation"
Status = "Success"
Message = ""
}
ValidateChangeLog $changeLogPath $versionString $changeLogStatus
# API review and package name validation
$fullPackageName = $pkgName
# If there's a groupId that means this is Java and pkgName = GroupId+ArtifactName
# but the VerifyAPIReview requires GroupId:ArtifactName
Write-Host "Package name before checking groupId: $fullPackageName"
if ($packageInfo.PSObject.Members.Name -contains "Group") {
$groupId = $packageInfo.Group
if ($groupId){
$fullPackageName = "${groupId}:$($packageInfo.ArtifactName)"
}
}
Write-Host "Checking API review status for package $fullPackageName"
$apireviewDetails = VerifyAPIReview $fullPackageName $packageInfo.Version $Language
$pkgValidationDetails= [PSCustomObject]@{
Name = $pkgName
Version = $packageInfo.Version
ChangeLogValidation = $changeLogStatus
APIReviewValidation = $apireviewDetails.ApiviewApproval
PackageNameValidation = $apireviewDetails.PackageNameApproval
}
$output = ConvertTo-Json $pkgValidationDetails
Write-Host "Output: $($output)"
# Create json token file in artifact path
$tokenFile = Join-Path $ArtifactPath "$($packageInfo.ArtifactName)-Validation.json"
$output | Out-File -FilePath $tokenFile -Encoding utf8
# Create DevOps work item
$updatedWi = CreateUpdatePackageWorkItem $packageInfo
# Update validation status in package work item
if ($updatedWi) {
Write-Host "Updating validation status in package work item."
$updatedWi = UpdateValidationStatus $pkgValidationDetails $BuildDefinition $PipelineUrl
}
# Fail the build if any validation is not successful for a release build
Write-Host "Change log status:" $changeLogStatus.Status
Write-Host "API Review status:" $apireviewDetails.ApiviewApproval.Status
Write-Host "Package Name status:" $apireviewDetails.PackageNameApproval.Status
if ($IsReleaseBuild)
{
if (!$updatedWi -or $changeLogStatus.Status -ne "Success" -or $apireviewDetails.ApiviewApproval.Status -ne "Approved" -or $apireviewDetails.PackageNameApproval.Status -ne "Approved") {
Write-Error "At least one of the Validations above failed for package $pkgName with version $versionString."
exit 1
}
}
}
CheckAzLoginAndDevOpsExtensionInstall
CheckDevOpsAccess
Write-Host "Artifact path: $($ArtifactPath)"
Write-Host "Artifact List: $($ArtifactList -join ', ')"
Write-Host "Package Info Files: $($PackageInfoFiles -join ', ')"
Write-Host "IsReleaseBuild: $IsReleaseBuild"
# Check if package config file is present. This file has package version, SDK type etc info.
if (-not $ConfigFileDir) {
$ConfigFileDir = Join-Path -Path $ArtifactPath "PackageInfo"
}
foreach ($artifact in $ArtifactList)
Write-Host "Config file path: $($ConfigFileDir)"
# Initialize working variable
$ProcessedPackageInfoFiles = @()
if ($ArtifactList -and $ArtifactList.Count -gt 0)
{
Write-Host "Processing $($artifact.name)"
ProcessPackage -PackageName $artifact.name -ConfigFileDir $ConfigFileDir
}
# Multiple artifacts mode (existing usage)
Write-Host "Using ArtifactList parameter with $($ArtifactList.Count) artifacts"
foreach ($artifact in $ArtifactList)
{
$pkgPropPath = Join-Path -Path $ConfigFileDir "$($artifact.name).json"
if (Test-Path $pkgPropPath) {
$ProcessedPackageInfoFiles += $pkgPropPath
}
else {
Write-Warning "Package property file path $pkgPropPath is invalid."
}
}
}
elseif ($PackageInfoFiles -and $PackageInfoFiles.Count -gt 0)
{
# Direct PackageInfoFiles (new method)
Write-Host "Using PackageInfoFiles parameter with $($PackageInfoFiles.Count) files"
$ProcessedPackageInfoFiles = $PackageInfoFiles
}
# Validate that we have package info files to process
if (-not $ProcessedPackageInfoFiles -or $ProcessedPackageInfoFiles.Count -eq 0) {
Write-Error "No package info files found after processing parameters."
exit 1
}
Write-Host "Processed Package Info Files: $($ProcessedPackageInfoFiles -join ', ')"
# Process all packages using the processed PackageInfoFiles array
foreach ($packageInfoFile in $ProcessedPackageInfoFiles)
{
$packageInfo = Get-Content $packageInfoFile | ConvertFrom-Json
ProcessPackage -packageInfo $packageInfo
}