From 7a02ab023b8ec93879e89a0cd886c3b941f08bbb Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Thu, 26 Mar 2020 10:18:42 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools repository (#46) --- eng/common/Extract-ReleaseNotes.ps1 | 65 +++ eng/common/InterdependencyGraph.html | 356 +++++++++++++ eng/common/README.md | 12 + .../TestResources/New-TestResources.cmd | 17 + .../TestResources/New-TestResources.ps1 | 479 +++++++++++++++++ .../TestResources/New-TestResources.ps1.md | 410 +++++++++++++++ eng/common/TestResources/README.md | 29 ++ .../TestResources/Remove-TestResources.cmd | 17 + .../TestResources/Remove-TestResources.ps1 | 173 +++++++ .../TestResources/Remove-TestResources.ps1.md | 224 ++++++++ .../TestResources/deploy-test-resources.yml | 73 +++ .../TestResources/remove-test-resources.yml | 48 ++ eng/common/Update-Change-Log.ps1 | 135 +++++ .../templates/steps/create-pull-request.yml | 60 +++ .../steps/create-tags-and-git-release.yml | 18 + .../templates/steps/publish-blobs.yml | 23 + eng/common/scripts/Submit-PullRequest.ps1 | 108 ++++ .../scripts/copy-docs-to-blobstorage.ps1 | 326 ++++++++++++ .../scripts/create-tags-and-git-release.ps1 | 481 ++++++++++++++++++ eng/common/scripts/git-branch-push.ps1 | 145 ++++++ 20 files changed, 3199 insertions(+) create mode 100644 eng/common/Extract-ReleaseNotes.ps1 create mode 100644 eng/common/InterdependencyGraph.html create mode 100644 eng/common/README.md create mode 100644 eng/common/TestResources/New-TestResources.cmd create mode 100644 eng/common/TestResources/New-TestResources.ps1 create mode 100644 eng/common/TestResources/New-TestResources.ps1.md create mode 100644 eng/common/TestResources/README.md create mode 100644 eng/common/TestResources/Remove-TestResources.cmd create mode 100644 eng/common/TestResources/Remove-TestResources.ps1 create mode 100644 eng/common/TestResources/Remove-TestResources.ps1.md create mode 100644 eng/common/TestResources/deploy-test-resources.yml create mode 100644 eng/common/TestResources/remove-test-resources.yml create mode 100644 eng/common/Update-Change-Log.ps1 create mode 100644 eng/common/pipelines/templates/steps/create-pull-request.yml create mode 100644 eng/common/pipelines/templates/steps/create-tags-and-git-release.yml create mode 100644 eng/common/pipelines/templates/steps/publish-blobs.yml create mode 100644 eng/common/scripts/Submit-PullRequest.ps1 create mode 100644 eng/common/scripts/copy-docs-to-blobstorage.ps1 create mode 100644 eng/common/scripts/create-tags-and-git-release.ps1 create mode 100644 eng/common/scripts/git-branch-push.ps1 diff --git a/eng/common/Extract-ReleaseNotes.ps1 b/eng/common/Extract-ReleaseNotes.ps1 new file mode 100644 index 000000000..6fff6a11d --- /dev/null +++ b/eng/common/Extract-ReleaseNotes.ps1 @@ -0,0 +1,65 @@ +# given a CHANGELOG.md file, extract the relevant info we need to decorate a release +param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [String]$VersionString +) + +$ErrorActionPreference = 'Stop' + +$RELEASE_TITLE_REGEX = "(?^\#+.*(?\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?))" + +$releaseNotes = @{} +$contentArrays = @{} +if ($ChangeLogLocation.Length -eq 0) +{ + return $releaseNotes +} + +try +{ + $contents = Get-Content $ChangeLogLocation + + # walk the document, finding where the version specifiers are and creating lists + $version = "" + foreach($line in $contents){ + if ($line -match $RELEASE_TITLE_REGEX) + { + $version = $matches["version"] + $contentArrays[$version] = @() + } + + $contentArrays[$version] += $line + } + + # resolve each of discovered version specifier string arrays into real content + foreach($key in $contentArrays.Keys) + { + $releaseNotes[$key] = New-Object PSObject -Property @{ + ReleaseVersion = $key + ReleaseContent = $contentArrays[$key] -join [Environment]::NewLine + } + } +} +catch +{ + Write-Host "Error parsing $ChangeLogLocation." + Write-Host $_.Exception.Message +} + +if ([System.String]::IsNullOrEmpty($VersionString)) +{ + return $releaseNotes +} +else +{ + if ($releaseNotes.ContainsKey($VersionString)) + { + $releaseNotesForVersion = $releaseNotes[$VersionString].ReleaseContent + $processedNotes = $releaseNotesForVersion -Split [Environment]::NewLine | where { $_ -notmatch $RELEASE_TITLE_REGEX } + return $processedNotes -Join [Environment]::NewLine + } + Write-Error "Release Notes for the Specified version ${VersionString} was not found" + exit 1 +} + diff --git a/eng/common/InterdependencyGraph.html b/eng/common/InterdependencyGraph.html new file mode 100644 index 000000000..21f78563e --- /dev/null +++ b/eng/common/InterdependencyGraph.html @@ -0,0 +1,356 @@ + + + +Interdependency Graph + + + + + + + + +
+

Dependency Graph

+ + +
+
+
+ + + diff --git a/eng/common/README.md b/eng/common/README.md new file mode 100644 index 000000000..7e9e197fc --- /dev/null +++ b/eng/common/README.md @@ -0,0 +1,12 @@ +# Common Engineering System + +The `eng/common` directory contains engineering files that are common across the various azure-sdk language repos. +It should remain relatively small and only contain textual based files like scripts, configs, or templates. It +should not contain binary files as they don't play well with git. + +# Updating + +Any updates to files in the `eng/common` directory should be made in the [azure-sdk-tools](https://github.com/azure/azure-sdk-tools) repo. +All changes made will cause a PR to created in all subscribed azure-sdk language repos which will blindly replace all contents of +the `eng/common` directory in that repo. For that reason do **NOT** make changes to files in this directory in the individual azure-sdk +languages repos as they will be overwritten the next time an update is taken from the common azure-sdk-tools repo. \ No newline at end of file diff --git a/eng/common/TestResources/New-TestResources.cmd b/eng/common/TestResources/New-TestResources.cmd new file mode 100644 index 000000000..94b0c1f2d --- /dev/null +++ b/eng/common/TestResources/New-TestResources.cmd @@ -0,0 +1,17 @@ +@echo off + +REM Copyright (c) Microsoft Corporation. All rights reserved. +REM Licensed under the MIT License. + +setlocal + +for /f "usebackq delims=" %%i in (`where pwsh 2^>nul`) do ( + set _cmd=%%i +) + +if "%_cmd%"=="" ( + echo Error: PowerShell not found. Please visit https://github.com/powershell/powershell for install instructions. + exit /b 2 +) + +call "%_cmd%" -NoLogo -NoProfile -File "%~dpn0.ps1" %* diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 new file mode 100644 index 000000000..29c404528 --- /dev/null +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -0,0 +1,479 @@ +#!/usr/bin/env pwsh + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +#Requires -Version 6.0 +#Requires -PSEdition Core +#Requires -Modules @{ModuleName='Az.Accounts'; ModuleVersion='1.6.4'} +#Requires -Modules @{ModuleName='Az.Resources'; ModuleVersion='1.8.0'} + +[CmdletBinding(DefaultParameterSetName = 'Default', SupportsShouldProcess = $true, ConfirmImpact = 'Medium')] +param ( + # Limit $BaseName to enough characters to be under limit plus prefixes, and https://docs.microsoft.com/azure/architecture/best-practices/resource-naming. + [Parameter(Mandatory = $true, Position = 0)] + [ValidatePattern('^[-a-zA-Z0-9\.\(\)_]{0,80}(?<=[a-zA-Z0-9\(\)])$')] + [string] $BaseName, + + [Parameter(Mandatory = $true)] + [string] $ServiceDirectory, + + [Parameter(Mandatory = $true)] + [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')] + [string] $TestApplicationId, + + [Parameter()] + [string] $TestApplicationSecret, + + [Parameter()] + [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')] + [string] $TestApplicationOid, + + [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $TenantId, + + [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)] + [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')] + [string] $ProvisionerApplicationId, + + [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)] + [string] $ProvisionerApplicationSecret, + + [Parameter()] + [ValidateRange(0, [int]::MaxValue)] + [int] $DeleteAfterHours, + + [Parameter()] + [string] $Location = '', + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string] $Environment = 'AzureCloud', + + [Parameter()] + [ValidateNotNullOrEmpty()] + [hashtable] $AdditionalParameters, + + [Parameter()] + [switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID), + + [Parameter()] + [switch] $Force +) + +# By default stop for any error. +if (!$PSBoundParameters.ContainsKey('ErrorAction')) { + $ErrorActionPreference = 'Stop' +} + +function Log($Message) { + Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) +} + +function Retry([scriptblock] $Action, [int] $Attempts = 5) { + $attempt = 0 + $sleep = 5 + + while ($attempt -lt $Attempts) { + try { + $attempt++ + return $Action.Invoke() + } catch { + if ($attempt -lt $Attempts) { + $sleep *= 2 + + Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..." + Start-Sleep -Seconds $sleep + } else { + Write-Error -ErrorRecord $_ + } + } + } +} + +# Support actions to invoke on exit. +$exitActions = @({ + if ($exitActions.Count -gt 1) { + Write-Verbose 'Running registered exit actions' + } +}) + +trap { + # Like using try..finally in PowerShell, but without keeping track of more braces or tabbing content. + $exitActions.Invoke() +} + +# Enumerate test resources to deploy. Fail if none found. +$root = [System.IO.Path]::Combine("$PSScriptRoot/../sdk", $ServiceDirectory) | Resolve-Path +$templateFileName = 'test-resources.json' +$templateFiles = @() + +Write-Verbose "Checking for '$templateFileName' files under '$root'" +Get-ChildItem -Path $root -Filter $templateFileName -Recurse | ForEach-Object { + $templateFile = $_.FullName + + Write-Verbose "Found template '$templateFile'" + $templateFiles += $templateFile +} + +if (!$templateFiles) { + Write-Warning -Message "No template files found under '$root'" + exit +} + +# If no location is specified use safe default locations for the given +# environment. If no matching environment is found $Location remains an empty +# string. +if (!$Location) { + $defaultLocations = @{ + 'AzureCloud' = 'westus2'; + 'AzureUSGovernment' = 'usgovvirginia'; + 'AzureChinaCloud' = 'chinaeast2'; + } + + if ($defaultLocations.ContainsKey($Environment)) { + $Location = $defaultLocations[$Environment] + } else { + Write-Error "Location cannot be empty and there is no default location for Environment: '$Environment'" + } + + Write-Verbose "Location was not set. Using default location for environment: '$Location'" +} + +# Log in if requested; otherwise, the user is expected to already be authenticated via Connect-AzAccount. +if ($ProvisionerApplicationId) { + $null = Disable-AzContextAutosave -Scope Process + + Log "Logging into service principal '$ProvisionerApplicationId'" + $provisionerSecret = ConvertTo-SecureString -String $ProvisionerApplicationSecret -AsPlainText -Force + $provisionerCredential = [System.Management.Automation.PSCredential]::new($ProvisionerApplicationId, $provisionerSecret) + + $provisionerAccount = Retry { + Connect-AzAccount -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment + } + + $exitActions += { + Write-Verbose "Logging out of service principal '$($provisionerAccount.Context.Account)'" + $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context + } +} + +# Get test application OID from ID if not already provided. +if ($TestApplicationId -and !$TestApplicationOid) { + $testServicePrincipal = Retry { + Get-AzADServicePrincipal -ApplicationId $TestApplicationId + } + + if ($testServicePrincipal -and $testServicePrincipal.Id) { + $script:TestApplicationOid = $testServicePrincipal.Id + } +} + +# Format the resource group name based on resource group naming recommendations and limitations. +$resourceGroupName = if ($CI) { + $BaseName = 't' + (New-Guid).ToString('n').Substring(0, 16) + Write-Verbose "Generated base name '$BaseName' for CI build" + + # If the ServiceDirectory is an absolute path use the last directory name + # (e.g. D:\foo\bar\ -> bar) + $serviceName = if (Split-Path -IsAbsolute $ServiceDirectory) { + Split-Path -Leaf $ServiceDirectory + } else { + $ServiceDirectory + } + + "rg-{0}-$BaseName" -f ($serviceName -replace '[\\\/:]', '-').Substring(0, [Math]::Min($serviceName.Length, 90 - $BaseName.Length - 4)).Trim('-') +} else { + "rg-$BaseName" +} + +# Tag the resource group to be deleted after a certain number of hours if specified. +$tags = @{ + Creator = if ($env:USER) { $env:USER } else { "${env:USERNAME}" } + ServiceDirectory = $ServiceDirectory +} + +if ($PSBoundParameters.ContainsKey('DeleteAfterHours')) { + $deleteAfter = [DateTime]::UtcNow.AddHours($DeleteAfterHours) + $tags.Add('DeleteAfter', $deleteAfter.ToString('o')) +} + +if ($CI) { + # Add tags for the current CI job. + $tags += @{ + BuildId = "${env:BUILD_BUILDID}" + BuildJob = "${env:AGENT_JOBNAME}" + BuildNumber = "${env:BUILD_BUILDNUMBER}" + BuildReason = "${env:BUILD_REASON}" + } + + # Set the resource group name variable. + Write-Host "Setting variable 'AZURE_RESOURCEGROUP_NAME': $resourceGroupName" + Write-Host "##vso[task.setvariable variable=AZURE_RESOURCEGROUP_NAME;]$resourceGroupName" +} + +Log "Creating resource group '$resourceGroupName' in location '$Location'" +$resourceGroup = Retry { + New-AzResourceGroup -Name "$resourceGroupName" -Location $Location -Tag $tags -Force:$Force +} + +if ($resourceGroup.ProvisioningState -eq 'Succeeded') { + # New-AzResourceGroup would've written an error and stopped the pipeline by default anyway. + Write-Verbose "Successfully created resource group '$($resourceGroup.ResourceGroupName)'" +} + +# Populate the template parameters and merge any additional specified. +$templateParameters = @{ + baseName = $BaseName + testApplicationId = $TestApplicationId + testApplicationOid = "$TestApplicationOid" +} + +if ($TenantId) { + $templateParameters.Add('tenantId', $TenantId) +} +if ($TestApplicationSecret) { + $templateParameters.Add('testApplicationSecret', $TestApplicationSecret) +} +if ($AdditionalParameters) { + $templateParameters += $AdditionalParameters +} + +# Try to detect the shell based on the parent process name (e.g. launch via shebang). +$shell, $shellExportFormat = if (($parentProcessName = (Get-Process -Id $PID).Parent.ProcessName) -and $parentProcessName -eq 'cmd') { + 'cmd', 'set {0}={1}' +} elseif (@('bash', 'csh', 'tcsh', 'zsh') -contains $parentProcessName) { + 'shell', 'export {0}={1}' +} else { + 'PowerShell', '$env:{0} = ''{1}''' +} + +foreach ($templateFile in $templateFiles) { + # Deployment fails if we pass in more parameters than are defined. + Write-Verbose "Removing unnecessary parameters from template '$templateFile'" + $templateJson = Get-Content -LiteralPath $templateFile | ConvertFrom-Json + $templateParameterNames = $templateJson.parameters.PSObject.Properties.Name + + $templateFileParameters = $templateParameters.Clone() + foreach ($key in $templateParameters.Keys) { + if ($templateParameterNames -notcontains $key) { + Write-Verbose "Removing unnecessary parameter '$key'" + $templateFileParameters.Remove($key) + } + } + + $preDeploymentScript = $templateFile | Split-Path | Join-Path -ChildPath 'test-resources-pre.ps1' + if (Test-Path $preDeploymentScript) { + Log "Invoking pre-deployment script '$preDeploymentScript'" + &$preDeploymentScript -ResourceGroupName $resourceGroupName @PSBoundParameters + } + + Log "Deploying template '$templateFile' to resource group '$($resourceGroup.ResourceGroupName)'" + $deployment = Retry { + New-AzResourceGroupDeployment -Name $BaseName -ResourceGroupName $resourceGroup.ResourceGroupName -TemplateFile $templateFile -TemplateParameterObject $templateFileParameters + } + + if ($deployment.ProvisioningState -eq 'Succeeded') { + # New-AzResourceGroupDeployment would've written an error and stopped the pipeline by default anyway. + Write-Verbose "Successfully deployed template '$templateFile' to resource group '$($resourceGroup.ResourceGroupName)'" + } + + if ($deployment.Outputs.Count -and !$CI) { + # Write an extra new line to isolate the environment variables for easy reading. + Log "Persist the following environment variables based on your detected shell ($shell):`n" + } + + $deploymentOutputs = @{} + foreach ($key in $deployment.Outputs.Keys) { + $variable = $deployment.Outputs[$key] + + # Work around bug that makes the first few characters of environment variables be lowercase. + $key = $key.ToUpperInvariant() + + if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') { + $deploymentOutputs[$key] = $variable.Value + + if ($CI) { + # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. + # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. + Write-Host "Setting variable '$key': ***" + Write-Host "##vso[task.setvariable variable=_$key;issecret=true;]$($variable.Value)" + Write-Host "##vso[task.setvariable variable=$key;]$($variable.Value)" + } else { + Write-Host ($shellExportFormat -f $key, $variable.Value) + } + } + } + + if ($key) { + # Isolate the environment variables for easy reading. + Write-Host "`n" + $key = $null + } + + $postDeploymentScript = $templateFile | Split-Path | Join-Path -ChildPath 'test-resources-post.ps1' + if (Test-Path $postDeploymentScript) { + Log "Invoking post-deployment script '$postDeploymentScript'" + &$postDeploymentScript -ResourceGroupName $resourceGroupName -DeploymentOutputs $deploymentOutputs @PSBoundParameters + } +} + +$exitActions.Invoke() + +<# +.SYNOPSIS +Deploys live test resources defined for a service directory to Azure. + +.DESCRIPTION +Deploys live test resouces specified in test-resources.json files to a resource +group. + +This script searches the directory specified in $ServiceDirectory recursively +for files named test-resources.json. All found test-resources.json files will be +deployed to the test resource group. + +If no test-resources.json files are located the script exits without making +changes to the Azure environment. + +A service principal must first be created before this script is run and passed +to $TestApplicationId and $TestApplicationSecret. Test resources will grant this +service principal access. + +This script uses credentials already specified in Connect-AzAccount or those +specified in $ProvisionerApplicationId and $ProvisionerApplicationSecret. + +.PARAMETER BaseName +A name to use in the resource group and passed to the ARM template as 'baseName'. +Limit $BaseName to enough characters to be under limit plus prefixes specified in +the ARM template. See also https://docs.microsoft.com/azure/architecture/best-practices/resource-naming + +Note: The value specified for this parameter will be overriden and generated +by New-TestResources.ps1 if $CI is specified. + +.PARAMETER ServiceDirectory +A directory under 'sdk' in the repository root - optionally with subdirectories +specified - in which to discover ARM templates named 'test-resources.json'. +This can also be an absolute path or specify parent directories. + +.PARAMETER TestApplicationId +The AAD Application ID to authenticate the test runner against deployed +resources. Passed to the ARM template as 'testApplicationId'. + +This application is used by the test runner to execute tests against the +live test resources. + +.PARAMETER TestApplicationSecret +Optional service principal secret (password) to authenticate the test runner +against deployed resources. Passed to the ARM template as +'testApplicationSecret'. + +This application is used by the test runner to execute tests against the +live test resources. + +.PARAMETER TestApplicationOid +Service Principal Object ID of the AAD Test application. This is used to assign +permissions to the AAD application so it can access tested features on the live +test resources (e.g. Role Assignments on resources). It is passed as to the ARM +template as 'testApplicationOid' + +For more information on the relationship between AAD Applications and Service +Principals see: https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals + +.PARAMETER TenantId +The tenant ID of a service principal when a provisioner is specified. The same +Tenant ID is used for Test Application and Provisioner Application. This value +is passed to the ARM template as 'tenantId'. + +.PARAMETER ProvisionerApplicationId +The AAD Application ID used to provision test resources when a provisioner is +specified. + +If none is specified New-TestResources.ps1 uses the TestApplicationId. + +This value is not passed to the ARM template. + +.PARAMETER ProvisionerApplicationSecret +A service principal secret (password) used to provision test resources when a +provisioner is specified. + +If none is specified New-TestResources.ps1 uses the TestApplicationSecret. + +This value is not passed to the ARM template. + +.PARAMETER DeleteAfterHours +Optional. Positive integer number of hours from the current time to set the +'DeleteAfter' tag on the created resource group. The computed value is a +timestamp of the form "2020-03-04T09:07:04.3083910Z". + +If this value is not specified no 'DeleteAfter' tag will be assigned to the +created resource group. + +An optional cleanup process can delete resource groups whose "DeleteAfter" +timestamp is less than the current time. + +This isused for CI automation. + +.PARAMETER Location +Optional location where resources should be created. By default this is +'westus2'. + +.PARAMETER AdditionalParameters +Optional key-value pairs of parameters to pass to the ARM template(s). + +.PARAMETER Environment +Name of the cloud environment. The default is the Azure Public Cloud +('PublicCloud') + +.PARAMETER CI +Indicates the script is run as part of a Continuous Integration / Continuous +Deployment (CI/CD) build (only Azure Pipelines is currently supported). + +.PARAMETER Force +Force creation of resources instead of being prompted. + +.EXAMPLE +$subscriptionId = "REPLACE_WITH_SUBSCRIPTION_ID" +Connect-AzAccount -Subscription $subscriptionId +$testAadApp = New-AzADServicePrincipal -Role Owner -DisplayName 'azure-sdk-live-test-app' +.\eng\common\LiveTestResources\New-TestResources.ps1 ` + -BaseName 'myalias' ` + -ServiceDirectory 'keyvault' ` + -TestApplicationId $testAadApp.ApplicationId.ToString() ` + -TestApplicationSecret (ConvertFrom-SecureString $testAadApp.Secret -AsPlainText) + +Run this in a desktop environment to create new AAD apps and Service Principals +that can be used to provision resources and run live tests. + +Requires PowerShell 7 to use ConvertFrom-SecureString -AsPlainText or convert +the SecureString to plaintext by another means. + +.EXAMPLE +eng/New-TestResources.ps1 ` + -BaseName 'Generated' ` + -ServiceDirectory '$(ServiceDirectory)' ` + -TenantId '$(TenantId)' ` + -ProvisionerApplicationId '$(ProvisionerId)' ` + -ProvisionerApplicationSecret '$(ProvisionerSecret)' ` + -TestApplicationId '$(TestAppId)' ` + -TestApplicationSecret '$(TestAppSecret)' ` + -DeleteAfterHours 24 ` + -CI ` + -Force ` + -Verbose + +Run this in an Azure DevOps CI (with approrpiate variables configured) before +executing live tests. The script will output variables as secrets (to enable +log redaction). + +.OUTPUTS +Entries from the ARM templates' "output" section in environment variable syntax +(e.g. $env:RESOURCE_NAME='<< resource name >>') that can be used for running +live tests. + +If run in -CI mode the environment variables will be output in syntax that Azure +DevOps can consume. + +.LINK +Remove-TestResources.ps1 +#> diff --git a/eng/common/TestResources/New-TestResources.ps1.md b/eng/common/TestResources/New-TestResources.ps1.md new file mode 100644 index 000000000..4c9785125 --- /dev/null +++ b/eng/common/TestResources/New-TestResources.ps1.md @@ -0,0 +1,410 @@ +--- +external help file: -help.xml +Module Name: +online version: +schema: 2.0.0 +--- + +# New-TestResources.ps1 + +## SYNOPSIS +Deploys live test resources defined for a service directory to Azure. + +## SYNTAX + +### Default (Default) +``` +New-TestResources.ps1 [-BaseName] -ServiceDirectory -TestApplicationId + [-TestApplicationSecret ] [-TestApplicationOid ] [-DeleteAfterHours ] + [-Location ] [-Environment ] [-AdditionalParameters ] [-CI] [-Force] [-WhatIf] + [-Confirm] [] +``` + +### Provisioner +``` +New-TestResources.ps1 [-BaseName] -ServiceDirectory -TestApplicationId + [-TestApplicationSecret ] [-TestApplicationOid ] -TenantId + -ProvisionerApplicationId -ProvisionerApplicationSecret [-DeleteAfterHours ] + [-Location ] [-Environment ] [-AdditionalParameters ] [-CI] [-Force] [-WhatIf] + [-Confirm] [] +``` + +## DESCRIPTION +Deploys live test resouces specified in test-resources.json files to a resource +group. + +This script searches the directory specified in $ServiceDirectory recursively +for files named test-resources.json. +All found test-resources.json files will be +deployed to the test resource group. + +If no test-resources.json files are located the script exits without making +changes to the Azure environment. + +A service principal must first be created before this script is run and passed +to $TestApplicationId and $TestApplicationSecret. +Test resources will grant this +service principal access. + +This script uses credentials already specified in Connect-AzAccount or those +specified in $ProvisionerApplicationId and $ProvisionerApplicationSecret. + +## EXAMPLES + +### EXAMPLE 1 +``` +$subscriptionId = "REPLACE_WITH_SUBSCRIPTION_ID" +Connect-AzAccount -Subscription $subscriptionId +$testAadApp = New-AzADServicePrincipal -Role Owner -DisplayName 'azure-sdk-live-test-app' +.\eng\common\LiveTestResources\New-TestResources.ps1 ` + -BaseName 'myalias' ` + -ServiceDirectory 'keyvault' ` + -TestApplicationId $testAadApp.ApplicationId.ToString() ` + -TestApplicationSecret (ConvertFrom-SecureString $testAadApp.Secret -AsPlainText) +``` + +Run this in a desktop environment to create new AAD apps and Service Principals +that can be used to provision resources and run live tests. + +Requires PowerShell 7 to use ConvertFrom-SecureString -AsPlainText or convert +the SecureString to plaintext by another means. + +### EXAMPLE 2 +``` +eng/New-TestResources.ps1 ` + -BaseName 'Generated' ` + -ServiceDirectory '$(ServiceDirectory)' ` + -TenantId '$(TenantId)' ` + -ProvisionerApplicationId '$(ProvisionerId)' ` + -ProvisionerApplicationSecret '$(ProvisionerSecret)' ` + -TestApplicationId '$(TestAppId)' ` + -TestApplicationSecret '$(TestAppSecret)' ` + -DeleteAfterHours 24 ` + -CI ` + -Force ` + -Verbose +``` + +Run this in an Azure DevOps CI (with approrpiate variables configured) before +executing live tests. +The script will output variables as secrets (to enable +log redaction). + +## PARAMETERS + +### -BaseName +A name to use in the resource group and passed to the ARM template as 'baseName'. +Limit $BaseName to enough characters to be under limit plus prefixes specified in +the ARM template. +See also https://docs.microsoft.com/azure/architecture/best-practices/resource-naming + +Note: The value specified for this parameter will be overriden and generated +by New-TestResources.ps1 if $CI is specified. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: True +Position: 1 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ServiceDirectory +A directory under 'sdk' in the repository root - optionally with subdirectories +specified - in which to discover ARM templates named 'test-resources.json'. +This can also be an absolute path or specify parent directories. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -TestApplicationId +The AAD Application ID to authenticate the test runner against deployed +resources. +Passed to the ARM template as 'testApplicationId'. + +This application is used by the test runner to execute tests against the +live test resources. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -TestApplicationSecret +Optional service principal secret (password) to authenticate the test runner +against deployed resources. +Passed to the ARM template as +'testApplicationSecret'. + +This application is used by the test runner to execute tests against the +live test resources. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -TestApplicationOid +Service Principal Object ID of the AAD Test application. +This is used to assign +permissions to the AAD application so it can access tested features on the live +test resources (e.g. +Role Assignments on resources). +It is passed as to the ARM +template as 'testApplicationOid' + +For more information on the relationship between AAD Applications and Service +Principals see: https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -TenantId +The tenant ID of a service principal when a provisioner is specified. +The same +Tenant ID is used for Test Application and Provisioner Application. +This value +is passed to the ARM template as 'tenantId'. + +```yaml +Type: String +Parameter Sets: Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ProvisionerApplicationId +The AAD Application ID used to provision test resources when a provisioner is +specified. + +If none is specified New-TestResources.ps1 uses the TestApplicationId. + +This value is not passed to the ARM template. + +```yaml +Type: String +Parameter Sets: Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ProvisionerApplicationSecret +A service principal secret (password) used to provision test resources when a +provisioner is specified. + +If none is specified New-TestResources.ps1 uses the TestApplicationSecret. + +This value is not passed to the ARM template. + +```yaml +Type: String +Parameter Sets: Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -DeleteAfterHours +Optional. +Positive integer number of hours from the current time to set the +'DeleteAfter' tag on the created resource group. +The computed value is a +timestamp of the form "2020-03-04T09:07:04.3083910Z". + +If this value is not specified no 'DeleteAfter' tag will be assigned to the +created resource group. + +An optional cleanup process can delete resource groups whose "DeleteAfter" +timestamp is less than the current time. + +This isused for CI automation. + +```yaml +Type: Int32 +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: 0 +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Location +Optional location where resources should be created. +By default this is +'westus2'. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: Westus2 +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Environment +Name of the cloud environment. +The default is the Azure Public Cloud +('PublicCloud') + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: AzureCloud +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -AdditionalParameters +Optional key-value pairs of parameters to pass to the ARM template(s). + +```yaml +Type: Hashtable +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -CI +Indicates the script is run as part of a Continuous Integration / Continuous +Deployment (CI/CD) build (only Azure Pipelines is currently supported). + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: ($null -ne $env:SYSTEM_TEAMPROJECTID) +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Force +Force creation of resources instead of being prompted. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: False +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -WhatIf +Shows what would happen if the cmdlet runs. +The cmdlet is not run. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216). + +## OUTPUTS + +### Entries from the ARM templates' "output" section in environment variable syntax +### (e.g. $env:RESOURCE_NAME='<< resource name >>') that can be used for running +### live tests. +### If run in -CI mode the environment variables will be output in syntax that Azure +### DevOps can consume. +## NOTES + +## RELATED LINKS + +[Remove-TestResources.ps1](./New-TestResources.ps1.md) + diff --git a/eng/common/TestResources/README.md b/eng/common/TestResources/README.md new file mode 100644 index 000000000..8eb911419 --- /dev/null +++ b/eng/common/TestResources/README.md @@ -0,0 +1,29 @@ +# Live Test Resource Management + +Live test runs require pre-existing resources in Azure. This set of PowerShell +commands automates creation and teardown of live test resources for Desktop and +CI scenarios. + +* [New-TestResources.ps1](./New-TestResources.ps1.md) - Create new test resources +for the given service. +* [Remove-TestResources.ps1](./New-TestResources.ps1.md) - Deletes resources + +## On the Desktop + +Run `New-TestResources.ps1` on your desktop to create live test resources for a +given service (e.g. Storage, Key Vault, etc.). The command will output +environment variables that need to be set when running the live tests. + +See examples for how to create the needed Service Principals and execute live +tests. + +## In CI + +The `New-TestResources.ps1` script is invoked on each test job to create an +isolated environment for live tests. Test resource isolation makes it easier to +parallelize test runs. + +## Other + +PowerShell markdown documentation created with +[PlatyPS](https://github.com/PowerShell/platyPS) \ No newline at end of file diff --git a/eng/common/TestResources/Remove-TestResources.cmd b/eng/common/TestResources/Remove-TestResources.cmd new file mode 100644 index 000000000..94b0c1f2d --- /dev/null +++ b/eng/common/TestResources/Remove-TestResources.cmd @@ -0,0 +1,17 @@ +@echo off + +REM Copyright (c) Microsoft Corporation. All rights reserved. +REM Licensed under the MIT License. + +setlocal + +for /f "usebackq delims=" %%i in (`where pwsh 2^>nul`) do ( + set _cmd=%%i +) + +if "%_cmd%"=="" ( + echo Error: PowerShell not found. Please visit https://github.com/powershell/powershell for install instructions. + exit /b 2 +) + +call "%_cmd%" -NoLogo -NoProfile -File "%~dpn0.ps1" %* diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 new file mode 100644 index 000000000..494fe1c87 --- /dev/null +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -0,0 +1,173 @@ +#!/usr/bin/env pwsh + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +#Requires -Version 6.0 +#Requires -PSEdition Core +#Requires -Modules @{ModuleName='Az.Accounts'; ModuleVersion='1.6.4'} +#Requires -Modules @{ModuleName='Az.Resources'; ModuleVersion='1.8.0'} + +[CmdletBinding(DefaultParameterSetName = 'Default', SupportsShouldProcess = $true, ConfirmImpact = 'Medium')] +param ( + # Limit $BaseName to enough characters to be under limit plus prefixes, and https://docs.microsoft.com/azure/architecture/best-practices/resource-naming. + [Parameter(ParameterSetName = 'Default', Mandatory = $true, Position = 0)] + [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true, Position = 0)] + [ValidatePattern('^[-a-zA-Z0-9\.\(\)_]{0,80}(?<=[a-zA-Z0-9\(\)])$')] + [string] $BaseName, + + [Parameter(ParameterSetName = 'ResourceGroup', Mandatory = $true)] + [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)] + [string] $ResourceGroupName, + + [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true)] + [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $TenantId, + + [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true)] + [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)] + [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')] + [string] $ProvisionerApplicationId, + + [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true)] + [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)] + [string] $ProvisionerApplicationSecret, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string] $Environment = 'AzureCloud', + + [Parameter()] + [switch] $Force +) + +# By default stop for any error. +if (!$PSBoundParameters.ContainsKey('ErrorAction')) { + $ErrorActionPreference = 'Stop' +} + +function Log($Message) { + Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) +} + +function Retry([scriptblock] $Action, [int] $Attempts = 5) { + $attempt = 0 + $sleep = 5 + + while ($attempt -lt $Attempts) { + try { + $attempt++ + return $Action.Invoke() + } catch { + if ($attempt -lt $Attempts) { + $sleep *= 2 + + Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..." + Start-Sleep -Seconds $sleep + } else { + Write-Error -ErrorRecord $_ + } + } + } +} + +# Support actions to invoke on exit. +$exitActions = @({ + if ($exitActions.Count -gt 1) { + Write-Verbose 'Running registered exit actions.' + } +}) + +trap { + # Like using try..finally in PowerShell, but without keeping track of more braces or tabbing content. + $exitActions.Invoke() +} + +if ($ProvisionerApplicationId) { + $null = Disable-AzContextAutosave -Scope Process + + Log "Logging into service principal '$ProvisionerApplicationId'" + $provisionerSecret = ConvertTo-SecureString -String $ProvisionerApplicationSecret -AsPlainText -Force + $provisionerCredential = [System.Management.Automation.PSCredential]::new($ProvisionerApplicationId, $provisionerSecret) + $provisionerAccount = Retry { + Connect-AzAccount -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment + } + + $exitActions += { + Write-Verbose "Logging out of service principal '$($provisionerAccount.Context.Account)'" + $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context + } +} + +if (!$ResourceGroupName) { + # Format the resource group name like in New-TestResources.ps1. + $ResourceGroupName = "rg-$BaseName" +} + +Log "Deleting resource group '$ResourceGroupName'" +if (Retry { Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force }) { + Write-Verbose "Successfully deleted resource group '$ResourceGroupName'" +} + +$exitActions.Invoke() + +<# +.SYNOPSIS +Deletes the resource group deployed for a service directory from Azure. + +.DESCRIPTION +Removes a resource group and all its resources previously deployed using +New-TestResources.ps1. + +If you are not currently logged into an account in the Az PowerShell module, +you will be asked to log in with Connect-AzAccount. Alternatively, you (or a +build pipeline) can pass $ProvisionerApplicationId and +$ProvisionerApplicationSecret to authenticate a service principal with access to +create resources. + +.PARAMETER BaseName +A name to use in the resource group and passed to the ARM template as 'baseName'. +This will delete the resource group named 'rg-' + +.PARAMETER ResourceGroupName +The name of the resource group to delete. + +.PARAMETER TenantId +The tenant ID of a service principal when a provisioner is specified. + +.PARAMETER ProvisionerApplicationId +A service principal ID to provision test resources when a provisioner is specified. + +.PARAMETER ProvisionerApplicationSecret +A service principal secret (password) to provision test resources when a provisioner is specified. + +.PARAMETER Environment +Name of the cloud environment. The default is the Azure Public Cloud +('PublicCloud') + +.PARAMETER Force +Force removal of resource group without asking for user confirmation + +.EXAMPLE +./Remove-TestResources.ps1 -BaseName uuid123 -Force + +Use the currently logged-in account to delete the resource group by the name of +'rg-uuid123' + +.EXAMPLE +eng/Remove-TestResources.ps1 ` + -ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}" ` + -TenantId '$(TenantId)' ` + -ProvisionerApplicationId '$(AppId)' ` + -ProvisionerApplicationSecret '$(AppSecret)' ` + -Force ` + -Verbose ` + +When run in the context of an Azure DevOps pipeline, this script removes the +resource group whose name is stored in the environment variable +AZURE_RESOURCEGROUP_NAME. + +.LINK +New-TestResources.ps1 +#> diff --git a/eng/common/TestResources/Remove-TestResources.ps1.md b/eng/common/TestResources/Remove-TestResources.ps1.md new file mode 100644 index 000000000..03803d8e5 --- /dev/null +++ b/eng/common/TestResources/Remove-TestResources.ps1.md @@ -0,0 +1,224 @@ +--- +external help file: -help.xml +Module Name: +online version: +schema: 2.0.0 +--- + +# Remove-TestResources.ps1 + +## SYNOPSIS +Deletes the resource group deployed for a service directory from Azure. + +## SYNTAX + +### Default (Default) +``` +Remove-TestResources.ps1 [-BaseName] [-Environment ] [-Force] [-WhatIf] [-Confirm] + [] +``` + +### Default+Provisioner +``` +Remove-TestResources.ps1 [-BaseName] -TenantId -ProvisionerApplicationId + -ProvisionerApplicationSecret [-Environment ] [-Force] [-WhatIf] [-Confirm] + [] +``` + +### ResourceGroup+Provisioner +``` +Remove-TestResources.ps1 -ResourceGroupName -TenantId -ProvisionerApplicationId + -ProvisionerApplicationSecret [-Environment ] [-Force] [-WhatIf] [-Confirm] + [] +``` + +### ResourceGroup +``` +Remove-TestResources.ps1 -ResourceGroupName [-Environment ] [-Force] [-WhatIf] [-Confirm] + [] +``` + +## DESCRIPTION +Removes a resource group and all its resources previously deployed using +New-TestResources.ps1. + +If you are not currently logged into an account in the Az PowerShell module, +you will be asked to log in with Connect-AzAccount. +Alternatively, you (or a +build pipeline) can pass $ProvisionerApplicationId and +$ProvisionerApplicationSecret to authenticate a service principal with access to +create resources. + +## EXAMPLES + +### EXAMPLE 1 +``` +./Remove-TestResources.ps1 -BaseName uuid123 -Force +``` + +Use the currently logged-in account to delete the resource group by the name of +'rg-uuid123' + +### EXAMPLE 2 +``` +eng/Remove-TestResources.ps1 ` + -ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}" ` + -TenantId '$(TenantId)' ` + -ProvisionerApplicationId '$(AppId)' ` + -ProvisionerApplicationSecret '$(AppSecret)' ` + -Force ` + -Verbose ` +``` + +When run in the context of an Azure DevOps pipeline, this script removes the +resource group whose name is stored in the environment variable +AZURE_RESOURCEGROUP_NAME. + +## PARAMETERS + +### -BaseName +A name to use in the resource group and passed to the ARM template as 'baseName'. +This will delete the resource group named 'rg-\' + +```yaml +Type: String +Parameter Sets: Default, Default+Provisioner +Aliases: + +Required: True +Position: 1 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ResourceGroupName +The name of the resource group to delete. + +```yaml +Type: String +Parameter Sets: ResourceGroup+Provisioner, ResourceGroup +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -TenantId +The tenant ID of a service principal when a provisioner is specified. + +```yaml +Type: String +Parameter Sets: Default+Provisioner, ResourceGroup+Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ProvisionerApplicationId +A service principal ID to provision test resources when a provisioner is specified. + +```yaml +Type: String +Parameter Sets: Default+Provisioner, ResourceGroup+Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ProvisionerApplicationSecret +A service principal secret (password) to provision test resources when a provisioner is specified. + +```yaml +Type: String +Parameter Sets: Default+Provisioner, ResourceGroup+Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Environment +Name of the cloud environment. +The default is the Azure Public Cloud +('PublicCloud') + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: AzureCloud +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Force +Force removal of resource group without asking for user confirmation + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: False +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -WhatIf +Shows what would happen if the cmdlet runs. +The cmdlet is not run. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216). + +## RELATED LINKS + +[New-TestResources.ps1](./New-TestResources.ps1.md) + diff --git a/eng/common/TestResources/deploy-test-resources.yml b/eng/common/TestResources/deploy-test-resources.yml new file mode 100644 index 000000000..c2445a4c4 --- /dev/null +++ b/eng/common/TestResources/deploy-test-resources.yml @@ -0,0 +1,73 @@ +# Deploys resources to a cloud type specified by the variable (not parameter) +# 'CloudType'. Use this as part of a matrix to deploy resources to a particular +# cloud instance. Normally we would use template parameters instead of variables +# but matrix variables are not available during template expansion so any +# benefits of parameters are lost. + +parameters: + ServiceDirectory: not-set + ArmTemplateParameters: '@{}' + DeleteAfterHours: 24 + Location: '' + +steps: + # New-TestResources command requires Az module + - pwsh: Install-Module -Name Az -Scope CurrentUser -AllowClobber -Force -Verbose + displayName: Install Azure PowerShell module + + - pwsh: > + eng/common/TestResources/New-TestResources.ps1 + -BaseName 'Generated' + -ServiceDirectory '${{ parameters.ServiceDirectory }}' + -TenantId '$(aad-azure-sdk-test-tenant-id)' + -TestApplicationId '$(aad-azure-sdk-test-client-id)' + -TestApplicationSecret '$(aad-azure-sdk-test-client-secret)' + -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id)' + -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret)' + -AdditionalParameters ${{ parameters.ArmTemplateParameters }} + -DeleteAfterHours ${{ parameters.DeleteAfterHours }} + -Location '${{ parameters.Location }}' + -Environment 'AzureCloud' + -CI + -Force + -Verbose + displayName: Deploy test resources (AzureCloud) + condition: and(succeeded(), eq(variables['CloudType'], 'AzureCloud')) + + - pwsh: > + eng/common/TestResources/New-TestResources.ps1 + -BaseName 'Generated' + -ServiceDirectory '${{ parameters.ServiceDirectory }}' + -TenantId '$(aad-azure-sdk-test-tenant-id-gov)' + -TestApplicationId '$(aad-azure-sdk-test-client-id-gov)' + -TestApplicationSecret '$(aad-azure-sdk-test-client-secret-gov)' + -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id-gov)' + -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret-gov)' + -AdditionalParameters ${{ parameters.ArmTemplateParameters }} + -DeleteAfterHours ${{ parameters.DeleteAfterHours }} + -Location '${{ parameters.Location }}' + -Environment 'AzureUSGovernment' + -CI + -Force + -Verbose + displayName: Deploy test resources (AzureUSGovernment) + condition: and(succeeded(), eq(variables['CloudType'], 'AzureUSGovernment')) + + - pwsh: > + eng/common/TestResources/New-TestResources.ps1 + -BaseName 'Generated' + -ServiceDirectory '${{ parameters.ServiceDirectory }}' + -TenantId '$(aad-azure-sdk-test-tenant-id-cn)' + -TestApplicationId '$(aad-azure-sdk-test-client-id-cn)' + -TestApplicationSecret '$(aad-azure-sdk-test-client-secret-cn)' + -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id-cn)' + -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret-cn)' + -AdditionalParameters ${{ parameters.ArmTemplateParameters }} + -DeleteAfterHours ${{ parameters.DeleteAfterHours }} + -Location '${{ parameters.Location }}' + -Environment 'AzureChinaCloud' + -CI + -Force + -Verbose + displayName: Deploy test resources (AzureChinaCloud) + condition: and(succeeded(), eq(variables['CloudType'], 'AzureChinaCloud')) \ No newline at end of file diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml new file mode 100644 index 000000000..fec875c0a --- /dev/null +++ b/eng/common/TestResources/remove-test-resources.yml @@ -0,0 +1,48 @@ +# Removes resources from a cloud type specified by the variable (not parameter) +# 'CloudType'. Use this as part of a matrix to remove resources from a +# particular cloud instance. Normally we would use template variables instead of +# parameters but matrix variables are not available during template expansion +# so any benefits of parameters are lost. + +# Assumes steps in deploy-test-resources.yml was run previously. Requires +# environment variable: AZURE_RESOURCEGROUP_NAME and Az PowerShell module + +steps: + - pwsh: > + eng/common/TestResources/Remove-TestResources.ps1 + -ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}" + -TenantId '$(aad-azure-sdk-test-tenant-id)' + -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id)' + -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret)' + -Environment 'AzureCloud' + -Force + -Verbose + displayName: Remove test resources (AzureCloud) + condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), eq(variables['CloudType'], 'AzureCloud')) + continueOnError: true + + - pwsh: > + eng/common/TestResources/Remove-TestResources.ps1 + -ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}" + -TenantId '$(aad-azure-sdk-test-tenant-id-gov)' + -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id-gov)' + -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret-gov)' + -Environment 'AzureUSGovernment' + -Force + -Verbose + displayName: Remove test resources (AzureUSGovernment) + condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), eq(variables['CloudType'], 'AzureUSGovernment')) + continueOnError: true + + - pwsh: > + eng/common/TestResources/Remove-TestResources.ps1 + -ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}" + -TenantId '$(aad-azure-sdk-test-tenant-id-cn)' + -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id-cn)' + -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret-cn)' + -Environment 'AzureChinaCloud' + -Force + -Verbose + displayName: Remove test resources (AzureChinaCloud) + condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), eq(variables['CloudType'], 'AzureChinaCloud')) + continueOnError: true diff --git a/eng/common/Update-Change-Log.ps1 b/eng/common/Update-Change-Log.ps1 new file mode 100644 index 000000000..bc48cbd63 --- /dev/null +++ b/eng/common/Update-Change-Log.ps1 @@ -0,0 +1,135 @@ +# Note: This script will add or replace version title in change log + +# Parameter description +# Version : Version to add or replace in change log +# ChangeLogPath: Path to change log file. If change log path is set to directory then script will probe for change log file in that path +# Unreleased: Default is true. If it is set to false, then today's date will be set in verion title. If it is True then title will show "Unreleased" +# ReplaceVersion: This is useful when replacing current version title with new title.( Helpful to update the title before package release) + +param ( + [Parameter(Mandatory = $true)] + [String]$Version, + [Parameter(Mandatory = $true)] + [String]$ChangeLogPath, + [String]$Unreleased = $True, + [String]$ReplaceVersion = $False +) + + +$RELEASE_TITLE_REGEX = "(?^\#+.*(?\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?))" +$UNRELEASED_TAG = "(Unreleased)" +function Version-Matches($line) +{ + return ($line -match $RELEASE_TITLE_REGEX) +} + +function Get-ChangelogPath($Path) +{ + # Check if CHANGELOG.md is present in path + $ChangeLogPath = Join-Path -Path $Path -ChildPath "CHANGELOG.md" + if ((Test-Path -Path $ChangeLogPath) -eq $False){ + # Check if change log exists with name HISTORY.md + $ChangeLogPath = Join-Path -Path $Path -ChildPath "HISTORY.md" + if ((Test-Path -Path $ChangeLogPath) -eq $False){ + Write-Host "Change log is not found in path[$Path]" + exit(1) + } + } + + Write-Host "Change log is found at path [$ChangeLogPath]" + return $ChangeLogPath +} + + +function Get-VersionTitle($Version, $Unreleased) +{ + # Generate version title + $newVersionTitle = "## $Version $UNRELEASED_TAG" + if ($Unreleased -eq $False){ + $releaseDate = Get-Date -Format "(yyyy-MM-dd)" + $newVersionTitle = "## $Version $releaseDate" + } + return $newVersionTitle +} + + +function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Version, $Unreleased, $ReplaceVersion) +{ + + # version parameter is to pass new version to add or replace + # Unreleased parameter can be set to False to set today's date instead of "Unreleased in title" + # ReplaceVersion param can be set to true to replace current version title( useful at release time to change title) + + # find index of current version + $Index = 0 + $CurrentTitle = "" + for(; $Index -lt $ChangelogLines.Count; $Index++){ + if (Version-Matches($ChangelogLines[$Index])){ + $CurrentTitle = $ChangelogLines[$Index] + Write-Host "Current Version title: $CurrentTitle" + break + } + } + + # Generate version title + $newVersionTitle = Get-VersionTitle -Version $Version -Unreleased $Unreleased + + if( $newVersionTitle -eq $CurrentTitle){ + Write-Host "No change is required in change log. Version is already present." + exit(0) + } + + # update change log script is triggered for all packages with current version for Java ( or any language where version is maintained in common file) + # Do not add new line or replace existing title when version is already present and script is triggered to add new line + if (($ReplaceVersion -eq $False) -and ($Unreleased -eq $True) -and $CurrentTitle.Contains($Version)){ + Write-Host "Version is already present in change log." + exit(0) + } + + if (($ReplaceVersion -eq $True) -and ($Unreleased -eq $False) -and (-not $CurrentTitle.Contains($UNRELEASED_TAG))){ + Write-Host "Version is already present in change log with a release date." + exit(0) + } + + # if current version title already has new version then we should replace title to update it + if ($CurrentTitle.Contains($Version) -and $ReplaceVersion -eq $False){ + Write-Host "Version is already present in title. Updating version title" + $ReplaceVersion = $True + } + + # if version is already found and not replacing then nothing to do + if ($ReplaceVersion -eq $False){ + Write-Host "Adding version title $newVersionTitle" + $ChangelogLines.insert($Index, "") + $ChangelogLines.insert($Index, "") + $ChangelogLines.insert($Index, $newVersionTitle) + } + else{ + # Script is executed to replace an existing version title + Write-Host "Replacing current version title to $newVersionTitle" + $ChangelogLines[$index] = $newVersionTitle + } + + return $ChangelogLines +} + + +# Make sure path is valid +if ((Test-Path -Path $ChangeLogPath) -eq $False){ + Write-Host "Change log path is invalid. [$ChangeLogPath]" + exit(1) +} + +# probe change log path if path is directory +if (Test-Path -Path $ChangeLogPath -PathType Container) +{ + $ChangeLogPath = Get-ChangelogPath -Path $ChangeLogPath +} + +# Read current change logs and add/update version +$ChangelogLines = [System.Collections.ArrayList](Get-Content -Path $ChangeLogPath) +$NewContents = Get-NewChangeLog -ChangelogLines $ChangelogLines -Version $Version -Unreleased $Unreleased -ReplaceVersion $ReplaceVersion + +Write-Host "Writing change log to file [$ChangeLogPath]" +Set-Content -Path $ChangeLogPath $NewContents +Write-Host "Version is added/updated in change log" diff --git a/eng/common/pipelines/templates/steps/create-pull-request.yml b/eng/common/pipelines/templates/steps/create-pull-request.yml new file mode 100644 index 000000000..e0212a7e0 --- /dev/null +++ b/eng/common/pipelines/templates/steps/create-pull-request.yml @@ -0,0 +1,60 @@ +# Expects azuresdk-github-pat is set to the PAT for azure-sdk +# Expects the buildtools to be cloned + +parameters: + BaseBranchName: master + PRBranchName: not-specified + PROwner: azure-sdk + CommitMsg: not-specified + RepoOwner: Azure + RepoName: not-specified + PushArgs: + WorkingDirectory: $(System.DefaultWorkingDirectory) + PRTitle: not-specified + +steps: + +- pwsh: | + echo "git add ." + git add . + + echo "git diff --name-status --cached --exit-code" + git diff --name-status --cached --exit-code + + if ($LastExitCode -ne 0) { + echo "##vso[task.setvariable variable=HasChanges]$true" + echo "Changes detected so setting HasChanges=true" + } + else { + echo "##vso[task.setvariable variable=HasChanges]$false" + echo "No changes so skipping code push" + } + + displayName: Check for changes + workingDirectory: ${{ parameters.WorkingDirectory }} + ignoreLASTEXITCODE: true + +- pwsh: | + eng/common/scripts/git-branch-push.ps1 ` + -PRBranchName "${{ parameters.PRBranchName }}" ` + -CommitMsg "${{ parameters.CommitMsg }}" ` + -GitUrl "https://$(azuresdk-github-pat)@github.com/${{ parameters.PROwner }}/${{ parameters.RepoName }}.git" ` + -PushArgs "${{ parameters.PushArgs }}" + + displayName: Push changes + workingDirectory: ${{ parameters.WorkingDirectory }} + condition: and(succeeded(), eq(variables['HasChanges'], 'true')) + +- pwsh: | + eng/common/scripts/Submit-PullRequest.ps1 ` + -RepoOwner "${{ parameters.RepoOwner }}" ` + -RepoName "${{ parameters.RepoName }}" ` + -BaseBranch "${{ parameters.BaseBranchName }}" ` + -PROwner "${{ parameters.PROwner }}" ` + -PRBranch "${{ parameters.PRBranchName }}" ` + -AuthToken "$(azuresdk-github-pat)" ` + -PRTitle "${{ parameters.PRTitle }}" + + displayName: Create pull request + workingDirectory: ${{ parameters.WorkingDirectory }} + condition: and(succeeded(), eq(variables['HasChanges'], 'true')) \ No newline at end of file diff --git a/eng/common/pipelines/templates/steps/create-tags-and-git-release.yml b/eng/common/pipelines/templates/steps/create-tags-and-git-release.yml new file mode 100644 index 000000000..78d91abe8 --- /dev/null +++ b/eng/common/pipelines/templates/steps/create-tags-and-git-release.yml @@ -0,0 +1,18 @@ +parameters: + ArtifactLocation: 'not-specified' + PackageRepository: 'not-specified' + ReleaseSha: 'not-specified' + RepoId: 'not-specified' + WorkingDirectory: '' + +steps: +- task: PowerShell@2 + displayName: 'Verify Package Tags and Create Git Releases' + inputs: + targetType: filePath + filePath: eng/common/scripts/create-tags-and-git-release.ps1 + arguments: -artifactLocation ${{parameters.ArtifactLocation}} -packageRepository ${{parameters.PackageRepository}} -releaseSha ${{parameters.ReleaseSha}} -repoId ${{parameters.RepoId}} -workingDirectory '${{parameters.WorkingDirectory}}' + pwsh: true + timeoutInMinutes: 5 + env: + GH_TOKEN: $(azuresdk-github-pat) \ No newline at end of file diff --git a/eng/common/pipelines/templates/steps/publish-blobs.yml b/eng/common/pipelines/templates/steps/publish-blobs.yml new file mode 100644 index 000000000..ce0d9f708 --- /dev/null +++ b/eng/common/pipelines/templates/steps/publish-blobs.yml @@ -0,0 +1,23 @@ +parameters: + FolderForUpload: '' + BlobSASKey: '' + TargetLanguage: '' + BlobName: '' + ScriptPath: '' + +steps: +- pwsh: | + Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://aka.ms/downloadazcopy-v10-windows" ` + -OutFile "azcopy.zip" | Wait-Process; Expand-Archive -Path "azcopy.zip" -DestinationPath "$(Build.BinariesDirectory)/azcopy/" + workingDirectory: $(Build.BinariesDirectory) + displayName: Download and Extract azcopy Zip + +- task: Powershell@2 + inputs: + targetType: 'filePath' + filePath: ${{ parameters.ScriptPath }} + arguments: -AzCopy $(Resolve-Path "$(Build.BinariesDirectory)/azcopy/azcopy_windows_amd64_*/azcopy.exe")[0] -DocLocation "${{ parameters.FolderForUpload }}" -SASKey "${{ parameters.BlobSASKey }}" -Language "${{ parameters.TargetLanguage }}" -BlobName "${{ parameters.BlobName }}" + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Copy Docs to Blob + continueOnError: false \ No newline at end of file diff --git a/eng/common/scripts/Submit-PullRequest.ps1 b/eng/common/scripts/Submit-PullRequest.ps1 new file mode 100644 index 000000000..f9e43478f --- /dev/null +++ b/eng/common/scripts/Submit-PullRequest.ps1 @@ -0,0 +1,108 @@ + #!/usr/bin/env pwsh -c + +<# +.DESCRIPTION +Creates a GitHub pull request for a given branch if it doesn't already exist +.PARAMETER RepoOwner +The GitHub repository owner to create the pull request against. +.PARAMETER RepoName +The GitHub repository name to create the pull request against. +.PARAMETER BaseBranch +The base or target branch we want the pull request to be against. +.PARAMETER PROwner +The owner of the branch we want to create a pull request for. +.PARAMETER PRBranch +The branch which we want to create a pull request for. +.PARAMETER AuthToken +A personal access token +#> +[CmdletBinding(SupportsShouldProcess = $true)] +param( + [Parameter(Mandatory = $true)] + $RepoOwner, + + [Parameter(Mandatory = $true)] + $RepoName, + + [Parameter(Mandatory = $true)] + $BaseBranch, + + [Parameter(Mandatory = $true)] + $PROwner, + + [Parameter(Mandatory = $true)] + $PRBranch, + + [Parameter(Mandatory = $true)] + $AuthToken, + + [Parameter(Mandatory = $true)] + $PRTitle, + $PRBody = $PRTitle +) + +$ErrorActionPreference = 'stop' +Set-StrictMode -Version 1 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +$headers = @{ + Authorization = "bearer $AuthToken" +} + +$query = 'query ($repoOwner: String!, $repoName: String!, $baseRefName: String!) { + repository(owner: $repoOwner, name: $repoName) { + pullRequests(baseRefName: $baseRefName, states: OPEN, first: 100) { + totalCount + nodes { + number + headRef { + name + repository { + name + owner { + login + } + } + } + } + } + } + }' + + +$data = @{ + query = $query + variables = @{ + repoOwner = $RepoOwner + repoName = $RepoName + baseRefName = $BaseBranch + } +} + +$resp = Invoke-RestMethod -Method Post -Headers $headers ` + https://api.github.com/graphql ` + -Body ($data | ConvertTo-Json) +$resp | Write-Verbose + +$matchingPr = $resp.data.repository.pullRequests.nodes ` + | ? { $_.headRef.name -eq $PRBranch -and $_.headRef.repository.owner.login -eq $PROwner } ` + | select -First 1 + +if ($matchingPr) { + Write-Host -f green "Pull request already exists https://github.com/$RepoOwner/$RepoName/pull/$($matchingPr.number)" +} +else { + $data = @{ + title = $PRTitle + head = "${PROwner}:${PRBranch}" + base = $BaseBranch + body = $PRBody + maintainer_can_modify = $true + } + + $resp = Invoke-RestMethod -Method POST -Headers $headers ` + https://api.github.com/repos/$RepoOwner/$RepoName/pulls ` + -Body ($data | ConvertTo-Json) + $resp | Write-Verbose + Write-Host -f green "Pull request created https://github.com/$RepoOwner/$RepoName/pull/$($resp.number)" +} diff --git a/eng/common/scripts/copy-docs-to-blobstorage.ps1 b/eng/common/scripts/copy-docs-to-blobstorage.ps1 new file mode 100644 index 000000000..383415b79 --- /dev/null +++ b/eng/common/scripts/copy-docs-to-blobstorage.ps1 @@ -0,0 +1,326 @@ +# Note, due to how `Expand-Archive` is leveraged in this script, +# powershell core is a requirement for successful execution. +param ( + $AzCopy, + $DocLocation, + $SASKey, + $Language, + $BlobName, + $ExitOnError=1 +) +$Language = $Language.ToLower() + +# Regex inspired but simplified from https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string +$SEMVER_REGEX = "^(?0|[1-9]\d*)\.(?0|[1-9]\d*)\.(?0|[1-9]\d*)(?:-?(?[a-zA-Z-]*)(?:\.?(?0|[1-9]\d*)))?$" + +function ToSemVer($version){ + if ($version -match $SEMVER_REGEX) + { + if($matches['prelabel'] -eq $null) { + # artifically provide these values for non-prereleases to enable easy sorting of them later than prereleases. + $prelabel = "zzz" + $prenumber = 999; + $isPre = $false; + } + else { + $prelabel = $matches["prelabel"] + $prenumber = [int]$matches["prenumber"] + $isPre = $true; + } + + New-Object PSObject -Property @{ + Major = [int]$matches['major'] + Minor = [int]$matches['minor'] + Patch = [int]$matches['patch'] + PrereleaseLabel = $prelabel + PrereleaseNumber = $prenumber + IsPrerelease = $isPre + RawVersion = $version + } + } + else + { + if ($ExitOnError) + { + throw "Unable to convert $version to valid semver and hard exit on error is enabled. Exiting." + } + else + { + return $null + } + } +} + +function SortSemVersions($versions) +{ + return $versions | Sort -Property Major, Minor, Patch, PrereleaseLabel, PrereleaseNumber -Descending +} + +function Sort-Versions +{ + Param ( + [Parameter(Mandatory=$true)] [string[]]$VersionArray + ) + + # standard init and sorting existing + $versionsObject = New-Object PSObject -Property @{ + OriginalVersionArray = $VersionArray + SortedVersionArray = @() + LatestGAPackage = "" + RawVersionsList = "" + LatestPreviewPackage = "" + } + + if ($VersionArray.Count -eq 0) + { + return $versionsObject + } + + $versionsObject.SortedVersionArray = SortSemVersions -versions ($VersionArray | % { ToSemVer $_}) + $versionsObject.RawVersionsList = $versionsObject.SortedVersionArray | % { $_.RawVersion } + + # handle latest and preview + # we only want to hold onto the latest preview if its NEWER than the latest GA. + # this means that the latest preview package either A) has to be the latest value in the VersionArray + # or B) set to nothing. We'll handle the set to nothing case a bit later. + $versionsObject.LatestPreviewPackage = $versionsObject.SortedVersionArray[0].RawVersion + $gaVersions = $versionsObject.SortedVersionArray | ? { !$_.IsPrerelease } + + # we have a GA package + if ($gaVersions.Count -ne 0) + { + # GA is the newest non-preview package + $versionsObject.LatestGAPackage = $gaVersions[0].RawVersion + + # in the case where latest preview == latestGA (because of our default selection earlier) + if ($versionsObject.LatestGAPackage -eq $versionsObject.LatestPreviewPackage) + { + # latest is newest, unset latest preview + $versionsObject.LatestPreviewPackage = "" + } + } + + return $versionsObject +} + +function Get-Existing-Versions +{ + Param ( + [Parameter(Mandatory=$true)] [String]$PkgName + ) + $versionUri = "$($BlobName)/`$web/$($Language)/$($PkgName)/versioning/versions" + Write-Host "Heading to $versionUri to retrieve known versions" + + try { + return ((Invoke-RestMethod -Uri $versionUri -MaximumRetryCount 3 -RetryIntervalSec 5) -Split "\n" | % {$_.Trim()} | ? { return $_ }) + } + catch { + # Handle 404. If it's 404, this is the first time we've published this package. + if ($_.Exception.Response.StatusCode.value__ -eq 404){ + Write-Host "Version file does not exist. This is the first time we have published this package." + } + else { + # If it's not a 404. exit. We don't know what's gone wrong. + Write-Host "Exception getting version file. Aborting" + Write-Host $_ + exit(1) + } + } +} + +function Update-Existing-Versions +{ + Param ( + [Parameter(Mandatory=$true)] [String]$PkgName, + [Parameter(Mandatory=$true)] [String]$PkgVersion, + [Parameter(Mandatory=$true)] [String]$DocDest + ) + $existingVersions = @(Get-Existing-Versions -PkgName $PkgName) + + Write-Host "Before I update anything, I am seeing $existingVersions" + + if (!$existingVersions) + { + $existingVersions = @() + $existingVersions += $PkgVersion + Write-Host "No existing versions. Adding $PkgVersion." + } + else + { + $existingVersions += $pkgVersion + Write-Host "Already Existing Versions. Adding $PkgVersion." + } + + $existingVersions = $existingVersions | Select-Object -Unique + + # newest first + $sortedVersionObj = (Sort-Versions -VersionArray $existingVersions) + + Write-Host $sortedVersionObj + Write-Host $sortedVersionObj.LatestGAPackage + Write-Host $sortedVersionObj.LatestPreviewPackage + + # write to file. to get the correct performance with "actually empty" files, we gotta do the newline + # join ourselves. This way we have absolute control over the trailing whitespace. + $sortedVersionObj.RawVersionsList -join "`n" | Out-File -File "$($DocLocation)/versions" -Force -NoNewLine + $sortedVersionObj.LatestGAPackage | Out-File -File "$($DocLocation)/latest-ga" -Force -NoNewLine + $sortedVersionObj.LatestPreviewPackage | Out-File -File "$($DocLocation)/latest-preview" -Force -NoNewLine + + & $($AzCopy) cp "$($DocLocation)/versions" "$($DocDest)/$($PkgName)/versioning/versions$($SASKey)" + & $($AzCopy) cp "$($DocLocation)/latest-preview" "$($DocDest)/$($PkgName)/versioning/latest-preview$($SASKey)" + & $($AzCopy) cp "$($DocLocation)/latest-ga" "$($DocDest)/$($PkgName)/versioning/latest-ga$($SASKey)" +} + +function Upload-Blobs +{ + Param ( + [Parameter(Mandatory=$true)] [String]$DocDir, + [Parameter(Mandatory=$true)] [String]$PkgName, + [Parameter(Mandatory=$true)] [String]$DocVersion + ) + #eg : $BlobName = "https://azuresdkdocs.blob.core.windows.net" + $DocDest = "$($BlobName)/`$web/$($Language)" + + Write-Host "DocDest $($DocDest)" + Write-Host "PkgName $($PkgName)" + Write-Host "DocVersion $($DocVersion)" + Write-Host "DocDir $($DocDir)" + Write-Host "Final Dest $($DocDest)/$($PkgName)/$($DocVersion)" + + Write-Host "Uploading $($PkgName)/$($DocVersion) to $($DocDest)..." + & $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/$($DocVersion)$($SASKey)" --recursive=true + + Write-Host "Handling versioning files under $($DocDest)/$($PkgName)/versioning/" + Update-Existing-Versions -PkgName $PkgName -PkgVersion $DocVersion -DocDest $DocDest +} + + +if ($Language -eq "javascript") +{ + $PublishedDocs = Get-ChildItem "$($DocLocation)/documentation" | Where-Object -FilterScript {$_.Name.EndsWith(".zip")} + + foreach ($Item in $PublishedDocs) { + $PkgName = "azure-$($Item.BaseName)" + Write-Host $PkgName + Expand-Archive -Force -Path "$($DocLocation)/documentation/$($Item.Name)" -DestinationPath "$($DocLocation)/documentation/$($Item.BaseName)" + $dirList = Get-ChildItem "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)" -Attributes Directory + + if($dirList.Length -eq 1){ + $DocVersion = $dirList[0].Name + Write-Host "Uploading Doc for $($PkgName) Version:- $($DocVersion)..." + Upload-Blobs -DocDir "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)/$($DocVersion)" -PkgName $PkgName -DocVersion $DocVersion + } + else{ + Write-Host "found more than 1 folder under the documentation for package - $($Item.Name)" + } + } +} + +if ($Language -eq "dotnet") +{ + $PublishedPkgs = Get-ChildItem "$($DocLocation)/packages" | Where-Object -FilterScript {$_.Name.EndsWith(".nupkg") -and -not $_.Name.EndsWith(".symbols.nupkg")} + $PublishedDocs = Get-ChildItem "$($DocLocation)" | Where-Object -FilterScript {$_.Name.StartsWith("Docs.")} + + foreach ($Item in $PublishedDocs) { + $PkgName = $Item.Name.Remove(0, 5) + $PkgFullName = $PublishedPkgs | Where-Object -FilterScript {$_.Name -match "$($PkgName).\d"} + + if (($PkgFullName | Measure-Object).count -eq 1) + { + $DocVersion = $PkgFullName[0].BaseName.Remove(0, $PkgName.Length + 1) + + Write-Host "Start Upload for $($PkgName)/$($DocVersion)" + Write-Host "DocDir $($Item)" + Write-Host "PkgName $($PkgName)" + Write-Host "DocVersion $($DocVersion)" + Upload-Blobs -DocDir "$($Item)" -PkgName $PkgName -DocVersion $DocVersion + } + else + { + Write-Host "Package with the same name Exists. Upload Skipped" + continue + } + } +} + +if ($Language -eq "python") +{ + $PublishedDocs = Get-ChildItem "$DocLocation" | Where-Object -FilterScript {$_.Name.EndsWith(".zip")} + + foreach ($Item in $PublishedDocs) { + $PkgName = $Item.BaseName + $ZippedDocumentationPath = Join-Path -Path $DocLocation -ChildPath $Item.Name + $UnzippedDocumentationPath = Join-Path -Path $DocLocation -ChildPath $PkgName + $VersionFileLocation = Join-Path -Path $UnzippedDocumentationPath -ChildPath "version.txt" + + Expand-Archive -Force -Path $ZippedDocumentationPath -DestinationPath $UnzippedDocumentationPath + + $Version = $(Get-Content $VersionFileLocation).Trim() + + Write-Host "Discovered Package Name: $PkgName" + Write-Host "Discovered Package Version: $Version" + Write-Host "Directory for Upload: $UnzippedDocumentationPath" + + Upload-Blobs -DocDir $UnzippedDocumentationPath -PkgName $PkgName -DocVersion $Version + } +} + +if ($Language -eq "java") +{ + $PublishedDocs = Get-ChildItem "$DocLocation" | Where-Object -FilterScript {$_.Name.EndsWith("-javadoc.jar")} + foreach ($Item in $PublishedDocs) { + $UnjarredDocumentationPath = "" + try { + $PkgName = $Item.BaseName + # The jar's unpacking command doesn't allow specifying a target directory + # and will unjar all of the files in whatever the current directory is. + # Create a subdirectory to unjar into, set the location, unjar and then + # set the location back to its original location. + $UnjarredDocumentationPath = Join-Path -Path $DocLocation -ChildPath $PkgName + New-Item -ItemType directory -Path "$UnjarredDocumentationPath" + $CurrentLocation = Get-Location + Set-Location $UnjarredDocumentationPath + jar -xf "$($Item.FullName)" + Set-Location $CurrentLocation + + # Get the POM file for the artifact we're processing + $PomFile = $Item.FullName.Substring(0,$Item.FullName.LastIndexOf(("-javadoc.jar"))) + ".pom" + Write-Host "PomFile $($PomFile)" + + # Pull the version from the POM + [xml]$PomXml = Get-Content $PomFile + $Version = $PomXml.project.version + $ArtifactId = $PomXml.project.artifactId + + Write-Host "Start Upload for $($PkgName)/$($Version)" + Write-Host "DocDir $($UnjarredDocumentationPath)" + Write-Host "PkgName $($ArtifactId)" + Write-Host "DocVersion $($Version)" + + Upload-Blobs -DocDir $UnjarredDocumentationPath -PkgName $ArtifactId -DocVersion $Version + + } Finally { + if (![string]::IsNullOrEmpty($UnjarredDocumentationPath)) { + if (Test-Path -Path $UnjarredDocumentationPath) { + Write-Host "Cleaning up $UnjarredDocumentationPath" + Remove-Item -Recurse -Force $UnjarredDocumentationPath + } + } + } + } +} + +if ($Language -eq "c") +{ + # The documentation publishing process for C differs from the other + # langauges in this file because this script is invoked once per library + # publishing. It is not, for example, invoked once per service publishing. + # This is also the case for other langauge publishing steps above... Those + # loops are left over from previous versions of this script which were used + # to publish multiple docs packages in a single invocation. + $pkgInfo = Get-Content $DocLocation/package-info.json | ConvertFrom-Json + $pkgName = $pkgInfo.name + $pkgVersion = $pkgInfo.version + + Upload-Blobs -DocDir $DocLocation -PkgName $pkgName -DocVersion $pkgVersion +} \ No newline at end of file diff --git a/eng/common/scripts/create-tags-and-git-release.ps1 b/eng/common/scripts/create-tags-and-git-release.ps1 new file mode 100644 index 000000000..99d2c6dc4 --- /dev/null +++ b/eng/common/scripts/create-tags-and-git-release.ps1 @@ -0,0 +1,481 @@ +# ASSUMPTIONS +# * that `npm` cli is present for querying available npm packages +# * that an environment variable $env:GH_TOKEN is populated with the appropriate PAT to allow pushing of github releases + +param ( + # used by VerifyPackages + $artifactLocation, # the root of the artifact folder. DevOps $(System.ArtifactsDirectory) + $workingDirectory, # directory that package artifacts will be extracted into for examination (if necessary) + $packageRepository, # used to indicate destination against which we will check the existing version. + # valid options: PyPI, Nuget, NPM, Maven, C + # used by CreateTags + $releaseSha, # the SHA for the artifacts. DevOps: $(Release.Artifacts..SourceVersion) or $(Build.SourceVersion) + + # used by Git Release + $repoOwner = "", # the owning organization of the repository. EG "Azure" + $repoName = "", # the name of the repository. EG "azure-sdk-for-java" + $repoId = "$repoOwner/$repoName", # full repo id. EG azure/azure-sdk-for-net DevOps: $(Build.Repository.Id), + [switch]$forceCreate = $false +) + +$VERSION_REGEX = "(?\d+)(\.(?\d+))?(\.(?\d+))?((?
[^0-9][^\s]+))?"
+$SDIST_PACKAGE_REGEX = "^(?.*)\-(?$VERSION_REGEX$)"
+
+# Posts a github release for each item of the pkgList variable. SilentlyContinue
+function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) {
+  foreach ($pkgInfo in $pkgList) {
+    Write-Host "Creating release $($pkgInfo.Tag)"
+
+    $releaseNotes = ""
+    if ($pkgInfo.ReleaseNotes[$pkgInfo.PackageVersion].ReleaseContent -ne $null) {
+      $releaseNotes = $pkgInfo.ReleaseNotes[$pkgInfo.PackageVersion].ReleaseContent
+    }
+
+    $isPrerelease = $False
+    if ($pkgInfo.PackageVersion -match $VERSION_REGEX) {
+      $preReleaseLabel = $matches["pre"]
+      $isPrerelease = ![string]::IsNullOrEmpty($preReleaseLabel)
+    }
+
+    $url = $releaseApiUrl
+    $body = ConvertTo-Json @{
+      tag_name         = $pkgInfo.Tag
+      target_commitish = $releaseSha
+      name             = $pkgInfo.Tag
+      draft            = $False
+      prerelease       = $isPrerelease
+      body             = $releaseNotes
+    }
+
+    $headers = @{
+      "Content-Type"  = "application/json"
+      "Authorization" = "token $($env:GH_TOKEN)"
+    }
+
+    FireAPIRequest -url $url -body $body -headers $headers -method "Post"
+  }
+}
+
+function FireAPIRequest($url, $method, $body = $null, $headers = $null) {
+  $attempts = 1
+
+  while ($attempts -le 3) {
+    try {
+      return Invoke-RestMethod -Method $method -Uri $url -Body $body -Headers $headers
+    }
+    catch {
+      $response = $_.Exception.Response
+
+      $statusCode = $response.StatusCode.value__
+      $statusDescription = $response.StatusDescription
+
+      if ($statusCode) {
+        Write-Host "API request attempt number $attempts to $url failed with statuscode $statusCode"
+        Write-Host $statusDescription
+
+        Write-Host "Rate Limit Details:"
+        Write-Host "Total: $($response.Headers.GetValues("X-RateLimit-Limit"))"
+        Write-Host "Remaining: $($response.Headers.GetValues("X-RateLimit-Remaining"))"
+        Write-Host "Reset Epoch: $($response.Headers.GetValues("X-RateLimit-Reset"))"
+      }
+      else {
+        Write-Host "API request attempt number $attempts to $url failed with no statuscode present, exception follows:"
+        Write-Host $_.Exception.Response
+        Write-Host $_.Exception
+      }
+
+      if ($attempts -ge 3) {
+        Write-Host "Abandoning Request $url after 3 attempts."
+        exit(1)
+      }
+
+      Start-Sleep -s 10
+    }
+
+    $attempts += 1
+  }
+}
+
+# Parse out package publishing information given a maven POM file
+function ParseMavenPackage($pkg, $workingDirectory) {
+  [xml]$contentXML = Get-Content $pkg
+
+  $pkgId = $contentXML.project.artifactId
+  $pkgVersion = $contentXML.project.version
+  $groupId = if ($contentXML.project.groupId -eq $null) { $contentXML.project.parent.groupId } else { $contentXML.project.groupId }
+
+  # if it's a snapshot. return $null (as we don't want to create tags for this, but we also don't want to fail)
+  if ($pkgVersion.Contains("SNAPSHOT")) {
+    return $null
+  }
+
+  $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-changelog.md")[0]
+
+  return New-Object PSObject -Property @{
+    PackageId      = $pkgId
+    PackageVersion = $pkgVersion
+    Deployable     = $forceCreate -or !(IsMavenPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion -groupId $groupId.Replace(".", "/"))
+    ReleaseNotes   = $releaseNotes
+  }
+}
+
+# Returns the maven (really sonatype) publish status of a package id and version.
+function IsMavenPackageVersionPublished($pkgId, $pkgVersion, $groupId) {
+  try {
+
+    $uri = "https://oss.sonatype.org/content/repositories/releases/$groupId/$pkgId/$pkgVersion/$pkgId-$pkgVersion.pom"
+    $pomContent = Invoke-RestMethod -Method "GET" -Uri $uri
+
+    if ($pomContent -ne $null -or $pomContent.Length -eq 0) {
+      return $true
+    }
+    else {
+      return $false
+    }
+  }
+  catch {
+    $statusCode = $_.Exception.Response.StatusCode.value__
+    $statusDescription = $_.Exception.Response.StatusDescription
+
+    # if this is 404ing, then this pkg has never been published before
+    if ($statusCode -eq 404) {
+      return $false
+    }
+
+    Write-Host "VersionCheck to maven for packageId $pkgId failed with statuscode $statusCode"
+    Write-Host $statusDescription
+    exit(1)
+  }
+}
+
+# make certain to always take the package json closest to the top
+function ResolvePkgJson($workFolder) {
+  $pathsWithComplexity = @()
+  foreach ($file in (Get-ChildItem -Path $workFolder -Recurse -Include "package.json")) {
+    $complexity = ($file.FullName -Split { $_ -eq "/" -or $_ -eq "\" }).Length
+    $pathsWithComplexity += New-Object PSObject -Property @{
+      Path       = $file
+      Complexity = $complexity
+    }
+  }
+
+  return ($pathsWithComplexity | Sort-Object -Property Complexity)[0].Path
+}
+
+# Parse out package publishing information given a .tgz npm artifact
+function ParseNPMPackage($pkg, $workingDirectory) {
+  $workFolder = "$workingDirectory$($pkg.Basename)"
+  $origFolder = Get-Location
+  mkdir $workFolder
+  cd $workFolder
+
+  tar -xzf $pkg
+
+  $packageJSON = ResolvePkgJson -workFolder $workFolder | Get-Content | ConvertFrom-Json
+  $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
+
+  cd $origFolder
+  Remove-Item $workFolder -Force  -Recurse -ErrorAction SilentlyContinue
+
+  $pkgId = $packageJSON.name
+  $pkgVersion = $packageJSON.version
+
+  $resultObj = New-Object PSObject -Property @{
+    PackageId      = $pkgId
+    PackageVersion = $pkgVersion
+    Deployable     = $forceCreate -or !(IsNPMPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion)
+    ReleaseNotes   = $releaseNotes
+  }
+
+  return $resultObj
+}
+
+# Returns the npm publish status of a package id and version.
+function IsNPMPackageVersionPublished($pkgId, $pkgVersion) {
+  $npmVersions = (npm show $pkgId versions)
+
+  if ($LastExitCode -ne 0) {
+    npm ping
+
+    if ($LastExitCode -eq 0) {
+      return $False
+    }
+
+    Write-Host "Could not find a deployed version of $pkgId, and NPM connectivity check failed."
+    exit(1)
+  }
+
+  $npmVersionList = $npmVersions.split(",") | % { return $_.replace("[", "").replace("]", "").Trim() }
+  return $npmVersionList.Contains($pkgVersion)
+}
+
+# Parse out package publishing information given a nupkg ZIP format.
+function ParseNugetPackage($pkg, $workingDirectory) {
+  $workFolder = "$workingDirectory$($pkg.Basename)"
+  $origFolder = Get-Location
+  $zipFileLocation = "$workFolder/$($pkg.Basename).zip"
+  mkdir $workFolder
+
+  Copy-Item -Path $pkg -Destination $zipFileLocation
+  Expand-Archive -Path $zipFileLocation -DestinationPath $workFolder
+  [xml] $packageXML = Get-ChildItem -Path "$workFolder/*.nuspec" | Get-Content
+  $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
+
+  Remove-Item $workFolder -Force  -Recurse -ErrorAction SilentlyContinue
+  $pkgId = $packageXML.package.metadata.id
+  $pkgVersion = $packageXML.package.metadata.version
+
+  return New-Object PSObject -Property @{
+    PackageId      = $pkgId
+    PackageVersion = $pkgVersion
+    Deployable     = $forceCreate -or !(IsNugetPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion)
+    ReleaseNotes   = $releaseNotes
+  }
+}
+
+# Returns the nuget publish status of a package id and version.
+function IsNugetPackageVersionPublished($pkgId, $pkgVersion) {
+
+  $nugetUri = "https://api.nuget.org/v3-flatcontainer/$($pkgId.ToLowerInvariant())/index.json"
+
+  try {
+    $nugetVersions = Invoke-RestMethod -Method "GET" -Uri $nugetUri
+
+    return $nugetVersions.versions.Contains($pkgVersion)
+  }
+  catch {
+    $statusCode = $_.Exception.Response.StatusCode.value__
+    $statusDescription = $_.Exception.Response.StatusDescription
+
+    # if this is 404ing, then this pkg has never been published before
+    if ($statusCode -eq 404) {
+      return $False
+    }
+
+    Write-Host "Nuget Invocation failed:"
+    Write-Host "StatusCode:" $statusCode
+    Write-Host "StatusDescription:" $statusDescription
+    exit(1)
+  }
+
+}
+
+# Parse out package publishing information given a python sdist of ZIP format.
+function ParsePyPIPackage($pkg, $workingDirectory) {
+  $pkg.Basename -match $SDIST_PACKAGE_REGEX | Out-Null
+
+  $pkgId = $matches["package"]
+  $pkgVersion = $matches["versionstring"]
+
+  $workFolder = "$workingDirectory$($pkg.Basename)"
+  $origFolder = Get-Location
+  mkdir $workFolder
+
+  Expand-Archive -Path $pkg -DestinationPath $workFolder
+  $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
+  Remove-Item $workFolder -Force  -Recurse -ErrorAction SilentlyContinue
+
+  return New-Object PSObject -Property @{
+    PackageId      = $pkgId
+    PackageVersion = $pkgVersion
+    Deployable     = $forceCreate -or !(IsPythonPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion)
+    ReleaseNotes   = $releaseNotes
+  }
+}
+
+function ParseCArtifact($pkg, $workingDirectory) {
+  $packageInfo = Get-Content -Raw -Path $pkg | ConvertFrom-JSON
+  $packageArtifactLocation = (Get-ItemProperty $pkg).Directory.FullName
+
+  $releaseNotes = ExtractReleaseNotes -changeLogLocation @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "CHANGELOG.md")[0]
+
+  return New-Object PSObject -Property @{
+    PackageId      = $packageInfo.name
+    PackageVersion = $packageInfo.version
+    # Artifact info is always considered deployable for C becasue it is not
+    # deployed anywhere. Dealing with duplicate tags happens downstream in
+    # CheckArtifactShaAgainstTagsList
+    Deployable     = $true
+    ReleaseNotes   = $releaseNotes
+  }
+}
+
+# Returns the pypi publish status of a package id and version.
+function IsPythonPackageVersionPublished($pkgId, $pkgVersion) {
+  try {
+    $existingVersion = (Invoke-RestMethod -Method "Get" -Uri "https://pypi.org/pypi/$pkgId/$pkgVersion/json").info.version
+
+    # if existingVersion exists, then it's already been published
+    return $True
+  }
+  catch {
+    $statusCode = $_.Exception.Response.StatusCode.value__
+    $statusDescription = $_.Exception.Response.StatusDescription
+
+    # if this is 404ing, then this pkg has never been published before
+    if ($statusCode -eq 404) {
+      return $False
+    }
+
+    Write-Host "PyPI Invocation failed:"
+    Write-Host "StatusCode:" $statusCode
+    Write-Host "StatusDescription:" $statusDescription
+    exit(1)
+  }
+}
+
+# Retrieves the list of all tags that exist on the target repository
+function GetExistingTags($apiUrl) {
+  try {
+    return (Invoke-RestMethod -Method "GET" -Uri "$apiUrl/git/refs/tags"  ) | % { $_.ref.Replace("refs/tags/", "") }
+  }
+  catch {
+    $statusCode = $_.Exception.Response.StatusCode.value__
+    $statusDescription = $_.Exception.Response.StatusDescription
+
+    Write-Host "Failed to retrieve tags from repository."
+    Write-Host "StatusCode:" $statusCode
+    Write-Host "StatusDescription:" $statusDescription
+
+    # Return an empty list if there are no tags in the repo
+    if ($statusCode -eq 404) {
+      return @()
+    }
+
+    exit(1)
+  }
+}
+
+# Walk across all build artifacts, check them against the appropriate repository, return a list of tags/releases
+function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha) {
+  $pkgList = [array]@()
+  $ParsePkgInfoFn = ""
+  $packagePattern = ""
+
+  switch ($pkgRepository) {
+    "Maven" {
+      $ParsePkgInfoFn = "ParseMavenPackage"
+      $packagePattern = "*.pom"
+      break
+    }
+    "Nuget" {
+      $ParsePkgInfoFn = "ParseNugetPackage"
+      $packagePattern = "*.nupkg"
+      break
+    }
+    "NPM" {
+      $ParsePkgInfoFn = "ParseNPMPackage"
+      $packagePattern = "*.tgz"
+      break
+    }
+    "PyPI" {
+      $ParsePkgInfoFn = "ParsePyPIPackage"
+      $packagePattern = "*.zip"
+      break
+    }
+    "C" {
+      $ParsePkgInfoFn = "ParseCArtifact"
+      $packagePattern = "*.json"
+    }
+    default {
+      Write-Host "Unrecognized Language: $language"
+      exit(1)
+    }
+  }
+
+  $pkgs = (Get-ChildItem -Path $artifactLocation -Include $packagePattern -Recurse -File)
+
+  Write-Host $pkgs
+
+  foreach ($pkg in $pkgs) {
+    try {
+      $parsedPackage = &$ParsePkgInfoFn -pkg $pkg -workingDirectory $workingDirectory
+
+      if ($parsedPackage -eq $null) {
+        continue
+      }
+
+      if ($parsedPackage.Deployable -ne $True) {
+        Write-Host "Package $($parsedPackage.PackageId) is marked with version $($parsedPackage.PackageVersion), the version $($parsedPackage.PackageVersion) has already been deployed to the target repository."
+        Write-Host "Maybe a pkg version wasn't updated properly?"
+        exit(1)
+      }
+
+      $pkgList += New-Object PSObject -Property @{
+        PackageId      = $parsedPackage.PackageId
+        PackageVersion = $parsedPackage.PackageVersion
+        Tag            = ($parsedPackage.PackageId + "_" + $parsedPackage.PackageVersion)
+        ReleaseNotes   = $parsedPackage.ReleaseNotes
+      }
+    }
+    catch {
+      Write-Host $_.Exception.Message
+      exit(1)
+    }
+  }
+
+  $results = ([array]$pkgList | Sort-Object -Property Tag -uniq)
+
+  $existingTags = GetExistingTags($apiUrl)
+  $intersect = $results | % { $_.Tag } | ? { $existingTags -contains $_ }
+
+  if ($intersect.Length -gt 0) {
+    CheckArtifactShaAgainstTagsList -priorExistingTagList $intersect -releaseSha $releaseSha -apiUrl $apiUrl
+
+    # all the tags are clean. remove them from the list of releases we will publish.
+    $results = $results | ? { -not ($intersect -contains $_.Tag ) }
+  }
+
+  return $results
+}
+
+# given a set of tags that we want to release, we need to ensure that if they already DO exist.
+# if they DO exist, quietly exit if the commit sha of the artifact matches that of the tag
+# if the commit sha does not match, exit with error and report both problem shas
+function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $apiUrl) {
+  $headers = @{
+    "Content-Type"  = "application/json"
+    "Authorization" = "token $($env:GH_TOKEN)"
+  }
+
+  $unmatchedTags = @()
+
+  foreach ($tag in $priorExistingTagList) {
+    $tagSha = (FireAPIRequest -Method "Get" -Url "$apiUrl/git/refs/tags/$tag" -Headers $headers)."object".sha
+
+    if ($tagSha -eq $releaseSha) {
+      Write-Host "This package has already been released. The existing tag commit SHA $releaseSha matches the artifact SHA being processed. Skipping release step for this tag."
+    }
+    else {
+      Write-Host "The artifact SHA $releaseSha does not match that of the currently existing tag."
+      Write-Host "Tag with issues is $tag with commit SHA $tagSha"
+
+      $unmatchedTags += $tag
+    }
+  }
+
+  if ($unmatchedTags.Length -gt 0) {
+    Write-Host "Tags already existing with different SHA versions. Exiting."
+    exit(1)
+  }
+}
+
+$apiUrl = "https://api.github.com/repos/$repoId"
+Write-Host "Using API URL $apiUrl"
+
+# VERIFY PACKAGES
+$pkgList = VerifyPackages -pkgRepository $packageRepository -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha
+
+if ($pkgList) {
+  Write-Host "Given the visible artifacts, github releases will be created for the following:"
+
+  foreach ($packageInfo in $pkgList) {
+    Write-Host $packageInfo.Tag
+  }
+
+  # CREATE TAGS and RELEASES
+  CreateReleases -pkgList $pkgList -releaseApiUrl $apiUrl/releases -releaseSha $releaseSha
+}
+else {
+  Write-Host "After processing, no packages required release."
+}
diff --git a/eng/common/scripts/git-branch-push.ps1 b/eng/common/scripts/git-branch-push.ps1
new file mode 100644
index 000000000..813688b7d
--- /dev/null
+++ b/eng/common/scripts/git-branch-push.ps1
@@ -0,0 +1,145 @@
+ #!/usr/bin/env pwsh -c
+
+<#
+.DESCRIPTION
+Create local branch of the given repo and attempt to push changes. The push may fail if
+there has been other changes pushed to the same branch, if so, fetch, rebase and try again.
+.PARAMETER PRBranchName
+The name of the github branch the changes are being put into
+.PARAMETER CommitMsg
+The message for this particular commit
+.PARAMETER GitUrl
+The GitHub repository URL
+.PARAMETER PushArgs
+Optional arguments to the push command
+#>
+[CmdletBinding(SupportsShouldProcess = $true)]
+param(
+    [Parameter(Mandatory = $true)]
+    [string] $PRBranchName,
+
+    [Parameter(Mandatory = $true)]
+    [string] $CommitMsg,
+
+    [Parameter(Mandatory = $true)]
+    [string] $GitUrl,
+
+    [Parameter(Mandatory = $false)]
+    [string] $PushArgs = ""
+)
+
+# This is necessay because of the janky git command output writing to stderr.
+# Without explicitly setting the ErrorActionPreference to continue the script
+# would fail the first time git wrote command output.
+$ErrorActionPreference = "Continue"
+
+Write-Host "git remote add azure-sdk-fork $GitUrl"
+git remote add azure-sdk-fork $GitUrl
+if ($LASTEXITCODE -ne 0)
+{
+    Write-Error "Unable to add remote LASTEXITCODE=$($LASTEXITCODE), see command output above."
+    exit $LASTEXITCODE
+}
+
+Write-Host "git fetch azure-sdk-fork"
+git fetch azure-sdk-fork
+if ($LASTEXITCODE -ne 0)
+{
+    Write-Error "Unable to fetch remote LASTEXITCODE=$($LASTEXITCODE), see command output above."
+    exit $LASTEXITCODE
+}
+
+Write-Host "git checkout -b $PRBranchName"
+git checkout -b $PRBranchName
+if ($LASTEXITCODE -ne 0)
+{
+    Write-Error "Unable to create branch LASTEXITCODE=$($LASTEXITCODE), see command output above."
+    exit $LASTEXITCODE
+}
+
+Write-Host "git -c user.name=`"azure-sdk`" -c user.email=`"azuresdk@microsoft.com`" commit -am `"$($CommitMsg)`""
+git -c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com" commit -am "$($CommitMsg)"
+if ($LASTEXITCODE -ne 0)
+{
+    Write-Error "Unable to add files and create commit LASTEXITCODE=$($LASTEXITCODE), see command output above."
+    exit $LASTEXITCODE
+}
+
+# The number of retries can be increased if necessary. In theory, the number of retries
+# should be the max number of libraries in the largest pipeline -1 as everything except
+# the first commit could hit issues and need to rebase. The reason this isn't set to that
+# is because the largest pipeline is cognitive services which has 18 libraries in its
+# pipeline and that just seemed a bit too large and 10 seemed like a good starting value.
+$numberOfRetries = 10
+$needsRetry = $false
+$tryNumber = 0
+do 
+{ 
+    $needsRetry = $false
+    Write-Host "git push azure-sdk-fork $PRBranchName $PushArgs"
+    git push azure-sdk-fork $PRBranchName $PushArgs
+    $tryNumber++
+    if ($LASTEXITCODE -ne 0)
+    {
+        $needsRetry = $true
+        Write-Host "Git push failed with LASTEXITCODE=$($LASTEXITCODE) Need to fetch and rebase: attempt number=$($tryNumber)"
+        Write-Host "git fetch azure-sdk-fork"
+        git fetch azure-sdk-fork
+        if ($LASTEXITCODE -ne 0)
+        {
+            Write-Error "Unable to fetch remote LASTEXITCODE=$($LASTEXITCODE), see command output above."
+            exit $LASTEXITCODE
+        }
+
+        try 
+        {
+            $TempPatchFile = New-TemporaryFile
+            Write-Host "git diff ${PRBranchName}~ ${PRBranchName} --output $TempPatchFile"
+            git diff ${PRBranchName}~ ${PRBranchName} --output $TempPatchFile
+            if ($LASTEXITCODE -ne 0)
+            {
+                Write-Error "Unable to create diff file LASTEXITCODE=$($LASTEXITCODE), see command output above."
+                continue
+            }
+
+            Write-Host "git reset --hard azure-sdk-fork/${PRBranchName}"
+            git reset --hard azure-sdk-fork/${PRBranchName}
+            if ($LASTEXITCODE -ne 0)
+            {
+                Write-Error "Unable to hard reset branch LASTEXITCODE=$($LASTEXITCODE), see command output above."
+                continue
+            }
+
+            # -C0 means to use no extra before or after lines of context to enable us to avoid adjacent line merge conflicts
+            Write-Host "git apply -C0 $TempPatchFile"
+            git apply -C0 $TempPatchFile
+            if ($LASTEXITCODE -ne 0)
+            {
+                Write-Error "Unable to apply diff file LASTEXITCODE=$($LASTEXITCODE), see command output above."
+                continue
+            }
+
+            Write-Host "git -c user.name=`"azure-sdk`" -c user.email=`"azuresdk@microsoft.com`" commit -am `"$($CommitMsg)`""
+            git -c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com" commit -am "$($CommitMsg)"
+            if ($LASTEXITCODE -ne 0)
+            {
+                Write-Error "Unable to commit LASTEXITCODE=$($LASTEXITCODE), see command output above."
+                continue
+            }
+        }
+        finally
+        {
+            if ( Test-Path $TempPatchFile )
+            {
+                Remove-Item $TempPatchFile
+            }
+        }
+    }
+
+} while($needsRetry -and $tryNumber -le $numberOfRetries) 
+
+if ($LASTEXITCODE -ne 0)
+{
+    Write-Error "Unable to push commit after $($tryNumber) retries LASTEXITCODE=$($LASTEXITCODE), see command output above."
+    exit $LASTEXITCODE
+}