Sync eng/common directory with azure-sdk-tools for PR 7385 (#5223)

* Update organization of the eng/common/testproxy folder
* Add merge-proxy-tags.ps1 and readme to eng/common/testproxy/scripts/tag-merge/
* Extract common assets script functionality to common-asset-functions.ps1

---------

Co-authored-by: Scott Beddall (from Dev Box) <scbedd@microsoft.com>
This commit is contained in:
Azure SDK Bot 2023-12-06 15:18:40 -08:00 committed by GitHub
parent af6d5dcd07
commit 818d22778f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 677 additions and 273 deletions

View File

@ -0,0 +1,275 @@
class Assets {
[string]$AssetsRepo = $DefaultAssetsRepo
[string]$AssetsRepoPrefixPath = ""
[string]$TagPrefix = ""
[string]$Tag = ""
Assets(
[string]$AssetsRepoPrefixPath,
[string]$TagPrefix
) {
$this.TagPrefix = $TagPrefix
$this.AssetsRepoPrefixPath = $AssetsRepoPrefixPath
}
}
class Version {
[int]$Year
[int]$Month
[int]$Day
[int]$Revision
Version(
[string]$VersionString
) {
if ($VersionString -match "(?<year>20\d{2})(?<month>\d{2})(?<day>\d{2}).(?<revision>\d+)") {
$this.Year = [int]$Matches["year"]
$this.Month = [int]$Matches["month"]
$this.Day = [int]$Matches["day"]
$this.Revision = [int]$Matches["revision"]
}
else {
# This should be a Write-Error however powershell apparently cannot utilize that
# in the constructor in certain cases
Write-Warning "Version String '$($VersionString)' is invalid and cannot be parsed"
exit 1
}
}
[bool] IsGreaterEqual([string]$OtherVersionString) {
[Version]$OtherVersion = [Version]::new($OtherVersionString)
if ($this.Year -lt $OtherVersion.Year) {
return $false
}
elseif ($this.Year -eq $OtherVersion.Year) {
if ($this.Month -lt $OtherVersion.Month) {
return $false
}
elseif ($this.Month -eq $OtherVersion.Month) {
if ($this.Day -lt $OtherVersion.Day) {
return $false
}
elseif ($this.Day -eq $OtherVersion.Day) {
if ($this.Revision -lt $OtherVersion.Revision) {
return $false
}
}
}
}
return $true
}
}
Function Test-Exe-In-Path {
Param([string] $ExeToLookFor, [bool]$ExitOnError = $true)
if ($null -eq (Get-Command $ExeToLookFor -ErrorAction SilentlyContinue)) {
if ($ExitOnError) {
Write-Error "Unable to find $ExeToLookFor in your PATH"
exit 1
}
else {
return $false
}
}
return $true
}
Function Test-TestProxyVersion {
param(
[string] $TestProxyExe
)
Write-Host "$TestProxyExe --version"
[string] $output = & "$TestProxyExe" --version
[Version]$CurrentProxyVersion = [Version]::new($output)
if (!$CurrentProxyVersion.IsGreaterEqual($MinTestProxyVersion)) {
Write-Error "$TestProxyExe version, $output, is less than the minimum version $MinTestProxyVersion"
Write-Error "Please refer to https://github.com/Azure/azure-sdk-tools/blob/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md#installation to upgrade your $TestProxyExe"
exit 1
}
}
Function Get-Repo-Language {
$GitRepoOnDiskErr = "This script can only be called from within an azure-sdk-for-<lang> repository on disk."
# Git remote -v is going to give us output similar to the following
# origin git@github.com:Azure/azure-sdk-for-java.git (fetch)
# origin git@github.com:Azure/azure-sdk-for-java.git (push)
# upstream git@github.com:Azure/azure-sdk-for-java (fetch)
# upstream git@github.com:Azure/azure-sdk-for-java (push)
# We're really only trying to get the language from the git remote
Write-Host "git remote -v"
[array] $remotes = & git remote -v
foreach ($line in $remotes) {
Write-Host "$line"
}
# Git remote -v returned "fatal: not a git repository (or any of the parent directories): .git"
# and the list of remotes will be null
if (-not $remotes) {
Write-Error $GitRepoOnDiskErr
exit 1
}
# The regular expression needed to be updated to handle the following types of input:
# origin git@github.com:Azure/azure-sdk-for-python.git (fetch)
# origin git@github.com:Azure/azure-sdk-for-python-pr.git (fetch)
# fork git@github.com:UserName/azure-sdk-for-python (fetch)
# azure-sdk https://github.com/azure-sdk/azure-sdk-for-net.git (fetch)
# origin https://github.com/Azure/azure-sdk-for-python/ (fetch)
# ForEach-Object splits the string on whitespace so each of the above strings is actually
# 3 different strings. The first and last pieces won't match anything, the middle string
# will match what is below. If the regular expression needs to be updated the following
# link below will go to a regex playground
# https://regex101.com/r/auOnAr/1
$lang = $remotes[0] | ForEach-Object { if ($_ -match "azure-sdk-for-(?<lang>[^\-\.\/ ]+)") {
#Return the named language match
return $Matches["lang"]
}
}
if ([String]::IsNullOrWhitespace($lang)) {
Write-Error $GitRepoOnDiskErr
exit 1
}
Write-Host "Current language=$lang"
return $lang
}
Function Get-Repo-Root($StartDir=$null) {
[string] $currentDir = Get-Location
if ($StartDir){
$currentDir = $StartDir
}
# -1 to strip off the trialing directory separator
return $currentDir.Substring(0, $currentDir.LastIndexOf("sdk") - 1)
}
Function New-Assets-Json-File {
param(
[Parameter(Mandatory = $true)]
[string] $Language
)
$AssetsRepoPrefixPath = $Language
[string] $currentDir = Get-Location
$sdkDir = "$([IO.Path]::DirectorySeparatorChar)sdk$([IO.Path]::DirectorySeparatorChar)"
# if we're not in a <reporoot>/sdk/<ServiceDirectory> or deeper then this script isn't
# being run in the right place
if (-not $currentDir.contains($sdkDir)) {
Write-Error "This script needs to be run at an sdk/<ServiceDirectory> or deeper."
exit 1
}
$TagPrefix = $currentDir.Substring($currentDir.LastIndexOf("sdk") + 4)
$TagPrefix = $TagPrefix.Replace("\", "/")
$TagPrefix = "$($AssetsRepoPrefixPath)/$($TagPrefix)"
[Assets]$Assets = [Assets]::new($AssetsRepoPrefixPath, $TagPrefix)
$AssetsJson = $Assets | ConvertTo-Json
$AssetsFileName = Join-Path -Path $currentDir -ChildPath "assets.json"
Write-Host "Writing file $AssetsFileName with the following contents"
Write-Host $AssetsJson
$Assets | ConvertTo-Json | Out-File $AssetsFileName
return $AssetsFileName
}
# Invoke the proxy command and echo the output.
Function Invoke-ProxyCommand {
param(
[string] $TestProxyExe,
[string] $CommandString,
[string] $TargetDirectory
)
$updatedDirectory = $TargetDirectory.Replace("`\", "/")
# CommandString just a string indicating the proxy arguments. In the default case of running against the proxy tool, can just be used directly.
# However, in the case of docker, we need to append a bunch more arguments to the string.
if ($TestProxyExe -eq "docker" -or $TestProxyExe -eq "podman"){
$token = $env:GIT_TOKEN
$committer = $env:GIT_COMMIT_OWNER
$email = $env:GIT_COMMIT_EMAIL
if (-not $committer) {
$committer = & git config --global user.name
}
if (-not $email) {
$email = & git config --global user.email
}
if(-not $token -or -not $committer -or -not $email){
Write-Error ("When running this transition script in `"docker`" or `"podman`" mode, " `
+ "the environment variables GIT_TOKEN, GIT_COMMIT_OWNER, and GIT_COMMIT_EMAIL must be set to reflect the appropriate user. ")
exit 1
}
$targetImage = if ($env:TRANSITION_SCRIPT_DOCKER_TAG) { $env:TRANSITION_SCRIPT_DOCKER_TAG } else { "azsdkengsys.azurecr.io/engsys/test-proxy:latest" }
$CommandString = @(
"run --rm --name transition.test.proxy",
"-v `"${updatedDirectory}:/srv/testproxy`"",
"-e `"GIT_TOKEN=${token}`"",
"-e `"GIT_COMMIT_OWNER=${committer}`"",
"-e `"GIT_COMMIT_EMAIL=${email}`"",
$targetImage,
"test-proxy",
$CommandString
) -join " "
}
Write-Host "$TestProxyExe $CommandString"
[array] $output = & "$TestProxyExe" $CommandString.Split(" ") --storage-location="$updatedDirectory"
# echo the command output
foreach ($line in $output) {
Write-Host "$line"
}
}
# Get the shorthash directory under PROXY_ASSETS_FOLDER
Function Get-AssetsRoot {
param(
[string] $AssetsJsonFile,
[string] $TestProxyExe
)
$repoRoot = Get-Repo-Root
$relPath = [IO.Path]::GetRelativePath($repoRoot, $AssetsJsonFile).Replace("`\", "/")
$assetsJsonDirectory = Split-Path $relPath
[array] $output = & "$TestProxyExe" config locate -a "$relPath" --storage-location="$repoRoot"
$assetsDirectory = $output[-1]
return Join-Path $assetsDirectory $assetsJsonDirectory
}
Function Move-AssetsFromLangRepo {
param(
[string] $AssetsRoot
)
$filter = $LangRecordingDirs[$language]
Write-Host "Language recording directory name=$filter"
Write-Host "Get-ChildItem -Recurse -Filter ""*.json"" | Where-Object { if ($filter.Contains(""*"")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains ""$filter"" }"
$filesToMove = Get-ChildItem -Recurse -Filter "*.json" | Where-Object { if ($filter.Contains("*")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains "$filter" } }
[string] $currentDir = Get-Location
foreach ($fromFile in $filesToMove) {
$relPath = [IO.Path]::GetRelativePath($currentDir, $fromFile)
$toFile = Join-Path -Path $AssetsRoot -ChildPath $relPath
# Write-Host "Moving from=$fromFile"
# Write-Host " to=$toFile"
$toPath = Split-Path -Path $toFile
Write-Host $toFile
if (!(Test-Path $toPath)) {
New-Item -Path $toPath -ItemType Directory -Force | Out-Null
}
Move-Item -LiteralPath $fromFile -Destination $toFile -Force
}
}

View File

@ -78,274 +78,7 @@ $LangRecordingDirs = @{"cpp" = "recordings";
"python" = "recordings";
};
class Assets {
[string]$AssetsRepo = $DefaultAssetsRepo
[string]$AssetsRepoPrefixPath = ""
[string]$TagPrefix = ""
[string]$Tag = ""
Assets(
[string]$AssetsRepoPrefixPath,
[string]$TagPrefix
) {
$this.TagPrefix = $TagPrefix
$this.AssetsRepoPrefixPath = $AssetsRepoPrefixPath
}
}
class Version {
[int]$Year
[int]$Month
[int]$Day
[int]$Revision
Version(
[string]$VersionString
) {
if ($VersionString -match "(?<year>20\d{2})(?<month>\d{2})(?<day>\d{2}).(?<revision>\d+)") {
$this.Year = [int]$Matches["year"]
$this.Month = [int]$Matches["month"]
$this.Day = [int]$Matches["day"]
$this.Revision = [int]$Matches["revision"]
}
else {
# This should be a Write-Error however powershell apparently cannot utilize that
# in the constructor in certain cases
Write-Warning "Version String '$($VersionString)' is invalid and cannot be parsed"
exit 1
}
}
[bool] IsGreaterEqual([string]$OtherVersionString) {
[Version]$OtherVersion = [Version]::new($OtherVersionString)
if ($this.Year -lt $OtherVersion.Year) {
return $false
}
elseif ($this.Year -eq $OtherVersion.Year) {
if ($this.Month -lt $OtherVersion.Month) {
return $false
}
elseif ($this.Month -eq $OtherVersion.Month) {
if ($this.Day -lt $OtherVersion.Day) {
return $false
}
elseif ($this.Day -eq $OtherVersion.Day) {
if ($this.Revision -lt $OtherVersion.Revision) {
return $false
}
}
}
}
return $true
}
}
Function Test-Exe-In-Path {
Param([string] $ExeToLookFor, [bool]$ExitOnError = $true)
if ($null -eq (Get-Command $ExeToLookFor -ErrorAction SilentlyContinue)) {
if ($ExitOnError) {
Write-Error "Unable to find $ExeToLookFor in your PATH"
exit 1
}
else {
return $false
}
}
return $true
}
Function Test-TestProxyVersion {
param(
[string] $TestProxyExe
)
Write-Host "$TestProxyExe --version"
[string] $output = & "$TestProxyExe" --version
[Version]$CurrentProxyVersion = [Version]::new($output)
if (!$CurrentProxyVersion.IsGreaterEqual($MinTestProxyVersion)) {
Write-Error "$TestProxyExe version, $output, is less than the minimum version $MinTestProxyVersion"
Write-Error "Please refer to https://github.com/Azure/azure-sdk-tools/blob/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md#installation to upgrade your $TestProxyExe"
exit 1
}
}
Function Get-Repo-Language {
$GitRepoOnDiskErr = "This script can only be called from within an azure-sdk-for-<lang> repository on disk."
# Git remote -v is going to give us output similar to the following
# origin git@github.com:Azure/azure-sdk-for-java.git (fetch)
# origin git@github.com:Azure/azure-sdk-for-java.git (push)
# upstream git@github.com:Azure/azure-sdk-for-java (fetch)
# upstream git@github.com:Azure/azure-sdk-for-java (push)
# We're really only trying to get the language from the git remote
Write-Host "git remote -v"
[array] $remotes = & git remote -v
foreach ($line in $remotes) {
Write-Host "$line"
}
# Git remote -v returned "fatal: not a git repository (or any of the parent directories): .git"
# and the list of remotes will be null
if (-not $remotes) {
Write-Error $GitRepoOnDiskErr
exit 1
}
# The regular expression needed to be updated to handle the following types of input:
# origin git@github.com:Azure/azure-sdk-for-python.git (fetch)
# origin git@github.com:Azure/azure-sdk-for-python-pr.git (fetch)
# fork git@github.com:UserName/azure-sdk-for-python (fetch)
# azure-sdk https://github.com/azure-sdk/azure-sdk-for-net.git (fetch)
# origin https://github.com/Azure/azure-sdk-for-python/ (fetch)
# ForEach-Object splits the string on whitespace so each of the above strings is actually
# 3 different strings. The first and last pieces won't match anything, the middle string
# will match what is below. If the regular expression needs to be updated the following
# link below will go to a regex playground
# https://regex101.com/r/auOnAr/1
$lang = $remotes[0] | ForEach-Object { if ($_ -match "azure-sdk-for-(?<lang>[^\-\.\/ ]+)") {
#Return the named language match
return $Matches["lang"]
}
}
if ([String]::IsNullOrWhitespace($lang)) {
Write-Error $GitRepoOnDiskErr
exit 1
}
Write-Host "Current language=$lang"
return $lang
}
Function Get-Repo-Root {
[string] $currentDir = Get-Location
# -1 to strip off the trialing directory separator
return $currentDir.Substring(0, $currentDir.LastIndexOf("sdk") - 1)
}
Function New-Assets-Json-File {
param(
[Parameter(Mandatory = $true)]
[string] $Language
)
$AssetsRepoPrefixPath = $Language
[string] $currentDir = Get-Location
$sdkDir = "$([IO.Path]::DirectorySeparatorChar)sdk$([IO.Path]::DirectorySeparatorChar)"
# if we're not in a <reporoot>/sdk/<ServiceDirectory> or deeper then this script isn't
# being run in the right place
if (-not $currentDir.contains($sdkDir)) {
Write-Error "This script needs to be run at an sdk/<ServiceDirectory> or deeper."
exit 1
}
$TagPrefix = $currentDir.Substring($currentDir.LastIndexOf("sdk") + 4)
$TagPrefix = $TagPrefix.Replace("\", "/")
$TagPrefix = "$($AssetsRepoPrefixPath)/$($TagPrefix)"
[Assets]$Assets = [Assets]::new($AssetsRepoPrefixPath, $TagPrefix)
$AssetsJson = $Assets | ConvertTo-Json
$AssetsFileName = Join-Path -Path $currentDir -ChildPath "assets.json"
Write-Host "Writing file $AssetsFileName with the following contents"
Write-Host $AssetsJson
$Assets | ConvertTo-Json | Out-File $AssetsFileName
return $AssetsFileName
}
# Invoke the proxy command and echo the output.
Function Invoke-ProxyCommand {
param(
[string] $TestProxyExe,
[string] $CommandArgs,
[string] $TargetDirectory
)
$updatedDirectory = $TargetDirectory.Replace("`\", "/")
if ($TestProxyExe -eq "docker" -or $TestProxyExe -eq "podman"){
$token = $env:GIT_TOKEN
$committer = $env:GIT_COMMIT_OWNER
$email = $env:GIT_COMMIT_EMAIL
if (-not $committer) {
$committer = & git config --global user.name
}
if (-not $email) {
$email = & git config --global user.email
}
if(-not $token -or -not $committer -or -not $email){
Write-Error ("When running this transition script in `"docker`" or `"podman`" mode, " `
+ "the environment variables GIT_TOKEN, GIT_COMMIT_OWNER, and GIT_COMMIT_EMAIL must be set to reflect the appropriate user. ")
exit 1
}
$targetImage = if ($env:TRANSITION_SCRIPT_DOCKER_TAG) { $env:TRANSITION_SCRIPT_DOCKER_TAG } else { "azsdkengsys.azurecr.io/engsys/test-proxy:latest" }
$CommandArgs = @(
"run --rm --name transition.test.proxy",
"-v `"${updatedDirectory}:/srv/testproxy`"",
"-e `"GIT_TOKEN=${token}`"",
"-e `"GIT_COMMIT_OWNER=${committer}`"",
"-e `"GIT_COMMIT_EMAIL=${email}`"",
$targetImage,
"test-proxy",
$CommandArgs
) -join " "
}
Write-Host "$TestProxyExe $CommandArgs"
[array] $output = & "$TestProxyExe" $CommandArgs.Split(" ") --storage-location="$updatedDirectory"
# echo the command output
foreach ($line in $output) {
Write-Host "$line"
}
}
# Get the shorthash directory under PROXY_ASSETS_FOLDER
Function Get-AssetsRoot {
param(
[string] $AssetsJsonFile,
[string] $TestProxyExe
)
$repoRoot = Get-Repo-Root
$relPath = [IO.Path]::GetRelativePath($repoRoot, $AssetsJsonFile).Replace("`\", "/")
$assetsJsonDirectory = Split-Path $relPath
[array] $output = & "$TestProxyExe" config locate -a "$relPath" --storage-location="$repoRoot"
$assetsDirectory = $output[-1]
return Join-Path $assetsDirectory $assetsJsonDirectory
}
Function Move-AssetsFromLangRepo {
param(
[string] $AssetsRoot
)
$filter = $LangRecordingDirs[$language]
Write-Host "Language recording directory name=$filter"
Write-Host "Get-ChildItem -Recurse -Filter ""*.json"" | Where-Object { if ($filter.Contains(""*"")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains ""$filter"" }"
$filesToMove = Get-ChildItem -Recurse -Filter "*.json" | Where-Object { if ($filter.Contains("*")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains "$filter" } }
[string] $currentDir = Get-Location
foreach ($fromFile in $filesToMove) {
$relPath = [IO.Path]::GetRelativePath($currentDir, $fromFile)
$toFile = Join-Path -Path $AssetsRoot -ChildPath $relPath
# Write-Host "Moving from=$fromFile"
# Write-Host " to=$toFile"
$toPath = Split-Path -Path $toFile
Write-Host $toFile
if (!(Test-Path $toPath)) {
New-Item -Path $toPath -ItemType Directory -Force | Out-Null
}
Move-Item -LiteralPath $fromFile -Destination $toFile -Force
}
}
. (Join-Path $PSScriptRoot "common-asset-functions.ps1")
Test-Exe-In-Path -ExeToLookFor $GitExe
$language = Get-Repo-Language
@ -402,7 +135,7 @@ if ($InitialPush) {
# Execute a restore on the current assets.json, it'll prep the root directory that
# the recordings need to be copied into
$CommandArgs = "restore --assets-json-path $assetsJsonRelPath"
Invoke-ProxyCommand -TestProxyExe $TestProxyExe -CommandArgs $CommandArgs -TargetDirectory $repoRoot
Invoke-ProxyCommand -TestProxyExe $TestProxyExe -CommandString $CommandArgs -TargetDirectory $repoRoot
$assetsRoot = (Get-AssetsRoot -AssetsJsonFile $assetsJsonFile -TestProxyExe $TestProxyExe)
Write-Host "assetsRoot=$assetsRoot"
@ -410,7 +143,7 @@ if ($InitialPush) {
Move-AssetsFromLangRepo -AssetsRoot $assetsRoot
$CommandArgs = "push --assets-json-path $assetsJsonRelPath"
Invoke-ProxyCommand -TestProxyExe $TestProxyExe -CommandArgs $CommandArgs -TargetDirectory $repoRoot
Invoke-ProxyCommand -TestProxyExe $TestProxyExe -CommandString $CommandArgs -TargetDirectory $repoRoot
# Verify that the assets.json file was updated
$updatedAssets = Get-Content $assetsJsonFile | Out-String | ConvertFrom-Json

View File

@ -10,7 +10,7 @@ param(
[Parameter(mandatory=$true)] [string] $TargetVersion
)
$versionFile = Join-Path $PSScriptRoot "target_version.txt"
$versionFile = Join-Path $PSScriptRoot ".." "target_version.txt"
$existingVersionText = Get-Content -Raw -Path $versionFile
$existingVersion = $existingVersionText.Trim()

View File

@ -0,0 +1,94 @@
# Merge Proxy Tags Script
This script is intended to allow simpler combination of proxy tags. This is necessary due a few facts:
- Feature teams often need to combine their efforts while adding features. This means parallel re-recording or addition of tests.
- Instead of recordings being directly alongside the feature work, able to be merged simultaneously, now recordings are a single external reference from `assets.json`.
This script merely allows the abstraction of some of this "combination" work.
## Usage
### PreReqs
- Must have `git` available on your PATH
- Must have the `test-proxy` available on your PATH
- `test-proxy` is honored when the proxy is installed as a `dotnet tool`
- `Azure.Sdk.Tools.TestProxy` is honored when the standalone executable is on your PATH
- Preference for `dotnet tool` if present
### Call the script
```powershell
cd "path/to/language/repo/root"
./eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 sdk/storage/azure-storage-blob/assets.json integration/example/storage_feature_addition2 integration/example/storage_feature_addition1
# ^ Combined Tag 1 ^ Combined Tag 2
test-proxy push -a sdk/storage/azure-storage-blob/assets.json
```
### Resolve Conflicts
If the script ends early to a `git conflict` occurring, the script leaves the asset repo in a resolvable state.
- `cd` to the working directory described in the output from the script before it starts working. ("The work will be complete in...")
- `git status` to identify which files are conflicted
You will see something along these lines:
```bash
C:/repo/azure-sdk-for-python/.assets/eDscgL1p9G/python |>git status
HEAD detached from python/storage/azure-storage-blob_12c8154ae2
You are currently cherry-picking commit 1fd0865.
(fix conflicts and run "git cherry-pick --continue")
(use "git cherry-pick --skip" to skip this patch)
(use "git cherry-pick --abort" to cancel the cherry-pick operation)
You are in a sparse checkout with 100% of tracked files present.
Unmerged paths:
(use "git add <file>..." to mark resolution)
both added: sdk/storage/azure-storage-blob/tests/recordings/test_append_blob_async.pyTestStorageAppendBlobAsynctest_append_blob_from_text_new.json
no changes added to commit (use "git add" and/or "git commit -a")
```
Resolve the conflicts in the file, then add it using `git add <filename>`. Once the conflict is fully resolved, use
```bash
C:/repo/azure-sdk-for-python/.assets/eDscgL1p9G/python [???]|>git cherry-pick --continue
[detached HEAD 236e234] add the same file names as what was present in tag integration/example/storage_feature_addition2. In this case, the files themselves are just different enough from integration/example/storage_feature_addition2 that we should intentionally cause a conflict
Date: Fri Dec 1 16:57:52 2023 -0800
1 file changed, 2 insertions(+), 2 deletions(-)
```
Once you've resolved the conflict, re-run the same script. The results of the cherry-pick resolution will be visible.
```bash
C:/repo/azure-sdk-for-python [test-storage-tag-combination]|>eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 sdk/storage/azure-storage-blob/assets.json integration/example/storage_feature_addition2 integration/example/storage_feature_addition2_conflict integration/example/storage_feature_addition1
Excluding tag integration/example/storage_feature_addition2 because we have already done work against it in a previous script invocation.
Excluding tag integration/example/storage_feature_addition2_conflict because we have already done work against it in a previous script invocation.
This script has detected the presence of a .mergeprogress file within folder C:\repo\azure-sdk-for-python.
If the presence of a previous execution of this script is surprising, delete the .assets folder and .mergeprogress file before invoking the script again.
Attempting to continue from a previous run, and excluding:
- integration/example/storage_feature_addition2
- integration/example/storage_feature_addition2_conflict
But continuing with:
- integration/example/storage_feature_addition1
If the above looks correct, press enter, otherwise, ctrl-c:
```
On successful result, the following will be present:
```
Successfully combined 3 tags. Invoke "test-proxy push C:\repo\azure-sdk-for-python\sdk\storage\azure-storage-blob\assets.json" to push the results as a new tag.
```
Just follow the instructions to push your combined tag!
### Push the result
Once the script has completed successfully, `test-proxy push` the results!
```bash
test-proxy push sdk/storage/azure-storage-blob/assets.json
```

View File

@ -0,0 +1,302 @@
<#
.SYNOPSIS
Merge multiple asset tagss worth of content into a single asset tag.
.DESCRIPTION
USAGE: merge-proxy-tags.ps1 path/to/target_assets_json. TAG1 TAG2 TAG3
Attempts to merge the contents of multiple assets tags into a single new local changeset, which can be `test-proxy push`-ed.
In the case one of the targeted tags exists in the targeted assets.json, that tag will always be the start point.
1. test-proxy restore -a <assets-file> -> populate .assets
2. test-proxy config locate -a <assets-file> -> get location of cloned git repo
3. walk the incoming tags, cherry-picking their changes directly into the changeset _in context_
4. In the case of a discovered git conflict, the process ends. A list of which tags merged and which didn't will be presented to the user.
4a. Users should resolve the git conflicts themselves.
4b. If the conflict was on the final tag, resolve the conflict (leaving it uncommitted tyvm), and test-proxy push, you're done.
4c. If the conflict was _not_ on the final tag, resolve the conflict, commit it, and then re-run this script with the SAME arguments as before.
This script requires that test-proxy or azure.sdk.tools.testproxy should be on the PATH.
.PARAMETER AssetsJson
The script uses a target assets.json to resolve where specifically on disk the tag merging should take place.
.PARAMETER TargetTags
The set of tags whose contents should be combined. Any number of tags > 1 is allowed.
#>
param(
[Parameter(Position=0)]
[string] $AssetsJson,
[Parameter(Position=1, ValueFromRemainingArguments=$true)]
[string[]] $TargetTags
)
. (Join-Path $PSScriptRoot ".." ".." "onboarding" "common-asset-functions.ps1")
function Git-Command-With-Result($CommandString, $WorkingDirectory) {
Write-Host "git $CommandString"
if ($WorkingDirectory){
Push-Location $WorkingDirectory
}
$result = Invoke-Expression "git $CommandString"
if ($WorkingDirectory) {
Pop-Location
}
return [PSCustomObject]@{
ExitCode = $lastexitcode
Output = $result
}
}
function Git-Command($CommandString, $WorkingDirectory, $HardExit=$true) {
$result = Git-Command-With-Result $CommandString $WorkingDirectory
if ($result.ExitCode -ne 0 -and $HardExit) {
Write-Error $result.Output
exit 1
}
return $result.Output
}
function Resolve-Proxy {
# this script requires the presence of git
Test-Exe-In-Path -ExeToLookFor "git" | Out-Null
$testProxyExe = "test-proxy"
# this script requires the presence of the test-proxy on the PATH
$proxyToolPresent = Test-Exe-In-Path -ExeToLookFor "test-proxy" -ExitOnError $false
$proxyStandalonePresent = Test-Exe-In-Path -ExeToLookFor "Azure.Sdk.Tools.TestProxy" -ExitOnError $false
if (-not $proxyToolPresent -and -not $proxyStandalonePresent) {
Write-Error "This script requires the presence of a test-proxy executable to complete its operations. Exiting."
exit 1
}
if (-not $proxyToolPresent) {
$testProxyExe = "Azure.Sdk.Tools.TestProxy"
}
return $testProxyExe
}
function Call-Proxy {
param(
[string] $TestProxyExe,
[string] $CommandArgs,
[string] $MountDirectory,
[boolean] $Output = $true
)
$CommandArgs += " --storage-location=$MountDirectory"
if ($Output -eq $true) {
Write-Host "$TestProxyExe $CommandArgs"
}
[array] $output = & "$TestProxyExe" $CommandArgs.Split(" ")
if ($lastexitcode -ne 0) {
foreach($line in $output) {
Write-Host $line
}
Write-Error "Proxy exe exited with unexpected non-zero exit code."
exit 1
}
if ($Output -eq $true) {
foreach($line in $output) {
Write-Host $line
}
}
return $output
}
function Locate-Assets-Slice($ProxyExe, $AssetsJson, $MountDirectory) {
$CommandString = "config locate -a $AssetsJson"
$output = Call-Proxy -TestProxyExe $ProxyExe -CommandArgs $CommandString -MountDirectory $MountDirectory -Output $false
return $output[-1].Trim()
}
function Get-Tag-SHA($TagName, $WorkingDirectory) {
$results = Git-Command "ls-remote origin $TagName" $WorkingDirectory
if ($results -and $lastexitcode -eq 0) {
$arr = $results -split '\s+'
return $arr[0]
}
Write-Error "Unable to fetch tag SHA for $TagName. The tag does not exist on the repository."
exit 1
}
function Start-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory) {
$alreadyCombinedTags = Load-Incomplete-Progress $MountDirectory
$TargetTags = $TargetTags | Where-Object { $_ -notin $alreadyCombinedTags }
if ($alreadyCombinedTags) {
Write-Host "This script has detected the presence of a .mergeprogress file within folder $MountDirectory."
Write-Host "If the presence of a previous execution of this script is surprising, delete the .assets folder and .mergeprogress file before invoking the script again."
Write-Host "Attempting to continue from a previous run, and excluding:"
foreach($Tag in $alreadyCombinedTags) {
Write-Host " - " -NoNewLine
Write-Host "$Tag" -ForegroundColor Green
}
Write-Host "But continuing with:"
foreach($Tag in $TargetTags){
Write-Host " - " -NoNewLine
Write-Host "$Tag" -ForegroundColor Green
}
}
else {
Write-Host "`nThis script will attempt to merge the following tag" -NoNewLine
if ($TargetTags.Length -gt 1) {
Write-Host "s" -NoNewLine
}
Write-Host ":"
foreach($Tag in $TargetTags) {
Write-Host " - " -NoNewLine
Write-Host "$Tag" -ForegroundColor Green
}
Write-Host "`nTargeting the assets slice targeted by " -NoNewLine
Write-Host "$AssetsJson." -ForegroundColor Green
Write-Host "`nThe work will be completed in " -NoNewLine
Write-Host $AssetsRepoLocation -ForegroundColor Green -NoNewLine
Write-Host "."
}
Read-Host -Prompt "If the above looks correct, press enter, otherwise, ctrl-c"
}
function Finish-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory) {
$len = $TargetTags.Length
Write-Host "`nSuccessfully combined $len tags. Invoke `"test-proxy push " -NoNewLine
Write-Host $AssetsJson -ForegroundColor Green -NoNewLine
Write-Host "`" to push the results as a new tag."
}
function Resolve-Target-Tags($AssetsJson, $TargetTags, $MountDirectory) {
$inprogress = Load-Incomplete-Progress $MountDirectory
$jsonContent = Get-Content -Raw -Path $AssetsJson
$jsonObj = $JsonContent | ConvertFrom-Json
$existingTarget = $jsonObj.Tag
return $TargetTags | Where-Object {
if ($_ -eq $existingTarget) {
Write-Host "Excluding tag $($_) from tag input list, it is present in assets.json."
}
$_ -ne $existingTarget
} | Where-Object {
if ($_ -in $inprogress) {
Write-Host "Excluding tag $($_) because we have already done work against it in a previous script invocation."
}
$_ -notin $inprogress
}
}
function Save-Incomplete-Progress($Tag, $MountDirectory) {
$progressFile = (Join-Path $MountDirectory ".mergeprogress")
[array] $existingTags = @()
if (Test-Path $progressFile) {
$existingTags = (Get-Content -Path $progressFile) -split "`n" | ForEach-Object { $_.Trim() }
}
$existingTags = $existingTags + $Tag | Select-Object -Unique
Set-Content -Path $progressFile -Value ($existingTags -join "`n") | Out-Null
return $existingTags
}
function Load-Incomplete-Progress($MountDirectory) {
$progressFile = (Join-Path $MountDirectory ".mergeprogress")
[array] $existingTags = @()
if (Test-Path $progressFile) {
$existingTags = ((Get-Content -Path $progressFile) -split "`n" | ForEach-Object { $_.Trim() })
}
return $existingTags
}
function Cleanup-Incomplete-Progress($MountDirectory) {
$progressFile = (Join-Path $MountDirectory ".mergeprogress")
if (Test-Path $progressFile) {
Remove-Item $progressFile | Out-Null
}
}
function Prepare-Assets($ProxyExe, $MountDirectory, $AssetsJson) {
$inprogress = Load-Incomplete-Progress $MountDirectory
if ($inprogress.Length -eq 0) {
Call-Proxy -TestProxyExe $ProxyExe -CommandArgs "reset -y -a $AssetsJson" -MountDirectory $MountDirectory -Output $false
}
}
function Combine-Tags($RemainingTags, $AssetsRepoLocation, $MountDirectory){
foreach($Tag in $RemainingTags) {
$tagSha = Get-Tag-SHA $Tag $AssetsRepoLocation
$existingTags = Save-Incomplete-Progress $Tag $MountDirectory
$cherryPickResult = Git-Command-With-Result "cherry-pick $tagSha" - $AssetsRepoLocation -HardExit $false
if ($cherryPickResult.ExitCode -ne 0) {
Write-Host "Conflicts while cherry-picking $Tag. Resolve the the conflict over in `"$AssetsRepoLocation`", and re-run this script with the same arguments as before." -ForegroundColor Red
exit 1
}
}
$pushedTags = Load-Incomplete-Progress $MountDirectory
$testFile = Get-ChildItem -Recurse -Path $AssetsRepoLocation | Where-Object { !$_.PSIsContainer } | Select-Object -First 1
Add-Content -Path $testFile -Value "`n"
# if we have successfully gotten to the end without any non-zero exit codes...delete the mergeprogress file, we're g2g
Cleanup-Incomplete-Progress $MountDirectory
return $pushedTags
}
$ErrorActionPreference = "Stop"
# resolve the proxy location so that we can invoke it easily
$proxyExe = Resolve-Proxy
$AssetsJson = Resolve-Path $AssetsJson
# figure out where the root of the repo for the passed assets.json is. We need it to properly set the mounting
# directory so that the test-proxy restore operations work IN PLACE with existing tooling
$mountDirectory = Get-Repo-Root -StartDir $AssetsJson
# ensure we actually have the .assets folder that we can cherry-pick on top of
Prepare-Assets $proxyExe $mountDirectory $AssetsJson
# using the mountingDirectory and the assets.json location, we can figure out where the assets slice actually lives within the .assets folder.
# we will use this to invoke individual cherry-picks before pushing up the result
$assetsRepoLocation = Locate-Assets-Slice $proxyExe $AssetsJson $mountDirectory
# resolve the tags that we will go after. If the target assets.json contains one of these tags, that tag is _already present_
# because the entire point of this script is to run in context of a set of recordings in the repo
$tags = Resolve-Target-Tags $AssetsJson $TargetTags $mountDirectory
Start-Message $AssetsJson $Tags $AssetsRepoLocation $mountDirectory
$CombinedTags = Combine-Tags $Tags $AssetsRepoLocation $mountDirectory
Finish-Message $AssetsJson $CombinedTags $AssetsRepoLocation $mountDirectory

View File

@ -15,7 +15,7 @@ steps:
condition: and(succeeded(), ${{ parameters.condition }}, ne('${{ parameters.targetVersion }}', ''))
inputs:
targetType: filePath
filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/override-proxy-version.ps1'
filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/scripts/override-proxy-version.ps1'
arguments: '-TargetVersion "${{ parameters.targetVersion }}"'
pwsh: true

View File

@ -16,7 +16,7 @@ steps:
condition: and(succeeded(), ${{ parameters.condition }}, ne('${{ parameters.targetVersion }}', ''))
inputs:
targetType: filePath
filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/override-proxy-version.ps1'
filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/scripts/override-proxy-version.ps1'
arguments: '-TargetVersion "${{ parameters.targetVersion }}"'
pwsh: true