diff --git a/eng/common/testproxy/onboarding/common-asset-functions.ps1 b/eng/common/testproxy/onboarding/common-asset-functions.ps1 new file mode 100644 index 000000000..3caa6654c --- /dev/null +++ b/eng/common/testproxy/onboarding/common-asset-functions.ps1 @@ -0,0 +1,275 @@ +class Assets { + [string]$AssetsRepo = $DefaultAssetsRepo + [string]$AssetsRepoPrefixPath = "" + [string]$TagPrefix = "" + [string]$Tag = "" + Assets( + [string]$AssetsRepoPrefixPath, + [string]$TagPrefix + ) { + $this.TagPrefix = $TagPrefix + $this.AssetsRepoPrefixPath = $AssetsRepoPrefixPath + } +} + +class Version { + [int]$Year + [int]$Month + [int]$Day + [int]$Revision + Version( + [string]$VersionString + ) { + if ($VersionString -match "(?20\d{2})(?\d{2})(?\d{2}).(?\d+)") { + $this.Year = [int]$Matches["year"] + $this.Month = [int]$Matches["month"] + $this.Day = [int]$Matches["day"] + $this.Revision = [int]$Matches["revision"] + } + else { + # This should be a Write-Error however powershell apparently cannot utilize that + # in the constructor in certain cases + Write-Warning "Version String '$($VersionString)' is invalid and cannot be parsed" + exit 1 + } + } + [bool] IsGreaterEqual([string]$OtherVersionString) { + [Version]$OtherVersion = [Version]::new($OtherVersionString) + if ($this.Year -lt $OtherVersion.Year) { + return $false + } + elseif ($this.Year -eq $OtherVersion.Year) { + if ($this.Month -lt $OtherVersion.Month) { + return $false + } + elseif ($this.Month -eq $OtherVersion.Month) { + if ($this.Day -lt $OtherVersion.Day) { + return $false + } + elseif ($this.Day -eq $OtherVersion.Day) { + if ($this.Revision -lt $OtherVersion.Revision) { + return $false + } + } + } + } + return $true + } +} + +Function Test-Exe-In-Path { + Param([string] $ExeToLookFor, [bool]$ExitOnError = $true) + if ($null -eq (Get-Command $ExeToLookFor -ErrorAction SilentlyContinue)) { + if ($ExitOnError) { + Write-Error "Unable to find $ExeToLookFor in your PATH" + exit 1 + } + else { + return $false + } + } + + return $true +} + +Function Test-TestProxyVersion { + param( + [string] $TestProxyExe + ) + + Write-Host "$TestProxyExe --version" + [string] $output = & "$TestProxyExe" --version + + [Version]$CurrentProxyVersion = [Version]::new($output) + if (!$CurrentProxyVersion.IsGreaterEqual($MinTestProxyVersion)) { + Write-Error "$TestProxyExe version, $output, is less than the minimum version $MinTestProxyVersion" + Write-Error "Please refer to https://github.com/Azure/azure-sdk-tools/blob/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md#installation to upgrade your $TestProxyExe" + exit 1 + } +} + +Function Get-Repo-Language { + + $GitRepoOnDiskErr = "This script can only be called from within an azure-sdk-for- repository on disk." + # Git remote -v is going to give us output similar to the following + # origin git@github.com:Azure/azure-sdk-for-java.git (fetch) + # origin git@github.com:Azure/azure-sdk-for-java.git (push) + # upstream git@github.com:Azure/azure-sdk-for-java (fetch) + # upstream git@github.com:Azure/azure-sdk-for-java (push) + # We're really only trying to get the language from the git remote + Write-Host "git remote -v" + [array] $remotes = & git remote -v + foreach ($line in $remotes) { + Write-Host "$line" + } + + # Git remote -v returned "fatal: not a git repository (or any of the parent directories): .git" + # and the list of remotes will be null + if (-not $remotes) { + Write-Error $GitRepoOnDiskErr + exit 1 + } + + # The regular expression needed to be updated to handle the following types of input: + # origin git@github.com:Azure/azure-sdk-for-python.git (fetch) + # origin git@github.com:Azure/azure-sdk-for-python-pr.git (fetch) + # fork git@github.com:UserName/azure-sdk-for-python (fetch) + # azure-sdk https://github.com/azure-sdk/azure-sdk-for-net.git (fetch) + # origin https://github.com/Azure/azure-sdk-for-python/ (fetch) + # ForEach-Object splits the string on whitespace so each of the above strings is actually + # 3 different strings. The first and last pieces won't match anything, the middle string + # will match what is below. If the regular expression needs to be updated the following + # link below will go to a regex playground + # https://regex101.com/r/auOnAr/1 + $lang = $remotes[0] | ForEach-Object { if ($_ -match "azure-sdk-for-(?[^\-\.\/ ]+)") { + #Return the named language match + return $Matches["lang"] + } + } + + if ([String]::IsNullOrWhitespace($lang)) { + Write-Error $GitRepoOnDiskErr + exit 1 + } + + Write-Host "Current language=$lang" + return $lang +} + +Function Get-Repo-Root($StartDir=$null) { + [string] $currentDir = Get-Location + + if ($StartDir){ + $currentDir = $StartDir + } + + # -1 to strip off the trialing directory separator + return $currentDir.Substring(0, $currentDir.LastIndexOf("sdk") - 1) +} + +Function New-Assets-Json-File { + param( + [Parameter(Mandatory = $true)] + [string] $Language + ) + $AssetsRepoPrefixPath = $Language + + [string] $currentDir = Get-Location + + $sdkDir = "$([IO.Path]::DirectorySeparatorChar)sdk$([IO.Path]::DirectorySeparatorChar)" + + # if we're not in a /sdk/ or deeper then this script isn't + # being run in the right place + if (-not $currentDir.contains($sdkDir)) { + Write-Error "This script needs to be run at an sdk/ or deeper." + exit 1 + } + + $TagPrefix = $currentDir.Substring($currentDir.LastIndexOf("sdk") + 4) + $TagPrefix = $TagPrefix.Replace("\", "/") + $TagPrefix = "$($AssetsRepoPrefixPath)/$($TagPrefix)" + [Assets]$Assets = [Assets]::new($AssetsRepoPrefixPath, $TagPrefix) + + $AssetsJson = $Assets | ConvertTo-Json + + $AssetsFileName = Join-Path -Path $currentDir -ChildPath "assets.json" + Write-Host "Writing file $AssetsFileName with the following contents" + Write-Host $AssetsJson + $Assets | ConvertTo-Json | Out-File $AssetsFileName + + return $AssetsFileName +} + +# Invoke the proxy command and echo the output. +Function Invoke-ProxyCommand { + param( + [string] $TestProxyExe, + [string] $CommandString, + [string] $TargetDirectory + ) + $updatedDirectory = $TargetDirectory.Replace("`\", "/") + + # CommandString just a string indicating the proxy arguments. In the default case of running against the proxy tool, can just be used directly. + # However, in the case of docker, we need to append a bunch more arguments to the string. + if ($TestProxyExe -eq "docker" -or $TestProxyExe -eq "podman"){ + $token = $env:GIT_TOKEN + $committer = $env:GIT_COMMIT_OWNER + $email = $env:GIT_COMMIT_EMAIL + + if (-not $committer) { + $committer = & git config --global user.name + } + + if (-not $email) { + $email = & git config --global user.email + } + + if(-not $token -or -not $committer -or -not $email){ + Write-Error ("When running this transition script in `"docker`" or `"podman`" mode, " ` + + "the environment variables GIT_TOKEN, GIT_COMMIT_OWNER, and GIT_COMMIT_EMAIL must be set to reflect the appropriate user. ") + exit 1 + } + + $targetImage = if ($env:TRANSITION_SCRIPT_DOCKER_TAG) { $env:TRANSITION_SCRIPT_DOCKER_TAG } else { "azsdkengsys.azurecr.io/engsys/test-proxy:latest" } + + $CommandString = @( + "run --rm --name transition.test.proxy", + "-v `"${updatedDirectory}:/srv/testproxy`"", + "-e `"GIT_TOKEN=${token}`"", + "-e `"GIT_COMMIT_OWNER=${committer}`"", + "-e `"GIT_COMMIT_EMAIL=${email}`"", + $targetImage, + "test-proxy", + $CommandString + ) -join " " + } + + Write-Host "$TestProxyExe $CommandString" + [array] $output = & "$TestProxyExe" $CommandString.Split(" ") --storage-location="$updatedDirectory" + # echo the command output + foreach ($line in $output) { + Write-Host "$line" + } +} + +# Get the shorthash directory under PROXY_ASSETS_FOLDER +Function Get-AssetsRoot { + param( + [string] $AssetsJsonFile, + [string] $TestProxyExe + ) + $repoRoot = Get-Repo-Root + $relPath = [IO.Path]::GetRelativePath($repoRoot, $AssetsJsonFile).Replace("`\", "/") + $assetsJsonDirectory = Split-Path $relPath + + [array] $output = & "$TestProxyExe" config locate -a "$relPath" --storage-location="$repoRoot" + $assetsDirectory = $output[-1] + + return Join-Path $assetsDirectory $assetsJsonDirectory +} + +Function Move-AssetsFromLangRepo { + param( + [string] $AssetsRoot + ) + $filter = $LangRecordingDirs[$language] + Write-Host "Language recording directory name=$filter" + Write-Host "Get-ChildItem -Recurse -Filter ""*.json"" | Where-Object { if ($filter.Contains(""*"")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains ""$filter"" }" + $filesToMove = Get-ChildItem -Recurse -Filter "*.json" | Where-Object { if ($filter.Contains("*")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains "$filter" } } + [string] $currentDir = Get-Location + + foreach ($fromFile in $filesToMove) { + $relPath = [IO.Path]::GetRelativePath($currentDir, $fromFile) + + $toFile = Join-Path -Path $AssetsRoot -ChildPath $relPath + # Write-Host "Moving from=$fromFile" + # Write-Host " to=$toFile" + $toPath = Split-Path -Path $toFile + + Write-Host $toFile + if (!(Test-Path $toPath)) { + New-Item -Path $toPath -ItemType Directory -Force | Out-Null + } + Move-Item -LiteralPath $fromFile -Destination $toFile -Force + } +} \ No newline at end of file diff --git a/eng/common/testproxy/onboarding/generate-assets-json.ps1 b/eng/common/testproxy/onboarding/generate-assets-json.ps1 index 3011a19e5..bd825eebb 100644 --- a/eng/common/testproxy/onboarding/generate-assets-json.ps1 +++ b/eng/common/testproxy/onboarding/generate-assets-json.ps1 @@ -78,274 +78,7 @@ $LangRecordingDirs = @{"cpp" = "recordings"; "python" = "recordings"; }; -class Assets { - [string]$AssetsRepo = $DefaultAssetsRepo - [string]$AssetsRepoPrefixPath = "" - [string]$TagPrefix = "" - [string]$Tag = "" - Assets( - [string]$AssetsRepoPrefixPath, - [string]$TagPrefix - ) { - $this.TagPrefix = $TagPrefix - $this.AssetsRepoPrefixPath = $AssetsRepoPrefixPath - } -} - -class Version { - [int]$Year - [int]$Month - [int]$Day - [int]$Revision - Version( - [string]$VersionString - ) { - if ($VersionString -match "(?20\d{2})(?\d{2})(?\d{2}).(?\d+)") { - $this.Year = [int]$Matches["year"] - $this.Month = [int]$Matches["month"] - $this.Day = [int]$Matches["day"] - $this.Revision = [int]$Matches["revision"] - } - else { - # This should be a Write-Error however powershell apparently cannot utilize that - # in the constructor in certain cases - Write-Warning "Version String '$($VersionString)' is invalid and cannot be parsed" - exit 1 - } - } - [bool] IsGreaterEqual([string]$OtherVersionString) { - [Version]$OtherVersion = [Version]::new($OtherVersionString) - if ($this.Year -lt $OtherVersion.Year) { - return $false - } - elseif ($this.Year -eq $OtherVersion.Year) { - if ($this.Month -lt $OtherVersion.Month) { - return $false - } - elseif ($this.Month -eq $OtherVersion.Month) { - if ($this.Day -lt $OtherVersion.Day) { - return $false - } - elseif ($this.Day -eq $OtherVersion.Day) { - if ($this.Revision -lt $OtherVersion.Revision) { - return $false - } - } - } - } - return $true - } -} - -Function Test-Exe-In-Path { - Param([string] $ExeToLookFor, [bool]$ExitOnError = $true) - if ($null -eq (Get-Command $ExeToLookFor -ErrorAction SilentlyContinue)) { - if ($ExitOnError) { - Write-Error "Unable to find $ExeToLookFor in your PATH" - exit 1 - } - else { - return $false - } - } - - return $true -} - -Function Test-TestProxyVersion { - param( - [string] $TestProxyExe - ) - - Write-Host "$TestProxyExe --version" - [string] $output = & "$TestProxyExe" --version - - [Version]$CurrentProxyVersion = [Version]::new($output) - if (!$CurrentProxyVersion.IsGreaterEqual($MinTestProxyVersion)) { - Write-Error "$TestProxyExe version, $output, is less than the minimum version $MinTestProxyVersion" - Write-Error "Please refer to https://github.com/Azure/azure-sdk-tools/blob/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md#installation to upgrade your $TestProxyExe" - exit 1 - } -} - -Function Get-Repo-Language { - - $GitRepoOnDiskErr = "This script can only be called from within an azure-sdk-for- repository on disk." - # Git remote -v is going to give us output similar to the following - # origin git@github.com:Azure/azure-sdk-for-java.git (fetch) - # origin git@github.com:Azure/azure-sdk-for-java.git (push) - # upstream git@github.com:Azure/azure-sdk-for-java (fetch) - # upstream git@github.com:Azure/azure-sdk-for-java (push) - # We're really only trying to get the language from the git remote - Write-Host "git remote -v" - [array] $remotes = & git remote -v - foreach ($line in $remotes) { - Write-Host "$line" - } - - # Git remote -v returned "fatal: not a git repository (or any of the parent directories): .git" - # and the list of remotes will be null - if (-not $remotes) { - Write-Error $GitRepoOnDiskErr - exit 1 - } - - # The regular expression needed to be updated to handle the following types of input: - # origin git@github.com:Azure/azure-sdk-for-python.git (fetch) - # origin git@github.com:Azure/azure-sdk-for-python-pr.git (fetch) - # fork git@github.com:UserName/azure-sdk-for-python (fetch) - # azure-sdk https://github.com/azure-sdk/azure-sdk-for-net.git (fetch) - # origin https://github.com/Azure/azure-sdk-for-python/ (fetch) - # ForEach-Object splits the string on whitespace so each of the above strings is actually - # 3 different strings. The first and last pieces won't match anything, the middle string - # will match what is below. If the regular expression needs to be updated the following - # link below will go to a regex playground - # https://regex101.com/r/auOnAr/1 - $lang = $remotes[0] | ForEach-Object { if ($_ -match "azure-sdk-for-(?[^\-\.\/ ]+)") { - #Return the named language match - return $Matches["lang"] - } - } - - if ([String]::IsNullOrWhitespace($lang)) { - Write-Error $GitRepoOnDiskErr - exit 1 - } - - Write-Host "Current language=$lang" - return $lang -} - -Function Get-Repo-Root { - [string] $currentDir = Get-Location - # -1 to strip off the trialing directory separator - return $currentDir.Substring(0, $currentDir.LastIndexOf("sdk") - 1) -} - -Function New-Assets-Json-File { - param( - [Parameter(Mandatory = $true)] - [string] $Language - ) - $AssetsRepoPrefixPath = $Language - - [string] $currentDir = Get-Location - - $sdkDir = "$([IO.Path]::DirectorySeparatorChar)sdk$([IO.Path]::DirectorySeparatorChar)" - - # if we're not in a /sdk/ or deeper then this script isn't - # being run in the right place - if (-not $currentDir.contains($sdkDir)) { - Write-Error "This script needs to be run at an sdk/ or deeper." - exit 1 - } - - $TagPrefix = $currentDir.Substring($currentDir.LastIndexOf("sdk") + 4) - $TagPrefix = $TagPrefix.Replace("\", "/") - $TagPrefix = "$($AssetsRepoPrefixPath)/$($TagPrefix)" - [Assets]$Assets = [Assets]::new($AssetsRepoPrefixPath, $TagPrefix) - - $AssetsJson = $Assets | ConvertTo-Json - - $AssetsFileName = Join-Path -Path $currentDir -ChildPath "assets.json" - Write-Host "Writing file $AssetsFileName with the following contents" - Write-Host $AssetsJson - $Assets | ConvertTo-Json | Out-File $AssetsFileName - - return $AssetsFileName -} - -# Invoke the proxy command and echo the output. -Function Invoke-ProxyCommand { - param( - [string] $TestProxyExe, - [string] $CommandArgs, - [string] $TargetDirectory - ) - $updatedDirectory = $TargetDirectory.Replace("`\", "/") - - if ($TestProxyExe -eq "docker" -or $TestProxyExe -eq "podman"){ - $token = $env:GIT_TOKEN - $committer = $env:GIT_COMMIT_OWNER - $email = $env:GIT_COMMIT_EMAIL - - if (-not $committer) { - $committer = & git config --global user.name - } - - if (-not $email) { - $email = & git config --global user.email - } - - if(-not $token -or -not $committer -or -not $email){ - Write-Error ("When running this transition script in `"docker`" or `"podman`" mode, " ` - + "the environment variables GIT_TOKEN, GIT_COMMIT_OWNER, and GIT_COMMIT_EMAIL must be set to reflect the appropriate user. ") - exit 1 - } - - $targetImage = if ($env:TRANSITION_SCRIPT_DOCKER_TAG) { $env:TRANSITION_SCRIPT_DOCKER_TAG } else { "azsdkengsys.azurecr.io/engsys/test-proxy:latest" } - - $CommandArgs = @( - "run --rm --name transition.test.proxy", - "-v `"${updatedDirectory}:/srv/testproxy`"", - "-e `"GIT_TOKEN=${token}`"", - "-e `"GIT_COMMIT_OWNER=${committer}`"", - "-e `"GIT_COMMIT_EMAIL=${email}`"", - $targetImage, - "test-proxy", - $CommandArgs - ) -join " " - } - - Write-Host "$TestProxyExe $CommandArgs" - [array] $output = & "$TestProxyExe" $CommandArgs.Split(" ") --storage-location="$updatedDirectory" - # echo the command output - foreach ($line in $output) { - Write-Host "$line" - } -} - -# Get the shorthash directory under PROXY_ASSETS_FOLDER -Function Get-AssetsRoot { - param( - [string] $AssetsJsonFile, - [string] $TestProxyExe - ) - $repoRoot = Get-Repo-Root - $relPath = [IO.Path]::GetRelativePath($repoRoot, $AssetsJsonFile).Replace("`\", "/") - $assetsJsonDirectory = Split-Path $relPath - - [array] $output = & "$TestProxyExe" config locate -a "$relPath" --storage-location="$repoRoot" - $assetsDirectory = $output[-1] - - return Join-Path $assetsDirectory $assetsJsonDirectory -} - -Function Move-AssetsFromLangRepo { - param( - [string] $AssetsRoot - ) - $filter = $LangRecordingDirs[$language] - Write-Host "Language recording directory name=$filter" - Write-Host "Get-ChildItem -Recurse -Filter ""*.json"" | Where-Object { if ($filter.Contains(""*"")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains ""$filter"" }" - $filesToMove = Get-ChildItem -Recurse -Filter "*.json" | Where-Object { if ($filter.Contains("*")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains "$filter" } } - [string] $currentDir = Get-Location - - foreach ($fromFile in $filesToMove) { - $relPath = [IO.Path]::GetRelativePath($currentDir, $fromFile) - - $toFile = Join-Path -Path $AssetsRoot -ChildPath $relPath - # Write-Host "Moving from=$fromFile" - # Write-Host " to=$toFile" - $toPath = Split-Path -Path $toFile - - Write-Host $toFile - if (!(Test-Path $toPath)) { - New-Item -Path $toPath -ItemType Directory -Force | Out-Null - } - Move-Item -LiteralPath $fromFile -Destination $toFile -Force - } -} +. (Join-Path $PSScriptRoot "common-asset-functions.ps1") Test-Exe-In-Path -ExeToLookFor $GitExe $language = Get-Repo-Language @@ -402,7 +135,7 @@ if ($InitialPush) { # Execute a restore on the current assets.json, it'll prep the root directory that # the recordings need to be copied into $CommandArgs = "restore --assets-json-path $assetsJsonRelPath" - Invoke-ProxyCommand -TestProxyExe $TestProxyExe -CommandArgs $CommandArgs -TargetDirectory $repoRoot + Invoke-ProxyCommand -TestProxyExe $TestProxyExe -CommandString $CommandArgs -TargetDirectory $repoRoot $assetsRoot = (Get-AssetsRoot -AssetsJsonFile $assetsJsonFile -TestProxyExe $TestProxyExe) Write-Host "assetsRoot=$assetsRoot" @@ -410,7 +143,7 @@ if ($InitialPush) { Move-AssetsFromLangRepo -AssetsRoot $assetsRoot $CommandArgs = "push --assets-json-path $assetsJsonRelPath" - Invoke-ProxyCommand -TestProxyExe $TestProxyExe -CommandArgs $CommandArgs -TargetDirectory $repoRoot + Invoke-ProxyCommand -TestProxyExe $TestProxyExe -CommandString $CommandArgs -TargetDirectory $repoRoot # Verify that the assets.json file was updated $updatedAssets = Get-Content $assetsJsonFile | Out-String | ConvertFrom-Json diff --git a/eng/common/testproxy/override-proxy-version.ps1 b/eng/common/testproxy/scripts/override-proxy-version.ps1 similarity index 92% rename from eng/common/testproxy/override-proxy-version.ps1 rename to eng/common/testproxy/scripts/override-proxy-version.ps1 index d1e6ff5bf..031b0c17f 100644 --- a/eng/common/testproxy/override-proxy-version.ps1 +++ b/eng/common/testproxy/scripts/override-proxy-version.ps1 @@ -10,7 +10,7 @@ param( [Parameter(mandatory=$true)] [string] $TargetVersion ) -$versionFile = Join-Path $PSScriptRoot "target_version.txt" +$versionFile = Join-Path $PSScriptRoot ".." "target_version.txt" $existingVersionText = Get-Content -Raw -Path $versionFile $existingVersion = $existingVersionText.Trim() diff --git a/eng/common/testproxy/scripts/tag-merge/README.md b/eng/common/testproxy/scripts/tag-merge/README.md new file mode 100644 index 000000000..aa524282e --- /dev/null +++ b/eng/common/testproxy/scripts/tag-merge/README.md @@ -0,0 +1,94 @@ +# Merge Proxy Tags Script + +This script is intended to allow simpler combination of proxy tags. This is necessary due a few facts: + +- Feature teams often need to combine their efforts while adding features. This means parallel re-recording or addition of tests. +- Instead of recordings being directly alongside the feature work, able to be merged simultaneously, now recordings are a single external reference from `assets.json`. + +This script merely allows the abstraction of some of this "combination" work. + +## Usage + +### PreReqs + +- Must have `git` available on your PATH +- Must have the `test-proxy` available on your PATH + - `test-proxy` is honored when the proxy is installed as a `dotnet tool` + - `Azure.Sdk.Tools.TestProxy` is honored when the standalone executable is on your PATH + - Preference for `dotnet tool` if present + +### Call the script + +```powershell +cd "path/to/language/repo/root" +./eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 sdk/storage/azure-storage-blob/assets.json integration/example/storage_feature_addition2 integration/example/storage_feature_addition1 +# ^ Combined Tag 1 ^ Combined Tag 2 +test-proxy push -a sdk/storage/azure-storage-blob/assets.json +``` + +### Resolve Conflicts + +If the script ends early to a `git conflict` occurring, the script leaves the asset repo in a resolvable state. + +- `cd` to the working directory described in the output from the script before it starts working. ("The work will be complete in...") +- `git status` to identify which files are conflicted + +You will see something along these lines: + +```bash +C:/repo/azure-sdk-for-python/.assets/eDscgL1p9G/python |>git status +HEAD detached from python/storage/azure-storage-blob_12c8154ae2 +You are currently cherry-picking commit 1fd0865. + (fix conflicts and run "git cherry-pick --continue") + (use "git cherry-pick --skip" to skip this patch) + (use "git cherry-pick --abort" to cancel the cherry-pick operation) + +You are in a sparse checkout with 100% of tracked files present. + +Unmerged paths: + (use "git add ..." to mark resolution) + both added: sdk/storage/azure-storage-blob/tests/recordings/test_append_blob_async.pyTestStorageAppendBlobAsynctest_append_blob_from_text_new.json + +no changes added to commit (use "git add" and/or "git commit -a") +``` + +Resolve the conflicts in the file, then add it using `git add `. Once the conflict is fully resolved, use + +```bash +C:/repo/azure-sdk-for-python/.assets/eDscgL1p9G/python [???]|>git cherry-pick --continue +[detached HEAD 236e234] add the same file names as what was present in tag integration/example/storage_feature_addition2. In this case, the files themselves are just different enough from integration/example/storage_feature_addition2 that we should intentionally cause a conflict + Date: Fri Dec 1 16:57:52 2023 -0800 + 1 file changed, 2 insertions(+), 2 deletions(-) +``` + +Once you've resolved the conflict, re-run the same script. The results of the cherry-pick resolution will be visible. + +```bash +C:/repo/azure-sdk-for-python [test-storage-tag-combination]|>eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 sdk/storage/azure-storage-blob/assets.json integration/example/storage_feature_addition2 integration/example/storage_feature_addition2_conflict integration/example/storage_feature_addition1 +Excluding tag integration/example/storage_feature_addition2 because we have already done work against it in a previous script invocation. +Excluding tag integration/example/storage_feature_addition2_conflict because we have already done work against it in a previous script invocation. +This script has detected the presence of a .mergeprogress file within folder C:\repo\azure-sdk-for-python. +If the presence of a previous execution of this script is surprising, delete the .assets folder and .mergeprogress file before invoking the script again. +Attempting to continue from a previous run, and excluding: + - integration/example/storage_feature_addition2 + - integration/example/storage_feature_addition2_conflict +But continuing with: + - integration/example/storage_feature_addition1 +If the above looks correct, press enter, otherwise, ctrl-c: +``` + +On successful result, the following will be present: + +``` +Successfully combined 3 tags. Invoke "test-proxy push C:\repo\azure-sdk-for-python\sdk\storage\azure-storage-blob\assets.json" to push the results as a new tag. +``` + +Just follow the instructions to push your combined tag! + +### Push the result + +Once the script has completed successfully, `test-proxy push` the results! + +```bash +test-proxy push sdk/storage/azure-storage-blob/assets.json +``` diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 new file mode 100644 index 000000000..866a5f787 --- /dev/null +++ b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 @@ -0,0 +1,302 @@ +<# +.SYNOPSIS +Merge multiple asset tagss worth of content into a single asset tag. + +.DESCRIPTION +USAGE: merge-proxy-tags.ps1 path/to/target_assets_json. TAG1 TAG2 TAG3 + +Attempts to merge the contents of multiple assets tags into a single new local changeset, which can be `test-proxy push`-ed. + +In the case one of the targeted tags exists in the targeted assets.json, that tag will always be the start point. + +1. test-proxy restore -a -> populate .assets +2. test-proxy config locate -a -> get location of cloned git repo +3. walk the incoming tags, cherry-picking their changes directly into the changeset _in context_ +4. In the case of a discovered git conflict, the process ends. A list of which tags merged and which didn't will be presented to the user. + 4a. Users should resolve the git conflicts themselves. + 4b. If the conflict was on the final tag, resolve the conflict (leaving it uncommitted tyvm), and test-proxy push, you're done. + 4c. If the conflict was _not_ on the final tag, resolve the conflict, commit it, and then re-run this script with the SAME arguments as before. + +This script requires that test-proxy or azure.sdk.tools.testproxy should be on the PATH. + +.PARAMETER AssetsJson +The script uses a target assets.json to resolve where specifically on disk the tag merging should take place. + +.PARAMETER TargetTags +The set of tags whose contents should be combined. Any number of tags > 1 is allowed. + +#> +param( + [Parameter(Position=0)] + [string] $AssetsJson, + [Parameter(Position=1, ValueFromRemainingArguments=$true)] + [string[]] $TargetTags +) + +. (Join-Path $PSScriptRoot ".." ".." "onboarding" "common-asset-functions.ps1") + +function Git-Command-With-Result($CommandString, $WorkingDirectory) { + Write-Host "git $CommandString" + + if ($WorkingDirectory){ + Push-Location $WorkingDirectory + } + + $result = Invoke-Expression "git $CommandString" + + if ($WorkingDirectory) { + Pop-Location + } + + return [PSCustomObject]@{ + ExitCode = $lastexitcode + Output = $result + } +} + +function Git-Command($CommandString, $WorkingDirectory, $HardExit=$true) { + $result = Git-Command-With-Result $CommandString $WorkingDirectory + + if ($result.ExitCode -ne 0 -and $HardExit) { + Write-Error $result.Output + exit 1 + } + + return $result.Output +} + +function Resolve-Proxy { + # this script requires the presence of git + Test-Exe-In-Path -ExeToLookFor "git" | Out-Null + + $testProxyExe = "test-proxy" + # this script requires the presence of the test-proxy on the PATH + $proxyToolPresent = Test-Exe-In-Path -ExeToLookFor "test-proxy" -ExitOnError $false + $proxyStandalonePresent = Test-Exe-In-Path -ExeToLookFor "Azure.Sdk.Tools.TestProxy" -ExitOnError $false + + if (-not $proxyToolPresent -and -not $proxyStandalonePresent) { + Write-Error "This script requires the presence of a test-proxy executable to complete its operations. Exiting." + exit 1 + } + + if (-not $proxyToolPresent) { + $testProxyExe = "Azure.Sdk.Tools.TestProxy" + } + + return $testProxyExe +} + +function Call-Proxy { + param( + [string] $TestProxyExe, + [string] $CommandArgs, + [string] $MountDirectory, + [boolean] $Output = $true + ) + + $CommandArgs += " --storage-location=$MountDirectory" + + if ($Output -eq $true) { + Write-Host "$TestProxyExe $CommandArgs" + } + + [array] $output = & "$TestProxyExe" $CommandArgs.Split(" ") + + if ($lastexitcode -ne 0) { + foreach($line in $output) { + Write-Host $line + } + Write-Error "Proxy exe exited with unexpected non-zero exit code." + exit 1 + } + + if ($Output -eq $true) { + foreach($line in $output) { + Write-Host $line + } + } + + return $output +} + +function Locate-Assets-Slice($ProxyExe, $AssetsJson, $MountDirectory) { + $CommandString = "config locate -a $AssetsJson" + + $output = Call-Proxy -TestProxyExe $ProxyExe -CommandArgs $CommandString -MountDirectory $MountDirectory -Output $false + + return $output[-1].Trim() +} + +function Get-Tag-SHA($TagName, $WorkingDirectory) { + $results = Git-Command "ls-remote origin $TagName" $WorkingDirectory + + if ($results -and $lastexitcode -eq 0) { + $arr = $results -split '\s+' + + return $arr[0] + } + + Write-Error "Unable to fetch tag SHA for $TagName. The tag does not exist on the repository." + exit 1 +} + +function Start-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory) { + $alreadyCombinedTags = Load-Incomplete-Progress $MountDirectory + + $TargetTags = $TargetTags | Where-Object { $_ -notin $alreadyCombinedTags } + + if ($alreadyCombinedTags) { + Write-Host "This script has detected the presence of a .mergeprogress file within folder $MountDirectory." + Write-Host "If the presence of a previous execution of this script is surprising, delete the .assets folder and .mergeprogress file before invoking the script again." + Write-Host "Attempting to continue from a previous run, and excluding:" + + foreach($Tag in $alreadyCombinedTags) { + Write-Host " - " -NoNewLine + Write-Host "$Tag" -ForegroundColor Green + } + Write-Host "But continuing with:" + + foreach($Tag in $TargetTags){ + Write-Host " - " -NoNewLine + Write-Host "$Tag" -ForegroundColor Green + } + } + else { + Write-Host "`nThis script will attempt to merge the following tag" -NoNewLine + if ($TargetTags.Length -gt 1) { + Write-Host "s" -NoNewLine + } + Write-Host ":" + foreach($Tag in $TargetTags) { + Write-Host " - " -NoNewLine + Write-Host "$Tag" -ForegroundColor Green + } + Write-Host "`nTargeting the assets slice targeted by " -NoNewLine + Write-Host "$AssetsJson." -ForegroundColor Green + Write-Host "`nThe work will be completed in " -NoNewLine + Write-Host $AssetsRepoLocation -ForegroundColor Green -NoNewLine + Write-Host "." + } + + Read-Host -Prompt "If the above looks correct, press enter, otherwise, ctrl-c" +} + +function Finish-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory) { + $len = $TargetTags.Length + + Write-Host "`nSuccessfully combined $len tags. Invoke `"test-proxy push " -NoNewLine + Write-Host $AssetsJson -ForegroundColor Green -NoNewLine + Write-Host "`" to push the results as a new tag." +} + +function Resolve-Target-Tags($AssetsJson, $TargetTags, $MountDirectory) { + $inprogress = Load-Incomplete-Progress $MountDirectory + + $jsonContent = Get-Content -Raw -Path $AssetsJson + $jsonObj = $JsonContent | ConvertFrom-Json + + $existingTarget = $jsonObj.Tag + + return $TargetTags | Where-Object { + if ($_ -eq $existingTarget) { + Write-Host "Excluding tag $($_) from tag input list, it is present in assets.json." + } + $_ -ne $existingTarget + } | Where-Object { + if ($_ -in $inprogress) { + Write-Host "Excluding tag $($_) because we have already done work against it in a previous script invocation." + } + $_ -notin $inprogress + } +} + +function Save-Incomplete-Progress($Tag, $MountDirectory) { + $progressFile = (Join-Path $MountDirectory ".mergeprogress") + [array] $existingTags = @() + if (Test-Path $progressFile) { + $existingTags = (Get-Content -Path $progressFile) -split "`n" | ForEach-Object { $_.Trim() } + } + + $existingTags = $existingTags + $Tag | Select-Object -Unique + + Set-Content -Path $progressFile -Value ($existingTags -join "`n") | Out-Null + + return $existingTags +} + +function Load-Incomplete-Progress($MountDirectory) { + $progressFile = (Join-Path $MountDirectory ".mergeprogress") + [array] $existingTags = @() + if (Test-Path $progressFile) { + $existingTags = ((Get-Content -Path $progressFile) -split "`n" | ForEach-Object { $_.Trim() }) + } + + return $existingTags +} + +function Cleanup-Incomplete-Progress($MountDirectory) { + $progressFile = (Join-Path $MountDirectory ".mergeprogress") + + if (Test-Path $progressFile) { + Remove-Item $progressFile | Out-Null + } +} + +function Prepare-Assets($ProxyExe, $MountDirectory, $AssetsJson) { + $inprogress = Load-Incomplete-Progress $MountDirectory + + if ($inprogress.Length -eq 0) { + Call-Proxy -TestProxyExe $ProxyExe -CommandArgs "reset -y -a $AssetsJson" -MountDirectory $MountDirectory -Output $false + } +} + +function Combine-Tags($RemainingTags, $AssetsRepoLocation, $MountDirectory){ + foreach($Tag in $RemainingTags) { + $tagSha = Get-Tag-SHA $Tag $AssetsRepoLocation + $existingTags = Save-Incomplete-Progress $Tag $MountDirectory + $cherryPickResult = Git-Command-With-Result "cherry-pick $tagSha" - $AssetsRepoLocation -HardExit $false + + if ($cherryPickResult.ExitCode -ne 0) { + Write-Host "Conflicts while cherry-picking $Tag. Resolve the the conflict over in `"$AssetsRepoLocation`", and re-run this script with the same arguments as before." -ForegroundColor Red + exit 1 + } + } + + $pushedTags = Load-Incomplete-Progress $MountDirectory + + $testFile = Get-ChildItem -Recurse -Path $AssetsRepoLocation | Where-Object { !$_.PSIsContainer } | Select-Object -First 1 + Add-Content -Path $testFile -Value "`n" + + # if we have successfully gotten to the end without any non-zero exit codes...delete the mergeprogress file, we're g2g + Cleanup-Incomplete-Progress $MountDirectory + + return $pushedTags +} + +$ErrorActionPreference = "Stop" + +# resolve the proxy location so that we can invoke it easily +$proxyExe = Resolve-Proxy + +$AssetsJson = Resolve-Path $AssetsJson + +# figure out where the root of the repo for the passed assets.json is. We need it to properly set the mounting +# directory so that the test-proxy restore operations work IN PLACE with existing tooling +$mountDirectory = Get-Repo-Root -StartDir $AssetsJson + +# ensure we actually have the .assets folder that we can cherry-pick on top of +Prepare-Assets $proxyExe $mountDirectory $AssetsJson + +# using the mountingDirectory and the assets.json location, we can figure out where the assets slice actually lives within the .assets folder. +# we will use this to invoke individual cherry-picks before pushing up the result +$assetsRepoLocation = Locate-Assets-Slice $proxyExe $AssetsJson $mountDirectory + +# resolve the tags that we will go after. If the target assets.json contains one of these tags, that tag is _already present_ +# because the entire point of this script is to run in context of a set of recordings in the repo +$tags = Resolve-Target-Tags $AssetsJson $TargetTags $mountDirectory + +Start-Message $AssetsJson $Tags $AssetsRepoLocation $mountDirectory + +$CombinedTags = Combine-Tags $Tags $AssetsRepoLocation $mountDirectory + +Finish-Message $AssetsJson $CombinedTags $AssetsRepoLocation $mountDirectory diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index 6e749801f..307b5032a 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -15,7 +15,7 @@ steps: condition: and(succeeded(), ${{ parameters.condition }}, ne('${{ parameters.targetVersion }}', '')) inputs: targetType: filePath - filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/override-proxy-version.ps1' + filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/scripts/override-proxy-version.ps1' arguments: '-TargetVersion "${{ parameters.targetVersion }}"' pwsh: true diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 22e4a7da4..7aea55d47 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -16,7 +16,7 @@ steps: condition: and(succeeded(), ${{ parameters.condition }}, ne('${{ parameters.targetVersion }}', '')) inputs: targetType: filePath - filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/override-proxy-version.ps1' + filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/scripts/override-proxy-version.ps1' arguments: '-TargetVersion "${{ parameters.targetVersion }}"' pwsh: true