Sync eng/common directory with azure-sdk-tools for PR 6919 (#4988)

* Add autorest-preview pipeline

* Add emitternpminstall

* Always install

* Use shorter leg name

* Add short circuiting to emitter install

* Use language matrix function

* Don't look for subfolders in the matrix package folders

* Revert unnecessary eng/common changes

* Rewrite GetPullRequestUrl to Get-BuildSourceDescription

* Remove alias from Invoke-LoggedCommand

* Use invoke-expression instead of shelling out

* Add better job splitting

* Replace Folder with Directory

---------

Co-authored-by: Patrick Hallisey <pahallis@microsoft.com>

eng/common/pipelines/templates/steps/sparse-checkout.yml

@@ -29,7 +29,7 @@ steps:
             if (!$dir) {
               $dir = "./$($repository.Name)"
             }
-            New-Item $dir -ItemType Directory -Force
+            New-Item $dir -ItemType Directory -Force | Out-Null
             Push-Location $dir
 
             if (Test-Path .git/info/sparse-checkout) {
@@ -70,9 +70,14 @@ steps:
 
             # sparse-checkout commands after initial checkout will auto-checkout again
             if (!$hasInitialized) {
-              Write-Host "git -c advice.detachedHead=false checkout $($repository.Commitish)"
+              # Remove refs/heads/ prefix from branch names
+              $commitish = $repository.Commitish -replace '^refs/heads/', ''
+
+              # use -- to prevent git from interpreting the commitish as a path
+              Write-Host "git -c advice.detachedHead=false checkout $commitish --"
+
               # This will use the default branch if repo.Commitish is empty
-              git -c advice.detachedHead=false checkout $($repository.Commitish)
+              git -c advice.detachedHead=false checkout $commitish --
             } else {
               Write-Host "Skipping checkout as repo has already been initialized"
             }

eng/common/scripts/Get-BuildSourceDescription.ps1

@@ -0,0 +1,24 @@
+param(
+  [string]$Variable,
+  [switch]$IsOutput
+)
+
+$repoUrl = $env:BUILD_REPOSITORY_URI
+$sourceBranch = $env:BUILD_SOURCEBRANCH
+
+$description = "[$sourceBranch]($repoUrl/tree/$sourceBranch)" 
+if ($sourceBranch -match "^refs/heads/(.+)$") {
+    $description = "Branch: [$($Matches[1])]($repoUrl/tree/$sourceBranch)"
+} elseif ($sourceBranch -match "^refs/tags/(.+)$") {
+    $description = "Tag: [$($Matches[1])]($repoUrl/tree/$sourceBranch)"
+} elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") {
+    $description = "Pull request: $repoUrl/pull/$($Matches[1])"
+}
+
+if ($IsOutput) {
+  Write-Host "Setting output variable '$Variable' to '$description'"
+  Write-Host "##vso[task.setvariable variable=$Variable;isoutput=true]$description"
+} else {
+  Write-Host "Setting variable '$Variable' to '$description'"
+  Write-Host "##vso[task.setvariable variable=$Variable]$description"
+}

eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1

@@ -0,0 +1,42 @@
+function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput)
+{
+    $pipelineBuild = !!$env:TF_BUILD
+    $startTime = Get-Date
+
+    if($pipelineBuild -and $GroupOutput) {
+        Write-Host "##[group]$Command"
+    } else {
+        Write-Host "> $Command"
+    }
+
+    if($ExecutePath) {
+      Push-Location $ExecutePath
+    }
+
+    try {
+      Invoke-Expression $Command
+
+      $duration = (Get-Date) - $startTime
+
+      if($pipelineBuild -and $GroupOutput) {
+        Write-Host "##[endgroup]"
+      }
+
+      if($LastExitCode -ne 0)
+      {
+          if($pipelineBuild) {
+              Write-Error "##[error]Command failed to execute ($duration): $Command`n"
+          } else {
+              Write-Error "Command failed to execute ($duration): $Command`n"
+          }
+      }
+      else {
+          Write-Host "Command succeeded ($duration)`n"
+      }
+    }
+    finally {
+      if($ExecutePath) {
+        Pop-Location
+      }
+    }
+}

eng/common/scripts/New-RegenerateMatrix.ps1

@@ -0,0 +1,102 @@
+[CmdLetBinding()]
+param (
+  [Parameter()]
+  [string]$OutputDirectory,
+
+  [Parameter()]
+  [string]$OutputVariableName,
+
+  [Parameter()]
+  [int]$JobCount = 8,
+
+  # The minimum number of items per job. If the number of items is less than this, then the number of jobs will be reduced.
+  [Parameter()]
+  [int]$MinimumPerJob = 10,
+
+  [Parameter()]
+  [string]$OnlyTypespec
+)
+
+. (Join-Path $PSScriptRoot common.ps1)
+
+[bool]$OnlyTypespec = $OnlyTypespec -in @("true", "t", "1", "yes", "y")
+
+# Divide the items into groups of approximately equal size.
+function Split-Items([array]$Items) {
+  # given $Items.Length = 22 and $JobCount = 5
+  # then $itemsPerGroup = 4
+  # and $largeJobCount = 2
+  # and $group.Length = 5, 5, 4, 4, 4
+  $itemCount = $Items.Length
+  $jobsForMinimum = $itemCount -lt $MinimumPerJob ? 1 : [math]::Floor($itemCount / $MinimumPerJob)
+
+  if ($JobCount -gt $jobsForMinimum) {
+    $JobCount = $jobsForMinimum
+  }
+  
+  $itemsPerGroup = [math]::Floor($itemCount / $JobCount)
+  $largeJobCount = $itemCount % $itemsPerGroup
+  $groups = [object[]]::new($JobCount)
+
+  $i = 0
+  for ($g = 0; $g -lt $JobCount; $g++) {
+    $groupLength = if ($g -lt $largeJobCount) { $itemsPerGroup + 1 } else { $itemsPerGroup }
+    $group = [object[]]::new($groupLength)
+    $groups[$g] = $group
+    for ($gi = 0; $gi -lt $groupLength; $gi++) {
+      $group[$gi] = $Items[$i++]
+    }
+  }
+
+  Write-Host "$itemCount items split into $JobCount groups of approximately $itemsPerGroup items each."
+
+  return , $groups
+}
+
+# ensure the output directory exists
+New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null
+
+if (Test-Path "Function:$GetDirectoriesForGenerationFn") {
+  $directoriesForGeneration = &$GetDirectoriesForGenerationFn
+}
+else {
+  $directoriesForGeneration = Get-ChildItem "$RepoRoot/sdk" -Directory | Get-ChildItem -Directory
+}
+
+if ($OnlyTypespec) {
+  $directoriesForGeneration = $directoriesForGeneration | Where-Object { Test-Path "$_/tsp-location.yaml" }
+}
+
+[array]$packageDirectories = $directoriesForGeneration
+| Sort-Object -Property FullName
+| ForEach-Object {
+  [ordered]@{
+    "PackageDirectory" = "$($_.Parent.Name)/$($_.Name)"
+    "ServiceArea"   = $_.Parent.Name
+  }
+}
+
+$batches = Split-Items -Items $packageDirectories
+
+$matrix = [ordered]@{}
+for ($i = 0; $i -lt $batches.Length; $i++) {
+  $batch = $batches[$i]
+  $json = $batch.PackageDirectory | ConvertTo-Json -AsArray
+
+  $firstPrefix = $batch[0].ServiceArea.Substring(0, 2)
+  $lastPrefix = $batch[-1].ServiceArea.Substring(0, 2)
+  
+  $key = "$firstPrefix`_$lastPrefix`_$i"
+  $fileName = "$key.json"
+  
+  Write-Host "`n`n=================================="
+  Write-Host $fileName
+  Write-Host "=================================="
+  $json | Out-Host
+  $json | Out-File "$OutputDirectory/$fileName"
+
+  $matrix[$key] = [ordered]@{ "JobKey" = $key; "DirectoryList" = $fileName }
+}
+
+$compressed = ConvertTo-Json $matrix -Depth 100 -Compress
+Write-Output "##vso[task.setVariable variable=$OutputVariableName;isOutput=true]$compressed"

eng/common/scripts/TypeSpec-Project-Generate.ps1

@@ -11,6 +11,7 @@ param (
 
 $ErrorActionPreference = "Stop"
 . $PSScriptRoot/Helpers/PSModule-Helpers.ps1
+. $PSScriptRoot/Helpers/CommandInvocation-Helpers.ps1
 . $PSScriptRoot/common.ps1
 Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
 
@@ -21,38 +22,30 @@ function NpmInstallForProject([string]$workingDirectory) {
         Write-Host "Generating from $currentDur"
 
         if (Test-Path "package.json") {
+            Write-Host "Removing existing package.json"
             Remove-Item -Path "package.json" -Force
         }
 
         if (Test-Path ".npmrc") {
+            Write-Host "Removing existing .nprc"
             Remove-Item -Path ".npmrc" -Force
         }
 
         if (Test-Path "node_modules") {
+            Write-Host "Removing existing node_modules"
             Remove-Item -Path "node_modules" -Force -Recurse
         }
 
         if (Test-Path "package-lock.json") {
+            Write-Host "Removing existing package-lock.json"
             Remove-Item -Path "package-lock.json" -Force
         }
 
-        #default to root/eng/emitter-package.json but you can override by writing
-        #Get-${Language}-EmitterPackageJsonPath in your Language-Settings.ps1
         $replacementPackageJson = Join-Path $PSScriptRoot "../../emitter-package.json"
-        if (Test-Path "Function:$GetEmitterPackageJsonPathFn") {
-            $replacementPackageJson = &$GetEmitterPackageJsonPathFn
-        }
 
         Write-Host("Copying package.json from $replacementPackageJson")
         Copy-Item -Path $replacementPackageJson -Destination "package.json" -Force
-
-        #default to root/eng/emitter-package-lock.json but you can override by writing
-        #Get-${Language}-EmitterPackageLockPath in your Language-Settings.ps1
         $emitterPackageLock = Join-Path $PSScriptRoot "../../emitter-package-lock.json"
-        if (Test-Path "Function:$GetEmitterPackageLockPathFn") {
-            $emitterPackageLock = &$GetEmitterPackageLockPathFn
-        }
-
         $usingLockFile = Test-Path $emitterPackageLock
 
         if ($usingLockFile) {
@@ -68,12 +61,10 @@ function NpmInstallForProject([string]$workingDirectory) {
         }
 
         if ($usingLockFile) {
-            Write-Host "> npm ci"
-            npm ci
+            Invoke-LoggedCommand "npm ci"
         }
         else {
-            Write-Host "> npm install"
-            npm install
+            Invoke-LoggedCommand "npm install"
         }
 
         if ($LASTEXITCODE) { exit $LASTEXITCODE }

eng/common/scripts/Update-GeneratedSdks.ps1

@@ -0,0 +1,16 @@
+[CmdLetBinding()]
+param(
+  [Parameter(Mandatory)]
+  [string]$PackageDirectoriesFile
+)
+
+. $PSScriptRoot/common.ps1
+. $PSScriptRoot/Helpers/CommandInvocation-Helpers.ps1
+
+$ErrorActionPreference = 'Stop'
+
+if (Test-Path "Function:$UpdateGeneratedSdksFn") {
+    &$UpdateGeneratedSdksFn $PackageDirectoriesFile
+} else {
+    Write-Error "Function $UpdateGeneratedSdksFn not implemented in Language-Settings.ps1"
+}

eng/common/scripts/common.ps1

@@ -60,8 +60,8 @@ $GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme"
 $GetRepositoryLinkFn = "Get-${Language}-RepositoryLink"
 $GetEmitterAdditionalOptionsFn = "Get-${Language}-EmitterAdditionalOptions"
 $GetEmitterNameFn = "Get-${Language}-EmitterName"
-$GetEmitterPackageJsonPathFn = "Get-${Language}-EmitterPackageJsonPath"
-$GetEmitterPackageLockPathFn = "Get-${Language}-EmitterPackageLockPath"
+$GetDirectoriesForGenerationFn = "Get-${Language}-DirectoriesForGeneration"
+$UpdateGeneratedSdksFn = "Update-${Language}-GeneratedSdks"
 
 # Expected to be set in eng/scripts/docs/Docs-Onboarding.ps1
 $SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding"