Sync eng/common directory with azure-sdk-tools for PR 6272 (#4698)

* Support localspecrepo if pass in this parameter * Support regen sdk code based on local typespecs * Added reference doc to error message --------- Co-authored-by: raychen <raychen@microsoft.com>
2023-06-05 18:12:43 -07:00 · 2023-06-05 18:12:43 -07:00 · 2a4d1835b8
commit 2a4d1835b8
parent 184303a893
3 changed files with 161 additions and 117 deletions
--- a/eng/common/scripts/TypeSpec-Project-Generate.ps1
+++ b/eng/common/scripts/TypeSpec-Project-Generate.ps1
@ -38,7 +38,7 @@ function NpmInstallForProject([string]$workingDirectory) {

        #default to root/eng/emitter-package.json but you can override by writing
        #Get-${Language}-EmitterPackageJsonPath in your Language-Settings.ps1
-        $replacementPackageJson = "$PSScriptRoot/../../emitter-package.json"
+        $replacementPackageJson = Join-Path $PSScriptRoot "../../emitter-package.json"
        if (Test-Path "Function:$GetEmitterPackageJsonPathFn") {
            $replacementPackageJson = &$GetEmitterPackageJsonPathFn
        }
@ -52,7 +52,7 @@ function NpmInstallForProject([string]$workingDirectory) {
            Write-Host "Package.json contains '-alpha.' in the version, Creating .npmrc using public/azure-sdk-for-js-test-autorest feed."
            "registry=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ `n`nalways-auth=true" | Out-File '.npmrc'
        }
-        
+
        npm install --no-lock-file
        if ($LASTEXITCODE) { exit $LASTEXITCODE }
    }
@ -112,3 +112,4 @@ $shouldCleanUp = !$SaveInputs
 if ($shouldCleanUp) {
    Remove-Item $tempFolder -Recurse -Force
 }
+exit 0
--- a/eng/common/scripts/TypeSpec-Project-Process.ps1
+++ b/eng/common/scripts/TypeSpec-Project-Process.ps1
@ -16,31 +16,20 @@ param (
 Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module

 function CreateUpdate-TspLocation([System.Object]$tspConfig, [string]$TypeSpecProjectDirectory, [string]$CommitHash, [string]$repo, [string]$repoRoot) {
-  $serviceDir = ""
  $additionalDirs = @()
-
-  # Parse tspcofig.yaml to get service-dir, additionalDirectories, and package-dir
-  if ($tspConfig["parameters"] -and $tspConfig["parameters"]["service-dir"]) {
-    $serviceDir = $tspConfig["parameters"]["service-dir"]["default"];
-  }
-  else {
-    Write-Error "Missing service-dir in parameters section of tspconfig.yaml."
-    exit 1
-  }
  if ($tspConfig["parameters"]["dependencies"] -and $tspConfig["parameters"]["dependencies"]["additionalDirectories"]) {
    $additionalDirs = $tspConfig["parameters"]["dependencies"]["additionalDirectories"];
  }

-  $packageDir = Get-PackageDir $tspConfig
-
  # Create service-dir if not exist
-  $serviceDir = Join-Path $repoRoot $serviceDir
+  $serviceDir = Get-ServiceDir $tspConfig $repoRoot
  if (!(Test-Path -Path $serviceDir)) {
    New-Item -Path $serviceDir -ItemType Directory | Out-Null
    Write-Host "created service folder $serviceDir"
  }

  # Create package-dir if not exist
+  $packageDir = Get-PackageDir $tspConfig
  $packageDir = Join-Path $serviceDir $packageDir
  if (!(Test-Path -Path $packageDir)) {
    New-Item -Path $packageDir -ItemType Directory | Out-Null
@ -71,6 +60,20 @@ function CreateUpdate-TspLocation([System.Object]$tspConfig, [string]$TypeSpecPr
  return $packageDir
 }

+function Get-ServiceDir([System.Object]$tspConfig, [string]$repoRoot) {
+  $serviceDir = ""
+  if ($tspConfig["parameters"] -and $tspConfig["parameters"]["service-dir"]) {
+    $serviceDir = $tspConfig["parameters"]["service-dir"]["default"];
+  }
+  else {
+    Write-Error "Missing service-dir in parameters section of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema."
+    exit 1
+  }
+
+  # Create service-dir if not exist
+  $serviceDir = Join-Path $repoRoot $serviceDir
+  return $serviceDir
+}
 function Get-PackageDir([System.Object]$tspConfig) {
  $emitterName = ""
  if (Test-Path "Function:$GetEmitterNameFn") {
@ -85,18 +88,27 @@ function Get-PackageDir([System.Object]$tspConfig) {
    $packageDir = $tspConfig["options"]["$emitterName"]["package-dir"]
  }
  else {
-    Write-Error "Missing package-dir in $emitterName options of tspconfig.yaml."
+    Write-Error "Missing package-dir in $emitterName options of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema."
    exit 1
  }
  return $packageDir
 }

-$repoRootPath =  (Join-Path $PSScriptRoot .. .. ..)
-$repoRootPath = Resolve-Path $repoRootPath
-$repoRootPath = $repoRootPath -replace "\\", "/"
-$tspConfigPath = Join-Path $repoRootPath 'tspconfig.yaml'
+function Get-TspLocationFolder([System.Object]$tspConfig, [string]$repoRoot) {
+  $serviceDir = Get-ServiceDir $tspConfig $repoRoot
+  $packageDir = Get-PackageDir $tspConfig
+  $packageDir = Join-Path $serviceDir $packageDir
+  return $packageDir
+}
+
+$sdkRepoRootPath =  (Join-Path $PSScriptRoot .. .. ..)
+$sdkRepoRootPath = Resolve-Path $sdkRepoRootPath
+$sdkRepoRootPath = $sdkRepoRootPath -replace "\\", "/"
+$tspConfigPath = Join-Path $sdkRepoRootPath 'tspconfig.yaml'
 $tmpTspConfigPath = $tspConfigPath
 $repo = ""
+$specRepoRoot = ""
+$generateFromLocalTypeSpec = $false
 # remote url scenario
 # example url of tspconfig.yaml: https://github.com/Azure/azure-rest-api-specs-pr/blob/724ccc4d7ef7655c0b4d5c5ac4a5513f19bbef35/specification/containerservice/Fleet.Management/tspconfig.yaml
 if ($TypeSpecProjectDirectory -match '^https://github.com/(?<repo>Azure/azure-rest-api-specs(-pr)?)/blob/(?<commit>[0-9a-f]{40})/(?<path>.*)/tspconfig.yaml$') {
@ -121,26 +133,26 @@ if ($TypeSpecProjectDirectory -match '^https://github.com/(?<repo>Azure/azure-re
    exit 1
  }
  $TypeSpecProjectDirectory = $TypeSpecProjectDirectory.Replace("\", "/")
-  if ($TypeSpecProjectDirectory -match "^.*/(?<path>specification/.*)$") {
+  if ($TypeSpecProjectDirectory -match "(?<repoRoot>^.*)/(?<path>specification/.*)$") {
    $TypeSpecProjectDirectory = $Matches["path"]
+    $specRepoRoot = $Matches["repoRoot"]
  } else {
    Write-Error "$TypeSpecProjectDirectory doesn't have 'specification' in path."
    exit 1
  }
-  if (!$CommitHash) {
-    Write-Error "Parameter of Commithash is not provided in the local path scenario."
-    exit 1
+  if (!$CommitHash -or !$RepoUrl) {
+    Write-Warning "Parameter of Commithash or RepoUrl are not provided along with the local path of tspconfig.yaml, trying to re-generate sdk code based on the local type specs."
+    $generateFromLocalTypeSpec = $true
  }
-  if (!$RepoUrl) {
-    Write-Error "Parameter of RepoUrl:$RepoUrl is not provided in the local path scenario."
-    exit 1
-  }
-  if ($RepoUrl -match "^https://github.com/(?<repo>[^/]*/azure-rest-api-specs(-pr)?).*") {
-    $repo = $Matches["repo"]
-  }
-  else {
-    Write-Error "Parameter 'RepoUrl' has incorrect value:$RepoUrl. It should be similar like 'https://github.com/Azure/azure-rest-api-specs'"
-    exit 1
+  
+  if ($RepoUrl) {
+    if ($RepoUrl -match "^https://github.com/(?<repo>[^/]*/azure-rest-api-specs(-pr)?).*") {
+      $repo = $Matches["repo"]
+    }
+    else {
+      Write-Error "Parameter 'RepoUrl' has incorrect value:$RepoUrl. It should be similar like 'https://github.com/Azure/azure-rest-api-specs'"
+      exit 1
+    }
  }
 }

@ -150,12 +162,23 @@ $tspConfigYaml = Get-Content $tspConfigPath -Raw | ConvertFrom-Yaml
 if (Test-Path $tmpTspConfigPath) {
  Remove-Item $tspConfigPath
 }
-# call CreateUpdate-TspLocation function
-$sdkProjectFolder = CreateUpdate-TspLocation $tspConfigYaml $TypeSpecProjectDirectory $CommitHash $repo $repoRootPath
+
+$sdkProjectFolder = ""
+if ($generateFromLocalTypeSpec) {
+  $sdkProjectFolder = Get-TspLocationFolder $tspConfigYaml $sdkRepoRootPath
+  $tspLocationYamlPath = Join-Path $sdkProjectFolder "tsp-location.yaml"
+  if (!(Test-Path -Path $tspLocationYamlPath)) {
+    Write-Error "Failed to find tsp-location.yaml in '$sdkProjectFolder', please make sure to provide CommitHash and RepoUrl parameters along with the local path of tspconfig.yaml in order to create tsp-location.yaml."
+    exit 1
+  }
+} else {
+  # call CreateUpdate-TspLocation function
+  $sdkProjectFolder = CreateUpdate-TspLocation $tspConfigYaml $TypeSpecProjectDirectory $CommitHash $repo $sdkRepoRootPath  
+}

 # call TypeSpec-Project-Sync.ps1
 $syncScript = Join-Path $PSScriptRoot TypeSpec-Project-Sync.ps1
-& $syncScript $sdkProjectFolder
+& $syncScript $sdkProjectFolder $specRepoRoot
 if ($LASTEXITCODE) { exit $LASTEXITCODE }

 # call TypeSpec-Project-Generate.ps1
--- a/eng/common/scripts/TypeSpec-Project-Sync.ps1
+++ b/eng/common/scripts/TypeSpec-Project-Sync.ps1
@ -2,9 +2,11 @@

 [CmdletBinding()]
 param (
-    [Parameter(Position=0)]
-    [ValidateNotNullOrEmpty()]
-    [string] $ProjectDirectory
+  [Parameter(Position = 0)]
+  [ValidateNotNullOrEmpty()]
+  [string] $ProjectDirectory,
+  [Parameter(Position = 1)]
+  [string] $LocalSpecRepoPath
 )

 $ErrorActionPreference = "Stop"
@ -13,115 +15,133 @@ Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
 $sparseCheckoutFile = ".git/info/sparse-checkout"

 function AddSparseCheckoutPath([string]$subDirectory) {
-    if (!(Test-Path $sparseCheckoutFile) -or !((Get-Content $sparseCheckoutFile).Contains($subDirectory))) {
-        Write-Output $subDirectory >> .git/info/sparse-checkout
-    }
+  if (!(Test-Path $sparseCheckoutFile) -or !((Get-Content $sparseCheckoutFile).Contains($subDirectory))) {
+    Write-Output $subDirectory >> .git/info/sparse-checkout
+  }
 }

 function CopySpecToProjectIfNeeded([string]$specCloneRoot, [string]$mainSpecDir, [string]$dest, [string[]]$specAdditionalSubDirectories) {
-    $source = "$specCloneRoot/$mainSpecDir"
-    Copy-Item -Path $source -Destination $dest -Recurse -Force
-    Write-Host "Copying spec from $source to $dest"
+  $source = Join-Path $specCloneRoot $mainSpecDir
+  Copy-Item -Path $source -Destination $dest -Recurse -Force
+  Write-Host "Copying spec from $source to $dest"

-    foreach ($additionalDir in $specAdditionalSubDirectories) {
-        $source = "$specCloneRoot/$additionalDir"
-        Write-Host "Copying spec from $source to $dest"
-        Copy-Item -Path $source -Destination $dest -Recurse -Force
-    }
+  foreach ($additionalDir in $specAdditionalSubDirectories) {
+    $source = Join-Path $specCloneRoot $additionalDir
+    Write-Host "Copying spec from $source to $dest"
+    Copy-Item -Path $source -Destination $dest -Recurse -Force
+  }
 }

 function UpdateSparseCheckoutFile([string]$mainSpecDir, [string[]]$specAdditionalSubDirectories) {
-    AddSparseCheckoutPath $mainSpecDir
-    foreach ($subDir in $specAdditionalSubDirectories) {
-        AddSparseCheckoutPath $subDir
-    }
+  AddSparseCheckoutPath $mainSpecDir
+  foreach ($subDir in $specAdditionalSubDirectories) {
+    Write-Host "Adding $subDir to sparse checkout"
+    AddSparseCheckoutPath $subDir
+  }
 }

 function GetGitRemoteValue([string]$repo) {
-    Push-Location $ProjectDirectory
-    $result = ""
-    try {
-        $gitRemotes = (git remote -v)
-        foreach ($remote in $gitRemotes) {
-            if ($remote.StartsWith("origin")) {
-                if ($remote -match 'https://github.com/\S+') {
-                    $result = "https://github.com/$repo.git"
-                    break
-                } elseif ($remote -match "git@github.com:\S+"){
-                    $result = "git@github.com:$repo.git"
-                    break
-                } else {
-                    throw "Unknown git remote format found: $remote"
-                }
-            }
+  Push-Location $ProjectDirectory
+  $result = ""
+  try {
+    $gitRemotes = (git remote -v)
+    foreach ($remote in $gitRemotes) {
+      Write-Host "Checking remote $remote"
+      if ($remote.StartsWith("origin") -or $remote.StartsWith("main")) {
+        if ($remote -match 'https://(.*)?github.com/\S+') {
+          $result = "https://github.com/$repo.git"
+          break
        }
+        elseif ($remote -match "(.*)?git@github.com:\S+") {
+          $result = "git@github.com:$repo.git"
+          break
+        }
+        else {
+          throw "Unknown git remote format found: $remote"
+        }
+      }
    }
-    finally {
-        Pop-Location
-    }
-
-    return $result
+  }
+  finally {
+    Pop-Location
+  }
+  Write-Host "Found git remote $result"
+  return $result
 }

 function InitializeSparseGitClone([string]$repo) {
-    git clone --no-checkout --filter=tree:0 $repo .
-    if ($LASTEXITCODE) { exit $LASTEXITCODE }
-    git sparse-checkout init
-    if ($LASTEXITCODE) { exit $LASTEXITCODE }
-    Remove-Item $sparseCheckoutFile -Force
+  git clone --no-checkout --filter=tree:0 $repo .
+  if ($LASTEXITCODE) { exit $LASTEXITCODE }
+  git sparse-checkout init
+  if ($LASTEXITCODE) { exit $LASTEXITCODE }
+  Remove-Item $sparseCheckoutFile -Force
 }

 function GetSpecCloneDir([string]$projectName) {
-    Push-Location $ProjectDirectory
-    try {
-        $root = git rev-parse --show-toplevel
-    }
-    finally {
-        Pop-Location
-    }
+  Push-Location $ProjectDirectory
+  try {
+    $root = git rev-parse --show-toplevel
+  }
+  finally {
+    Pop-Location
+  }

-    $sparseSpecCloneDir = "$root/../sparse-spec/$projectName"
-    New-Item $sparseSpecCloneDir -Type Directory -Force | Out-Null
-    $createResult = Resolve-Path $sparseSpecCloneDir
-    return $createResult
+  $sparseSpecCloneDir = "$root/../sparse-spec/$projectName"
+  New-Item $sparseSpecCloneDir -Type Directory -Force | Out-Null
+  $createResult = Resolve-Path $sparseSpecCloneDir
+  return $createResult
 }

 $typespecConfigurationFile = Resolve-Path "$ProjectDirectory/tsp-location.yaml"
 Write-Host "Reading configuration from $typespecConfigurationFile"
 $configuration = Get-Content -Path $typespecConfigurationFile -Raw | ConvertFrom-Yaml

-$pieces = $typespecConfigurationFile.Path.Replace("\","/").Split("/")
+$pieces = $typespecConfigurationFile.Path.Replace("\", "/").Split("/")
 $projectName = $pieces[$pieces.Count - 2]

 $specSubDirectory = $configuration["directory"]

-if ( $configuration["repo"] -and $configuration["commit"]) {
-    $specCloneDir = GetSpecCloneDir $projectName
-    $gitRemoteValue = GetGitRemoteValue $configuration["repo"]
-
-    Write-Host "Setting up sparse clone for $projectName at $specCloneDir"
-
-    Push-Location $specCloneDir.Path
-    try {
-        if (!(Test-Path ".git")) {
-            InitializeSparseGitClone $gitRemoteValue
-        }
-        UpdateSparseCheckoutFile $specSubDirectory $configuration["additionalDirectories"]
-        git checkout $configuration["commit"]
-        if ($LASTEXITCODE) { exit $LASTEXITCODE }
-    }
-    finally {
-        Pop-Location
-    }
-} elseif ( $configuration["spec-root-dir"] ) {
-    $specCloneDir = $configuration["spec-root-dir"]
+# use local spec repo if provided
+if ($LocalSpecRepoPath) {
+  $specCloneDir = $LocalSpecRepoPath
 }
+elseif ($configuration["repo"] -and $configuration["commit"]) {
+  # use sparse clone if repo and commit are provided
+  $specCloneDir = GetSpecCloneDir $projectName
+  $gitRemoteValue = GetGitRemoteValue $configuration["repo"]

+  Write-Host "from tsplocation.yaml 'repo' is:"$configuration["repo"]
+  Write-Host "Setting up sparse clone for $projectName at $specCloneDir"
+
+  Push-Location $specCloneDir.Path
+  try {
+    if (!(Test-Path ".git")) {
+      Write-Host "Initializing sparse clone for repo: $gitRemoteValue"
+      InitializeSparseGitClone $gitRemoteValue
+    }
+    Write-Host "Updating sparse checkout file with directory:$specSubDirectory"
+    UpdateSparseCheckoutFile $specSubDirectory $configuration["additionalDirectories"]
+    $commit = $configuration["commit"]
+    Write-Host "git checkout commit: $commit"
+    git checkout $configuration["commit"]
+    if ($LASTEXITCODE) { exit $LASTEXITCODE }
+  }
+  finally {
+    Pop-Location
+  }
+}
+else {
+  # write error if neither local spec repo nor repo and commit are provided
+  Write-Error "Must contain both 'repo' and 'commit' in tsp-location.yaml or input 'localSpecRepoPath' parameter."
+  exit 1
+}

 $tempTypeSpecDir = "$ProjectDirectory/TempTypeSpecFiles"
 New-Item $tempTypeSpecDir -Type Directory -Force | Out-Null
 CopySpecToProjectIfNeeded `
-    -specCloneRoot $specCloneDir `
-    -mainSpecDir $specSubDirectory `
-    -dest $tempTypeSpecDir `
-    -specAdditionalSubDirectories $configuration["additionalDirectories"]
+  -specCloneRoot $specCloneDir `
+  -mainSpecDir $specSubDirectory `
+  -dest $tempTypeSpecDir `
+  -specAdditionalSubDirectories $configuration["additionalDirectories"]
+
+exit 0