Sync eng/common directory with azure-sdk-tools repository (#251)

Co-authored-by: Daniel Jurek <djurek@microsoft.com>

eng/common/TestResources/deploy-test-resources.yml

@@ -13,7 +13,7 @@ parameters:
 #   "TestApplicationId": "<test app id>",
 #   "TestApplicationSecret": "<test app secret>",
 #   "ProvisionerApplicationId": "<provisoner app id>",
-#   "ProvisoinerApplicationSecret": "<provisoner app secert>",
+#   "ProvisionerApplicationSecret": "<provisoner app secert>",
 #   "Environment": "AzureCloud | AzureGov | AzureChina | <other environment>"
 # }
 

eng/common/TestResources/remove-test-resources.yml

@@ -32,5 +32,5 @@ steps:
         -Force `
         -Verbose
     displayName: Remove test resources
-    condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), succeededOrFailed())
+    condition: ne(variables['AZURE_RESOURCEGROUP_NAME'], '')
     continueOnError: true

eng/common/Update-Change-Log.ps1

@@ -27,10 +27,10 @@ function Get-ChangelogPath($Path)
 { 
    # Check if CHANGELOG.md is present in path 
    $ChangeLogPath = Join-Path -Path $Path -ChildPath "CHANGELOG.md"
-   if ((Test-Path -Path $ChangeLogPath) -eq $False){
+   if ((Test-Path -Path $ChangeLogPath) -eq $False) {
       # Check if change log exists with name HISTORY.md
       $ChangeLogPath = Join-Path -Path $Path -ChildPath "HISTORY.md"
-      if ((Test-Path -Path $ChangeLogPath) -eq $False){
+      if ((Test-Path -Path $ChangeLogPath) -eq $False) {
          Write-Host "Change log is not found in path[$Path]"
          exit(1)
       }
@@ -45,7 +45,7 @@ function Get-VersionTitle($Version, $Unreleased)
 {
    # Generate version title
    $newVersionTitle = "## $Version $UNRELEASED_TAG"
-   if ($Unreleased -eq $False){
+   if ($Unreleased -eq $False) {
       $releaseDate = Get-Date -Format "(yyyy-MM-dd)"
       $newVersionTitle = "## $Version $releaseDate"
    }
@@ -67,10 +67,10 @@ function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Versi
    # Version increment tool passes replaceversion as False and Unreleased as True
    $is_version_increment = $ReplaceVersion -eq $False -and $Unreleased -eq $True
 
-   for(; $Index -lt $ChangelogLines.Count; $Index++){
-      if (Version-Matches($ChangelogLines[$Index])){
+   for (; $Index -lt $ChangelogLines.Count; $Index++) {
+      if (Version-Matches($ChangelogLines[$Index])) {
          # Find current title in change log
-         if( -not $CurrentTitle){
+         if( -not $CurrentTitle) {
             $CurrentTitle = $ChangelogLines[$Index]
             $CurrentIndex = $Index
             Write-Host "Current Version title: $CurrentTitle"
@@ -80,7 +80,7 @@ function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Versi
          # update change log script is triggered for all packages with current version for Java ( or any language where version is maintained in common file)
          # and this can cause an issue if someone changes changelog manually to prepare for release without updating actual version in central version file
          # Do not add new line or replace existing title when version is already present and script is triggered to add new line
-         if ($is_version_increment -and $ChangelogLines[$Index].Contains($Version)){
+         if ($is_version_increment -and $ChangelogLines[$Index].Contains($Version)) {
             Write-Host "Version is already present in change log."
             exit(0)
          }
@@ -90,26 +90,24 @@ function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Versi
    # Generate version title
    $newVersionTitle = Get-VersionTitle -Version $Version -Unreleased $Unreleased
 
-   if( $newVersionTitle -eq $CurrentTitle){
+   if( $newVersionTitle -eq $CurrentTitle) {
       Write-Host "No change is required in change log. Version is already present."
       exit(0)
    }
 
-   
-
-   if (($ReplaceVersion -eq $True) -and ($Unreleased -eq $False) -and (-not $CurrentTitle.Contains($UNRELEASED_TAG))){
+   if (($ReplaceVersion -eq $True) -and ($Unreleased -eq $False) -and $CurrentTitle.Contains($version) -and (-not $CurrentTitle.Contains($UNRELEASED_TAG))) {
       Write-Host "Version is already present in change log with a release date."
       exit(0)
    }
 
    # if current version title already has new version then we should replace title to update it
-   if ($CurrentTitle.Contains($Version) -and $ReplaceVersion -eq $False){
+   if ($CurrentTitle.Contains($Version) -and $ReplaceVersion -eq $False) {
       Write-Host "Version is already present in title. Updating version title"
       $ReplaceVersion = $True
    }
 
    # if version is already found and not replacing then nothing to do
-   if ($ReplaceVersion -eq $False){
+   if ($ReplaceVersion -eq $False) {
       Write-Host "Adding version title $newVersionTitle"
       $ChangelogLines.insert($CurrentIndex, "")
       $ChangelogLines.insert($CurrentIndex, "")
@@ -121,24 +119,28 @@ function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Versi
       $ChangelogLines[$CurrentIndex] = $newVersionTitle
    }
 
-   return $ChangelogLines      
+   return $ChangelogLines
 }
 
 
 # Make sure path is valid
-if ((Test-Path -Path $ChangeLogPath) -eq $False){
+if ((Test-Path -Path $ChangeLogPath) -eq $False) {
    Write-Host "Change log path is invalid. [$ChangeLogPath]"
    exit(1)
 }
 
 # probe change log path if path is directory 
-if (Test-Path -Path $ChangeLogPath -PathType Container)
-{   
+if (Test-Path -Path $ChangeLogPath -PathType Container) {
    $ChangeLogPath = Get-ChangelogPath -Path $ChangeLogPath
 }
 
 # Read current change logs and add/update version
 $ChangelogLines = [System.Collections.ArrayList](Get-Content -Path $ChangeLogPath)
+
+if ($null -eq $ChangelogLines) { 
+   $ChangelogLines = @() 
+}
+
 $NewContents = Get-NewChangeLog -ChangelogLines $ChangelogLines -Version $Version -Unreleased $Unreleased -ReplaceVersion $ReplaceVersion
 
 Write-Host "Writing change log to file [$ChangeLogPath]"

eng/common/pipelines/templates/steps/cosmos-emulator.yml

@@ -0,0 +1,16 @@
+parameters:
+  EmulatorMsiUrl: "https://aka.ms/cosmosdb-emulator"
+  StartParameters: ''
+
+steps:
+  - powershell: |
+      $targetDir = $env:temp
+      Write-Host "Downloading and extracting Cosmos DB Emulator - ${{ parameters.EmulatorMsiUrl }}"
+      Write-Host "Target Dir: $targetDir"
+      msiexec /a ${{ parameters.EmulatorMsiUrl }} TARGETDIR=$targetDir /qn | wait-process
+    displayName: Download and Extract Public Cosmos DB Emulator
+  - powershell: |
+      Write-Host "Launching Cosmos DB Emulator"
+      Import-Module "$env:temp\Azure Cosmos DB Emulator\PSModules\Microsoft.Azure.CosmosDB.Emulator"
+      Start-CosmosDbEmulator -NoUI ${{ parameters.StartParameters }}
+    displayName: Start Cosmos DB Emulator

eng/common/pipelines/templates/steps/create-pull-request.yml

@@ -12,6 +12,8 @@ parameters:
   WorkingDirectory: $(System.DefaultWorkingDirectory)
   PRTitle: not-specified
   ScriptDirectory: eng/common/scripts
+  GHReviewersVariable: '' 
+  GHTeamReviewersVariable: ''
 
 steps:
 
@@ -63,3 +65,18 @@ steps:
       -PRBranch "${{ parameters.PRBranchName }}"
       -AuthToken "$(azuresdk-github-pat)"
       -PRTitle "${{ parameters.PRTitle }}"
+
+- task: PowerShell@2
+  displayName: Tag a Reviewer on PR
+  condition: and(succeeded(), eq(variables['HasChanges'], 'true'))
+  inputs:
+    pwsh: true
+    workingDirectory: ${{ parameters.WorkingDirectory }}
+    filePath: ${{ parameters.ScriptDirectory }}/add-pullrequest-reviewers.ps1
+    arguments: >
+      -RepoOwner "${{ parameters.RepoOwner }}"
+      -RepoName "${{ parameters.RepoName }}"
+      -AuthToken "$(azuresdk-github-pat)"
+      -GitHubUsers "$(${{ parameters.GHReviewersVariable }})"
+      -GitHubTeams "$(${{ parameters.GHTeamReviewersVariable }})"
+      -PRNumber "$(Submitted.PullRequest.Number)"

eng/common/pipelines/templates/steps/daily-dev-build-variable.yml

@@ -0,0 +1,12 @@
+# This script fragment is used across our repos to set a variable "SetDevVersion" which
+# is used when this pipeline is going to be generating and publishing daily dev builds.
+
+steps:
+- pwsh: |
+    $setDailyDevBuild = "false"
+    if (('$(Build.Reason)' -eq 'Schedule') -and ('$(System.TeamProject)' -eq 'internal')) {
+      $setDailyDevBuild = "true"
+    }
+    echo "##vso[task.setvariable variable=SetDevVersion]$setDailyDevBuild"
+  displayName: "Setup Versioning Properties"
+  condition: eq(variables['SetDevVersion'], '')

eng/common/pipelines/templates/steps/docs-metadata-release.yml

@@ -12,6 +12,8 @@ parameters:
   ArtifactName: ''
   Language: ''
   DocRepoDestinationPath: '' #usually docs-ref-services/
+  GHReviewersVariable: '' 
+  GHTeamReviewersVariable: '' # externally set, as eng-common does not have the identity-resolver. Run as pre-step
 
 steps:
 - pwsh: |
@@ -56,3 +58,5 @@ steps:
     BaseBranchName: smoke-test
     WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo
     ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }}
+    GHReviewersVariable: ${{ parameters.GHReviewersVariable }}
+    GHTeamReviewersVariable: ${{ parameters.GHTeamReviewersVariable }} 

eng/common/pipelines/templates/steps/get-pr-owners.yml

@@ -0,0 +1,46 @@
+parameters: 
+  TargetVariable: ''
+  ServiceDirectory: ''
+
+steps:
+  - pwsh: |
+      git clone https://github.com/Azure/azure-sdk-tools.git $(Build.SourcesDirectory)/tools_repo
+      cd $(Build.SourcesDirectory)/tools_repo
+      git checkout 564ad63ae72d18422533fa1da9d396e7703c1cb5
+    displayName: Setup Identity Resolver
+
+  - pwsh: |
+      $result = dotnet run -v q -- `
+        --aad-app-id-var APP_ID `
+        --aad-app-secret-var APP_SECRET `
+        --aad-tenant-var AAD_TENANT `
+        --kusto-url-var KUSTO_URL `
+        --kusto-database-var KUSTO_DB `
+        --kusto-table-var KUSTO_TABLE `
+        --identity "$(Build.QueuedBy)"
+      $resolvedIdentity = $result[-1] | ConvertFrom-Json
+
+      Write-Host $resolvedIdentity
+
+      Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$($resolvedIdentity.GithubUserName)"
+    displayName: 'Resolving Queuing User'
+    workingDirectory: $(Build.SourcesDirectory)/tools_repo/tools/notification-configuration/identity-resolver
+    env:
+      APP_ID: $(notification-aad-app-id)
+      APP_SECRET: $(notification-aad-secret)
+      AAD_TENANT: $(notification-aad-tenant)
+      KUSTO_URL: $(notification-kusto-url)
+      KUSTO_DB: $(notification-kusto-db)
+      KUSTO_TABLE: $(notification-kusto-table)
+
+  - pwsh: |
+      Remove-Item -Force -Recurse $(Build.SourcesDirectory)/tools_repo
+    displayName: Clean Up Cloned Tools Repo
+
+  - pwsh: |
+      $originalValue = "$(${{ parameters.TargetVariable }})"
+      $result = $(Build.SourcesDirectory)/eng/common/scripts/get-codeowners.ps1 -TargetDirectory /sdk/${{ parameters.ServiceDirectory }}/ -RootDirectory $(Build.SourcesDirectory)
+      if ($result) {
+        Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$originalValue,$result"
+      }
+    displayName: Add CodeOwners if Present

eng/common/pipelines/templates/steps/verify-links.yml

@@ -1,5 +1,7 @@
 parameters:
   Directory: 'not-specified'
+  IgnoreLinksFile: "$(Build.SourcesDirectory)/eng/ignore-links.txt"
+  
 
 steps:
   - task: PowerShell@2
@@ -9,4 +11,4 @@ steps:
       workingDirectory: $(Build.SourcesDirectory)/${{ parameters.Directory }}
       filePath: eng/common/scripts/Verify-Links.ps1 
       arguments: >
-        -urls $(dir -r -i *.md) -rootUrl "file://$(Build.SourcesDirectory)/${{ parameters.Directory }}"
+        -urls $(dir -r -i *.md) -rootUrl "file://$(Build.SourcesDirectory)/${{ parameters.Directory }}" -recursive:$false -ignoreLinksFile ${{ parameters.IgnoreLinksFile }}

eng/common/scripts/ChangeLog-Operations.ps1

@@ -0,0 +1,122 @@
+# Common Changelog Operations
+
+$RELEASE_TITLE_REGEX = "(?<releaseNoteTitle>^\#+.*(?<version>\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?)(\s(?<releaseStatus>\(Unreleased\)|\(\d{4}-\d{2}-\d{2}\)))?)"
+
+# Returns a Collection of changeLogEntry object containing changelog info for all version present in the gived CHANGELOG
+function Get-ChangeLogEntries {
+  param (
+    [Parameter(Mandatory = $true)]
+    [String]$ChangeLogLocation
+  )
+
+  $changeLogEntries = @{}
+  if (!(Test-Path $ChangeLogLocation)) {
+    Write-Error "ChangeLog[${ChangeLogLocation}] does not exist"
+    return $null
+  }
+
+  try {
+    $contents = Get-Content $ChangeLogLocation
+    # walk the document, finding where the version specifiers are and creating lists
+    $changeLogEntry = $null
+    foreach ($line in $contents) {
+      if ($line -match $RELEASE_TITLE_REGEX) {
+        $changeLogEntry = [pscustomobject]@{ 
+          ReleaseVersion = $matches["version"]
+          ReleaseStatus  = $matches["releaseStatus"]
+          ReleaseTitle   = $line
+          ReleaseContent = @() # Release content without the version title
+        }
+        $changeLogEntries[$changeLogEntry.ReleaseVersion] = $changeLogEntry
+      }
+      else {
+        if ($changeLogEntry) {
+          $changeLogEntry.ReleaseContent += $line
+        }
+      }
+    }
+  }
+  catch {
+    Write-Host "Error parsing $ChangeLogLocation."
+    Write-Host $_.Exception.Message
+  }
+  return $changeLogEntries
+}
+
+# Returns single changeLogEntry object containing the ChangeLog for a particular version
+function Get-ChangeLogEntry {
+  param (
+    [Parameter(Mandatory = $true)]
+    [String]$ChangeLogLocation,
+    [Parameter(Mandatory = $true)]
+    [String]$VersionString
+  )
+  $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $ChangeLogLocation
+
+  if ($changeLogEntries -and $changeLogEntries.ContainsKey($VersionString)) {
+    return $changeLogEntries[$VersionString]
+  }
+  return $null
+}
+
+#Returns the changelog for a particular version as string
+function Get-ChangeLogEntryAsString {
+  param (
+    [Parameter(Mandatory = $true)]
+    [String]$ChangeLogLocation,
+    [Parameter(Mandatory = $true)]
+    [String]$VersionString
+  )
+
+  $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
+  return ChangeLogEntryAsString $changeLogEntry
+}
+
+function ChangeLogEntryAsString($changeLogEntry) {
+  if (!$changeLogEntry) {
+    return "[Missing change log entry]"
+  }
+  [string]$releaseTitle = $changeLogEntry.ReleaseTitle
+  [string]$releaseContent = $changeLogEntry.ReleaseContent -Join [Environment]::NewLine
+  return $releaseTitle, $releaseContent -Join [Environment]::NewLine
+}
+
+function Confirm-ChangeLogEntry {
+  param (
+    [Parameter(Mandatory = $true)]
+    [String]$ChangeLogLocation,
+    [Parameter(Mandatory = $true)]
+    [String]$VersionString,
+    [boolean]$ForRelease = $false
+  )
+
+  $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
+
+  if (!$changeLogEntry) {
+    Write-Error "ChangeLog[${ChangeLogLocation}] does not have an entry for version ${VersionString}."
+    return $false
+  }
+
+  Write-Host "Found the following change log entry for version '${VersionString}' in [${ChangeLogLocation}]."
+  Write-Host "-----"
+  Write-Host (ChangeLogEntryAsString $changeLogEntry)
+  Write-Host "-----"
+
+  if ([System.String]::IsNullOrEmpty($changeLogEntry.ReleaseStatus)) {
+    Write-Error "Entry does not have a correct release status. Please ensure the status is set to a date '(yyyy-MM-dd)' or '(Unreleased)' if not yet released."
+    return $false
+  }
+
+  if ($ForRelease -eq $True) {
+    if ($changeLogEntry.ReleaseStatus -eq "(Unreleased)") {
+      Write-Error "Entry has no release date set. Please ensure to set a release date with format 'yyyy-MM-dd'."
+      return $false
+    }
+
+    if ([System.String]::IsNullOrWhiteSpace($changeLogEntry.ReleaseContent)) {
+      Write-Error "Entry has no content. Please ensure to provide some content of what changed in this version."
+      return $false
+    }
+  }
+  return $true
+}

eng/common/scripts/Package-Properties.ps1

@@ -0,0 +1,182 @@
+# Helper functions for retireving useful information from azure-sdk-for-* repo
+# Example Use : Import-Module .\eng\common\scripts\modules
+class PackageProps
+{
+    [string]$pkgName
+    [string]$pkgVersion
+    [string]$pkgDirectoryPath
+    [string]$pkgServiceName
+    [string]$pkgReadMePath
+    [string]$pkgChangeLogPath
+    [string]$pkgGroup
+
+    PackageProps([string]$pkgName,[string]$pkgVersion,[string]$pkgDirectoryPath,[string]$pkgServiceName)
+    {
+        $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName)
+    }
+
+    PackageProps([string]$pkgName,[string]$pkgVersion,[string]$pkgDirectoryPath,[string]$pkgServiceName,[string]$pkgGroup="")
+    {
+        $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName, $pkgGroup)
+    }
+
+    hidden [void]Initialize(
+        [string]$pkgName,
+        [string]$pkgVersion,
+        [string]$pkgDirectoryPath,
+        [string]$pkgServiceName
+    )
+    {
+        $this.pkgName = $pkgName
+        $this.pkgVersion = $pkgVersion
+        $this.pkgDirectoryPath = $pkgDirectoryPath
+        $this.pkgServiceName = $pkgServiceName
+
+        if (Test-Path (Join-Path $pkgDirectoryPath "README.md"))
+        {
+            $this.pkgReadMePath = Join-Path $pkgDirectoryPath "README.md"
+        } 
+        else
+        {
+            $this.pkgReadMePath = $null
+        }
+
+        if (Test-Path (Join-Path $pkgDirectoryPath "CHANGELOG.md"))
+        {
+            $this.pkgChangeLogPath = Join-Path $pkgDirectoryPath "CHANGELOG.md"
+        } 
+        else
+        {
+            $this.pkgChangeLogPath = $null
+        }
+    }
+
+    hidden [void]Initialize(
+        [string]$pkgName,
+        [string]$pkgVersion,
+        [string]$pkgDirectoryPath,
+        [string]$pkgServiceName,
+        [string]$pkgGroup
+    )
+    {
+        $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName)
+        $this.pkgGroup = $pkgGroup
+    }
+}
+
+# Takes package name and service Name
+# Returns important properties of the package as related to the language repo
+# Returns a PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath }
+# Note: python is required for parsing python package properties.
+function Get-PkgProperties
+{
+    Param
+    (
+        [Parameter(Mandatory=$true)]
+        [string]$PackageName,
+        [Parameter(Mandatory=$true)]
+        [string]$ServiceName
+    )
+
+    $pkgDirectoryName = $null
+    $pkgDirectoryPath = $null
+    $serviceDirectoryPath = Join-Path $RepoRoot "sdk" $ServiceName
+    if (!(Test-Path $serviceDirectoryPath))
+    {
+        Write-Error "Service Directory $ServiceName does not exist"
+        exit 1
+    }
+
+    $directoriesPresent = Get-ChildItem $serviceDirectoryPath -Directory
+
+    foreach ($directory in $directoriesPresent)
+    {
+        $pkgDirectoryPath = Join-Path $serviceDirectoryPath $directory.Name
+        if ($ExtractPkgProps)
+        {
+            $pkgProps = &$ExtractPkgProps -pkgPath $pkgDirectoryPath -serviceName $ServiceName -pkgName $PackageName
+        }
+        else
+        {
+            Write-Error "The function '${ExtractPkgProps}' was not found."
+        }
+
+        if ($pkgProps -ne $null)
+        {
+            return $pkgProps
+        }
+    }
+    Write-Error "Failed to retrive Properties for $PackageName"
+}
+
+# Takes ServiceName and Repo Root Directory
+# Returns important properties for each package in the specified service, or entire repo if the serviceName is not specified
+# Returns an Table of service key to array values of PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath }
+function Get-AllPkgProperties ([string]$ServiceName=$null)
+{
+    $pkgPropsResult = @()
+
+    if ([string]::IsNullOrEmpty($ServiceName))
+    {
+        $searchDir = Join-Path $RepoRoot "sdk"
+        foreach ($dir in (Get-ChildItem $searchDir -Directory))
+        {
+            $serviceDir = Join-Path $searchDir $dir.Name
+
+            if (Test-Path (Join-Path $serviceDir "ci.yml"))
+            {
+                $activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml")
+                if ($activePkgList -ne $null)
+                {
+                    $pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -serviceName $dir.Name -pkgPropsResult $pkgPropsResult
+                }
+            }
+        }
+    } 
+    else
+    {
+        $serviceDir = Join-Path $RepoRoot "sdk" $ServiceName
+        if (Test-Path (Join-Path $serviceDir "ci.yml"))
+        {
+            $activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml")
+            if ($activePkgList -ne $null)
+            {
+                $pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -serviceName $ServiceName -pkgPropsResult $pkgPropsResult
+            }
+        }
+    }
+
+    return $pkgPropsResult
+}
+
+function Operate-OnPackages ($activePkgList, $serviceName, [Array]$pkgPropsResult)
+{
+    foreach ($pkg in $activePkgList)
+    {
+        $pkgProps = Get-PkgProperties -PackageName $pkg["name"] -ServiceName $serviceName
+        $pkgPropsResult += $pkgProps
+    }
+    return $pkgPropsResult
+}
+
+function Get-PkgListFromYml ($ciYmlPath)
+{
+    $ProgressPreference = "SilentlyContinue"
+    Register-PSRepository -Default -ErrorAction:SilentlyContinue
+    Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser
+    $ciYmlContent = Get-Content $ciYmlPath -Raw
+    $ciYmlObj = ConvertFrom-Yaml $ciYmlContent -Ordered
+    if ($ciYmlObj.Contains("stages"))
+    {
+      $artifactsInCI = $ciYmlObj["stages"][0]["parameters"]["Artifacts"]
+    }
+    elseif ($ciYmlObj.Contains("extends")) 
+    {
+      $artifactsInCI = $ciYmlObj["extends"]["parameters"]["Artifacts"]
+    }
+    if ($artifactsInCI -eq $null)
+    {
+        Write-Error "Failed to retrive package names in ci $ciYmlPath"
+    }
+    return $artifactsInCI
+}

eng/common/scripts/Submit-PullRequest.ps1

@@ -58,6 +58,9 @@ $resp | Write-Verbose
 
 if ($resp.Count -gt 0) {
     Write-Host -f green "Pull request already exists $($resp[0].html_url)"
+
+    # setting variable to reference the pull request by number
+    Write-Host "##vso[task.setvariable variable=Submitted.PullRequest.Number]$($resp[0].number)"
 }
 else {
   $data = @{
@@ -80,4 +83,7 @@ else {
 
   $resp | Write-Verbose
   Write-Host -f green "Pull request created https://github.com/$RepoOwner/$RepoName/pull/$($resp.number)"
+
+  # setting variable to reference the pull request by number
+  Write-Host "##vso[task.setvariable variable=Submitted.PullRequest.Number]$($resp.number)"
 }

eng/common/scripts/Verify-ChangeLog.ps1

@@ -1,31 +1,48 @@
 # Wrapper Script for ChangeLog Verification
 param (
-    [String]$ChangeLogLocation,
-    [String]$VersionString,
-    [string]$PackageName,
-    [string]$ServiceName,
-    [string]$RepoRoot,
-    [ValidateSet("net","java","js","python")]
-    [string]$Language,
-    [string]$RepoName,
-    [boolean]$ForRelease=$False
+  [String]$ChangeLogLocation,
+  [String]$VersionString,
+  [string]$PackageName,
+  [string]$ServiceName,
+  [string]$RepoRoot,
+  [ValidateSet("net", "java", "js", "python")]
+  [string]$Language,
+  [string]$RepoName,
+  [boolean]$ForRelease = $False
 )
 
+$ProgressPreference = "SilentlyContinue"
 . (Join-Path $PSScriptRoot SemVer.ps1)
 Import-Module (Join-Path $PSScriptRoot modules ChangeLog-Operations.psm1)
 
-if ((Test-Path $ChangeLogLocation) -and -not([System.String]::IsNullOrEmpty($VersionString)))
+$validChangeLog = $false
+if ($ChangeLogLocation -and $VersionString) 
 {
-    Confirm-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString -ForRelease $ForRelease
+  $validChangeLog = Confirm-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString -ForRelease $ForRelease
 }
-else 
+else
 {
-    Import-Module (Join-Path $PSScriptRoot modules Package-Properties.psm1)
-    if ([System.String]::IsNullOrEmpty($Language))
+  Import-Module (Join-Path $PSScriptRoot modules Package-Properties.psm1)
+  if ([System.String]::IsNullOrEmpty($Language))
+  {
+    if ($RepoName -match "azure-sdk-for-(?<lang>[^-]+)")
     {
-        $Language = $RepoName.Substring($RepoName.LastIndexOf('-') + 1)
+      $Language = $matches["lang"]
     }
+    else
+    {
+      Write-Error "Failed to set Language automatically. Please pass the appropriate Language as a parameter."
+      exit 1
+    }
+  }
 
-    $PackageProp = Get-PkgProperties -PackageName $PackageName -ServiceName $ServiceName -Language $Language -RepoRoot $RepoRoot
-    Confirm-ChangeLogEntry -ChangeLogLocation $PackageProp.pkgChangeLogPath -VersionString $PackageProp.pkgVersion -ForRelease $ForRelease
-}
+  $PackageProp = Get-PkgProperties -PackageName $PackageName -ServiceName $ServiceName -Language $Language -RepoRoot $RepoRoot
+  $validChangeLog = Confirm-ChangeLogEntry -ChangeLogLocation $PackageProp.pkgChangeLogPath -VersionString $PackageProp.pkgVersion -ForRelease $ForRelease
+}
+
+if (!$validChangeLog)
+{
+  exit 1
+}
+
+exit 0

eng/common/scripts/Verify-Links.ps1

@@ -87,7 +87,7 @@ function ResolveUri ([System.Uri]$referralUri, [string]$link)
     return $null
   }
 
-  if ($null -ne $ignoreLinks -and $ignoreLinks.Contains($link)) {
+  if ($null -ne $ignoreLinks -and ($ignoreLinks.Contains($link) -or $ignoreLinks.Contains($linkUri.ToString()))) {
     Write-Verbose "Ignoring invalid link $linkUri because it is in the ignore file."
     return $null
   }
@@ -125,7 +125,18 @@ function CheckLink ([System.Uri]$linkUri)
   }
   else {
     try {
-      $response = Invoke-WebRequest -Uri $linkUri
+      $headRequestSucceeded = $true
+      try {
+        # Attempt HEAD request first
+        $response = Invoke-WebRequest -Uri $linkUri -Method HEAD
+      }
+      catch {
+        $headRequestSucceeded = $false
+      }
+      if (!$headRequestSucceeded) {
+        # Attempt a GET request if the HEAD request failed.
+        $response = Invoke-WebRequest -Uri $linkUri -Method GET
+      }
       $statusCode = $response.StatusCode
       if ($statusCode -ne 200) {
         Write-Host "[$statusCode] while requesting $linkUri"

eng/common/scripts/add-pullrequest-reviewers.ps1

@@ -0,0 +1,79 @@
+param(
+    [Parameter(Mandatory = $true)]
+    $RepoOwner,
+
+    [Parameter(Mandatory = $true)]
+    $RepoName,
+
+    [Parameter(Mandatory = $false)]
+    $GitHubUsers = "",
+
+    [Parameter(Mandatory = $false)]
+    $GitHubTeams = "",
+
+    [Parameter(Mandatory = $true)]
+    $PRNumber,
+  
+    [Parameter(Mandatory = $true)]
+    $AuthToken
+)
+
+# at least one of these needs to be populated
+if (-not $GitHubUsers -and -not $GitHubTeams) {
+  Write-Host "No user provided for addition, exiting."
+  exit 0
+}
+
+$userAdditions = @($GitHubUsers.Split(",") | % { $_.Trim() } | ? { return $_ })
+$teamAdditions = @($GitHubTeams.Split(",") | % { $_.Trim() } | ? { return $_ })
+
+$headers = @{
+  Authorization = "bearer $AuthToken"
+}
+$uri = "https://api.github.com/repos/$RepoOwner/$RepoName/pulls/$PRNumber/requested_reviewers"
+
+try {
+  $resp = Invoke-RestMethod -Headers $headers $uri -MaximumRetryCount 3
+}
+catch {
+  Write-Error "Invoke-RestMethod [$uri] failed with exception:`n$_"
+  exit 1
+}
+
+# the response object takes this form: https://developer.github.com/v3/pulls/review_requests/#response-1
+# before we can push a new reviewer, we need to pull the simple Ids out of the complex objects that came back in the response
+$userReviewers = @($resp.users | % { return $_.login })
+$teamReviewers = @($resp.teams | % { return $_.slug })
+
+if (!$usersReviewers) { $modifiedUserReviewers = @() } else { $modifiedUserReviewers = $usersReviewers.Clone() }
+$modifiedUserReviewers += ($modifiedUserReviewers | ? { !$usersReviews.Contains($_) })
+
+if ($teamReviewers) { $modifiedTeamReviewers = @() } else { $modifiedTeamReviewers = $teamReviewers.Clone() }
+$modifiedTeamReviewers += ($modifiedUserReviewers | ? { !$teamReviewers.Contains($_) })
+
+$detectedUserDiffs = Compare-Object -ReferenceObject $userReviewers -DifferenceObject $modifiedUserReviewers
+$detectedTeamDiffs = Compare-Object -ReferenceObject $teamReviewers -DifferenceObject $modifiedTeamReviewers
+
+# Compare-Object returns values when there is a difference between the comparied objects.
+# we only want to run the update if there IS a difference.
+if ($detectedUserDiffs -or $detectedTeamDiffs) {
+  $postResp = @{}
+
+  if ($modifiedUserReviewers) { $postResp["reviewers"] = $modifiedUserReviewers }
+  if ($modifiedTeamReviewers) { $postResp["team_reviewers"] = $modifiedTeamReviewers }
+
+  $postResp = $postResp | ConvertTo-Json
+
+  try {
+    $resp = Invoke-RestMethod -Method Post -Headers $headers -Body $postResp -Uri $uri -MaximumRetryCount 3
+    $resp | Write-Verbose
+  }
+  catch {
+    Write-Error "Unable to update PR reviewers. `n$_"
+  }
+}
+else {
+  $results = $GitHubUsers + $GitHubTeams
+  Write-Host "Reviewers $results already added. Exiting."
+  exit(0)
+}

eng/common/scripts/artifact-metadata-parsing.ps1

@@ -406,7 +406,7 @@ function IsPythonPackageVersionPublished($pkgId, $pkgVersion) {
 # Retrieves the list of all tags that exist on the target repository
 function GetExistingTags($apiUrl) {
   try {
-    return (Invoke-WebRequest -Method "GET" -Uri "$apiUrl/git/refs/tags"  -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") }
+    return (Invoke-RestMethod -Method "GET" -Uri "$apiUrl/git/refs/tags"  -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") }
   }
   catch {
     Write-Host $_

eng/common/scripts/common.ps1

@@ -0,0 +1,27 @@
+$global:RepoRoot = Resolve-Path "${PSScriptRoot}..\..\..\.."
+$global:EngDir = Join-Path $global:RepoRoot "eng"
+$global:EngCommonDir = Join-Path $global:EngDir "common"
+$global:EngCommonScriptsDir = Join-Path $global:EngCommonDir "scripts"
+$global:EngScriptsDir = Join-Path $global:EngDir "scripts"
+
+# Import required scripts
+. (Join-Path $global:EngCommonScriptsDir SemVer.ps1)
+. (Join-Path $global:EngCommonScriptsDir Changelog-Operations.ps1)
+. (Join-Path $global:EngCommonScriptsDir Package-Properties.ps1)
+
+# Setting expected from common languages settings
+$global:Language = "Unknown"
+$global:PackageRepository = "Unknown"
+$global:packagePattern = "Unknown"
+$global:MetadataUri = "Unknown"
+
+# Import common language settings
+$EngScriptsLanguageSettings = Join-path $global:EngScriptsDir "Language-Settings.ps1"
+if (Test-Path $EngScriptsLanguageSettings) {
+  . $EngScriptsLanguageSettings
+}
+
+# Transformed Functions
+$GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo"
+$GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile"
+$PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs"

eng/common/scripts/get-codeowners.ps1

@@ -0,0 +1,40 @@
+param (
+  $TargetDirectory, # should be in relative form from root of repo. EG: sdk/servicebus
+  $RootDirectory # ideally $(Build.SourcesDirectory)
+)
+
+$codeOwnersLocation = Join-Path $RootDirectory -ChildPath ".github/CODEOWNERS"
+
+if (!(Test-Path $codeOwnersLocation)) {
+  Write-Host "Unable to find CODEOWNERS file in target directory $RootDirectory"
+  exit 1
+}
+
+$codeOwnersContent = Get-Content $codeOwnersLocation
+
+$ownedFolders = @{}
+
+foreach ($contentLine in $codeOwnersContent) {
+  if (-not $contentLine.StartsWith("#") -and $contentLine){
+    $splitLine = $contentLine -split "\s+"
+    
+    # CODEOWNERS file can also have labels present after the owner aliases
+    # gh aliases start with @ in codeowners. don't pass on to API calls
+    $ownedFolders[$splitLine[0].ToLower()] = ($splitLine[1..$($splitLine.Length)] `
+      | ? { $_.StartsWith("@") } `
+      | % { return $_.substring(1) }) -join ","
+  }
+}
+
+$results = $ownedFolders[$TargetDirectory.ToLower()]
+
+if ($results) {
+  Write-Host "Discovered code owners for path $TargetDirectory are $results."
+  return $results
+}
+else {
+  Write-Host "Unable to match path $TargetDirectory in CODEOWNERS file located at $codeOwnersLocation."
+  Write-Host $ownedFolders | ConvertTo-Json
+  return ""
+}
+

eng/common/scripts/modules/ChangeLog-Operations.psm1

@@ -11,8 +11,8 @@ function Get-ChangeLogEntries {
 
   $changeLogEntries = @{}
   if (!(Test-Path $ChangeLogLocation)) {
-    Write-Host "ChangeLog '{0}' was not found" -f $ChangeLogLocation
-    exit 1
+    Write-Error "ChangeLog[${ChangeLogLocation}] does not exist"
+    return $null
   }
 
   try {
@@ -51,14 +51,12 @@ function Get-ChangeLogEntry {
     [Parameter(Mandatory = $true)]
     [String]$VersionString
   )
-
   $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $ChangeLogLocation
 
-  if ($changeLogEntries.ContainsKey($VersionString)) {
+  if ($changeLogEntries -and $changeLogEntries.ContainsKey($VersionString)) {
     return $changeLogEntries[$VersionString]
   }
-  Write-Error "Release Notes for the Specified version ${VersionString} was not found"
-  exit 1
+  return $null
 }
 
 #Returns the changelog for a particular version as string
@@ -70,9 +68,16 @@ function Get-ChangeLogEntryAsString {
     [String]$VersionString
   )
 
-  $changeLogEntries = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
-  [string]$releaseTitle = $changeLogEntries.ReleaseTitle
-  [string]$releaseContent = $changeLogEntries.ReleaseContent -Join [Environment]::NewLine
+  $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
+  return ChangeLogEntryAsString $changeLogEntry
+}
+
+function ChangeLogEntryAsString($changeLogEntry) {
+  if (!$changeLogEntry) {
+    return "[Missing change log entry]"
+  }
+  [string]$releaseTitle = $changeLogEntry.ReleaseTitle
+  [string]$releaseContent = $changeLogEntry.ReleaseContent -Join [Environment]::NewLine
   return $releaseTitle, $releaseContent -Join [Environment]::NewLine
 }
 
@@ -87,27 +92,33 @@ function Confirm-ChangeLogEntry {
 
   $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
 
+  if (!$changeLogEntry) {
+    Write-Error "ChangeLog[${ChangeLogLocation}] does not have an entry for version ${VersionString}."
+    return $false
+  }
+
+  Write-Host "Found the following change log entry for version '${VersionString}' in [${ChangeLogLocation}]."
+  Write-Host "-----"
+  Write-Host (ChangeLogEntryAsString $changeLogEntry)
+  Write-Host "-----"
+
   if ([System.String]::IsNullOrEmpty($changeLogEntry.ReleaseStatus)) {
-    Write-Host ("##[error]Changelog '{0}' has wrong release note title" -f $ChangeLogLocation)
-    Write-Host "##[info]Ensure the release date is included i.e. (yyyy-MM-dd) or (Unreleased) if not yet released"
-    exit 1
+    Write-Error "Entry does not have a correct release status. Please ensure the status is set to a date '(yyyy-MM-dd)' or '(Unreleased)' if not yet released."
+    return $false
   }
 
   if ($ForRelease -eq $True) {
     if ($changeLogEntry.ReleaseStatus -eq "(Unreleased)") {
-      Write-Host ("##[error]No release date set. Please set a release date with format 'yyyy-MM-dd' in the heading for version '{0}' in the changelog '{1}'." -f $VersionString, $ChangelogLocation)
-      exit 1
+      Write-Error "Entry has no release date set. Please ensure to set a release date with format 'yyyy-MM-dd'."
+      return $false
     }
 
     if ([System.String]::IsNullOrWhiteSpace($changeLogEntry.ReleaseContent)) {
-      Write-Host ("##[error]Empty Release Notes for '{0}' in '{1}'" -f $VersionString, $ChangeLogLocation)
-      Write-Host "##[info]Please ensure there is a release notes entry before releasing the package."
-      exit 1
+      Write-Error "Entry has no content. Please ensure to provide some content of what changed in this version."
+      return $false
     }
   }
-
-  Write-Host $changeLogEntry.ReleaseTitle
-  Write-Host $changeLogEntry.ReleaseContent
+  return $true
 }
  
 Export-ModuleMember -Function 'Get-ChangeLogEntries'

eng/common/scripts/modules/Package-Properties.psm1

@@ -3,13 +3,24 @@
 class PackageProps
 {
     [string]$pkgName
-    [AzureEngSemanticVersion]$pkgVersion
+    [string]$pkgVersion
     [string]$pkgDirectoryPath
     [string]$pkgServiceName
     [string]$pkgReadMePath
     [string]$pkgChangeLogPath
+    [string]$pkgGroup
 
-    PackageProps(
+    PackageProps([string]$pkgName,[string]$pkgVersion,[string]$pkgDirectoryPath,[string]$pkgServiceName)
+    {
+        $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName)
+    }
+
+    PackageProps([string]$pkgName,[string]$pkgVersion,[string]$pkgDirectoryPath,[string]$pkgServiceName,[string]$pkgGroup="")
+    {
+        $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName, $pkgGroup)
+    }
+
+    hidden [void]Initialize(
         [string]$pkgName,
         [string]$pkgVersion,
         [string]$pkgDirectoryPath,
@@ -17,11 +28,7 @@ class PackageProps
     )
     {
         $this.pkgName = $pkgName
-        $this.pkgVersion = [AzureEngSemanticVersion]::ParseVersionString($pkgVersion)
-        if ($this.pkgVersion -eq $null)
-        {
-            Write-Error "Invalid version in $pkgDirectoryPath"
-        }
+        $this.pkgVersion = $pkgVersion
         $this.pkgDirectoryPath = $pkgDirectoryPath
         $this.pkgServiceName = $pkgServiceName
 
@@ -43,9 +50,19 @@ class PackageProps
             $this.pkgChangeLogPath = $null
         }
     }
-}
 
-Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser
+    hidden [void]Initialize(
+        [string]$pkgName,
+        [string]$pkgVersion,
+        [string]$pkgDirectoryPath,
+        [string]$pkgServiceName,
+        [string]$pkgGroup
+    )
+    {
+        $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName)
+        $this.pkgGroup = $pkgGroup
+    }
+}
 
 function Extract-PkgProps ($pkgPath, $serviceName, $pkgName, $lang)
 {
@@ -126,10 +143,11 @@ function Extract-JavaPkgProps ($pkgPath, $serviceName, $pkgName)
         $projectData.load($projectPath)
         $projectPkgName = $projectData.project.artifactId
         $pkgVersion = $projectData.project.version
+        $pkgGroup = $projectData.project.groupId
 
         if ($projectPkgName -eq $pkgName)
         {
-            return [PackageProps]::new($pkgName, $pkgVersion.ToString(), $pkgPath, $serviceName)
+            return [PackageProps]::new($pkgName, $pkgVersion.ToString(), $pkgPath, $serviceName, $pkgGroup)
         }
     }
     return $null
@@ -237,10 +255,19 @@ function Operate-OnPackages ($activePkgList, $serviceName, $language, $repoRoot,
 
 function Get-PkgListFromYml ($ciYmlPath)
 {
+    $ProgressPreference = "SilentlyContinue"
+    Register-PSRepository -Default -ErrorAction:SilentlyContinue
+    Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser
     $ciYmlContent = Get-Content $ciYmlPath -Raw
     $ciYmlObj = ConvertFrom-Yaml $ciYmlContent -Ordered
-    $artifactsInCI = $ciYmlObj["stages"][0]["parameters"]["Artifacts"]
-
+    if ($ciYmlObj.Contains("stages"))
+    {
+      $artifactsInCI = $ciYmlObj["stages"][0]["parameters"]["Artifacts"]
+    }
+    elseif ($ciYmlObj.Contains("extends")) 
+    {
+      $artifactsInCI = $ciYmlObj["extends"]["parameters"]["Artifacts"]
+    }
     if ($artifactsInCI -eq $null)
     {
         Write-Error "Failed to retrive package names in ci $ciYmlPath"