Sync eng/common directory with azure-sdk-tools for PR 7758 (#5410)
* Add 1es changes for job/matrix generation and publish * Use more flexible pool filter for prev/next pool matches * consolidate displayname definition * use linux pool variables for generate matrix job * Fix publish artifact * Use single publish task for publish artifact --------- Co-authored-by: Ben Broderick Phillips <bebroder@microsoft.com>
This commit is contained in:
Azure SDK Bot
GitHub
parent
e2dcf52dc1
commit
4be69dddcb
140
eng/common/pipelines/templates/jobs/generate-job-matrix.yml
Normal file
140
eng/common/pipelines/templates/jobs/generate-job-matrix.yml
Normal file
@ -0,0 +1,140 @@
|
||||
parameters:
|
||||
- name: AdditionalParameters
|
||||
type: object
|
||||
- name: DependsOn
|
||||
type: object
|
||||
default: null
|
||||
- name: CloudConfig
|
||||
type: object
|
||||
default: {}
|
||||
- name: MatrixConfigs
|
||||
type: object
|
||||
default: []
|
||||
- name: MatrixFilters
|
||||
type: object
|
||||
default: []
|
||||
- name: MatrixReplace
|
||||
type: object
|
||||
default: {}
|
||||
- name: JobTemplatePath
|
||||
type: string
|
||||
# Set this to false to do a full checkout for private repositories with the azure pipelines service connection
|
||||
- name: SparseCheckout
|
||||
type: boolean
|
||||
default: true
|
||||
- name: SparseCheckoutPaths
|
||||
type: object
|
||||
default: []
|
||||
- name: Pool
|
||||
type: string
|
||||
default: $(LINUXPOOL)
|
||||
- name: OsVmImage
|
||||
type: string
|
||||
default: $(LINUXVMIMAGE)
|
||||
- name: Os
|
||||
type: string
|
||||
default: linux
|
||||
# This parameter is only necessary if there are multiple invocations of this template within the SAME STAGE.
|
||||
# When that occurs, provide a name other than the default value.
|
||||
- name: GenerateJobName
|
||||
type: string
|
||||
default: 'generate_job_matrix'
|
||||
- name: PreGenerationSteps
|
||||
type: stepList
|
||||
default: []
|
||||
# Mappings to OS name required at template compile time by 1es pipeline templates
|
||||
- name: Pools
|
||||
type: object
|
||||
default:
|
||||
- name: Linux
|
||||
filter: .*Linux.*Pool$
|
||||
os: linux
|
||||
- name: Windows
|
||||
filter: .*Windows.*Pool$
|
||||
os: windows
|
||||
- name: Mac
|
||||
filter: .*MacPool$
|
||||
os: macOS
|
||||
|
||||
jobs:
|
||||
- job: ${{ parameters.GenerateJobName }}
|
||||
variables:
|
||||
- template: /eng/pipelines/templates/variables/image.yml
|
||||
- name: skipComponentGovernanceDetection
|
||||
value: true
|
||||
- name: displayNameFilter
|
||||
value: $[ coalesce(variables.jobMatrixFilter, '.*') ]
|
||||
pool:
|
||||
name: ${{ parameters.Pool }}
|
||||
vmImage: ${{ parameters.OsVmImage }}
|
||||
os: ${{ parameters.Os }}
|
||||
${{ if parameters.DependsOn }}:
|
||||
dependsOn: ${{ parameters.DependsOn }}
|
||||
steps:
|
||||
# Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
|
||||
# as we require the github service connection to be loaded.
|
||||
- ${{ if and(parameters.SparseCheckout, not(contains(variables['Build.DefinitionName'], '-pr - '))) }}:
|
||||
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
|
||||
parameters:
|
||||
${{ if ne(length(parameters.SparseCheckoutPaths), 0) }}:
|
||||
Paths: ${{ parameters.SparseCheckoutPaths }}
|
||||
${{ if and(eq(length(parameters.SparseCheckoutPaths), 0), ne(parameters.AdditionalParameters.ServiceDirectory, '')) }}:
|
||||
Paths:
|
||||
- "sdk/${{ parameters.AdditionalParameters.ServiceDirectory }}"
|
||||
|
||||
- ${{ parameters.PreGenerationSteps }}
|
||||
|
||||
- ${{ each config in parameters.MatrixConfigs }}:
|
||||
- ${{ each pool in parameters.Pools }}:
|
||||
- ${{ if eq(config.GenerateVMJobs, 'true') }}:
|
||||
- task: Powershell@2
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
|
||||
arguments: >
|
||||
-ConfigPath ${{ config.Path }}
|
||||
-Selection ${{ config.Selection }}
|
||||
-DisplayNameFilter '$(displayNameFilter)'
|
||||
-Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
|
||||
-Replace '${{ join(''',''', parameters.MatrixReplace) }}'
|
||||
-NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
|
||||
displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
|
||||
name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
|
||||
|
||||
- ${{ if eq(config.GenerateContainerJobs, 'true') }}:
|
||||
- task: Powershell@2
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
|
||||
arguments: >
|
||||
-ConfigPath ${{ config.Path }}
|
||||
-Selection ${{ config.Selection }}
|
||||
-DisplayNameFilter '$(displayNameFilter)'
|
||||
-Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
|
||||
-NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
|
||||
displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
|
||||
name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
|
||||
|
||||
- ${{ each config in parameters.MatrixConfigs }}:
|
||||
- ${{ each pool in parameters.Pools }}:
|
||||
- ${{ if eq(config.GenerateVMJobs, 'true') }}:
|
||||
- template: ${{ parameters.JobTemplatePath }}
|
||||
parameters:
|
||||
UsePlatformContainer: false
|
||||
OSName: ${{ pool.os }}
|
||||
Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
|
||||
DependsOn: ${{ parameters.GenerateJobName }}
|
||||
CloudConfig: ${{ parameters.CloudConfig }}
|
||||
${{ each param in parameters.AdditionalParameters }}:
|
||||
${{ param.key }}: ${{ param.value }}
|
||||
|
||||
- ${{ if eq(config.GenerateContainerJobs, 'true') }}:
|
||||
- template: ${{ parameters.JobTemplatePath }}
|
||||
parameters:
|
||||
UsePlatformContainer: true
|
||||
OSName: ${{ pool.os }}
|
||||
Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
|
||||
DependsOn: ${{ parameters.GenerateJobName }}
|
||||
CloudConfig: ${{ parameters.CloudConfig }}
|
||||
${{ each param in parameters.AdditionalParameters }}:
|
||||
${{ param.key }}: ${{ param.value }}
|
||||
@ -0,0 +1,30 @@
|
||||
# This step is used to prevent duplication of artifact publishes when there is an issue that would prevent the overall success of the job.
|
||||
# Ensuring that we only publish when successful (and two a differently named artifact otherwise) will allow easy retry on a build pipeline
|
||||
# without running into the "cannot override artifact" failure when we finally do get a passing run.
|
||||
|
||||
# ArtifactName - The name of the artifact in the "successful" case.
|
||||
# ArtifactPath - The path we will be publishing.
|
||||
# CustomCondition - Used if there is additional logic necessary to prevent attempt of publish.
|
||||
|
||||
parameters:
|
||||
ArtifactName: ''
|
||||
ArtifactPath: ''
|
||||
CustomCondition: true
|
||||
|
||||
steps:
|
||||
- pwsh: |
|
||||
Write-Host "##vso[task.setvariable variable=PublishArtifactName;]${{ parameters.ArtifactName }}"
|
||||
condition: and(succeeded(), ${{ parameters.CustomCondition }})
|
||||
displayName: Set Artifact Name
|
||||
|
||||
- pwsh: |
|
||||
Write-Host "##vso[task.setvariable variable=PublishArtifactName;]${{ parameters.ArtifactName }}-FailedAttempt$(System.JobAttempt)"
|
||||
condition: and(failed(), ${{ parameters.CustomCondition }})
|
||||
displayName: Set Failed Artifact Name
|
||||
|
||||
- task: 1ES.PublishPipelineArtifact@1
|
||||
condition: and(succeeded(), ${{ parameters.CustomCondition }})
|
||||
displayName: 'Publish ${{ parameters.ArtifactName }} Artifacts'
|
||||
inputs:
|
||||
artifact: '$(PublishArtifactName)'
|
||||
path: '${{ parameters.ArtifactPath }}'
|
||||
Loading…
Reference in New Issue
Block a user