Sync eng/common directory with azure-sdk-tools for PR 13142 (#6855)
* quotes * generalize * cant use trigger as a job * Update eng/common/pipelines/templates/jobs/ai-eval-job.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * target repo name fix * compile time expression * compile time expression * show params for pipeline * fix the pipeline variables * another fix * fix variables --------- Co-authored-by: Juan Ospina <70209456+jeo02@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Azure SDK Bot
GitHub
parent
f1ccfabd25
commit
d857c1c7d1
@ -15,83 +15,34 @@ parameters:
|
||||
default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations'
|
||||
- name: OpenAIEndPoint
|
||||
type: string
|
||||
default: 'https://ai-prmarottai3149546654251245.openai.azure.com/'
|
||||
default: 'https://openai-shared.openai.azure.com/'
|
||||
- name: Model
|
||||
type: string
|
||||
default: 'gpt-5'
|
||||
- name: ToolsRepoName
|
||||
type: string
|
||||
default: 'azure-sdk-tools'
|
||||
- name: ToolsRepoOwner
|
||||
- name: EvalRepoOwner
|
||||
type: string
|
||||
default: 'Azure'
|
||||
- name: EvalRepoName
|
||||
type: string
|
||||
default: 'azure-sdk-tools'
|
||||
- name: EvalRepoCommit
|
||||
type: string
|
||||
default: 'main'
|
||||
|
||||
jobs:
|
||||
- job: Run_Eval
|
||||
variables:
|
||||
- template: /eng/pipelines/templates/variables/globals.yml
|
||||
- template: /eng/pipelines/templates/variables/image.yml
|
||||
displayName: 'Run AI Eval'
|
||||
pool:
|
||||
name: $(LINUXPOOL)
|
||||
image: $(LINUXVMIMAGE)
|
||||
os: linux
|
||||
variables:
|
||||
TargetRepoOwner: ${{ split(variables['Build.Repository.Name'], '/')[0] }}
|
||||
TargetRepoName: ${{ split(variables['Build.Repository.Name'], '/')[1] }}
|
||||
TargetRepoCommit: ${{ variables['Build.SourceVersion'] }}
|
||||
|
||||
steps:
|
||||
- task: UseDotNet@2
|
||||
displayName: "Use .NET SDK 9.0.x"
|
||||
retryCountOnTaskFailure: 3
|
||||
inputs:
|
||||
packageType: sdk
|
||||
version: 9.0.x
|
||||
performMultiLevelLookup: true
|
||||
|
||||
- task: UseDotNet@2
|
||||
displayName: "Use .NET SDK 8.0.x"
|
||||
retryCountOnTaskFailure: 3
|
||||
inputs:
|
||||
packageType: sdk
|
||||
version: 8.0.x
|
||||
performMultiLevelLookup: true
|
||||
|
||||
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
|
||||
extends:
|
||||
template: /eng/common/pipelines/templates/jobs/ai-eval-job.yml
|
||||
parameters:
|
||||
SkipCheckoutNone: true
|
||||
Repositories:
|
||||
- Name: $(Build.Repository.Name)
|
||||
Commitish: $(Build.SourceVersion)
|
||||
WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)
|
||||
- Name: ${{ parameters.ToolsRepoOwner }}/${{ parameters.ToolsRepoName }}
|
||||
WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.ToolsRepoName }}
|
||||
Paths:
|
||||
- 'tools/**'
|
||||
- eng/common/**
|
||||
- .github/copilot-instructions.md
|
||||
|
||||
- task: AzureCLI@2
|
||||
displayName: 'Run eval'
|
||||
inputs:
|
||||
azureSubscription: opensource-api-connection
|
||||
scriptType: 'bash'
|
||||
scriptLocation: 'inlineScript'
|
||||
workingDirectory: '${{ parameters.ToolsRepoName }}/${{ parameters.EvalProject }}'
|
||||
inlineScript: |
|
||||
echo "Logged in to Azure"
|
||||
echo "Running eval in project ${{ parameters.ToolsRepoName }}/${{ parameters.EvalProject }}"
|
||||
dotnet test /p:ArtifactsPackagesDir=$(Build.ArtifactStagingDirectory) --logger trx
|
||||
env:
|
||||
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
|
||||
DOTNET_CLI_TELEMETRY_OPTOUT: 1
|
||||
DOTNET_MULTILEVEL_LOOKUP: 0
|
||||
AZURE_OPENAI_MODEL_DEPLOYMENT_NAME: ${{ parameters.Model }}
|
||||
AZURE_OPENAI_ENDPOINT: ${{ parameters.OpenAIEndPoint }}
|
||||
REPOSITORY_NAME: $(Build.Repository.Name)
|
||||
COPILOT_INSTRUCTIONS_PATH_MCP_EVALS: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/.github/copilot-instructions.md
|
||||
|
||||
- task: PublishTestResults@2
|
||||
condition: succeededOrFailed()
|
||||
inputs:
|
||||
testResultsFiles: '**/*.trx'
|
||||
testRunTitle: $(System.JobDisplayName)
|
||||
testResultsFormat: 'VSTest'
|
||||
mergeTestResults: true
|
||||
EvalProject: ${{ parameters.EvalProject }}
|
||||
OpenAIEndPoint: ${{ parameters.OpenAIEndPoint }}
|
||||
Model: ${{ parameters.Model }}
|
||||
EvalRepoOwner: ${{ parameters.EvalRepoOwner }}
|
||||
EvalRepoName: ${{ parameters.EvalRepoName }}
|
||||
EvalRepoCommit: ${{ parameters.EvalRepoCommit }}
|
||||
TargetRepoOwner: $(TargetRepoOwner)
|
||||
TargetRepoName: $(TargetRepoName)
|
||||
TargetRepoCommit: $(TargetRepoCommit)
|
||||
100
eng/common/pipelines/templates/jobs/ai-eval-job.yml
Normal file
100
eng/common/pipelines/templates/jobs/ai-eval-job.yml
Normal file
@ -0,0 +1,100 @@
|
||||
parameters:
|
||||
- name: EvalProject
|
||||
type: string
|
||||
default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations'
|
||||
- name: OpenAIEndPoint
|
||||
type: string
|
||||
default: 'https://openai-shared.openai.azure.com/'
|
||||
- name: Model
|
||||
type: string
|
||||
default: 'gpt-5'
|
||||
# Repository where evals are located
|
||||
- name: EvalRepoOwner
|
||||
type: string
|
||||
default: 'Azure'
|
||||
- name: EvalRepoName
|
||||
type: string
|
||||
default: 'azure-sdk-tools'
|
||||
- name: EvalRepoCommit
|
||||
type: string
|
||||
default: 'main'
|
||||
# Target repository for copilot instructions
|
||||
- name: TargetRepoOwner
|
||||
type: string
|
||||
- name: TargetRepoName
|
||||
type: string
|
||||
- name: TargetRepoCommit
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
- job: Run_Eval
|
||||
variables:
|
||||
- template: /eng/pipelines/templates/variables/globals.yml
|
||||
- template: /eng/pipelines/templates/variables/image.yml
|
||||
displayName: 'Run AI Eval'
|
||||
pool:
|
||||
name: $(LINUXPOOL)
|
||||
image: $(LINUXVMIMAGE)
|
||||
os: linux
|
||||
|
||||
steps:
|
||||
- checkout: none
|
||||
- task: UseDotNet@2
|
||||
displayName: "Use .NET SDK 9.0.x"
|
||||
retryCountOnTaskFailure: 3
|
||||
inputs:
|
||||
packageType: sdk
|
||||
version: 9.0.x
|
||||
performMultiLevelLookup: true
|
||||
|
||||
- task: UseDotNet@2
|
||||
displayName: "Use .NET SDK 8.0.x"
|
||||
retryCountOnTaskFailure: 3
|
||||
inputs:
|
||||
packageType: sdk
|
||||
version: 8.0.x
|
||||
performMultiLevelLookup: true
|
||||
|
||||
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
|
||||
parameters:
|
||||
SkipCheckoutNone: true
|
||||
Repositories:
|
||||
- Name: ${{ parameters.EvalRepoOwner }}/${{ parameters.EvalRepoName }}
|
||||
Commitish: ${{ parameters.EvalRepoCommit }}
|
||||
WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.EvalRepoName }}
|
||||
- Name: ${{ parameters.TargetRepoOwner }}/${{ parameters.TargetRepoName }}
|
||||
Commitish: ${{ parameters.TargetRepoCommit }}
|
||||
WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.TargetRepoName }}
|
||||
Paths:
|
||||
- 'tools/**'
|
||||
- 'eng/common/**'
|
||||
- '.github/copilot-instructions.md'
|
||||
|
||||
- task: AzureCLI@2
|
||||
displayName: 'Run eval'
|
||||
condition: and(succeeded(), ne(variables['Skip.Eval'], 'true'))
|
||||
inputs:
|
||||
azureSubscription: opensource-api-connection
|
||||
scriptType: 'bash'
|
||||
scriptLocation: 'inlineScript'
|
||||
workingDirectory: '$(System.DefaultWorkingDirectory)/${{ parameters.EvalRepoName }}/${{ parameters.EvalProject }}'
|
||||
inlineScript: |
|
||||
echo "Logged in to Azure"
|
||||
echo "Running eval"
|
||||
dotnet test --logger trx
|
||||
env:
|
||||
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
|
||||
DOTNET_CLI_TELEMETRY_OPTOUT: 1
|
||||
DOTNET_MULTILEVEL_LOOKUP: 0
|
||||
AZURE_OPENAI_MODEL_DEPLOYMENT_NAME: ${{ parameters.Model }}
|
||||
AZURE_OPENAI_ENDPOINT: ${{ parameters.OpenAIEndPoint }}
|
||||
REPOSITORY_NAME: ${{ parameters.TargetRepoName }}
|
||||
COPILOT_INSTRUCTIONS_PATH_MCP_EVALS: $(System.DefaultWorkingDirectory)/${{ parameters.TargetRepoName }}/.github/copilot-instructions.md
|
||||
|
||||
- task: PublishTestResults@2
|
||||
condition: succeededOrFailed()
|
||||
inputs:
|
||||
testResultsFiles: '**/*.trx'
|
||||
testRunTitle: $(System.JobDisplayName)
|
||||
testResultsFormat: 'VSTest'
|
||||
mergeTestResults: true
|
||||
Loading…
Reference in New Issue
Block a user