d857c1c7d1
* quotes * generalize * cant use trigger as a job * Update eng/common/pipelines/templates/jobs/ai-eval-job.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * target repo name fix * compile time expression * compile time expression * show params for pipeline * fix the pipeline variables * another fix * fix variables --------- Co-authored-by: Juan Ospina <70209456+jeo02@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
101 lines
3.2 KiB
YAML
101 lines
3.2 KiB
YAML
parameters:
|
|
- name: EvalProject
|
|
type: string
|
|
default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations'
|
|
- name: OpenAIEndPoint
|
|
type: string
|
|
default: 'https://openai-shared.openai.azure.com/'
|
|
- name: Model
|
|
type: string
|
|
default: 'gpt-5'
|
|
# Repository where evals are located
|
|
- name: EvalRepoOwner
|
|
type: string
|
|
default: 'Azure'
|
|
- name: EvalRepoName
|
|
type: string
|
|
default: 'azure-sdk-tools'
|
|
- name: EvalRepoCommit
|
|
type: string
|
|
default: 'main'
|
|
# Target repository for copilot instructions
|
|
- name: TargetRepoOwner
|
|
type: string
|
|
- name: TargetRepoName
|
|
type: string
|
|
- name: TargetRepoCommit
|
|
type: string
|
|
|
|
jobs:
|
|
- job: Run_Eval
|
|
variables:
|
|
- template: /eng/pipelines/templates/variables/globals.yml
|
|
- template: /eng/pipelines/templates/variables/image.yml
|
|
displayName: 'Run AI Eval'
|
|
pool:
|
|
name: $(LINUXPOOL)
|
|
image: $(LINUXVMIMAGE)
|
|
os: linux
|
|
|
|
steps:
|
|
- checkout: none
|
|
- task: UseDotNet@2
|
|
displayName: "Use .NET SDK 9.0.x"
|
|
retryCountOnTaskFailure: 3
|
|
inputs:
|
|
packageType: sdk
|
|
version: 9.0.x
|
|
performMultiLevelLookup: true
|
|
|
|
- task: UseDotNet@2
|
|
displayName: "Use .NET SDK 8.0.x"
|
|
retryCountOnTaskFailure: 3
|
|
inputs:
|
|
packageType: sdk
|
|
version: 8.0.x
|
|
performMultiLevelLookup: true
|
|
|
|
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
|
|
parameters:
|
|
SkipCheckoutNone: true
|
|
Repositories:
|
|
- Name: ${{ parameters.EvalRepoOwner }}/${{ parameters.EvalRepoName }}
|
|
Commitish: ${{ parameters.EvalRepoCommit }}
|
|
WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.EvalRepoName }}
|
|
- Name: ${{ parameters.TargetRepoOwner }}/${{ parameters.TargetRepoName }}
|
|
Commitish: ${{ parameters.TargetRepoCommit }}
|
|
WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.TargetRepoName }}
|
|
Paths:
|
|
- 'tools/**'
|
|
- 'eng/common/**'
|
|
- '.github/copilot-instructions.md'
|
|
|
|
- task: AzureCLI@2
|
|
displayName: 'Run eval'
|
|
condition: and(succeeded(), ne(variables['Skip.Eval'], 'true'))
|
|
inputs:
|
|
azureSubscription: opensource-api-connection
|
|
scriptType: 'bash'
|
|
scriptLocation: 'inlineScript'
|
|
workingDirectory: '$(System.DefaultWorkingDirectory)/${{ parameters.EvalRepoName }}/${{ parameters.EvalProject }}'
|
|
inlineScript: |
|
|
echo "Logged in to Azure"
|
|
echo "Running eval"
|
|
dotnet test --logger trx
|
|
env:
|
|
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
|
|
DOTNET_CLI_TELEMETRY_OPTOUT: 1
|
|
DOTNET_MULTILEVEL_LOOKUP: 0
|
|
AZURE_OPENAI_MODEL_DEPLOYMENT_NAME: ${{ parameters.Model }}
|
|
AZURE_OPENAI_ENDPOINT: ${{ parameters.OpenAIEndPoint }}
|
|
REPOSITORY_NAME: ${{ parameters.TargetRepoName }}
|
|
COPILOT_INSTRUCTIONS_PATH_MCP_EVALS: $(System.DefaultWorkingDirectory)/${{ parameters.TargetRepoName }}/.github/copilot-instructions.md
|
|
|
|
- task: PublishTestResults@2
|
|
condition: succeededOrFailed()
|
|
inputs:
|
|
testResultsFiles: '**/*.trx'
|
|
testRunTitle: $(System.JobDisplayName)
|
|
testResultsFormat: 'VSTest'
|
|
mergeTestResults: true
|