Sync eng/common directory with azure-sdk-tools for PR 13142 (#6855)
* quotes * generalize * cant use trigger as a job * Update eng/common/pipelines/templates/jobs/ai-eval-job.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * target repo name fix * compile time expression * compile time expression * show params for pipeline * fix the pipeline variables * another fix * fix variables --------- Co-authored-by: Juan Ospina <70209456+jeo02@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Azure SDK Bot
GitHub
parent
f1ccfabd25
commit
d857c1c7d1
@ -15,83 +15,34 @@ parameters:
|
|||||||
default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations'
|
default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations'
|
||||||
- name: OpenAIEndPoint
|
- name: OpenAIEndPoint
|
||||||
type: string
|
type: string
|
||||||
default: 'https://ai-prmarottai3149546654251245.openai.azure.com/'
|
default: 'https://openai-shared.openai.azure.com/'
|
||||||
- name: Model
|
- name: Model
|
||||||
type: string
|
type: string
|
||||||
default: 'gpt-5'
|
default: 'gpt-5'
|
||||||
- name: ToolsRepoName
|
- name: EvalRepoOwner
|
||||||
type: string
|
|
||||||
default: 'azure-sdk-tools'
|
|
||||||
- name: ToolsRepoOwner
|
|
||||||
type: string
|
type: string
|
||||||
default: 'Azure'
|
default: 'Azure'
|
||||||
|
- name: EvalRepoName
|
||||||
|
type: string
|
||||||
|
default: 'azure-sdk-tools'
|
||||||
|
- name: EvalRepoCommit
|
||||||
|
type: string
|
||||||
|
default: 'main'
|
||||||
|
|
||||||
jobs:
|
variables:
|
||||||
- job: Run_Eval
|
TargetRepoOwner: ${{ split(variables['Build.Repository.Name'], '/')[0] }}
|
||||||
variables:
|
TargetRepoName: ${{ split(variables['Build.Repository.Name'], '/')[1] }}
|
||||||
- template: /eng/pipelines/templates/variables/globals.yml
|
TargetRepoCommit: ${{ variables['Build.SourceVersion'] }}
|
||||||
- template: /eng/pipelines/templates/variables/image.yml
|
|
||||||
displayName: 'Run AI Eval'
|
extends:
|
||||||
pool:
|
template: /eng/common/pipelines/templates/jobs/ai-eval-job.yml
|
||||||
name: $(LINUXPOOL)
|
parameters:
|
||||||
image: $(LINUXVMIMAGE)
|
EvalProject: ${{ parameters.EvalProject }}
|
||||||
os: linux
|
OpenAIEndPoint: ${{ parameters.OpenAIEndPoint }}
|
||||||
|
Model: ${{ parameters.Model }}
|
||||||
steps:
|
EvalRepoOwner: ${{ parameters.EvalRepoOwner }}
|
||||||
- task: UseDotNet@2
|
EvalRepoName: ${{ parameters.EvalRepoName }}
|
||||||
displayName: "Use .NET SDK 9.0.x"
|
EvalRepoCommit: ${{ parameters.EvalRepoCommit }}
|
||||||
retryCountOnTaskFailure: 3
|
TargetRepoOwner: $(TargetRepoOwner)
|
||||||
inputs:
|
TargetRepoName: $(TargetRepoName)
|
||||||
packageType: sdk
|
TargetRepoCommit: $(TargetRepoCommit)
|
||||||
version: 9.0.x
|
|
||||||
performMultiLevelLookup: true
|
|
||||||
|
|
||||||
- task: UseDotNet@2
|
|
||||||
displayName: "Use .NET SDK 8.0.x"
|
|
||||||
retryCountOnTaskFailure: 3
|
|
||||||
inputs:
|
|
||||||
packageType: sdk
|
|
||||||
version: 8.0.x
|
|
||||||
performMultiLevelLookup: true
|
|
||||||
|
|
||||||
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
|
|
||||||
parameters:
|
|
||||||
SkipCheckoutNone: true
|
|
||||||
Repositories:
|
|
||||||
- Name: $(Build.Repository.Name)
|
|
||||||
Commitish: $(Build.SourceVersion)
|
|
||||||
WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)
|
|
||||||
- Name: ${{ parameters.ToolsRepoOwner }}/${{ parameters.ToolsRepoName }}
|
|
||||||
WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.ToolsRepoName }}
|
|
||||||
Paths:
|
|
||||||
- 'tools/**'
|
|
||||||
- eng/common/**
|
|
||||||
- .github/copilot-instructions.md
|
|
||||||
|
|
||||||
- task: AzureCLI@2
|
|
||||||
displayName: 'Run eval'
|
|
||||||
inputs:
|
|
||||||
azureSubscription: opensource-api-connection
|
|
||||||
scriptType: 'bash'
|
|
||||||
scriptLocation: 'inlineScript'
|
|
||||||
workingDirectory: '${{ parameters.ToolsRepoName }}/${{ parameters.EvalProject }}'
|
|
||||||
inlineScript: |
|
|
||||||
echo "Logged in to Azure"
|
|
||||||
echo "Running eval in project ${{ parameters.ToolsRepoName }}/${{ parameters.EvalProject }}"
|
|
||||||
dotnet test /p:ArtifactsPackagesDir=$(Build.ArtifactStagingDirectory) --logger trx
|
|
||||||
env:
|
|
||||||
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
|
|
||||||
DOTNET_CLI_TELEMETRY_OPTOUT: 1
|
|
||||||
DOTNET_MULTILEVEL_LOOKUP: 0
|
|
||||||
AZURE_OPENAI_MODEL_DEPLOYMENT_NAME: ${{ parameters.Model }}
|
|
||||||
AZURE_OPENAI_ENDPOINT: ${{ parameters.OpenAIEndPoint }}
|
|
||||||
REPOSITORY_NAME: $(Build.Repository.Name)
|
|
||||||
COPILOT_INSTRUCTIONS_PATH_MCP_EVALS: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/.github/copilot-instructions.md
|
|
||||||
|
|
||||||
- task: PublishTestResults@2
|
|
||||||
condition: succeededOrFailed()
|
|
||||||
inputs:
|
|
||||||
testResultsFiles: '**/*.trx'
|
|
||||||
testRunTitle: $(System.JobDisplayName)
|
|
||||||
testResultsFormat: 'VSTest'
|
|
||||||
mergeTestResults: true
|
|
||||||
100
eng/common/pipelines/templates/jobs/ai-eval-job.yml
Normal file
100
eng/common/pipelines/templates/jobs/ai-eval-job.yml
Normal file
@ -0,0 +1,100 @@
|
|||||||
|
parameters:
|
||||||
|
- name: EvalProject
|
||||||
|
type: string
|
||||||
|
default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations'
|
||||||
|
- name: OpenAIEndPoint
|
||||||
|
type: string
|
||||||
|
default: 'https://openai-shared.openai.azure.com/'
|
||||||
|
- name: Model
|
||||||
|
type: string
|
||||||
|
default: 'gpt-5'
|
||||||
|
# Repository where evals are located
|
||||||
|
- name: EvalRepoOwner
|
||||||
|
type: string
|
||||||
|
default: 'Azure'
|
||||||
|
- name: EvalRepoName
|
||||||
|
type: string
|
||||||
|
default: 'azure-sdk-tools'
|
||||||
|
- name: EvalRepoCommit
|
||||||
|
type: string
|
||||||
|
default: 'main'
|
||||||
|
# Target repository for copilot instructions
|
||||||
|
- name: TargetRepoOwner
|
||||||
|
type: string
|
||||||
|
- name: TargetRepoName
|
||||||
|
type: string
|
||||||
|
- name: TargetRepoCommit
|
||||||
|
type: string
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
- job: Run_Eval
|
||||||
|
variables:
|
||||||
|
- template: /eng/pipelines/templates/variables/globals.yml
|
||||||
|
- template: /eng/pipelines/templates/variables/image.yml
|
||||||
|
displayName: 'Run AI Eval'
|
||||||
|
pool:
|
||||||
|
name: $(LINUXPOOL)
|
||||||
|
image: $(LINUXVMIMAGE)
|
||||||
|
os: linux
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- checkout: none
|
||||||
|
- task: UseDotNet@2
|
||||||
|
displayName: "Use .NET SDK 9.0.x"
|
||||||
|
retryCountOnTaskFailure: 3
|
||||||
|
inputs:
|
||||||
|
packageType: sdk
|
||||||
|
version: 9.0.x
|
||||||
|
performMultiLevelLookup: true
|
||||||
|
|
||||||
|
- task: UseDotNet@2
|
||||||
|
displayName: "Use .NET SDK 8.0.x"
|
||||||
|
retryCountOnTaskFailure: 3
|
||||||
|
inputs:
|
||||||
|
packageType: sdk
|
||||||
|
version: 8.0.x
|
||||||
|
performMultiLevelLookup: true
|
||||||
|
|
||||||
|
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
|
||||||
|
parameters:
|
||||||
|
SkipCheckoutNone: true
|
||||||
|
Repositories:
|
||||||
|
- Name: ${{ parameters.EvalRepoOwner }}/${{ parameters.EvalRepoName }}
|
||||||
|
Commitish: ${{ parameters.EvalRepoCommit }}
|
||||||
|
WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.EvalRepoName }}
|
||||||
|
- Name: ${{ parameters.TargetRepoOwner }}/${{ parameters.TargetRepoName }}
|
||||||
|
Commitish: ${{ parameters.TargetRepoCommit }}
|
||||||
|
WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.TargetRepoName }}
|
||||||
|
Paths:
|
||||||
|
- 'tools/**'
|
||||||
|
- 'eng/common/**'
|
||||||
|
- '.github/copilot-instructions.md'
|
||||||
|
|
||||||
|
- task: AzureCLI@2
|
||||||
|
displayName: 'Run eval'
|
||||||
|
condition: and(succeeded(), ne(variables['Skip.Eval'], 'true'))
|
||||||
|
inputs:
|
||||||
|
azureSubscription: opensource-api-connection
|
||||||
|
scriptType: 'bash'
|
||||||
|
scriptLocation: 'inlineScript'
|
||||||
|
workingDirectory: '$(System.DefaultWorkingDirectory)/${{ parameters.EvalRepoName }}/${{ parameters.EvalProject }}'
|
||||||
|
inlineScript: |
|
||||||
|
echo "Logged in to Azure"
|
||||||
|
echo "Running eval"
|
||||||
|
dotnet test --logger trx
|
||||||
|
env:
|
||||||
|
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
|
||||||
|
DOTNET_CLI_TELEMETRY_OPTOUT: 1
|
||||||
|
DOTNET_MULTILEVEL_LOOKUP: 0
|
||||||
|
AZURE_OPENAI_MODEL_DEPLOYMENT_NAME: ${{ parameters.Model }}
|
||||||
|
AZURE_OPENAI_ENDPOINT: ${{ parameters.OpenAIEndPoint }}
|
||||||
|
REPOSITORY_NAME: ${{ parameters.TargetRepoName }}
|
||||||
|
COPILOT_INSTRUCTIONS_PATH_MCP_EVALS: $(System.DefaultWorkingDirectory)/${{ parameters.TargetRepoName }}/.github/copilot-instructions.md
|
||||||
|
|
||||||
|
- task: PublishTestResults@2
|
||||||
|
condition: succeededOrFailed()
|
||||||
|
inputs:
|
||||||
|
testResultsFiles: '**/*.trx'
|
||||||
|
testRunTitle: $(System.JobDisplayName)
|
||||||
|
testResultsFormat: 'VSTest'
|
||||||
|
mergeTestResults: true
|
||||||
Loading…
Reference in New Issue
Block a user