From 5dff0dad92e47b4ca973797aa0139253587ef741 Mon Sep 17 00:00:00 2001 From: Danny Amirault Date: Wed, 18 Mar 2020 13:32:30 -0700 Subject: [PATCH] Initial CI pipeline work (#36) * Replicate C pipeline files to CPP * Adjust to build CPP * Remove pipeline pieces copied from C that are not needed for C++ build. * Remove extraneous debugging step. --- eng/pipelines/client.yml | 23 +++ eng/pipelines/doc-index.yml | 59 ++++++ .../templates/jobs/archetype-sdk-client.yml | 177 ++++++++++++++++++ .../stages/archetype-cpp-release.yml | 67 +++++++ .../templates/stages/archetype-sdk-client.yml | 22 +++ sdk/core/azure-core/version.txt | 1 + sdk/core/ci.yml | 24 +++ 7 files changed, 373 insertions(+) create mode 100644 eng/pipelines/client.yml create mode 100644 eng/pipelines/doc-index.yml create mode 100644 eng/pipelines/templates/jobs/archetype-sdk-client.yml create mode 100644 eng/pipelines/templates/stages/archetype-cpp-release.yml create mode 100644 eng/pipelines/templates/stages/archetype-sdk-client.yml create mode 100644 sdk/core/azure-core/version.txt create mode 100644 sdk/core/ci.yml diff --git a/eng/pipelines/client.yml b/eng/pipelines/client.yml new file mode 100644 index 000000000..c12d2927e --- /dev/null +++ b/eng/pipelines/client.yml @@ -0,0 +1,23 @@ +# DO NOT EDIT THIS FILE +# This file is generated automatically and any changes will be lost. + +trigger: + branches: + include: + - master + - feature/* + - release/* + - hotfix/* + +pr: + branches: + include: + - master + - feature/* + - release/* + - hotfix/* + +jobs: + - template: ./templates/jobs/archetype-sdk-client.yml + parameters: + BuildReleaseArtifacts: false diff --git a/eng/pipelines/doc-index.yml b/eng/pipelines/doc-index.yml new file mode 100644 index 000000000..fc9b7439b --- /dev/null +++ b/eng/pipelines/doc-index.yml @@ -0,0 +1,59 @@ +resources: + repositories: + - repository: azure-sdk-tools + type: github + name: azure/azure-sdk-tools + endpoint: azure + +jobs: + - job: GenerateDocIndex + pool: + vmImage: windows-2019 + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: '3.6' + + - template: eng/pipelines/templates/scripts/replace-relative-links.yml@azure-sdk-tools + parameters: + TargetFolder: '.' + RootFolder: '.' + BuildSHA: $(Build.SourceVersion) + RepoId: 'Azure/azure-sdk-for-cpp' + + - pwsh: | + Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://github.com/dotnet/docfx/releases/download/v2.43.2/docfx.zip" ` + -OutFile "docfx.zip" | Wait-Process; Expand-Archive -Path "docfx.zip" -DestinationPath "./docfx/" + workingDirectory: $(Build.SourcesDirectory) + displayName: Download and Extract DocFX + + - pwsh: >- + $(Build.SourcesDirectory)/eng/docs/index/Generate-DocIndex.ps1 + -RepoRoot $(Build.SourcesDirectory) + -DocGenDir "$(Build.SourcesDirectory)/eng/docs/index" + -verbose + displayName: 'Generate Doc Index' + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: '3.6' + + - template: eng/pipelines/templates/scripts/mashup-doc-index.yml@azure-sdk-tools + parameters: + SourceDirectory: $(Build.SourcesDirectory) + + - task: CopyFiles@2 + displayName: Copy HTML to Artifacts Directory + inputs: + sourceFolder: $(Build.SourcesDirectory)/docfx_project/ + content: '**\*' + targetFolder: $(Build.ArtifactStagingDirectory)/docfx_project + overWrite: true + + - task: PublishPipelineArtifact@0 + condition: succeeded() + inputs: + artifactName: "Doc.Index" + targetPath: $(Build.ArtifactStagingDirectory)/docfx_project/_site \ No newline at end of file diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml new file mode 100644 index 000000000..228284918 --- /dev/null +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -0,0 +1,177 @@ +parameters: + Artifacts: [] + ServiceDirectory: not-specified + BuildReleaseArtifacts: true + +jobs: +- job: Validate + strategy: + matrix: + Linux_x64: + vm.image: 'ubuntu-18.04' + vcpkg.deps: '' + VCPKG_DEFAULT_TRIPLET: 'x64-linux' + Win_x86: + vm.image: 'windows-2019' + vcpkg.deps: '' + VCPKG_DEFAULT_TRIPLET: 'x86-windows-static' + CMAKE_GENERATOR: 'Visual Studio 16 2019' + CMAKE_GENERATOR_PLATFORM: Win32 + Win_x64: + vm.image: 'windows-2019' + vcpkg.deps: '' + VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' + CMAKE_GENERATOR: 'Visual Studio 16 2019' + CMAKE_GENERATOR_PLATFORM: x64 + MacOS_x64: + vm.image: 'macOS-10.14' + vcpkg.deps: '' + VCPKG_DEFAULT_TRIPLET: 'x64-osx' + + pool: + vmImage: $(vm.image) + variables: + CMOCKA_XML_FILE: "%g-test-results.xml" + CMOCKA_MESSAGE_OUTPUT: "xml" + + steps: + - checkout: self + submodules: recursive + + # Mac OS specific requirements + - task: Bash@3 + inputs: + targetType: inline + script: | + sudo xcode-select --switch /Applications/Xcode_11.3.1.app + echo "xcode path:" + sudo xcode-select --print-path + + # Install gcc 9 + brew install gcc@9 + gcc --version + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + git rev-parse --verify HEAD + git status + + ./bootstrap-vcpkg.sh + + # Validate that vcpkg bootstrap succeeded + ./vcpkg version + if [ $? -ne 0 ] + then + echo "./bootstrap-vcpkg.sh FAILED" + exit 1 + fi + + echo "##vso[task.prependpath]$(pwd)" + echo "##vso[task.setvariable variable=VCPKG_INSTALLATION_ROOT;]$(pwd)" + + # Execute only for Mac and if there is at least one dependency to be installed + condition: and(succeeded(), contains(variables['vm.image'], 'macOS'), not(eq(variables['vcpkg.deps'], ''))) + displayName: vcpkg bootstrap + + + - script: | + vcpkg install $(vcpkg.deps) + displayName: vcpkg install dependencies + # Execute only if there is at least one dependency to be installed + condition: and(succeeded(), not(eq(variables['vcpkg.deps'], ''))) + + - task: CMake@1 + inputs: + cmakeArgs: --version + displayName: cmake version + - task: CMake@1 + inputs: + cmakeArgs: $(build.args) .. + displayName: cmake generate + - task: CMake@1 + inputs: + cmakeArgs: --build . + displayName: cmake build + - script: | + ctest -V + workingDirectory: build + displayName: ctest + +# Disable build for cpp - client +- ${{ if ne(parameters.ServiceDirectory, 'not-specified' )}}: + - job: GenerateReleaseArtifacts + pool: + vmImage: windows-2019 + steps: + # Generate package spec JSON files for tagging the repo + - ${{ each artifact in parameters.Artifacts }}: + - pwsh: >- + New-Item + -ItemType directory + -Path $(Build.ArtifactStagingDirectory) + -Name packages/${{ artifact.Name }} + displayName: Create package info artifact location + + - pwsh: | + $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }}/package-info.json + $version = Get-Content -Path sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/version.txt + $outputObject = @{ version = $version.ToString(); name = '${{ artifact.Name }}' } | ConvertTo-Json + + Set-Content -Path $outputPath -Value $outputObject + displayName: Create package info JSON file + + - pwsh: | + $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }} + $changelogPath = "sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/CHANGELOG.md" + + if (Test-Path $changeLogPath) { + Copy-Item sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/CHANGELOG.md $outputPath + } else { + Write-Warning "$changeLogPath does not exist" + } + displayName: Copy CHANGELOG.md to package artifact + + - task: PublishPipelineArtifact@1 + inputs: + artifactName: packages + path: $(Build.ArtifactStagingDirectory)/packages + + # Generate Documentation + - pwsh: | + $setupFile="doxygen-1.8.15-setup.exe"; + Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://azuresdktooling.blob.core.windows.net/dependencies/doxygen-1.8.15-setup.exe" ` + -OutFile $setupFile | Wait-Process; Start-Process -Filepath .\$setupFile -ArgumentList @("/VERYSILENT") -Wait + workingDirectory: $(Agent.TempDirectory) + displayName: Download and Install Doxygen + + - ${{ each artifact in parameters.Artifacts }}: + - pwsh: Write-Host "##vso[task.setvariable variable=PackageVersion]$(Get-Content .\sdk\${{ parameters.ServiceDirectory }}\${{ artifact.Path }}\version.txt)" + displayName: Set $(PackageVersion) from version.txt file - ${{ artifact.Name }} + - pwsh: >- + python eng/docs/api/generate_docs.py + --TemplateFile="eng/docs/api/Doxyfile.template" + --PackagePath="sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}" + --PackageName="${{ artifact.Name }}" + --PackageVersion="$(PackageVersion)" + --AssetsPath="eng/docs/api/assets" + --DoxygenPath="c:\\program files\\doxygen\\bin\\doxygen.exe" + displayName: Generate docs - ${{ artifact.Name }} + + - pwsh: New-Item -ItemType directory -Path $(Build.ArtifactStagingDirectory) -Name docs/${{ artifact.Name }} + displayName: Create doc artifact location - ${{ artifact.Name }} + + - task: CopyFiles@2 + inputs: + sourceFolder: sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/html + targetFolder: $(Build.ArtifactStagingDirectory)/docs/${{ artifact.Name }} + displayName: Copy documentation - ${{ artifact.Name }} + + - pwsh: | + $packageInfoPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }}/package-info.json + Copy-Item -Path $packageInfoPath -Destination $(Build.ArtifactStagingDirectory)/docs/${{ artifact.Name }}/package-info.json + displayName: Copy package-info.json to documentation path + + - task: PublishPipelineArtifact@1 + inputs: + artifactName: docs + path: $(Build.ArtifactStagingDirectory)/docs diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml new file mode 100644 index 000000000..6ce1fdfe3 --- /dev/null +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -0,0 +1,67 @@ +parameters: + Artifacts: [] + ArtifactName: 'not-specified' + +stages: + - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: + - ${{ each artifact in parameters.Artifacts }}: + - stage: Release_${{artifact.safeName}} + displayName: 'Release ${{artifact.name}}' + dependsOn: Build + condition: and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp-pr')) + + jobs: + - deployment: TagRepository + displayName: "Create release tag" + condition: ne(variables['Skip.TagRepository'], 'true') + environment: github + + pool: + vmImage: windows-2019 + + strategy: + runOnce: + deploy: + steps: + - checkout: none + - template: tools/clone-buildtools/clone-buildtools.yml@azure-sdk-build-tools + - pwsh: Get-ChildItem -Recurse $(Pipeline.Workspace)/packages/${{artifact.Name}} + displayName: Output Visible Artifacts + - pwsh: >- + $(Pipeline.Workspace)/azure-sdk-build-tools/scripts/create-tags-and-git-release.ps1 + -artifactLocation "$(Pipeline.Workspace)/packages/${{artifact.Name}}" + -packageRepository C + -releaseSha $(Build.SourceVersion) + -repoId $(Build.Repository.Name) + displayName: 'Verify Package Tags and Create Git Releases' + timeoutInMinutes: 5 + env: + GH_TOKEN: $(azuresdk-github-pat) + + - ${{if ne(artifact.options.skipPublishDocs, 'true')}}: + - deployment: PublishDocs + displayName: Publish Docs to GitHub pages + condition: ne(variables['Skip.PublishDocs'], 'true') + environment: githubio + + pool: + vmImage: windows-2019 + + strategy: + runOnce: + deploy: + steps: + - checkout: none + - pwsh: Write-Host 'publish docs' + - template: tools/clone-buildtools/clone-buildtools.yml@azure-sdk-build-tools + - pwsh: | + Get-ChildItem -Recurse $(Pipeline.Workspace)/docs/${{ artifact.Name }} + displayName: Output Visible Artifacts + - template: tools/generic-blob-upload/publish-blobs.yml@azure-sdk-build-tools + parameters: + FolderForUpload: '$(Pipeline.Workspace)/docs/${{ artifact.Name }}' + BlobSASKey: '$(azure-sdk-docs-prod-sas)' + BlobName: '$(azure-sdk-docs-prod-blob-name)' + TargetLanguage: 'cpp' + # we override the regular script path because we have cloned the build tools repo as a separate artifact. + ScriptPath: '$(Pipeline.Workspace)/azure-sdk-build-tools/scripts/copy-docs-to-blobstorage.ps1' diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml new file mode 100644 index 000000000..8a52ff2a9 --- /dev/null +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -0,0 +1,22 @@ +parameters: + Artifacts: [] + ServiceDirectory: not-specified + + +stages: + - stage: Build + jobs: + - template: ../jobs/archetype-sdk-client.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + Artifacts: ${{ parameters.Artifacts }} + + - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: + + - template: archetype-cpp-release.yml + parameters: + ServiceDirectory: ${{parameters.ServiceDirectory}} + DependsOn: + - Build + Artifacts: ${{parameters.Artifacts}} + ArtifactName: packages diff --git a/sdk/core/azure-core/version.txt b/sdk/core/azure-core/version.txt new file mode 100644 index 000000000..1b2282f7d --- /dev/null +++ b/sdk/core/azure-core/version.txt @@ -0,0 +1 @@ +1.0.0-dev.1 \ No newline at end of file diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml new file mode 100644 index 000000000..b74f3468b --- /dev/null +++ b/sdk/core/ci.yml @@ -0,0 +1,24 @@ +resources: + repositories: + - repository: azure-sdk-tools + type: github + name: Azure/azure-sdk-tools + endpoint: azure + - repository: azure-sdk-build-tools + type: git + name: internal/azure-sdk-build-tools + +trigger: + branches: + include: + - master + - feature/* + - release/* + - hotfix/* + +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml + parameters: + ServiceDirectory: core + Artifacts: + - Name: azure-core