From fc50c08b20dc5871b03a036444bd3fd43521c35f Mon Sep 17 00:00:00 2001 From: George Arama <50641385+gearama@users.noreply.github.com> Date: Fri, 21 Oct 2022 09:36:49 -0700 Subject: [PATCH] Storageblob (#4043) * first take * grrrrr * qwqwq * ewqw * eewew * put back * pipeline * Set read-only VCPKG_BINARY_SOURCES_SECRET * Pass InstallLanguageSteps through the perf template * blop pipeline * path * ';l'l; * add support for sync param * --sync flag support * typo * sync * sync to y * for PR * clang Co-authored-by: Daniel Jurek --- .vscode/cspell.json | 1 + cmake-modules/PerfTest.cmake | 11 +++++ eng/pipelines/templates/jobs/perf.yml | 9 ++--- sdk/core/perf.yml | 8 ++++ sdk/core/perf/src/options.cpp | 3 +- .../azure-storage-blobs/perf-resources.bicep | 21 ++++++++++ sdk/storage/azure-storage-blobs/perf.yml | 40 +++++++++++++++++++ .../test/perf/CMakeLists.txt | 7 +++- .../src/azure_storage_blobs_perf_test.cpp | 1 + vcpkg.json | 2 +- 10 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 cmake-modules/PerfTest.cmake create mode 100644 sdk/storage/azure-storage-blobs/perf-resources.bicep create mode 100644 sdk/storage/azure-storage-blobs/perf.yml diff --git a/.vscode/cspell.json b/.vscode/cspell.json index fbd5d8dda..cfbe17a05 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -74,6 +74,7 @@ "HRESULT", "IMDS", "Intel", + "issecret", "itfactor", "iusg", "jepio", diff --git a/cmake-modules/PerfTest.cmake b/cmake-modules/PerfTest.cmake new file mode 100644 index 000000000..001d7eae5 --- /dev/null +++ b/cmake-modules/PerfTest.cmake @@ -0,0 +1,11 @@ +macro(SetPerfDeps PACKAGE VAR_RESULT) + string(TOUPPER ${PACKAGE} SUFFIX) + string(CONCAT VAR_TRIGGER "VCPKG-" ${SUFFIX}) + message(STATUS "trigger name ${VAR_TRIGGER}") + if(DEFINED ENV{${VAR_TRIGGER}}) + find_package(${PACKAGE} $ENV{${VAR_TRIGGER}} EXACT) + add_compile_definitions(${VAR_RESULT}="$ENV{${VAR_TRIGGER}}") + else() + add_compile_definitions(${VAR_RESULT}="source") + endif() +endmacro() diff --git a/eng/pipelines/templates/jobs/perf.yml b/eng/pipelines/templates/jobs/perf.yml index a31dda892..72cf24308 100644 --- a/eng/pipelines/templates/jobs/perf.yml +++ b/eng/pipelines/templates/jobs/perf.yml @@ -23,6 +23,9 @@ parameters: - name: EnvVars type: object default: [] +- name: InstallLanguageSteps + type: object + default: [] extends: template: /eng/common/pipelines/templates/jobs/perf.yml @@ -38,8 +41,4 @@ extends: Iterations: ${{ parameters.Iterations }} AdditionalArguments: ${{ parameters.AdditionalArguments }} EnvVars: ${{ parameters.EnvVars}} - InstallLanguageSteps: - - template: /eng/pipelines/templates/steps/vcpkg.yml - #/eng/pipelines/templates/steps/vcpkg-clone.yml - parameters: - RepoOwner: Microsoft + InstallLanguageSteps: ${{ parameters.InstallLanguageSteps }} diff --git a/sdk/core/perf.yml b/sdk/core/perf.yml index a8494272b..6708e23df 100644 --- a/sdk/core/perf.yml +++ b/sdk/core/perf.yml @@ -30,3 +30,11 @@ extends: Arguments: ${{ parameters.Arguments }} Iterations: ${{ parameters.Iterations }} AdditionalArguments: ${{ parameters.AdditionalArguments }} + InstallLanguageSteps: + - pwsh: | + Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read" + displayName: Set Vcpkg Variables + + EnvVars: + # This is set in the InstallLanguageSteps + VCPKG_BINARY_SOURCES_SECRET: $(VCPKG_BINARY_SOURCES_SECRET) diff --git a/sdk/core/perf/src/options.cpp b/sdk/core/perf/src/options.cpp index ad404fe20..f01693612 100644 --- a/sdk/core/perf/src/options.cpp +++ b/sdk/core/perf/src/options.cpp @@ -85,5 +85,6 @@ std::vector Azure::Perf::GlobalTestOptions::GetOptionMe {"Rate", {"-r", "--rate"}, "Target throughput (ops/sec). Default to no throughput.", 1}, {"Warmup", {"-w", "--warmup"}, "Duration of warmup in seconds. Default to 5 seconds.", 1}, {"TestProxies", {"-x", "--test-proxies"}, "URIs of TestProxy Servers (separated by ';')", 1}, - {"help", {"-h", "--help"}, "Display help information.", 0}}; + {"help", {"-h", "--help"}, "Display help information.", 0}, + {"Sync", {"-y", "--sync"}, "Runs sync version of test, not implemented", 0}}; } diff --git a/sdk/storage/azure-storage-blobs/perf-resources.bicep b/sdk/storage/azure-storage-blobs/perf-resources.bicep new file mode 100644 index 000000000..f05d353d7 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/perf-resources.bicep @@ -0,0 +1,21 @@ +param baseName string = resourceGroup().name +param location string = resourceGroup().location + +resource storageAccount 'Microsoft.Storage/storageAccounts@2019-06-01' = { + name: '${baseName}blob' + location: location + kind: 'BlockBlobStorage' + sku: { + name: 'Premium_LRS' + } +} + +var name = storageAccount.name +var key = storageAccount.listKeys().keys[0].value +var connectionString = 'DefaultEndpointsProtocol=https;AccountName=${name};AccountKey=${key}' + +output AZURE_STORAGE_ACCOUNT_NAME string = name +output AZURE_STORAGE_ACCOUNT_KEY string = key +output AZURE_STORAGE_CONNECTION_STRING string = connectionString +output STANDARD_STORAGE_CONNECTION_STRING string = connectionString +output STORAGE_CONNECTION_STRING string = connectionString diff --git a/sdk/storage/azure-storage-blobs/perf.yml b/sdk/storage/azure-storage-blobs/perf.yml new file mode 100644 index 000000000..46eb0233c --- /dev/null +++ b/sdk/storage/azure-storage-blobs/perf.yml @@ -0,0 +1,40 @@ +parameters: +- name: PackageVersions + displayName: PackageVersions (regex of package versions to run) + type: string + default: '12|source' +- name: Tests + displayName: Tests (regex of tests to run) + type: string + default: '^(download|upload|list-blobs)$' +- name: Arguments + displayName: Arguments (regex of arguments to run) + type: string + default: '(10240)|(10485760)|(1073741824)|(5 )|(500 )|(50000 )' +- name: Iterations + displayName: Iterations (times to run each test) + type: number + default: '5' +- name: AdditionalArguments + displayName: AdditionalArguments (passed to PerfAutomation) + type: string + default: ' ' + +extends: + template: /eng/pipelines/templates/jobs/perf.yml + parameters: + ServiceDirectory: storage/azure-storage-blobs + Services: "^storage-blob$" + PackageVersions: ${{ parameters.PackageVersions }} + Tests: ${{ parameters.Tests }} + Arguments: ${{ parameters.Arguments }} + Iterations: ${{ parameters.Iterations }} + AdditionalArguments: ${{ parameters.AdditionalArguments }} + InstallLanguageSteps: + - pwsh: | + Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read" + displayName: Set Vcpkg Variables + + EnvVars: + # This is set in the InstallLanguageSteps + VCPKG_BINARY_SOURCES_SECRET: $(VCPKG_BINARY_SOURCES_SECRET) diff --git a/sdk/storage/azure-storage-blobs/test/perf/CMakeLists.txt b/sdk/storage/azure-storage-blobs/test/perf/CMakeLists.txt index 73159ded4..63a01d6d7 100644 --- a/sdk/storage/azure-storage-blobs/test/perf/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/test/perf/CMakeLists.txt @@ -6,6 +6,8 @@ cmake_minimum_required (VERSION 3.13) project(azure-storage-blobs-perf LANGUAGES CXX) set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED True) +include(AzureVcpkg) +az_vcpkg_integrate() if(BUILD_TRANSPORT_CURL) set(DOWNLOAD_WITH_LIBCURL inc/azure/storage/blobs/test/download_blob_transport_only.hpp) @@ -35,6 +37,8 @@ add_executable ( create_per_service_target_build(storage azure-storage-blobs-perf) +include(PerfTest) +SETPERFDEPS(azure-storage-blobs-cpp VCPKG_STORAGE_BLOB_VERSION) # Include the headers from the project. target_include_directories( azure-storage-blobs-perf @@ -47,7 +51,8 @@ if (MSVC) target_compile_options(azure-storage-blobs-perf PUBLIC /wd4996) endif() + # link the `azure-perf` lib together with any other library which will be used for the tests. -target_link_libraries(azure-storage-blobs-perf PRIVATE azure-storage-blobs azure-perf) +target_link_libraries(azure-storage-blobs-perf PRIVATE Azure::azure-storage-blobs azure-perf) # Make sure the project will appear in the test folder for Visual Studio CMake view set_target_properties(azure-storage-blobs-perf PROPERTIES FOLDER "Tests/Storage") diff --git a/sdk/storage/azure-storage-blobs/test/perf/src/azure_storage_blobs_perf_test.cpp b/sdk/storage/azure-storage-blobs/test/perf/src/azure_storage_blobs_perf_test.cpp index 04fd4018e..996f4e960 100644 --- a/sdk/storage/azure-storage-blobs/test/perf/src/azure_storage_blobs_perf_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/perf/src/azure_storage_blobs_perf_test.cpp @@ -16,6 +16,7 @@ int main(int argc, char** argv) { + std::cout << "Azure-storage-blobs VERSION " << VCPKG_STORAGE_BLOB_VERSION << std::endl; // Create the test list std::vector tests diff --git a/vcpkg.json b/vcpkg.json index 17fa92fee..b88dcc5b0 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -15,7 +15,7 @@ }, { "name": "opentelemetry-cpp", - "platform": "!uwp", + "platform": "!(windows & !static)", "version>=": "1.3.0" }, {