From b7e7e4e0008c0d8146f762531dcfeea55da990dd Mon Sep 17 00:00:00 2001 From: George Arama <50641385+gearama@users.noreply.github.com> Date: Tue, 17 Jan 2023 10:53:55 -0800 Subject: [PATCH] Kv perf pipeline (#4222) * secrets * perftestst for get Secret * dss * qfe * pipelines * comments * cleanup script * comment * key last * somemore stuff * clang --- CMakeSettings.json | 2 +- .../perf.yml | 40 ++++++ .../test/get_certificate_test.hpp | 2 +- .../azure-security-keyvault-keys/perf.yml | 40 ++++++ .../CMakeLists.txt | 2 +- .../azure-security-keyvault-secrets/perf.yml | 40 ++++++ .../test/perf/CMakeLists.txt | 38 +++++ .../keyvault/secrets/test/get_secret_test.hpp | 126 +++++++++++++++++ ...re_security_keyvault_secrets_perf_test.cpp | 18 +++ sdk/keyvault/cleanup/CMakeLists.txt | 16 +++ sdk/keyvault/cleanup/cleanup.cpp | 132 ++++++++++++++++++ 11 files changed, 453 insertions(+), 3 deletions(-) create mode 100644 sdk/keyvault/azure-security-keyvault-certificates/perf.yml create mode 100644 sdk/keyvault/azure-security-keyvault-keys/perf.yml create mode 100644 sdk/keyvault/azure-security-keyvault-secrets/perf.yml create mode 100644 sdk/keyvault/azure-security-keyvault-secrets/test/perf/CMakeLists.txt create mode 100644 sdk/keyvault/azure-security-keyvault-secrets/test/perf/inc/azure/keyvault/secrets/test/get_secret_test.hpp create mode 100644 sdk/keyvault/azure-security-keyvault-secrets/test/perf/src/azure_security_keyvault_secrets_perf_test.cpp create mode 100644 sdk/keyvault/cleanup/CMakeLists.txt create mode 100644 sdk/keyvault/cleanup/cleanup.cpp diff --git a/CMakeSettings.json b/CMakeSettings.json index b242dbc1a..f6173b0ff 100644 --- a/CMakeSettings.json +++ b/CMakeSettings.json @@ -7,7 +7,7 @@ "inheritEnvironments": [ "msvc_x64_x64" ], "buildRoot": "${projectDir}\\out\\build\\${name}", "installRoot": "${projectDir}\\out\\install\\${name}", - "cmakeCommandArgs": "-DINSTALL_GTEST=OFF -DBUILD_TESTING=ON -DBUILD_TRANSPORT_CURL=ON -DBUILD_SAMPLES=ON", + "cmakeCommandArgs": "-DINSTALL_GTEST=OFF -DBUILD_TESTING=ON -DBUILD_TRANSPORT_CURL=ON -DBUILD_SAMPLES=ON -DBUILD_PERFORMANCE_TESTS=ON", "buildCommandArgs": "-v", "ctestCommandArgs": "", "variables": [ diff --git a/sdk/keyvault/azure-security-keyvault-certificates/perf.yml b/sdk/keyvault/azure-security-keyvault-certificates/perf.yml new file mode 100644 index 000000000..55662e233 --- /dev/null +++ b/sdk/keyvault/azure-security-keyvault-certificates/perf.yml @@ -0,0 +1,40 @@ +parameters: +- name: PackageVersions + displayName: PackageVersions (regex of package versions to run) + type: string + default: '1|source' +- name: Tests + displayName: Tests (regex of tests to run) + type: string + default: '^(get-certificate)$' +- name: Arguments + displayName: Arguments (regex of arguments to run) + type: string + default: '.*' +- name: Iterations + displayName: Iterations (times to run each test) + type: number + default: '5' +- name: AdditionalArguments + displayName: AdditionalArguments (passed to PerfAutomation) + type: string + default: ' ' + +extends: + template: /eng/pipelines/templates/jobs/perf.yml + parameters: + ServiceDirectory: keyvault + Services: "^keyvault-certificates$" + PackageVersions: ${{ parameters.PackageVersions }} + Tests: ${{ parameters.Tests }} + Arguments: ${{ parameters.Arguments }} + Iterations: ${{ parameters.Iterations }} + AdditionalArguments: ${{ parameters.AdditionalArguments }} + InstallLanguageSteps: + - pwsh: | + Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read" + displayName: Set Vcpkg Variables + + EnvVars: + # This is set in the InstallLanguageSteps + VCPKG_BINARY_SOURCES_SECRET: $(VCPKG_BINARY_SOURCES_SECRET) diff --git a/sdk/keyvault/azure-security-keyvault-certificates/test/perf/inc/azure/keyvault/certificates/test/get_certificate_test.hpp b/sdk/keyvault/azure-security-keyvault-certificates/test/perf/inc/azure/keyvault/certificates/test/get_certificate_test.hpp index 4cfb141dd..fc48c3dae 100644 --- a/sdk/keyvault/azure-security-keyvault-certificates/test/perf/inc/azure/keyvault/certificates/test/get_certificate_test.hpp +++ b/sdk/keyvault/azure-security-keyvault-certificates/test/perf/inc/azure/keyvault/certificates/test/get_certificate_test.hpp @@ -28,7 +28,7 @@ namespace Azure { namespace Test { /** - * @brief A test to measure getting a key performance. + * @brief A test to measure getting a certificate performance. * */ class GetCertificate : public Azure::Perf::PerfTest { diff --git a/sdk/keyvault/azure-security-keyvault-keys/perf.yml b/sdk/keyvault/azure-security-keyvault-keys/perf.yml new file mode 100644 index 000000000..51d0f4dc3 --- /dev/null +++ b/sdk/keyvault/azure-security-keyvault-keys/perf.yml @@ -0,0 +1,40 @@ +parameters: +- name: PackageVersions + displayName: PackageVersions (regex of package versions to run) + type: string + default: '1|source' +- name: Tests + displayName: Tests (regex of tests to run) + type: string + default: '^(get-key)$' +- name: Arguments + displayName: Arguments (regex of arguments to run) + type: string + default: '.*' +- name: Iterations + displayName: Iterations (times to run each test) + type: number + default: '5' +- name: AdditionalArguments + displayName: AdditionalArguments (passed to PerfAutomation) + type: string + default: ' ' + +extends: + template: /eng/pipelines/templates/jobs/perf.yml + parameters: + ServiceDirectory: keyvault + Services: "^keyvault-keys$" + PackageVersions: ${{ parameters.PackageVersions }} + Tests: ${{ parameters.Tests }} + Arguments: ${{ parameters.Arguments }} + Iterations: ${{ parameters.Iterations }} + AdditionalArguments: ${{ parameters.AdditionalArguments }} + InstallLanguageSteps: + - pwsh: | + Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read" + displayName: Set Vcpkg Variables + + EnvVars: + # This is set in the InstallLanguageSteps + VCPKG_BINARY_SOURCES_SECRET: $(VCPKG_BINARY_SOURCES_SECRET) diff --git a/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt b/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt index 2020482a1..633be18b5 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt +++ b/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt @@ -114,7 +114,7 @@ if(BUILD_TESTING) endif() if (BUILD_PERFORMANCE_TESTS) - #add_subdirectory(test/perf) + add_subdirectory(test/perf) endif() if(BUILD_SAMPLES) diff --git a/sdk/keyvault/azure-security-keyvault-secrets/perf.yml b/sdk/keyvault/azure-security-keyvault-secrets/perf.yml new file mode 100644 index 000000000..df7c82139 --- /dev/null +++ b/sdk/keyvault/azure-security-keyvault-secrets/perf.yml @@ -0,0 +1,40 @@ +parameters: +- name: PackageVersions + displayName: PackageVersions (regex of package versions to run) + type: string + default: '1|source' +- name: Tests + displayName: Tests (regex of tests to run) + type: string + default: '^(get-secret|list-secrets)$' +- name: Arguments + displayName: Arguments (regex of arguments to run) + type: string + default: '.*' +- name: Iterations + displayName: Iterations (times to run each test) + type: number + default: '5' +- name: AdditionalArguments + displayName: AdditionalArguments (passed to PerfAutomation) + type: string + default: ' ' + +extends: + template: /eng/pipelines/templates/jobs/perf.yml + parameters: + ServiceDirectory: keyvault + Services: "^keyvault-secrets$" + PackageVersions: ${{ parameters.PackageVersions }} + Tests: ${{ parameters.Tests }} + Arguments: ${{ parameters.Arguments }} + Iterations: ${{ parameters.Iterations }} + AdditionalArguments: ${{ parameters.AdditionalArguments }} + InstallLanguageSteps: + - pwsh: | + Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read" + displayName: Set Vcpkg Variables + + EnvVars: + # This is set in the InstallLanguageSteps + VCPKG_BINARY_SOURCES_SECRET: $(VCPKG_BINARY_SOURCES_SECRET) diff --git a/sdk/keyvault/azure-security-keyvault-secrets/test/perf/CMakeLists.txt b/sdk/keyvault/azure-security-keyvault-secrets/test/perf/CMakeLists.txt new file mode 100644 index 000000000..cfd8104fa --- /dev/null +++ b/sdk/keyvault/azure-security-keyvault-secrets/test/perf/CMakeLists.txt @@ -0,0 +1,38 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT + +# Configure CMake project. +cmake_minimum_required (VERSION 3.13) +project(azure-security-keyvault-secrets-perf LANGUAGES CXX) +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +set( + AZURE_KEYVAULT_SECRETS_PERF_TEST_HEADER + inc/azure/keyvault/secrets/test/get_secret_test.hpp +) + +set( + AZURE_KEYVAULT_SECRETS_PERF_TEST_SOURCE + src/azure_security_keyvault_secrets_perf_test.cpp +) + +# Name the binary to be created. +add_executable ( + azure-security-keyvault-secrets-perf + ${AZURE_KEYVAULT_SECRETS_PERF_TEST_HEADER} ${AZURE_KEYVAULT_SECRETS_PERF_TEST_SOURCE} +) +create_per_service_target_build(keyvault azure-security-keyvault-secrets-perf) +create_map_file(azure-security-keyvault-secrets-perf azure-security-keyvault-secrets-perf.map) + +# Include the headers from the project. +target_include_directories( + azure-security-keyvault-secrets-perf + PUBLIC + $ +) + +# link the `azure-perf` lib together with any other library which will be used for the tests. +target_link_libraries(azure-security-keyvault-secrets-perf PRIVATE azure-identity azure-security-keyvault-secrets azure-perf) +# Make sure the project will appear in the test folder for Visual Studio CMake view +set_target_properties(azure-security-keyvault-secrets-perf PROPERTIES FOLDER "Tests/Keyvault") diff --git a/sdk/keyvault/azure-security-keyvault-secrets/test/perf/inc/azure/keyvault/secrets/test/get_secret_test.hpp b/sdk/keyvault/azure-security-keyvault-secrets/test/perf/inc/azure/keyvault/secrets/test/get_secret_test.hpp new file mode 100644 index 000000000..d16919311 --- /dev/null +++ b/sdk/keyvault/azure-security-keyvault-secrets/test/perf/inc/azure/keyvault/secrets/test/get_secret_test.hpp @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @file + * @brief Test the overhead of getting a secret. + * + */ + +#pragma once + +#include + +#include +#include +#include + +#include +#include +#include + +using namespace Azure::Core::_internal; +namespace Azure { namespace Security { namespace KeyVault { namespace Secrets { namespace Test { + + /** + * @brief A test to measure getting a secret performance. + * + */ + class GetSecret : public Azure::Perf::PerfTest { + private: + std::string m_vaultUrl; + std::string m_secretName; + std::string m_tenantId; + std::string m_clientId; + std::string m_secret; + std::shared_ptr m_credential; + std::unique_ptr m_client; + + public: + /** + * @brief Get the Ids and secret + * + */ + void Setup() override + { + m_vaultUrl = m_options.GetOptionOrDefault( + "vaultUrl", Environment::GetVariable("AZURE_KEYVAULT_URL")); + m_tenantId = m_options.GetOptionOrDefault( + "TenantId", Environment::GetVariable("AZURE_TENANT_ID")); + m_clientId = m_options.GetOptionOrDefault( + "ClientId", Environment::GetVariable("AZURE_CLIENT_ID")); + m_secret = m_options.GetOptionOrDefault( + "Secret", Environment::GetVariable("AZURE_CLIENT_SECRET")); + m_credential = std::make_shared( + m_tenantId, m_clientId, m_secret); + m_client = std::make_unique( + m_vaultUrl, + m_credential, + InitClientOptions()); + this->CreateRandomNameKey(); + } + + /** + * @brief Create a random named secret. + * + */ + void CreateRandomNameKey() + { + std::string name("perf"); + int suffixLen = 10; + static const char alphanum[] + = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + std::string suffix; + suffix.reserve(suffixLen); + + for (int i = 0; i < suffixLen; ++i) + { + suffix += alphanum[rand() % (sizeof(alphanum) - 1)]; + } + + m_secretName = name + suffix; + + auto secretResponse = m_client->SetSecret(m_secretName, "secretValue"); + } + + /** + * @brief Construct a new GetSecret test. + * + * @param options The test options. + */ + GetSecret(Azure::Perf::TestOptions options) : PerfTest(options) {} + + /** + * @brief Define the test + * + */ + void Run(Azure::Core::Context const&) override { auto t = m_client->GetSecret(m_secretName); } + + /** + * @brief Define the test options for the test. + * + * @return The list of test options. + */ + std::vector GetTestOptions() override + { + return { + {"vaultUrl", {"--vaultUrl"}, "The Key Vault Account.", 1, false}, + {"TenantId", {"--tenantId"}, "The tenant Id for the authentication.", 1, false}, + {"ClientId", {"--clientId"}, "The client Id for the authentication.", 1, false}, + {"Secret", {"--secret"}, "The secret for authentication.", 1, false, true}}; + } + + /** + * @brief Get the static Test Metadata for the test. + * + * @return Azure::Perf::TestMetadata describing the test. + */ + static Azure::Perf::TestMetadata GetTestMetadata() + { + return {"GetSecret", "Get a secret", [](Azure::Perf::TestOptions options) { + return std::make_unique( + options); + }}; + } + }; +}}}}} // namespace Azure::Security::KeyVault::Secrets::Test diff --git a/sdk/keyvault/azure-security-keyvault-secrets/test/perf/src/azure_security_keyvault_secrets_perf_test.cpp b/sdk/keyvault/azure-security-keyvault-secrets/test/perf/src/azure_security_keyvault_secrets_perf_test.cpp new file mode 100644 index 000000000..75d4376cf --- /dev/null +++ b/sdk/keyvault/azure-security-keyvault-secrets/test/perf/src/azure_security_keyvault_secrets_perf_test.cpp @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include + +#include "azure/keyvault/secrets/test/get_secret_test.hpp" + +int main(int argc, char** argv) +{ + + // Create the test list + std::vector tests{ + Azure::Security::KeyVault::Secrets::Test::GetSecret::GetTestMetadata()}; + + Azure::Perf::Program::Run(Azure::Core::Context::ApplicationContext, tests, argc, argv); + + return 0; +} diff --git a/sdk/keyvault/cleanup/CMakeLists.txt b/sdk/keyvault/cleanup/CMakeLists.txt new file mode 100644 index 000000000..46379193d --- /dev/null +++ b/sdk/keyvault/cleanup/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT + +cmake_minimum_required (VERSION 3.13) + +project (cleanup-keyvault LANGUAGES CXX) +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +add_executable ( + cleanup-keyvault + cleanup.cpp +) +create_per_service_target_build_for_sample(keyvault cleanup-keyvault) + +target_link_libraries(cleanup-keyvault PRIVATE azure-security-keyvault-secrets azure-security-keyvault-keys azure-security-keyvault-certificates azure-identity get-env-helper) diff --git a/sdk/keyvault/cleanup/cleanup.cpp b/sdk/keyvault/cleanup/cleanup.cpp new file mode 100644 index 000000000..d9c5ba96a --- /dev/null +++ b/sdk/keyvault/cleanup/cleanup.cpp @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @brief This sample provides the code implementation to use the Key Vault Secrets SDK client for + * C++ to create, get, update, delete and purge a secret. + * + * @remark The following environment variables must be set before running the sample. + * - AZURE_KEYVAULT_URL: To the Key Vault account URL. + * - AZURE_TENANT_ID: Tenant ID for the Azure account. + * - AZURE_CLIENT_ID: The Client ID to authenticate the request. + * - AZURE_CLIENT_SECRET: The client secret. + * + */ + +#include "get_env.hpp" + +#include +#include +#include +#include +#include +#include + +using namespace Azure::Security::KeyVault::Secrets; +using namespace Azure::Security::KeyVault::Certificates; +using namespace Azure::Security::KeyVault::Keys; +using namespace std::chrono_literals; + +int main() +{ + auto tenantId = std::getenv("AZURE_TENANT_ID"); + auto clientId = std::getenv("AZURE_CLIENT_ID"); + auto clientSecret = std::getenv("AZURE_CLIENT_SECRET"); + auto credential + = std::make_shared(tenantId, clientId, clientSecret); + + SecretClient secretClient(std::getenv("AZURE_KEYVAULT_URL"), credential); + KeyClient keyClient(std::getenv("AZURE_KEYVAULT_URL"), credential); + CertificateClient certClient(std::getenv("AZURE_KEYVAULT_URL"), credential); + + try + { + std::vector certOps; + std::vector keyOps; + std::vector secretOps; + + for (auto secrets = secretClient.GetPropertiesOfSecrets(); secrets.HasPage(); + secrets.MoveToNextPage()) + { + for (auto const& secret : secrets.Items) + { + try + { + secretOps.push_back(secretClient.StartDeleteSecret(secret.Name)); + std::cout << "DeleteSecret" << secret.Name << std::endl; + } + catch (...) + { + std::cout << "fail to delete secret " << secret.Name; + } + } + } + + for (auto certificates = certClient.GetPropertiesOfCertificates(); certificates.HasPage(); + certificates.MoveToNextPage()) + { + for (auto const& certificate : certificates.Items) + { + try + { + certOps.push_back(certClient.StartDeleteCertificate(certificate.Name)); + std::cout << "Delete Certificate" << certificate.Name << std::endl; + } + catch (...) + { + std::cout << "fail to delete cert " << certificate.Name; + } + } + } + + for (auto keys = keyClient.GetPropertiesOfKeys(); keys.HasPage(); keys.MoveToNextPage()) + { + for (auto const& key : keys.Items) + { + try + { + keyOps.push_back(keyClient.StartDeleteKey(key.Name)); + std::cout << "DeleteKey" << key.Name << std::endl; + } + catch (...) + { + std::cout << "fail to delete key " << key.Name; + } + } + } + + for (auto op : certOps) + { + op.PollUntilDone(1s); + certClient.PurgeDeletedCertificate(op.Value().Name()); + std::cout << "Purge cert " << op.Value().Name() << std::endl; + } + + for (auto op : secretOps) + { + op.PollUntilDone(1s); + secretClient.PurgeDeletedSecret(op.Value().Name); + std::cout << "Purge secret " << op.Value().Name << std::endl; + } + + for (auto op : keyOps) + { + op.PollUntilDone(1s); + keyClient.PurgeDeletedKey(op.Value().Name()); + std::cout << "Purge Key " << op.Value().Name() << std::endl; + } + } + catch (Azure::Core::Credentials::AuthenticationException const& e) + { + std::cout << "Authentication Exception happened:" << std::endl << e.what() << std::endl; + return 1; + } + catch (Azure::Core::RequestFailedException const& e) + { + std::cout << "Key Vault Secret Client Exception happened:" << std::endl + << e.Message << std::endl; + return 1; + } + + return 0; +}