github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improved discoverability of tests; handle exceptions thrown during test execution (#4021)

Browse Source 
* Improved discoverability of tests on WIndows; handle exceptions thrown during test execution better
This commit is contained in:
Larry Osterman 2022-10-13 14:06:05 -07:00 committed by GitHub
parent 083a88e884
commit e488a81a03
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 246 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
CMakeSettings.json
View File

@ -290,7 +290,7 @@
"configurationType": "Debug",
"buildRoot": "${projectDir}\\out\\build\\${name}",
"installRoot": "${projectDir}\\out\\install\\${name}",
"cmakeCommandArgs": "-DINSTALL_GTEST=OFF -DBUILD_TESTING=ON -DBUILD_TRANSPORT_CURL=ON -DBUILD_SAMPLES=ON -DBUILD_PERFORMANCE_TESTS=ON",
"cmakeCommandArgs": "",
"buildCommandArgs": "-v",
"inheritEnvironments": [ "msvc_x64_x64" ],
"variables": [
@ -299,6 +299,88 @@
"value": "x64-windows-static",
"type": "STRING"
},
{
"name": "INSTALL_GTEST",
"value": "False",
"type": "BOOL"
},
{
"name": "BUILD_TESTING",
"value": "True",
"type": "BOOL"
},
{
"name": "BUILD_SAMPLES",
"value": "True",
"type": "BOOL"
},
{
"name": "BUILD_PERFORMANCE_TESTS",
"value": "True",
"type": "BOOL"
},
{
"name": "BUILD_TRANSPORT_WINHTTP",
"value": "True",
"type": "BOOL"
},
{
"name": "BUILD_TRANSPORT_CURL",
"value": "True",
"type": "BOOL"
},
{
"name": "MSVC_USE_STATIC_CRT",
"value": "True",
"type": "BOOL"
}
]
},
{
"name": "x64-ReleaseWithPerfTest",
"generator": "Ninja",
"configurationType": "RelWithDebInfo",
"buildRoot": "${projectDir}\\out\\build\\${name}",
"installRoot": "${projectDir}\\out\\install\\${name}",
"cmakeCommandArgs": "",
"buildCommandArgs": "-v",
"inheritEnvironments": [ "msvc_x64_x64" ],
"variables": [
{
"name": "VCPKG_TARGET_TRIPLET",
"value": "x64-windows-static",
"type": "STRING"
},
{
"name": "INSTALL_GTEST",
"value": "False",
"type": "BOOL"
},
{
"name": "BUILD_TESTING",
"value": "True",
"type": "BOOL"
},
{
"name": "BUILD_SAMPLES",
"value": "True",
"type": "BOOL"
},
{
"name": "BUILD_PERFORMANCE_TESTS",
"value": "True",
"type": "BOOL"
},
{
"name": "BUILD_TRANSPORT_WINHTTP",
"value": "True",
"type": "BOOL"
},
{
"name": "BUILD_TRANSPORT_CURL",
"value": "True",
"type": "BOOL"
},
{
"name": "MSVC_USE_STATIC_CRT",
"value": "True",

1
sdk/core/perf/README.md
View File

@ -46,6 +46,7 @@ usage: azure-perf-test testName [options]
>Note: You can use the option `-h` to print out the available options for a test name.
The next options can be used for any test:
| Option | Activators | Description | Default | Example |
| ---------- | --- | ---| ---| --- |
| Duration | -d, --duration | Duration of the test in seconds | 10 | -d 5

4
sdk/core/perf/inc/azure/perf/argagg.hpp
View File

@ -1436,7 +1436,7 @@ inline fmt_ostream::~fmt_ostream() { output << fmt_string(this->str()); }
inline std::string lstrip(const std::string& text)
{
auto result = text;
std::string result = text;
result.erase(result.begin(), std::find_if(result.begin(), result.end(), [](int ch) {
return !std::isspace(ch);
@ -1447,7 +1447,7 @@ inline std::string lstrip(const std::string& text)
inline std::string rstrip(const std::string& text)
{
auto result = text;
std::string result = text;
result.erase(
std::find_if(result.rbegin(), result.rend(), [](int ch) { return !std::isspace(ch); }).base(),

4
sdk/core/perf/src/arg_parser.cpp
View File

@ -18,12 +18,12 @@ argagg::parser_results Azure::Perf::Program::ArgParser::Parse(
// Option Name, Activate options, display message and number of expected args.
argagg::parser argParser;
auto optionsMetadata = Azure::Perf::GlobalTestOptions::GetOptionMetadata();
for (auto option : testOptions)
for (auto const& option : testOptions)
{
argParser.definitions.push_back(
{option.Name, option.Activators, option.DisplayMessage, option.ExpectedArgs});
}
for (auto option : optionsMetadata)
for (auto const& option : optionsMetadata)
{
argParser.definitions.push_back(
{option.Name, option.Activators, option.DisplayMessage, option.ExpectedArgs});

2
sdk/core/perf/src/base_test.cpp
View File

@ -32,7 +32,7 @@ public:
ProxyPolicy(ProxyPolicy const& other) : ProxyPolicy{other.m_testContext} {}
// move
ProxyPolicy(ProxyPolicy&& other) : m_testContext{other.m_testContext} {}
ProxyPolicy(ProxyPolicy&& other) noexcept : m_testContext{other.m_testContext} {}
std::unique_ptr<RawResponse> Send(
Request& request,

78
sdk/core/perf/src/program.cpp
View File

@ -6,23 +6,23 @@
#include <azure/core/internal/json/json.hpp>
#include <azure/core/internal/strings.hpp>
#include <azure/core/platform.hpp>
#include <chrono>
#include <csignal>
#include <iostream>
#include <thread>
namespace {
inline std::unique_ptr<Azure::Perf::PerfTest> PrintAvailableTests(
std::vector<Azure::Perf::TestMetadata> const& tests)
inline void PrintAvailableTests(std::vector<Azure::Perf::TestMetadata> const& tests)
{
std::cout << "No test name found in the input. Available tests to run:" << std::endl;
std::cout << std::endl << "Name\t\tDescription" << std::endl << "---\t\t---" << std::endl;
for (auto test : tests)
for (auto const& test : tests)
{
std::cout << test.Name << "\t\t" << test.Description << std::endl;
}
return nullptr;
}
inline Azure::Perf::TestMetadata const* GetTestMetadata(
@ -37,14 +37,17 @@ inline Azure::Perf::TestMetadata const* GetTestMetadata(
argagg::parser argParser;
auto args = argParser.parse(argc, argv, true);
auto testName = std::string(args.pos[0]);
for (auto& test : tests)
if (!args.pos.empty())
{
if (Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual(
test.Name, testName))
auto testName = std::string(args.pos[0]);
for (auto& test : tests)
{
return &test;
if (Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual(
test.Name, testName))
{
return &test;
}
}
}
return nullptr;
@ -79,12 +82,13 @@ inline void PrintOptions(
{
std::cout << std::endl << "=== Test Options ===" << std::endl;
Azure::Core::Json::_internal::json optionsAsJson;
for (auto option : testOptions)
for (auto const& option : testOptions)
{
try
{
optionsAsJson[option.Name]
= option.SensitiveData ? "***" : parsedArgs[option.Name].as<std::string>();
auto optionName{option.Name};
optionsAsJson[optionName]
= option.SensitiveData ? "***" : parsedArgs[optionName].as<std::string>();
}
catch (std::out_of_range const&)
{
@ -199,14 +203,14 @@ inline void RunTests(
std::vector<std::chrono::nanoseconds> lastCompletionTimes(parallelTestsCount);
/********************* Progress Reporter ******************************/
Azure::Core::Context progresToken;
Azure::Core::Context progressToken;
uint64_t lastCompleted = 0;
auto progressThread = std::thread(
[&title, &completedOperations, &lastCompletionTimes, &lastCompleted, &progresToken]() {
[&title, &completedOperations, &lastCompletionTimes, &lastCompleted, &progressToken]() {
std::cout << std::endl
<< "=== " << title << " ===" << std::endl
<< "Current\t\tTotal\t\tAverage" << std::endl;
while (!progresToken.IsCancelled())
while (!progressToken.IsCancelled())
{
using namespace std::chrono_literals;
std::this_thread::sleep_for(1000ms);
@ -228,7 +232,6 @@ inline void RunTests(
bool isCancelled = false;
// Azure::Context is not good performer for checking cancellation inside the test loop
auto manualCancellation = std::thread([&deadLineSeconds, &isCancelled] {
using namespace std::chrono_literals;
std::this_thread::sleep_for(deadLineSeconds);
isCancelled = true;
});
@ -251,7 +254,7 @@ inline void RunTests(
}
// Stop progress
progresToken.Cancel();
progressToken.Cancel();
progressThread.join();
std::cout << std::endl << "=== Results ===";
@ -278,6 +281,26 @@ void Azure::Perf::Program::Run(
int argc,
char** argv)
{
// Ensure that all calls to abort() no longer pop up a modal dialog on Windows.
#if defined(_DEBUG) && defined(_MSC_VER)
_CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_DEBUG);
#endif
// Declare a signal handler to report unhandled exceptions on Windows - this is not needed for other
// OS's as they will print the exception to stderr in their terminate() function.
#if defined(AZ_PLATFORM_WINDOWS)
signal(SIGABRT, [](int) {
try
{
throw;
}
catch (std::exception const& ex)
{
std::cout << "Exception thrown: " << ex.what() << std::endl;
}
});
#endif // AZ_PLATFORM_WINDOWS
// Parse args only to get the test name first
auto testMetadata = GetTestMetadata(tests, argc, argv);
auto const& testGenerator = testMetadata->Factory;
@ -285,8 +308,10 @@ void Azure::Perf::Program::Run(
{
// Wrong input. Print what are the options.
PrintAvailableTests(tests);
return;
}
// Initial test to get it's options, we can use a dummy parser results
argagg::parser_results argResults;
auto test = testGenerator(Azure::Perf::TestOptions(argResults));
@ -351,7 +376,7 @@ void Azure::Perf::Program::Run(
{
if (!options.TestProxies.empty())
{
std::cout << " - Creating test recordgins for each test using test-proxies..." << std::endl;
std::cout << " - Creating test recordings for each test using test-proxies..." << std::endl;
std::cout << " - Enabling test-proxy playback" << std::endl;
}
@ -375,20 +400,13 @@ void Azure::Perf::Program::Run(
/******************** Tests ******************************/
std::string iterationInfo;
try
for (int iteration = 0; iteration < options.Iterations; iteration++)
{
for (int iteration = 0; iteration < options.Iterations; iteration++)
if (iteration > 0)
{
if (iteration > 0)
{
iterationInfo.append(FormatNumber(iteration));
}
RunTests(context, parallelTest, options, "Test" + iterationInfo);
iterationInfo.append(FormatNumber(iteration));
}
}
catch (std::exception const& error)
{
std::cout << "Error: " << error.what();
RunTests(context, parallelTest, options, "Test" + iterationInfo);
}
std::cout << std::endl << "=== Pre-Cleanup ===" << std::endl;

2
sdk/core/perf/test/inc/azure/perf/test/curl_http_client_get_test.hpp
View File

@ -36,7 +36,7 @@ namespace Azure { namespace Perf { namespace Test {
*/
void GlobalSetup() override
{
_detail::HttpClient = std::make_unique<Azure::Core::Http::CurlTransport>();
m_httpClient = std::make_unique<Azure::Core::Http::CurlTransport>();
}
/**

9
sdk/core/perf/test/inc/azure/perf/test/http_client_get_test.hpp
View File

@ -20,10 +20,6 @@
namespace Azure { namespace Perf { namespace Test {
namespace _detail {
static std::unique_ptr<Azure::Core::Http::HttpTransport> HttpClient;
} // namespace _detail
/**
* @brief A performance test that defines a test option.
*
@ -31,6 +27,7 @@ namespace Azure { namespace Perf { namespace Test {
class HttpClientGetTest : public Azure::Perf::PerfTest {
protected:
Azure::Core::Url m_url;
static std::unique_ptr<Azure::Core::Http::HttpTransport> m_httpClient;
public:
/**
@ -57,7 +54,7 @@ namespace Azure { namespace Perf { namespace Test {
void Run(Azure::Core::Context const& ctx) override
{
Azure::Core::Http::Request request(Azure::Core::Http::HttpMethod::Get, m_url);
auto response = _detail::HttpClient->Send(request, ctx);
auto response = m_httpClient->Send(request, ctx);
// Read the body from network
auto bodyStream = response->ExtractBodyStream();
response->SetBody(bodyStream->ReadToEnd(ctx));
@ -74,4 +71,6 @@ namespace Azure { namespace Perf { namespace Test {
}
};
std::unique_ptr<Azure::Core::Http::HttpTransport> HttpClientGetTest::m_httpClient;
}}} // namespace Azure::Perf::Test

99
sdk/core/perf/test/inc/azure/perf/test/http_pipeline_get_test.hpp Normal file
View File

@ -0,0 +1,99 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// SPDX-License-Identifier: MIT
/**
* @file
* @brief Performance test measuring the use of an HTTP pipeline (and optionally test proxy).
*
*/
#pragma once
#include <azure/perf.hpp>
#include <azure/core/http/http.hpp>
#include <azure/core/http/transport.hpp>
#include <azure/core/internal/http/pipeline.hpp>
#include <azure/core/io/body_stream.hpp>
#include <memory>
#include <vector>
namespace Azure { namespace Perf { namespace Test {
/**
* @brief A performance test that defines a test option.
*
*/
class HttpPipelineGetTest : public Azure::Perf::PerfTest {
Azure::Core::Url m_url;
public:
/**
* @brief Construct a new Extended Options Test object.
*
* @param options The command-line parsed options.
*/
HttpPipelineGetTest(Azure::Perf::TestOptions options) : PerfTest(options) {}
/**
* @brief Get and set the URL option
*
*/
void Setup() override
{
m_url = Azure::Core::Url(m_options.GetMandatoryOption<std::string>("url"));
}
/**
* @brief Set up the HTTP client
*
*/
void GlobalSetup() override {}
/**
* @brief Get the static Test Metadata for the test.
*
* @return Azure::Perf::TestMetadata describing the test.
*/
static Azure::Perf::TestMetadata GetTestMetadata()
{
return {
"httpPipelineGet",
"Send an HTTP GET request to a configurable URL using Azure Pipelines.",
[](Azure::Perf::TestOptions options) {
return std::make_unique<Azure::Perf::Test::HttpPipelineGetTest>(options);
}};
}
/**
* @brief Define the test options for the test.
*
* @return The list of test options.
*/
std::vector<Azure::Perf::TestOption> GetTestOptions() override
{
return {{"url", {"--url"}, "Url to send the HTTP request. *Required parameter.", 1, true}};
}
/**
* @brief The test definition
*
* @param ctx The cancellation token.
*/
void Run(Azure::Core::Context const& ctx) override
{
Azure::Core::_internal::ClientOptions clientOptions;
ConfigureClientOptions(clientOptions);
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRequest;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetry;
Azure::Core::Http::_internal::HttpPipeline pipeline(
clientOptions, "PipelineTest", "na", std::move(perRequest), std::move(perRetry));
Azure::Core::Http::Request request(Azure::Core::Http::HttpMethod::Get, m_url);
auto response = pipeline.Send(request, ctx);
response->GetBody();
}
};
}}} // namespace Azure::Perf::Test

2
sdk/core/perf/test/inc/azure/perf/test/win_http_client_get_test.hpp
View File

@ -34,7 +34,7 @@ namespace Azure { namespace Perf { namespace Test {
*/
void GlobalSetup() override
{
_detail::HttpClient = std::make_unique<Azure::Core::Http::WinHttpTransport>();
m_httpClient = std::make_unique<Azure::Core::Http::WinHttpTransport>();
}
/**

5
sdk/core/perf/test/src/perf_test.cpp
View File

@ -5,6 +5,7 @@
#include "azure/perf/test/delay_test.hpp"
#include "azure/perf/test/extended_options_test.hpp"
#include "azure/perf/test/http_pipeline_get_test.hpp"
#if defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER)
#include "azure/perf/test/curl_http_client_get_test.hpp"
#endif
@ -24,7 +25,9 @@ int main(int argc, char** argv)
Azure::Perf::Test::NoOp::GetTestMetadata(),
Azure::Perf::Test::ExtendedOptionsTest::GetTestMetadata(),
Azure::Perf::Test::DelayTest::GetTestMetadata(),
Azure::Perf::Test::ExceptionTest::GetTestMetadata()};
Azure::Perf::Test::ExceptionTest::GetTestMetadata(),
Azure::Perf::Test::HttpPipelineGetTest::GetTestMetadata(),
};
#if defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER)
tests.emplace_back(Azure::Perf::Test::CurlHttpClientGetTest::GetTestMetadata());