From da1451586a6ef50bac4e3dead9eba2c094c2d91a Mon Sep 17 00:00:00 2001 From: Victor Vazquez Date: Fri, 18 Jun 2021 10:05:52 -0700 Subject: [PATCH] Upload blob perf test + RandomStream class for utility (#2433) * random stream * upload blob perf test * upload perf blob test * undo --- sdk/core/perf/CMakeLists.txt | 4 +- .../perf/inc/azure/perf/random_stream.hpp | 53 +++++++++++ sdk/core/perf/src/random_stream.cpp | 83 +++++++++++++++++ sdk/core/perf/test/CMakeLists.txt | 16 ++++ sdk/core/perf/test/src/random_stream_test.cpp | 44 ++++++++++ .../storage/blobs/test/download_blob_test.hpp | 2 +- .../storage/blobs/test/upload_blob_test.hpp | 88 +++++++++++++++++++ .../src/azure_storage_blobs_perf_test.cpp | 4 +- 8 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 sdk/core/perf/inc/azure/perf/random_stream.hpp create mode 100644 sdk/core/perf/src/random_stream.cpp create mode 100644 sdk/core/perf/test/src/random_stream_test.cpp create mode 100644 sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/upload_blob_test.hpp diff --git a/sdk/core/perf/CMakeLists.txt b/sdk/core/perf/CMakeLists.txt index 811cb1bce..4ae4155be 100644 --- a/sdk/core/perf/CMakeLists.txt +++ b/sdk/core/perf/CMakeLists.txt @@ -14,6 +14,7 @@ set( inc/azure/perf/dynamic_test_options.hpp inc/azure/perf/options.hpp inc/azure/perf/program.hpp + inc/azure/perf/random_stream.hpp inc/azure/perf/test_metadata.hpp inc/azure/perf/test.hpp inc/azure/perf/test_options.hpp @@ -24,6 +25,7 @@ set( src/arg_parser.cpp src/options.cpp src/program.cpp + src/random_stream.cpp ) add_library(azure-perf ${AZURE_PERFORMANCE_HEADER} ${AZURE_PERFORMANCE_SOURCE}) @@ -42,7 +44,7 @@ endif() # make sure that users can consume the project as a library. add_library (Azure::Perf ALIAS azure-perf) -target_link_libraries(azure-perf PRIVATE azure-core) +target_link_libraries(azure-perf PUBLIC azure-core) set_target_properties(azure-perf PROPERTIES FOLDER "Core") diff --git a/sdk/core/perf/inc/azure/perf/random_stream.hpp b/sdk/core/perf/inc/azure/perf/random_stream.hpp new file mode 100644 index 000000000..e23cdbbdf --- /dev/null +++ b/sdk/core/perf/inc/azure/perf/random_stream.hpp @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @file + * @brief A random stream of any specific size. Useful for test cases. + * + */ + +#pragma once + +#include + +#include +#include + +namespace Azure { namespace Perf { + + /** + * @brief A random stream of any specific size. Useful for test cases. + * + */ + class RandomStream { + private: + /** + * @brief Wraps a stream and keep reading bytes from it by rewinding it until some length. + * + * @note Enables to create a stream with huge size by re-using a small buffer. + * + */ + class CircularStream : public Azure::Core::IO::BodyStream { + private: + std::unique_ptr> m_buffer; + size_t m_length; + size_t m_totalRead = 0; + Azure::Core::IO::MemoryBodyStream m_memoryStream; + + size_t OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) override; + + public: + CircularStream(size_t size); + + int64_t Length() const override { return this->m_length; } + void Rewind() override { m_totalRead = 0; } + }; + + public: + static std::unique_ptr Create(size_t size) + { + return std::make_unique(size); + } + }; +}} // namespace Azure::Perf diff --git a/sdk/core/perf/src/random_stream.cpp b/sdk/core/perf/src/random_stream.cpp new file mode 100644 index 000000000..cea0b9d87 --- /dev/null +++ b/sdk/core/perf/src/random_stream.cpp @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include + +#include "azure/perf/random_stream.hpp" + +#if defined(AZ_PLATFORM_WINDOWS) +#if !defined(WIN32_LEAN_AND_MEAN) +#define WIN32_LEAN_AND_MEAN +#endif +#if !defined(NOMINMAX) +#define NOMINMAX +#endif +#endif + +static thread_local std::mt19937_64 random_generator(std::random_device{}()); + +namespace { + +static uint8_t RandomChar() +{ + const uint8_t charset[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + std::uniform_int_distribution distribution(0, sizeof(charset) - 2); + return charset[distribution(random_generator)]; +} + +void RandomBuffer(uint8_t* buffer, size_t length) +{ + uint8_t* start_addr = buffer; + uint8_t* end_addr = buffer + length; + + const size_t rand_int_size = sizeof(uint64_t); + while (uintptr_t(start_addr) % rand_int_size != 0 && start_addr < end_addr) + { + *(start_addr++) = RandomChar(); + } + + std::uniform_int_distribution distribution(0ULL, std::numeric_limits::max()); + while (start_addr + rand_int_size <= end_addr) + { + *reinterpret_cast(start_addr) = distribution(random_generator); + start_addr += rand_int_size; + } + while (start_addr < end_addr) + { + *(start_addr++) = RandomChar(); + } +} + +} // namespace + +Azure::Perf::RandomStream::CircularStream::CircularStream(size_t size) + : m_buffer(std::make_unique>(1024 * 1024)), m_length(size), + m_memoryStream(*m_buffer) +{ + RandomBuffer(m_buffer->data(), 1024 * 1024); +} + +size_t Azure::Perf::RandomStream::CircularStream::OnRead( + uint8_t* buffer, + size_t count, + Azure::Core::Context const& context) +{ + size_t available = m_length - m_totalRead; + if (available == 0) + { + return 0; + } + + size_t toRead = std::min(count, available); + auto read = m_memoryStream.Read(buffer, toRead, context); + + // Circurl implementation. Rewind the stream every time we reach the end + if (read == 0) // No more bytes to read from. + { + m_memoryStream.Rewind(); + read = m_memoryStream.Read(buffer, toRead, context); + } + + m_totalRead += read; + return read; +} diff --git a/sdk/core/perf/test/CMakeLists.txt b/sdk/core/perf/test/CMakeLists.txt index 0a6ef639a..11ef47f80 100644 --- a/sdk/core/perf/test/CMakeLists.txt +++ b/sdk/core/perf/test/CMakeLists.txt @@ -40,3 +40,19 @@ target_include_directories( target_link_libraries(azure-perf-test PRIVATE azure-core azure-perf) # Make sure the project will appear in the test folder for Visual Studio CMake view set_target_properties(azure-perf-test PROPERTIES FOLDER "Tests/Core") + +# Unit tests +include(GoogleTest) + +add_executable ( + azure-perf-unit-test + src/random_stream_test.cpp +) + +target_link_libraries(azure-perf-unit-test PRIVATE azure-perf gtest gtest_main) + +gtest_discover_tests(azure-perf-unit-test + TEST_PREFIX azure-perf-unittest. + NO_PRETTY_TYPES + NO_PRETTY_VALUES) + diff --git a/sdk/core/perf/test/src/random_stream_test.cpp b/sdk/core/perf/test/src/random_stream_test.cpp new file mode 100644 index 000000000..6b797431b --- /dev/null +++ b/sdk/core/perf/test/src/random_stream_test.cpp @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include + +#include + +TEST(circular_stream, basic) +{ + size_t const totalSize = 1024 * 1024 * 3; // should give 5 loops + size_t const chunk = 1024 * 1024; + auto r_stream = Azure::Perf::RandomStream::Create(totalSize); + uint8_t buffer[chunk]; + uint8_t buffer2[chunk]; + + // 1st read + auto count = r_stream->Read(buffer, chunk, Azure::Core::Context::ApplicationContext); + EXPECT_EQ(count, chunk); + + // 2nd read + count = r_stream->Read(buffer2, chunk, Azure::Core::Context::ApplicationContext); + EXPECT_EQ(count, chunk); + for (size_t i = 0; i != chunk; i++) + { + EXPECT_EQ(buffer[i], buffer2[i]); + } + + // 3nd read + count = r_stream->Read(buffer, chunk, Azure::Core::Context::ApplicationContext); + EXPECT_EQ(count, chunk); + for (size_t i = 0; i != chunk; i++) + { + EXPECT_EQ(buffer[i], buffer2[i]); + } + + // 4nd read + count = r_stream->Read(buffer, chunk, Azure::Core::Context::ApplicationContext); + EXPECT_EQ(count, 0); + // should not change buffer + for (size_t i = 0; i != chunk; i++) + { + EXPECT_EQ(buffer[i], buffer2[i]); + } +} diff --git a/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/download_blob_test.hpp b/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/download_blob_test.hpp index 524ae82a3..f7d5b40ca 100644 --- a/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/download_blob_test.hpp +++ b/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/download_blob_test.hpp @@ -37,7 +37,7 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test { DownloadBlob(Azure::Perf::TestOptions options) : BlobsTest(options) {} /** - * @brief Upload 5Mb to be downloaded in the test + * @brief The size to upload on setup is defined by a mandatory parameter. * */ void Setup() override diff --git a/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/upload_blob_test.hpp b/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/upload_blob_test.hpp new file mode 100644 index 000000000..bcca1b4bc --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/upload_blob_test.hpp @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @file + * @brief Test the performance of uploading a block blob. + * + */ + +#pragma once + +#include +#include +#include + +#include "azure/storage/blobs/test/blob_base_test.hpp" + +#include +#include +#include + +namespace Azure { namespace Storage { namespace Blobs { namespace Test { + + /** + * @brief A test to measure uploading a blob. + * + */ + class UploadBlob : public Azure::Storage::Blobs::Test::BlobsTest { + private: + // C++ can upload and download from contiguos memory or file only + std::vector m_uploadBuffer; + + public: + /** + * @brief Construct a new UploadBlob test. + * + * @param options The test options. + */ + UploadBlob(Azure::Perf::TestOptions options) : BlobsTest(options) {} + + /** + * @brief The size to upload on setup is defined by a mandatory parameter. + * + */ + void Setup() override + { + // Call base to create blob client + BlobsTest::Setup(); + + long size = m_options.GetMandatoryOption("Size"); + m_uploadBuffer = Azure::Perf::RandomStream::Create(size)->ReadToEnd( + Azure::Core::Context::ApplicationContext); + } + + /** + * @brief Define the test + * + */ + void Run(Azure::Core::Context const&) override + { + m_blobClient->UploadFrom(m_uploadBuffer.data(), m_uploadBuffer.size()); + } + + /** + * @brief Define the test options for the test. + * + * @return The list of test options. + */ + std::vector GetTestOptions() override + { + // TODO: Merge with base options + return {{"Size", {"--size", "-s"}, "Size of payload (in bytes)", 1, true}}; + } + + /** + * @brief Get the static Test Metadata for the test. + * + * @return Azure::Perf::TestMetadata describing the test. + */ + static Azure::Perf::TestMetadata GetTestMetadata() + { + return {"UploadBlob", "Upload a blob.", [](Azure::Perf::TestOptions options) { + return std::make_unique(options); + }}; + } + }; + +}}}} // namespace Azure::Storage::Blobs::Test diff --git a/sdk/storage/azure-storage-blobs/test/perf/src/azure_storage_blobs_perf_test.cpp b/sdk/storage/azure-storage-blobs/test/perf/src/azure_storage_blobs_perf_test.cpp index f02f110dd..ee78db23e 100644 --- a/sdk/storage/azure-storage-blobs/test/perf/src/azure_storage_blobs_perf_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/perf/src/azure_storage_blobs_perf_test.cpp @@ -4,13 +4,15 @@ #include #include "azure/storage/blobs/test/download_blob_test.hpp" +#include "azure/storage/blobs/test/upload_blob_test.hpp" int main(int argc, char** argv) { // Create the test list std::vector tests{ - Azure::Storage::Blobs::Test::DownloadBlob::GetTestMetadata()}; + Azure::Storage::Blobs::Test::DownloadBlob::GetTestMetadata(), + Azure::Storage::Blobs::Test::UploadBlob::GetTestMetadata()}; Azure::Perf::Program::Run(Azure::Core::Context::ApplicationContext, tests, argc, argv);