From 0f37994e44e7cb69058ebf98a78849061a0b0596 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Fri, 17 Jul 2020 11:16:18 +0800 Subject: [PATCH] add perf benchmark code (#319) * Add multithread perf testcase * add large scale upload testcase * Fix compiler error on GCC 9 * Fix failed testcase --- sdk/storage/test/CMakeLists.txt | 1 + sdk/storage/test/blobs/large_scale_test.cpp | 61 ++++++++++++++ .../test/blobs/performance_benchmark.cpp | 82 +++++++++++++++++-- sdk/storage/test/test_base.cpp | 6 +- 4 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 sdk/storage/test/blobs/large_scale_test.cpp diff --git a/sdk/storage/test/CMakeLists.txt b/sdk/storage/test/CMakeLists.txt index 4018ecc70..d0b8e2a65 100644 --- a/sdk/storage/test/CMakeLists.txt +++ b/sdk/storage/test/CMakeLists.txt @@ -17,6 +17,7 @@ add_executable ( blobs/page_blob_client_test.hpp blobs/page_blob_client_test.cpp blobs/performance_benchmark.cpp + blobs/large_scale_test.cpp datalake/service_client_test.hpp datalake/service_client_test.cpp datalake/file_system_client_test.hpp diff --git a/sdk/storage/test/blobs/large_scale_test.cpp b/sdk/storage/test/blobs/large_scale_test.cpp new file mode 100644 index 000000000..83c215980 --- /dev/null +++ b/sdk/storage/test/blobs/large_scale_test.cpp @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "blobs/blob.hpp" +#include "common/file_io.hpp" +#include "test_base.hpp" + +#include + +namespace Azure { namespace Storage { namespace Test { + + TEST(BlobsLargeScaleTest, DISABLED_LargeScaleUpload) + { + const std::string containerName = "large-scale-test"; + auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( + StandardStorageConnectionString(), containerName); + try + { + containerClient.Create(); + } + catch (StorageError& e) + { + if (e.StatusCode != Azure::Core::Http::HttpStatusCode::Conflict) + { + throw; + } + } + auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( + StandardStorageConnectionString(), containerName, "LargeScale" + RandomString()); + + const std::string sourceFile = ""; + std::size_t fileSize = 0; + constexpr std::size_t concurrency = 16; + + ASSERT_FALSE(sourceFile.empty()); + { + Details::FileReader reader(sourceFile); + fileSize = static_cast(reader.GetFileSize()); + } + + Blobs::UploadBlobOptions options; + options.Concurrency = concurrency; + auto timer_start = std::chrono::steady_clock::now(); + try + { + auto res = blockBlobClient.UploadFromFile(sourceFile, options); + } + catch (std::exception& e) + { + std::cout << e.what() << std::endl; + } + auto timer_end = std::chrono::steady_clock::now(); + + double speed = static_cast(fileSize) / 1_MB + / std::chrono::duration_cast(timer_end - timer_start).count() + * 1000; + std::cout << "Upload " << static_cast(fileSize) / 1_GB << "GiB, speed " << speed + << "MiB/s" << std::endl; + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/test/blobs/performance_benchmark.cpp b/sdk/storage/test/blobs/performance_benchmark.cpp index 5c076e561..b53d931c4 100644 --- a/sdk/storage/test/blobs/performance_benchmark.cpp +++ b/sdk/storage/test/blobs/performance_benchmark.cpp @@ -1,22 +1,26 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // SPDX-License-Identifier: MIT -#include "block_blob_client_test.hpp" +#include "blob_container_client_test.hpp" #include +#include +#include namespace Azure { namespace Storage { namespace Test { - TEST_F(BlockBlobClientTest, DISABLED_SingleThreadPerf) + TEST_F(BlobContainerClientTest, DISABLED_SingleThreadPerf) { auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName, RandomString()); + StandardStorageConnectionString(), m_containerName, "SingleThreadPerf" + RandomString()); constexpr std::size_t bufferSize = static_cast(1_GB); std::vector buffer = RandomBuffer(bufferSize); { + Blobs::UploadBlobOptions options; + options.ChunkSize = 8_MB; auto timer_start = std::chrono::steady_clock::now(); - auto res = blockBlobClient.UploadFromBuffer(buffer.data(), buffer.size()); + auto res = blockBlobClient.UploadFromBuffer(buffer.data(), buffer.size(), options); auto timer_end = std::chrono::steady_clock::now(); double speed = static_cast(bufferSize) / 1_MB @@ -25,8 +29,11 @@ namespace Azure { namespace Storage { namespace Test { std::cout << "Upload speed: " << speed << "MiB/s" << std::endl; } { + Blobs::DownloadBlobToBufferOptions options; + options.InitialChunkSize = 8_MB; + options.ChunkSize = 8_MB; auto timer_start = std::chrono::steady_clock::now(); - auto res = blockBlobClient.DownloadToBuffer(buffer.data(), buffer.size()); + auto res = blockBlobClient.DownloadToBuffer(buffer.data(), buffer.size(), options); auto timer_end = std::chrono::steady_clock::now(); double speed = static_cast(bufferSize) / 1_MB @@ -36,4 +43,69 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(BlobContainerClientTest, DISABLED_MultiThreadPerf) + { + constexpr int concurrency = 64; + std::vector blockBlobClients; + for (int i = 0; i < concurrency; ++i) + { + blockBlobClients.emplace_back( + Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( + StandardStorageConnectionString(), + m_containerName, + "MultiThreadPerf" + RandomString())); + } + + constexpr std::size_t bufferSize = static_cast(1_GB); + std::vector buffer = RandomBuffer(bufferSize); + + { + std::vector> futures; + Blobs::UploadBlobOptions options; + options.ChunkSize = 8_MB; + auto timer_start = std::chrono::steady_clock::now(); + for (int i = 0; i < concurrency; ++i) + { + futures.emplace_back( + std::async(std::launch::async, [&blockBlobClients, &buffer, &options, i]() { + auto res + = blockBlobClients[i].UploadFromBuffer(buffer.data(), buffer.size(), options); + })); + } + for (auto& f : futures) + { + f.get(); + } + auto timer_end = std::chrono::steady_clock::now(); + double speed = static_cast(bufferSize) * concurrency / 1_MB + / std::chrono::duration_cast(timer_end - timer_start).count() + * 1000; + std::cout << "Upload speed: " << speed << "MiB/s" << std::endl; + } + { + std::vector> futures; + Blobs::DownloadBlobToBufferOptions options; + options.InitialChunkSize = 8_MB; + options.ChunkSize = 8_MB; + auto timer_start = std::chrono::steady_clock::now(); + for (int i = 0; i < concurrency; ++i) + { + futures.emplace_back( + std::async(std::launch::async, [&blockBlobClients, &buffer, &options, i]() { + auto res + = blockBlobClients[i].DownloadToBuffer(buffer.data(), buffer.size(), options); + })); + } + for (auto& f : futures) + { + f.get(); + } + auto timer_end = std::chrono::steady_clock::now(); + double speed = static_cast(bufferSize) * concurrency / 1_MB + / std::chrono::duration_cast(timer_end - timer_start).count() + * 1000; + std::cout << "Download speed: " << speed << "MiB/s" << std::endl; + } + } + }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/test/test_base.cpp b/sdk/storage/test/test_base.cpp index 2b4d14d0d..5e37e1ac1 100644 --- a/sdk/storage/test/test_base.cpp +++ b/sdk/storage/test/test_base.cpp @@ -156,7 +156,11 @@ namespace Azure { namespace Storage { namespace Test { int64_t fileSize = ftell(fin); std::vector fileContent(static_cast(fileSize)); fseek(fin, 0, SEEK_SET); - fread(fileContent.data(), static_cast(fileSize), 1, fin); + std::size_t elementsRead = fread(fileContent.data(), static_cast(fileSize), 1, fin); + if (elementsRead != 1 && fileSize != 0) + { + throw std::runtime_error("failed to read file"); + } fclose(fin); return fileContent; }