Upload blob perf test + RandomStream class for utility (#2433)

* random stream

* upload blob perf test

* upload perf blob test

* undo

sdk/core/perf/CMakeLists.txt

@@ -14,6 +14,7 @@ set(
   inc/azure/perf/dynamic_test_options.hpp
   inc/azure/perf/options.hpp
   inc/azure/perf/program.hpp
+  inc/azure/perf/random_stream.hpp
   inc/azure/perf/test_metadata.hpp
   inc/azure/perf/test.hpp
   inc/azure/perf/test_options.hpp
@@ -24,6 +25,7 @@ set(
   src/arg_parser.cpp
   src/options.cpp
   src/program.cpp
+  src/random_stream.cpp
 )
 
 add_library(azure-perf ${AZURE_PERFORMANCE_HEADER} ${AZURE_PERFORMANCE_SOURCE})
@@ -42,7 +44,7 @@ endif()
 
 # make sure that users can consume the project as a library.
 add_library (Azure::Perf ALIAS azure-perf)
-target_link_libraries(azure-perf PRIVATE azure-core)
+target_link_libraries(azure-perf PUBLIC azure-core)
 
 set_target_properties(azure-perf PROPERTIES FOLDER "Core")
 

sdk/core/perf/inc/azure/perf/random_stream.hpp

@@ -0,0 +1,53 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+/**
+ * @file
+ * @brief A random stream of any specific size. Useful for test cases.
+ *
+ */
+
+#pragma once
+
+#include <azure/core/io/body_stream.hpp>
+
+#include <memory>
+#include <random>
+
+namespace Azure { namespace Perf {
+
+  /**
+   * @brief A random stream of any specific size. Useful for test cases.
+   *
+   */
+  class RandomStream {
+  private:
+    /**
+     * @brief Wraps a stream and keep reading bytes from it by rewinding it until some length.
+     *
+     * @note Enables to create a stream with huge size by re-using a small buffer.
+     *
+     */
+    class CircularStream : public Azure::Core::IO::BodyStream {
+    private:
+      std::unique_ptr<std::vector<uint8_t>> m_buffer;
+      size_t m_length;
+      size_t m_totalRead = 0;
+      Azure::Core::IO::MemoryBodyStream m_memoryStream;
+
+      size_t OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) override;
+
+    public:
+      CircularStream(size_t size);
+
+      int64_t Length() const override { return this->m_length; }
+      void Rewind() override { m_totalRead = 0; }
+    };
+
+  public:
+    static std::unique_ptr<Azure::Core::IO::BodyStream> Create(size_t size)
+    {
+      return std::make_unique<CircularStream>(size);
+    }
+  };
+}} // namespace Azure::Perf

sdk/core/perf/src/random_stream.cpp

@@ -0,0 +1,83 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+#include <azure/core/platform.hpp>
+
+#include "azure/perf/random_stream.hpp"
+
+#if defined(AZ_PLATFORM_WINDOWS)
+#if !defined(WIN32_LEAN_AND_MEAN)
+#define WIN32_LEAN_AND_MEAN
+#endif
+#if !defined(NOMINMAX)
+#define NOMINMAX
+#endif
+#endif
+
+static thread_local std::mt19937_64 random_generator(std::random_device{}());
+
+namespace {
+
+static uint8_t RandomChar()
+{
+  const uint8_t charset[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+  std::uniform_int_distribution<size_t> distribution(0, sizeof(charset) - 2);
+  return charset[distribution(random_generator)];
+}
+
+void RandomBuffer(uint8_t* buffer, size_t length)
+{
+  uint8_t* start_addr = buffer;
+  uint8_t* end_addr = buffer + length;
+
+  const size_t rand_int_size = sizeof(uint64_t);
+  while (uintptr_t(start_addr) % rand_int_size != 0 && start_addr < end_addr)
+  {
+    *(start_addr++) = RandomChar();
+  }
+
+  std::uniform_int_distribution<uint64_t> distribution(0ULL, std::numeric_limits<uint64_t>::max());
+  while (start_addr + rand_int_size <= end_addr)
+  {
+    *reinterpret_cast<uint64_t*>(start_addr) = distribution(random_generator);
+    start_addr += rand_int_size;
+  }
+  while (start_addr < end_addr)
+  {
+    *(start_addr++) = RandomChar();
+  }
+}
+
+} // namespace
+
+Azure::Perf::RandomStream::CircularStream::CircularStream(size_t size)
+    : m_buffer(std::make_unique<std::vector<uint8_t>>(1024 * 1024)), m_length(size),
+      m_memoryStream(*m_buffer)
+{
+  RandomBuffer(m_buffer->data(), 1024 * 1024);
+}
+
+size_t Azure::Perf::RandomStream::CircularStream::OnRead(
+    uint8_t* buffer,
+    size_t count,
+    Azure::Core::Context const& context)
+{
+  size_t available = m_length - m_totalRead;
+  if (available == 0)
+  {
+    return 0;
+  }
+
+  size_t toRead = std::min(count, available);
+  auto read = m_memoryStream.Read(buffer, toRead, context);
+
+  // Circurl implementation. Rewind the stream every time we reach the end
+  if (read == 0) // No more bytes to read from.
+  {
+    m_memoryStream.Rewind();
+    read = m_memoryStream.Read(buffer, toRead, context);
+  }
+
+  m_totalRead += read;
+  return read;
+}

sdk/core/perf/test/CMakeLists.txt

@@ -40,3 +40,19 @@ target_include_directories(
 target_link_libraries(azure-perf-test PRIVATE azure-core azure-perf)
 # Make sure the project will appear in the test folder for Visual Studio CMake view
 set_target_properties(azure-perf-test PROPERTIES FOLDER "Tests/Core")
+
+# Unit tests
+include(GoogleTest)
+
+add_executable (
+  azure-perf-unit-test
+    src/random_stream_test.cpp
+)
+
+target_link_libraries(azure-perf-unit-test PRIVATE azure-perf gtest gtest_main)
+
+gtest_discover_tests(azure-perf-unit-test
+     TEST_PREFIX azure-perf-unittest.
+     NO_PRETTY_TYPES
+     NO_PRETTY_VALUES)
+

sdk/core/perf/test/src/random_stream_test.cpp

@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+#include <gtest/gtest.h>
+
+#include <azure/perf/random_stream.hpp>
+
+TEST(circular_stream, basic)
+{
+  size_t const totalSize = 1024 * 1024 * 3; // should give 5 loops
+  size_t const chunk = 1024 * 1024;
+  auto r_stream = Azure::Perf::RandomStream::Create(totalSize);
+  uint8_t buffer[chunk];
+  uint8_t buffer2[chunk];
+
+  // 1st read
+  auto count = r_stream->Read(buffer, chunk, Azure::Core::Context::ApplicationContext);
+  EXPECT_EQ(count, chunk);
+
+  // 2nd read
+  count = r_stream->Read(buffer2, chunk, Azure::Core::Context::ApplicationContext);
+  EXPECT_EQ(count, chunk);
+  for (size_t i = 0; i != chunk; i++)
+  {
+    EXPECT_EQ(buffer[i], buffer2[i]);
+  }
+
+  // 3nd read
+  count = r_stream->Read(buffer, chunk, Azure::Core::Context::ApplicationContext);
+  EXPECT_EQ(count, chunk);
+  for (size_t i = 0; i != chunk; i++)
+  {
+    EXPECT_EQ(buffer[i], buffer2[i]);
+  }
+
+  // 4nd read
+  count = r_stream->Read(buffer, chunk, Azure::Core::Context::ApplicationContext);
+  EXPECT_EQ(count, 0);
+  // should not change buffer
+  for (size_t i = 0; i != chunk; i++)
+  {
+    EXPECT_EQ(buffer[i], buffer2[i]);
+  }
+}

sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/download_blob_test.hpp

@@ -37,7 +37,7 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test {
     DownloadBlob(Azure::Perf::TestOptions options) : BlobsTest(options) {}
 
     /**
-     * @brief Upload 5Mb to be downloaded in the test
+     * @brief The size to upload on setup is defined by a mandatory parameter.
      *
      */
     void Setup() override

sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/upload_blob_test.hpp

@@ -0,0 +1,88 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+/**
+ * @file
+ * @brief Test the performance of uploading a block blob.
+ *
+ */
+
+#pragma once
+
+#include <azure/core/io/body_stream.hpp>
+#include <azure/perf.hpp>
+#include <azure/perf/random_stream.hpp>
+
+#include "azure/storage/blobs/test/blob_base_test.hpp"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace Azure { namespace Storage { namespace Blobs { namespace Test {
+
+  /**
+   * @brief A test to measure uploading a blob.
+   *
+   */
+  class UploadBlob : public Azure::Storage::Blobs::Test::BlobsTest {
+  private:
+    // C++ can upload and download from contiguos memory or file only
+    std::vector<uint8_t> m_uploadBuffer;
+
+  public:
+    /**
+     * @brief Construct a new UploadBlob test.
+     *
+     * @param options The test options.
+     */
+    UploadBlob(Azure::Perf::TestOptions options) : BlobsTest(options) {}
+
+    /**
+     * @brief The size to upload on setup is defined by a mandatory parameter.
+     *
+     */
+    void Setup() override
+    {
+      // Call base to create blob client
+      BlobsTest::Setup();
+
+      long size = m_options.GetMandatoryOption<long>("Size");
+      m_uploadBuffer = Azure::Perf::RandomStream::Create(size)->ReadToEnd(
+          Azure::Core::Context::ApplicationContext);
+    }
+
+    /**
+     * @brief Define the test
+     *
+     */
+    void Run(Azure::Core::Context const&) override
+    {
+      m_blobClient->UploadFrom(m_uploadBuffer.data(), m_uploadBuffer.size());
+    }
+
+    /**
+     * @brief Define the test options for the test.
+     *
+     * @return The list of test options.
+     */
+    std::vector<Azure::Perf::TestOption> GetTestOptions() override
+    {
+      // TODO: Merge with base options
+      return {{"Size", {"--size", "-s"}, "Size of payload (in bytes)", 1, true}};
+    }
+
+    /**
+     * @brief Get the static Test Metadata for the test.
+     *
+     * @return Azure::Perf::TestMetadata describing the test.
+     */
+    static Azure::Perf::TestMetadata GetTestMetadata()
+    {
+      return {"UploadBlob", "Upload a blob.", [](Azure::Perf::TestOptions options) {
+                return std::make_unique<Azure::Storage::Blobs::Test::UploadBlob>(options);
+              }};
+    }
+  };
+
+}}}} // namespace Azure::Storage::Blobs::Test

sdk/storage/azure-storage-blobs/test/perf/src/azure_storage_blobs_perf_test.cpp

@@ -4,13 +4,15 @@
 #include <azure/perf.hpp>
 
 #include "azure/storage/blobs/test/download_blob_test.hpp"
+#include "azure/storage/blobs/test/upload_blob_test.hpp"
 
 int main(int argc, char** argv)
 {
 
   // Create the test list
   std::vector<Azure::Perf::TestMetadata> tests{
-      Azure::Storage::Blobs::Test::DownloadBlob::GetTestMetadata()};
+      Azure::Storage::Blobs::Test::DownloadBlob::GetTestMetadata(),
+      Azure::Storage::Blobs::Test::UploadBlob::GetTestMetadata()};
 
   Azure::Perf::Program::Run(Azure::Core::Context::ApplicationContext, tests, argc, argv);