From b9299478ee8e2efa42ca4f4e187700cfa37ad267 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Thu, 20 Aug 2020 10:54:28 +0800 Subject: [PATCH] [Storage Blob Service] Concurrent upload for Page Blob (#486) --- sdk/storage/inc/blobs/append_blob_client.hpp | 4 +- sdk/storage/inc/blobs/blob_options.hpp | 2 + sdk/storage/inc/blobs/blob_responses.hpp | 7 + sdk/storage/inc/blobs/page_blob_client.hpp | 42 +++++- sdk/storage/src/blobs/append_blob_client.cpp | 4 +- sdk/storage/src/blobs/page_blob_client.cpp | 104 +++++++++++++- .../test/blobs/block_blob_client_test.cpp | 131 ++++++++++-------- .../test/blobs/page_blob_client_test.cpp | 110 +++++++++++++++ 8 files changed, 331 insertions(+), 73 deletions(-) diff --git a/sdk/storage/inc/blobs/append_blob_client.hpp b/sdk/storage/inc/blobs/append_blob_client.hpp index 9df9ccde1..9f716863e 100644 --- a/sdk/storage/inc/blobs/append_blob_client.hpp +++ b/sdk/storage/inc/blobs/append_blob_client.hpp @@ -115,7 +115,7 @@ namespace Azure { namespace Storage { namespace Blobs { * @return A CreateAppendBlobResult describing the newly created append blob. */ Azure::Core::Response Create( - const CreateAppendBlobOptions& options = CreateAppendBlobOptions()); + const CreateAppendBlobOptions& options = CreateAppendBlobOptions()) const; /** * @brief Commits a new block of data, represented by the content BodyStream to the end @@ -129,7 +129,7 @@ namespace Azure { namespace Storage { namespace Blobs { */ Azure::Core::Response AppendBlock( Azure::Core::Http::BodyStream* content, - const AppendBlockOptions& options = AppendBlockOptions()); + const AppendBlockOptions& options = AppendBlockOptions()) const; /** * @brief Commits a new block of data, represented by the content BodyStream to the end diff --git a/sdk/storage/inc/blobs/blob_options.hpp b/sdk/storage/inc/blobs/blob_options.hpp index 16320063b..93d712790 100644 --- a/sdk/storage/inc/blobs/blob_options.hpp +++ b/sdk/storage/inc/blobs/blob_options.hpp @@ -920,6 +920,8 @@ namespace Azure { namespace Storage { namespace Blobs { int Concurrency = 1; }; + using UploadPageBlobFromOptions = UploadBlockBlobFromOptions; + /** * @brief Optional parameters for BlockBlobClient::StageBlock. */ diff --git a/sdk/storage/inc/blobs/blob_responses.hpp b/sdk/storage/inc/blobs/blob_responses.hpp index fa3d51a2c..290582e67 100644 --- a/sdk/storage/inc/blobs/blob_responses.hpp +++ b/sdk/storage/inc/blobs/blob_responses.hpp @@ -27,6 +27,13 @@ namespace Azure { namespace Storage { namespace Blobs { using UploadBlockBlobFromResult = UploadBlockBlobResult; + struct UploadPageBlobFromResult + { + Azure::Core::Nullable ServerEncrypted; + Azure::Core::Nullable EncryptionKeySha256; + Azure::Core::Nullable EncryptionScope; + }; + struct PageRange { int64_t Offset; diff --git a/sdk/storage/inc/blobs/page_blob_client.hpp b/sdk/storage/inc/blobs/page_blob_client.hpp index 17d296011..32dbcdc44 100644 --- a/sdk/storage/inc/blobs/page_blob_client.hpp +++ b/sdk/storage/inc/blobs/page_blob_client.hpp @@ -120,7 +120,7 @@ namespace Azure { namespace Storage { namespace Blobs { */ Azure::Core::Response Create( int64_t blobContentLength, - const CreatePageBlobOptions& options = CreatePageBlobOptions()); + const CreatePageBlobOptions& options = CreatePageBlobOptions()) const; /** * @brief Writes content to a range of pages in a page blob, starting at offset. @@ -135,7 +135,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Response UploadPages( Azure::Core::Http::BodyStream* content, int64_t offset, - const UploadPageBlobPagesOptions& options = UploadPageBlobPagesOptions()); + const UploadPageBlobPagesOptions& options = UploadPageBlobPagesOptions()) const; /** * @brief Writes a range of pages to a page blob where the contents are read from a @@ -160,7 +160,34 @@ namespace Azure { namespace Storage { namespace Blobs { int64_t sourceOffset, int64_t sourceLength, int64_t destinationoffset, - const UploadPageBlobPagesFromUriOptions& options = UploadPageBlobPagesFromUriOptions()); + const UploadPageBlobPagesFromUriOptions& options + = UploadPageBlobPagesFromUriOptions()) const; + + /** + * @brief Creates a new page blob, or updates the content of an existing page blob. Updating + * an existing page blob overwrites any existing metadata on the blob. + * + * @param buffer A memory buffer containing the content to upload. + * @param bufferSize Size of the memory buffer. + * @param options Optional parameters to execute this function. + * @return A UploadPageBlobPagesResult describing the state of the updated page blob. + */ + Azure::Core::Response UploadFrom( + const uint8_t* buffer, + std::size_t bufferSize, + const UploadPageBlobFromOptions& options = UploadPageBlobFromOptions()) const; + + /** + * @brief Creates a new page blob, or updates the content of an existing page blob. Updating + * an existing page blob overwrites any existing metadata on the blob. + * + * @param file A file containing the content to upload. + * @param options Optional parameters to execute this function. + * @return A UploadPageBlobFromResult describing the state of the updated page blob. + */ + Azure::Core::Response UploadFrom( + const std::string& file, + const UploadPageBlobFromOptions& options = UploadPageBlobFromOptions()) const; /** * @brief Clears one or more pages from the page blob, as specificed by offset and length. @@ -175,7 +202,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Response ClearPages( int64_t offset, int64_t length, - const ClearPageBlobPagesOptions& options = ClearPageBlobPagesOptions()); + const ClearPageBlobPagesOptions& options = ClearPageBlobPagesOptions()) const; /** * @brief Resizes the page blob to the specified size (which must be a multiple of 512). If the @@ -189,7 +216,7 @@ namespace Azure { namespace Storage { namespace Blobs { */ Azure::Core::Response Resize( int64_t blobContentLength, - const ResizePageBlobOptions& options = ResizePageBlobOptions()); + const ResizePageBlobOptions& options = ResizePageBlobOptions()) const; /** * @brief Returns the list of valid page ranges for a page blob or snapshot of a page blob. @@ -198,7 +225,7 @@ namespace Azure { namespace Storage { namespace Blobs { * @return A GetPageBlobPageRangesResult describing the valid page ranges for this blob. */ Azure::Core::Response GetPageRanges( - const GetPageBlobPageRangesOptions& options = GetPageBlobPageRangesOptions()); + const GetPageBlobPageRangesOptions& options = GetPageBlobPageRangesOptions()) const; /** * @brief Starts copying a snapshot of the sourceUri page blob to this page blob. The snapshot @@ -213,7 +240,8 @@ namespace Azure { namespace Storage { namespace Blobs { */ Azure::Core::Response StartCopyIncremental( const std::string& sourceUri, - const StartCopyPageBlobIncrementalOptions& options = StartCopyPageBlobIncrementalOptions()); + const StartCopyPageBlobIncrementalOptions& options + = StartCopyPageBlobIncrementalOptions()) const; private: explicit PageBlobClient(BlobClient blobClient); diff --git a/sdk/storage/src/blobs/append_blob_client.cpp b/sdk/storage/src/blobs/append_blob_client.cpp index 106e58242..b659f8ed5 100644 --- a/sdk/storage/src/blobs/append_blob_client.cpp +++ b/sdk/storage/src/blobs/append_blob_client.cpp @@ -73,7 +73,7 @@ namespace Azure { namespace Storage { namespace Blobs { } Azure::Core::Response AppendBlobClient::Create( - const CreateAppendBlobOptions& options) + const CreateAppendBlobOptions& options) const { BlobRestClient::AppendBlob::CreateAppendBlobOptions protocolLayerOptions; protocolLayerOptions.HttpHeaders = options.HttpHeaders; @@ -96,7 +96,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Response AppendBlobClient::AppendBlock( Azure::Core::Http::BodyStream* content, - const AppendBlockOptions& options) + const AppendBlockOptions& options) const { BlobRestClient::AppendBlob::AppendBlockOptions protocolLayerOptions; protocolLayerOptions.ContentMd5 = options.ContentMd5; diff --git a/sdk/storage/src/blobs/page_blob_client.cpp b/sdk/storage/src/blobs/page_blob_client.cpp index e1d62e755..c02f60a01 100644 --- a/sdk/storage/src/blobs/page_blob_client.cpp +++ b/sdk/storage/src/blobs/page_blob_client.cpp @@ -3,7 +3,9 @@ #include "blobs/page_blob_client.hpp" +#include "common/concurrent_transfer.hpp" #include "common/constants.hpp" +#include "common/file_io.hpp" #include "common/storage_common.hpp" namespace Azure { namespace Storage { namespace Blobs { @@ -72,7 +74,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Response PageBlobClient::Create( int64_t blobContentLength, - const CreatePageBlobOptions& options) + const CreatePageBlobOptions& options) const { BlobRestClient::PageBlob::CreatePageBlobOptions protocolLayerOptions; protocolLayerOptions.BlobContentLength = blobContentLength; @@ -99,7 +101,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Response PageBlobClient::UploadPages( Azure::Core::Http::BodyStream* content, int64_t offset, - const UploadPageBlobPagesOptions& options) + const UploadPageBlobPagesOptions& options) const { BlobRestClient::PageBlob::UploadPageBlobPagesOptions protocolLayerOptions; protocolLayerOptions.Range = std::make_pair(offset, offset + content->Length() - 1); @@ -126,7 +128,7 @@ namespace Azure { namespace Storage { namespace Blobs { int64_t sourceOffset, int64_t sourceLength, int64_t destinationoffset, - const UploadPageBlobPagesFromUriOptions& options) + const UploadPageBlobPagesFromUriOptions& options) const { BlobRestClient::PageBlob::UploadPageBlobPagesFromUriOptions protocolLayerOptions; protocolLayerOptions.SourceUri = sourceUri; @@ -155,7 +157,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Response PageBlobClient::ClearPages( int64_t offset, int64_t length, - const ClearPageBlobPagesOptions& options) + const ClearPageBlobPagesOptions& options) const { BlobRestClient::PageBlob::ClearPageBlobPagesOptions protocolLayerOptions; protocolLayerOptions.Range = std::make_pair(offset, offset + length - 1); @@ -175,9 +177,97 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } + Azure::Core::Response PageBlobClient::UploadFrom( + const uint8_t* buffer, + std::size_t bufferSize, + const UploadPageBlobFromOptions& options) const + { + BlobRestClient::PageBlob::CreatePageBlobOptions createOptions; + createOptions.BlobContentLength = bufferSize; + createOptions.HttpHeaders = options.HttpHeaders; + createOptions.Metadata = options.Metadata; + createOptions.Tier = options.Tier; + if (m_customerProvidedKey.HasValue()) + { + createOptions.EncryptionKey = m_customerProvidedKey.GetValue().Key; + createOptions.EncryptionKeySha256 = m_customerProvidedKey.GetValue().KeyHash; + createOptions.EncryptionAlgorithm = m_customerProvidedKey.GetValue().Algorithm; + } + createOptions.EncryptionScope = m_encryptionScope; + auto createResult = BlobRestClient::PageBlob::Create( + options.Context, *m_pipeline, m_blobUrl.ToString(), createOptions); + + constexpr int64_t c_defaultChunkSize = 8 * 1024 * 1024; + int64_t chunkSize + = options.ChunkSize.HasValue() ? options.ChunkSize.GetValue() : c_defaultChunkSize; + + auto uploadPageFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { + unused(chunkId, numChunks); + Azure::Core::Http::MemoryBodyStream contentStream(buffer + offset, length); + UploadPageBlobPagesOptions uploadPagesOptions; + uploadPagesOptions.Context = options.Context; + UploadPages(&contentStream, offset, uploadPagesOptions); + }; + + Details::ConcurrentTransfer(0, bufferSize, chunkSize, options.Concurrency, uploadPageFunc); + + UploadPageBlobFromResult result; + result.ServerEncrypted = createResult->ServerEncrypted; + result.EncryptionKeySha256 = createResult->EncryptionKeySha256; + result.EncryptionScope = createResult->EncryptionScope; + return Azure::Core::Response( + std::move(result), + std::make_unique(std::move(createResult.GetRawResponse()))); + } + + Azure::Core::Response PageBlobClient::UploadFrom( + const std::string& file, + const UploadPageBlobFromOptions& options) const + { + Details::FileReader fileReader(file); + + BlobRestClient::PageBlob::CreatePageBlobOptions createOptions; + createOptions.BlobContentLength = fileReader.GetFileSize(); + createOptions.HttpHeaders = options.HttpHeaders; + createOptions.Metadata = options.Metadata; + createOptions.Tier = options.Tier; + if (m_customerProvidedKey.HasValue()) + { + createOptions.EncryptionKey = m_customerProvidedKey.GetValue().Key; + createOptions.EncryptionKeySha256 = m_customerProvidedKey.GetValue().KeyHash; + createOptions.EncryptionAlgorithm = m_customerProvidedKey.GetValue().Algorithm; + } + createOptions.EncryptionScope = m_encryptionScope; + auto createResult = BlobRestClient::PageBlob::Create( + options.Context, *m_pipeline, m_blobUrl.ToString(), createOptions); + + constexpr int64_t c_defaultChunkSize = 8 * 1024 * 1024; + int64_t chunkSize + = options.ChunkSize.HasValue() ? options.ChunkSize.GetValue() : c_defaultChunkSize; + + auto uploadPageFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { + unused(chunkId, numChunks); + Azure::Core::Http::FileBodyStream contentStream(fileReader.GetHandle(), offset, length); + UploadPageBlobPagesOptions uploadPagesOptions; + uploadPagesOptions.Context = options.Context; + UploadPages(&contentStream, offset, uploadPagesOptions); + }; + + Details::ConcurrentTransfer( + 0, fileReader.GetFileSize(), chunkSize, options.Concurrency, uploadPageFunc); + + UploadPageBlobFromResult result; + result.ServerEncrypted = createResult->ServerEncrypted; + result.EncryptionKeySha256 = createResult->EncryptionKeySha256; + result.EncryptionScope = createResult->EncryptionScope; + return Azure::Core::Response( + std::move(result), + std::make_unique(std::move(createResult.GetRawResponse()))); + } + Azure::Core::Response PageBlobClient::Resize( int64_t blobContentLength, - const ResizePageBlobOptions& options) + const ResizePageBlobOptions& options) const { BlobRestClient::PageBlob::ResizePageBlobOptions protocolLayerOptions; protocolLayerOptions.BlobContentLength = blobContentLength; @@ -198,7 +288,7 @@ namespace Azure { namespace Storage { namespace Blobs { } Azure::Core::Response PageBlobClient::GetPageRanges( - const GetPageBlobPageRangesOptions& options) + const GetPageBlobPageRangesOptions& options) const { BlobRestClient::PageBlob::GetPageBlobPageRangesOptions protocolLayerOptions; protocolLayerOptions.PreviousSnapshot = options.PreviousSnapshot; @@ -236,7 +326,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Response PageBlobClient::StartCopyIncremental( const std::string& sourceUri, - const StartCopyPageBlobIncrementalOptions& options) + const StartCopyPageBlobIncrementalOptions& options) const { BlobRestClient::PageBlob::StartCopyPageBlobIncrementalOptions protocolLayerOptions; protocolLayerOptions.CopySource = sourceUri; diff --git a/sdk/storage/test/blobs/block_blob_client_test.cpp b/sdk/storage/test/blobs/block_blob_client_test.cpp index e59f97577..3f7ab6484 100644 --- a/sdk/storage/test/blobs/block_blob_client_test.cpp +++ b/sdk/storage/test/blobs/block_blob_client_test.cpp @@ -568,68 +568,89 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlockBlobClientTest, ConcurrentUpload) { - std::string tempFilename = RandomString(); + std::vector blobContent = RandomBuffer(static_cast(8_MB)); - auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName, RandomString()); + auto testUploadFromBuffer = [&](int concurrency, int64_t blobSize) { + auto blockBlobClient = m_blobContainerClient->GetBlockBlobClient(RandomString()); + + Azure::Storage::Blobs::UploadBlockBlobFromOptions options; + options.ChunkSize = 1_MB; + options.Concurrency = concurrency; + options.HttpHeaders = m_blobUploadOptions.HttpHeaders; + options.HttpHeaders.ContentMd5.clear(); + options.Metadata = m_blobUploadOptions.Metadata; + options.Tier = m_blobUploadOptions.Tier; + auto res = blockBlobClient.UploadFrom( + blobContent.data(), static_cast(blobSize), options); + EXPECT_FALSE(res->ETag.empty()); + EXPECT_FALSE(res->LastModified.empty()); + auto properties = *blockBlobClient.GetProperties(); + properties.HttpHeaders.ContentMd5.clear(); + EXPECT_EQ(properties.ContentLength, blobSize); + EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); + EXPECT_EQ(properties.Metadata, options.Metadata); + EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue()); + EXPECT_EQ(properties.ETag, res->ETag); + EXPECT_EQ(properties.LastModified, res->LastModified); + std::vector downloadContent(static_cast(blobSize), '\x00'); + blockBlobClient.DownloadTo(downloadContent.data(), static_cast(blobSize)); + EXPECT_EQ( + downloadContent, + std::vector( + blobContent.begin(), blobContent.begin() + static_cast(blobSize))); + }; + + auto testUploadFromFile = [&](int concurrency, int64_t blobSize) { + auto blockBlobClient = m_blobContainerClient->GetBlockBlobClient(RandomString()); + + Azure::Storage::Blobs::UploadBlockBlobFromOptions options; + options.ChunkSize = 1_MB; + options.Concurrency = concurrency; + options.HttpHeaders = m_blobUploadOptions.HttpHeaders; + options.HttpHeaders.ContentMd5.clear(); + options.Metadata = m_blobUploadOptions.Metadata; + options.Tier = m_blobUploadOptions.Tier; + + std::string tempFilename = RandomString(); + { + Azure::Storage::Details::FileWriter fileWriter(tempFilename); + fileWriter.Write(blobContent.data(), blobSize, 0); + } + auto res = blockBlobClient.UploadFrom(tempFilename, options); + EXPECT_FALSE(res->ETag.empty()); + EXPECT_FALSE(res->LastModified.empty()); + auto properties = *blockBlobClient.GetProperties(); + properties.HttpHeaders.ContentMd5.clear(); + EXPECT_EQ(properties.ContentLength, blobSize); + EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); + EXPECT_EQ(properties.Metadata, options.Metadata); + EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue()); + EXPECT_EQ(properties.ETag, res->ETag); + EXPECT_EQ(properties.LastModified, res->LastModified); + std::vector downloadContent(static_cast(blobSize), '\x00'); + blockBlobClient.DownloadTo(downloadContent.data(), static_cast(blobSize)); + EXPECT_EQ( + downloadContent, + std::vector( + blobContent.begin(), blobContent.begin() + static_cast(blobSize))); + DeleteFile(tempFilename); + }; + + std::vector> futures; for (int c : {1, 2, 5}) { - for (int64_t length : + for (int64_t l : {0ULL, 1ULL, 2ULL, 2_KB, 4_KB, 999_KB, 1_MB, 2_MB - 1, 3_MB, 5_MB, 8_MB - 1234, 8_MB}) { - Azure::Storage::Blobs::UploadBlockBlobFromOptions options; - options.ChunkSize = 1_MB; - options.Concurrency = c; - options.HttpHeaders = m_blobUploadOptions.HttpHeaders; - options.HttpHeaders.ContentMd5.clear(); - options.Metadata = m_blobUploadOptions.Metadata; - options.Tier = m_blobUploadOptions.Tier; - { - auto res = blockBlobClient.UploadFrom( - m_blobContent.data(), static_cast(length), options); - EXPECT_FALSE(res->ETag.empty()); - EXPECT_FALSE(res->LastModified.empty()); - auto properties = *blockBlobClient.GetProperties(); - properties.HttpHeaders.ContentMd5.clear(); - EXPECT_EQ(properties.ContentLength, length); - EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); - EXPECT_EQ(properties.Metadata, options.Metadata); - EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue()); - EXPECT_EQ(properties.ETag, res->ETag); - EXPECT_EQ(properties.LastModified, res->LastModified); - std::vector downloadContent(static_cast(length), '\x00'); - blockBlobClient.DownloadTo(downloadContent.data(), static_cast(length)); - EXPECT_EQ( - downloadContent, - std::vector( - m_blobContent.begin(), m_blobContent.begin() + static_cast(length))); - } - { - { - Azure::Storage::Details::FileWriter fileWriter(tempFilename); - fileWriter.Write(m_blobContent.data(), length, 0); - } - auto res = blockBlobClient.UploadFrom(tempFilename, options); - EXPECT_FALSE(res->ETag.empty()); - EXPECT_FALSE(res->LastModified.empty()); - auto properties = *blockBlobClient.GetProperties(); - properties.HttpHeaders.ContentMd5.clear(); - EXPECT_EQ(properties.ContentLength, length); - EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); - EXPECT_EQ(properties.Metadata, options.Metadata); - EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue()); - EXPECT_EQ(properties.ETag, res->ETag); - EXPECT_EQ(properties.LastModified, res->LastModified); - std::vector downloadContent(static_cast(length), '\x00'); - blockBlobClient.DownloadTo(downloadContent.data(), static_cast(length)); - EXPECT_EQ( - downloadContent, - std::vector( - m_blobContent.begin(), m_blobContent.begin() + static_cast(length))); - } + ASSERT_GE(blobContent.size(), static_cast(l)); + futures.emplace_back(std::async(std::launch::async, testUploadFromBuffer, c, l)); + futures.emplace_back(std::async(std::launch::async, testUploadFromFile, c, l)); } } - DeleteFile(tempFilename); + for (auto& f : futures) + { + f.get(); + } } TEST_F(BlockBlobClientTest, DownloadError) diff --git a/sdk/storage/test/blobs/page_blob_client_test.cpp b/sdk/storage/test/blobs/page_blob_client_test.cpp index 693323cc7..95858dc10 100644 --- a/sdk/storage/test/blobs/page_blob_client_test.cpp +++ b/sdk/storage/test/blobs/page_blob_client_test.cpp @@ -2,7 +2,12 @@ // SPDX-License-Identifier: MIT #include "page_blob_client_test.hpp" + +#include +#include + #include "common/crypt.hpp" +#include "common/file_io.hpp" namespace Azure { namespace Storage { namespace Test { @@ -244,4 +249,109 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_THROW(pageBlobClient.UploadPages(&pageContent, 0, options), StorageError); } + TEST_F(PageBlobClientTest, ConcurrentUploadFromNonExistingFile) + { + auto pageBlobClient = m_blobContainerClient->GetPageBlobClient(RandomString()); + std::string emptyFilename = RandomString(); + EXPECT_THROW(pageBlobClient.UploadFrom(emptyFilename), std::runtime_error); + EXPECT_THROW(pageBlobClient.Delete(), StorageError); + } + + TEST_F(PageBlobClientTest, ConcurrentUploadEmptyBlob) + { + auto pageBlobClient = m_blobContainerClient->GetPageBlobClient(RandomString()); + + std::vector emptyContent; + pageBlobClient.UploadFrom(emptyContent.data(), emptyContent.size()); + EXPECT_NO_THROW(pageBlobClient.Delete()); + + std::string emptyFilename = RandomString(); + { + Details::FileWriter writer(emptyFilename); + } + pageBlobClient.UploadFrom(emptyFilename); + EXPECT_NO_THROW(pageBlobClient.Delete()); + + DeleteFile(emptyFilename); + } + + TEST_F(PageBlobClientTest, ConcurrentUpload) + { + std::vector blobContent = RandomBuffer(static_cast(8_MB)); + + auto testUploadFromBuffer = [&](int concurrency, int64_t blobSize) { + auto pageBlobClient = m_blobContainerClient->GetPageBlobClient(RandomString()); + + Azure::Storage::Blobs::UploadPageBlobFromOptions options; + options.ChunkSize = 512_KB; + options.Concurrency = concurrency; + options.HttpHeaders = m_blobUploadOptions.HttpHeaders; + options.HttpHeaders.ContentMd5.clear(); + options.Metadata = m_blobUploadOptions.Metadata; + + auto res = pageBlobClient.UploadFrom( + blobContent.data(), static_cast(blobSize), options); + EXPECT_TRUE(res->ServerEncrypted.HasValue()); + + auto properties = *pageBlobClient.GetProperties(); + properties.HttpHeaders.ContentMd5.clear(); + EXPECT_EQ(properties.ContentLength, blobSize); + EXPECT_EQ(properties.Metadata, options.Metadata); + std::vector downloadContent(static_cast(blobSize), '\x00'); + pageBlobClient.DownloadTo(downloadContent.data(), static_cast(blobSize)); + EXPECT_EQ( + downloadContent, + std::vector( + blobContent.begin(), blobContent.begin() + static_cast(blobSize))); + }; + + auto testUploadFromFile = [&](int concurrency, int64_t blobSize) { + auto pageBlobClient = m_blobContainerClient->GetPageBlobClient(RandomString()); + + Azure::Storage::Blobs::UploadPageBlobFromOptions options; + options.ChunkSize = 512_KB; + options.Concurrency = concurrency; + options.HttpHeaders = m_blobUploadOptions.HttpHeaders; + options.HttpHeaders.ContentMd5.clear(); + options.Metadata = m_blobUploadOptions.Metadata; + + std::string tempFilename = RandomString(); + { + Azure::Storage::Details::FileWriter fileWriter(tempFilename); + fileWriter.Write(blobContent.data(), blobSize, 0); + } + + auto res = pageBlobClient.UploadFrom(tempFilename, options); + EXPECT_TRUE(res->ServerEncrypted.HasValue()); + + auto properties = *pageBlobClient.GetProperties(); + properties.HttpHeaders.ContentMd5.clear(); + EXPECT_EQ(properties.ContentLength, blobSize); + EXPECT_EQ(properties.Metadata, options.Metadata); + std::vector downloadContent(static_cast(blobSize), '\x00'); + pageBlobClient.DownloadTo(downloadContent.data(), static_cast(blobSize)); + EXPECT_EQ( + downloadContent, + std::vector( + blobContent.begin(), blobContent.begin() + static_cast(blobSize))); + + DeleteFile(tempFilename); + }; + + std::vector> futures; + for (int c : {1, 2, 5}) + { + for (int64_t l : {0ULL, 512ULL, 1_KB, 4_KB, 1_MB, 4_MB + 512}) + { + ASSERT_GE(blobContent.size(), static_cast(l)); + futures.emplace_back(std::async(std::launch::async, testUploadFromBuffer, c, l)); + futures.emplace_back(std::async(std::launch::async, testUploadFromFile, c, l)); + } + } + for (auto& f : futures) + { + f.get(); + } + } + }}} // namespace Azure::Storage::Test