From 95aa6361ef7655d8e9187a0f8bdd740e8c0d0443 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Wed, 8 Jul 2020 13:26:22 +0800 Subject: [PATCH] Concurrent upload block blob from file, concurrent download blob to file (#247) * concurrent download to file * concurrent upload block blob from file * update sample * Change default download chunk size from 8MB to 4MB * fix CI --- .../templates/jobs/archetype-sdk-client.yml | 2 +- sdk/storage/CMakeLists.txt | 2 + sdk/storage/inc/blobs/block_blob_client.hpp | 14 +- sdk/storage/inc/common/file_io.hpp | 50 +++++ sdk/storage/sample/blob_getting_started.cpp | 21 +- sdk/storage/src/blobs/blob_client.cpp | 183 +++++++++++++++--- sdk/storage/src/blobs/block_blob_client.cpp | 62 ++++++ sdk/storage/src/common/file_io.cpp | 129 ++++++++++++ sdk/storage/test/block_blob_client_test.cpp | 153 ++++++++++++--- sdk/storage/test/test_base.cpp | 19 ++ sdk/storage/test/test_base.hpp | 4 + 11 files changed, 571 insertions(+), 68 deletions(-) create mode 100644 sdk/storage/inc/common/file_io.hpp create mode 100644 sdk/storage/src/common/file_io.cpp diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index dfaa4664a..ba3ed1fe5 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -82,7 +82,7 @@ jobs: pool: vmImage: windows-2019 variables: - VcpkgDependencies: curl[winssl] + VcpkgDependencies: curl[winssl] libxml2 VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' steps: - pwsh: | diff --git a/sdk/storage/CMakeLists.txt b/sdk/storage/CMakeLists.txt index dfb1d45b7..0ac64b7e2 100644 --- a/sdk/storage/CMakeLists.txt +++ b/sdk/storage/CMakeLists.txt @@ -19,6 +19,7 @@ set (AZURE_STORAGE_BLOB_HEADER inc/common/crypt.hpp inc/common/xml_wrapper.hpp inc/common/concurrent_transfer.hpp + inc/common/file_io.hpp inc/blobs/blob.hpp inc/blobs/blob_service_client.hpp inc/blobs/blob_container_client.hpp @@ -58,6 +59,7 @@ set (AZURE_STORAGE_BLOB_SOURCE src/common/storage_error.cpp src/common/crypt.cpp src/common/xml_wrapper.cpp + src/common/file_io.cpp src/blobs/blob_service_client.cpp src/blobs/blob_container_client.cpp src/blobs/blob_client.cpp diff --git a/sdk/storage/inc/blobs/block_blob_client.hpp b/sdk/storage/inc/blobs/block_blob_client.hpp index cdef42de4..b6d6f9b3f 100644 --- a/sdk/storage/inc/blobs/block_blob_client.hpp +++ b/sdk/storage/inc/blobs/block_blob_client.hpp @@ -120,13 +120,25 @@ namespace Azure { namespace Storage { namespace Blobs { * @param buffer A memory buffer containing the content to upload. * @param bufferSize Size of the memory buffer. * @param options Optional parameters to execute this function. - * @return A BlockBlobInfo describing the state of the updated block blob. + * @return A BlobContentInfo describing the state of the updated block blob. */ BlobContentInfo UploadFromBuffer( const uint8_t* buffer, std::size_t bufferSize, const UploadBlobOptions& options = UploadBlobOptions()) const; + /** + * @brief Creates a new block blob, or updates the content of an existing block blob. Updating + * an existing block blob overwrites any existing metadata on the blob. + * + * @param buffer A file containing the content to upload. + * @param options Optional parameters to execute this function. + * @return A BlobContentInfo describing the state of the updated block blob. + */ + BlobContentInfo UploadFromFile( + const std::string& file, + const UploadBlobOptions& options = UploadBlobOptions()) const; + /** * @brief Creates a new block as part of a block blob's staging area to be eventually * committed via the CommitBlockList operation. diff --git a/sdk/storage/inc/common/file_io.hpp b/sdk/storage/inc/common/file_io.hpp new file mode 100644 index 000000000..759c940ca --- /dev/null +++ b/sdk/storage/inc/common/file_io.hpp @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef _WIN32 +#include +#endif + +#include +#include + +namespace Azure { namespace Storage { namespace Details { + +#ifdef _WIN32 + using FileHandle = HANDLE; +#else + using FileHandle = int; +#endif + + class FileReader { + public: + FileReader(const std::string& filename); + + ~FileReader(); + + FileHandle GetHandle() const { return m_handle; } + + int64_t GetFileSize() const { return m_fileSize; } + + private: + FileHandle m_handle; + int64_t m_fileSize; + }; + + class FileWriter { + public: + FileWriter(const std::string& filename); + + ~FileWriter(); + + FileHandle GetHandle() const { return m_handle; } + + void Write(const uint8_t* buffer, int64_t length, int64_t offset); + + private: + FileHandle m_handle; + }; + +}}} // namespace Azure::Storage::Details diff --git a/sdk/storage/sample/blob_getting_started.cpp b/sdk/storage/sample/blob_getting_started.cpp index d83a81d5b..0586fac8d 100644 --- a/sdk/storage/sample/blob_getting_started.cpp +++ b/sdk/storage/sample/blob_getting_started.cpp @@ -29,27 +29,20 @@ void BlobsGettingStarted() BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName); - auto blobContentStream = Azure::Core::Http::MemoryBodyStream( - reinterpret_cast(blobContent.data()), blobContent.length()); - blobClient.Upload(blobContentStream); + blobClient.UploadFromBuffer( + reinterpret_cast(blobContent.data()), blobContent.size()); std::map blobMetadata = {{"key1", "value1"}, {"key2", "value2"}}; blobClient.SetMetadata(blobMetadata); - auto blobDownloadContent = blobClient.Download(); - blobContent.resize(static_cast(blobDownloadContent.BodyStream->Length())); - Azure::Core::Context context; - Azure::Core::Http::BodyStream::ReadToCount( - context, - *blobDownloadContent.BodyStream, - reinterpret_cast(&blobContent[0]), - blobDownloadContent.BodyStream->Length()); - - std::cout << blobContent << std::endl; - auto properties = blobClient.GetProperties(); for (auto metadata : properties.Metadata) { std::cout << metadata.first << ":" << metadata.second << std::endl; } + blobContent.resize(static_cast(properties.ContentLength)); + + blobClient.DownloadToBuffer(reinterpret_cast(&blobContent[0]), blobContent.size()); + + std::cout << blobContent << std::endl; } diff --git a/sdk/storage/src/blobs/blob_client.cpp b/sdk/storage/src/blobs/blob_client.cpp index e2efb767d..e7b628456 100644 --- a/sdk/storage/src/blobs/blob_client.cpp +++ b/sdk/storage/src/blobs/blob_client.cpp @@ -8,6 +8,7 @@ #include "blobs/page_blob_client.hpp" #include "common/common_headers_request_policy.hpp" #include "common/concurrent_transfer.hpp" +#include "common/file_io.hpp" #include "common/shared_key_policy.hpp" #include "common/storage_common.hpp" #include "http/curl/curl.hpp" @@ -149,7 +150,7 @@ namespace Azure { namespace Storage { namespace Blobs { std::size_t bufferSize, const DownloadBlobToBufferOptions& options) const { - constexpr int64_t c_defaultChunkSize = 8 * 1024 * 1024; + constexpr int64_t c_defaultChunkSize = 4 * 1024 * 1024; // Just start downloading using an initial chunk. If it's a small blob, we'll get the whole // thing in one shot. If it's a large blob, we'll get its full size in Content-Range and can @@ -222,28 +223,28 @@ namespace Azure { namespace Storage { namespace Blobs { BlobDownloadInfo ret = returnTypeConverter(firstChunk); // Keep downloading the remaining in parallel - auto downloadChunkFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { + auto downloadChunkFunc + = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { + DownloadBlobOptions chunkOptions; + chunkOptions.Context = options.Context; + chunkOptions.Offset = offset; + chunkOptions.Length = length; + auto chunk = Download(chunkOptions); + int64_t bytesRead = Azure::Core::Http::BodyStream::ReadToCount( + chunkOptions.Context, + *chunk.BodyStream, + buffer + (offset - firstChunkOffset), + chunkOptions.Length.GetValue()); + if (bytesRead != chunkOptions.Length.GetValue()) + { + throw std::runtime_error("error when reading body stream"); + } - DownloadBlobOptions chunkOptions; - chunkOptions.Context = options.Context; - chunkOptions.Offset = offset; - chunkOptions.Length = length; - auto chunk = Download(chunkOptions); - int64_t bytesRead = Azure::Core::Http::BodyStream::ReadToCount( - chunkOptions.Context, - *chunk.BodyStream, - buffer + (offset - firstChunkOffset), - chunkOptions.Length.GetValue()); - if (bytesRead != chunkOptions.Length.GetValue()) - { - throw std::runtime_error("error when reading body stream"); - } - - if (chunkId == numChunks - 1) - { - ret = returnTypeConverter(chunk); - } - }; + if (chunkId == numChunks - 1) + { + ret = returnTypeConverter(chunk); + } + }; int64_t remainingOffset = firstChunkOffset + firstChunkLength; int64_t remainingSize = blobRangeSize - firstChunkLength; @@ -261,11 +262,139 @@ namespace Azure { namespace Storage { namespace Blobs { } Details::ConcurrentTransfer( - remainingOffset, - remainingSize, - chunkSize, - options.Concurrency, - downloadChunkFunc); + remainingOffset, remainingSize, chunkSize, options.Concurrency, downloadChunkFunc); + ret.ContentLength = blobRangeSize; + return ret; + } + + BlobDownloadInfo BlobClient::DownloadToFile( + const std::string& file, + const DownloadBlobToFileOptions& options) const + { + constexpr int64_t c_defaultChunkSize = 4 * 1024 * 1024; + + // Just start downloading using an initial chunk. If it's a small blob, we'll get the whole + // thing in one shot. If it's a large blob, we'll get its full size in Content-Range and can + // keep downloading it in chunks. + int64_t firstChunkOffset = options.Offset.HasValue() ? options.Offset.GetValue() : 0; + int64_t firstChunkLength = c_defaultChunkSize; + if (options.InitialChunkSize.HasValue()) + { + firstChunkLength = options.InitialChunkSize.GetValue(); + } + if (options.Length.HasValue()) + { + firstChunkLength = std::min(firstChunkLength, options.Length.GetValue()); + } + + DownloadBlobOptions firstChunkOptions; + firstChunkOptions.Context = options.Context; + firstChunkOptions.Offset = options.Offset; + if (firstChunkOptions.Offset.HasValue()) + { + firstChunkOptions.Length = firstChunkLength; + } + + Details::FileWriter fileWriter(file); + + auto firstChunk = Download(firstChunkOptions); + + int64_t blobSize; + int64_t blobRangeSize; + if (firstChunkOptions.Offset.HasValue()) + { + blobSize = std::stoll(firstChunk.ContentRange.GetValue().substr( + firstChunk.ContentRange.GetValue().find('/') + 1)); + blobRangeSize = blobSize - firstChunkOffset; + if (options.Length.HasValue()) + { + blobRangeSize = std::min(blobRangeSize, options.Length.GetValue()); + } + } + else + { + blobSize = firstChunk.BodyStream->Length(); + blobRangeSize = blobSize; + } + firstChunkLength = std::min(firstChunkLength, blobRangeSize); + + auto bodyStreamToFile = [](Azure::Core::Http::BodyStream& stream, + Details::FileWriter& fileWriter, + int64_t offset, + int64_t length, + Azure::Core::Context& context) { + constexpr std::size_t bufferSize = 4 * 1024 * 1024; + std::vector buffer(bufferSize); + while (length > 0) + { + int64_t readSize = std::min(static_cast(bufferSize), length); + int64_t bytesRead + = Azure::Core::Http::BodyStream::ReadToCount(context, stream, buffer.data(), readSize); + if (bytesRead != readSize) + { + throw std::runtime_error("error when reading body stream"); + } + fileWriter.Write(buffer.data(), bytesRead, offset); + length -= bytesRead; + offset += bytesRead; + } + }; + + bodyStreamToFile( + *firstChunk.BodyStream, fileWriter, 0, firstChunkLength, firstChunkOptions.Context); + firstChunk.BodyStream.reset(); + + auto returnTypeConverter = [](BlobDownloadResponse& response) { + BlobDownloadInfo ret; + ret.ETag = std::move(response.ETag); + ret.LastModified = std::move(response.LastModified); + ret.HttpHeaders = std::move(response.HttpHeaders); + ret.Metadata = std::move(response.Metadata); + ret.BlobType = response.BlobType; + ret.ServerEncrypted = response.ServerEncrypted; + ret.EncryptionKeySHA256 = std::move(response.EncryptionKeySHA256); + return ret; + }; + BlobDownloadInfo ret = returnTypeConverter(firstChunk); + + // Keep downloading the remaining in parallel + auto downloadChunkFunc + = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { + DownloadBlobOptions chunkOptions; + chunkOptions.Context = options.Context; + chunkOptions.Offset = offset; + chunkOptions.Length = length; + auto chunk = Download(chunkOptions); + bodyStreamToFile( + *chunk.BodyStream, + fileWriter, + offset - firstChunkOffset, + chunkOptions.Length.GetValue(), + chunkOptions.Context); + + if (chunkId == numChunks - 1) + { + ret = returnTypeConverter(chunk); + } + }; + + int64_t remainingOffset = firstChunkOffset + firstChunkLength; + int64_t remainingSize = blobRangeSize - firstChunkLength; + int64_t chunkSize; + if (options.ChunkSize.HasValue()) + { + chunkSize = options.ChunkSize.GetValue(); + } + else + { + int64_t c_grainSize = 4 * 1024; + chunkSize = remainingSize / options.Concurrency; + chunkSize = (std::max(chunkSize, int64_t(1)) + c_grainSize - 1) / c_grainSize * c_grainSize; + chunkSize = std::min(chunkSize, c_defaultChunkSize); + } + + Details::ConcurrentTransfer( + remainingOffset, remainingSize, chunkSize, options.Concurrency, downloadChunkFunc); ret.ContentLength = blobRangeSize; return ret; } diff --git a/sdk/storage/src/blobs/block_blob_client.cpp b/sdk/storage/src/blobs/block_blob_client.cpp index 02b41baa4..3dd09da88 100644 --- a/sdk/storage/src/blobs/block_blob_client.cpp +++ b/sdk/storage/src/blobs/block_blob_client.cpp @@ -5,6 +5,7 @@ #include "common/concurrent_transfer.hpp" #include "common/crypt.hpp" +#include "common/file_io.hpp" #include "common/storage_common.hpp" namespace Azure { namespace Storage { namespace Blobs { @@ -135,6 +136,67 @@ namespace Azure { namespace Storage { namespace Blobs { return commitBlockListResponse; } + BlobContentInfo BlockBlobClient::UploadFromFile( + const std::string& file, + const UploadBlobOptions& options) const + { + constexpr int64_t c_defaultBlockSize = 8 * 1024 * 1024; + constexpr int64_t c_maximumNumberBlocks = 50000; + constexpr int64_t c_grainSize = 4 * 1024; + + Details::FileReader fileReader(file); + + int64_t chunkSize = c_defaultBlockSize; + if (options.ChunkSize.HasValue()) + { + chunkSize = options.ChunkSize.GetValue(); + } + else + { + int64_t minBlockSize + = (fileReader.GetFileSize() + c_maximumNumberBlocks - 1) / c_maximumNumberBlocks; + chunkSize = std::max(chunkSize, minBlockSize); + chunkSize = (chunkSize + c_grainSize - 1) / c_grainSize * c_grainSize; + } + + std::vector> blockIds; + auto getBlockId = [](int64_t id) { + constexpr std::size_t c_blockIdLength = 64; + std::string blockId = std::to_string(id); + blockId = std::string(c_blockIdLength - blockId.length(), '0') + blockId; + return Base64Encode(blockId); + }; + + auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { + Azure::Core::Http::FileBodyStream contentStream(fileReader.GetHandle(), offset, length); + StageBlockOptions chunkOptions; + chunkOptions.Context = options.Context; + auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions); + if (chunkId == numChunks - 1) + { + blockIds.resize(static_cast(numChunks)); + } + }; + + Details::ConcurrentTransfer( + 0, fileReader.GetFileSize(), chunkSize, options.Concurrency, uploadBlockFunc); + + for (std::size_t i = 0; i < blockIds.size(); ++i) + { + blockIds[i].first = BlockType::Uncommitted; + blockIds[i].second = getBlockId(static_cast(i)); + } + CommitBlockListOptions commitBlockListOptions; + commitBlockListOptions.Context = options.Context; + commitBlockListOptions.HttpHeaders = options.HttpHeaders; + commitBlockListOptions.Metadata = options.Metadata; + commitBlockListOptions.Tier = options.Tier; + auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions); + commitBlockListResponse.ContentCRC64.Reset(); + commitBlockListResponse.ContentMD5.Reset(); + return commitBlockListResponse; + } + BlockInfo BlockBlobClient::StageBlock( const std::string& blockId, Azure::Core::Http::BodyStream& content, diff --git a/sdk/storage/src/common/file_io.cpp b/sdk/storage/src/common/file_io.cpp new file mode 100644 index 000000000..3d58f4130 --- /dev/null +++ b/sdk/storage/src/common/file_io.cpp @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "common/file_io.hpp" + +#ifndef _WIN32 +#include +#include +#include +#include +#endif + +#include +#include + +namespace Azure { namespace Storage { namespace Details { + +#ifdef _WIN32 + FileReader::FileReader(const std::string& filename) + { + m_handle = CreateFile( + filename.data(), + GENERIC_READ, + FILE_SHARE_READ, + nullptr, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (m_handle == INVALID_HANDLE_VALUE) + { + throw std::runtime_error("failed to open file"); + } + + LARGE_INTEGER fileSize; + BOOL ret = GetFileSizeEx(m_handle, &fileSize); + if (!ret) + { + CloseHandle(m_handle); + throw std::runtime_error("failed to get size of file"); + } + m_fileSize = fileSize.QuadPart; + } + + FileReader::~FileReader() { CloseHandle(m_handle); } + + FileWriter::FileWriter(const std::string& filename) + { + m_handle = CreateFile( + filename.data(), + GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (m_handle == INVALID_HANDLE_VALUE) + { + throw std::runtime_error("failed to open file"); + } + } + + FileWriter::~FileWriter() { CloseHandle(m_handle); } + + void FileWriter::Write(const uint8_t* buffer, int64_t length, int64_t offset) + { + if (length > std::numeric_limits::max()) + { + throw std::runtime_error("failed to write file"); + } + + OVERLAPPED overlapped; + std::memset(&overlapped, 0, sizeof(overlapped)); + overlapped.Offset = static_cast(static_cast(offset)); + overlapped.OffsetHigh = static_cast(static_cast(offset) >> 32); + + DWORD bytesWritten; + BOOL ret = WriteFile(m_handle, buffer, static_cast(length), &bytesWritten, &overlapped); + if (!ret) + { + throw std::runtime_error("failed to write file"); + } + } +#else + FileReader::FileReader(const std::string& filename) + { + m_handle = open(filename.data(), O_RDONLY); + if (m_handle == -1) + { + throw std::runtime_error("failed to open file"); + } + m_fileSize = lseek(m_handle, 0, SEEK_END); + if (m_fileSize == -1) + { + close(m_handle); + throw std::runtime_error("failed to get size of file"); + } + } + + FileReader::~FileReader() { close(m_handle); } + + FileWriter::FileWriter(const std::string& filename) + { + m_handle = open( + filename.data(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (m_handle == -1) + { + throw std::runtime_error("failed to open file"); + } + } + + FileWriter::~FileWriter() { close(m_handle); } + + void FileWriter::Write(const uint8_t* buffer, int64_t length, int64_t offset) + { + if (static_cast(length) > std::numeric_limits::max() + || offset > static_cast(std::numeric_limits::max())) + { + throw std::runtime_error("failed to write file"); + } + ssize_t bytesWritten + = pwrite(m_handle, buffer, static_cast(length), static_cast(offset)); + if (bytesWritten != length) + { + throw std::runtime_error("failed to write file"); + } + } +#endif + +}}} // namespace Azure::Storage::Details diff --git a/sdk/storage/test/block_blob_client_test.cpp b/sdk/storage/test/block_blob_client_test.cpp index 3aca953c4..e9d7a8b1a 100644 --- a/sdk/storage/test/block_blob_client_test.cpp +++ b/sdk/storage/test/block_blob_client_test.cpp @@ -4,6 +4,7 @@ #include "block_blob_client_test.hpp" #include "common/crypt.hpp" +#include "common/file_io.hpp" #include #include @@ -256,6 +257,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlockBlobClientTest, ConcurrentDownload) { + std::string tempFilename = RandomString(); std::vector downloadBuffer = m_blobContent; for (int c : {1, 2, 4}) { @@ -267,6 +269,11 @@ namespace Azure { namespace Storage { namespace Test { auto res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size()); EXPECT_EQ(downloadBuffer, m_blobContent); EXPECT_EQ(static_cast(res.ContentLength), downloadBuffer.size()); + res = m_blockBlobClient->DownloadToFile(tempFilename); + auto downloadFile = ReadFile(tempFilename); + EXPECT_EQ(downloadFile, m_blobContent); + EXPECT_EQ(static_cast(res.ContentLength), downloadFile.size()); + DeleteFile(tempFilename); // download whole blob downloadBuffer.assign(downloadBuffer.size(), '\x00'); @@ -274,6 +281,11 @@ namespace Azure { namespace Storage { namespace Test { res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size()); EXPECT_EQ(downloadBuffer, m_blobContent); EXPECT_EQ(static_cast(res.ContentLength), downloadBuffer.size()); + res = m_blockBlobClient->DownloadToFile(tempFilename); + downloadFile = ReadFile(tempFilename); + EXPECT_EQ(downloadFile, m_blobContent); + EXPECT_EQ(static_cast(res.ContentLength), downloadFile.size()); + DeleteFile(tempFilename); // download whole blob downloadBuffer.assign(downloadBuffer.size(), '\x00'); @@ -282,6 +294,11 @@ namespace Azure { namespace Storage { namespace Test { res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size()); EXPECT_EQ(downloadBuffer, m_blobContent); EXPECT_EQ(static_cast(res.ContentLength), downloadBuffer.size()); + res = m_blockBlobClient->DownloadToFile(tempFilename); + downloadFile = ReadFile(tempFilename); + EXPECT_EQ(downloadFile, m_blobContent); + EXPECT_EQ(static_cast(res.ContentLength), downloadFile.size()); + DeleteFile(tempFilename); // download whole blob downloadBuffer.assign(downloadBuffer.size(), '\x00'); @@ -290,6 +307,11 @@ namespace Azure { namespace Storage { namespace Test { res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size() * 2); EXPECT_EQ(downloadBuffer, m_blobContent); EXPECT_EQ(static_cast(res.ContentLength), downloadBuffer.size()); + res = m_blockBlobClient->DownloadToFile(tempFilename); + downloadFile = ReadFile(tempFilename); + EXPECT_EQ(downloadFile, m_blobContent); + EXPECT_EQ(static_cast(res.ContentLength), downloadFile.size()); + DeleteFile(tempFilename); options.InitialChunkSize = 4_KB; options.ChunkSize = 4_KB; @@ -311,14 +333,28 @@ namespace Azure { namespace Storage { namespace Test { std::vector( m_blobContent.begin() + static_cast(offset), m_blobContent.begin() + static_cast(offset) - + downloadContent.size())); + + static_cast(actualLength))); EXPECT_EQ(res.ContentLength, actualLength); + + std::string tempFilename2 = RandomString(); + res = m_blockBlobClient->DownloadToFile(tempFilename2, optionsCopy); + auto downloadFile = ReadFile(tempFilename2); + EXPECT_EQ( + downloadFile, + std::vector( + m_blobContent.begin() + static_cast(offset), + m_blobContent.begin() + static_cast(offset) + + static_cast(actualLength))); + EXPECT_EQ(res.ContentLength, actualLength); + DeleteFile(tempFilename2); } else { EXPECT_THROW( m_blockBlobClient->DownloadToBuffer(nullptr, 8 * 1024 * 1024, optionsCopy), std::runtime_error); + EXPECT_THROW( + m_blockBlobClient->DownloadToFile(tempFilename, optionsCopy), std::runtime_error); } }; @@ -365,6 +401,8 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlockBlobClientTest, ConcurrentDownloadEmptyBlob) { + std::string tempFilename = RandomString(); + std::vector emptyContent; auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( StandardStorageConnectionString(), m_containerName, RandomString()); @@ -381,6 +419,15 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders); EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata); EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob); + res = blockBlobClient.DownloadToFile(tempFilename); + EXPECT_EQ(res.ContentLength, 0); + EXPECT_FALSE(res.ETag.empty()); + EXPECT_FALSE(res.LastModified.empty()); + EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob); + EXPECT_TRUE(ReadFile(tempFilename).empty()); + DeleteFile(tempFilename); res = blockBlobClient.DownloadToBuffer(emptyContent.data(), static_cast(8_MB)); EXPECT_EQ(res.ContentLength, 0); @@ -389,6 +436,15 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders); EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata); EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob); + res = blockBlobClient.DownloadToFile(tempFilename); + EXPECT_EQ(res.ContentLength, 0); + EXPECT_FALSE(res.ETag.empty()); + EXPECT_FALSE(res.LastModified.empty()); + EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob); + EXPECT_TRUE(ReadFile(tempFilename).empty()); + DeleteFile(tempFilename); for (int c : {1, 2}) { @@ -405,18 +461,29 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders); EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata); EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob); + res = blockBlobClient.DownloadToFile(tempFilename, options); + EXPECT_EQ(res.ContentLength, 0); + EXPECT_FALSE(res.ETag.empty()); + EXPECT_FALSE(res.LastModified.empty()); + EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob); + EXPECT_TRUE(ReadFile(tempFilename).empty()); + DeleteFile(tempFilename); options.Offset = 0; EXPECT_THROW( blockBlobClient.DownloadToBuffer( emptyContent.data(), static_cast(8_MB), options), std::runtime_error); + EXPECT_THROW(blockBlobClient.DownloadToFile(tempFilename, options), std::runtime_error); options.Offset = 1; EXPECT_THROW( blockBlobClient.DownloadToBuffer( emptyContent.data(), static_cast(8_MB), options), std::runtime_error); + EXPECT_THROW(blockBlobClient.DownloadToFile(tempFilename, options), std::runtime_error); options.Offset = 0; options.Length = 1; @@ -424,6 +491,7 @@ namespace Azure { namespace Storage { namespace Test { blockBlobClient.DownloadToBuffer( emptyContent.data(), static_cast(8_MB), options), std::runtime_error); + EXPECT_THROW(blockBlobClient.DownloadToFile(tempFilename, options), std::runtime_error); options.Offset = 100; options.Length = 100; @@ -431,11 +499,14 @@ namespace Azure { namespace Storage { namespace Test { blockBlobClient.DownloadToBuffer( emptyContent.data(), static_cast(8_MB), options), std::runtime_error); + EXPECT_THROW(blockBlobClient.DownloadToFile(tempFilename, options), std::runtime_error); } } TEST_F(BlockBlobClientTest, ConcurrentUpload) { + std::string tempFilename = RandomString(); + auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( StandardStorageConnectionString(), m_containerName, RandomString()); for (int c : {1, 2, 5}) @@ -449,32 +520,64 @@ namespace Azure { namespace Storage { namespace Test { options.HttpHeaders = m_blobUploadOptions.HttpHeaders; options.Metadata = m_blobUploadOptions.Metadata; options.Tier = m_blobUploadOptions.Tier; - - auto res = blockBlobClient.UploadFromBuffer( - m_blobContent.data(), static_cast(length), options); - EXPECT_FALSE(res.RequestId.empty()); - EXPECT_FALSE(res.Version.empty()); - EXPECT_FALSE(res.Date.empty()); - EXPECT_FALSE(res.ETag.empty()); - EXPECT_FALSE(res.LastModified.empty()); - EXPECT_FALSE(res.SequenceNumber.HasValue()); - EXPECT_FALSE(res.ContentCRC64.HasValue()); - EXPECT_FALSE(res.ContentMD5.HasValue()); - auto properties = blockBlobClient.GetProperties(); - EXPECT_EQ(properties.ContentLength, length); - EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); - EXPECT_EQ(properties.Metadata, options.Metadata); - EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue()); - EXPECT_EQ(properties.ETag, res.ETag); - EXPECT_EQ(properties.LastModified, res.LastModified); - std::vector downloadContent(static_cast(length), '\x00'); - blockBlobClient.DownloadToBuffer(downloadContent.data(), static_cast(length)); - EXPECT_EQ( - downloadContent, - std::vector( - m_blobContent.begin(), m_blobContent.begin() + static_cast(length))); + { + auto res = blockBlobClient.UploadFromBuffer( + m_blobContent.data(), static_cast(length), options); + EXPECT_FALSE(res.RequestId.empty()); + EXPECT_FALSE(res.Version.empty()); + EXPECT_FALSE(res.Date.empty()); + EXPECT_FALSE(res.ETag.empty()); + EXPECT_FALSE(res.LastModified.empty()); + EXPECT_FALSE(res.SequenceNumber.HasValue()); + EXPECT_FALSE(res.ContentCRC64.HasValue()); + EXPECT_FALSE(res.ContentMD5.HasValue()); + auto properties = blockBlobClient.GetProperties(); + EXPECT_EQ(properties.ContentLength, length); + EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); + EXPECT_EQ(properties.Metadata, options.Metadata); + EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue()); + EXPECT_EQ(properties.ETag, res.ETag); + EXPECT_EQ(properties.LastModified, res.LastModified); + std::vector downloadContent(static_cast(length), '\x00'); + blockBlobClient.DownloadToBuffer( + downloadContent.data(), static_cast(length)); + EXPECT_EQ( + downloadContent, + std::vector( + m_blobContent.begin(), m_blobContent.begin() + static_cast(length))); + } + { + { + Azure::Storage::Details::FileWriter fileWriter(tempFilename); + fileWriter.Write(m_blobContent.data(), length, 0); + } + auto res = blockBlobClient.UploadFromFile(tempFilename, options); + EXPECT_FALSE(res.RequestId.empty()); + EXPECT_FALSE(res.Version.empty()); + EXPECT_FALSE(res.Date.empty()); + EXPECT_FALSE(res.ETag.empty()); + EXPECT_FALSE(res.LastModified.empty()); + EXPECT_FALSE(res.SequenceNumber.HasValue()); + EXPECT_FALSE(res.ContentCRC64.HasValue()); + EXPECT_FALSE(res.ContentMD5.HasValue()); + auto properties = blockBlobClient.GetProperties(); + EXPECT_EQ(properties.ContentLength, length); + EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); + EXPECT_EQ(properties.Metadata, options.Metadata); + EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue()); + EXPECT_EQ(properties.ETag, res.ETag); + EXPECT_EQ(properties.LastModified, res.LastModified); + std::vector downloadContent(static_cast(length), '\x00'); + blockBlobClient.DownloadToBuffer( + downloadContent.data(), static_cast(length)); + EXPECT_EQ( + downloadContent, + std::vector( + m_blobContent.begin(), m_blobContent.begin() + static_cast(length))); + } } } + DeleteFile(tempFilename); } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/test/test_base.cpp b/sdk/storage/test/test_base.cpp index 7af15a90d..481251a78 100644 --- a/sdk/storage/test/test_base.cpp +++ b/sdk/storage/test/test_base.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -132,4 +133,22 @@ namespace Azure { namespace Storage { namespace Test { } } + std::vector ReadFile(const std::string& filename) + { + FILE* fin = fopen(filename.data(), "rb"); + if (!fin) + { + throw std::runtime_error("failed to open file"); + } + fseek(fin, 0, SEEK_END); + int64_t fileSize = ftell(fin); + std::vector fileContent(static_cast(fileSize)); + fseek(fin, 0, SEEK_SET); + fread(fileContent.data(), static_cast(fileSize), 1, fin); + fclose(fin); + return fileContent; + } + + void DeleteFile(const std::string& filename) { std::remove(filename.data()); } + }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/test/test_base.hpp b/sdk/storage/test/test_base.hpp index fa299dbcf..ef6a935e5 100644 --- a/sdk/storage/test/test_base.hpp +++ b/sdk/storage/test/test_base.hpp @@ -41,4 +41,8 @@ namespace Azure { namespace Storage { namespace Test { return ReadBodyStream(stream); } + std::vector ReadFile(const std::string& filename); + + void DeleteFile(const std::string& filename); + }}} // namespace Azure::Storage::Test