github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Concurrent upload block blob from file, concurrent download blob to file (#247)

Browse Source 
* concurrent download to file

* concurrent upload block blob from file

* update sample

* Change default download chunk size from 8MB to 4MB

* fix CI
This commit is contained in:
JinmingHu 2020-07-08 13:26:22 +08:00 committed by GitHub
parent ed9fd2d179
commit 95aa6361ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 571 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
eng/pipelines/templates/jobs/archetype-sdk-client.yml
View File

@ -82,7 +82,7 @@ jobs:
pool:
vmImage: windows-2019
variables:
VcpkgDependencies: curl[winssl]
VcpkgDependencies: curl[winssl] libxml2
VCPKG_DEFAULT_TRIPLET: 'x64-windows-static'
steps:
- pwsh: |

2
sdk/storage/CMakeLists.txt
View File

@ -19,6 +19,7 @@ set (AZURE_STORAGE_BLOB_HEADER
inc/common/crypt.hpp
inc/common/xml_wrapper.hpp
inc/common/concurrent_transfer.hpp
inc/common/file_io.hpp
inc/blobs/blob.hpp
inc/blobs/blob_service_client.hpp
inc/blobs/blob_container_client.hpp
@ -58,6 +59,7 @@ set (AZURE_STORAGE_BLOB_SOURCE
src/common/storage_error.cpp
src/common/crypt.cpp
src/common/xml_wrapper.cpp
src/common/file_io.cpp
src/blobs/blob_service_client.cpp
src/blobs/blob_container_client.cpp
src/blobs/blob_client.cpp

14
sdk/storage/inc/blobs/block_blob_client.hpp
View File

@ -120,13 +120,25 @@ namespace Azure { namespace Storage { namespace Blobs {
* @param buffer A memory buffer containing the content to upload.
* @param bufferSize Size of the memory buffer.
* @param options Optional parameters to execute this function.
* @return A BlockBlobInfo describing the state of the updated block blob.
* @return A BlobContentInfo describing the state of the updated block blob.
*/
BlobContentInfo UploadFromBuffer(
const uint8_t* buffer,
std::size_t bufferSize,
const UploadBlobOptions& options = UploadBlobOptions()) const;
/**
* @brief Creates a new block blob, or updates the content of an existing block blob. Updating
* an existing block blob overwrites any existing metadata on the blob.
*
* @param buffer A file containing the content to upload.
* @param options Optional parameters to execute this function.
* @return A BlobContentInfo describing the state of the updated block blob.
*/
BlobContentInfo UploadFromFile(
const std::string& file,
const UploadBlobOptions& options = UploadBlobOptions()) const;
/**
* @brief Creates a new block as part of a block blob's staging area to be eventually
* committed via the CommitBlockList operation.

50
sdk/storage/inc/common/file_io.hpp Normal file
View File

@ -0,0 +1,50 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// SPDX-License-Identifier: MIT
#pragma once
#ifdef _WIN32
#include <Windows.h>
#endif
#include <cstdint>
#include <string>
namespace Azure { namespace Storage { namespace Details {
#ifdef _WIN32
using FileHandle = HANDLE;
#else
using FileHandle = int;
#endif
class FileReader {
public:
FileReader(const std::string& filename);
~FileReader();
FileHandle GetHandle() const { return m_handle; }
int64_t GetFileSize() const { return m_fileSize; }
private:
FileHandle m_handle;
int64_t m_fileSize;
};
class FileWriter {
public:
FileWriter(const std::string& filename);
~FileWriter();
FileHandle GetHandle() const { return m_handle; }
void Write(const uint8_t* buffer, int64_t length, int64_t offset);
private:
FileHandle m_handle;
};
}}} // namespace Azure::Storage::Details

21
sdk/storage/sample/blob_getting_started.cpp
View File

@ -29,27 +29,20 @@ void BlobsGettingStarted()
BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName);
auto blobContentStream = Azure::Core::Http::MemoryBodyStream(
reinterpret_cast<const uint8_t*>(blobContent.data()), blobContent.length());
blobClient.Upload(blobContentStream);
blobClient.UploadFromBuffer(
reinterpret_cast<const uint8_t*>(blobContent.data()), blobContent.size());
std::map<std::string, std::string> blobMetadata = {{"key1", "value1"}, {"key2", "value2"}};
blobClient.SetMetadata(blobMetadata);
auto blobDownloadContent = blobClient.Download();
blobContent.resize(static_cast<std::size_t>(blobDownloadContent.BodyStream->Length()));
Azure::Core::Context context;
Azure::Core::Http::BodyStream::ReadToCount(
context,
*blobDownloadContent.BodyStream,
reinterpret_cast<uint8_t*>(&blobContent[0]),
blobDownloadContent.BodyStream->Length());
std::cout << blobContent << std::endl;
auto properties = blobClient.GetProperties();
for (auto metadata : properties.Metadata)
{
std::cout << metadata.first << ":" << metadata.second << std::endl;
}
blobContent.resize(static_cast<std::size_t>(properties.ContentLength));
blobClient.DownloadToBuffer(reinterpret_cast<uint8_t*>(&blobContent[0]), blobContent.size());
std::cout << blobContent << std::endl;
}

183
sdk/storage/src/blobs/blob_client.cpp
View File

@ -8,6 +8,7 @@
#include "blobs/page_blob_client.hpp"
#include "common/common_headers_request_policy.hpp"
#include "common/concurrent_transfer.hpp"
#include "common/file_io.hpp"
#include "common/shared_key_policy.hpp"
#include "common/storage_common.hpp"
#include "http/curl/curl.hpp"
@ -149,7 +150,7 @@ namespace Azure { namespace Storage { namespace Blobs {
std::size_t bufferSize,
const DownloadBlobToBufferOptions& options) const
{
constexpr int64_t c_defaultChunkSize = 8 * 1024 * 1024;
constexpr int64_t c_defaultChunkSize = 4 * 1024 * 1024;
// Just start downloading using an initial chunk. If it's a small blob, we'll get the whole
// thing in one shot. If it's a large blob, we'll get its full size in Content-Range and can
@ -222,28 +223,28 @@ namespace Azure { namespace Storage { namespace Blobs {
BlobDownloadInfo ret = returnTypeConverter(firstChunk);
// Keep downloading the remaining in parallel
auto downloadChunkFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
auto downloadChunkFunc
= [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
DownloadBlobOptions chunkOptions;
chunkOptions.Context = options.Context;
chunkOptions.Offset = offset;
chunkOptions.Length = length;
auto chunk = Download(chunkOptions);
int64_t bytesRead = Azure::Core::Http::BodyStream::ReadToCount(
chunkOptions.Context,
*chunk.BodyStream,
buffer + (offset - firstChunkOffset),
chunkOptions.Length.GetValue());
if (bytesRead != chunkOptions.Length.GetValue())
{
throw std::runtime_error("error when reading body stream");
}
DownloadBlobOptions chunkOptions;
chunkOptions.Context = options.Context;
chunkOptions.Offset = offset;
chunkOptions.Length = length;
auto chunk = Download(chunkOptions);
int64_t bytesRead = Azure::Core::Http::BodyStream::ReadToCount(
chunkOptions.Context,
*chunk.BodyStream,
buffer + (offset - firstChunkOffset),
chunkOptions.Length.GetValue());
if (bytesRead != chunkOptions.Length.GetValue())
{
throw std::runtime_error("error when reading body stream");
}
if (chunkId == numChunks - 1)
{
ret = returnTypeConverter(chunk);
}
};
if (chunkId == numChunks - 1)
{
ret = returnTypeConverter(chunk);
}
};
int64_t remainingOffset = firstChunkOffset + firstChunkLength;
int64_t remainingSize = blobRangeSize - firstChunkLength;
@ -261,11 +262,139 @@ namespace Azure { namespace Storage { namespace Blobs {
}
Details::ConcurrentTransfer(
remainingOffset,
remainingSize,
chunkSize,
options.Concurrency,
downloadChunkFunc);
remainingOffset, remainingSize, chunkSize, options.Concurrency, downloadChunkFunc);
ret.ContentLength = blobRangeSize;
return ret;
}
BlobDownloadInfo BlobClient::DownloadToFile(
const std::string& file,
const DownloadBlobToFileOptions& options) const
{
constexpr int64_t c_defaultChunkSize = 4 * 1024 * 1024;
// Just start downloading using an initial chunk. If it's a small blob, we'll get the whole
// thing in one shot. If it's a large blob, we'll get its full size in Content-Range and can
// keep downloading it in chunks.
int64_t firstChunkOffset = options.Offset.HasValue() ? options.Offset.GetValue() : 0;
int64_t firstChunkLength = c_defaultChunkSize;
if (options.InitialChunkSize.HasValue())
{
firstChunkLength = options.InitialChunkSize.GetValue();
}
if (options.Length.HasValue())
{
firstChunkLength = std::min(firstChunkLength, options.Length.GetValue());
}
DownloadBlobOptions firstChunkOptions;
firstChunkOptions.Context = options.Context;
firstChunkOptions.Offset = options.Offset;
if (firstChunkOptions.Offset.HasValue())
{
firstChunkOptions.Length = firstChunkLength;
}
Details::FileWriter fileWriter(file);
auto firstChunk = Download(firstChunkOptions);
int64_t blobSize;
int64_t blobRangeSize;
if (firstChunkOptions.Offset.HasValue())
{
blobSize = std::stoll(firstChunk.ContentRange.GetValue().substr(
firstChunk.ContentRange.GetValue().find('/') + 1));
blobRangeSize = blobSize - firstChunkOffset;
if (options.Length.HasValue())
{
blobRangeSize = std::min(blobRangeSize, options.Length.GetValue());
}
}
else
{
blobSize = firstChunk.BodyStream->Length();
blobRangeSize = blobSize;
}
firstChunkLength = std::min(firstChunkLength, blobRangeSize);
auto bodyStreamToFile = [](Azure::Core::Http::BodyStream& stream,
Details::FileWriter& fileWriter,
int64_t offset,
int64_t length,
Azure::Core::Context& context) {
constexpr std::size_t bufferSize = 4 * 1024 * 1024;
std::vector<uint8_t> buffer(bufferSize);
while (length > 0)
{
int64_t readSize = std::min(static_cast<int64_t>(bufferSize), length);
int64_t bytesRead
= Azure::Core::Http::BodyStream::ReadToCount(context, stream, buffer.data(), readSize);
if (bytesRead != readSize)
{
throw std::runtime_error("error when reading body stream");
}
fileWriter.Write(buffer.data(), bytesRead, offset);
length -= bytesRead;
offset += bytesRead;
}
};
bodyStreamToFile(
*firstChunk.BodyStream, fileWriter, 0, firstChunkLength, firstChunkOptions.Context);
firstChunk.BodyStream.reset();
auto returnTypeConverter = [](BlobDownloadResponse& response) {
BlobDownloadInfo ret;
ret.ETag = std::move(response.ETag);
ret.LastModified = std::move(response.LastModified);
ret.HttpHeaders = std::move(response.HttpHeaders);
ret.Metadata = std::move(response.Metadata);
ret.BlobType = response.BlobType;
ret.ServerEncrypted = response.ServerEncrypted;
ret.EncryptionKeySHA256 = std::move(response.EncryptionKeySHA256);
return ret;
};
BlobDownloadInfo ret = returnTypeConverter(firstChunk);
// Keep downloading the remaining in parallel
auto downloadChunkFunc
= [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
DownloadBlobOptions chunkOptions;
chunkOptions.Context = options.Context;
chunkOptions.Offset = offset;
chunkOptions.Length = length;
auto chunk = Download(chunkOptions);
bodyStreamToFile(
*chunk.BodyStream,
fileWriter,
offset - firstChunkOffset,
chunkOptions.Length.GetValue(),
chunkOptions.Context);
if (chunkId == numChunks - 1)
{
ret = returnTypeConverter(chunk);
}
};
int64_t remainingOffset = firstChunkOffset + firstChunkLength;
int64_t remainingSize = blobRangeSize - firstChunkLength;
int64_t chunkSize;
if (options.ChunkSize.HasValue())
{
chunkSize = options.ChunkSize.GetValue();
}
else
{
int64_t c_grainSize = 4 * 1024;
chunkSize = remainingSize / options.Concurrency;
chunkSize = (std::max(chunkSize, int64_t(1)) + c_grainSize - 1) / c_grainSize * c_grainSize;
chunkSize = std::min(chunkSize, c_defaultChunkSize);
}
Details::ConcurrentTransfer(
remainingOffset, remainingSize, chunkSize, options.Concurrency, downloadChunkFunc);
ret.ContentLength = blobRangeSize;
return ret;
}

62
sdk/storage/src/blobs/block_blob_client.cpp
View File

@ -5,6 +5,7 @@
#include "common/concurrent_transfer.hpp"
#include "common/crypt.hpp"
#include "common/file_io.hpp"
#include "common/storage_common.hpp"
namespace Azure { namespace Storage { namespace Blobs {
@ -135,6 +136,67 @@ namespace Azure { namespace Storage { namespace Blobs {
return commitBlockListResponse;
}
BlobContentInfo BlockBlobClient::UploadFromFile(
const std::string& file,
const UploadBlobOptions& options) const
{
constexpr int64_t c_defaultBlockSize = 8 * 1024 * 1024;
constexpr int64_t c_maximumNumberBlocks = 50000;
constexpr int64_t c_grainSize = 4 * 1024;
Details::FileReader fileReader(file);
int64_t chunkSize = c_defaultBlockSize;
if (options.ChunkSize.HasValue())
{
chunkSize = options.ChunkSize.GetValue();
}
else
{
int64_t minBlockSize
= (fileReader.GetFileSize() + c_maximumNumberBlocks - 1) / c_maximumNumberBlocks;
chunkSize = std::max(chunkSize, minBlockSize);
chunkSize = (chunkSize + c_grainSize - 1) / c_grainSize * c_grainSize;
}
std::vector<std::pair<BlockType, std::string>> blockIds;
auto getBlockId = [](int64_t id) {
constexpr std::size_t c_blockIdLength = 64;
std::string blockId = std::to_string(id);
blockId = std::string(c_blockIdLength - blockId.length(), '0') + blockId;
return Base64Encode(blockId);
};
auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
Azure::Core::Http::FileBodyStream contentStream(fileReader.GetHandle(), offset, length);
StageBlockOptions chunkOptions;
chunkOptions.Context = options.Context;
auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions);
if (chunkId == numChunks - 1)
{
blockIds.resize(static_cast<std::size_t>(numChunks));
}
};
Details::ConcurrentTransfer(
0, fileReader.GetFileSize(), chunkSize, options.Concurrency, uploadBlockFunc);
for (std::size_t i = 0; i < blockIds.size(); ++i)
{
blockIds[i].first = BlockType::Uncommitted;
blockIds[i].second = getBlockId(static_cast<int64_t>(i));
}
CommitBlockListOptions commitBlockListOptions;
commitBlockListOptions.Context = options.Context;
commitBlockListOptions.HttpHeaders = options.HttpHeaders;
commitBlockListOptions.Metadata = options.Metadata;
commitBlockListOptions.Tier = options.Tier;
auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions);
commitBlockListResponse.ContentCRC64.Reset();
commitBlockListResponse.ContentMD5.Reset();
return commitBlockListResponse;
}
BlockInfo BlockBlobClient::StageBlock(
const std::string& blockId,
Azure::Core::Http::BodyStream& content,

129
sdk/storage/src/common/file_io.cpp Normal file
View File

@ -0,0 +1,129 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// SPDX-License-Identifier: MIT
#include "common/file_io.hpp"
#ifndef _WIN32
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#endif
#include <limits>
#include <stdexcept>
namespace Azure { namespace Storage { namespace Details {
#ifdef _WIN32
FileReader::FileReader(const std::string& filename)
{
m_handle = CreateFile(
filename.data(),
GENERIC_READ,
FILE_SHARE_READ,
nullptr,
OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL,
NULL);
if (m_handle == INVALID_HANDLE_VALUE)
{
throw std::runtime_error("failed to open file");
}
LARGE_INTEGER fileSize;
BOOL ret = GetFileSizeEx(m_handle, &fileSize);
if (!ret)
{
CloseHandle(m_handle);
throw std::runtime_error("failed to get size of file");
}
m_fileSize = fileSize.QuadPart;
}
FileReader::~FileReader() { CloseHandle(m_handle); }
FileWriter::FileWriter(const std::string& filename)
{
m_handle = CreateFile(
filename.data(),
GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE,
nullptr,
CREATE_ALWAYS,
FILE_ATTRIBUTE_NORMAL,
NULL);
if (m_handle == INVALID_HANDLE_VALUE)
{
throw std::runtime_error("failed to open file");
}
}
FileWriter::~FileWriter() { CloseHandle(m_handle); }
void FileWriter::Write(const uint8_t* buffer, int64_t length, int64_t offset)
{
if (length > std::numeric_limits<DWORD>::max())
{
throw std::runtime_error("failed to write file");
}
OVERLAPPED overlapped;
std::memset(&overlapped, 0, sizeof(overlapped));
overlapped.Offset = static_cast<DWORD>(static_cast<uint64_t>(offset));
overlapped.OffsetHigh = static_cast<DWORD>(static_cast<uint64_t>(offset) >> 32);
DWORD bytesWritten;
BOOL ret = WriteFile(m_handle, buffer, static_cast<DWORD>(length), &bytesWritten, &overlapped);
if (!ret)
{
throw std::runtime_error("failed to write file");
}
}
#else
FileReader::FileReader(const std::string& filename)
{
m_handle = open(filename.data(), O_RDONLY);
if (m_handle == -1)
{
throw std::runtime_error("failed to open file");
}
m_fileSize = lseek(m_handle, 0, SEEK_END);
if (m_fileSize == -1)
{
close(m_handle);
throw std::runtime_error("failed to get size of file");
}
}
FileReader::~FileReader() { close(m_handle); }
FileWriter::FileWriter(const std::string& filename)
{
m_handle = open(
filename.data(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
if (m_handle == -1)
{
throw std::runtime_error("failed to open file");
}
}
FileWriter::~FileWriter() { close(m_handle); }
void FileWriter::Write(const uint8_t* buffer, int64_t length, int64_t offset)
{
if (static_cast<uint64_t>(length) > std::numeric_limits<size_t>::max()
|| offset > static_cast<int64_t>(std::numeric_limits<off_t>::max()))
{
throw std::runtime_error("failed to write file");
}
ssize_t bytesWritten
= pwrite(m_handle, buffer, static_cast<size_t>(length), static_cast<off_t>(offset));
if (bytesWritten != length)
{
throw std::runtime_error("failed to write file");
}
}
#endif
}}} // namespace Azure::Storage::Details

153
sdk/storage/test/block_blob_client_test.cpp
View File

@ -4,6 +4,7 @@
#include "block_blob_client_test.hpp"
#include "common/crypt.hpp"
#include "common/file_io.hpp"
#include <future>
#include <random>
@ -256,6 +257,7 @@ namespace Azure { namespace Storage { namespace Test {
TEST_F(BlockBlobClientTest, ConcurrentDownload)
{
std::string tempFilename = RandomString();
std::vector<uint8_t> downloadBuffer = m_blobContent;
for (int c : {1, 2, 4})
{
@ -267,6 +269,11 @@ namespace Azure { namespace Storage { namespace Test {
auto res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size());
EXPECT_EQ(downloadBuffer, m_blobContent);
EXPECT_EQ(static_cast<std::size_t>(res.ContentLength), downloadBuffer.size());
res = m_blockBlobClient->DownloadToFile(tempFilename);
auto downloadFile = ReadFile(tempFilename);
EXPECT_EQ(downloadFile, m_blobContent);
EXPECT_EQ(static_cast<std::size_t>(res.ContentLength), downloadFile.size());
DeleteFile(tempFilename);
// download whole blob
downloadBuffer.assign(downloadBuffer.size(), '\x00');
@ -274,6 +281,11 @@ namespace Azure { namespace Storage { namespace Test {
res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size());
EXPECT_EQ(downloadBuffer, m_blobContent);
EXPECT_EQ(static_cast<std::size_t>(res.ContentLength), downloadBuffer.size());
res = m_blockBlobClient->DownloadToFile(tempFilename);
downloadFile = ReadFile(tempFilename);
EXPECT_EQ(downloadFile, m_blobContent);
EXPECT_EQ(static_cast<std::size_t>(res.ContentLength), downloadFile.size());
DeleteFile(tempFilename);
// download whole blob
downloadBuffer.assign(downloadBuffer.size(), '\x00');
@ -282,6 +294,11 @@ namespace Azure { namespace Storage { namespace Test {
res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size());
EXPECT_EQ(downloadBuffer, m_blobContent);
EXPECT_EQ(static_cast<std::size_t>(res.ContentLength), downloadBuffer.size());
res = m_blockBlobClient->DownloadToFile(tempFilename);
downloadFile = ReadFile(tempFilename);
EXPECT_EQ(downloadFile, m_blobContent);
EXPECT_EQ(static_cast<std::size_t>(res.ContentLength), downloadFile.size());
DeleteFile(tempFilename);
// download whole blob
downloadBuffer.assign(downloadBuffer.size(), '\x00');
@ -290,6 +307,11 @@ namespace Azure { namespace Storage { namespace Test {
res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size() * 2);
EXPECT_EQ(downloadBuffer, m_blobContent);
EXPECT_EQ(static_cast<std::size_t>(res.ContentLength), downloadBuffer.size());
res = m_blockBlobClient->DownloadToFile(tempFilename);
downloadFile = ReadFile(tempFilename);
EXPECT_EQ(downloadFile, m_blobContent);
EXPECT_EQ(static_cast<std::size_t>(res.ContentLength), downloadFile.size());
DeleteFile(tempFilename);
options.InitialChunkSize = 4_KB;
options.ChunkSize = 4_KB;
@ -311,14 +333,28 @@ namespace Azure { namespace Storage { namespace Test {
std::vector<uint8_t>(
m_blobContent.begin() + static_cast<std::size_t>(offset),
m_blobContent.begin() + static_cast<std::size_t>(offset)
+ downloadContent.size()));
+ static_cast<std::size_t>(actualLength)));
EXPECT_EQ(res.ContentLength, actualLength);
std::string tempFilename2 = RandomString();
res = m_blockBlobClient->DownloadToFile(tempFilename2, optionsCopy);
auto downloadFile = ReadFile(tempFilename2);
EXPECT_EQ(
downloadFile,
std::vector<uint8_t>(
m_blobContent.begin() + static_cast<std::size_t>(offset),
m_blobContent.begin() + static_cast<std::size_t>(offset)
+ static_cast<std::size_t>(actualLength)));
EXPECT_EQ(res.ContentLength, actualLength);
DeleteFile(tempFilename2);
}
else
{
EXPECT_THROW(
m_blockBlobClient->DownloadToBuffer(nullptr, 8 * 1024 * 1024, optionsCopy),
std::runtime_error);
EXPECT_THROW(
m_blockBlobClient->DownloadToFile(tempFilename, optionsCopy), std::runtime_error);
}
};
@ -365,6 +401,8 @@ namespace Azure { namespace Storage { namespace Test {
TEST_F(BlockBlobClientTest, ConcurrentDownloadEmptyBlob)
{
std::string tempFilename = RandomString();
std::vector<uint8_t> emptyContent;
auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString(
StandardStorageConnectionString(), m_containerName, RandomString());
@ -381,6 +419,15 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders);
EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata);
EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob);
res = blockBlobClient.DownloadToFile(tempFilename);
EXPECT_EQ(res.ContentLength, 0);
EXPECT_FALSE(res.ETag.empty());
EXPECT_FALSE(res.LastModified.empty());
EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders);
EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata);
EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob);
EXPECT_TRUE(ReadFile(tempFilename).empty());
DeleteFile(tempFilename);
res = blockBlobClient.DownloadToBuffer(emptyContent.data(), static_cast<std::size_t>(8_MB));
EXPECT_EQ(res.ContentLength, 0);
@ -389,6 +436,15 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders);
EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata);
EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob);
res = blockBlobClient.DownloadToFile(tempFilename);
EXPECT_EQ(res.ContentLength, 0);
EXPECT_FALSE(res.ETag.empty());
EXPECT_FALSE(res.LastModified.empty());
EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders);
EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata);
EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob);
EXPECT_TRUE(ReadFile(tempFilename).empty());
DeleteFile(tempFilename);
for (int c : {1, 2})
{
@ -405,18 +461,29 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders);
EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata);
EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob);
res = blockBlobClient.DownloadToFile(tempFilename, options);
EXPECT_EQ(res.ContentLength, 0);
EXPECT_FALSE(res.ETag.empty());
EXPECT_FALSE(res.LastModified.empty());
EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders);
EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata);
EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob);
EXPECT_TRUE(ReadFile(tempFilename).empty());
DeleteFile(tempFilename);
options.Offset = 0;
EXPECT_THROW(
blockBlobClient.DownloadToBuffer(
emptyContent.data(), static_cast<std::size_t>(8_MB), options),
std::runtime_error);
EXPECT_THROW(blockBlobClient.DownloadToFile(tempFilename, options), std::runtime_error);
options.Offset = 1;
EXPECT_THROW(
blockBlobClient.DownloadToBuffer(
emptyContent.data(), static_cast<std::size_t>(8_MB), options),
std::runtime_error);
EXPECT_THROW(blockBlobClient.DownloadToFile(tempFilename, options), std::runtime_error);
options.Offset = 0;
options.Length = 1;
@ -424,6 +491,7 @@ namespace Azure { namespace Storage { namespace Test {
blockBlobClient.DownloadToBuffer(
emptyContent.data(), static_cast<std::size_t>(8_MB), options),
std::runtime_error);
EXPECT_THROW(blockBlobClient.DownloadToFile(tempFilename, options), std::runtime_error);
options.Offset = 100;
options.Length = 100;
@ -431,11 +499,14 @@ namespace Azure { namespace Storage { namespace Test {
blockBlobClient.DownloadToBuffer(
emptyContent.data(), static_cast<std::size_t>(8_MB), options),
std::runtime_error);
EXPECT_THROW(blockBlobClient.DownloadToFile(tempFilename, options), std::runtime_error);
}
}
TEST_F(BlockBlobClientTest, ConcurrentUpload)
{
std::string tempFilename = RandomString();
auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString(
StandardStorageConnectionString(), m_containerName, RandomString());
for (int c : {1, 2, 5})
@ -449,32 +520,64 @@ namespace Azure { namespace Storage { namespace Test {
options.HttpHeaders = m_blobUploadOptions.HttpHeaders;
options.Metadata = m_blobUploadOptions.Metadata;
options.Tier = m_blobUploadOptions.Tier;
auto res = blockBlobClient.UploadFromBuffer(
m_blobContent.data(), static_cast<std::size_t>(length), options);
EXPECT_FALSE(res.RequestId.empty());
EXPECT_FALSE(res.Version.empty());
EXPECT_FALSE(res.Date.empty());
EXPECT_FALSE(res.ETag.empty());
EXPECT_FALSE(res.LastModified.empty());
EXPECT_FALSE(res.SequenceNumber.HasValue());
EXPECT_FALSE(res.ContentCRC64.HasValue());
EXPECT_FALSE(res.ContentMD5.HasValue());
auto properties = blockBlobClient.GetProperties();
EXPECT_EQ(properties.ContentLength, length);
EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders);
EXPECT_EQ(properties.Metadata, options.Metadata);
EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue());
EXPECT_EQ(properties.ETag, res.ETag);
EXPECT_EQ(properties.LastModified, res.LastModified);
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(length), '\x00');
blockBlobClient.DownloadToBuffer(downloadContent.data(), static_cast<std::size_t>(length));
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
m_blobContent.begin(), m_blobContent.begin() + static_cast<std::size_t>(length)));
{
auto res = blockBlobClient.UploadFromBuffer(
m_blobContent.data(), static_cast<std::size_t>(length), options);
EXPECT_FALSE(res.RequestId.empty());
EXPECT_FALSE(res.Version.empty());
EXPECT_FALSE(res.Date.empty());
EXPECT_FALSE(res.ETag.empty());
EXPECT_FALSE(res.LastModified.empty());
EXPECT_FALSE(res.SequenceNumber.HasValue());
EXPECT_FALSE(res.ContentCRC64.HasValue());
EXPECT_FALSE(res.ContentMD5.HasValue());
auto properties = blockBlobClient.GetProperties();
EXPECT_EQ(properties.ContentLength, length);
EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders);
EXPECT_EQ(properties.Metadata, options.Metadata);
EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue());
EXPECT_EQ(properties.ETag, res.ETag);
EXPECT_EQ(properties.LastModified, res.LastModified);
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(length), '\x00');
blockBlobClient.DownloadToBuffer(
downloadContent.data(), static_cast<std::size_t>(length));
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
m_blobContent.begin(), m_blobContent.begin() + static_cast<std::size_t>(length)));
}
{
{
Azure::Storage::Details::FileWriter fileWriter(tempFilename);
fileWriter.Write(m_blobContent.data(), length, 0);
}
auto res = blockBlobClient.UploadFromFile(tempFilename, options);
EXPECT_FALSE(res.RequestId.empty());
EXPECT_FALSE(res.Version.empty());
EXPECT_FALSE(res.Date.empty());
EXPECT_FALSE(res.ETag.empty());
EXPECT_FALSE(res.LastModified.empty());
EXPECT_FALSE(res.SequenceNumber.HasValue());
EXPECT_FALSE(res.ContentCRC64.HasValue());
EXPECT_FALSE(res.ContentMD5.HasValue());
auto properties = blockBlobClient.GetProperties();
EXPECT_EQ(properties.ContentLength, length);
EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders);
EXPECT_EQ(properties.Metadata, options.Metadata);
EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue());
EXPECT_EQ(properties.ETag, res.ETag);
EXPECT_EQ(properties.LastModified, res.LastModified);
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(length), '\x00');
blockBlobClient.DownloadToBuffer(
downloadContent.data(), static_cast<std::size_t>(length));
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
m_blobContent.begin(), m_blobContent.begin() + static_cast<std::size_t>(length)));
}
}
}
DeleteFile(tempFilename);
}
}}} // namespace Azure::Storage::Test

19
sdk/storage/test/test_base.cpp
View File

@ -9,6 +9,7 @@
#include <algorithm>
#include <cctype>
#include <cstdio>
#include <cstdlib>
#include <limits>
#include <random>
@ -132,4 +133,22 @@ namespace Azure { namespace Storage { namespace Test {
}
}
std::vector<uint8_t> ReadFile(const std::string& filename)
{
FILE* fin = fopen(filename.data(), "rb");
if (!fin)
{
throw std::runtime_error("failed to open file");
}
fseek(fin, 0, SEEK_END);
int64_t fileSize = ftell(fin);
std::vector<uint8_t> fileContent(static_cast<std::size_t>(fileSize));
fseek(fin, 0, SEEK_SET);
fread(fileContent.data(), static_cast<size_t>(fileSize), 1, fin);
fclose(fin);
return fileContent;
}
void DeleteFile(const std::string& filename) { std::remove(filename.data()); }
}}} // namespace Azure::Storage::Test

4
sdk/storage/test/test_base.hpp
View File

@ -41,4 +41,8 @@ namespace Azure { namespace Storage { namespace Test {
return ReadBodyStream(stream);
}
std::vector<uint8_t> ReadFile(const std::string& filename);
void DeleteFile(const std::string& filename);
}}} // namespace Azure::Storage::Test