github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

default chunk size for concurrent upload is nullable (#2162)

Browse Source
This commit is contained in:
JinmingHu 2021-04-28 12:17:07 +08:00 committed by GitHub
parent ff10813ba0
commit 0cc4f0a34b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 40 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sdk/storage/azure-storage-blobs/CHANGELOG.md
View File

@ -5,6 +5,7 @@
### Breaking Changes
- Renamed `HasMorePages()` in paged response to `HasPage()`.
- Default chunk size for concurrent upload was changed to nullable.
## 12.0.0-beta.10 (2021-04-16)

2
sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp
View File

@ -743,7 +743,7 @@ namespace Azure { namespace Storage { namespace Blobs {
* @brief The maximum number of bytes in a single request. This value cannot be larger than
* 4000 MiB.
*/
int64_t ChunkSize = 4 * 1024 * 1024;
Azure::Nullable<int64_t> ChunkSize;
/**
* @brief The maximum number of threads that may be used in a parallel transfer.

38
sdk/storage/azure-storage-blobs/src/block_blob_client.cpp
View File

@ -110,9 +110,26 @@ namespace Azure { namespace Storage { namespace Blobs {
const UploadBlockBlobFromOptions& options,
const Azure::Core::Context& context) const
{
constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL;
constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL;
constexpr int64_t MaxBlockNumber = 50000;
constexpr int64_t BlockGrainSize = 1 * 1024 * 1024;
int64_t chunkSize = std::min(MaxStageBlockSize, options.TransferOptions.ChunkSize);
int64_t chunkSize;
if (options.TransferOptions.ChunkSize.HasValue())
{
chunkSize = options.TransferOptions.ChunkSize.Value();
}
else
{
int64_t minChunkSize = (bufferSize + MaxBlockNumber - 1) / MaxBlockNumber;
minChunkSize = (minChunkSize + BlockGrainSize - 1) / BlockGrainSize * BlockGrainSize;
chunkSize = std::max(DefaultStageBlockSize, minChunkSize);
}
if (chunkSize > MaxStageBlockSize)
{
throw Azure::Core::RequestFailedException("Block size is too big");
}
if (bufferSize <= static_cast<std::size_t>(options.TransferOptions.SingleUploadThreshold))
{
@ -172,7 +189,10 @@ namespace Azure { namespace Storage { namespace Blobs {
const UploadBlockBlobFromOptions& options,
const Azure::Core::Context& context) const
{
constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL;
constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL;
constexpr int64_t MaxBlockNumber = 50000;
constexpr int64_t BlockGrainSize = 1 * 1024 * 1024;
{
Azure::Core::IO::FileBodyStream contentStream(fileName);
@ -209,7 +229,21 @@ namespace Azure { namespace Storage { namespace Blobs {
}
};
int64_t chunkSize = std::min(MaxStageBlockSize, options.TransferOptions.ChunkSize);
int64_t chunkSize;
if (options.TransferOptions.ChunkSize.HasValue())
{
chunkSize = options.TransferOptions.ChunkSize.Value();
}
else
{
int64_t minChunkSize = (fileReader.GetFileSize() + MaxBlockNumber - 1) / MaxBlockNumber;
minChunkSize = (minChunkSize + BlockGrainSize - 1) / BlockGrainSize * BlockGrainSize;
chunkSize = std::max(DefaultStageBlockSize, minChunkSize);
}
if (chunkSize > MaxStageBlockSize)
{
throw Azure::Core::RequestFailedException("Block size is too big");
}
_internal::ConcurrentTransfer(
0,

1
sdk/storage/azure-storage-files-datalake/CHANGELOG.md
View File

@ -5,6 +5,7 @@
### Breaking Changes
- Renamed `HasMorePages()` in paged response to `HasPage()`.
- Default chunk size for concurrent upload was changed to nullable.
## 12.0.0-beta.10 (2021-04-16)

2
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp
View File

@ -571,7 +571,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* @brief The maximum number of bytes in a single request. This value cannot be larger than
* 4000 MiB.
*/
int64_t ChunkSize = 4 * 1024 * 1024;
Azure::Nullable<int64_t> ChunkSize;
/**
* @brief The maximum number of threads that may be used in a parallel transfer.