From 66c7518dce0b48b39ffba68152a4c39c4269d245 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Tue, 7 Jul 2020 10:54:05 +0800 Subject: [PATCH] Suggested changes in API review (#244) * Rename Properties->HttpHeaders * Rename BlobXxxItems->Items * Remove MaxResults in response of list operations * specify time format in comments for GetUSerDelegationKey API * per operation pipelines and per retry pipelines * Define some response types and rename ListBlobsFlat * Add different return types for some APIs * Rename BlobDownloadInfo->BlobDownloadResponse * Assign default value to BlobContentLength to supporess compiler warnings * add concurrent download to buffer * concurrent upload block blob from buffer --- sdk/storage/CMakeLists.txt | 7 +- sdk/storage/inc/blobs/blob_client.hpp | 89 +++- .../inc/blobs/blob_container_client.hpp | 33 +- sdk/storage/inc/blobs/blob_options.hpp | 156 +++++-- sdk/storage/inc/blobs/blob_service_client.hpp | 7 +- sdk/storage/inc/blobs/block_blob_client.hpp | 14 + .../internal/protocol/blob_rest_client.hpp | 432 ++++++++++-------- .../inc/common/concurrent_transfer.hpp | 70 +++ sdk/storage/src/blobs/append_blob_client.cpp | 2 +- sdk/storage/src/blobs/blob_client.cpp | 174 ++++++- .../src/blobs/blob_container_client.cpp | 33 +- sdk/storage/src/blobs/blob_service_client.cpp | 26 +- sdk/storage/src/blobs/block_blob_client.cpp | 64 ++- sdk/storage/src/blobs/page_blob_client.cpp | 2 +- sdk/storage/test/append_blob_client_test.cpp | 15 +- .../test/blob_container_client_test.cpp | 17 +- sdk/storage/test/blob_service_client_test.cpp | 6 +- sdk/storage/test/block_blob_client_test.cpp | 295 +++++++++++- sdk/storage/test/page_blob_client_test.cpp | 15 +- 19 files changed, 1082 insertions(+), 375 deletions(-) create mode 100644 sdk/storage/inc/common/concurrent_transfer.hpp diff --git a/sdk/storage/CMakeLists.txt b/sdk/storage/CMakeLists.txt index b937a8e83..dfb1d45b7 100644 --- a/sdk/storage/CMakeLists.txt +++ b/sdk/storage/CMakeLists.txt @@ -18,6 +18,7 @@ set (AZURE_STORAGE_BLOB_HEADER inc/common/shared_key_policy.hpp inc/common/crypt.hpp inc/common/xml_wrapper.hpp + inc/common/concurrent_transfer.hpp inc/blobs/blob.hpp inc/blobs/blob_service_client.hpp inc/blobs/blob_container_client.hpp @@ -85,11 +86,15 @@ set(AZURE_STORAGE_SOURCE add_library(azure-storage ${AZURE_STORAGE_HEADER} ${AZURE_STORAGE_SOURCE}) +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) +set(THREADS_PREFER_PTHREAD_FLAG TRUE) +find_package(Threads REQUIRED) + find_package(LibXml2 REQUIRED) target_include_directories(azure-storage PUBLIC ${LIBXML2_INCLUDE_DIR} $ $ $) -target_link_libraries(azure-storage azure-core ${LIBXML2_LIBRARIES}) +target_link_libraries(azure-storage Threads::Threads azure-core ${LIBXML2_LIBRARIES}) if(MSVC) target_link_libraries(azure-storage bcrypt) diff --git a/sdk/storage/inc/blobs/blob_client.hpp b/sdk/storage/inc/blobs/blob_client.hpp index dc4f0dbe5..5abf2a643 100644 --- a/sdk/storage/inc/blobs/blob_client.hpp +++ b/sdk/storage/inc/blobs/blob_client.hpp @@ -14,6 +14,18 @@ namespace Azure { namespace Storage { namespace Blobs { + struct BlobDownloadInfo + { + std::string ETag; + std::string LastModified; + int64_t ContentLength = 0; + BlobHttpHeaders HttpHeaders; + std::map Metadata; + Blobs::BlobType BlobType = Blobs::BlobType::Unknown; + Azure::Core::Nullable ServerEncrypted; + Azure::Core::Nullable EncryptionKeySHA256; + }; + class BlockBlobClient; class AppendBlobClient; class PageBlobClient; @@ -144,11 +156,12 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Sets system properties on the blob. * - * @param options Optional - * parameters to execute this function. - * @return A BlobInfo describing the updated blob. + * @param httpHeaders The standard HTTP header system properties to set. + * @param options Optional parameters to execute this function. + * @return A SetBlobHttpHeadersResponse describing the updated blob. */ - BlobInfo SetHttpHeaders( + SetBlobHttpHeadersResponse SetHttpHeaders( + BlobHttpHeaders httpHeaders, const SetBlobHttpHeadersOptions& options = SetBlobHttpHeadersOptions()) const; /** @@ -158,10 +171,9 @@ namespace Azure { namespace Storage { namespace Blobs { * @param metadata Custom metadata to set for this blob. * @param * options Optional parameters to execute this function. - * @return A BlobInfo describing - * the updated blob. + * @return A SetBlobMetadataResponse describing the updated blob. */ - BlobInfo SetMetadata( + SetBlobMetadataResponse SetMetadata( std::map metadata, const SetBlobMetadataOptions& options = SetBlobMetadataOptions()) const; @@ -172,10 +184,9 @@ namespace Azure { namespace Storage { namespace Blobs { * @param Tier Indicates the tier to be set on the blob. * @param options Optional * parameters to execute this function. - * @return A BasicResponse on successfully setting - * the tier. + * @return A SetAccessTierResponse on successfully setting the tier. */ - BasicResponse SetAccessTier( + SetAccessTierResponse SetAccessTier( AccessTier Tier, const SetAccessTierOptions& options = SetAccessTierOptions()) const; @@ -201,23 +212,50 @@ namespace Azure { namespace Storage { namespace Blobs { * * @param copyId ID of the copy operation to abort. * @param options Optional parameters to execute this function. - * @return A BasicResponse - * on successfully aborting. + * @return A AbortCopyBlobResponse on successfully aborting. */ - BasicResponse AbortCopyFromUri( + AbortCopyBlobResponse AbortCopyFromUri( const std::string& copyId, const AbortCopyFromUriOptions& options = AbortCopyFromUriOptions()) const; /** - * @brief Downloads a blob from the service, including its metadata and properties. + * @brief Downloads a blob or a blob range from the service, including its metadata and + * properties. + * + * @param options Optional parameters to execute this function. + * @return A BlobDownloadResponse describing the downloaded blob. + * BlobDownloadResponse.BodyStream contains the blob's data. + */ + BlobDownloadResponse Download(const DownloadBlobOptions& options = DownloadBlobOptions()) const; - * * + /** + * @brief Downloads a blob or a blob range from the service to a memory buffer using parallel + * requests. + * + * @param buffer A memory buffer to write the blob content to. + * @param bufferSize Size of the memory buffer. Size must be larger or equal to size of the blob + * or blob range. * @param options Optional parameters to execute this function. * @return A * BlobDownloadInfo describing the downloaded blob. - * BlobDownloadInfo.BodyStream contains the blob's data. */ - BlobDownloadInfo Download(const DownloadBlobOptions& options = DownloadBlobOptions()) const; + BlobDownloadInfo DownloadToBuffer( + uint8_t* buffer, + std::size_t bufferSize, + const DownloadBlobToBufferOptions& options = DownloadBlobToBufferOptions()) const; + + /** + * @brief Downloads a blob or a blob range from the service to a file using parallel + * requests. + * + * @param file A file path to write the downloaded content to. + * @param options Optional parameters to execute this function. + * @return A + * BlobDownloadInfo describing the downloaded blob. + */ + BlobDownloadInfo DownloadToFile( + const std::string& file, + const DownloadBlobToFileOptions& options = DownloadBlobToFileOptions()) const; /** * @brief Creates a read-only snapshot of a blob. @@ -236,10 +274,9 @@ namespace Azure { namespace Storage { namespace Blobs { * snapshots. You can delete both at the same time using DeleteBlobOptions.DeleteSnapshots. * * @param options Optional parameters to execute this function. - * @return A - * BasicResponse on successfully deleting. + * @return A DeleteBlobResponse on successfully deleting. */ - BasicResponse Delete(const DeleteBlobOptions& options = DeleteBlobOptions()) const; + DeleteBlobResponse Delete(const DeleteBlobOptions& options = DeleteBlobOptions()) const; /** * @brief Restores the contents and metadata of a soft deleted blob and any associated @@ -247,16 +284,22 @@ namespace Azure { namespace Storage { namespace Blobs { * * @param options Optional parameters to execute this * function. - * @return A BasicResponse on successfully deleting. + * @return A UndeleteBlobResponse on successfully deleting. */ - BasicResponse Undelete(const UndeleteBlobOptions& options = UndeleteBlobOptions()) const; + UndeleteBlobResponse Undelete(const UndeleteBlobOptions& options = UndeleteBlobOptions()) const; protected: UrlBuilder m_blobUrl; std::shared_ptr m_pipeline; private: - BlobClient() = default; + explicit BlobClient( + UrlBuilder blobUri, + std::shared_ptr pipeline) + : m_blobUrl(std::move(blobUri)), m_pipeline(std::move(pipeline)) + { + } + friend class BlobContainerClient; }; }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/inc/blobs/blob_container_client.hpp b/sdk/storage/inc/blobs/blob_container_client.hpp index a25a5b50c..fe3f0ba95 100644 --- a/sdk/storage/inc/blobs/blob_container_client.hpp +++ b/sdk/storage/inc/blobs/blob_container_client.hpp @@ -119,7 +119,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Gets the container's primary uri endpoint. - * + * * @return The * container's primary uri endpoint. */ @@ -128,7 +128,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Creates a new container under the specified account. If the container with the * same name already exists, the operation fails. - * + * * @param options Optional * parameters to execute this function. * @return A BlobContainerInfo describing the newly @@ -140,13 +140,12 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Marks the specified container for deletion. The container and any blobs * contained within it are later deleted during garbage collection. - * + * * @param * options Optional parameters to execute this function. - * @return A BasicResponse if - * successful. + * @return A DeleteContainerResponse if successful. */ - BasicResponse Delete( + DeleteContainerResponse Delete( const DeleteBlobContainerOptions& options = DeleteBlobContainerOptions()) const; /** @@ -167,31 +166,37 @@ namespace Azure { namespace Storage { namespace Blobs { * @param metadata Custom metadata to set for this container. * @param options * Optional parameters to execute this function. - * @return A BlobContainerInfo if - * successful. + * @return A SetContainerMetadataResponse if successful. */ - BlobContainerInfo SetMetadata( + SetContainerMetadataResponse SetMetadata( std::map metadata, SetBlobContainerMetadataOptions options = SetBlobContainerMetadataOptions()) const; /** * @brief Returns a single segment of blobs in this container, starting from the * specified Marker, Use an empty Marker to start enumeration from the beginning and the - * NextMarker if it's not empty to make subsequent calls to ListBlobs to continue enumerating - * the blobs segment by segment. Blobs are ordered lexicographically by name. A Delimiter can be - * used to traverse a virtual hierarchy of blobs as though it were a file system. + * NextMarker if it's not empty to make subsequent calls to ListBlobsFlat to continue + * enumerating the blobs segment by segment. Blobs are ordered lexicographically by name. A + * Delimiter can be used to traverse a virtual hierarchy of blobs as though it were a file + * system. * * @param options Optional parameters to execute this function. * @return A * BlobsFlatSegment describing a segment of the blobs in the container. */ - BlobsFlatSegment ListBlobs(const ListBlobsOptions& options = ListBlobsOptions()) const; + BlobsFlatSegment ListBlobsFlat(const ListBlobsOptions& options = ListBlobsOptions()) const; private: UrlBuilder m_containerUrl; std::shared_ptr m_pipeline; - BlobContainerClient() = default; + explicit BlobContainerClient( + UrlBuilder containerUri, + std::shared_ptr pipeline) + : m_containerUrl(std::move(containerUri)), m_pipeline(std::move(pipeline)) + { + } + friend class BlobServiceClient; }; diff --git a/sdk/storage/inc/blobs/blob_options.hpp b/sdk/storage/inc/blobs/blob_options.hpp index f6050968e..4693478d0 100644 --- a/sdk/storage/inc/blobs/blob_options.hpp +++ b/sdk/storage/inc/blobs/blob_options.hpp @@ -17,10 +17,16 @@ namespace Azure { namespace Storage { namespace Blobs { struct BlobServiceClientOptions { /** - * @brief Transport pipeline policies for authentication, retries, etc., that are - * applied to every request. + * @brief Transport pipeline policies for authentication, additional HTTP headers, etc., that + * are applied to every request. */ - std::vector> policies; + std::vector> PerOperationPolicies; + + /** + * @brief Transport pipeline policies for authentication, additional HTTP headers, etc., that + * are applied to every retrial. + */ + std::vector> PerRetryPolicies; }; /** @@ -77,10 +83,16 @@ namespace Azure { namespace Storage { namespace Blobs { struct BlobContainerClientOptions { /** - * @brief Transport pipeline policies for authentication, retries, etc., that are - * applied to every request. + * @brief Transport pipeline policies for authentication, additional HTTP headers, etc., that + * are applied to every request. */ - std::vector> policies; + std::vector> PerOperationPolicies; + + /** + * @brief Transport pipeline policies for authentication, additional HTTP headers, etc., that + * are applied to every retrial. + */ + std::vector> PerRetryPolicies; }; /** @@ -157,7 +169,7 @@ namespace Azure { namespace Storage { namespace Blobs { }; /** - * @brief Optional parameters for BlobContainerClient::ListBlobs. + * @brief Optional parameters for BlobContainerClient::ListBlobsFlat. */ struct ListBlobsOptions { @@ -205,10 +217,16 @@ namespace Azure { namespace Storage { namespace Blobs { struct BlobClientOptions { /** - * @brief Transport pipeline policies for authentication, retries, etc., that are - * applied to every request. + * @brief Transport pipeline policies for authentication, additional HTTP headers, etc., that + * are applied to every request. */ - std::vector> policies; + std::vector> PerOperationPolicies; + + /** + * @brief Transport pipeline policies for authentication, additional HTTP headers, etc., that + * are applied to every retrial. + */ + std::vector> PerRetryPolicies; }; /** @@ -278,36 +296,6 @@ namespace Azure { namespace Storage { namespace Blobs { */ Azure::Core::Context Context; - /** - * @brief The MIME content type of the blob. - */ - std::string ContentType; - - /** - * @brief Specifies which content encodings have been applied to the blob. - */ - std::string ContentEncoding; - - /** - * @brief Specifies the natural languages used by this resource. - */ - std::string ContentLanguage; - - /** - * @brief Sets the blob’s MD5 hash. - */ - std::string ContentMD5; - - /** - * @brief Sets the blob's cache control. - */ - std::string CacheControl; - - /** - * @brief Sets the blob’s Content-Disposition header. - */ - std::string ContentDisposition; - /** * @brief Specify this header to perform the operation only if the resource has been * modified since the specified time. @@ -545,6 +533,50 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Nullable IfNoneMatch; }; + /** + * @brief Optional parameters for BlobClient::DownloadToBuffer. + */ + struct DownloadBlobToBufferOptions + { + /** + * @brief Context for cancelling long running operations. + */ + Azure::Core::Context Context; + + /** + * @brief Downloads only the bytes of the blob from this offset. + */ + Azure::Core::Nullable Offset; + + /** + * @brief Returns at most this number of bytes of the blob from the offset. Null means + * download until the end. + */ + Azure::Core::Nullable Length; + + /** + * @brief The size of the first range request in bytes. Blobs smaller than this limit will be + * downloaded in a single request. Blobs larger than this limit will continue being downloaded + * in chunks of size ChunkSize. + */ + Azure::Core::Nullable InitialChunkSize; + + /** + * @brief The maximum number of bytes in a single request. + */ + Azure::Core::Nullable ChunkSize; + + /** + * @brief The maximum number of threads that may be used in a parallel transfer. + */ + int Concurrency = 1; + }; + + /** + * @brief Optional parameters for BlobClient::DownloadToFile. + */ + using DownloadBlobToFileOptions = DownloadBlobToBufferOptions; + /** * @brief Optional parameters for BlobClient::CreateSnapshot. */ @@ -676,7 +708,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief The standard HTTP header system properties to set. */ - BlobHttpHeaders Properties; + BlobHttpHeaders HttpHeaders; /** * @brief Name-value pairs associated with the blob as metadata. @@ -714,6 +746,42 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Nullable IfNoneMatch; }; + /** + * @brief Optional parameters for BlockBlobClient::UploadFromBuffer. + */ + struct UploadBlobOptions + { + /** + * @brief Context for cancelling long running operations. + */ + Azure::Core::Context Context; + + /** + * @brief The standard HTTP header system properties to set. + */ + BlobHttpHeaders HttpHeaders; + + /** + * @brief Name-value pairs associated with the blob as metadata. + */ + std::map Metadata; + + /** + * @brief Indicates the tier to be set on blob. + */ + Azure::Core::Nullable Tier; + + /** + * @brief The maximum number of bytes in a single request. + */ + Azure::Core::Nullable ChunkSize; + + /** + * @brief The maximum number of threads that may be used in a parallel transfer. + */ + int Concurrency = 1; + }; + /** * @brief Optional parameters for BlockBlobClient::StageBlock. */ @@ -818,7 +886,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief The standard HTTP header system properties to set. */ - BlobHttpHeaders Properties; + BlobHttpHeaders HttpHeaders; /** * @brief Name-value pairs associated with the blob as metadata. @@ -906,7 +974,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief The standard HTTP header system properties to set. */ - BlobHttpHeaders Properties; + BlobHttpHeaders HttpHeaders; /** * @brief Name-value pairs associated with the blob as metadata. @@ -1100,7 +1168,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief The standard HTTP header system properties to set. */ - BlobHttpHeaders Properties; + BlobHttpHeaders HttpHeaders; /** * @brief Name-value pairs associated with the blob as metadata. diff --git a/sdk/storage/inc/blobs/blob_service_client.hpp b/sdk/storage/inc/blobs/blob_service_client.hpp index 7e38185e0..b0febaf10 100644 --- a/sdk/storage/inc/blobs/blob_service_client.hpp +++ b/sdk/storage/inc/blobs/blob_service_client.hpp @@ -110,9 +110,10 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Retrieves a key that can be used to delegate Active Directory authorization to * shared access signatures. * - * @param startsOn Start time for the key's validity. The time should be specified in UTC. - * @param expiresOn Expiration of the key's validity. - * The time should be specified in UTC. + * @param startsOn Start time for the key's validity, in ISO date format. The time should be + * specified in UTC. + * @param expiresOn Expiration of the key's validity, in ISO date format. The time should be + * specified in UTC. * @param options Optional parameters to execute * this function. * @return A deserialized UserDelegationKey instance. diff --git a/sdk/storage/inc/blobs/block_blob_client.hpp b/sdk/storage/inc/blobs/block_blob_client.hpp index a1f888acb..cdef42de4 100644 --- a/sdk/storage/inc/blobs/block_blob_client.hpp +++ b/sdk/storage/inc/blobs/block_blob_client.hpp @@ -113,6 +113,20 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Http::BodyStream& content, const UploadBlockBlobOptions& options = UploadBlockBlobOptions()) const; + /** + * @brief Creates a new block blob, or updates the content of an existing block blob. Updating + * an existing block blob overwrites any existing metadata on the blob. + * + * @param buffer A memory buffer containing the content to upload. + * @param bufferSize Size of the memory buffer. + * @param options Optional parameters to execute this function. + * @return A BlockBlobInfo describing the state of the updated block blob. + */ + BlobContentInfo UploadFromBuffer( + const uint8_t* buffer, + std::size_t bufferSize, + const UploadBlobOptions& options = UploadBlobOptions()) const; + /** * @brief Creates a new block as part of a block blob's staging area to be eventually * committed via the CommitBlockList operation. diff --git a/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp b/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp index fe4d5b838..1018dc5a3 100644 --- a/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp +++ b/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp @@ -28,6 +28,14 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Http::BodyStream, std::function>; + struct AbortCopyBlobResponse + { + std::string RequestId; + std::string Date; + std::string Version; + Azure::Core::Nullable ClientRequestId; + }; // struct AbortCopyBlobResponse + enum class AccessTier { Unknown, @@ -172,14 +180,6 @@ namespace Azure { namespace Storage { namespace Blobs { throw std::runtime_error("cannot convert " + access_tier + " to AccessTier"); } - struct BasicResponse - { - std::string RequestId; - std::string Date; - std::string Version; - Azure::Core::Nullable ClientRequestId; - }; // struct BasicResponse - struct BlobAppendInfo { std::string RequestId; @@ -276,17 +276,6 @@ namespace Azure { namespace Storage { namespace Blobs { std::string ContentDisposition; }; // struct BlobHttpHeaders - struct BlobInfo - { - std::string RequestId; - std::string Date; - std::string Version; - Azure::Core::Nullable ClientRequestId; - std::string ETag; - std::string LastModified; - Azure::Core::Nullable SequenceNumber; - }; // struct BlobInfo - enum class BlobLeaseState { Available, @@ -562,6 +551,22 @@ namespace Azure { namespace Storage { namespace Blobs { throw std::runtime_error("cannot convert " + copy_status + " to CopyStatus"); } + struct DeleteBlobResponse + { + std::string RequestId; + std::string Date; + std::string Version; + Azure::Core::Nullable ClientRequestId; + }; // struct DeleteBlobResponse + + struct DeleteContainerResponse + { + std::string RequestId; + std::string Date; + std::string Version; + Azure::Core::Nullable ClientRequestId; + }; // struct DeleteContainerResponse + enum class DeleteSnapshotsOption { None, @@ -813,6 +818,54 @@ namespace Azure { namespace Storage { namespace Blobs { throw std::runtime_error("cannot convert " + rehydrate_priority + " to RehydratePriority"); } + struct SetAccessTierResponse + { + std::string RequestId; + std::string Date; + std::string Version; + Azure::Core::Nullable ClientRequestId; + }; // struct SetAccessTierResponse + + struct SetBlobHttpHeadersResponse + { + std::string RequestId; + std::string Date; + std::string Version; + Azure::Core::Nullable ClientRequestId; + std::string ETag; + std::string LastModified; + Azure::Core::Nullable SequenceNumber; + }; // struct SetBlobHttpHeadersResponse + + struct SetBlobMetadataResponse + { + std::string RequestId; + std::string Date; + std::string Version; + Azure::Core::Nullable ClientRequestId; + std::string ETag; + std::string LastModified; + Azure::Core::Nullable SequenceNumber; + }; // struct SetBlobMetadataResponse + + struct SetContainerMetadataResponse + { + std::string RequestId; + std::string Date; + std::string Version; + Azure::Core::Nullable ClientRequestId; + std::string ETag; + std::string LastModified; + }; // struct SetContainerMetadataResponse + + struct UndeleteBlobResponse + { + std::string RequestId; + std::string Date; + std::string Version; + Azure::Core::Nullable ClientRequestId; + }; // struct UndeleteBlobResponse + struct UserDelegationKey { std::string RequestId; @@ -885,7 +938,7 @@ namespace Azure { namespace Storage { namespace Blobs { Blobs::CopyStatus CopyStatus = Blobs::CopyStatus::Unknown; }; // struct BlobCopyInfo - struct BlobDownloadInfo + struct BlobDownloadResponse { std::string RequestId; std::string Date; @@ -895,7 +948,7 @@ namespace Azure { namespace Storage { namespace Blobs { std::string ETag; std::string LastModified; Azure::Core::Nullable ContentRange; - BlobHttpHeaders Properties; + BlobHttpHeaders HttpHeaders; std::map Metadata; Azure::Core::Nullable SequenceNumber; // only for page blob Azure::Core::Nullable CommittedBlockCount; // only for append blob @@ -907,14 +960,14 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Nullable LeaseStatus; Azure::Core::Nullable ServerEncrypted; Azure::Core::Nullable EncryptionKeySHA256; - }; // struct BlobDownloadInfo + }; // struct BlobDownloadResponse struct BlobItem { std::string Name; bool Deleted = false; std::string Snapshot; - BlobHttpHeaders Properties; + BlobHttpHeaders HttpHeaders; std::map Metadata; std::string CreationTime; std::string LastModified; @@ -945,12 +998,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Nullable LeaseState; Azure::Core::Nullable LeaseStatus; int64_t ContentLength = 0; - std::string ContentType; - std::string ContentEncoding; - std::string ContentLanguage; - std::string ContentMD5; - std::string CacheControl; - std::string ContentDisposition; + BlobHttpHeaders HttpHeaders; Azure::Core::Nullable SequenceNumber; // only for page blob Azure::Core::Nullable CommittedBlockCount; // only for append blob Azure::Core::Nullable ServerEncrypted; @@ -977,9 +1025,8 @@ namespace Azure { namespace Storage { namespace Blobs { std::string Prefix; std::string Marker; std::string NextMarker; - Azure::Core::Nullable MaxResults; std::string Delimiter; - std::vector BlobItems; + std::vector Items; }; // struct BlobsFlatSegment struct ListContainersSegment @@ -992,8 +1039,7 @@ namespace Azure { namespace Storage { namespace Blobs { std::string Prefix; std::string Marker; std::string NextMarker; - Azure::Core::Nullable MaxResults; - std::vector BlobContainerItems; + std::vector Items; }; // struct ListContainersSegment class BlobRestClient { @@ -1201,7 +1247,6 @@ namespace Azure { namespace Storage { namespace Blobs { k_Prefix, k_Marker, k_NextMarker, - k_MaxResults, k_Containers, k_Container, k_Unknown, @@ -1243,10 +1288,6 @@ namespace Azure { namespace Storage { namespace Blobs { { path.emplace_back(XmlTagName::k_NextMarker); } - else if (std::strcmp(node.Name, "MaxResults") == 0) - { - path.emplace_back(XmlTagName::k_MaxResults); - } else if (std::strcmp(node.Name, "Containers") == 0) { path.emplace_back(XmlTagName::k_Containers); @@ -1262,7 +1303,7 @@ namespace Azure { namespace Storage { namespace Blobs { if (path.size() == 3 && path[0] == XmlTagName::k_EnumerationResults && path[1] == XmlTagName::k_Containers && path[2] == XmlTagName::k_Container) { - ret.BlobContainerItems.emplace_back(BlobContainerItemFromXml(reader)); + ret.Items.emplace_back(BlobContainerItemFromXml(reader)); path.pop_back(); } } @@ -1285,12 +1326,6 @@ namespace Azure { namespace Storage { namespace Blobs { { ret.NextMarker = node.Value; } - else if ( - path.size() == 2 && path[0] == XmlTagName::k_EnumerationResults - && path[1] == XmlTagName::k_MaxResults) - { - ret.MaxResults = std::stoi(node.Value); - } } else if (node.Type == XmlNodeType::Attribute) { @@ -1746,13 +1781,13 @@ namespace Azure { namespace Storage { namespace Blobs { return request; } - static BasicResponse DeleteParseResponse( + static DeleteContainerResponse DeleteParseResponse( Azure::Core::Context context, std::unique_ptr pHttpResponse) { unused(context); Azure::Core::Http::Response& httpResponse = *pHttpResponse; - BasicResponse response; + DeleteContainerResponse response; auto http_status_code = static_cast::type>( httpResponse.GetStatusCode()); @@ -1772,7 +1807,7 @@ namespace Azure { namespace Storage { namespace Blobs { return response; } - static BasicResponse Delete( + static DeleteContainerResponse Delete( Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, @@ -1931,13 +1966,13 @@ namespace Azure { namespace Storage { namespace Blobs { return request; } - static BlobContainerInfo SetMetadataParseResponse( + static SetContainerMetadataResponse SetMetadataParseResponse( Azure::Core::Context context, std::unique_ptr pHttpResponse) { unused(context); Azure::Core::Http::Response& httpResponse = *pHttpResponse; - BlobContainerInfo response; + SetContainerMetadataResponse response; auto http_status_code = static_cast::type>( httpResponse.GetStatusCode()); @@ -1959,7 +1994,7 @@ namespace Azure { namespace Storage { namespace Blobs { return response; } - static BlobContainerInfo SetMetadata( + static SetContainerMetadataResponse SetMetadata( Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, @@ -2094,7 +2129,6 @@ namespace Azure { namespace Storage { namespace Blobs { k_Prefix, k_Marker, k_NextMarker, - k_MaxResults, k_Delimiter, k_Blobs, k_Blob, @@ -2137,10 +2171,6 @@ namespace Azure { namespace Storage { namespace Blobs { { path.emplace_back(XmlTagName::k_NextMarker); } - else if (std::strcmp(node.Name, "MaxResults") == 0) - { - path.emplace_back(XmlTagName::k_MaxResults); - } else if (std::strcmp(node.Name, "Delimiter") == 0) { path.emplace_back(XmlTagName::k_Delimiter); @@ -2160,7 +2190,7 @@ namespace Azure { namespace Storage { namespace Blobs { if (path.size() == 3 && path[0] == XmlTagName::k_EnumerationResults && path[1] == XmlTagName::k_Blobs && path[2] == XmlTagName::k_Blob) { - ret.BlobItems.emplace_back(BlobItemFromXml(reader)); + ret.Items.emplace_back(BlobItemFromXml(reader)); path.pop_back(); } } @@ -2183,12 +2213,6 @@ namespace Azure { namespace Storage { namespace Blobs { { ret.NextMarker = node.Value; } - else if ( - path.size() == 2 && path[0] == XmlTagName::k_EnumerationResults - && path[1] == XmlTagName::k_MaxResults) - { - ret.MaxResults = std::stoi(node.Value); - } else if ( path.size() == 2 && path[0] == XmlTagName::k_EnumerationResults && path[1] == XmlTagName::k_Delimiter) @@ -2385,37 +2409,37 @@ namespace Azure { namespace Storage { namespace Blobs { path.size() == 2 && path[0] == XmlTagName::k_Properties && path[1] == XmlTagName::k_ContentType) { - ret.Properties.ContentType = node.Value; + ret.HttpHeaders.ContentType = node.Value; } else if ( path.size() == 2 && path[0] == XmlTagName::k_Properties && path[1] == XmlTagName::k_ContentEncoding) { - ret.Properties.ContentEncoding = node.Value; + ret.HttpHeaders.ContentEncoding = node.Value; } else if ( path.size() == 2 && path[0] == XmlTagName::k_Properties && path[1] == XmlTagName::k_ContentLanguage) { - ret.Properties.ContentLanguage = node.Value; + ret.HttpHeaders.ContentLanguage = node.Value; } else if ( path.size() == 2 && path[0] == XmlTagName::k_Properties && path[1] == XmlTagName::k_ContentMD5) { - ret.Properties.ContentMD5 = node.Value; + ret.HttpHeaders.ContentMD5 = node.Value; } else if ( path.size() == 2 && path[0] == XmlTagName::k_Properties && path[1] == XmlTagName::k_CacheControl) { - ret.Properties.CacheControl = node.Value; + ret.HttpHeaders.CacheControl = node.Value; } else if ( path.size() == 2 && path[0] == XmlTagName::k_Properties && path[1] == XmlTagName::k_ContentDisposition) { - ret.Properties.ContentDisposition = node.Value; + ret.HttpHeaders.ContentDisposition = node.Value; } else if ( path.size() == 2 && path[0] == XmlTagName::k_Properties @@ -2604,13 +2628,13 @@ namespace Azure { namespace Storage { namespace Blobs { return request; } - static BlobDownloadInfo DownloadParseResponse( + static BlobDownloadResponse DownloadParseResponse( Azure::Core::Context context, std::unique_ptr pHttpResponse) { unused(context); Azure::Core::Http::Response& httpResponse = *pHttpResponse; - BlobDownloadInfo response; + BlobDownloadResponse response; auto http_status_code = static_cast::type>( httpResponse.GetStatusCode()); @@ -2639,44 +2663,44 @@ namespace Azure { namespace Storage { namespace Blobs { { response.ContentCRC64 = response_content_crc64_iterator->second; } - auto response_properties_content_type_iterator + auto response_http_headers_content_type_iterator = httpResponse.GetHeaders().find("Content-Type"); - if (response_properties_content_type_iterator != httpResponse.GetHeaders().end()) + if (response_http_headers_content_type_iterator != httpResponse.GetHeaders().end()) { - response.Properties.ContentType = response_properties_content_type_iterator->second; + response.HttpHeaders.ContentType = response_http_headers_content_type_iterator->second; } - auto response_properties_content_encoding_iterator + auto response_http_headers_content_encoding_iterator = httpResponse.GetHeaders().find("Content-Encoding"); - if (response_properties_content_encoding_iterator != httpResponse.GetHeaders().end()) + if (response_http_headers_content_encoding_iterator != httpResponse.GetHeaders().end()) { - response.Properties.ContentEncoding - = response_properties_content_encoding_iterator->second; + response.HttpHeaders.ContentEncoding + = response_http_headers_content_encoding_iterator->second; } - auto response_properties_content_language_iterator + auto response_http_headers_content_language_iterator = httpResponse.GetHeaders().find("Content-Language"); - if (response_properties_content_language_iterator != httpResponse.GetHeaders().end()) + if (response_http_headers_content_language_iterator != httpResponse.GetHeaders().end()) { - response.Properties.ContentLanguage - = response_properties_content_language_iterator->second; + response.HttpHeaders.ContentLanguage + = response_http_headers_content_language_iterator->second; } - auto response_properties_cache_control_iterator + auto response_http_headers_cache_control_iterator = httpResponse.GetHeaders().find("Cache-Control"); - if (response_properties_cache_control_iterator != httpResponse.GetHeaders().end()) + if (response_http_headers_cache_control_iterator != httpResponse.GetHeaders().end()) { - response.Properties.CacheControl = response_properties_cache_control_iterator->second; + response.HttpHeaders.CacheControl = response_http_headers_cache_control_iterator->second; } - auto response_properties_content_md5_iterator + auto response_http_headers_content_md5_iterator = httpResponse.GetHeaders().find("Content-MD5"); - if (response_properties_content_md5_iterator != httpResponse.GetHeaders().end()) + if (response_http_headers_content_md5_iterator != httpResponse.GetHeaders().end()) { - response.Properties.ContentMD5 = response_properties_content_md5_iterator->second; + response.HttpHeaders.ContentMD5 = response_http_headers_content_md5_iterator->second; } - auto response_properties_content_disposition_iterator + auto response_http_headers_content_disposition_iterator = httpResponse.GetHeaders().find("Content-Disposition"); - if (response_properties_content_disposition_iterator != httpResponse.GetHeaders().end()) + if (response_http_headers_content_disposition_iterator != httpResponse.GetHeaders().end()) { - response.Properties.ContentDisposition - = response_properties_content_disposition_iterator->second; + response.HttpHeaders.ContentDisposition + = response_http_headers_content_disposition_iterator->second; } for (auto i = httpResponse.GetHeaders().lower_bound("x-ms-meta-"); i != httpResponse.GetHeaders().end() && i->first.substr(0, 10) == "x-ms-meta-"; @@ -2735,7 +2759,7 @@ namespace Azure { namespace Storage { namespace Blobs { return response; } - static BlobDownloadInfo Download( + static BlobDownloadResponse Download( Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, @@ -2796,13 +2820,13 @@ namespace Azure { namespace Storage { namespace Blobs { return request; } - static BasicResponse DeleteParseResponse( + static DeleteBlobResponse DeleteParseResponse( Azure::Core::Context context, std::unique_ptr pHttpResponse) { unused(context); Azure::Core::Http::Response& httpResponse = *pHttpResponse; - BasicResponse response; + DeleteBlobResponse response; auto http_status_code = static_cast::type>( httpResponse.GetStatusCode()); @@ -2822,7 +2846,7 @@ namespace Azure { namespace Storage { namespace Blobs { return response; } - static BasicResponse Delete( + static DeleteBlobResponse Delete( Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, @@ -2857,13 +2881,13 @@ namespace Azure { namespace Storage { namespace Blobs { return request; } - static BasicResponse UndeleteParseResponse( + static UndeleteBlobResponse UndeleteParseResponse( Azure::Core::Context context, std::unique_ptr pHttpResponse) { unused(context); Azure::Core::Http::Response& httpResponse = *pHttpResponse; - BasicResponse response; + UndeleteBlobResponse response; auto http_status_code = static_cast::type>( httpResponse.GetStatusCode()); @@ -2883,7 +2907,7 @@ namespace Azure { namespace Storage { namespace Blobs { return response; } - static BasicResponse Undelete( + static UndeleteBlobResponse Undelete( Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, @@ -2987,38 +3011,44 @@ namespace Azure { namespace Storage { namespace Blobs { response.LeaseDuration = response_lease_duration_iterator->second; } response.ContentLength = std::stoll(httpResponse.GetHeaders().at("Content-Length")); - auto response_content_type_iterator = httpResponse.GetHeaders().find("Content-Type"); - if (response_content_type_iterator != httpResponse.GetHeaders().end()) + auto response_http_headers_content_type_iterator + = httpResponse.GetHeaders().find("Content-Type"); + if (response_http_headers_content_type_iterator != httpResponse.GetHeaders().end()) { - response.ContentType = response_content_type_iterator->second; + response.HttpHeaders.ContentType = response_http_headers_content_type_iterator->second; } - auto response_content_encoding_iterator + auto response_http_headers_content_encoding_iterator = httpResponse.GetHeaders().find("Content-Encoding"); - if (response_content_encoding_iterator != httpResponse.GetHeaders().end()) + if (response_http_headers_content_encoding_iterator != httpResponse.GetHeaders().end()) { - response.ContentEncoding = response_content_encoding_iterator->second; + response.HttpHeaders.ContentEncoding + = response_http_headers_content_encoding_iterator->second; } - auto response_content_language_iterator + auto response_http_headers_content_language_iterator = httpResponse.GetHeaders().find("Content-Language"); - if (response_content_language_iterator != httpResponse.GetHeaders().end()) + if (response_http_headers_content_language_iterator != httpResponse.GetHeaders().end()) { - response.ContentLanguage = response_content_language_iterator->second; + response.HttpHeaders.ContentLanguage + = response_http_headers_content_language_iterator->second; } - auto response_cache_control_iterator = httpResponse.GetHeaders().find("Cache-Control"); - if (response_cache_control_iterator != httpResponse.GetHeaders().end()) + auto response_http_headers_cache_control_iterator + = httpResponse.GetHeaders().find("Cache-Control"); + if (response_http_headers_cache_control_iterator != httpResponse.GetHeaders().end()) { - response.CacheControl = response_cache_control_iterator->second; + response.HttpHeaders.CacheControl = response_http_headers_cache_control_iterator->second; } - auto response_content_md5_iterator = httpResponse.GetHeaders().find("Content-MD5"); - if (response_content_md5_iterator != httpResponse.GetHeaders().end()) + auto response_http_headers_content_md5_iterator + = httpResponse.GetHeaders().find("Content-MD5"); + if (response_http_headers_content_md5_iterator != httpResponse.GetHeaders().end()) { - response.ContentMD5 = response_content_md5_iterator->second; + response.HttpHeaders.ContentMD5 = response_http_headers_content_md5_iterator->second; } - auto response_content_disposition_iterator + auto response_http_headers_content_disposition_iterator = httpResponse.GetHeaders().find("Content-Disposition"); - if (response_content_disposition_iterator != httpResponse.GetHeaders().end()) + if (response_http_headers_content_disposition_iterator != httpResponse.GetHeaders().end()) { - response.ContentDisposition = response_content_disposition_iterator->second; + response.HttpHeaders.ContentDisposition + = response_http_headers_content_disposition_iterator->second; } auto response_sequence_number_iterator = httpResponse.GetHeaders().find("x-ms-blob-sequence-number"); @@ -3113,12 +3143,7 @@ namespace Azure { namespace Storage { namespace Blobs { struct SetHttpHeadersOptions { Azure::Core::Nullable Timeout; - std::string ContentType; - std::string ContentEncoding; - std::string ContentLanguage; - std::string ContentMD5; - std::string CacheControl; - std::string ContentDisposition; + BlobHttpHeaders HttpHeaders; Azure::Core::Nullable EncryptionKey; Azure::Core::Nullable EncryptionKeySHA256; Azure::Core::Nullable EncryptionAlgorithm; @@ -3142,29 +3167,30 @@ namespace Azure { namespace Storage { namespace Blobs { { request.AddQueryParameter("timeout", std::to_string(options.Timeout.GetValue())); } - if (!options.ContentType.empty()) + if (!options.HttpHeaders.ContentType.empty()) { - request.AddHeader("x-ms-blob-content-type", options.ContentType); + request.AddHeader("x-ms-blob-content-type", options.HttpHeaders.ContentType); } - if (!options.ContentEncoding.empty()) + if (!options.HttpHeaders.ContentEncoding.empty()) { - request.AddHeader("x-ms-blob-content-encoding", options.ContentEncoding); + request.AddHeader("x-ms-blob-content-encoding", options.HttpHeaders.ContentEncoding); } - if (!options.ContentLanguage.empty()) + if (!options.HttpHeaders.ContentLanguage.empty()) { - request.AddHeader("x-ms-blob-content-language", options.ContentLanguage); + request.AddHeader("x-ms-blob-content-language", options.HttpHeaders.ContentLanguage); } - if (!options.CacheControl.empty()) + if (!options.HttpHeaders.CacheControl.empty()) { - request.AddHeader("x-ms-blob-cache-control", options.CacheControl); + request.AddHeader("x-ms-blob-cache-control", options.HttpHeaders.CacheControl); } - if (!options.ContentMD5.empty()) + if (!options.HttpHeaders.ContentMD5.empty()) { - request.AddHeader("x-ms-blob-content-md5", options.ContentMD5); + request.AddHeader("x-ms-blob-content-md5", options.HttpHeaders.ContentMD5); } - if (!options.ContentDisposition.empty()) + if (!options.HttpHeaders.ContentDisposition.empty()) { - request.AddHeader("x-ms-blob-content-disposition", options.ContentDisposition); + request.AddHeader( + "x-ms-blob-content-disposition", options.HttpHeaders.ContentDisposition); } if (options.EncryptionKey.HasValue()) { @@ -3197,13 +3223,13 @@ namespace Azure { namespace Storage { namespace Blobs { return request; } - static BlobInfo SetHttpHeadersParseResponse( + static SetBlobHttpHeadersResponse SetHttpHeadersParseResponse( Azure::Core::Context context, std::unique_ptr pHttpResponse) { unused(context); Azure::Core::Http::Response& httpResponse = *pHttpResponse; - BlobInfo response; + SetBlobHttpHeadersResponse response; auto http_status_code = static_cast::type>( httpResponse.GetStatusCode()); @@ -3231,7 +3257,7 @@ namespace Azure { namespace Storage { namespace Blobs { return response; } - static BlobInfo SetHttpHeaders( + static SetBlobHttpHeadersResponse SetHttpHeaders( Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, @@ -3316,13 +3342,13 @@ namespace Azure { namespace Storage { namespace Blobs { return request; } - static BlobInfo SetMetadataParseResponse( + static SetBlobMetadataResponse SetMetadataParseResponse( Azure::Core::Context context, std::unique_ptr pHttpResponse) { unused(context); Azure::Core::Http::Response& httpResponse = *pHttpResponse; - BlobInfo response; + SetBlobMetadataResponse response; auto http_status_code = static_cast::type>( httpResponse.GetStatusCode()); @@ -3344,7 +3370,7 @@ namespace Azure { namespace Storage { namespace Blobs { return response; } - static BlobInfo SetMetadata( + static SetBlobMetadataResponse SetMetadata( Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, @@ -3388,13 +3414,13 @@ namespace Azure { namespace Storage { namespace Blobs { return request; } - static BasicResponse SetAccessTierParseResponse( + static SetAccessTierResponse SetAccessTierParseResponse( Azure::Core::Context context, std::unique_ptr pHttpResponse) { unused(context); Azure::Core::Http::Response& httpResponse = *pHttpResponse; - BasicResponse response; + SetAccessTierResponse response; auto http_status_code = static_cast::type>( httpResponse.GetStatusCode()); @@ -3414,7 +3440,7 @@ namespace Azure { namespace Storage { namespace Blobs { return response; } - static BasicResponse SetAccessTier( + static SetAccessTierResponse SetAccessTier( Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, @@ -3603,13 +3629,13 @@ namespace Azure { namespace Storage { namespace Blobs { return request; } - static BasicResponse AbortCopyFromUriParseResponse( + static AbortCopyBlobResponse AbortCopyFromUriParseResponse( Azure::Core::Context context, std::unique_ptr pHttpResponse) { unused(context); Azure::Core::Http::Response& httpResponse = *pHttpResponse; - BasicResponse response; + AbortCopyBlobResponse response; auto http_status_code = static_cast::type>( httpResponse.GetStatusCode()); @@ -3629,7 +3655,7 @@ namespace Azure { namespace Storage { namespace Blobs { return response; } - static BasicResponse AbortCopyFromUri( + static AbortCopyBlobResponse AbortCopyFromUri( Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, @@ -3783,7 +3809,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Nullable Timeout; Azure::Core::Nullable ContentMD5; Azure::Core::Nullable ContentCRC64; - BlobHttpHeaders Properties; + BlobHttpHeaders HttpHeaders; std::map Metadata; Azure::Core::Nullable LeaseId; Azure::Core::Nullable Tier; @@ -3829,29 +3855,30 @@ namespace Azure { namespace Storage { namespace Blobs { { request.AddHeader("x-ms-content-crc64", options.ContentCRC64.GetValue()); } - if (!options.Properties.ContentType.empty()) + if (!options.HttpHeaders.ContentType.empty()) { - request.AddHeader("x-ms-blob-content-type", options.Properties.ContentType); + request.AddHeader("x-ms-blob-content-type", options.HttpHeaders.ContentType); } - if (!options.Properties.ContentEncoding.empty()) + if (!options.HttpHeaders.ContentEncoding.empty()) { - request.AddHeader("x-ms-blob-content-encoding", options.Properties.ContentEncoding); + request.AddHeader("x-ms-blob-content-encoding", options.HttpHeaders.ContentEncoding); } - if (!options.Properties.ContentLanguage.empty()) + if (!options.HttpHeaders.ContentLanguage.empty()) { - request.AddHeader("x-ms-blob-content-language", options.Properties.ContentLanguage); + request.AddHeader("x-ms-blob-content-language", options.HttpHeaders.ContentLanguage); } - if (!options.Properties.CacheControl.empty()) + if (!options.HttpHeaders.CacheControl.empty()) { - request.AddHeader("x-ms-blob-cache-control", options.Properties.CacheControl); + request.AddHeader("x-ms-blob-cache-control", options.HttpHeaders.CacheControl); } - if (!options.Properties.ContentMD5.empty()) + if (!options.HttpHeaders.ContentMD5.empty()) { - request.AddHeader("x-ms-blob-content-md5", options.Properties.ContentMD5); + request.AddHeader("x-ms-blob-content-md5", options.HttpHeaders.ContentMD5); } - if (!options.Properties.ContentDisposition.empty()) + if (!options.HttpHeaders.ContentDisposition.empty()) { - request.AddHeader("x-ms-blob-content-disposition", options.Properties.ContentDisposition); + request.AddHeader( + "x-ms-blob-content-disposition", options.HttpHeaders.ContentDisposition); } std::set metadataKeys; for (const auto& pair : options.Metadata) @@ -4236,7 +4263,7 @@ namespace Azure { namespace Storage { namespace Blobs { { Azure::Core::Nullable Timeout; std::vector> BlockList; - BlobHttpHeaders Properties; + BlobHttpHeaders HttpHeaders; std::map Metadata; Azure::Core::Nullable LeaseId; Azure::Core::Nullable EncryptionKey; @@ -4275,29 +4302,30 @@ namespace Azure { namespace Storage { namespace Blobs { { request.AddQueryParameter("timeout", std::to_string(options.Timeout.GetValue())); } - if (!options.Properties.ContentType.empty()) + if (!options.HttpHeaders.ContentType.empty()) { - request.AddHeader("x-ms-blob-content-type", options.Properties.ContentType); + request.AddHeader("x-ms-blob-content-type", options.HttpHeaders.ContentType); } - if (!options.Properties.ContentEncoding.empty()) + if (!options.HttpHeaders.ContentEncoding.empty()) { - request.AddHeader("x-ms-blob-content-encoding", options.Properties.ContentEncoding); + request.AddHeader("x-ms-blob-content-encoding", options.HttpHeaders.ContentEncoding); } - if (!options.Properties.ContentLanguage.empty()) + if (!options.HttpHeaders.ContentLanguage.empty()) { - request.AddHeader("x-ms-blob-content-language", options.Properties.ContentLanguage); + request.AddHeader("x-ms-blob-content-language", options.HttpHeaders.ContentLanguage); } - if (!options.Properties.CacheControl.empty()) + if (!options.HttpHeaders.CacheControl.empty()) { - request.AddHeader("x-ms-blob-cache-control", options.Properties.CacheControl); + request.AddHeader("x-ms-blob-cache-control", options.HttpHeaders.CacheControl); } - if (!options.Properties.ContentMD5.empty()) + if (!options.HttpHeaders.ContentMD5.empty()) { - request.AddHeader("x-ms-blob-content-md5", options.Properties.ContentMD5); + request.AddHeader("x-ms-blob-content-md5", options.HttpHeaders.ContentMD5); } - if (!options.Properties.ContentDisposition.empty()) + if (!options.HttpHeaders.ContentDisposition.empty()) { - request.AddHeader("x-ms-blob-content-disposition", options.Properties.ContentDisposition); + request.AddHeader( + "x-ms-blob-content-disposition", options.HttpHeaders.ContentDisposition); } std::set metadataKeys; for (const auto& pair : options.Metadata) @@ -4667,9 +4695,9 @@ namespace Azure { namespace Storage { namespace Blobs { struct CreateOptions { Azure::Core::Nullable Timeout; - int64_t BlobContentLength; + int64_t BlobContentLength = -1; Azure::Core::Nullable SequenceNumber; - BlobHttpHeaders Properties; + BlobHttpHeaders HttpHeaders; std::map Metadata; Azure::Core::Nullable LeaseId; Azure::Core::Nullable Tier; @@ -4695,29 +4723,30 @@ namespace Azure { namespace Storage { namespace Blobs { { request.AddQueryParameter("timeout", std::to_string(options.Timeout.GetValue())); } - if (!options.Properties.ContentType.empty()) + if (!options.HttpHeaders.ContentType.empty()) { - request.AddHeader("x-ms-blob-content-type", options.Properties.ContentType); + request.AddHeader("x-ms-blob-content-type", options.HttpHeaders.ContentType); } - if (!options.Properties.ContentEncoding.empty()) + if (!options.HttpHeaders.ContentEncoding.empty()) { - request.AddHeader("x-ms-blob-content-encoding", options.Properties.ContentEncoding); + request.AddHeader("x-ms-blob-content-encoding", options.HttpHeaders.ContentEncoding); } - if (!options.Properties.ContentLanguage.empty()) + if (!options.HttpHeaders.ContentLanguage.empty()) { - request.AddHeader("x-ms-blob-content-language", options.Properties.ContentLanguage); + request.AddHeader("x-ms-blob-content-language", options.HttpHeaders.ContentLanguage); } - if (!options.Properties.CacheControl.empty()) + if (!options.HttpHeaders.CacheControl.empty()) { - request.AddHeader("x-ms-blob-cache-control", options.Properties.CacheControl); + request.AddHeader("x-ms-blob-cache-control", options.HttpHeaders.CacheControl); } - if (!options.Properties.ContentMD5.empty()) + if (!options.HttpHeaders.ContentMD5.empty()) { - request.AddHeader("x-ms-blob-content-md5", options.Properties.ContentMD5); + request.AddHeader("x-ms-blob-content-md5", options.HttpHeaders.ContentMD5); } - if (!options.Properties.ContentDisposition.empty()) + if (!options.HttpHeaders.ContentDisposition.empty()) { - request.AddHeader("x-ms-blob-content-disposition", options.Properties.ContentDisposition); + request.AddHeader( + "x-ms-blob-content-disposition", options.HttpHeaders.ContentDisposition); } std::set metadataKeys; for (const auto& pair : options.Metadata) @@ -5326,7 +5355,7 @@ namespace Azure { namespace Storage { namespace Blobs { struct ResizeOptions { Azure::Core::Nullable Timeout; - int64_t BlobContentLength; + int64_t BlobContentLength = -1; Azure::Core::Nullable LeaseId; Azure::Core::Nullable IfSequenceNumberLessThanOrEqualTo; Azure::Core::Nullable IfSequenceNumberLessThan; @@ -5843,7 +5872,7 @@ namespace Azure { namespace Storage { namespace Blobs { struct CreateOptions { Azure::Core::Nullable Timeout; - BlobHttpHeaders Properties; + BlobHttpHeaders HttpHeaders; std::map Metadata; Azure::Core::Nullable LeaseId; Azure::Core::Nullable EncryptionKey; @@ -5868,29 +5897,30 @@ namespace Azure { namespace Storage { namespace Blobs { { request.AddQueryParameter("timeout", std::to_string(options.Timeout.GetValue())); } - if (!options.Properties.ContentType.empty()) + if (!options.HttpHeaders.ContentType.empty()) { - request.AddHeader("x-ms-blob-content-type", options.Properties.ContentType); + request.AddHeader("x-ms-blob-content-type", options.HttpHeaders.ContentType); } - if (!options.Properties.ContentEncoding.empty()) + if (!options.HttpHeaders.ContentEncoding.empty()) { - request.AddHeader("x-ms-blob-content-encoding", options.Properties.ContentEncoding); + request.AddHeader("x-ms-blob-content-encoding", options.HttpHeaders.ContentEncoding); } - if (!options.Properties.ContentLanguage.empty()) + if (!options.HttpHeaders.ContentLanguage.empty()) { - request.AddHeader("x-ms-blob-content-language", options.Properties.ContentLanguage); + request.AddHeader("x-ms-blob-content-language", options.HttpHeaders.ContentLanguage); } - if (!options.Properties.CacheControl.empty()) + if (!options.HttpHeaders.CacheControl.empty()) { - request.AddHeader("x-ms-blob-cache-control", options.Properties.CacheControl); + request.AddHeader("x-ms-blob-cache-control", options.HttpHeaders.CacheControl); } - if (!options.Properties.ContentMD5.empty()) + if (!options.HttpHeaders.ContentMD5.empty()) { - request.AddHeader("x-ms-blob-content-md5", options.Properties.ContentMD5); + request.AddHeader("x-ms-blob-content-md5", options.HttpHeaders.ContentMD5); } - if (!options.Properties.ContentDisposition.empty()) + if (!options.HttpHeaders.ContentDisposition.empty()) { - request.AddHeader("x-ms-blob-content-disposition", options.Properties.ContentDisposition); + request.AddHeader( + "x-ms-blob-content-disposition", options.HttpHeaders.ContentDisposition); } std::set metadataKeys; for (const auto& pair : options.Metadata) diff --git a/sdk/storage/inc/common/concurrent_transfer.hpp b/sdk/storage/inc/common/concurrent_transfer.hpp new file mode 100644 index 000000000..df6cebe9e --- /dev/null +++ b/sdk/storage/inc/common/concurrent_transfer.hpp @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include +#include + +namespace Azure { namespace Storage { namespace Details { + + inline void ConcurrentTransfer( + int64_t offset, + int64_t length, + int64_t chunkSize, + int concurrency, + // offset, length, chunk id, number of chunks + std::function transferFunc) + { + std::atomic numWorkingThreads{concurrency}; + std::atomic nextChunkId{0}; + std::atomic failed{false}; + + const auto numChunks = (length + chunkSize - 1) / chunkSize; + + auto threadFunc = [&]() { + while (true) + { + int chunkId = nextChunkId.fetch_add(1); + if (chunkId >= numChunks || failed) + { + break; + } + int64_t chunkOffset = offset + chunkSize * chunkId; + int64_t chunkLength = std::min(length - chunkSize * chunkId, chunkSize); + try + { + transferFunc(chunkOffset, chunkLength, chunkId, numChunks); + } + catch (std::exception&) + { + if (failed.exchange(true) == false) + { + numWorkingThreads.fetch_sub(1); + throw; + } + } + } + numWorkingThreads.fetch_sub(1); + }; + + std::vector> threadHandles; + for (int i = 0; i < concurrency - 1; ++i) + { + threadHandles.emplace_back(std::async(std::launch::async, threadFunc)); + } + threadFunc(); + for (auto& handle : threadHandles) + { + handle.get(); + } + + if (numWorkingThreads != 0) + { + std::abort(); + } + } + +}}} // namespace Azure::Storage::Details diff --git a/sdk/storage/src/blobs/append_blob_client.cpp b/sdk/storage/src/blobs/append_blob_client.cpp index c763579dc..bf12317a1 100644 --- a/sdk/storage/src/blobs/append_blob_client.cpp +++ b/sdk/storage/src/blobs/append_blob_client.cpp @@ -60,7 +60,7 @@ namespace Azure { namespace Storage { namespace Blobs { BlobContentInfo AppendBlobClient::Create(const CreateAppendBlobOptions& options) { BlobRestClient::AppendBlob::CreateOptions protocolLayerOptions; - protocolLayerOptions.Properties = options.Properties; + protocolLayerOptions.HttpHeaders = options.HttpHeaders; protocolLayerOptions.Metadata = options.Metadata; protocolLayerOptions.IfModifiedSince = options.IfModifiedSince; protocolLayerOptions.IfUnmodifiedSince = options.IfUnmodifiedSince; diff --git a/sdk/storage/src/blobs/blob_client.cpp b/sdk/storage/src/blobs/blob_client.cpp index b5c717406..e2efb767d 100644 --- a/sdk/storage/src/blobs/blob_client.cpp +++ b/sdk/storage/src/blobs/blob_client.cpp @@ -7,6 +7,7 @@ #include "blobs/block_blob_client.hpp" #include "blobs/page_blob_client.hpp" #include "common/common_headers_request_policy.hpp" +#include "common/concurrent_transfer.hpp" #include "common/shared_key_policy.hpp" #include "common/storage_common.hpp" #include "http/curl/curl.hpp" @@ -41,7 +42,12 @@ namespace Azure { namespace Storage { namespace Blobs { : m_blobUrl(blobUri) { std::vector> policies; - for (const auto& p : options.policies) + for (const auto& p : options.PerOperationPolicies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + // TODO: Retry policy goes here + for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); } @@ -59,7 +65,12 @@ namespace Azure { namespace Storage { namespace Blobs { : m_blobUrl(blobUri) { std::vector> policies; - for (const auto& p : options.policies) + for (const auto& p : options.PerOperationPolicies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + // TODO: Retry policy goes here + for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); } @@ -75,7 +86,12 @@ namespace Azure { namespace Storage { namespace Blobs { : m_blobUrl(blobUri) { std::vector> policies; - for (const auto& p : options.policies) + for (const auto& p : options.PerOperationPolicies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + // TODO: Retry policy goes here + for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); } @@ -105,7 +121,7 @@ namespace Azure { namespace Storage { namespace Blobs { return newClient; } - BlobDownloadInfo BlobClient::Download(const DownloadBlobOptions& options) const + BlobDownloadResponse BlobClient::Download(const DownloadBlobOptions& options) const { BlobRestClient::Blob::DownloadOptions protocolLayerOptions; if (options.Offset.HasValue() && options.Length.HasValue()) @@ -128,6 +144,132 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } + BlobDownloadInfo BlobClient::DownloadToBuffer( + uint8_t* buffer, + std::size_t bufferSize, + const DownloadBlobToBufferOptions& options) const + { + constexpr int64_t c_defaultChunkSize = 8 * 1024 * 1024; + + // Just start downloading using an initial chunk. If it's a small blob, we'll get the whole + // thing in one shot. If it's a large blob, we'll get its full size in Content-Range and can + // keep downloading it in chunks. + int64_t firstChunkOffset = options.Offset.HasValue() ? options.Offset.GetValue() : 0; + int64_t firstChunkLength = c_defaultChunkSize; + if (options.InitialChunkSize.HasValue()) + { + firstChunkLength = options.InitialChunkSize.GetValue(); + } + if (options.Length.HasValue()) + { + firstChunkLength = std::min(firstChunkLength, options.Length.GetValue()); + } + + DownloadBlobOptions firstChunkOptions; + firstChunkOptions.Context = options.Context; + firstChunkOptions.Offset = options.Offset; + if (firstChunkOptions.Offset.HasValue()) + { + firstChunkOptions.Length = firstChunkLength; + } + + auto firstChunk = Download(firstChunkOptions); + + int64_t blobSize; + int64_t blobRangeSize; + if (firstChunkOptions.Offset.HasValue()) + { + blobSize = std::stoll(firstChunk.ContentRange.GetValue().substr( + firstChunk.ContentRange.GetValue().find('/') + 1)); + blobRangeSize = blobSize - firstChunkOffset; + if (options.Length.HasValue()) + { + blobRangeSize = std::min(blobRangeSize, options.Length.GetValue()); + } + } + else + { + blobSize = firstChunk.BodyStream->Length(); + blobRangeSize = blobSize; + } + firstChunkLength = std::min(firstChunkLength, blobRangeSize); + + if (static_cast(blobRangeSize) > bufferSize) + { + throw std::runtime_error( + "buffer is not big enough, blob range size is " + std::to_string(blobRangeSize)); + } + + int64_t bytesRead = Azure::Core::Http::BodyStream::ReadToCount( + firstChunkOptions.Context, *firstChunk.BodyStream, buffer, firstChunkLength); + if (bytesRead != firstChunkLength) + { + throw std::runtime_error("error when reading body stream"); + } + firstChunk.BodyStream.reset(); + + auto returnTypeConverter = [](BlobDownloadResponse& response) { + BlobDownloadInfo ret; + ret.ETag = std::move(response.ETag); + ret.LastModified = std::move(response.LastModified); + ret.HttpHeaders = std::move(response.HttpHeaders); + ret.Metadata = std::move(response.Metadata); + ret.BlobType = response.BlobType; + ret.ServerEncrypted = response.ServerEncrypted; + ret.EncryptionKeySHA256 = std::move(response.EncryptionKeySHA256); + return ret; + }; + BlobDownloadInfo ret = returnTypeConverter(firstChunk); + + // Keep downloading the remaining in parallel + auto downloadChunkFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { + + DownloadBlobOptions chunkOptions; + chunkOptions.Context = options.Context; + chunkOptions.Offset = offset; + chunkOptions.Length = length; + auto chunk = Download(chunkOptions); + int64_t bytesRead = Azure::Core::Http::BodyStream::ReadToCount( + chunkOptions.Context, + *chunk.BodyStream, + buffer + (offset - firstChunkOffset), + chunkOptions.Length.GetValue()); + if (bytesRead != chunkOptions.Length.GetValue()) + { + throw std::runtime_error("error when reading body stream"); + } + + if (chunkId == numChunks - 1) + { + ret = returnTypeConverter(chunk); + } + }; + + int64_t remainingOffset = firstChunkOffset + firstChunkLength; + int64_t remainingSize = blobRangeSize - firstChunkLength; + int64_t chunkSize; + if (options.ChunkSize.HasValue()) + { + chunkSize = options.ChunkSize.GetValue(); + } + else + { + int64_t c_grainSize = 4 * 1024; + chunkSize = remainingSize / options.Concurrency; + chunkSize = (std::max(chunkSize, int64_t(1)) + c_grainSize - 1) / c_grainSize * c_grainSize; + chunkSize = std::min(chunkSize, c_defaultChunkSize); + } + + Details::ConcurrentTransfer( + remainingOffset, + remainingSize, + chunkSize, + options.Concurrency, + downloadChunkFunc); + ret.ContentLength = blobRangeSize; + return ret; + } + BlobProperties BlobClient::GetProperties(const GetBlobPropertiesOptions& options) const { BlobRestClient::Blob::GetPropertiesOptions protocolLayerOptions; @@ -139,15 +281,12 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } - BlobInfo BlobClient::SetHttpHeaders(const SetBlobHttpHeadersOptions& options) const + SetBlobHttpHeadersResponse BlobClient::SetHttpHeaders( + BlobHttpHeaders httpHeaders, + const SetBlobHttpHeadersOptions& options) const { BlobRestClient::Blob::SetHttpHeadersOptions protocolLayerOptions; - protocolLayerOptions.ContentType = options.ContentType; - protocolLayerOptions.ContentEncoding = options.ContentEncoding; - protocolLayerOptions.ContentLanguage = options.ContentLanguage; - protocolLayerOptions.ContentMD5 = options.ContentMD5; - protocolLayerOptions.CacheControl = options.CacheControl; - protocolLayerOptions.ContentDisposition = options.ContentDisposition; + protocolLayerOptions.HttpHeaders = std::move(httpHeaders); protocolLayerOptions.IfModifiedSince = options.IfModifiedSince; protocolLayerOptions.IfUnmodifiedSince = options.IfUnmodifiedSince; protocolLayerOptions.IfMatch = options.IfMatch; @@ -156,7 +295,7 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } - BlobInfo BlobClient::SetMetadata( + SetBlobMetadataResponse BlobClient::SetMetadata( std::map metadata, const SetBlobMetadataOptions& options) const { @@ -170,8 +309,9 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } - BasicResponse BlobClient::SetAccessTier(AccessTier Tier, const SetAccessTierOptions& options) - const + SetAccessTierResponse BlobClient::SetAccessTier( + AccessTier Tier, + const SetAccessTierOptions& options) const { BlobRestClient::Blob::SetAccessTierOptions protocolLayerOptions; protocolLayerOptions.Tier = Tier; @@ -203,7 +343,7 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } - BasicResponse BlobClient::AbortCopyFromUri( + AbortCopyBlobResponse BlobClient::AbortCopyFromUri( const std::string& copyId, const AbortCopyFromUriOptions& options) const { @@ -227,7 +367,7 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } - BasicResponse BlobClient::Delete(const DeleteBlobOptions& options) const + DeleteBlobResponse BlobClient::Delete(const DeleteBlobOptions& options) const { BlobRestClient::Blob::DeleteOptions protocolLayerOptions; protocolLayerOptions.DeleteSnapshots = options.DeleteSnapshots; @@ -239,7 +379,7 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } - BasicResponse BlobClient::Undelete(const UndeleteBlobOptions& options) const + UndeleteBlobResponse BlobClient::Undelete(const UndeleteBlobOptions& options) const { BlobRestClient::Blob::UndeleteOptions protocolLayerOptions; return BlobRestClient::Blob::Undelete( diff --git a/sdk/storage/src/blobs/blob_container_client.cpp b/sdk/storage/src/blobs/blob_container_client.cpp index 9985e8e34..b9b3e2529 100644 --- a/sdk/storage/src/blobs/blob_container_client.cpp +++ b/sdk/storage/src/blobs/blob_container_client.cpp @@ -40,7 +40,12 @@ namespace Azure { namespace Storage { namespace Blobs { : BlobContainerClient(containerUri, options) { std::vector> policies; - for (const auto& p : options.policies) + for (const auto& p : options.PerOperationPolicies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + // TODO: Retry policy goes here + for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); } @@ -58,7 +63,12 @@ namespace Azure { namespace Storage { namespace Blobs { : BlobContainerClient(containerUri, options) { std::vector> policies; - for (const auto& p : options.policies) + for (const auto& p : options.PerOperationPolicies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + // TODO: Retry policy goes here + for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); } @@ -76,7 +86,12 @@ namespace Azure { namespace Storage { namespace Blobs { : m_containerUrl(containerUri) { std::vector> policies; - for (const auto& p : options.policies) + for (const auto& p : options.PerOperationPolicies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + // TODO: Retry policy goes here + for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); } @@ -90,10 +105,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto blobUri = m_containerUrl; blobUri.AppendPath(blobName); - BlobClient blobClient; - blobClient.m_blobUrl = std::move(blobUri); - blobClient.m_pipeline = m_pipeline; - return blobClient; + return BlobClient(std::move(blobUri), m_pipeline); } BlockBlobClient BlobContainerClient::GetBlockBlobClient(const std::string& blobName) const @@ -120,7 +132,8 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions); } - BasicResponse BlobContainerClient::Delete(const DeleteBlobContainerOptions& options) const + DeleteContainerResponse BlobContainerClient::Delete( + const DeleteBlobContainerOptions& options) const { BlobRestClient::Container::DeleteOptions protocolLayerOptions; protocolLayerOptions.IfModifiedSince = options.IfModifiedSince; @@ -138,7 +151,7 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions); } - BlobContainerInfo BlobContainerClient::SetMetadata( + SetContainerMetadataResponse BlobContainerClient::SetMetadata( std::map metadata, SetBlobContainerMetadataOptions options) const { @@ -149,7 +162,7 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions); } - BlobsFlatSegment BlobContainerClient::ListBlobs(const ListBlobsOptions& options) const + BlobsFlatSegment BlobContainerClient::ListBlobsFlat(const ListBlobsOptions& options) const { BlobRestClient::Container::ListBlobsOptions protocolLayerOptions; protocolLayerOptions.Prefix = options.Prefix; diff --git a/sdk/storage/src/blobs/blob_service_client.cpp b/sdk/storage/src/blobs/blob_service_client.cpp index 254f64f8e..bb77e866d 100644 --- a/sdk/storage/src/blobs/blob_service_client.cpp +++ b/sdk/storage/src/blobs/blob_service_client.cpp @@ -35,7 +35,12 @@ namespace Azure { namespace Storage { namespace Blobs { : m_serviceUrl(serviceUri) { std::vector> policies; - for (const auto& p : options.policies) + for (const auto& p : options.PerOperationPolicies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + // TODO: Retry policy goes here + for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); } @@ -53,7 +58,12 @@ namespace Azure { namespace Storage { namespace Blobs { : m_serviceUrl(serviceUri) { std::vector> policies; - for (const auto& p : options.policies) + for (const auto& p : options.PerOperationPolicies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + // TODO: Retry policy goes here + for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); } @@ -71,7 +81,12 @@ namespace Azure { namespace Storage { namespace Blobs { : m_serviceUrl(serviceUri) { std::vector> policies; - for (const auto& p : options.policies) + for (const auto& p : options.PerOperationPolicies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + // TODO: Retry policy goes here + for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); } @@ -86,10 +101,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto containerUri = m_serviceUrl; containerUri.AppendPath(containerName); - BlobContainerClient containerClient; - containerClient.m_containerUrl = std::move(containerUri); - containerClient.m_pipeline = m_pipeline; - return containerClient; + return BlobContainerClient(std::move(containerUri), m_pipeline); } ListContainersSegment BlobServiceClient::ListBlobContainersSegment( diff --git a/sdk/storage/src/blobs/block_blob_client.cpp b/sdk/storage/src/blobs/block_blob_client.cpp index dad6c8b13..02b41baa4 100644 --- a/sdk/storage/src/blobs/block_blob_client.cpp +++ b/sdk/storage/src/blobs/block_blob_client.cpp @@ -3,6 +3,8 @@ #include "blobs/block_blob_client.hpp" +#include "common/concurrent_transfer.hpp" +#include "common/crypt.hpp" #include "common/storage_common.hpp" namespace Azure { namespace Storage { namespace Blobs { @@ -64,7 +66,7 @@ namespace Azure { namespace Storage { namespace Blobs { BlobRestClient::BlockBlob::UploadOptions protocolLayerOptions; protocolLayerOptions.ContentMD5 = options.ContentMD5; protocolLayerOptions.ContentCRC64 = options.ContentCRC64; - protocolLayerOptions.Properties = options.Properties; + protocolLayerOptions.HttpHeaders = options.HttpHeaders; protocolLayerOptions.Metadata = options.Metadata; protocolLayerOptions.Tier = options.Tier; protocolLayerOptions.IfModifiedSince = options.IfModifiedSince; @@ -75,6 +77,64 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_blobUrl.ToString(), content, protocolLayerOptions); } + BlobContentInfo BlockBlobClient::UploadFromBuffer( + const uint8_t* buffer, + std::size_t bufferSize, + const UploadBlobOptions& options) const + { + constexpr int64_t c_defaultBlockSize = 8 * 1024 * 1024; + constexpr int64_t c_maximumNumberBlocks = 50000; + constexpr int64_t c_grainSize = 4 * 1024; + + int64_t chunkSize = c_defaultBlockSize; + if (options.ChunkSize.HasValue()) + { + chunkSize = options.ChunkSize.GetValue(); + } + else + { + int64_t minBlockSize = (bufferSize + c_maximumNumberBlocks - 1) / c_maximumNumberBlocks; + chunkSize = std::max(chunkSize, minBlockSize); + chunkSize = (chunkSize + c_grainSize - 1) / c_grainSize * c_grainSize; + } + + std::vector> blockIds; + auto getBlockId = [](int64_t id) { + constexpr std::size_t c_blockIdLength = 64; + std::string blockId = std::to_string(id); + blockId = std::string(c_blockIdLength - blockId.length(), '0') + blockId; + return Base64Encode(blockId); + }; + + auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { + Azure::Core::Http::MemoryBodyStream contentStream(buffer + offset, length); + StageBlockOptions chunkOptions; + chunkOptions.Context = options.Context; + auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions); + if (chunkId == numChunks - 1) + { + blockIds.resize(static_cast(numChunks)); + } + }; + + Details::ConcurrentTransfer(0, bufferSize, chunkSize, options.Concurrency, uploadBlockFunc); + + for (std::size_t i = 0; i < blockIds.size(); ++i) + { + blockIds[i].first = BlockType::Uncommitted; + blockIds[i].second = getBlockId(static_cast(i)); + } + CommitBlockListOptions commitBlockListOptions; + commitBlockListOptions.Context = options.Context; + commitBlockListOptions.HttpHeaders = options.HttpHeaders; + commitBlockListOptions.Metadata = options.Metadata; + commitBlockListOptions.Tier = options.Tier; + auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions); + commitBlockListResponse.ContentCRC64.Reset(); + commitBlockListResponse.ContentMD5.Reset(); + return commitBlockListResponse; + } + BlockInfo BlockBlobClient::StageBlock( const std::string& blockId, Azure::Core::Http::BodyStream& content, @@ -126,7 +186,7 @@ namespace Azure { namespace Storage { namespace Blobs { { BlobRestClient::BlockBlob::CommitBlockListOptions protocolLayerOptions; protocolLayerOptions.BlockList = blockIds; - protocolLayerOptions.Properties = options.Properties; + protocolLayerOptions.HttpHeaders = options.HttpHeaders; protocolLayerOptions.Metadata = options.Metadata; protocolLayerOptions.Tier = options.Tier; protocolLayerOptions.IfModifiedSince = options.IfModifiedSince; diff --git a/sdk/storage/src/blobs/page_blob_client.cpp b/sdk/storage/src/blobs/page_blob_client.cpp index eb4e39353..397b82788 100644 --- a/sdk/storage/src/blobs/page_blob_client.cpp +++ b/sdk/storage/src/blobs/page_blob_client.cpp @@ -62,7 +62,7 @@ namespace Azure { namespace Storage { namespace Blobs { BlobRestClient::PageBlob::CreateOptions protocolLayerOptions; protocolLayerOptions.BlobContentLength = blobContentLength; protocolLayerOptions.SequenceNumber = options.SequenceNumber; - protocolLayerOptions.Properties = options.Properties; + protocolLayerOptions.HttpHeaders = options.HttpHeaders; protocolLayerOptions.Metadata = options.Metadata; protocolLayerOptions.Tier = options.Tier; protocolLayerOptions.IfModifiedSince = options.IfModifiedSince; diff --git a/sdk/storage/test/append_blob_client_test.cpp b/sdk/storage/test/append_blob_client_test.cpp index 5c2df8555..5f6e80d5a 100644 --- a/sdk/storage/test/append_blob_client_test.cpp +++ b/sdk/storage/test/append_blob_client_test.cpp @@ -22,17 +22,18 @@ namespace Azure { namespace Storage { namespace Test { m_blobContent.resize(100); RandomBuffer(reinterpret_cast(&m_blobContent[0]), m_blobContent.size()); m_blobUploadOptions.Metadata = {{"key1", "V1"}, {"KEY2", "Value2"}}; - m_blobUploadOptions.Properties.ContentType = "application/x-binary"; - m_blobUploadOptions.Properties.ContentLanguage = "en-US"; - m_blobUploadOptions.Properties.ContentDisposition = "attachment"; - m_blobUploadOptions.Properties.CacheControl = "no-cache"; - m_blobUploadOptions.Properties.ContentEncoding = "identify"; - m_blobUploadOptions.Properties.ContentMD5 = ""; + m_blobUploadOptions.HttpHeaders.ContentType = "application/x-binary"; + m_blobUploadOptions.HttpHeaders.ContentLanguage = "en-US"; + m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment"; + m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache"; + m_blobUploadOptions.HttpHeaders.ContentEncoding = "identify"; + m_blobUploadOptions.HttpHeaders.ContentMD5 = ""; m_appendBlobClient->Create(m_blobUploadOptions); auto blockContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); m_appendBlobClient->AppendBlock(blockContent); - m_blobUploadOptions.Properties.ContentMD5 = m_appendBlobClient->GetProperties().ContentMD5; + m_blobUploadOptions.HttpHeaders.ContentMD5 + = m_appendBlobClient->GetProperties().HttpHeaders.ContentMD5; } void AppendBlobClientTest::TearDownTestSuite() { BlobContainerClientTest::TearDownTestSuite(); } diff --git a/sdk/storage/test/blob_container_client_test.cpp b/sdk/storage/test/blob_container_client_test.cpp index 97b858a4e..3758d5320 100644 --- a/sdk/storage/test/blob_container_client_test.cpp +++ b/sdk/storage/test/blob_container_client_test.cpp @@ -105,17 +105,16 @@ namespace Azure { namespace Storage { namespace Test { std::set listBlobs; do { - auto res = m_blobContainerClient->ListBlobs(options); + auto res = m_blobContainerClient->ListBlobsFlat(options); EXPECT_FALSE(res.RequestId.empty()); EXPECT_FALSE(res.Date.empty()); ; EXPECT_FALSE(res.Version.empty()); EXPECT_FALSE(res.ServiceEndpoint.empty()); - EXPECT_EQ(res.MaxResults.GetValue(), options.MaxResults.GetValue()); EXPECT_EQ(res.Container, m_containerName); options.Marker = res.NextMarker; - for (const auto& blob : res.BlobItems) + for (const auto& blob : res.Items) { EXPECT_FALSE(blob.Name.empty()); EXPECT_FALSE(blob.CreationTime.empty()); @@ -133,9 +132,9 @@ namespace Azure { namespace Storage { namespace Test { listBlobs.clear(); do { - auto res = m_blobContainerClient->ListBlobs(options); + auto res = m_blobContainerClient->ListBlobsFlat(options); options.Marker = res.NextMarker; - for (const auto& blob : res.BlobItems) + for (const auto& blob : res.Items) { listBlobs.insert(blob.Name); } @@ -164,10 +163,10 @@ namespace Azure { namespace Storage { namespace Test { std::set listBlobs; while (true) { - auto res = m_blobContainerClient->ListBlobs(options); + auto res = m_blobContainerClient->ListBlobsFlat(options); EXPECT_EQ(res.Delimiter, options.Delimiter.GetValue()); EXPECT_EQ(res.Prefix, options.Prefix.GetValue()); - for (const auto& blob : res.BlobItems) + for (const auto& blob : res.Items) { listBlobs.insert(blob.Name); } @@ -175,9 +174,9 @@ namespace Azure { namespace Storage { namespace Test { { options.Marker = res.NextMarker; } - else if (!res.BlobItems.empty()) + else if (!res.Items.empty()) { - options.Prefix = res.BlobItems[0].Name + delimiter; + options.Prefix = res.Items[0].Name + delimiter; if (options.Marker.HasValue()) { options.Marker.Reset(); diff --git a/sdk/storage/test/blob_service_client_test.cpp b/sdk/storage/test/blob_service_client_test.cpp index defd8a2c4..94c2952b1 100644 --- a/sdk/storage/test/blob_service_client_test.cpp +++ b/sdk/storage/test/blob_service_client_test.cpp @@ -53,10 +53,9 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(res.Date.empty()); EXPECT_FALSE(res.Version.empty()); EXPECT_FALSE(res.ServiceEndpoint.empty()); - EXPECT_EQ(res.MaxResults.GetValue(), options.MaxResults.GetValue()); options.Marker = res.NextMarker; - for (const auto& container : res.BlobContainerItems) + for (const auto& container : res.Items) { listContainers.insert(container.Name); } @@ -76,10 +75,9 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(res.Date.empty()); EXPECT_FALSE(res.Version.empty()); EXPECT_FALSE(res.ServiceEndpoint.empty()); - EXPECT_EQ(res.MaxResults.GetValue(), options.MaxResults.GetValue()); options.Marker = res.NextMarker; - for (const auto& container : res.BlobContainerItems) + for (const auto& container : res.Items) { EXPECT_FALSE(container.Name.empty()); EXPECT_FALSE(container.ETag.empty()); diff --git a/sdk/storage/test/block_blob_client_test.cpp b/sdk/storage/test/block_blob_client_test.cpp index 3d8e3ef92..3aca953c4 100644 --- a/sdk/storage/test/block_blob_client_test.cpp +++ b/sdk/storage/test/block_blob_client_test.cpp @@ -5,6 +5,10 @@ #include "common/crypt.hpp" +#include +#include +#include + namespace Azure { namespace Storage { namespace Blobs { bool operator==( @@ -37,17 +41,18 @@ namespace Azure { namespace Storage { namespace Test { m_blobContent.resize(static_cast(8_MB)); RandomBuffer(reinterpret_cast(&m_blobContent[0]), m_blobContent.size()); m_blobUploadOptions.Metadata = {{"key1", "V1"}, {"KEY2", "Value2"}}; - m_blobUploadOptions.Properties.ContentType = "application/x-binary"; - m_blobUploadOptions.Properties.ContentLanguage = "en-US"; - m_blobUploadOptions.Properties.ContentDisposition = "attachment"; - m_blobUploadOptions.Properties.CacheControl = "no-cache"; - m_blobUploadOptions.Properties.ContentEncoding = "identity"; - m_blobUploadOptions.Properties.ContentMD5 = ""; + m_blobUploadOptions.HttpHeaders.ContentType = "application/x-binary"; + m_blobUploadOptions.HttpHeaders.ContentLanguage = "en-US"; + m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment"; + m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache"; + m_blobUploadOptions.HttpHeaders.ContentEncoding = "identity"; + m_blobUploadOptions.HttpHeaders.ContentMD5 = ""; m_blobUploadOptions.Tier = Azure::Storage::Blobs::AccessTier::Hot; auto blobContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); m_blockBlobClient->Upload(blobContent, m_blobUploadOptions); - m_blobUploadOptions.Properties.ContentMD5 = m_blockBlobClient->GetProperties().ContentMD5; + m_blobUploadOptions.HttpHeaders.ContentMD5 + = m_blockBlobClient->GetProperties().HttpHeaders.ContentMD5; } void BlockBlobClientTest::TearDownTestSuite() { BlobContainerClientTest::TearDownTestSuite(); } @@ -76,7 +81,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(res.Version.empty()); EXPECT_FALSE(res.ETag.empty()); EXPECT_FALSE(res.LastModified.empty()); - EXPECT_EQ(res.Properties, m_blobUploadOptions.Properties); + EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders); EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata); EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob); Azure::Storage::Blobs::DownloadBlobOptions options; @@ -92,6 +97,35 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(res.ContentRange.GetValue().empty()); } + TEST_F(BlockBlobClientTest, DownloadEmpty) + { + std::vector emptyContent; + auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( + StandardStorageConnectionString(), m_containerName, RandomString()); + auto blobContent + = Azure::Core::Http::MemoryBodyStream(emptyContent.data(), emptyContent.size()); + blockBlobClient.Upload(blobContent); + blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders); + blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata); + + auto res = blockBlobClient.Download(); + EXPECT_EQ(res.BodyStream->Length(), 0); + EXPECT_FALSE(res.RequestId.empty()); + EXPECT_FALSE(res.Date.empty()); + EXPECT_FALSE(res.Version.empty()); + EXPECT_FALSE(res.ETag.empty()); + EXPECT_FALSE(res.LastModified.empty()); + EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob); + + Azure::Storage::Blobs::DownloadBlobOptions options; + options.Offset = 0; + EXPECT_THROW(blockBlobClient.Download(options), std::runtime_error); + options.Length = 1; + EXPECT_THROW(blockBlobClient.Download(options), std::runtime_error); + } + TEST_F(BlockBlobClientTest, CopyFromUri) { auto blobClient = m_blobContainerClient->GetBlobClient(RandomString()); @@ -136,7 +170,9 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_THROW(snapshotClient.SetMetadata({}), std::runtime_error); EXPECT_THROW( snapshotClient.SetAccessTier(Azure::Storage::Blobs::AccessTier::Cool), std::runtime_error); - EXPECT_THROW(snapshotClient.SetHttpHeaders(), std::runtime_error); + EXPECT_THROW( + snapshotClient.SetHttpHeaders(Azure::Storage::Blobs::BlobHttpHeaders()), + std::runtime_error); Azure::Storage::Blobs::CreateSnapshotOptions options; options.Metadata = {{"snapshotkey1", "snapshotvalue1"}, {"snapshotKEY2", "SNAPSHOTVALUE2"}}; @@ -155,14 +191,7 @@ namespace Azure { namespace Storage { namespace Test { blockBlobClient.Upload(blobContent); blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata); blockBlobClient.SetAccessTier(Azure::Storage::Blobs::AccessTier::Cool); - Azure::Storage::Blobs::SetBlobHttpHeadersOptions options; - options.ContentType = m_blobUploadOptions.Properties.ContentType; - options.ContentEncoding = m_blobUploadOptions.Properties.ContentEncoding; - options.ContentLanguage = m_blobUploadOptions.Properties.ContentLanguage; - options.ContentMD5 = m_blobUploadOptions.Properties.ContentMD5; - options.CacheControl = m_blobUploadOptions.Properties.CacheControl; - options.ContentDisposition = m_blobUploadOptions.Properties.ContentDisposition; - blockBlobClient.SetHttpHeaders(options); + blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders); auto res = blockBlobClient.GetProperties(); EXPECT_FALSE(res.RequestId.empty()); @@ -173,12 +202,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(res.CreationTime.empty()); EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata); EXPECT_EQ(res.ContentLength, static_cast(m_blobContent.size())); - EXPECT_EQ(res.ContentType, options.ContentType); - EXPECT_EQ(res.ContentEncoding, options.ContentEncoding); - EXPECT_EQ(res.ContentLanguage, options.ContentLanguage); - EXPECT_EQ(res.ContentMD5, options.ContentMD5); - EXPECT_EQ(res.CacheControl, options.CacheControl); - EXPECT_EQ(res.ContentDisposition, options.ContentDisposition); + EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders); EXPECT_EQ(res.Tier.GetValue(), Azure::Storage::Blobs::AccessTier::Cool); EXPECT_FALSE(res.AccessTierChangeTime.GetValue().empty()); } @@ -196,7 +220,7 @@ namespace Azure { namespace Storage { namespace Test { = Azure::Core::Http::MemoryBodyStream(block1Content.data(), block1Content.size()); blockBlobClient.StageBlock(blockId1, blockContent); Azure::Storage::Blobs::CommitBlockListOptions options; - options.Properties = m_blobUploadOptions.Properties; + options.HttpHeaders = m_blobUploadOptions.HttpHeaders; options.Metadata = m_blobUploadOptions.Metadata; blockBlobClient.CommitBlockList( {{Azure::Storage::Blobs::BlockType::Uncommitted, blockId1}}, options); @@ -230,4 +254,227 @@ namespace Azure { namespace Storage { namespace Test { */ } + TEST_F(BlockBlobClientTest, ConcurrentDownload) + { + std::vector downloadBuffer = m_blobContent; + for (int c : {1, 2, 4}) + { + Azure::Storage::Blobs::DownloadBlobToBufferOptions options; + options.Concurrency = c; + + // download whole blob + downloadBuffer.assign(downloadBuffer.size(), '\x00'); + auto res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size()); + EXPECT_EQ(downloadBuffer, m_blobContent); + EXPECT_EQ(static_cast(res.ContentLength), downloadBuffer.size()); + + // download whole blob + downloadBuffer.assign(downloadBuffer.size(), '\x00'); + options.Offset = 0; + res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size()); + EXPECT_EQ(downloadBuffer, m_blobContent); + EXPECT_EQ(static_cast(res.ContentLength), downloadBuffer.size()); + + // download whole blob + downloadBuffer.assign(downloadBuffer.size(), '\x00'); + options.Offset = 0; + options.Length = downloadBuffer.size(); + res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size()); + EXPECT_EQ(downloadBuffer, m_blobContent); + EXPECT_EQ(static_cast(res.ContentLength), downloadBuffer.size()); + + // download whole blob + downloadBuffer.assign(downloadBuffer.size(), '\x00'); + options.Offset = 0; + options.Length = downloadBuffer.size() * 2; + res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size() * 2); + EXPECT_EQ(downloadBuffer, m_blobContent); + EXPECT_EQ(static_cast(res.ContentLength), downloadBuffer.size()); + + options.InitialChunkSize = 4_KB; + options.ChunkSize = 4_KB; + + auto downloadRange = [&](int64_t offset, int64_t length) { + int64_t actualLength + = std::min(length, static_cast(m_blobContent.size()) - offset); + + auto optionsCopy = options; + optionsCopy.Offset = offset; + optionsCopy.Length = length; + if (actualLength > 0) + { + std::vector downloadContent(static_cast(actualLength), '\x00'); + auto res = m_blockBlobClient->DownloadToBuffer( + downloadContent.data(), static_cast(actualLength), optionsCopy); + EXPECT_EQ( + downloadContent, + std::vector( + m_blobContent.begin() + static_cast(offset), + m_blobContent.begin() + static_cast(offset) + + downloadContent.size())); + EXPECT_EQ(res.ContentLength, actualLength); + } + else + { + EXPECT_THROW( + m_blockBlobClient->DownloadToBuffer(nullptr, 8 * 1024 * 1024, optionsCopy), + std::runtime_error); + } + }; + + // random range + std::vector> downloadRangeTasks; + std::mt19937_64 random_generator(std::random_device{}()); + for (int i = 0; i < 16; ++i) + { + std::uniform_int_distribution offsetDistribution(0, m_blobContent.size() - 1); + int64_t offset = offsetDistribution(random_generator); + std::uniform_int_distribution lengthDistribution(1, 64_KB); + int64_t length = lengthDistribution(random_generator); + downloadRangeTasks.emplace_back( + std::async(std::launch::async, downloadRange, offset, length)); + } + downloadRangeTasks.emplace_back(std::async(std::launch::async, downloadRange, 0, 1)); + downloadRangeTasks.emplace_back(std::async(std::launch::async, downloadRange, 1, 1)); + downloadRangeTasks.emplace_back( + std::async(std::launch::async, downloadRange, m_blobContent.size() - 1, 1)); + downloadRangeTasks.emplace_back( + std::async(std::launch::async, downloadRange, m_blobContent.size() - 1, 2)); + downloadRangeTasks.emplace_back( + std::async(std::launch::async, downloadRange, m_blobContent.size(), 1)); + downloadRangeTasks.emplace_back( + std::async(std::launch::async, downloadRange, m_blobContent.size() + 1, 2)); + + for (auto& task : downloadRangeTasks) + { + task.get(); + } + + // buffer not big enough + options.Offset = 1; + for (int64_t length : {1ULL, 2ULL, 4_KB, 5_KB, 8_KB, 11_KB, 20_KB}) + { + options.Length = length; + EXPECT_THROW( + m_blockBlobClient->DownloadToBuffer( + downloadBuffer.data(), static_cast(length - 1), options), + std::runtime_error); + } + } + } + + TEST_F(BlockBlobClientTest, ConcurrentDownloadEmptyBlob) + { + std::vector emptyContent; + auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( + StandardStorageConnectionString(), m_containerName, RandomString()); + auto blobContent + = Azure::Core::Http::MemoryBodyStream(emptyContent.data(), emptyContent.size()); + blockBlobClient.Upload(blobContent); + blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders); + blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata); + + auto res = blockBlobClient.DownloadToBuffer(emptyContent.data(), 0); + EXPECT_EQ(res.ContentLength, 0); + EXPECT_FALSE(res.ETag.empty()); + EXPECT_FALSE(res.LastModified.empty()); + EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob); + + res = blockBlobClient.DownloadToBuffer(emptyContent.data(), static_cast(8_MB)); + EXPECT_EQ(res.ContentLength, 0); + EXPECT_FALSE(res.ETag.empty()); + EXPECT_FALSE(res.LastModified.empty()); + EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob); + + for (int c : {1, 2}) + { + Azure::Storage::Blobs::DownloadBlobToBufferOptions options; + options.InitialChunkSize = 10; + options.ChunkSize = 10; + options.Concurrency = c; + + res = blockBlobClient.DownloadToBuffer( + emptyContent.data(), static_cast(8_MB), options); + EXPECT_EQ(res.ContentLength, 0); + EXPECT_FALSE(res.ETag.empty()); + EXPECT_FALSE(res.LastModified.empty()); + EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob); + + options.Offset = 0; + EXPECT_THROW( + blockBlobClient.DownloadToBuffer( + emptyContent.data(), static_cast(8_MB), options), + std::runtime_error); + + options.Offset = 1; + EXPECT_THROW( + blockBlobClient.DownloadToBuffer( + emptyContent.data(), static_cast(8_MB), options), + std::runtime_error); + + options.Offset = 0; + options.Length = 1; + EXPECT_THROW( + blockBlobClient.DownloadToBuffer( + emptyContent.data(), static_cast(8_MB), options), + std::runtime_error); + + options.Offset = 100; + options.Length = 100; + EXPECT_THROW( + blockBlobClient.DownloadToBuffer( + emptyContent.data(), static_cast(8_MB), options), + std::runtime_error); + } + } + + TEST_F(BlockBlobClientTest, ConcurrentUpload) + { + auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( + StandardStorageConnectionString(), m_containerName, RandomString()); + for (int c : {1, 2, 5}) + { + for (int64_t length : + {0ULL, 1ULL, 2ULL, 2_KB, 4_KB, 999_KB, 1_MB, 2_MB - 1, 3_MB, 5_MB, 8_MB - 1234, 8_MB}) + { + Azure::Storage::Blobs::UploadBlobOptions options; + options.ChunkSize = 1_MB; + options.Concurrency = c; + options.HttpHeaders = m_blobUploadOptions.HttpHeaders; + options.Metadata = m_blobUploadOptions.Metadata; + options.Tier = m_blobUploadOptions.Tier; + + auto res = blockBlobClient.UploadFromBuffer( + m_blobContent.data(), static_cast(length), options); + EXPECT_FALSE(res.RequestId.empty()); + EXPECT_FALSE(res.Version.empty()); + EXPECT_FALSE(res.Date.empty()); + EXPECT_FALSE(res.ETag.empty()); + EXPECT_FALSE(res.LastModified.empty()); + EXPECT_FALSE(res.SequenceNumber.HasValue()); + EXPECT_FALSE(res.ContentCRC64.HasValue()); + EXPECT_FALSE(res.ContentMD5.HasValue()); + auto properties = blockBlobClient.GetProperties(); + EXPECT_EQ(properties.ContentLength, length); + EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); + EXPECT_EQ(properties.Metadata, options.Metadata); + EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue()); + EXPECT_EQ(properties.ETag, res.ETag); + EXPECT_EQ(properties.LastModified, res.LastModified); + std::vector downloadContent(static_cast(length), '\x00'); + blockBlobClient.DownloadToBuffer(downloadContent.data(), static_cast(length)); + EXPECT_EQ( + downloadContent, + std::vector( + m_blobContent.begin(), m_blobContent.begin() + static_cast(length))); + } + } + } + }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/test/page_blob_client_test.cpp b/sdk/storage/test/page_blob_client_test.cpp index 2891a575e..4028a11f9 100644 --- a/sdk/storage/test/page_blob_client_test.cpp +++ b/sdk/storage/test/page_blob_client_test.cpp @@ -22,17 +22,18 @@ namespace Azure { namespace Storage { namespace Test { m_blobContent.resize(static_cast(1_KB)); RandomBuffer(reinterpret_cast(&m_blobContent[0]), m_blobContent.size()); m_blobUploadOptions.Metadata = {{"key1", "V1"}, {"KEY2", "Value2"}}; - m_blobUploadOptions.Properties.ContentType = "application/x-binary"; - m_blobUploadOptions.Properties.ContentLanguage = "en-US"; - m_blobUploadOptions.Properties.ContentDisposition = "attachment"; - m_blobUploadOptions.Properties.CacheControl = "no-cache"; - m_blobUploadOptions.Properties.ContentEncoding = "identity"; - m_blobUploadOptions.Properties.ContentMD5 = ""; + m_blobUploadOptions.HttpHeaders.ContentType = "application/x-binary"; + m_blobUploadOptions.HttpHeaders.ContentLanguage = "en-US"; + m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment"; + m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache"; + m_blobUploadOptions.HttpHeaders.ContentEncoding = "identity"; + m_blobUploadOptions.HttpHeaders.ContentMD5 = ""; m_pageBlobClient->Create(m_blobContent.size(), m_blobUploadOptions); auto pageContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); m_pageBlobClient->UploadPages(pageContent, 0); - m_blobUploadOptions.Properties.ContentMD5 = m_pageBlobClient->GetProperties().ContentMD5; + m_blobUploadOptions.HttpHeaders.ContentMD5 + = m_pageBlobClient->GetProperties().HttpHeaders.ContentMD5; } void PageBlobClientTest::TearDownTestSuite() { BlobContainerClientTest::TearDownTestSuite(); }