From 204c025d095e2b1a74f0f1ded806e4205dd554f0 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Thu, 16 Jul 2020 15:49:55 +0800 Subject: [PATCH] suggested changes in API review (#312) * suggested changes in API review * Rename UrlBuilder->UriBuilder * add perf test case * Integrate retrypolicy into blob clients * remove `Content-Length` header for GET/HEAD/DELETE http requests * Remove temp files in ut --- sdk/storage/CMakeLists.txt | 6 +- sdk/storage/inc/blobs/append_blob_client.hpp | 2 +- sdk/storage/inc/blobs/blob_client.hpp | 6 +- .../inc/blobs/blob_container_client.hpp | 6 +- sdk/storage/inc/blobs/blob_options.hpp | 63 +++++++------- sdk/storage/inc/blobs/blob_service_client.hpp | 4 +- sdk/storage/inc/blobs/block_blob_client.hpp | 6 +- .../internal/protocol/blob_rest_client.hpp | 10 --- sdk/storage/inc/blobs/page_blob_client.hpp | 22 ++--- sdk/storage/inc/common/storage_credential.hpp | 10 +-- ...rl_builder.hpp => storage_uri_builder.hpp} | 6 +- .../inc/datalake/datalake_utilities.hpp | 4 +- .../inc/datalake/file_system_client.hpp | 10 +-- sdk/storage/inc/datalake/path_client.hpp | 10 +-- sdk/storage/inc/datalake/service_client.hpp | 6 +- sdk/storage/src/blobs/append_blob_client.cpp | 42 +++++----- sdk/storage/src/blobs/blob_client.cpp | 83 ++++++++++--------- .../src/blobs/blob_container_client.cpp | 21 +++-- sdk/storage/src/blobs/blob_service_client.cpp | 9 +- sdk/storage/src/blobs/block_blob_client.cpp | 38 ++++----- sdk/storage/src/blobs/page_blob_client.cpp | 74 ++++++++--------- sdk/storage/src/common/shared_key_policy.cpp | 4 +- sdk/storage/src/common/storage_credential.cpp | 8 +- ...rl_builder.cpp => storage_uri_builder.cpp} | 18 ++-- .../src/datalake/datalake_utilities.cpp | 4 +- sdk/storage/test/CMakeLists.txt | 1 + .../test/blobs/append_blob_client_test.cpp | 20 ++--- .../test/blobs/blob_container_client_test.cpp | 6 +- .../test/blobs/block_blob_client_test.cpp | 16 ++-- .../test/blobs/page_blob_client_test.cpp | 6 +- .../test/blobs/performance_benchmark.cpp | 39 +++++++++ sdk/storage/test/test_base.hpp | 14 +++- 32 files changed, 314 insertions(+), 260 deletions(-) rename sdk/storage/inc/common/{storage_url_builder.hpp => storage_uri_builder.hpp} (96%) rename sdk/storage/src/common/{storage_url_builder.cpp => storage_uri_builder.cpp} (91%) create mode 100644 sdk/storage/test/blobs/performance_benchmark.cpp diff --git a/sdk/storage/CMakeLists.txt b/sdk/storage/CMakeLists.txt index 92c02542d..409b00253 100644 --- a/sdk/storage/CMakeLists.txt +++ b/sdk/storage/CMakeLists.txt @@ -14,7 +14,7 @@ set (AZURE_STORAGE_BLOB_HEADER inc/common/constants.hpp inc/common/storage_common.hpp inc/common/storage_credential.hpp - inc/common/storage_url_builder.hpp + inc/common/storage_uri_builder.hpp inc/common/common_headers_request_policy.hpp inc/common/shared_key_policy.hpp inc/common/crypt.hpp @@ -38,7 +38,7 @@ set (AZURE_STORAGE_DATALAKE_HEADER inc/common/xml_wrapper.hpp inc/common/storage_error.hpp inc/common/storage_credential.hpp - inc/common/storage_url_builder.hpp + inc/common/storage_uri_builder.hpp inc/common/common_headers_request_policy.hpp inc/common/shared_key_policy.hpp inc/common/token_credential_policy.hpp @@ -56,7 +56,7 @@ set (AZURE_STORAGE_DATALAKE_HEADER set (AZURE_STORAGE_BLOB_SOURCE src/common/storage_credential.cpp - src/common/storage_url_builder.cpp + src/common/storage_uri_builder.cpp src/common/common_headers_request_policy.cpp src/common/shared_key_policy.cpp src/common/storage_error.cpp diff --git a/sdk/storage/inc/blobs/append_blob_client.hpp b/sdk/storage/inc/blobs/append_blob_client.hpp index e3399a318..eaccefcfa 100644 --- a/sdk/storage/inc/blobs/append_blob_client.hpp +++ b/sdk/storage/inc/blobs/append_blob_client.hpp @@ -117,7 +117,7 @@ namespace Azure { namespace Storage { namespace Blobs { * @return A BlobAppendInfo describing the state of the updated append blob. */ BlobAppendInfo AppendBlock( - Azure::Core::Http::BodyStream& content, + Azure::Core::Http::BodyStream* content, const AppendBlockOptions& options = AppendBlockOptions()); /** diff --git a/sdk/storage/inc/blobs/blob_client.hpp b/sdk/storage/inc/blobs/blob_client.hpp index 5abf2a643..aaec3d5e1 100644 --- a/sdk/storage/inc/blobs/blob_client.hpp +++ b/sdk/storage/inc/blobs/blob_client.hpp @@ -5,7 +5,7 @@ #include "blob_options.hpp" #include "common/storage_credential.hpp" -#include "common/storage_url_builder.hpp" +#include "common/storage_uri_builder.hpp" #include "internal/protocol/blob_rest_client.hpp" #include @@ -289,12 +289,12 @@ namespace Azure { namespace Storage { namespace Blobs { UndeleteBlobResponse Undelete(const UndeleteBlobOptions& options = UndeleteBlobOptions()) const; protected: - UrlBuilder m_blobUrl; + UriBuilder m_blobUrl; std::shared_ptr m_pipeline; private: explicit BlobClient( - UrlBuilder blobUri, + UriBuilder blobUri, std::shared_ptr pipeline) : m_blobUrl(std::move(blobUri)), m_pipeline(std::move(pipeline)) { diff --git a/sdk/storage/inc/blobs/blob_container_client.hpp b/sdk/storage/inc/blobs/blob_container_client.hpp index 5c9f1816c..a1599a4d0 100644 --- a/sdk/storage/inc/blobs/blob_container_client.hpp +++ b/sdk/storage/inc/blobs/blob_container_client.hpp @@ -6,7 +6,7 @@ #include "blob_options.hpp" #include "blobs/blob_client.hpp" #include "common/storage_credential.hpp" -#include "common/storage_url_builder.hpp" +#include "common/storage_uri_builder.hpp" #include "internal/protocol/blob_rest_client.hpp" #include @@ -202,11 +202,11 @@ namespace Azure { namespace Storage { namespace Blobs { const ListBlobsOptions& options = ListBlobsOptions()) const; private: - UrlBuilder m_containerUrl; + UriBuilder m_containerUrl; std::shared_ptr m_pipeline; explicit BlobContainerClient( - UrlBuilder containerUri, + UriBuilder containerUri, std::shared_ptr pipeline) : m_containerUrl(std::move(containerUri)), m_pipeline(std::move(pipeline)) { diff --git a/sdk/storage/inc/blobs/blob_options.hpp b/sdk/storage/inc/blobs/blob_options.hpp index 8f2c91d32..6d08c68ed 100644 --- a/sdk/storage/inc/blobs/blob_options.hpp +++ b/sdk/storage/inc/blobs/blob_options.hpp @@ -190,7 +190,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - ContainerAccessConditions Conditions; + ContainerAccessConditions AccessConditions; }; /** @@ -206,7 +206,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - LeaseAccessConditions Conditions; + LeaseAccessConditions AccessConditions; }; /** @@ -222,7 +222,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - ContainerAccessConditions Conditions; + ContainerAccessConditions AccessConditions; }; /** @@ -314,7 +314,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; /** @@ -330,7 +330,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; /** @@ -346,7 +346,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; /** @@ -389,7 +389,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; /** * @brief Optional conditions that the source must meet to perform this operation. @@ -422,7 +422,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - LeaseAccessConditions Conditions; + LeaseAccessConditions AccessConditions; }; /** @@ -449,7 +449,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; /** @@ -517,7 +517,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; /** @@ -540,7 +540,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; /** @@ -596,7 +596,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; /** @@ -662,7 +662,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - LeaseAccessConditions Conditions; + LeaseAccessConditions AccessConditions; }; /** @@ -703,7 +703,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - LeaseAccessConditions Conditions; + LeaseAccessConditions AccessConditions; /** * @brief Optional conditions that the source must meet to perform this operation. @@ -739,7 +739,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; /** @@ -761,7 +761,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - LeaseAccessConditions Conditions; + LeaseAccessConditions AccessConditions; }; /** @@ -787,7 +787,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; /** @@ -817,7 +817,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - AppendBlobAccessConditions Conditions; + AppendBlobAccessConditions AccessConditions; }; /** @@ -858,7 +858,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - AppendBlobAccessConditions Conditions; + AppendBlobAccessConditions AccessConditions; }; /** @@ -870,6 +870,11 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Context for cancelling long running operations. */ Azure::Core::Context Context; + + /** + * @brief The sequence number is a user-controlled value that you can use to track requests. The + * value of the sequence number must be between 0 and 2^63 - 1. + */ Azure::Core::Nullable SequenceNumber; /** @@ -890,7 +895,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; /** @@ -920,7 +925,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - PageBlobAccessConditions Conditions; + PageBlobAccessConditions AccessConditions; }; /** @@ -950,7 +955,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - PageBlobAccessConditions Conditions; + PageBlobAccessConditions AccessConditions; }; /** @@ -966,7 +971,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - PageBlobAccessConditions Conditions; + PageBlobAccessConditions AccessConditions; }; /** @@ -982,7 +987,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; /** @@ -1012,19 +1017,21 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Nullable PreviousSnapshotUrl; /** - * @brief Optionally specifies the offset of range over which to list ranges. + * @brief Optionally specifies the offset of range over which to list ranges. This offset must + * be a modulus of 512. */ Azure::Core::Nullable Offset; /** - * @brief Optionally specifies the length of range over which to list ranges. + * @brief Optionally specifies the length of range over which to list ranges. The length must be + * a modulus of 512. */ Azure::Core::Nullable Length; /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; /** @@ -1040,7 +1047,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that must be met to perform this operation. */ - BlobAccessConditions Conditions; + BlobAccessConditions AccessConditions; }; }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/inc/blobs/blob_service_client.hpp b/sdk/storage/inc/blobs/blob_service_client.hpp index b0febaf10..e911ca61f 100644 --- a/sdk/storage/inc/blobs/blob_service_client.hpp +++ b/sdk/storage/inc/blobs/blob_service_client.hpp @@ -6,7 +6,7 @@ #include "blob_options.hpp" #include "blobs/blob_container_client.hpp" #include "common/storage_credential.hpp" -#include "common/storage_url_builder.hpp" +#include "common/storage_uri_builder.hpp" #include "internal/protocol/blob_rest_client.hpp" #include @@ -124,7 +124,7 @@ namespace Azure { namespace Storage { namespace Blobs { const GetUserDelegationKeyOptions& options = GetUserDelegationKeyOptions()) const; protected: - UrlBuilder m_serviceUrl; + UriBuilder m_serviceUrl; std::shared_ptr m_pipeline; }; }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/inc/blobs/block_blob_client.hpp b/sdk/storage/inc/blobs/block_blob_client.hpp index b6d6f9b3f..0f2633944 100644 --- a/sdk/storage/inc/blobs/block_blob_client.hpp +++ b/sdk/storage/inc/blobs/block_blob_client.hpp @@ -110,7 +110,7 @@ namespace Azure { namespace Storage { namespace Blobs { * @return A BlobContentInfo describing the state of the updated block blob. */ BlobContentInfo Upload( - Azure::Core::Http::BodyStream& content, + Azure::Core::Http::BodyStream* content, const UploadBlockBlobOptions& options = UploadBlockBlobOptions()) const; /** @@ -131,7 +131,7 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Creates a new block blob, or updates the content of an existing block blob. Updating * an existing block blob overwrites any existing metadata on the blob. * - * @param buffer A file containing the content to upload. + * @param file A file containing the content to upload. * @param options Optional parameters to execute this function. * @return A BlobContentInfo describing the state of the updated block blob. */ @@ -151,7 +151,7 @@ namespace Azure { namespace Storage { namespace Blobs { */ BlockInfo StageBlock( const std::string& blockId, - Azure::Core::Http::BodyStream& content, + Azure::Core::Http::BodyStream* content, const StageBlockOptions& options = StageBlockOptions()) const; /** diff --git a/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp b/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp index 21701731a..3831e2619 100644 --- a/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp +++ b/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp @@ -1118,7 +1118,6 @@ namespace Azure { namespace Storage { namespace Blobs { { unused(body); auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); - request.AddHeader("Content-Length", "0"); request.AddHeader("x-ms-version", "2019-12-12"); if (options.Timeout.HasValue()) { @@ -1820,7 +1819,6 @@ namespace Azure { namespace Storage { namespace Blobs { { unused(body); auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Delete, url); - request.AddHeader("Content-Length", "0"); request.AddQueryParameter("restype", "container"); request.AddHeader("x-ms-version", "2019-12-12"); if (options.Timeout.HasValue()) @@ -1897,7 +1895,6 @@ namespace Azure { namespace Storage { namespace Blobs { { unused(body); auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Head, url); - request.AddHeader("Content-Length", "0"); request.AddQueryParameter("restype", "container"); request.AddHeader("x-ms-version", "2019-12-12"); if (options.Timeout.HasValue()) @@ -2094,7 +2091,6 @@ namespace Azure { namespace Storage { namespace Blobs { { unused(body); auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); - request.AddHeader("Content-Length", "0"); request.AddHeader("x-ms-version", "2019-12-12"); if (options.Timeout.HasValue()) { @@ -2194,7 +2190,6 @@ namespace Azure { namespace Storage { namespace Blobs { { unused(body); auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); - request.AddHeader("Content-Length", "0"); request.AddHeader("x-ms-version", "2019-12-12"); if (options.Timeout.HasValue()) { @@ -2907,7 +2902,6 @@ namespace Azure { namespace Storage { namespace Blobs { { unused(body); auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); - request.AddHeader("Content-Length", "0"); request.AddHeader("x-ms-version", "2019-12-12"); if (options.Timeout.HasValue()) { @@ -3125,7 +3119,6 @@ namespace Azure { namespace Storage { namespace Blobs { { unused(body); auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Delete, url); - request.AddHeader("Content-Length", "0"); request.AddHeader("x-ms-version", "2019-12-12"); if (options.Timeout.HasValue()) { @@ -3277,7 +3270,6 @@ namespace Azure { namespace Storage { namespace Blobs { { unused(body); auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Head, url); - request.AddHeader("Content-Length", "0"); request.AddHeader("x-ms-version", "2019-12-12"); if (options.Timeout.HasValue()) { @@ -4802,7 +4794,6 @@ namespace Azure { namespace Storage { namespace Blobs { { unused(body); auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); - request.AddHeader("Content-Length", "0"); request.AddQueryParameter("comp", "blocklist"); if (options.ListType.HasValue()) { @@ -5840,7 +5831,6 @@ namespace Azure { namespace Storage { namespace Blobs { { unused(body); auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); - request.AddHeader("Content-Length", "0"); request.AddQueryParameter("comp", "pagelist"); if (options.PreviousSnapshot.HasValue()) { diff --git a/sdk/storage/inc/blobs/page_blob_client.hpp b/sdk/storage/inc/blobs/page_blob_client.hpp index db59ff8b8..8e95c5c3b 100644 --- a/sdk/storage/inc/blobs/page_blob_client.hpp +++ b/sdk/storage/inc/blobs/page_blob_client.hpp @@ -141,7 +141,7 @@ namespace Azure { namespace Storage { namespace Blobs { * PageInfo describing the state of the updated pages. */ PageInfo UploadPages( - Azure::Core::Http::BodyStream& content, + Azure::Core::Http::BodyStream* content, int64_t offset, const UploadPagesOptions& options = UploadPagesOptions()); @@ -153,15 +153,14 @@ namespace Azure { namespace Storage { namespace Blobs { * uri of up to 2 KB in length that specifies a blob. The source blob must either be public or * must be authenticated via a shared access signature. If the source blob is public, no * authentication is required to perform the operation. - * @param sourceOffset Only upload a - * part of the blob in the sourceUri from the specified offset. - * @param sourceLength Only - * upload specified length of the blob in the sourceUri. - * @param destinationOffset - * Specifies the starting offset for the content to be written. Given that pages must be aligned - * with 512-byte boundaries, the start offset must be a modulus of 512. - * @param options - * Optional parameters to execute this function. + * @param sourceOffset Only upload a part of the blob in the sourceUri from the specified + * offset. This offset doesn't need to be a modulus of 512. + * @param sourceLength Only upload specified length of the blob in the sourceUri. This length + * must be a modulus of 512. + * @param destinationOffset Specifies the starting offset for the content to be written. Given + * that pages must be aligned with 512-byte boundaries, the start offset must be a modulus of + * 512. + * @param options Optional parameters to execute this function. * @return A PageInfo describing the state * of the updated pages. */ @@ -177,7 +176,8 @@ namespace Azure { namespace Storage { namespace Blobs { * * @param offset Specifies the starting offset for the content to be cleared. Given that pages * must be aligned with 512-byte boundaries, the start offset must be a modulus of 512. - * @param length Specifies the length of the content to be cleared. + * @param length Specifies the length of the content to be cleared. The length must be a modulus + * of 512. * @param options Optional parameters to execute this function. * @return A PageInfo describing the state of the updated pages. */ diff --git a/sdk/storage/inc/common/storage_credential.hpp b/sdk/storage/inc/common/storage_credential.hpp index b3aabd400..9d110f2bb 100644 --- a/sdk/storage/inc/common/storage_credential.hpp +++ b/sdk/storage/inc/common/storage_credential.hpp @@ -3,7 +3,7 @@ #pragma once -#include "common/storage_url_builder.hpp" +#include "common/storage_uri_builder.hpp" #include #include @@ -67,10 +67,10 @@ namespace Azure { namespace Storage { struct ConnectionStringParts { - UrlBuilder BlobServiceUri; - UrlBuilder FileServiceUri; - UrlBuilder QueueServiceUri; - UrlBuilder DataLakeServiceUri; + UriBuilder BlobServiceUri; + UriBuilder FileServiceUri; + UriBuilder QueueServiceUri; + UriBuilder DataLakeServiceUri; std::shared_ptr KeyCredential; }; diff --git a/sdk/storage/inc/common/storage_url_builder.hpp b/sdk/storage/inc/common/storage_uri_builder.hpp similarity index 96% rename from sdk/storage/inc/common/storage_url_builder.hpp rename to sdk/storage/inc/common/storage_uri_builder.hpp index e39358b7c..78d5d261e 100644 --- a/sdk/storage/inc/common/storage_url_builder.hpp +++ b/sdk/storage/inc/common/storage_uri_builder.hpp @@ -9,12 +9,12 @@ namespace Azure { namespace Storage { - class UrlBuilder { + class UriBuilder { public: - UrlBuilder() {} + UriBuilder() {} // url must be url-encoded - explicit UrlBuilder(const std::string& url); + explicit UriBuilder(const std::string& url); void SetScheme(const std::string& scheme) { m_scheme = scheme; } diff --git a/sdk/storage/inc/datalake/datalake_utilities.hpp b/sdk/storage/inc/datalake/datalake_utilities.hpp index 956df7dea..57f16be0f 100644 --- a/sdk/storage/inc/datalake/datalake_utilities.hpp +++ b/sdk/storage/inc/datalake/datalake_utilities.hpp @@ -3,13 +3,13 @@ #pragma once -#include "common/storage_url_builder.hpp" +#include "common/storage_uri_builder.hpp" #include #include namespace Azure { namespace Storage { namespace DataLake { namespace Details { - UrlBuilder GetBlobUriFromDfsUri(const UrlBuilder& dfsUri); + UriBuilder GetBlobUriFromDfsUri(const UriBuilder& dfsUri); std::map DeserializeMetadata( const std::string& dataLakePropertiesString); diff --git a/sdk/storage/inc/datalake/file_system_client.hpp b/sdk/storage/inc/datalake/file_system_client.hpp index afc232d19..88b2d8afd 100644 --- a/sdk/storage/inc/datalake/file_system_client.hpp +++ b/sdk/storage/inc/datalake/file_system_client.hpp @@ -4,7 +4,7 @@ #pragma once #include "common/storage_credential.hpp" -#include "common/storage_url_builder.hpp" +#include "common/storage_uri_builder.hpp" #include "datalake/service_client.hpp" #include "datalake_options.hpp" #include "http/pipeline.hpp" @@ -130,13 +130,13 @@ namespace Azure { namespace Storage { namespace DataLake { const ListPathsOptions& options = ListPathsOptions()) const; private: - UrlBuilder m_dfsUri; - UrlBuilder m_blobUri; + UriBuilder m_dfsUri; + UriBuilder m_blobUri; std::shared_ptr m_pipeline; explicit FileSystemClient( - UrlBuilder dfsUri, - UrlBuilder blobUri, + UriBuilder dfsUri, + UriBuilder blobUri, std::shared_ptr pipeline) : m_dfsUri(std::move(dfsUri)), m_blobUri(std::move(blobUri)), m_pipeline(std::move(pipeline)) diff --git a/sdk/storage/inc/datalake/path_client.hpp b/sdk/storage/inc/datalake/path_client.hpp index e9296fc69..5d47d9f9e 100644 --- a/sdk/storage/inc/datalake/path_client.hpp +++ b/sdk/storage/inc/datalake/path_client.hpp @@ -4,7 +4,7 @@ #pragma once #include "common/storage_credential.hpp" -#include "common/storage_url_builder.hpp" +#include "common/storage_uri_builder.hpp" #include "datalake/file_system_client.hpp" #include "datalake_options.hpp" #include "http/pipeline.hpp" @@ -311,13 +311,13 @@ namespace Azure { namespace Storage { namespace DataLake { ReadPathResponse Read(const PathReadOptions& options = PathReadOptions()) const; private: - UrlBuilder m_dfsUri; - UrlBuilder m_blobUri; + UriBuilder m_dfsUri; + UriBuilder m_blobUri; std::shared_ptr m_pipeline; explicit PathClient( - UrlBuilder dfsUri, - UrlBuilder blobUri, + UriBuilder dfsUri, + UriBuilder blobUri, std::shared_ptr pipeline) : m_dfsUri(std::move(dfsUri)), m_blobUri(std::move(blobUri)), m_pipeline(std::move(pipeline)) diff --git a/sdk/storage/inc/datalake/service_client.hpp b/sdk/storage/inc/datalake/service_client.hpp index 1c6f3112d..e45094da7 100644 --- a/sdk/storage/inc/datalake/service_client.hpp +++ b/sdk/storage/inc/datalake/service_client.hpp @@ -4,7 +4,7 @@ #pragma once #include "common/storage_credential.hpp" -#include "common/storage_url_builder.hpp" +#include "common/storage_uri_builder.hpp" #include "datalake_options.hpp" #include "http/pipeline.hpp" #include "protocol/datalake_rest_client.hpp" @@ -75,8 +75,8 @@ namespace Azure { namespace Storage { namespace DataLake { const ListFileSystemsOptions& options = ListFileSystemsOptions()) const; private: - UrlBuilder m_dfsUri; - UrlBuilder m_blobUri; + UriBuilder m_dfsUri; + UriBuilder m_blobUri; std::shared_ptr m_pipeline; }; }}} // namespace Azure::Storage::DataLake diff --git a/sdk/storage/src/blobs/append_blob_client.cpp b/sdk/storage/src/blobs/append_blob_client.cpp index 7c210bb69..06f937fe3 100644 --- a/sdk/storage/src/blobs/append_blob_client.cpp +++ b/sdk/storage/src/blobs/append_blob_client.cpp @@ -63,31 +63,31 @@ namespace Azure { namespace Storage { namespace Blobs { BlobRestClient::AppendBlob::CreateOptions protocolLayerOptions; protocolLayerOptions.HttpHeaders = options.HttpHeaders; protocolLayerOptions.Metadata = options.Metadata; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::AppendBlob::Create( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } BlobAppendInfo AppendBlobClient::AppendBlock( - Azure::Core::Http::BodyStream& content, + Azure::Core::Http::BodyStream* content, const AppendBlockOptions& options) { BlobRestClient::AppendBlob::AppendBlockOptions protocolLayerOptions; protocolLayerOptions.ContentMD5 = options.ContentMD5; protocolLayerOptions.ContentCRC64 = options.ContentCRC64; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.MaxSize = options.Conditions.MaxSize; - protocolLayerOptions.AppendPosition = options.Conditions.AppendPosition; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.MaxSize = options.AccessConditions.MaxSize; + protocolLayerOptions.AppendPosition = options.AccessConditions.AppendPosition; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::AppendBlob::AppendBlock( - options.Context, *m_pipeline, m_blobUrl.ToString(), content, protocolLayerOptions); + options.Context, *m_pipeline, m_blobUrl.ToString(), *content, protocolLayerOptions); } BlobAppendInfo AppendBlobClient::AppendBlockFromUri( @@ -111,13 +111,13 @@ namespace Azure { namespace Storage { namespace Blobs { } protocolLayerOptions.ContentMD5 = options.ContentMD5; protocolLayerOptions.ContentCRC64 = options.ContentCRC64; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.MaxSize = options.Conditions.MaxSize; - protocolLayerOptions.AppendPosition = options.Conditions.AppendPosition; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.MaxSize = options.AccessConditions.MaxSize; + protocolLayerOptions.AppendPosition = options.AccessConditions.AppendPosition; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::AppendBlob::AppendBlockFromUri( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } diff --git a/sdk/storage/src/blobs/blob_client.cpp b/sdk/storage/src/blobs/blob_client.cpp index e2c1c1cac..8b77ada49 100644 --- a/sdk/storage/src/blobs/blob_client.cpp +++ b/sdk/storage/src/blobs/blob_client.cpp @@ -48,7 +48,8 @@ namespace Azure { namespace Storage { namespace Blobs { { policies.emplace_back(std::unique_ptr(p->Clone())); } - // TODO: Retry policy goes here + policies.emplace_back( + std::make_unique(Azure::Core::Http::RetryOptions())); for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); @@ -71,7 +72,8 @@ namespace Azure { namespace Storage { namespace Blobs { { policies.emplace_back(std::unique_ptr(p->Clone())); } - // TODO: Retry policy goes here + policies.emplace_back( + std::make_unique(Azure::Core::Http::RetryOptions())); for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); @@ -92,7 +94,8 @@ namespace Azure { namespace Storage { namespace Blobs { { policies.emplace_back(std::unique_ptr(p->Clone())); } - // TODO: Retry policy goes here + policies.emplace_back( + std::make_unique(Azure::Core::Http::RetryOptions())); for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); @@ -137,11 +140,11 @@ namespace Azure { namespace Storage { namespace Blobs { options.Offset.GetValue(), std::numeric_limits>::max()); } - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::Blob::Download( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); @@ -404,11 +407,11 @@ namespace Azure { namespace Storage { namespace Blobs { BlobProperties BlobClient::GetProperties(const GetBlobPropertiesOptions& options) const { BlobRestClient::Blob::GetPropertiesOptions protocolLayerOptions; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::Blob::GetProperties( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } @@ -419,11 +422,11 @@ namespace Azure { namespace Storage { namespace Blobs { { BlobRestClient::Blob::SetHttpHeadersOptions protocolLayerOptions; protocolLayerOptions.HttpHeaders = std::move(httpHeaders); - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::Blob::SetHttpHeaders( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } @@ -434,11 +437,11 @@ namespace Azure { namespace Storage { namespace Blobs { { BlobRestClient::Blob::SetMetadataOptions protocolLayerOptions; protocolLayerOptions.Metadata = std::move(metadata); - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::Blob::SetMetadata( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } @@ -463,14 +466,14 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.SourceUri = sourceUri; protocolLayerOptions.Tier = options.Tier; protocolLayerOptions.RehydratePriority = options.RehydratePriority; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; protocolLayerOptions.SourceLeaseId = options.SourceConditions.LeaseId; protocolLayerOptions.SourceIfModifiedSince = options.SourceConditions.IfModifiedSince; - protocolLayerOptions.SourceIfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; + protocolLayerOptions.SourceIfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; protocolLayerOptions.SourceIfMatch = options.SourceConditions.IfMatch; protocolLayerOptions.SourceIfNoneMatch = options.SourceConditions.IfNoneMatch; return BlobRestClient::Blob::StartCopyFromUri( @@ -483,7 +486,7 @@ namespace Azure { namespace Storage { namespace Blobs { { BlobRestClient::Blob::AbortCopyFromUriOptions protocolLayerOptions; protocolLayerOptions.CopyId = copyId; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; return BlobRestClient::Blob::AbortCopyFromUri( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } @@ -492,11 +495,11 @@ namespace Azure { namespace Storage { namespace Blobs { { BlobRestClient::Blob::CreateSnapshotOptions protocolLayerOptions; protocolLayerOptions.Metadata = options.Metadata; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::Blob::CreateSnapshot( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } @@ -505,11 +508,11 @@ namespace Azure { namespace Storage { namespace Blobs { { BlobRestClient::Blob::DeleteOptions protocolLayerOptions; protocolLayerOptions.DeleteSnapshots = options.DeleteSnapshots; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::Blob::Delete( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } diff --git a/sdk/storage/src/blobs/blob_container_client.cpp b/sdk/storage/src/blobs/blob_container_client.cpp index 39c4fb309..d0e065c4f 100644 --- a/sdk/storage/src/blobs/blob_container_client.cpp +++ b/sdk/storage/src/blobs/blob_container_client.cpp @@ -44,7 +44,8 @@ namespace Azure { namespace Storage { namespace Blobs { { policies.emplace_back(std::unique_ptr(p->Clone())); } - // TODO: Retry policy goes here + policies.emplace_back( + std::make_unique(Azure::Core::Http::RetryOptions())); for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); @@ -67,7 +68,8 @@ namespace Azure { namespace Storage { namespace Blobs { { policies.emplace_back(std::unique_ptr(p->Clone())); } - // TODO: Retry policy goes here + policies.emplace_back( + std::make_unique(Azure::Core::Http::RetryOptions())); for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); @@ -90,7 +92,8 @@ namespace Azure { namespace Storage { namespace Blobs { { policies.emplace_back(std::unique_ptr(p->Clone())); } - // TODO: Retry policy goes here + policies.emplace_back( + std::make_unique(Azure::Core::Http::RetryOptions())); for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); @@ -136,9 +139,9 @@ namespace Azure { namespace Storage { namespace Blobs { const DeleteBlobContainerOptions& options) const { BlobRestClient::Container::DeleteOptions protocolLayerOptions; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; return BlobRestClient::Container::Delete( options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions); } @@ -147,7 +150,7 @@ namespace Azure { namespace Storage { namespace Blobs { const GetBlobContainerPropertiesOptions& options) const { BlobRestClient::Container::GetPropertiesOptions protocolLayerOptions; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; return BlobRestClient::Container::GetProperties( options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions); } @@ -158,8 +161,8 @@ namespace Azure { namespace Storage { namespace Blobs { { BlobRestClient::Container::SetMetadataOptions protocolLayerOptions; protocolLayerOptions.Metadata = metadata; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; return BlobRestClient::Container::SetMetadata( options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions); } diff --git a/sdk/storage/src/blobs/blob_service_client.cpp b/sdk/storage/src/blobs/blob_service_client.cpp index 7cf45d2e2..fb69893da 100644 --- a/sdk/storage/src/blobs/blob_service_client.cpp +++ b/sdk/storage/src/blobs/blob_service_client.cpp @@ -39,7 +39,8 @@ namespace Azure { namespace Storage { namespace Blobs { { policies.emplace_back(std::unique_ptr(p->Clone())); } - // TODO: Retry policy goes here + policies.emplace_back( + std::make_unique(Azure::Core::Http::RetryOptions())); for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); @@ -62,7 +63,8 @@ namespace Azure { namespace Storage { namespace Blobs { { policies.emplace_back(std::unique_ptr(p->Clone())); } - // TODO: Retry policy goes here + policies.emplace_back( + std::make_unique(Azure::Core::Http::RetryOptions())); for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); @@ -85,7 +87,8 @@ namespace Azure { namespace Storage { namespace Blobs { { policies.emplace_back(std::unique_ptr(p->Clone())); } - // TODO: Retry policy goes here + policies.emplace_back( + std::make_unique(Azure::Core::Http::RetryOptions())); for (const auto& p : options.PerRetryPolicies) { policies.emplace_back(std::unique_ptr(p->Clone())); diff --git a/sdk/storage/src/blobs/block_blob_client.cpp b/sdk/storage/src/blobs/block_blob_client.cpp index e63f68969..9933e0cac 100644 --- a/sdk/storage/src/blobs/block_blob_client.cpp +++ b/sdk/storage/src/blobs/block_blob_client.cpp @@ -62,7 +62,7 @@ namespace Azure { namespace Storage { namespace Blobs { } BlobContentInfo BlockBlobClient::Upload( - Azure::Core::Http::BodyStream& content, + Azure::Core::Http::BodyStream* content, const UploadBlockBlobOptions& options) const { BlobRestClient::BlockBlob::UploadOptions protocolLayerOptions; @@ -71,13 +71,13 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.HttpHeaders = options.HttpHeaders; protocolLayerOptions.Metadata = options.Metadata; protocolLayerOptions.Tier = options.Tier; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::BlockBlob::Upload( - options.Context, *m_pipeline, m_blobUrl.ToString(), content, protocolLayerOptions); + options.Context, *m_pipeline, m_blobUrl.ToString(), *content, protocolLayerOptions); } BlobContentInfo BlockBlobClient::UploadFromBuffer( @@ -113,7 +113,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Http::MemoryBodyStream contentStream(buffer + offset, length); StageBlockOptions chunkOptions; chunkOptions.Context = options.Context; - auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions); + auto blockInfo = StageBlock(getBlockId(chunkId), &contentStream, chunkOptions); if (chunkId == numChunks - 1) { blockIds.resize(static_cast(numChunks)); @@ -173,7 +173,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Http::FileBodyStream contentStream(fileReader.GetHandle(), offset, length); StageBlockOptions chunkOptions; chunkOptions.Context = options.Context; - auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions); + auto blockInfo = StageBlock(getBlockId(chunkId), &contentStream, chunkOptions); if (chunkId == numChunks - 1) { blockIds.resize(static_cast(numChunks)); @@ -201,16 +201,16 @@ namespace Azure { namespace Storage { namespace Blobs { BlockInfo BlockBlobClient::StageBlock( const std::string& blockId, - Azure::Core::Http::BodyStream& content, + Azure::Core::Http::BodyStream* content, const StageBlockOptions& options) const { BlobRestClient::BlockBlob::StageBlockOptions protocolLayerOptions; protocolLayerOptions.BlockId = blockId; protocolLayerOptions.ContentMD5 = options.ContentMD5; protocolLayerOptions.ContentCRC64 = options.ContentCRC64; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; return BlobRestClient::BlockBlob::StageBlock( - options.Context, *m_pipeline, m_blobUrl.ToString(), content, protocolLayerOptions); + options.Context, *m_pipeline, m_blobUrl.ToString(), *content, protocolLayerOptions); } BlockInfo BlockBlobClient::StageBlockFromUri( @@ -236,7 +236,7 @@ namespace Azure { namespace Storage { namespace Blobs { } protocolLayerOptions.ContentMD5 = options.ContentMD5; protocolLayerOptions.ContentCRC64 = options.ContentCRC64; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; protocolLayerOptions.SourceIfModifiedSince = options.SourceConditions.IfModifiedSince; protocolLayerOptions.SourceIfUnmodifiedSince = options.SourceConditions.IfUnmodifiedSince; protocolLayerOptions.SourceIfMatch = options.SourceConditions.IfMatch; @@ -254,11 +254,11 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.HttpHeaders = options.HttpHeaders; protocolLayerOptions.Metadata = options.Metadata; protocolLayerOptions.Tier = options.Tier; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::BlockBlob::CommitBlockList( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } @@ -267,7 +267,7 @@ namespace Azure { namespace Storage { namespace Blobs { { BlobRestClient::BlockBlob::GetBlockListOptions protocolLayerOptions; protocolLayerOptions.ListType = options.ListType; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; return BlobRestClient::BlockBlob::GetBlockList( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } diff --git a/sdk/storage/src/blobs/page_blob_client.cpp b/sdk/storage/src/blobs/page_blob_client.cpp index 45c2926c1..4a6f1b417 100644 --- a/sdk/storage/src/blobs/page_blob_client.cpp +++ b/sdk/storage/src/blobs/page_blob_client.cpp @@ -66,31 +66,31 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.HttpHeaders = options.HttpHeaders; protocolLayerOptions.Metadata = options.Metadata; protocolLayerOptions.Tier = options.Tier; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::PageBlob::Create( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } PageInfo PageBlobClient::UploadPages( - Azure::Core::Http::BodyStream& content, + Azure::Core::Http::BodyStream* content, int64_t offset, const UploadPagesOptions& options) { BlobRestClient::PageBlob::UploadPagesOptions protocolLayerOptions; - protocolLayerOptions.Range = std::make_pair(offset, offset + content.Length() - 1); + protocolLayerOptions.Range = std::make_pair(offset, offset + content->Length() - 1); protocolLayerOptions.ContentMD5 = options.ContentMD5; protocolLayerOptions.ContentCRC64 = options.ContentCRC64; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::PageBlob::UploadPages( - options.Context, *m_pipeline, m_blobUrl.ToString(), content, protocolLayerOptions); + options.Context, *m_pipeline, m_blobUrl.ToString(), *content, protocolLayerOptions); } PageInfo PageBlobClient::UploadPagesFromUri( @@ -108,11 +108,11 @@ namespace Azure { namespace Storage { namespace Blobs { = std::make_pair(destinationoffset, destinationoffset + sourceLength - 1); protocolLayerOptions.ContentMD5 = options.ContentMD5; protocolLayerOptions.ContentCRC64 = options.ContentCRC64; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::PageBlob::UploadPagesFromUri( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } @@ -124,11 +124,11 @@ namespace Azure { namespace Storage { namespace Blobs { { BlobRestClient::PageBlob::ClearPagesOptions protocolLayerOptions; protocolLayerOptions.Range = std::make_pair(offset, offset + length - 1); - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::PageBlob::ClearPages( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } @@ -139,11 +139,11 @@ namespace Azure { namespace Storage { namespace Blobs { { BlobRestClient::PageBlob::ResizeOptions protocolLayerOptions; protocolLayerOptions.BlobContentLength = blobContentLength; - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::PageBlob::Resize( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } @@ -158,11 +158,11 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.Range = std::make_pair( options.Offset.GetValue(), options.Offset.GetValue() + options.Length.GetValue() - 1); } - protocolLayerOptions.LeaseId = options.Conditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; auto protocolLayerResponse = BlobRestClient::PageBlob::GetPageRanges( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); @@ -191,10 +191,10 @@ namespace Azure { namespace Storage { namespace Blobs { { BlobRestClient::PageBlob::CopyIncrementalOptions protocolLayerOptions; protocolLayerOptions.CopySource = sourceUri; - protocolLayerOptions.IfModifiedSince = options.Conditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.Conditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.Conditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.Conditions.IfNoneMatch; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; return BlobRestClient::PageBlob::CopyIncremental( options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions); } diff --git a/sdk/storage/src/common/shared_key_policy.cpp b/sdk/storage/src/common/shared_key_policy.cpp index 1e59f8516..f8a3530c9 100644 --- a/sdk/storage/src/common/shared_key_policy.cpp +++ b/sdk/storage/src/common/shared_key_policy.cpp @@ -4,7 +4,7 @@ #include "common/shared_key_policy.hpp" #include "common/crypt.hpp" -#include "common/storage_url_builder.hpp" +#include "common/storage_uri_builder.hpp" #include #include @@ -65,7 +65,7 @@ namespace Azure { namespace Storage { ordered_kv.clear(); // canonicalized resource - UrlBuilder resourceUrl(request.GetEncodedUrl()); + UriBuilder resourceUrl(request.GetEncodedUrl()); string_to_sign += "/" + m_credential->AccountName + "/" + resourceUrl.GetPath() + "\n"; for (const auto& query : resourceUrl.GetQuery()) { diff --git a/sdk/storage/src/common/storage_credential.cpp b/sdk/storage/src/common/storage_credential.cpp index b739020c9..3b86a7401 100644 --- a/sdk/storage/src/common/storage_credential.cpp +++ b/sdk/storage/src/common/storage_credential.cpp @@ -59,28 +59,28 @@ namespace Azure { namespace Storage { namespace Details { { endpoint = defaultEndpointsProtocol + "://" + accountName + ".blob." + EndpointSuffix; } - connectionStringParts.BlobServiceUri = UrlBuilder(std::move(endpoint)); + connectionStringParts.BlobServiceUri = UriBuilder(std::move(endpoint)); endpoint = getWithDefault(connectionStringMap, "DfsEndpoint"); if (endpoint.empty() && !accountName.empty()) { endpoint = defaultEndpointsProtocol + "://" + accountName + ".dfs." + EndpointSuffix; } - connectionStringParts.DataLakeServiceUri = UrlBuilder(std::move(endpoint)); + connectionStringParts.DataLakeServiceUri = UriBuilder(std::move(endpoint)); endpoint = getWithDefault(connectionStringMap, "FileEndpoint"); if (endpoint.empty() && !accountName.empty()) { endpoint = defaultEndpointsProtocol + "://" + accountName + ".file." + EndpointSuffix; } - connectionStringParts.FileServiceUri = UrlBuilder(std::move(endpoint)); + connectionStringParts.FileServiceUri = UriBuilder(std::move(endpoint)); endpoint = getWithDefault(connectionStringMap, "QueueEndpoint"); if (endpoint.empty() && !accountName.empty()) { endpoint = defaultEndpointsProtocol + "://" + accountName + ".queue." + EndpointSuffix; } - connectionStringParts.QueueServiceUri = UrlBuilder(std::move(endpoint)); + connectionStringParts.QueueServiceUri = UriBuilder(std::move(endpoint)); std::string accountKey = getWithDefault(connectionStringMap, "AccountKey"); if (!accountKey.empty()) diff --git a/sdk/storage/src/common/storage_url_builder.cpp b/sdk/storage/src/common/storage_uri_builder.cpp similarity index 91% rename from sdk/storage/src/common/storage_url_builder.cpp rename to sdk/storage/src/common/storage_uri_builder.cpp index c9dc8a85a..3487f243f 100644 --- a/sdk/storage/src/common/storage_url_builder.cpp +++ b/sdk/storage/src/common/storage_uri_builder.cpp @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // SPDX-License-Identifier: MIT -#include "common/storage_url_builder.hpp" +#include "common/storage_uri_builder.hpp" #include #include @@ -9,7 +9,7 @@ namespace Azure { namespace Storage { - UrlBuilder::UrlBuilder(const std::string& url) + UriBuilder::UriBuilder(const std::string& url) { std::string::const_iterator pos = url.begin(); @@ -61,12 +61,12 @@ namespace Azure { namespace Storage { = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~"; static const char* subdelimiters = "!$&'()*+,;="; - std::string UrlBuilder::EncodeHost(const std::string& host) + std::string UriBuilder::EncodeHost(const std::string& host) { return EncodeImpl(host, [](int c) { return c > 127; }); } - std::string UrlBuilder::EncodePath(const std::string& path) + std::string UriBuilder::EncodePath(const std::string& path) { const static std::vector shouldEncodeTable = []() { const std::string pathCharacters @@ -86,7 +86,7 @@ namespace Azure { namespace Storage { return EncodeImpl(path, [](int c) { return shouldEncodeTable[c]; }); } - std::string UrlBuilder::EncodeQuery(const std::string& query) + std::string UriBuilder::EncodeQuery(const std::string& query) { const static std::vector shouldEncodeTable = []() { std::string queryCharacters = std::string(unreserved) + std::string(subdelimiters) + "%/:@?"; @@ -109,7 +109,7 @@ namespace Azure { namespace Storage { return EncodeImpl(query, [](int c) { return shouldEncodeTable[c]; }); } - std::string UrlBuilder::EncodeFragment(const std::string& fragment) + std::string UriBuilder::EncodeFragment(const std::string& fragment) { const static std::vector shouldEncodeTable = []() { std::string queryCharacters = std::string(unreserved) + std::string(subdelimiters) + "%/:@?"; @@ -128,7 +128,7 @@ namespace Azure { namespace Storage { return EncodeImpl(fragment, [](int c) { return shouldEncodeTable[c]; }); } - std::string UrlBuilder::EncodeImpl( + std::string UriBuilder::EncodeImpl( const std::string& source, const std::function& shouldEncode) { @@ -152,7 +152,7 @@ namespace Azure { namespace Storage { return encoded; } - void UrlBuilder::SetQuery(const std::string& query) + void UriBuilder::SetQuery(const std::string& query) { m_query.clear(); @@ -185,7 +185,7 @@ namespace Azure { namespace Storage { } } - std::string UrlBuilder::ToString() const + std::string UriBuilder::ToString() const { std::string full_url; if (!m_scheme.empty()) diff --git a/sdk/storage/src/datalake/datalake_utilities.cpp b/sdk/storage/src/datalake/datalake_utilities.cpp index 355f239b2..28fcddb17 100644 --- a/sdk/storage/src/datalake/datalake_utilities.cpp +++ b/sdk/storage/src/datalake/datalake_utilities.cpp @@ -7,9 +7,9 @@ #include "datalake/protocol/datalake_rest_client.hpp" namespace Azure { namespace Storage { namespace DataLake { namespace Details { - UrlBuilder GetBlobUriFromDfsUri(const UrlBuilder& dfsUri) + UriBuilder GetBlobUriFromDfsUri(const UriBuilder& dfsUri) { - UrlBuilder result = dfsUri; + UriBuilder result = dfsUri; auto hoststr = result.GetHost(); auto pos = hoststr.find(".dfs."); if (pos != std::string::npos) diff --git a/sdk/storage/test/CMakeLists.txt b/sdk/storage/test/CMakeLists.txt index 6b1e0b764..4018ecc70 100644 --- a/sdk/storage/test/CMakeLists.txt +++ b/sdk/storage/test/CMakeLists.txt @@ -16,6 +16,7 @@ add_executable ( blobs/append_blob_client_test.cpp blobs/page_blob_client_test.hpp blobs/page_blob_client_test.cpp + blobs/performance_benchmark.cpp datalake/service_client_test.hpp datalake/service_client_test.cpp datalake/file_system_client_test.hpp diff --git a/sdk/storage/test/blobs/append_blob_client_test.cpp b/sdk/storage/test/blobs/append_blob_client_test.cpp index c38f9ad51..c679c0d9a 100644 --- a/sdk/storage/test/blobs/append_blob_client_test.cpp +++ b/sdk/storage/test/blobs/append_blob_client_test.cpp @@ -31,7 +31,7 @@ namespace Azure { namespace Storage { namespace Test { m_appendBlobClient->Create(m_blobUploadOptions); auto blockContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - m_appendBlobClient->AppendBlock(blockContent); + m_appendBlobClient->AppendBlock(&blockContent); m_blobUploadOptions.HttpHeaders.ContentMD5 = m_appendBlobClient->GetProperties().HttpHeaders.ContentMD5; } @@ -51,27 +51,27 @@ namespace Azure { namespace Storage { namespace Test { auto blockContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - appendBlobClient.AppendBlock(blockContent); + appendBlobClient.AppendBlock(&blockContent); properties = appendBlobClient.GetProperties(); EXPECT_EQ(properties.CommittedBlockCount.GetValue(), 1); EXPECT_EQ(properties.ContentLength, static_cast(m_blobContent.size())); Azure::Storage::Blobs::AppendBlockOptions options; - options.Conditions.AppendPosition = 1_MB; + options.AccessConditions.AppendPosition = 1_MB; blockContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - EXPECT_THROW(appendBlobClient.AppendBlock(blockContent, options), std::runtime_error); - options.Conditions.AppendPosition = properties.ContentLength; + EXPECT_THROW(appendBlobClient.AppendBlock(&blockContent, options), std::runtime_error); + options.AccessConditions.AppendPosition = properties.ContentLength; blockContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - appendBlobClient.AppendBlock(blockContent, options); + appendBlobClient.AppendBlock(&blockContent, options); properties = appendBlobClient.GetProperties(); options = Azure::Storage::Blobs::AppendBlockOptions(); - options.Conditions.MaxSize = properties.ContentLength + m_blobContent.size() - 1; + options.AccessConditions.MaxSize = properties.ContentLength + m_blobContent.size() - 1; blockContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - EXPECT_THROW(appendBlobClient.AppendBlock(blockContent, options), std::runtime_error); - options.Conditions.MaxSize = properties.ContentLength + m_blobContent.size(); + EXPECT_THROW(appendBlobClient.AppendBlock(&blockContent, options), std::runtime_error); + options.AccessConditions.MaxSize = properties.ContentLength + m_blobContent.size(); blockContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - appendBlobClient.AppendBlock(blockContent, options); + appendBlobClient.AppendBlock(&blockContent, options); // TODO: AppendBlockFromUri must be authorized with SAS, but we don't have SAS for now. diff --git a/sdk/storage/test/blobs/blob_container_client_test.cpp b/sdk/storage/test/blobs/blob_container_client_test.cpp index 80c4aeae8..ef887327f 100644 --- a/sdk/storage/test/blobs/blob_container_client_test.cpp +++ b/sdk/storage/test/blobs/blob_container_client_test.cpp @@ -86,7 +86,7 @@ namespace Azure { namespace Storage { namespace Test { std::string blobName = prefix1 + baseName + std::to_string(i); auto blobClient = m_blobContainerClient->GetBlockBlobClient(blobName); auto emptyContent = Azure::Core::Http::MemoryBodyStream(nullptr, 0); - blobClient.Upload(emptyContent); + blobClient.Upload(&emptyContent); p1Blobs.insert(blobName); p1p2Blobs.insert(blobName); } @@ -95,7 +95,7 @@ namespace Azure { namespace Storage { namespace Test { std::string blobName = prefix2 + baseName + std::to_string(i); auto blobClient = m_blobContainerClient->GetBlockBlobClient(blobName); auto emptyContent = Azure::Core::Http::MemoryBodyStream(nullptr, 0); - blobClient.Upload(emptyContent); + blobClient.Upload(&emptyContent); p2Blobs.insert(blobName); p1p2Blobs.insert(blobName); } @@ -156,7 +156,7 @@ namespace Azure { namespace Storage { namespace Test { std::string blobName = blobNamePrefix + delimiter + RandomString(); auto blobClient = m_blobContainerClient->GetBlockBlobClient(blobName); auto emptyContent = Azure::Core::Http::MemoryBodyStream(nullptr, 0); - blobClient.Upload(emptyContent); + blobClient.Upload(&emptyContent); blobs.insert(blobName); } } diff --git a/sdk/storage/test/blobs/block_blob_client_test.cpp b/sdk/storage/test/blobs/block_blob_client_test.cpp index f94ff7234..03e6574ef 100644 --- a/sdk/storage/test/blobs/block_blob_client_test.cpp +++ b/sdk/storage/test/blobs/block_blob_client_test.cpp @@ -51,7 +51,7 @@ namespace Azure { namespace Storage { namespace Test { m_blobUploadOptions.Tier = Azure::Storage::Blobs::AccessTier::Hot; auto blobContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - m_blockBlobClient->Upload(blobContent, m_blobUploadOptions); + m_blockBlobClient->Upload(&blobContent, m_blobUploadOptions); m_blobUploadOptions.HttpHeaders.ContentMD5 = m_blockBlobClient->GetProperties().HttpHeaders.ContentMD5; } @@ -64,7 +64,7 @@ namespace Azure { namespace Storage { namespace Test { StandardStorageConnectionString(), m_containerName, RandomString()); auto blobContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - blockBlobClient.Upload(blobContent, m_blobUploadOptions); + blockBlobClient.Upload(&blobContent, m_blobUploadOptions); blockBlobClient.Delete(); EXPECT_THROW(blockBlobClient.Delete(), std::runtime_error); @@ -105,7 +105,7 @@ namespace Azure { namespace Storage { namespace Test { StandardStorageConnectionString(), m_containerName, RandomString()); auto blobContent = Azure::Core::Http::MemoryBodyStream(emptyContent.data(), emptyContent.size()); - blockBlobClient.Upload(blobContent); + blockBlobClient.Upload(&blobContent); blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders); blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata); @@ -167,7 +167,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(ReadBodyStream(snapshotClient.Download().BodyStream), m_blobContent); EXPECT_EQ(snapshotClient.GetProperties().Metadata, m_blobUploadOptions.Metadata); auto emptyContent = Azure::Core::Http::MemoryBodyStream(nullptr, 0); - EXPECT_THROW(snapshotClient.Upload(emptyContent), std::runtime_error); + EXPECT_THROW(snapshotClient.Upload(&emptyContent), std::runtime_error); EXPECT_THROW(snapshotClient.SetMetadata({}), std::runtime_error); EXPECT_THROW( snapshotClient.SetAccessTier(Azure::Storage::Blobs::AccessTier::Cool), std::runtime_error); @@ -189,7 +189,7 @@ namespace Azure { namespace Storage { namespace Test { StandardStorageConnectionString(), m_containerName, RandomString()); auto blobContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - blockBlobClient.Upload(blobContent); + blockBlobClient.Upload(&blobContent); blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata); blockBlobClient.SetAccessTier(Azure::Storage::Blobs::AccessTier::Cool); blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders); @@ -219,7 +219,7 @@ namespace Azure { namespace Storage { namespace Test { RandomBuffer(reinterpret_cast(&block1Content[0]), block1Content.size()); auto blockContent = Azure::Core::Http::MemoryBodyStream(block1Content.data(), block1Content.size()); - blockBlobClient.StageBlock(blockId1, blockContent); + blockBlobClient.StageBlock(blockId1, &blockContent); Azure::Storage::Blobs::CommitBlockListOptions options; options.HttpHeaders = m_blobUploadOptions.HttpHeaders; options.Metadata = m_blobUploadOptions.Metadata; @@ -355,6 +355,7 @@ namespace Azure { namespace Storage { namespace Test { std::runtime_error); EXPECT_THROW( m_blockBlobClient->DownloadToFile(tempFilename, optionsCopy), std::runtime_error); + DeleteFile(tempFilename); } }; @@ -408,7 +409,7 @@ namespace Azure { namespace Storage { namespace Test { StandardStorageConnectionString(), m_containerName, RandomString()); auto blobContent = Azure::Core::Http::MemoryBodyStream(emptyContent.data(), emptyContent.size()); - blockBlobClient.Upload(blobContent); + blockBlobClient.Upload(&blobContent); blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders); blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata); @@ -500,6 +501,7 @@ namespace Azure { namespace Storage { namespace Test { emptyContent.data(), static_cast(8_MB), options), std::runtime_error); EXPECT_THROW(blockBlobClient.DownloadToFile(tempFilename, options), std::runtime_error); + DeleteFile(tempFilename); } } diff --git a/sdk/storage/test/blobs/page_blob_client_test.cpp b/sdk/storage/test/blobs/page_blob_client_test.cpp index 0b7cbfb12..f125af756 100644 --- a/sdk/storage/test/blobs/page_blob_client_test.cpp +++ b/sdk/storage/test/blobs/page_blob_client_test.cpp @@ -31,7 +31,7 @@ namespace Azure { namespace Storage { namespace Test { m_pageBlobClient->Create(m_blobContent.size(), m_blobUploadOptions); auto pageContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - m_pageBlobClient->UploadPages(pageContent, 0); + m_pageBlobClient->UploadPages(&pageContent, 0); m_blobUploadOptions.HttpHeaders.ContentMD5 = m_pageBlobClient->GetProperties().HttpHeaders.ContentMD5; } @@ -71,7 +71,7 @@ namespace Azure { namespace Storage { namespace Test { StandardStorageConnectionString(), m_containerName, RandomString()); pageBlobClient.Create(8_KB, m_blobUploadOptions); auto pageContent = Azure::Core::Http::MemoryBodyStream(blobContent.data(), blobContent.size()); - pageBlobClient.UploadPages(pageContent, 2_KB); + pageBlobClient.UploadPages(&pageContent, 2_KB); // |_|_|x|x| |x|x|_|_| blobContent.insert(blobContent.begin(), static_cast(2_KB), '\x00'); blobContent.resize(static_cast(8_KB), '\x00'); @@ -104,7 +104,7 @@ namespace Azure { namespace Storage { namespace Test { // |_|_|_|x| |x|x|_|_| This is what's in snapshot blobContent.resize(static_cast(1_KB)); auto pageClient = Azure::Core::Http::MemoryBodyStream(blobContent.data(), blobContent.size()); - pageBlobClient.UploadPages(pageClient, 0); + pageBlobClient.UploadPages(&pageClient, 0); pageBlobClient.ClearPages(3_KB, 1_KB); // |x|_|_|_| |x|x|_|_| diff --git a/sdk/storage/test/blobs/performance_benchmark.cpp b/sdk/storage/test/blobs/performance_benchmark.cpp new file mode 100644 index 000000000..5c076e561 --- /dev/null +++ b/sdk/storage/test/blobs/performance_benchmark.cpp @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "block_blob_client_test.hpp" + +#include + +namespace Azure { namespace Storage { namespace Test { + + TEST_F(BlockBlobClientTest, DISABLED_SingleThreadPerf) + { + auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( + StandardStorageConnectionString(), m_containerName, RandomString()); + + constexpr std::size_t bufferSize = static_cast(1_GB); + std::vector buffer = RandomBuffer(bufferSize); + { + auto timer_start = std::chrono::steady_clock::now(); + auto res = blockBlobClient.UploadFromBuffer(buffer.data(), buffer.size()); + auto timer_end = std::chrono::steady_clock::now(); + + double speed = static_cast(bufferSize) / 1_MB + / std::chrono::duration_cast(timer_end - timer_start).count() + * 1000; + std::cout << "Upload speed: " << speed << "MiB/s" << std::endl; + } + { + auto timer_start = std::chrono::steady_clock::now(); + auto res = blockBlobClient.DownloadToBuffer(buffer.data(), buffer.size()); + auto timer_end = std::chrono::steady_clock::now(); + + double speed = static_cast(bufferSize) / 1_MB + / std::chrono::duration_cast(timer_end - timer_start).count() + * 1000; + std::cout << "Download speed: " << speed << "MiB/s" << std::endl; + } + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/test/test_base.hpp b/sdk/storage/test/test_base.hpp index bf9010d44..b03086f5f 100644 --- a/sdk/storage/test/test_base.hpp +++ b/sdk/storage/test/test_base.hpp @@ -15,10 +15,16 @@ namespace Azure { namespace Storage { namespace Test { const std::string& PremiumFileConnectionString(); const std::string& ADLSGen2ConnectionString(); - inline unsigned long long operator""_KB(unsigned long long x) { return x * 1024; } - inline unsigned long long operator""_MB(unsigned long long x) { return x * 1024 * 1024; } - inline unsigned long long operator""_GB(unsigned long long x) { return x * 1024 * 1024 * 1024; } - inline unsigned long long operator""_TB(unsigned long long x) + constexpr inline unsigned long long operator""_KB(unsigned long long x) { return x * 1024; } + constexpr inline unsigned long long operator""_MB(unsigned long long x) + { + return x * 1024 * 1024; + } + constexpr inline unsigned long long operator""_GB(unsigned long long x) + { + return x * 1024 * 1024 * 1024; + } + constexpr inline unsigned long long operator""_TB(unsigned long long x) { return x * 1024 * 1024 * 1024 * 1024; }