From dc04b2410ebb354dc0bc5e266b270724891c678c Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Thu, 30 Jun 2022 18:07:01 +0800 Subject: [PATCH] Azure Storage Blob Batch (#3776) --- sdk/storage/azure-storage-blobs/CHANGELOG.md | 2 + .../azure-storage-blobs/CMakeLists.txt | 3 + .../inc/azure/storage/blobs.hpp | 1 + .../inc/azure/storage/blobs/blob_batch.hpp | 154 +++++ .../inc/azure/storage/blobs/blob_client.hpp | 1 + .../storage/blobs/blob_container_client.hpp | 37 +- .../inc/azure/storage/blobs/blob_options.hpp | 7 + .../azure/storage/blobs/blob_responses.hpp | 7 + .../storage/blobs/blob_service_client.hpp | 30 + .../azure/storage/blobs/deferred_response.hpp | 42 ++ .../azure-storage-blobs/src/blob_batch.cpp | 625 ++++++++++++++++++ .../src/blob_container_client.cpp | 59 +- .../src/blob_service_client.cpp | 69 +- .../test/ut/CMakeLists.txt | 1 + .../test/ut/blob_batch_client_test.cpp | 264 ++++++++ .../BlobBatchClientTest.exceptions.json | 144 ++++ ...BlobBatchClientTest.submitdeletebatch.json | 383 +++++++++++ ...lobBatchClientTest.tokenauthorization.json | 125 ++++ 18 files changed, 1932 insertions(+), 22 deletions(-) create mode 100644 sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_batch.hpp create mode 100644 sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/deferred_response.hpp create mode 100644 sdk/storage/azure-storage-blobs/src/blob_batch.cpp create mode 100644 sdk/storage/azure-storage-blobs/test/ut/blob_batch_client_test.cpp create mode 100644 sdk/storage/azure-storage-blobs/test/ut/recordings/BlobBatchClientTest.exceptions.json create mode 100644 sdk/storage/azure-storage-blobs/test/ut/recordings/BlobBatchClientTest.submitdeletebatch.json create mode 100644 sdk/storage/azure-storage-blobs/test/ut/recordings/BlobBatchClientTest.tokenauthorization.json diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index 26ef613bf..5ae31580a 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -4,6 +4,8 @@ ### Features Added +- Added support for Blob Batch. + ### Breaking Changes ### Bugs Fixed diff --git a/sdk/storage/azure-storage-blobs/CMakeLists.txt b/sdk/storage/azure-storage-blobs/CMakeLists.txt index 5da8d43ef..8ecec312e 100644 --- a/sdk/storage/azure-storage-blobs/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/CMakeLists.txt @@ -41,6 +41,7 @@ endif() set( AZURE_STORAGE_BLOBS_HEADER inc/azure/storage/blobs/append_blob_client.hpp + inc/azure/storage/blobs/blob_batch.hpp inc/azure/storage/blobs/blob_client.hpp inc/azure/storage/blobs/blob_container_client.hpp inc/azure/storage/blobs/blob_lease_client.hpp @@ -49,6 +50,7 @@ set( inc/azure/storage/blobs/blob_sas_builder.hpp inc/azure/storage/blobs/blob_service_client.hpp inc/azure/storage/blobs/block_blob_client.hpp + inc/azure/storage/blobs/deferred_response.hpp inc/azure/storage/blobs/dll_import_export.hpp inc/azure/storage/blobs/page_blob_client.hpp inc/azure/storage/blobs/rest_client.hpp @@ -61,6 +63,7 @@ set( set( AZURE_STORAGE_BLOBS_SOURCE src/append_blob_client.cpp + src/blob_batch.cpp src/blob_client.cpp src/blob_container_client.cpp src/blob_lease_client.cpp diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs.hpp index 03eb6fce2..98d75e56c 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs.hpp @@ -11,6 +11,7 @@ #include #include "azure/storage/blobs/append_blob_client.hpp" +#include "azure/storage/blobs/blob_batch.hpp" #include "azure/storage/blobs/blob_client.hpp" #include "azure/storage/blobs/blob_container_client.hpp" #include "azure/storage/blobs/blob_lease_client.hpp" diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_batch.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_batch.hpp new file mode 100644 index 000000000..09a24d642 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_batch.hpp @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +#include "azure/storage/blobs/blob_client.hpp" +#include "azure/storage/blobs/blob_container_client.hpp" +#include "azure/storage/blobs/blob_service_client.hpp" +#include "azure/storage/blobs/deferred_response.hpp" + +namespace Azure { namespace Storage { namespace Blobs { + + namespace _detail { + extern const Core::Context::Key s_batchKey; + + class StringBodyStream final : public Core::IO::BodyStream { + public: + explicit StringBodyStream(std::string content) : m_content(std::move(content)) {} + StringBodyStream(const StringBodyStream&) = delete; + StringBodyStream& operator=(const StringBodyStream&) = delete; + StringBodyStream(StringBodyStream&& other) = default; + StringBodyStream& operator=(StringBodyStream&& other) = default; + ~StringBodyStream() override {} + int64_t Length() const override { return m_content.length(); } + void Rewind() override { m_offset = 0; } + + private: + size_t OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) override; + + private: + std::string m_content; + size_t m_offset = 0; + }; + + enum class BatchSubrequestType + { + DeleteBlob, + SetBlobAccessTier, + }; + + struct BatchSubrequest + { + explicit BatchSubrequest(BatchSubrequestType type) : Type(type) {} + virtual ~BatchSubrequest() = 0; + + BatchSubrequestType Type; + }; + + struct BlobBatchAccessHelper; + + std::shared_ptr ConstructBatchRequestPolicy( + const std::vector>& + servicePerRetryPolicies, + const std::vector>& + servicePerOperationPolicies, + const BlobClientOptions& options); + + std::shared_ptr ConstructBatchSubrequestPolicy( + std::unique_ptr&& tokenAuthPolicy, + std::unique_ptr&& sharedKeyAuthPolicy, + const BlobClientOptions& options); + } // namespace _detail + + /** + * @brief A batch object allows you to batch multiple operations in a single request via + * #Azure::Storage::Blobs::BlobServiceClient::SubmitBatch or + * #Azure::Storage::Blobs::BlobContainerClient::SubmitBatch. + */ + class BlobBatch final { + public: + /** + * @brief Adds a delete subrequest into batch object. + * + * @param blobContainerName Container name of the blob to delete. + * @param blobName Blob name of the blob to delete. + * @param options Optional parameters to execute the delete operation. + * @return A deferred response which can produce a Response after batch object + * is submitted. + */ + DeferredResponse DeleteBlob( + const std::string& blobContainerName, + const std::string& blobName, + const DeleteBlobOptions& options = DeleteBlobOptions()); + + /** + * @brief Adds a delete subrequest into batch object. + * + * @param blobUrl Url of the blob to delete. + * @param options Optional parameters to execute the delete operation. + * @return A deferred response which can produce a Response after batch object + * is submitted. + */ + DeferredResponse DeleteBlob( + const std::string& blobUrl, + const DeleteBlobOptions& options = DeleteBlobOptions()); + + /** + * @brief Adds a change tier subrequest into batch object. + * + * @param blobContainerName Container name of the blob to delete. + * @param blobName Blob name of the blob to delete. + * @param accessTier Indicates the tier to be set on the blob. + * @param options Optional parameters to execute the delete operation. + * @return A deferred response which can produce a Response after batch + * object is submitted. + */ + DeferredResponse SetBlobAccessTier( + const std::string& blobContainerName, + const std::string& blobName, + Models::AccessTier accessTier, + const SetBlobAccessTierOptions& options = SetBlobAccessTierOptions()); + + /** + * @brief Adds a change tier subrequest into batch object. + * + * @param blobUrl Url of the blob to delete. + * @param accessTier Indicates the tier to be set on the blob. + * @param options Optional parameters to execute the delete operation. + * @return A deferred response which can produce a Response after batch + * object is submitted. + */ + DeferredResponse SetBlobAccessTier( + const std::string& blobUrl, + Models::AccessTier accessTier, + const SetBlobAccessTierOptions& options = SetBlobAccessTierOptions()); + + private: + explicit BlobBatch(BlobServiceClient blobServiceClient); + explicit BlobBatch(BlobContainerClient blobContainerClient); + + BlobClient GetBlobClientForSubrequest(Core::Url url) const; + + private: + Core::Url m_url; + Nullable m_blobServiceClient; + Nullable m_blobContainerClient; + + std::vector> m_subrequests; + + friend class BlobServiceClient; + friend class BlobContainerClient; + friend struct _detail::BlobBatchAccessHelper; + }; + +}}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp index 89451b5a8..b3034f3f7 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp @@ -431,5 +431,6 @@ namespace Azure { namespace Storage { namespace Blobs { friend class Files::DataLake::DataLakeDirectoryClient; friend class Files::DataLake::DataLakeFileClient; friend class BlobLeaseClient; + friend class BlobBatch; }; }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp index d9ec5e276..acd4e485d 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp @@ -12,6 +12,7 @@ namespace Azure { namespace Storage { namespace Blobs { class BlobLeaseClient; + class BlobBatch; /** * The BlobContainerClient allows you to manipulate Azure Storage containers and their @@ -284,25 +285,41 @@ namespace Azure { namespace Storage { namespace Blobs { const UploadBlockBlobOptions& options = UploadBlockBlobOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; + /** + * @brief Creates a new batch object to collect subrequests that can be submitted together via + * SubmitBatch. + * + * @return A new batch object. + */ + BlobBatch CreateBatch(); + + /** + * @brief Submits a batch of subrequests. + * + * @param batch The batch object containing subrequests. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A SubmitBlobBatchResult. + * @remark This function will throw only if there's something wrong with the batch request + * (parent request). + */ + Response SubmitBatch( + const BlobBatch& batch, + const SubmitBlobBatchOptions& options = SubmitBlobBatchOptions(), + const Core::Context& context = Core::Context()) const; + private: Azure::Core::Url m_blobContainerUrl; std::shared_ptr m_pipeline; Azure::Nullable m_customerProvidedKey; Azure::Nullable m_encryptionScope; - explicit BlobContainerClient( - Azure::Core::Url blobContainerUrl, - std::shared_ptr pipeline, - Azure::Nullable customerProvidedKey, - Azure::Nullable encryptionScope) - : m_blobContainerUrl(std::move(blobContainerUrl)), m_pipeline(std::move(pipeline)), - m_customerProvidedKey(std::move(customerProvidedKey)), - m_encryptionScope(std::move(encryptionScope)) - { - } + std::shared_ptr m_batchRequestPipeline; + std::shared_ptr m_batchSubrequestPipeline; friend class BlobServiceClient; friend class BlobLeaseClient; + friend class BlobBatch; }; }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index 25dc5ae5a..912cee90e 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -1484,6 +1484,13 @@ namespace Azure { namespace Storage { namespace Blobs { { }; + /** + * @brief Optional parameters for #Azure::Storage::Blobs::BlobBatchClient::SubmitBatch. + */ + struct SubmitBlobBatchOptions final + { + }; + namespace _detail { inline std::string TagsToString(const std::map& tags) { diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp index edcb2f92c..573265ca4 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp @@ -163,6 +163,13 @@ namespace Azure { namespace Storage { std::string LeaseId; }; + /** + * @brief Response type for #Azure::Storage::Blobs::BlobBatchClient::SubmitBatch. + */ + struct SubmitBlobBatchResult final + { + }; + } // namespace Models /** diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_service_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_service_client.hpp index a1f6134e9..dc677d31f 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_service_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_service_client.hpp @@ -13,6 +13,8 @@ namespace Azure { namespace Storage { namespace Blobs { + class BlobBatch; + /** * The BlobServiceClient allows you to manipulate Azure Storage service resources and blob * containers. The storage account provides the top-level namespace for the Blob service. @@ -243,10 +245,38 @@ namespace Azure { namespace Storage { namespace Blobs { const RenameBlobContainerOptions& options = RenameBlobContainerOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; + /** + * @brief Creates a new batch object to collect subrequests that can be submitted together via + * SubmitBatch. + * + * @return A new batch object. + */ + BlobBatch CreateBatch(); + + /** + * @brief Submits a batch of subrequests. + * + * @param batch The batch object containing subrequests. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A SubmitBlobBatchResult. + * @remark This function will throw only if there's something wrong with the batch request + * (parent request). + */ + Response SubmitBatch( + const BlobBatch& batch, + const SubmitBlobBatchOptions& options = SubmitBlobBatchOptions(), + const Core::Context& context = Core::Context()) const; + private: Azure::Core::Url m_serviceUrl; std::shared_ptr m_pipeline; Azure::Nullable m_customerProvidedKey; Azure::Nullable m_encryptionScope; + + std::shared_ptr m_batchRequestPipeline; + std::shared_ptr m_batchSubrequestPipeline; + + friend class BlobBatch; }; }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/deferred_response.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/deferred_response.hpp new file mode 100644 index 000000000..ba6e4f63a --- /dev/null +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/deferred_response.hpp @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +#include + +namespace Azure { namespace Storage { + namespace Blobs { + class BlobBatch; + } + /** + * @brief Base type for a deferred response. + */ + template class DeferredResponse final { + public: + DeferredResponse(const DeferredResponse&) = delete; + DeferredResponse(DeferredResponse&&) = default; + DeferredResponse& operator=(const DeferredResponse&) = delete; + DeferredResponse& operator=(DeferredResponse&&) = default; + + /** + * @brief Gets the deferred response. + * + * @remark It's undefined behavior to call this function before the response or exception is + * available. + * + * @return The deferred response. An exception is thrown if error occurred. + */ + Response GetResponse() const { return m_func(); } + + private: + DeferredResponse(std::function()> func) : m_func(std::move(func)) {} + + private: + std::function()> m_func; + + friend class Blobs::BlobBatch; + }; +}} // namespace Azure::Storage diff --git a/sdk/storage/azure-storage-blobs/src/blob_batch.cpp b/sdk/storage/azure-storage-blobs/src/blob_batch.cpp new file mode 100644 index 000000000..3bcee5b8c --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/blob_batch.cpp @@ -0,0 +1,625 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "azure/storage/blobs/blob_batch.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "private/package_version.hpp" + +namespace Azure { namespace Storage { namespace Blobs { + + const Core::Context::Key _detail::s_batchKey; + + namespace _detail { + + struct BlobBatchAccessHelper + { + explicit BlobBatchAccessHelper(const BlobBatch& batch) : m_batch(&batch) {} + + const std::vector>& Subrequests() const + { + return m_batch->m_subrequests; + } + const BlobBatch* m_batch; + }; + + } // namespace _detail + + namespace { + const std::string LineEnding = "\r\n"; + const std::string BatchContentTypePrefix = "multipart/mixed; boundary="; + + static Core::Context::Key s_subrequestKey; + static Core::Context::Key s_subresponseKey; + + struct Parser final + { + explicit Parser(const std::string& str) + : startPos(str.data()), currPos(startPos), endPos(startPos + str.length()) + { + } + explicit Parser(const std::vector& str) + : startPos(reinterpret_cast(str.data())), + currPos(reinterpret_cast(startPos)), + endPos(reinterpret_cast(startPos) + str.size()) + { + } + const char* startPos; + const char* currPos; + const char* endPos; + + bool IsEnd() const { return currPos == endPos; } + + bool LookAhead(const std::string& expect) const + { + for (size_t i = 0; i < expect.length(); ++i) + { + if (currPos + i < endPos && currPos[i] == expect[i]) + { + continue; + } + return false; + } + return true; + } + + void Consume(const std::string& expect) + { + // This moves currPos + if (LookAhead(expect)) + { + currPos += expect.length(); + } + else + { + throw std::runtime_error( + "failed to parse response body at " + std::to_string(currPos - startPos)); + } + } + + const char* FindNext(const std::string& expect) const + { + return std::search(currPos, endPos, expect.begin(), expect.end()); + } + + const char* AfterNext(const std::string& expect) const + { + return std::min(endPos, FindNext(expect) + expect.length()); + } + + std::string GetBeforeNextAndConsume(const std::string& expect) + { + // This moves currPos + auto ePos = FindNext(expect); + std::string ret(currPos, ePos); + currPos = std::min(endPos, ePos + expect.length()); + return ret; + } + }; + + std::unique_ptr ParseRawResponse(const std::string& responseText) + { + Parser parser(responseText); + + parser.Consume("HTTP/"); + int32_t httpMajorVersion = std::stoi(parser.GetBeforeNextAndConsume(".")); + int32_t httpMinorVersion = std::stoi(parser.GetBeforeNextAndConsume(" ")); + int32_t httpStatusCode = std::stoi(parser.GetBeforeNextAndConsume(" ")); + const std::string httpReasonPhrase = parser.GetBeforeNextAndConsume(LineEnding); + + auto rawResponse = std::make_unique( + httpMajorVersion, + httpMinorVersion, + static_cast(httpStatusCode), + httpReasonPhrase); + + while (!parser.IsEnd()) + { + if (parser.LookAhead(LineEnding)) + { + break; + } + std::string headerName = parser.GetBeforeNextAndConsume(": "); + std::string headerValue = parser.GetBeforeNextAndConsume(LineEnding); + rawResponse->SetHeader(headerName, headerValue); + } + parser.Consume(LineEnding); + rawResponse->SetBody(std::vector(parser.currPos, parser.endPos)); + + return rawResponse; + } + + class RemoveXMsVersionPolicy final : public Core::Http::Policies::HttpPolicy { + public: + ~RemoveXMsVersionPolicy() override {} + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + std::unique_ptr Send( + Core::Http::Request& request, + Core::Http::Policies::NextHttpPolicy nextPolicy, + const Core::Context& context) const override + { + request.RemoveHeader(_internal::HttpHeaderXMsVersion); + return nextPolicy.Send(request, context); + } + }; + + class NoopTransportPolicy final : public Core::Http::Policies::HttpPolicy { + public: + ~NoopTransportPolicy() override {} + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + std::unique_ptr Send( + Core::Http::Request& request, + Core::Http::Policies::NextHttpPolicy nextPolicy, + const Core::Context& context) const override + { + (void)nextPolicy; + + std::string* subrequestText = nullptr; + context.TryGetValue(s_subrequestKey, subrequestText); + + if (subrequestText) + { + std::string requestText = request.GetMethod().ToString() + " /" + + request.GetUrl().GetRelativeUrl() + " HTTP/1.1" + LineEnding; + for (const auto& header : request.GetHeaders()) + { + requestText += header.first + ": " + header.second + LineEnding; + } + requestText += LineEnding; + *subrequestText = std::move(requestText); + + auto rawResponse = std::make_unique( + 1, 1, Core::Http::HttpStatusCode::Accepted, "Accepted"); + return rawResponse; + } + + std::string* subresponseText = nullptr; + context.TryGetValue(s_subresponseKey, subresponseText); + if (subresponseText) + { + return ParseRawResponse(*subresponseText); + } + AZURE_UNREACHABLE_CODE(); + } + }; + + class ConstructBatchRequestBodyPolicy final : public Core::Http::Policies::HttpPolicy { + public: + ConstructBatchRequestBodyPolicy( + std::function constructRequestFunction, + std::function&, const Core::Context&)> + parseResponseFunction) + : m_constructRequestFunction(std::move(constructRequestFunction)), + m_parseResponseFunction(std::move(parseResponseFunction)) + { + } + ~ConstructBatchRequestBodyPolicy() override {} + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + std::unique_ptr Send( + Core::Http::Request& request, + Core::Http::Policies::NextHttpPolicy nextPolicy, + const Core::Context& context) const override + { + m_constructRequestFunction(request, context); + auto rawResponse = nextPolicy.Send(request, context); + m_parseResponseFunction(rawResponse, context); + return rawResponse; + } + + private: + std::function m_constructRequestFunction; + std::function&, const Core::Context&)> + m_parseResponseFunction; + }; + + template + std::function()> CreateDeferredResponseFunc( + std::promise>>& promise) + { + return [&promise]() { + try + { + auto f = promise.get_future(); + AZURE_ASSERT_MSG( + f.wait_for(std::chrono::seconds(0)) == std::future_status::ready, + "GetResponse() is called when response is not ready."); + return f.get().Value(); + } + catch (std::future_error&) + { + AZURE_ASSERT_MSG(false, "GetResponse() can only be called once."); + } + AZURE_UNREACHABLE_CODE(); + }; + } + + struct DeleteBlobSubrequest final : public _detail::BatchSubrequest + { + DeleteBlobSubrequest() : BatchSubrequest(_detail::BatchSubrequestType::DeleteBlob) {} + + Nullable Client; + DeleteBlobOptions Options; + std::promise>> Promise; + }; + + struct SetBlobAccessTierSubrequest final : public _detail::BatchSubrequest + { + SetBlobAccessTierSubrequest() + : BatchSubrequest(_detail::BatchSubrequestType::SetBlobAccessTier) + { + } + + Nullable Client; + Models::AccessTier Tier; + SetBlobAccessTierOptions Options; + std::promise>> Promise; + }; + + void ConstructSubrequests(Core::Http::Request& request, const Core::Context& context) + { + const std::string boundary = "batch_" + Azure::Core::Uuid::CreateUuid().ToString(); + + auto getBatchBoundary = [&boundary, subRequestCounter = 0]() mutable { + std::string ret; + ret += "--" + boundary + LineEnding; + ret += "Content-Type: application/http" + LineEnding + "Content-Transfer-Encoding: binary" + + LineEnding + "Content-ID: " + std::to_string(subRequestCounter++) + LineEnding + + LineEnding; + return ret; + }; + + std::string requestBody; + + const BlobBatch* batch = nullptr; + context.TryGetValue(_detail::s_batchKey, batch); + + for (const auto& subrequestPtr : _detail::BlobBatchAccessHelper(*batch).Subrequests()) + { + if (subrequestPtr->Type == _detail::BatchSubrequestType::DeleteBlob) + { + auto& subrequest = *static_cast(subrequestPtr.get()); + requestBody += getBatchBoundary(); + std::string subrequestText; + subrequest.Client.Value().Delete( + subrequest.Options, Core::Context().WithValue(s_subrequestKey, &subrequestText)); + requestBody += subrequestText; + } + else if (subrequestPtr->Type == _detail::BatchSubrequestType::SetBlobAccessTier) + { + auto& subrequest = *static_cast(subrequestPtr.get()); + requestBody += getBatchBoundary(); + + std::string subrequestText; + subrequest.Client.Value().SetAccessTier( + subrequest.Tier, + subrequest.Options, + Core::Context().WithValue(s_subrequestKey, &subrequestText)); + requestBody += subrequestText; + } + else + { + AZURE_UNREACHABLE_CODE(); + } + } + requestBody += "--" + boundary + "--" + LineEnding; + + request.SetHeader(_internal::HttpHeaderContentType, BatchContentTypePrefix + boundary); + static_cast<_detail::StringBodyStream&>(*request.GetBodyStream()) + = _detail::StringBodyStream(std::move(requestBody)); + request.SetHeader( + _internal::HttpHeaderContentLength, std::to_string(request.GetBodyStream()->Length())); + } + + void ParseSubresponses( + std::unique_ptr& rawResponse, + const Core::Context& context) + { + if (rawResponse->GetStatusCode() != Core::Http::HttpStatusCode::Accepted + || rawResponse->GetHeaders().count(_internal::HttpHeaderContentType) == 0) + { + return; + } + + const std::string boundary = rawResponse->GetHeaders() + .at(std::string(_internal::HttpHeaderContentType)) + .substr(BatchContentTypePrefix.length()); + + const std::vector& responseBody + = rawResponse->ExtractBodyStream()->ReadToEnd(context); + Parser parser(responseBody); + + std::vector subresponses; + while (true) + { + parser.Consume("--" + boundary); + if (parser.LookAhead("--")) + { + parser.Consume("--"); + } + if (parser.IsEnd()) + { + break; + } + auto contentIdPos = parser.AfterNext("Content-ID: "); + auto responseStartPos = parser.AfterNext(LineEnding + LineEnding); + auto responseEndPos = parser.FindNext("--" + boundary); + if (contentIdPos != parser.endPos) + { + parser.currPos = contentIdPos; + auto idEndPos = parser.FindNext(LineEnding); + size_t id = static_cast(std::stoi(std::string(parser.currPos, idEndPos))); + if (subresponses.size() < id + 1) + { + subresponses.resize(id + 1); + } + subresponses[id] = std::string(responseStartPos, responseEndPos); + parser.currPos = responseEndPos; + } + else + { + rawResponse = ParseRawResponse(std::string(responseStartPos, responseEndPos)); + parser.currPos = responseEndPos; + return; + } + } + + const BlobBatch* batch = nullptr; + context.TryGetValue(_detail::s_batchKey, batch); + + size_t subresponseCounter = 0; + for (const auto& subrequestPtr : _detail::BlobBatchAccessHelper(*batch).Subrequests()) + { + if (subrequestPtr->Type == _detail::BatchSubrequestType::DeleteBlob) + { + auto& subrequest = *static_cast(subrequestPtr.get()); + try + { + auto response = subrequest.Client.Value().Delete( + subrequest.Options, + Core::Context().WithValue(s_subresponseKey, &subresponses[subresponseCounter++])); + subrequest.Promise.set_value(std::move(response)); + } + catch (...) + { + subrequest.Promise.set_exception(std::current_exception()); + } + } + else if (subrequestPtr->Type == _detail::BatchSubrequestType::SetBlobAccessTier) + { + auto& subrequest = *static_cast(subrequestPtr.get()); + try + { + auto response = subrequest.Client.Value().SetAccessTier( + subrequest.Tier, + subrequest.Options, + Core::Context().WithValue(s_subresponseKey, &subresponses[subresponseCounter++])); + subrequest.Promise.set_value(std::move(response)); + } + catch (...) + { + subrequest.Promise.set_exception(std::current_exception()); + } + } + else + { + AZURE_UNREACHABLE_CODE(); + } + } + } + } // namespace + + namespace _detail { + + size_t StringBodyStream::OnRead( + uint8_t* buffer, + size_t count, + Azure::Core::Context const& context) + { + (void)context; + size_t copy_length = std::min(count, m_content.length() - m_offset); + std::memcpy(buffer, &m_content[0] + m_offset, static_cast(copy_length)); + m_offset += copy_length; + return copy_length; + } + + BatchSubrequest::~BatchSubrequest() {} + + std::shared_ptr ConstructBatchRequestPolicy( + const std::vector>& + servicePerRetryPolicies, + const std::vector>& + servicePerOperationPolicies, + const BlobClientOptions& options) + { + std::vector> perRetryPolicies; + perRetryPolicies.push_back(std::make_unique( + [](Core::Http::Request& request, const Core::Context& context) { + ConstructSubrequests(request, context); + }, + [](std::unique_ptr& rawResponse, const Core::Context& context) { + ParseSubresponses(rawResponse, context); + })); + for (auto& policy : servicePerRetryPolicies) + { + perRetryPolicies.push_back(policy->Clone()); + } + std::vector> perOperationPolicies; + for (auto& policy : servicePerOperationPolicies) + { + perOperationPolicies.push_back(policy->Clone()); + } + return std::make_shared( + options, + _internal::BlobServicePackageName, + PackageVersion::ToString(), + std::move(perRetryPolicies), + std::move(perOperationPolicies)); + } + + std::shared_ptr ConstructBatchSubrequestPolicy( + std::unique_ptr&& tokenAuthPolicy, + std::unique_ptr&& sharedKeyAuthPolicy, + const BlobClientOptions& options) + { + std::vector> policies; + policies.emplace_back( + std::make_unique()); + policies.emplace_back( + std::make_unique( + _internal::BlobServicePackageName, PackageVersion::ToString(), options.Telemetry)); + for (auto& policy : options.PerOperationPolicies) + { + policies.emplace_back(policy->Clone()); + } + policies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>()); + if (tokenAuthPolicy) + { + policies.emplace_back(std::move(tokenAuthPolicy)); + } + for (auto& policy : options.PerRetryPolicies) + { + policies.emplace_back(policy->Clone()); + } + policies.emplace_back(std::make_unique()); + if (sharedKeyAuthPolicy) + { + policies.emplace_back(std::move(sharedKeyAuthPolicy)); + } + policies.push_back(std::make_unique()); + return std::make_shared(std::move(policies)); + } + } // namespace _detail + + BlobBatch::BlobBatch(BlobServiceClient blobServiceClient) + : m_blobServiceClient(std::move(blobServiceClient)) + { + m_url = m_blobServiceClient.Value().m_serviceUrl; + } + + BlobBatch::BlobBatch(BlobContainerClient blobContainerClient) + : m_blobContainerClient(std::move(blobContainerClient)) + { + auto serviceUrl = m_blobContainerClient.Value().m_blobContainerUrl; + auto path = serviceUrl.GetPath(); + if (!path.empty() && path.back() == '/') + { + path.pop_back(); + } + auto slash_pos = path.rfind('/'); + path = path.substr(0, slash_pos == std::string::npos ? 0 : (slash_pos + 1)); + serviceUrl.SetPath(path); + m_url = std::move(serviceUrl); + } + + BlobClient BlobBatch::GetBlobClientForSubrequest(Core::Url url) const + { + if (m_blobServiceClient.HasValue()) + { + auto blobClient = m_blobServiceClient.Value().GetBlobContainerClient("$").GetBlobClient("$"); + blobClient.m_blobUrl = std::move(url); + blobClient.m_pipeline = m_blobServiceClient.Value().m_batchSubrequestPipeline; + return blobClient; + } + else if (m_blobContainerClient.HasValue()) + { + auto blobClient = m_blobContainerClient->GetBlobClient("$"); + blobClient.m_blobUrl = std::move(url); + blobClient.m_pipeline = m_blobContainerClient.Value().m_batchSubrequestPipeline; + return blobClient; + } + AZURE_UNREACHABLE_CODE(); + } + + DeferredResponse BlobBatch::DeleteBlob( + const std::string& blobContainerName, + const std::string& blobName, + const DeleteBlobOptions& options) + { + auto blobUrl = m_url; + blobUrl.AppendPath(blobContainerName); + blobUrl.AppendPath(blobName); + auto op = std::make_shared(); + op->Client = GetBlobClientForSubrequest(std::move(blobUrl)); + op->Options = options; + DeferredResponse deferredResponse( + CreateDeferredResponseFunc(op->Promise)); + m_subrequests.push_back(std::move(op)); + return deferredResponse; + } + + DeferredResponse BlobBatch::DeleteBlob( + const std::string& blobUrl, + const DeleteBlobOptions& options) + { + auto op = std::make_shared(); + op->Client = GetBlobClientForSubrequest(Core::Url(blobUrl)); + op->Options = options; + DeferredResponse deferredResponse( + CreateDeferredResponseFunc(op->Promise)); + m_subrequests.push_back(std::move(op)); + return deferredResponse; + } + + DeferredResponse BlobBatch::SetBlobAccessTier( + const std::string& blobContainerName, + const std::string& blobName, + Models::AccessTier accessTier, + const SetBlobAccessTierOptions& options) + { + auto blobUrl = m_url; + blobUrl.AppendPath(blobContainerName); + blobUrl.AppendPath(blobName); + auto op = std::make_shared(); + op->Client = GetBlobClientForSubrequest(std::move(blobUrl)); + op->Tier = accessTier; + op->Options = options; + DeferredResponse deferredResponse( + CreateDeferredResponseFunc(op->Promise)); + m_subrequests.push_back(std::move(op)); + return deferredResponse; + } + + DeferredResponse BlobBatch::SetBlobAccessTier( + const std::string& blobUrl, + Models::AccessTier accessTier, + const SetBlobAccessTierOptions& options) + { + auto op = std::make_shared(); + op->Client = GetBlobClientForSubrequest(Core::Url(blobUrl)); + op->Tier = accessTier; + op->Options = options; + DeferredResponse deferredResponse( + CreateDeferredResponseFunc(op->Promise)); + m_subrequests.push_back(std::move(op)); + return deferredResponse; + } + +}}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp index d934f023f..d1b4bfac8 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp @@ -14,6 +14,7 @@ #include #include "azure/storage/blobs/append_blob_client.hpp" +#include "azure/storage/blobs/blob_batch.hpp" #include "azure/storage/blobs/block_blob_client.hpp" #include "azure/storage/blobs/page_blob_client.hpp" @@ -48,8 +49,8 @@ namespace Azure { namespace Storage { namespace Blobs { : BlobContainerClient(blobContainerUrl, options) { BlobClientOptions newOptions = options; - newOptions.PerRetryPolicies.emplace_back( - std::make_unique<_internal::SharedKeyPolicy>(credential)); + auto sharedKeyAuthPolicy = std::make_unique<_internal::SharedKeyPolicy>(credential); + newOptions.PerRetryPolicies.emplace_back(sharedKeyAuthPolicy->Clone()); std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -58,6 +59,13 @@ namespace Azure { namespace Storage { namespace Blobs { perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>()); perOperationPolicies.emplace_back( std::make_unique<_internal::StorageServiceVersionPolicy>(newOptions.ApiVersion)); + + m_batchRequestPipeline + = _detail::ConstructBatchRequestPolicy(perRetryPolicies, perOperationPolicies, newOptions); + + m_batchSubrequestPipeline + = _detail::ConstructBatchSubrequestPolicy(nullptr, std::move(sharedKeyAuthPolicy), options); + m_pipeline = std::make_shared( newOptions, _internal::BlobServicePackageName, @@ -77,15 +85,24 @@ namespace Azure { namespace Storage { namespace Blobs { perRetryPolicies.emplace_back(std::make_unique<_internal::StorageSwitchToSecondaryPolicy>( m_blobContainerUrl.GetHost(), options.SecondaryHostForRetryReads)); perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>()); + std::unique_ptr tokenAuthPolicy; { Azure::Core::Credentials::TokenRequestContext tokenContext; tokenContext.Scopes.emplace_back(_internal::StorageScope); - perRetryPolicies.emplace_back( - std::make_unique( - credential, tokenContext)); + tokenAuthPolicy = std::make_unique< + Azure::Core::Http::Policies::_internal::BearerTokenAuthenticationPolicy>( + credential, tokenContext); + perRetryPolicies.emplace_back(tokenAuthPolicy->Clone()); } perOperationPolicies.emplace_back( std::make_unique<_internal::StorageServiceVersionPolicy>(options.ApiVersion)); + + m_batchRequestPipeline + = _detail::ConstructBatchRequestPolicy(perRetryPolicies, perOperationPolicies, options); + + m_batchSubrequestPipeline + = _detail::ConstructBatchSubrequestPolicy(std::move(tokenAuthPolicy), nullptr, options); + m_pipeline = std::make_shared( options, _internal::BlobServicePackageName, @@ -107,6 +124,12 @@ namespace Azure { namespace Storage { namespace Blobs { perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>()); perOperationPolicies.emplace_back( std::make_unique<_internal::StorageServiceVersionPolicy>(options.ApiVersion)); + + m_batchRequestPipeline + = _detail::ConstructBatchRequestPolicy(perRetryPolicies, perOperationPolicies, options); + + m_batchSubrequestPipeline = _detail::ConstructBatchSubrequestPolicy(nullptr, nullptr, options); + m_pipeline = std::make_shared( options, _internal::BlobServicePackageName, @@ -444,4 +467,30 @@ namespace Azure { namespace Storage { namespace Blobs { std::move(blockBlobClient), std::move(response.RawResponse)); } + BlobBatch BlobContainerClient::CreateBatch() { return BlobBatch(*this); } + + Response BlobContainerClient::SubmitBatch( + const BlobBatch& batch, + const SubmitBlobBatchOptions& options, + const Core::Context& context) const + { + (void)options; + + if (!batch.m_blobContainerClient.HasValue()) + { + throw std::runtime_error("Batch is not container-scoped."); + } + + _detail::BlobContainerClient::SubmitBlobContainerBatchOptions protocolLayerOptions; + _detail::StringBodyStream bodyStream(std::string{}); + auto response = _detail::BlobContainerClient::SubmitBatch( + *m_batchRequestPipeline, + m_blobContainerUrl, + bodyStream, + protocolLayerOptions, + context.WithValue(_detail::s_batchKey, &batch)); + return Azure::Response( + Models::SubmitBlobBatchResult(), std::move(response.RawResponse)); + } + }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp index df0cdbe70..d4b580061 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp @@ -12,6 +12,7 @@ #include #include +#include "azure/storage/blobs/blob_batch.hpp" #include "private/package_version.hpp" namespace Azure { namespace Storage { namespace Blobs { @@ -41,8 +42,8 @@ namespace Azure { namespace Storage { namespace Blobs { : BlobServiceClient(serviceUrl, options) { BlobClientOptions newOptions = options; - newOptions.PerRetryPolicies.emplace_back( - std::make_unique<_internal::SharedKeyPolicy>(credential)); + auto sharedKeyPolicy = std::make_unique<_internal::SharedKeyPolicy>(credential); + newOptions.PerRetryPolicies.emplace_back(sharedKeyPolicy->Clone()); std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -51,6 +52,13 @@ namespace Azure { namespace Storage { namespace Blobs { perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>()); perOperationPolicies.emplace_back( std::make_unique<_internal::StorageServiceVersionPolicy>(newOptions.ApiVersion)); + + m_batchRequestPipeline + = _detail::ConstructBatchRequestPolicy(perRetryPolicies, perOperationPolicies, newOptions); + + m_batchSubrequestPipeline + = _detail::ConstructBatchSubrequestPolicy(nullptr, std::move(sharedKeyPolicy), options); + m_pipeline = std::make_shared( newOptions, _internal::BlobServicePackageName, @@ -70,15 +78,24 @@ namespace Azure { namespace Storage { namespace Blobs { perRetryPolicies.emplace_back(std::make_unique<_internal::StorageSwitchToSecondaryPolicy>( m_serviceUrl.GetHost(), options.SecondaryHostForRetryReads)); perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>()); + std::unique_ptr tokenAuthPolicy; { Azure::Core::Credentials::TokenRequestContext tokenContext; tokenContext.Scopes.emplace_back(_internal::StorageScope); - perRetryPolicies.emplace_back( - std::make_unique( - credential, tokenContext)); + tokenAuthPolicy = std::make_unique< + Azure::Core::Http::Policies::_internal::BearerTokenAuthenticationPolicy>( + credential, tokenContext); + perRetryPolicies.emplace_back(tokenAuthPolicy->Clone()); } perOperationPolicies.emplace_back( std::make_unique<_internal::StorageServiceVersionPolicy>(options.ApiVersion)); + + m_batchRequestPipeline + = _detail::ConstructBatchRequestPolicy(perRetryPolicies, perOperationPolicies, options); + + m_batchSubrequestPipeline + = _detail::ConstructBatchSubrequestPolicy(std::move(tokenAuthPolicy), nullptr, options); + m_pipeline = std::make_shared( options, _internal::BlobServicePackageName, @@ -100,6 +117,12 @@ namespace Azure { namespace Storage { namespace Blobs { perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>()); perOperationPolicies.emplace_back( std::make_unique<_internal::StorageServiceVersionPolicy>(options.ApiVersion)); + + m_batchRequestPipeline + = _detail::ConstructBatchRequestPolicy(perRetryPolicies, perOperationPolicies, options); + + m_batchSubrequestPipeline = _detail::ConstructBatchSubrequestPolicy(nullptr, nullptr, options); + m_pipeline = std::make_shared( options, _internal::BlobServicePackageName, @@ -113,8 +136,14 @@ namespace Azure { namespace Storage { namespace Blobs { { auto blobContainerUrl = m_serviceUrl; blobContainerUrl.AppendPath(_internal::UrlEncodePath(blobContainerName)); - return BlobContainerClient( - std::move(blobContainerUrl), m_pipeline, m_customerProvidedKey, m_encryptionScope); + + BlobContainerClient blobContainerClient(blobContainerUrl.GetAbsoluteUrl()); + blobContainerClient.m_pipeline = m_pipeline; + blobContainerClient.m_customerProvidedKey = m_customerProvidedKey; + blobContainerClient.m_encryptionScope = m_encryptionScope; + blobContainerClient.m_batchRequestPipeline = m_batchRequestPipeline; + blobContainerClient.m_batchSubrequestPipeline = m_batchSubrequestPipeline; + return blobContainerClient; } ListBlobContainersPagedResponse BlobServiceClient::ListBlobContainers( @@ -281,4 +310,30 @@ namespace Azure { namespace Storage { namespace Blobs { std::move(blobContainerClient), std::move(response.RawResponse)); } + BlobBatch BlobServiceClient::CreateBatch() { return BlobBatch(*this); } + + Response BlobServiceClient::SubmitBatch( + const BlobBatch& batch, + const SubmitBlobBatchOptions& options, + const Core::Context& context) const + { + (void)options; + + if (!batch.m_blobServiceClient.HasValue()) + { + throw std::runtime_error("Batch is container-scoped."); + } + + _detail::ServiceClient::SubmitServiceBatchOptions protocolLayerOptions; + _detail::StringBodyStream bodyStream(std::string{}); + auto response = _detail::ServiceClient::SubmitBatch( + *m_batchRequestPipeline, + m_serviceUrl, + bodyStream, + protocolLayerOptions, + context.WithValue(_detail::s_batchKey, &batch)); + return Azure::Response( + Models::SubmitBlobBatchResult(), std::move(response.RawResponse)); + } + }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt b/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt index 8b5ca8b38..15c0dd0c3 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt @@ -16,6 +16,7 @@ add_executable ( azure-storage-blobs-test append_blob_client_test.cpp append_blob_client_test.hpp + blob_batch_client_test.cpp blob_container_client_test.cpp blob_container_client_test.hpp blob_query_test.cpp diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_batch_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_batch_client_test.cpp new file mode 100644 index 000000000..afd561ac4 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_batch_client_test.cpp @@ -0,0 +1,264 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include + +#include "test/ut/test_base.hpp" + +namespace Azure { namespace Storage { namespace Test { + + class BlobBatchClientTest : public StorageTest { + private: + std::unique_ptr m_client; + + protected: + // Required to rename the test propertly once the test is started. + // We can only know the test instance name until the test instance is run. + Azure::Storage::Blobs::BlobServiceClient const& GetClientForTest(std::string const& testName) + { + // set the interceptor for the current test + m_testContext.RenameTest(testName); + return *m_client; + } + + void SetUp() override + { + StorageTest::SetUp(); + + auto options = InitClientOptions(); + m_client = std::make_unique( + Azure::Storage::Blobs::BlobServiceClient::CreateFromConnectionString( + StandardStorageConnectionString(), options)); + } + }; + + TEST_F(BlobBatchClientTest, SubmitDeleteBatch) + { + const std::string testName = GetTestNameLowerCase(); + + const std::string containerName1 = testName + "1"; + const std::string blob1Name = "b1"; + const std::string blob2Name = "b2"; + const std::string containerName2 = testName + "2"; + const std::string blob3Name = "b3"; + + auto serviceClient = GetClientForTest(testName); + auto container1Client = serviceClient.GetBlobContainerClient(containerName1); + container1Client.CreateIfNotExists(); + auto container2Client = serviceClient.GetBlobContainerClient(containerName2); + container2Client.CreateIfNotExists(); + auto blob1Client = container1Client.GetAppendBlobClient(blob1Name); + blob1Client.Create(); + auto blob2Client = container1Client.GetAppendBlobClient(blob2Name); + blob2Client.Create(); + auto blob3Client = container2Client.GetAppendBlobClient(blob3Name); + blob3Client.Create(); + blob3Client.CreateSnapshot(); + + auto batch = serviceClient.CreateBatch(); + auto delete1Response = batch.DeleteBlob(blob1Client.GetUrl()); + auto delete2Response = batch.DeleteBlob(containerName1, blob2Name); + Blobs::DeleteBlobOptions deleteOptions; + deleteOptions.DeleteSnapshots = Blobs::Models::DeleteSnapshotsOption::OnlySnapshots; + auto delete3Response = batch.DeleteBlob(blob3Client.GetUrl(), deleteOptions); + auto submitBatchResponse = serviceClient.SubmitBatch(batch); + + EXPECT_TRUE(delete1Response.GetResponse().Value.Deleted); + EXPECT_TRUE(delete2Response.GetResponse().Value.Deleted); + EXPECT_TRUE(delete3Response.GetResponse().Value.Deleted); + EXPECT_THROW(blob1Client.GetProperties(), StorageException); + EXPECT_THROW(blob2Client.GetProperties(), StorageException); + EXPECT_NO_THROW(blob3Client.GetProperties()); + + container1Client.Delete(); + container2Client.Delete(); + } + + TEST_F(BlobBatchClientTest, SubmitSetTierBatch_LIVEONLY_) + { + const std::string testName = GetTestNameLowerCase(); + + const std::string containerName = testName; + const std::string blob1Name = "b1"; + const std::string blob2Name = "b2"; + + auto containerSasToken = [&]() { + Sas::BlobSasBuilder sasBuilder; + sasBuilder.Protocol = Sas::SasProtocol::HttpsAndHttp; + sasBuilder.ExpiresOn = std::chrono::system_clock::now() + std::chrono::minutes(5); + sasBuilder.BlobContainerName = containerName; + sasBuilder.Resource = Sas::BlobSasResource::BlobContainer; + sasBuilder.SetPermissions(Sas::BlobContainerSasPermissions::All); + return sasBuilder.GenerateSasToken( + *_internal::ParseConnectionString(StandardStorageConnectionString()).KeyCredential); + }(); + + auto serviceClient = GetClientForTest(testName); + serviceClient.GetBlobContainerClient(containerName).CreateIfNotExists(); + auto containerClient = Blobs::BlobContainerClient( + serviceClient.GetBlobContainerClient(containerName).GetUrl() + containerSasToken, + InitClientOptions()); + auto blob1Client = containerClient.GetBlockBlobClient(blob1Name); + blob1Client.UploadFrom(nullptr, 0); + auto blob2Client = containerClient.GetBlockBlobClient(blob2Name); + blob2Client.UploadFrom(nullptr, 0); + + auto batch = containerClient.CreateBatch(); + auto setTier1Response + = batch.SetBlobAccessTier(containerName, blob1Name, Blobs::Models::AccessTier::Cool); + auto setTier2Response + = batch.SetBlobAccessTier(blob2Client.GetUrl(), Blobs::Models::AccessTier::Archive); + auto submitBatchResponse = containerClient.SubmitBatch(batch); + + EXPECT_NO_THROW(setTier1Response.GetResponse()); + EXPECT_NO_THROW(setTier2Response.GetResponse()); + EXPECT_EQ( + blob1Client.GetProperties().Value.AccessTier.Value(), Blobs::Models::AccessTier::Cool); + EXPECT_EQ( + blob2Client.GetProperties().Value.AccessTier.Value(), Blobs::Models::AccessTier::Archive); + + serviceClient.DeleteBlobContainer(containerName); + } + + TEST_F(BlobBatchClientTest, TokenAuthorization) + { + const std::string testName = GetTestNameLowerCase(); + + std::shared_ptr credential + = std::make_shared( + AadTenantId(), AadClientId(), AadClientSecret()); + Blobs::BlobClientOptions options; + + auto serviceClient = InitTestClient( + GetClientForTest(testName).GetUrl(), credential, options); + + const std::string containerName = testName; + const std::string blobName = "b1"; + + auto containerClient = serviceClient->GetBlobContainerClient(containerName); + containerClient.CreateIfNotExists(); + auto blobClient = containerClient.GetAppendBlobClient(blobName); + blobClient.Create(); + + auto batch = containerClient.CreateBatch(); + auto delete1Response = batch.DeleteBlob(blobClient.GetUrl()); + auto submitBatchResponse = containerClient.SubmitBatch(batch); + + EXPECT_TRUE(delete1Response.GetResponse().Value.Deleted); + + containerClient.Delete(); + } + + TEST_F(BlobBatchClientTest, Exceptions_LIVEONLY_) + { + const std::string testName = GetTestNameLowerCase(); + + const std::string containerName = testName; + const std::string blobName = "b1"; + + auto serviceClient = GetClientForTest(testName); + auto containerClient = serviceClient.GetBlobContainerClient(containerName); + containerClient.CreateIfNotExists(); + auto blobClient = containerClient.GetBlockBlobClient(blobName); + blobClient.UploadFrom(nullptr, 0); + + // Empty batch + auto batch = containerClient.CreateBatch(); + + try + { + containerClient.SubmitBatch(batch); + FAIL(); + } + catch (StorageException& e) + { + EXPECT_EQ(e.StatusCode, Azure::Core::Http::HttpStatusCode::BadRequest); + EXPECT_FALSE(e.ReasonPhrase.empty()); + EXPECT_FALSE(e.RequestId.empty()); + EXPECT_FALSE(e.ClientRequestId.empty()); + EXPECT_EQ(e.ErrorCode, "InvalidInput"); + } + catch (...) + { + FAIL(); + } + + // Partial failure + { + auto r1 = batch.SetBlobAccessTier(blobClient.GetUrl(), Blobs::Models::AccessTier::Hot); + auto r2 = batch.SetBlobAccessTier( + containerName, "BlobNameNotExists", Blobs::Models::AccessTier::Hot); + EXPECT_NO_THROW(containerClient.SubmitBatch(batch)); + EXPECT_NO_THROW(r1.GetResponse()); + EXPECT_THROW(r2.GetResponse(), StorageException); + } + + // Mixed operations + auto batch2 = containerClient.CreateBatch(); + batch2.SetBlobAccessTier(blobClient.GetUrl(), Blobs::Models::AccessTier::Cool); + batch2.DeleteBlob(blobClient.GetUrl()); + + try + { + containerClient.SubmitBatch(batch2); + FAIL(); + } + catch (StorageException& e) + { + EXPECT_EQ(e.StatusCode, Azure::Core::Http::HttpStatusCode::BadRequest); + EXPECT_FALSE(e.ReasonPhrase.empty()); + EXPECT_FALSE(e.RequestId.empty()); + EXPECT_FALSE(e.ClientRequestId.empty()); + EXPECT_EQ(e.ErrorCode, "AllBatchSubRequestsShouldBeSameApi"); + } + catch (...) + { + FAIL(); + } + + auto containerExpiredSasToken = [&]() { + Sas::BlobSasBuilder sasBuilder; + sasBuilder.Protocol = Sas::SasProtocol::HttpsAndHttp; + sasBuilder.ExpiresOn = std::chrono::system_clock::now() - std::chrono::minutes(5); + sasBuilder.BlobContainerName = containerName; + sasBuilder.Resource = Sas::BlobSasResource::BlobContainer; + sasBuilder.SetPermissions(Sas::BlobContainerSasPermissions::All); + return sasBuilder.GenerateSasToken( + *_internal::ParseConnectionString(StandardStorageConnectionString()).KeyCredential); + }(); + auto containerSasToken = [&]() { + Sas::BlobSasBuilder sasBuilder; + sasBuilder.Protocol = Sas::SasProtocol::HttpsAndHttp; + sasBuilder.ExpiresOn = std::chrono::system_clock::now() + std::chrono::minutes(5); + sasBuilder.BlobContainerName = containerName; + sasBuilder.Resource = Sas::BlobSasResource::BlobContainer; + sasBuilder.SetPermissions(Sas::BlobContainerSasPermissions::All); + return sasBuilder.GenerateSasToken( + *_internal::ParseConnectionString(StandardStorageConnectionString()).KeyCredential); + }(); + auto containerSasClient = Blobs::BlobContainerClient( + serviceClient.GetBlobContainerClient(containerName).GetUrl() + containerExpiredSasToken); + auto batch3 = containerSasClient.CreateBatch(); + batch3.DeleteBlob(blobClient.GetUrl() + containerSasToken); + try + { + containerSasClient.SubmitBatch(batch3); + FAIL(); + } + catch (StorageException& e) + { + EXPECT_EQ(e.StatusCode, Azure::Core::Http::HttpStatusCode::Forbidden); + EXPECT_FALSE(e.ReasonPhrase.empty()); + EXPECT_FALSE(e.RequestId.empty()); + EXPECT_FALSE(e.ClientRequestId.empty()); + EXPECT_EQ(e.ErrorCode, "AuthenticationFailed"); + } + catch (...) + { + FAIL(); + } + + containerClient.Delete(); + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobBatchClientTest.exceptions.json b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobBatchClientTest.exceptions.json new file mode 100644 index 000000000..baf719a51 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobBatchClientTest.exceptions.json @@ -0,0 +1,144 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "826b7081-9e8a-4486-4dbd-754bafda9877", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 28 Jun 2022 08:59:58 GMT", + "etag": "\"0x8DA58E4948E254B\"", + "last-modified": "Tue, 28 Jun 2022 08:59:59 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "826b7081-9e8a-4486-4dbd-754bafda9877", + "x-ms-request-id": "4f35247c-201e-009a-5dcd-8a441b000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/exceptions?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "805b8546-ca84-4de3-5cad-e9a4bf1d70ad", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "content-md5": "1B2M2Y8AsgTpgAmY7PhCfg==", + "date": "Tue, 28 Jun 2022 08:59:58 GMT", + "etag": "\"0x8DA58E494C38EF9\"", + "last-modified": "Tue, 28 Jun 2022 08:59:59 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "805b8546-ca84-4de3-5cad-e9a4bf1d70ad", + "x-ms-content-crc64": "AAAAAAAAAAA=", + "x-ms-request-id": "4f3524b7-201e-009a-10cd-8a441b000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02", + "x-ms-version-id": "2022-06-28T08:59:59.7978361Z" + }, + "Url": "https://REDACTED.blob.core.windows.net/exceptions/b1" + }, + { + "Headers": { + "content-type": "multipart/mixed; boundary=batch_ffe1c850-d4aa-4dd1-411c-120c5c215be2", + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "7391bd0f-94eb-4b07-5e9f-a3c79a7ae138", + "x-ms-version": "2020-10-02" + }, + "Method": "POST", + "Response": { + "BODY": "--batchresponse_706f8051-2324-4b39-8185-62332e77e61c\r\nContent-Type: application/http\r\n\r\nHTTP/1.1 400 One of the request inputs is not valid.\r\nx-ms-error-code: InvalidInput\r\nx-ms-request-id: 4f352503-201e-009a-56cd-8a441b000000\r\nx-ms-version: 2020-10-02\r\nx-ms-client-request-id: 7391bd0f-94eb-4b07-5e9f-a3c79a7ae138\r\nContent-Length: 221\r\nContent-Type: application/xml\r\nServer: Windows-Azure-Blob/1.0\r\n\r\n\nInvalidInputOne of the request inputs is not valid.\nRequestId:4f352503-201e-009a-56cd-8a441b000000\nTime:2022-06-28T09:00:00.2423323Z\r\n--batchresponse_706f8051-2324-4b39-8185-62332e77e61c--", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-type": "multipart/mixed; boundary=batchresponse_706f8051-2324-4b39-8185-62332e77e61c", + "date": "Tue, 28 Jun 2022 08:59:59 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "7391bd0f-94eb-4b07-5e9f-a3c79a7ae138", + "x-ms-request-id": "4f352503-201e-009a-56cd-8a441b000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/exceptions?comp=batch&restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "e8577d2f-290e-450c-5780-c0b616d93dd6", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202" + }, + "Url": "https://REDACTED.blob.core.windows.net/exceptions/b1?comp=tier" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "aeacc093-2a2c-4650-7acf-71b99a83fee9", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202" + }, + "Url": "https://REDACTED.blob.core.windows.net/exceptions/b1" + }, + { + "Headers": { + "content-type": "multipart/mixed; boundary=batch_44147b73-099b-48bf-593d-d5da18fc33a6", + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "0888f4dd-0214-4a2b-4cce-58018c2e88ea", + "x-ms-version": "2020-10-02" + }, + "Method": "POST", + "Response": { + "BODY": "--batchresponse_fab35cab-6d23-4c95-8dd1-edc59aafd881\r\nContent-Type: application/http\r\n\r\nHTTP/1.1 400 All batch subrequests should be the same api.\r\nx-ms-error-code: AllBatchSubRequestsShouldBeSameApi\r\nx-ms-request-id: 4f3525f8-201e-009a-28cd-8a441b000000\r\nx-ms-version: 2020-10-02\r\nx-ms-client-request-id: 0888f4dd-0214-4a2b-4cce-58018c2e88ea\r\nContent-Length: 249\r\nContent-Type: application/xml\r\nServer: Windows-Azure-Blob/1.0\r\n\r\n\nAllBatchSubRequestsShouldBeSameApiAll batch subrequests should be the same api.\nRequestId:4f3525f8-201e-009a-28cd-8a441b000000\nTime:2022-06-28T09:00:01.1408151Z\r\n--batchresponse_fab35cab-6d23-4c95-8dd1-edc59aafd881--", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-type": "multipart/mixed; boundary=batchresponse_fab35cab-6d23-4c95-8dd1-edc59aafd881", + "date": "Tue, 28 Jun 2022 09:00:00 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "0888f4dd-0214-4a2b-4cce-58018c2e88ea", + "x-ms-request-id": "4f3525f8-201e-009a-28cd-8a441b000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/exceptions?comp=batch&restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "f7d9ca6e-1362-4478-56c6-4588683fafa6", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Tue, 28 Jun 2022 09:00:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "f7d9ca6e-1362-4478-56c6-4588683fafa6", + "x-ms-request-id": "a432dc6f-d01e-0055-0fcd-8aca49000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/exceptions?restype=container" + } + ] +} diff --git a/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobBatchClientTest.submitdeletebatch.json b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobBatchClientTest.submitdeletebatch.json new file mode 100644 index 000000000..e49480e6c --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobBatchClientTest.submitdeletebatch.json @@ -0,0 +1,383 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "c4d555c2-46b5-4589-4208-ac374c2d53db", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 28 Jun 2022 08:59:41 GMT", + "etag": "\"0x8DA58E48A27F380\"", + "last-modified": "Tue, 28 Jun 2022 08:59:42 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "c4d555c2-46b5-4589-4208-ac374c2d53db", + "x-ms-request-id": "1f791515-601e-008b-5ecd-8adeaf000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch1?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "ec0e9d4b-555b-435e-70d0-e4cd7e234b5e", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 28 Jun 2022 08:59:41 GMT", + "etag": "\"0x8DA58E48A5B09E0\"", + "last-modified": "Tue, 28 Jun 2022 08:59:42 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "ec0e9d4b-555b-435e-70d0-e4cd7e234b5e", + "x-ms-request-id": "1f7915c0-601e-008b-72cd-8adeaf000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch2?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "b22b2bb1-6f86-410c-66df-0e12f55b1be6", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 28 Jun 2022 08:59:42 GMT", + "etag": "\"0x8DA58E48A9467D5\"", + "last-modified": "Tue, 28 Jun 2022 08:59:42 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "b22b2bb1-6f86-410c-66df-0e12f55b1be6", + "x-ms-request-id": "1f79163d-601e-008b-5ecd-8adeaf000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02", + "x-ms-version-id": "2022-06-28T08:59:42.7115989Z" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch1/b1" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "70f62543-8fac-4d16-42aa-47fa382ee321", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 28 Jun 2022 08:59:42 GMT", + "etag": "\"0x8DA58E48AC50DA1\"", + "last-modified": "Tue, 28 Jun 2022 08:59:43 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "70f62543-8fac-4d16-42aa-47fa382ee321", + "x-ms-request-id": "1f7916c7-601e-008b-57cd-8adeaf000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02", + "x-ms-version-id": "2022-06-28T08:59:43.0304161Z" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch1/b2" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "1cfcb7ff-8964-44db-7c21-19604e60a773", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 28 Jun 2022 08:59:42 GMT", + "etag": "\"0x8DA58E48AF760E5\"", + "last-modified": "Tue, 28 Jun 2022 08:59:43 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "1cfcb7ff-8964-44db-7c21-19604e60a773", + "x-ms-request-id": "1f791770-601e-008b-67cd-8adeaf000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02", + "x-ms-version-id": "2022-06-28T08:59:43.3612275Z" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch2/b3" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "4a05a52d-36df-4b3b-799a-b911c57f363a", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 28 Jun 2022 08:59:43 GMT", + "etag": "\"0x8DA58E48AF760E5\"", + "last-modified": "Tue, 28 Jun 2022 08:59:43 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "4a05a52d-36df-4b3b-799a-b911c57f363a", + "x-ms-request-id": "1f791826-601e-008b-11cd-8adeaf000000", + "x-ms-request-server-encrypted": "false", + "x-ms-snapshot": "2022-06-28T08:59:43.6820453Z", + "x-ms-version": "2020-10-02", + "x-ms-version-id": "2022-06-28T08:59:43.6830453Z" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch2/b3?comp=snapshot" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "66705a78-b4b8-4785-5a18-2403a38abd57", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch1/b1" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "89852895-6f69-4528-74f6-dddc86a96f88", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch1/b2" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "1eaf9c79-0de1-40dc-781e-cc3c3cf5552f", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch2/b3" + }, + { + "Headers": { + "content-type": "multipart/mixed; boundary=batch_d3d5b5bc-7b2f-452a-7089-d737922908bd", + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "8d3bdb6a-9b67-48e2-4d4f-e00b3ab2853b", + "x-ms-version": "2020-10-02" + }, + "Method": "POST", + "Response": { + "BODY": "--batchresponse_3b60398a-cb61-4c44-8ddb-c3b37829cbb1\r\nContent-Type: application/http\r\nContent-ID: 0\r\n\r\nHTTP/1.1 202 Accepted\r\nx-ms-delete-type-permanent: true\r\nx-ms-request-id: 1f7918a1-601e-008b-73cd-8adeaf1e293f\r\nx-ms-version: 2020-10-02\r\nx-ms-client-request-id: 66705a78-b4b8-4785-5a18-2403a38abd57\r\nServer: Windows-Azure-Blob/1.0\r\n\r\n--batchresponse_3b60398a-cb61-4c44-8ddb-c3b37829cbb1\r\nContent-Type: application/http\r\nContent-ID: 1\r\n\r\nHTTP/1.1 202 Accepted\r\nx-ms-delete-type-permanent: true\r\nx-ms-request-id: 1f7918a1-601e-008b-73cd-8adeaf1e2941\r\nx-ms-version: 2020-10-02\r\nx-ms-client-request-id: 89852895-6f69-4528-74f6-dddc86a96f88\r\nServer: Windows-Azure-Blob/1.0\r\n\r\n--batchresponse_3b60398a-cb61-4c44-8ddb-c3b37829cbb1\r\nContent-Type: application/http\r\nContent-ID: 2\r\n\r\nHTTP/1.1 202 Accepted\r\nx-ms-delete-type-permanent: true\r\nx-ms-request-id: 1f7918a1-601e-008b-73cd-8adeaf1e2942\r\nx-ms-version: 2020-10-02\r\nx-ms-client-request-id: 1eaf9c79-0de1-40dc-781e-cc3c3cf5552f\r\nServer: Windows-Azure-Blob/1.0\r\n\r\n--batchresponse_3b60398a-cb61-4c44-8ddb-c3b37829cbb1--", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-type": "multipart/mixed; boundary=batchresponse_3b60398a-cb61-4c44-8ddb-c3b37829cbb1", + "date": "Tue, 28 Jun 2022 08:59:43 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "8d3bdb6a-9b67-48e2-4d4f-e00b3ab2853b", + "x-ms-request-id": "1f7918a1-601e-008b-73cd-8adeaf000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net?comp=batch" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "f95d874a-41d1-4f33-60f2-c4504cf88b47", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "Server": "Windows-Azure-Blob/1.0", + "x-ms-client-request-id": "66705a78-b4b8-4785-5a18-2403a38abd57", + "x-ms-delete-type-permanent": "true", + "x-ms-request-id": "1f7918a1-601e-008b-73cd-8adeaf1e293f", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch1/b1" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "faa2b291-3ca7-4212-47b8-7c3ff1b057b9", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "Server": "Windows-Azure-Blob/1.0", + "x-ms-client-request-id": "89852895-6f69-4528-74f6-dddc86a96f88", + "x-ms-delete-type-permanent": "true", + "x-ms-request-id": "1f7918a1-601e-008b-73cd-8adeaf1e2941", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch1/b2" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "f4a2acba-2d48-4b75-5d90-d48bbbddf5cd", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "Server": "Windows-Azure-Blob/1.0", + "x-ms-client-request-id": "1eaf9c79-0de1-40dc-781e-cc3c3cf5552f", + "x-ms-delete-type-permanent": "true", + "x-ms-request-id": "1f7918a1-601e-008b-73cd-8adeaf1e2942", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch2/b3" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "6faf1d4d-d581-4919-6822-8202ee5d71cc", + "x-ms-version": "2020-10-02" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "The specified blob does not exist.", + "STATUS_CODE": "404", + "date": "Tue, 28 Jun 2022 08:59:44 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-client-request-id": "6faf1d4d-d581-4919-6822-8202ee5d71cc", + "x-ms-error-code": "BlobNotFound", + "x-ms-request-id": "1f7919bf-601e-008b-69cd-8adeaf000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch1/b1" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "b6b68bfc-20fe-4738-6d4c-deb09f681a69", + "x-ms-version": "2020-10-02" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "The specified blob does not exist.", + "STATUS_CODE": "404", + "date": "Tue, 28 Jun 2022 08:59:44 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-client-request-id": "b6b68bfc-20fe-4738-6d4c-deb09f681a69", + "x-ms-error-code": "BlobNotFound", + "x-ms-request-id": "1f791a2a-601e-008b-49cd-8adeaf000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch1/b2" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "6ad0e0bb-2072-4dc4-5f7d-e16fa12967c2", + "x-ms-version": "2020-10-02" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Tue, 28 Jun 2022 08:59:44 GMT", + "etag": "\"0x8DA58E48AF760E5\"", + "last-modified": "Tue, 28 Jun 2022 08:59:43 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "vary": "Origin", + "x-ms-blob-committed-block-count": "0", + "x-ms-blob-type": "AppendBlob", + "x-ms-client-request-id": "6ad0e0bb-2072-4dc4-5f7d-e16fa12967c2", + "x-ms-creation-time": "Tue, 28 Jun 2022 08:59:43 GMT", + "x-ms-is-current-version": "true", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-request-id": "1f791a9e-601e-008b-32cd-8adeaf000000", + "x-ms-server-encrypted": "true", + "x-ms-version": "2020-10-02", + "x-ms-version-id": "2022-06-28T08:59:43.6830453Z" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch2/b3" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "c9f892df-380a-4c35-7cc0-2fac5cf42322", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Tue, 28 Jun 2022 08:59:45 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "c9f892df-380a-4c35-7cc0-2fac5cf42322", + "x-ms-request-id": "1f791b04-601e-008b-07cd-8adeaf000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch1?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "26d494dc-1836-4876-6054-4fee745bfe84", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Tue, 28 Jun 2022 08:59:45 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "26d494dc-1836-4876-6054-4fee745bfe84", + "x-ms-request-id": "1f791b6c-601e-008b-5ecd-8adeaf000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/submitdeletebatch2?restype=container" + } + ] +} diff --git a/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobBatchClientTest.tokenauthorization.json b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobBatchClientTest.tokenauthorization.json new file mode 100644 index 000000000..7535b161d --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobBatchClientTest.tokenauthorization.json @@ -0,0 +1,125 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "75b45834-afba-4e35-407f-f6a08f365fd4", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 28 Jun 2022 08:59:54 GMT", + "etag": "\"0x8DA58E491EF7D15\"", + "last-modified": "Tue, 28 Jun 2022 08:59:55 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "75b45834-afba-4e35-407f-f6a08f365fd4", + "x-ms-request-id": "3ae4ef42-d01e-0018-7bcd-8a05a5000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/tokenauthorization?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "0cdd77c6-688c-45df-69cd-7a578b609496", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 28 Jun 2022 08:59:55 GMT", + "etag": "\"0x8DA58E4921FACD3\"", + "last-modified": "Tue, 28 Jun 2022 08:59:55 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "0cdd77c6-688c-45df-69cd-7a578b609496", + "x-ms-request-id": "3ae4efb9-d01e-0018-5ecd-8a05a5000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02", + "x-ms-version-id": "2022-06-28T08:59:55.3683667Z" + }, + "Url": "https://REDACTED.blob.core.windows.net/tokenauthorization/b1" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "030920c6-6578-4c1a-6f2c-de11eaf8814c", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202" + }, + "Url": "https://REDACTED.blob.core.windows.net/tokenauthorization/b1" + }, + { + "Headers": { + "content-type": "multipart/mixed; boundary=batch_635fcca6-0ac8-447f-7677-e48bd5f17210", + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "93311fa2-726f-43a0-72fa-a59527bae4e3", + "x-ms-version": "2020-10-02" + }, + "Method": "POST", + "Response": { + "BODY": "--batchresponse_4fa0dc98-76e2-4af3-a696-c3f8b59d5778\r\nContent-Type: application/http\r\nContent-ID: 0\r\n\r\nHTTP/1.1 202 Accepted\r\nx-ms-delete-type-permanent: true\r\nx-ms-request-id: 3ae4f1f2-d01e-0018-51cd-8a05a51e08dc\r\nx-ms-version: 2020-10-02\r\nx-ms-client-request-id: 030920c6-6578-4c1a-6f2c-de11eaf8814c\r\nServer: Windows-Azure-Blob/1.0\r\n\r\n--batchresponse_4fa0dc98-76e2-4af3-a696-c3f8b59d5778--", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-type": "multipart/mixed; boundary=batchresponse_4fa0dc98-76e2-4af3-a696-c3f8b59d5778", + "date": "Tue, 28 Jun 2022 08:59:57 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "93311fa2-726f-43a0-72fa-a59527bae4e3", + "x-ms-request-id": "3ae4f1f2-d01e-0018-51cd-8a05a5000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/tokenauthorization?comp=batch&restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "67e38e19-f43e-4408-59bc-cddf6a734d50", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "Server": "Windows-Azure-Blob/1.0", + "x-ms-client-request-id": "030920c6-6578-4c1a-6f2c-de11eaf8814c", + "x-ms-delete-type-permanent": "true", + "x-ms-request-id": "3ae4f1f2-d01e-0018-51cd-8a05a51e08dc", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/tokenauthorization/b1" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.2 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "13964705-9bc8-4a32-46f2-69b38ff0aac5", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Tue, 28 Jun 2022 08:59:57 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "13964705-9bc8-4a32-46f2-69b38ff0aac5", + "x-ms-request-id": "3ae4f3d7-d01e-0018-7ecd-8a05a5000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/tokenauthorization?restype=container" + } + ] +}