From 1dce74c642267dffce32e39f91a1b815bcaa426a Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Tue, 18 Aug 2020 09:11:59 +0800 Subject: [PATCH] [Storage Blobs Service] Undelete Container (#462) * Undelete Container --- .../inc/blobs/blob_container_client.hpp | 13 ++ sdk/storage/inc/blobs/blob_options.hpp | 13 +- .../inc/blobs/protocol/blob_rest_client.hpp | 175 +++++++++++++++--- sdk/storage/inc/common/storage_common.hpp | 1 + .../src/blobs/blob_container_client.cpp | 12 ++ sdk/storage/src/blobs/blob_service_client.cpp | 2 +- .../test/blobs/blob_container_client_test.cpp | 76 ++++++++ .../test/blobs/blob_service_client_test.cpp | 6 + 8 files changed, 268 insertions(+), 30 deletions(-) diff --git a/sdk/storage/inc/blobs/blob_container_client.hpp b/sdk/storage/inc/blobs/blob_container_client.hpp index 4a44c9ed0..bbac1651e 100644 --- a/sdk/storage/inc/blobs/blob_container_client.hpp +++ b/sdk/storage/inc/blobs/blob_container_client.hpp @@ -148,6 +148,19 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Response Delete( const DeleteContainerOptions& options = DeleteContainerOptions()) const; + /** + * @brief Restores a previously deleted container. + * + * @param deletedContainerName The name of the previously deleted container. + * @param deletedContainerVersion The version of the previously deleted container. + * @param options Optional parameters to execute this function. + * @return An UndeleteContainerResult if successful. + */ + Azure::Core::Response UndeleteContainer( + const std::string& deletedContainerName, + const std::string& deletedContainerVersion, + const UndeleteContainerOptions& options = UndeleteContainerOptions()) const; + /** * @brief Returns all user-defined metadata and system properties for the specified * container. The data returned does not include the container's list of blobs. diff --git a/sdk/storage/inc/blobs/blob_options.hpp b/sdk/storage/inc/blobs/blob_options.hpp index f8bd0a59a..885109eee 100644 --- a/sdk/storage/inc/blobs/blob_options.hpp +++ b/sdk/storage/inc/blobs/blob_options.hpp @@ -123,7 +123,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Specifies that the container's metadata be returned. */ - ListBlobContainersIncludeOption Include = ListBlobContainersIncludeOption::None; + ListBlobContainersIncludeItem Include = ListBlobContainersIncludeItem::None; }; /** @@ -280,6 +280,17 @@ namespace Azure { namespace Storage { namespace Blobs { ContainerAccessConditions AccessConditions; }; + /** + * @brief Optional parameters for BlobContainerClient::Undelete. + */ + struct UndeleteContainerOptions + { + /** + * @brief Context for cancelling long running operations. + */ + Azure::Core::Context Context; + }; + /** * @brief Optional parameters for BlobContainerClient::GetProperties. */ diff --git a/sdk/storage/inc/blobs/protocol/blob_rest_client.hpp b/sdk/storage/inc/blobs/protocol/blob_rest_client.hpp index e349d5ebe..308e21207 100644 --- a/sdk/storage/inc/blobs/protocol/blob_rest_client.hpp +++ b/sdk/storage/inc/blobs/protocol/blob_rest_client.hpp @@ -864,57 +864,59 @@ namespace Azure { namespace Storage { namespace Blobs { std::string Value; }; // struct GetUserDelegationKeyResult - enum class ListBlobContainersIncludeOption + enum class ListBlobContainersIncludeItem { None = 0, Metadata = 1, - }; // bitwise enum ListBlobContainersIncludeOption + Deleted = 2, + }; // bitwise enum ListBlobContainersIncludeItem - inline ListBlobContainersIncludeOption operator|( - ListBlobContainersIncludeOption lhs, - ListBlobContainersIncludeOption rhs) + inline ListBlobContainersIncludeItem operator|( + ListBlobContainersIncludeItem lhs, + ListBlobContainersIncludeItem rhs) { - using type = std::underlying_type_t; - return static_cast( + using type = std::underlying_type_t; + return static_cast( static_cast(lhs) | static_cast(rhs)); } - inline ListBlobContainersIncludeOption& operator|=( - ListBlobContainersIncludeOption& lhs, - ListBlobContainersIncludeOption rhs) + inline ListBlobContainersIncludeItem& operator|=( + ListBlobContainersIncludeItem& lhs, + ListBlobContainersIncludeItem rhs) { lhs = lhs | rhs; return lhs; } - inline ListBlobContainersIncludeOption operator&( - ListBlobContainersIncludeOption lhs, - ListBlobContainersIncludeOption rhs) + inline ListBlobContainersIncludeItem operator&( + ListBlobContainersIncludeItem lhs, + ListBlobContainersIncludeItem rhs) { - using type = std::underlying_type_t; - return static_cast( + using type = std::underlying_type_t; + return static_cast( static_cast(lhs) & static_cast(rhs)); } - inline ListBlobContainersIncludeOption& operator&=( - ListBlobContainersIncludeOption& lhs, - ListBlobContainersIncludeOption rhs) + inline ListBlobContainersIncludeItem& operator&=( + ListBlobContainersIncludeItem& lhs, + ListBlobContainersIncludeItem rhs) { lhs = lhs & rhs; return lhs; } - inline std::string ListBlobContainersIncludeOptionToString( - const ListBlobContainersIncludeOption& val) + inline std::string ListBlobContainersIncludeItemToString(const ListBlobContainersIncludeItem& val) { - ListBlobContainersIncludeOption value_list[] = { - ListBlobContainersIncludeOption::Metadata, + ListBlobContainersIncludeItem value_list[] = { + ListBlobContainersIncludeItem::Metadata, + ListBlobContainersIncludeItem::Deleted, }; const char* string_list[] = { "metadata", + "deleted", }; std::string ret; - for (std::size_t i = 0; i < sizeof(value_list) / sizeof(ListBlobContainersIncludeOption); ++i) + for (std::size_t i = 0; i < sizeof(value_list) / sizeof(ListBlobContainersIncludeItem); ++i) { if ((val & value_list[i]) == value_list[i]) { @@ -1245,6 +1247,10 @@ namespace Azure { namespace Storage { namespace Blobs { { }; // struct UndeleteBlobResult + struct UndeleteContainerResult + { + }; // struct UndeleteContainerResult + struct UploadBlockBlobResult { std::string ETag; @@ -1303,6 +1309,12 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Nullable LeaseDuration; BlobLeaseState LeaseState = BlobLeaseState::Available; BlobLeaseStatus LeaseStatus = BlobLeaseStatus::Unlocked; + std::string DefaultEncryptionScope; + bool PreventEncryptionScopeOverride = false; + bool IsDeleted = false; + Azure::Core::Nullable VersionId; + Azure::Core::Nullable DeletedOn; + Azure::Core::Nullable RemainingRetentionDays; }; // struct BlobContainerItem struct BlobGeoReplication @@ -1427,6 +1439,8 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Nullable LeaseDuration; BlobLeaseState LeaseState = BlobLeaseState::Available; BlobLeaseStatus LeaseStatus = BlobLeaseStatus::Unlocked; + std::string DefaultEncryptionScope; + bool PreventEncryptionScopeOverride = false; }; // struct GetContainerPropertiesResult struct StartCopyBlobFromUriResult @@ -1515,7 +1529,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Nullable Prefix; Azure::Core::Nullable Marker; Azure::Core::Nullable MaxResults; - ListBlobContainersIncludeOption IncludeMetadata = ListBlobContainersIncludeOption::None; + ListBlobContainersIncludeItem Include = ListBlobContainersIncludeItem::None; }; // struct ListContainersSegmentOptions static Azure::Core::Response ListBlobContainers( @@ -1544,11 +1558,11 @@ namespace Azure { namespace Storage { namespace Blobs { { request.AddQueryParameter("maxresults", std::to_string(options.MaxResults.GetValue())); } - std::string list_blob_containers_include_option - = ListBlobContainersIncludeOptionToString(options.IncludeMetadata); - if (!list_blob_containers_include_option.empty()) + std::string list_blob_containers_include_item + = ListBlobContainersIncludeItemToString(options.Include); + if (!list_blob_containers_include_item.empty()) { - request.AddQueryParameter("include", list_blob_containers_include_option); + request.AddQueryParameter("include", list_blob_containers_include_item); } auto pHttpResponse = pipeline.Send(context, request); Azure::Core::Http::RawResponse& httpResponse = *pHttpResponse; @@ -2297,7 +2311,13 @@ namespace Azure { namespace Storage { namespace Blobs { k_LeaseStatus, k_LeaseState, k_LeaseDuration, + k_DefaultEncryptionScope, + k_DenyEncryptionScopeOverride, k_Metadata, + k_Deleted, + k_Version, + k_DeletedTime, + k_RemainingRetentionDays, k_Unknown, }; std::vector path; @@ -2361,10 +2381,34 @@ namespace Azure { namespace Storage { namespace Blobs { { path.emplace_back(XmlTagName::k_LeaseDuration); } + else if (std::strcmp(node.Name, "DefaultEncryptionScope") == 0) + { + path.emplace_back(XmlTagName::k_DefaultEncryptionScope); + } + else if (std::strcmp(node.Name, "DenyEncryptionScopeOverride") == 0) + { + path.emplace_back(XmlTagName::k_DenyEncryptionScopeOverride); + } else if (std::strcmp(node.Name, "Metadata") == 0) { path.emplace_back(XmlTagName::k_Metadata); } + else if (std::strcmp(node.Name, "Deleted") == 0) + { + path.emplace_back(XmlTagName::k_Deleted); + } + else if (std::strcmp(node.Name, "Version") == 0) + { + path.emplace_back(XmlTagName::k_Version); + } + else if (std::strcmp(node.Name, "DeletedTime") == 0) + { + path.emplace_back(XmlTagName::k_DeletedTime); + } + else if (std::strcmp(node.Name, "RemainingRetentionDays") == 0) + { + path.emplace_back(XmlTagName::k_RemainingRetentionDays); + } else { path.emplace_back(XmlTagName::k_Unknown); @@ -2429,6 +2473,38 @@ namespace Azure { namespace Storage { namespace Blobs { { ret.LeaseDuration = node.Value; } + else if ( + path.size() == 2 && path[0] == XmlTagName::k_Properties + && path[1] == XmlTagName::k_DefaultEncryptionScope) + { + ret.DefaultEncryptionScope = node.Value; + } + else if ( + path.size() == 2 && path[0] == XmlTagName::k_Properties + && path[1] == XmlTagName::k_DenyEncryptionScopeOverride) + { + ret.PreventEncryptionScopeOverride = std::strcmp(node.Value, "true") == 0; + } + else if (path.size() == 1 && path[0] == XmlTagName::k_Deleted) + { + ret.IsDeleted = std::strcmp(node.Value, "true") == 0; + } + else if (path.size() == 1 && path[0] == XmlTagName::k_Version) + { + ret.VersionId = node.Value; + } + else if ( + path.size() == 2 && path[0] == XmlTagName::k_Properties + && path[1] == XmlTagName::k_DeletedTime) + { + ret.DeletedOn = node.Value; + } + else if ( + path.size() == 2 && path[0] == XmlTagName::k_Properties + && path[1] == XmlTagName::k_RemainingRetentionDays) + { + ret.RemainingRetentionDays = std::stoi(node.Value); + } } } return ret; @@ -3100,6 +3176,45 @@ namespace Azure { namespace Storage { namespace Blobs { std::move(response), std::move(pHttpResponse)); } + struct UndeleteContainerOptions + { + Azure::Core::Nullable Timeout; + std::string DeletedContainerName; + std::string DeletedContainerVersion; + }; // struct UndeleteContainerOptions + + static Azure::Core::Response Undelete( + const Azure::Core::Context& context, + Azure::Core::Http::HttpPipeline& pipeline, + const std::string& url, + const UndeleteContainerOptions& options) + { + unused(options); + auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); + request.AddHeader("Content-Length", "0"); + request.AddQueryParameter("restype", "container"); + request.AddQueryParameter("comp", "undelete"); + request.AddHeader("x-ms-version", c_ApiVersion); + if (options.Timeout.HasValue()) + { + request.AddQueryParameter("timeout", std::to_string(options.Timeout.GetValue())); + } + request.AddHeader("x-ms-deleted-container-name", options.DeletedContainerName); + request.AddHeader("x-ms-deleted-container-version", options.DeletedContainerVersion); + auto pHttpResponse = pipeline.Send(context, request); + Azure::Core::Http::RawResponse& httpResponse = *pHttpResponse; + UndeleteContainerResult response; + auto http_status_code + = static_cast::type>( + httpResponse.GetStatusCode()); + if (!(http_status_code == 201)) + { + throw StorageError::CreateFromResponse(context, std::move(pHttpResponse)); + } + return Azure::Core::Response( + std::move(response), std::move(pHttpResponse)); + } + struct GetContainerPropertiesOptions { Azure::Core::Nullable Timeout; @@ -3161,6 +3276,10 @@ namespace Azure { namespace Storage { namespace Blobs { { response.LeaseDuration = response_lease_duration_iterator->second; } + response.DefaultEncryptionScope + = httpResponse.GetHeaders().at("x-ms-default-encryption-scope"); + response.PreventEncryptionScopeOverride + = httpResponse.GetHeaders().at("x-ms-deny-encryption-scope-override") == "true"; return Azure::Core::Response( std::move(response), std::move(pHttpResponse)); } diff --git a/sdk/storage/inc/common/storage_common.hpp b/sdk/storage/inc/common/storage_common.hpp index 218eace71..649198efe 100644 --- a/sdk/storage/inc/common/storage_common.hpp +++ b/sdk/storage/inc/common/storage_common.hpp @@ -11,6 +11,7 @@ namespace Azure { namespace Storage { template void unused(T&&...) {} constexpr int32_t c_InfiniteLeaseDuration = -1; + constexpr static const char* c_AccountEncryptionKey = "$account-encryption-key"; std::string CreateUniqueLeaseId(); diff --git a/sdk/storage/src/blobs/blob_container_client.cpp b/sdk/storage/src/blobs/blob_container_client.cpp index 64b9f129b..5509a16ee 100644 --- a/sdk/storage/src/blobs/blob_container_client.cpp +++ b/sdk/storage/src/blobs/blob_container_client.cpp @@ -163,6 +163,18 @@ namespace Azure { namespace Storage { namespace Blobs { options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions); } + Azure::Core::Response BlobContainerClient::UndeleteContainer( + const std::string& deletedContainerName, + const std::string& deletedContainerVersion, + const UndeleteContainerOptions& options) const + { + BlobRestClient::Container::UndeleteContainerOptions protocolLayerOptions; + protocolLayerOptions.DeletedContainerName = deletedContainerName; + protocolLayerOptions.DeletedContainerVersion = deletedContainerVersion; + return BlobRestClient::Container::Undelete( + options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions); + } + Azure::Core::Response BlobContainerClient::GetProperties( const GetContainerPropertiesOptions& options) const { diff --git a/sdk/storage/src/blobs/blob_service_client.cpp b/sdk/storage/src/blobs/blob_service_client.cpp index 3a366cb0e..8aaef8846 100644 --- a/sdk/storage/src/blobs/blob_service_client.cpp +++ b/sdk/storage/src/blobs/blob_service_client.cpp @@ -127,7 +127,7 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.Prefix = options.Prefix; protocolLayerOptions.Marker = options.Marker; protocolLayerOptions.MaxResults = options.MaxResults; - protocolLayerOptions.IncludeMetadata = options.Include; + protocolLayerOptions.Include = options.Include; return BlobRestClient::Service::ListBlobContainers( options.Context, *m_pipeline, m_serviceUrl.ToString(), protocolLayerOptions); } diff --git a/sdk/storage/test/blobs/blob_container_client_test.cpp b/sdk/storage/test/blobs/blob_container_client_test.cpp index f515f5302..db43e4a9c 100644 --- a/sdk/storage/test/blobs/blob_container_client_test.cpp +++ b/sdk/storage/test/blobs/blob_container_client_test.cpp @@ -5,6 +5,9 @@ #include "blobs/blob_sas_builder.hpp" #include "common/crypt.hpp" +#include +#include + namespace Azure { namespace Storage { namespace Blobs { bool operator==( @@ -396,6 +399,11 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobContainerClientTest, EncryptionScope) { + { + auto properties = *m_blobContainerClient->GetProperties(); + EXPECT_EQ(properties.DefaultEncryptionScope, c_AccountEncryptionKey); + EXPECT_EQ(properties.PreventEncryptionScopeOverride, false); + } { std::string containerName = LowercaseRandomString(); std::string blobName = RandomString(); @@ -407,6 +415,11 @@ namespace Azure { namespace Storage { namespace Test { createOptions.DefaultEncryptionScope = c_TestEncryptionScope; createOptions.PreventEncryptionScopeOverride = true; EXPECT_NO_THROW(containerClient.Create(createOptions)); + auto properties = *containerClient.GetProperties(); + EXPECT_EQ(properties.DefaultEncryptionScope, createOptions.DefaultEncryptionScope.GetValue()); + EXPECT_EQ( + properties.PreventEncryptionScopeOverride, + createOptions.PreventEncryptionScopeOverride.GetValue()); auto appendBlobClient = containerClient.GetAppendBlobClient(blobName); auto blobContentInfo = appendBlobClient.Create(); EXPECT_TRUE(blobContentInfo->EncryptionScope.HasValue()); @@ -622,4 +635,67 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(containerClient.Delete(options)); } + TEST_F(BlobContainerClientTest, DISABLED_Undelete) + { + auto serviceClient = Azure::Storage::Blobs::BlobServiceClient::CreateFromConnectionString( + StandardStorageConnectionString()); + std::string containerName = LowercaseRandomString(); + auto containerClient = serviceClient.GetBlobContainerClient(containerName); + containerClient.Create(); + containerClient.Delete(); + + Blobs::BlobContainerItem deletedContainerItem; + { + Azure::Storage::Blobs::ListContainersSegmentOptions options; + options.Prefix = containerName; + options.Include = Blobs::ListBlobContainersIncludeItem::Deleted; + do + { + auto res = serviceClient.ListBlobContainersSegment(options); + options.Marker = res->NextMarker; + for (const auto& container : res->Items) + { + if (container.Name == containerName) + { + deletedContainerItem = container; + break; + } + } + } while (!options.Marker.GetValue().empty()); + } + EXPECT_EQ(deletedContainerItem.Name, containerName); + EXPECT_TRUE(deletedContainerItem.IsDeleted); + EXPECT_TRUE(deletedContainerItem.VersionId.HasValue()); + EXPECT_FALSE(deletedContainerItem.VersionId.GetValue().empty()); + EXPECT_TRUE(deletedContainerItem.DeletedOn.HasValue()); + EXPECT_FALSE(deletedContainerItem.DeletedOn.GetValue().empty()); + EXPECT_TRUE(deletedContainerItem.RemainingRetentionDays.HasValue()); + EXPECT_GE(deletedContainerItem.RemainingRetentionDays.GetValue(), 0); + + std::string containerName2 = LowercaseRandomString(); + auto containerClient2 = serviceClient.GetBlobContainerClient(containerName2); + for (int i = 0; i < 60; ++i) + { + try + { + containerClient2.UndeleteContainer( + deletedContainerItem.Name, deletedContainerItem.VersionId.GetValue()); + break; + } + catch (StorageError& e) + { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict + && e.ReasonPhrase == "The specified container is being deleted.") + { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + else + { + throw; + } + } + } + EXPECT_NO_THROW(containerClient2.GetProperties()); + } + }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/test/blobs/blob_service_client_test.cpp b/sdk/storage/test/blobs/blob_service_client_test.cpp index ffe48eae7..4430ae759 100644 --- a/sdk/storage/test/blobs/blob_service_client_test.cpp +++ b/sdk/storage/test/blobs/blob_service_client_test.cpp @@ -150,6 +150,12 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(container.Name.empty()); EXPECT_FALSE(container.ETag.empty()); EXPECT_FALSE(container.LastModified.empty()); + EXPECT_FALSE(container.IsDeleted); + EXPECT_FALSE(container.VersionId.HasValue()); + EXPECT_FALSE(container.DeletedOn.HasValue()); + EXPECT_FALSE(container.RemainingRetentionDays.HasValue()); + EXPECT_EQ(container.DefaultEncryptionScope, c_AccountEncryptionKey); + EXPECT_FALSE(container.PreventEncryptionScopeOverride); listContainers.insert(container.Name); } } while (!options.Marker.GetValue().empty());