From 4c5286803b0a15b9a95e35acc0ae7dcf1981c483 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Wed, 19 Aug 2020 09:13:13 +0800 Subject: [PATCH] [Storage Blob Service] ORS (#467) --- .../inc/blobs/protocol/blob_rest_client.hpp | 379 +++++++++++++----- .../test/blobs/blob_container_client_test.cpp | 4 +- .../test/blobs/blob_service_client_test.cpp | 2 +- 3 files changed, 292 insertions(+), 93 deletions(-) diff --git a/sdk/storage/inc/blobs/protocol/blob_rest_client.hpp b/sdk/storage/inc/blobs/protocol/blob_rest_client.hpp index 140c3168b..ff893d4e9 100644 --- a/sdk/storage/inc/blobs/protocol/blob_rest_client.hpp +++ b/sdk/storage/inc/blobs/protocol/blob_rest_client.hpp @@ -996,6 +996,48 @@ namespace Azure { namespace Storage { namespace Blobs { return ret; } + enum class ObjectReplicationStatus + { + Unknown, + Complete, + Failed, + }; // enum class ObjectReplicationStatus + + inline std::string ObjectReplicationStatusToString( + const ObjectReplicationStatus& object_replication_status) + { + switch (object_replication_status) + { + case ObjectReplicationStatus::Unknown: + return ""; + case ObjectReplicationStatus::Complete: + return "complete"; + case ObjectReplicationStatus::Failed: + return "failed"; + default: + return std::string(); + } + } + + inline ObjectReplicationStatus ObjectReplicationStatusFromString( + const std::string& object_replication_status) + { + if (object_replication_status == "") + { + return ObjectReplicationStatus::Unknown; + } + if (object_replication_status == "complete") + { + return ObjectReplicationStatus::Complete; + } + if (object_replication_status == "failed") + { + return ObjectReplicationStatus::Failed; + } + throw std::runtime_error( + "cannot convert " + object_replication_status + " to ObjectReplicationStatus"); + } + enum class PublicAccessType { Container, @@ -1317,42 +1359,16 @@ namespace Azure { namespace Storage { namespace Blobs { bool PreventEncryptionScopeOverride = false; bool IsDeleted = false; Azure::Core::Nullable VersionId; - Azure::Core::Nullable DeletedOn; + Azure::Core::Nullable DeletedTime; Azure::Core::Nullable RemainingRetentionDays; }; // struct BlobContainerItem struct BlobGeoReplication { BlobGeoReplicationStatus Status = BlobGeoReplicationStatus::Unknown; - Azure::Core::Nullable LastSyncedOn; + Azure::Core::Nullable LastSyncTime; }; // struct BlobGeoReplication - struct BlobItem - { - std::string Name; - bool Deleted = false; - std::string Snapshot; - Azure::Core::Nullable VersionId; - Azure::Core::Nullable IsCurrentVersion; - BlobHttpHeaders HttpHeaders; - std::map Metadata; - std::string CreationTime; - std::string LastModified; - std::string ETag; - int64_t ContentLength = 0; - Blobs::BlobType BlobType = Blobs::BlobType::Unknown; - Azure::Core::Nullable Tier; - Azure::Core::Nullable AccessTierInferred; - BlobLeaseStatus LeaseStatus = BlobLeaseStatus::Unlocked; - BlobLeaseState LeaseState = BlobLeaseState::Available; - Azure::Core::Nullable LeaseDuration; - Azure::Core::Nullable ServerEncrypted; - Azure::Core::Nullable EncryptionKeySha256; - Azure::Core::Nullable EncryptionScope; - Azure::Core::Nullable SequenceNumber; // only for page blobd - Azure::Core::Nullable IsSealed; // only for append blob - }; // struct BlobItem - struct BlobMetrics { std::string Version; @@ -1361,63 +1377,12 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Nullable IncludeApis; }; // struct BlobMetrics - struct DownloadBlobResult - { - std::unique_ptr BodyStream; - std::string ETag; - std::string LastModified; - Azure::Core::Nullable ContentRange; - BlobHttpHeaders HttpHeaders; - std::map Metadata; - Azure::Core::Nullable SequenceNumber; // only for page blob - Azure::Core::Nullable CommittedBlockCount; // only for append blob - Azure::Core::Nullable IsSealed; // only for append blob - Blobs::BlobType BlobType = Blobs::BlobType::Unknown; - Azure::Core::Nullable ContentMd5; // Md5 for the downloaded range - Azure::Core::Nullable ContentCrc64; - Azure::Core::Nullable LeaseDuration; - Azure::Core::Nullable LeaseState; - Azure::Core::Nullable LeaseStatus; - Azure::Core::Nullable ServerEncrypted; - Azure::Core::Nullable EncryptionKeySha256; - Azure::Core::Nullable EncryptionScope; - }; // struct DownloadBlobResult - struct GetAccountInfoResult { Blobs::SkuName SkuName = Blobs::SkuName::Unknown; Blobs::AccountKind AccountKind = Blobs::AccountKind::Unknown; }; // struct GetAccountInfoResult - struct GetBlobPropertiesResult - { - std::string ETag; - std::string LastModified; - std::string CreationTime; - std::map Metadata; - Blobs::BlobType BlobType = Blobs::BlobType::Unknown; - Azure::Core::Nullable LeaseDuration; - Azure::Core::Nullable LeaseState; - Azure::Core::Nullable LeaseStatus; - int64_t ContentLength = 0; - BlobHttpHeaders HttpHeaders; - Azure::Core::Nullable SequenceNumber; // only for page blob - Azure::Core::Nullable CommittedBlockCount; // only for append blob - Azure::Core::Nullable IsSealed; // only for append blob - Azure::Core::Nullable ServerEncrypted; - Azure::Core::Nullable EncryptionKeySha256; - Azure::Core::Nullable EncryptionScope; - Azure::Core::Nullable Tier; - Azure::Core::Nullable AccessTierInferred; - Azure::Core::Nullable ArchiveStatus; - Azure::Core::Nullable AccessTierChangeTime; - Azure::Core::Nullable CopyId; - Azure::Core::Nullable CopySource; - Azure::Core::Nullable CopyStatus; - Azure::Core::Nullable CopyProgress; - Azure::Core::Nullable CopyCompletionTime; - }; // struct GetBlobPropertiesResult - struct GetBlockListResult { std::string ETag; @@ -1451,6 +1416,12 @@ namespace Azure { namespace Storage { namespace Blobs { bool PreventEncryptionScopeOverride = false; }; // struct GetContainerPropertiesResult + struct ObjectReplicationRule + { + std::string RuleId; + ObjectReplicationStatus ReplicationStatus = ObjectReplicationStatus::Unknown; + }; // struct ObjectReplicationRule + struct StartCopyBlobFromUriResult { std::string ETag; @@ -1496,6 +1467,108 @@ namespace Azure { namespace Storage { namespace Blobs { BlobGeoReplication GeoReplication; }; // struct GetServiceStatisticsResult + struct ListContainersSegmentResult + { + std::string ServiceEndpoint; + std::string Prefix; + std::string Marker; + std::string NextMarker; + std::vector Items; + }; // struct ListContainersSegmentResult + + struct ObjectReplicationPolicy + { + std::string PolicyId; + std::vector Rules; + }; // struct ObjectReplicationPolicy + + struct BlobItem + { + std::string Name; + bool Deleted = false; + std::string Snapshot; + Azure::Core::Nullable VersionId; + Azure::Core::Nullable IsCurrentVersion; + BlobHttpHeaders HttpHeaders; + std::map Metadata; + std::string CreationTime; + std::string LastModified; + std::string ETag; + int64_t ContentLength = 0; + Blobs::BlobType BlobType = Blobs::BlobType::Unknown; + Azure::Core::Nullable Tier; + Azure::Core::Nullable AccessTierInferred; + BlobLeaseStatus LeaseStatus = BlobLeaseStatus::Unlocked; + BlobLeaseState LeaseState = BlobLeaseState::Available; + Azure::Core::Nullable LeaseDuration; + Azure::Core::Nullable ServerEncrypted; + Azure::Core::Nullable EncryptionKeySha256; + Azure::Core::Nullable EncryptionScope; + Azure::Core::Nullable SequenceNumber; // only for page blobd + Azure::Core::Nullable IsSealed; // only for append blob + std::vector + ObjectReplicationSourceProperties; // only valid for replication source blob + }; // struct BlobItem + + struct DownloadBlobResult + { + std::unique_ptr BodyStream; + std::string ETag; + std::string LastModified; + Azure::Core::Nullable ContentRange; + BlobHttpHeaders HttpHeaders; + std::map Metadata; + Azure::Core::Nullable SequenceNumber; // only for page blob + Azure::Core::Nullable CommittedBlockCount; // only for append blob + Azure::Core::Nullable IsSealed; // only for append blob + Blobs::BlobType BlobType = Blobs::BlobType::Unknown; + Azure::Core::Nullable ContentMd5; // Md5 for the downloaded range + Azure::Core::Nullable ContentCrc64; + Azure::Core::Nullable LeaseDuration; + Azure::Core::Nullable LeaseState; + Azure::Core::Nullable LeaseStatus; + Azure::Core::Nullable ServerEncrypted; + Azure::Core::Nullable EncryptionKeySha256; + Azure::Core::Nullable EncryptionScope; + Azure::Core::Nullable + ObjectReplicationDestinationPolicyId; // only valid for replication destination blob + std::vector + ObjectReplicationSourceProperties; // only valid for replication source blob + }; // struct DownloadBlobResult + + struct GetBlobPropertiesResult + { + std::string ETag; + std::string LastModified; + std::string CreationTime; + std::map Metadata; + Blobs::BlobType BlobType = Blobs::BlobType::Unknown; + Azure::Core::Nullable LeaseDuration; + Azure::Core::Nullable LeaseState; + Azure::Core::Nullable LeaseStatus; + int64_t ContentLength = 0; + BlobHttpHeaders HttpHeaders; + Azure::Core::Nullable SequenceNumber; // only for page blob + Azure::Core::Nullable CommittedBlockCount; // only for append blob + Azure::Core::Nullable IsSealed; // only for append blob + Azure::Core::Nullable ServerEncrypted; + Azure::Core::Nullable EncryptionKeySha256; + Azure::Core::Nullable EncryptionScope; + Azure::Core::Nullable Tier; + Azure::Core::Nullable AccessTierInferred; + Azure::Core::Nullable ArchiveStatus; + Azure::Core::Nullable AccessTierChangeTime; + Azure::Core::Nullable CopyId; + Azure::Core::Nullable CopySource; + Azure::Core::Nullable CopyStatus; + Azure::Core::Nullable CopyProgress; + Azure::Core::Nullable CopyCompletionTime; + Azure::Core::Nullable + ObjectReplicationDestinationPolicyId; // only valid for replication destination blob + std::vector + ObjectReplicationSourceProperties; // only valid for replication source blob + }; // struct GetBlobPropertiesResult + struct ListBlobsByHierarchySegmentResult { std::string ServiceEndpoint; @@ -1518,15 +1591,6 @@ namespace Azure { namespace Storage { namespace Blobs { std::vector Items; }; // struct ListBlobsFlatSegmentResult - struct ListContainersSegmentResult - { - std::string ServiceEndpoint; - std::string Prefix; - std::string Marker; - std::string NextMarker; - std::vector Items; - }; // struct ListContainersSegmentResult - class BlobRestClient { public: class Service { @@ -2505,7 +2569,7 @@ namespace Azure { namespace Storage { namespace Blobs { path.size() == 2 && path[0] == XmlTagName::k_Properties && path[1] == XmlTagName::k_DeletedTime) { - ret.DeletedOn = node.Value; + ret.DeletedTime = node.Value; } else if ( path.size() == 2 && path[0] == XmlTagName::k_Properties @@ -2654,7 +2718,7 @@ namespace Azure { namespace Storage { namespace Blobs { } else if (path.size() == 1 && path[0] == XmlTagName::k_LastSyncTime) { - ret.LastSyncedOn = node.Value; + ret.LastSyncTime = node.Value; } } } @@ -4187,6 +4251,7 @@ namespace Azure { namespace Storage { namespace Blobs { k_Sealed, k_xmsblobsequencenumber, k_Metadata, + k_OrMetadata, k_Unknown, }; std::vector path; @@ -4318,6 +4383,10 @@ namespace Azure { namespace Storage { namespace Blobs { { path.emplace_back(XmlTagName::k_Metadata); } + else if (std::strcmp(node.Name, "OrMetadata") == 0) + { + path.emplace_back(XmlTagName::k_OrMetadata); + } else { path.emplace_back(XmlTagName::k_Unknown); @@ -4327,6 +4396,12 @@ namespace Azure { namespace Storage { namespace Blobs { ret.Metadata = MetadataFromXml(reader); path.pop_back(); } + else if (path.size() == 1 && path[0] == XmlTagName::k_OrMetadata) + { + ret.ObjectReplicationSourceProperties + = ObjectReplicationSourcePropertiesFromXml(reader); + path.pop_back(); + } } else if (node.Type == XmlNodeType::Text) { @@ -4645,6 +4720,58 @@ namespace Azure { namespace Storage { namespace Blobs { return ret; } + static std::vector ObjectReplicationSourcePropertiesFromXml( + XmlReader& reader) + { + int depth = 0; + std::map> orPropertiesMap; + std::string policyId; + std::string ruleId; + while (true) + { + auto node = reader.Read(); + if (node.Type == XmlNodeType::End) + { + break; + } + else if (node.Type == XmlNodeType::StartTag) + { + ++depth; + std::string startTagName = node.Name; + if (startTagName.substr(0, 3) == "or-") + { + auto underscorePos = startTagName.find('_', 3); + policyId + = std::string(startTagName.begin() + 3, startTagName.begin() + underscorePos); + ruleId = startTagName.substr(underscorePos + 1); + } + } + else if (node.Type == XmlNodeType::EndTag) + { + if (depth-- == 0) + { + break; + } + } + if (depth == 1 && node.Type == XmlNodeType::Text) + { + ObjectReplicationRule rule; + rule.RuleId = std::move(ruleId); + rule.ReplicationStatus = ObjectReplicationStatusFromString(node.Value); + orPropertiesMap[policyId].emplace_back(std::move(rule)); + } + } + std::vector ret; + for (auto& property : orPropertiesMap) + { + ObjectReplicationPolicy policy; + policy.PolicyId = property.first; + policy.Rules = std::move(property.second); + ret.emplace_back(std::move(policy)); + } + return ret; + } + static void SetContainerAccessPolicyOptionsToXml( XmlWriter& writer, const SetContainerAccessPolicyOptions& options) @@ -4883,6 +5010,42 @@ namespace Azure { namespace Storage { namespace Blobs { response.IsSealed = response_is_sealed_iterator->second == "true"; } response.BlobType = BlobTypeFromString(httpResponse.GetHeaders().at("x-ms-blob-type")); + auto response_object_replication_destination_policy_id_iterator + = httpResponse.GetHeaders().find("x-ms-or-policy-id"); + if (response_object_replication_destination_policy_id_iterator + != httpResponse.GetHeaders().end()) + { + response.ObjectReplicationDestinationPolicyId + = response_object_replication_destination_policy_id_iterator->second; + } + { + std::map> orPropertiesMap; + for (auto i = httpResponse.GetHeaders().lower_bound("x-ms-or-"); + i != httpResponse.GetHeaders().end() && i->first.substr(0, 8) == "x-ms-or-"; + ++i) + { + const std::string& header = i->first; + auto underscorePos = header.find('_', 8); + if (underscorePos == std::string::npos) + { + continue; + } + std::string policyId = std::string(header.begin() + 8, header.begin() + underscorePos); + std::string ruleId = header.substr(underscorePos + 1); + + ObjectReplicationRule rule; + rule.RuleId = std::move(ruleId); + rule.ReplicationStatus = ObjectReplicationStatusFromString(i->second); + orPropertiesMap[policyId].emplace_back(std::move(rule)); + } + for (auto& property : orPropertiesMap) + { + ObjectReplicationPolicy policy; + policy.PolicyId = property.first; + policy.Rules = std::move(property.second); + response.ObjectReplicationSourceProperties.emplace_back(std::move(policy)); + } + } return Azure::Core::Response( std::move(response), std::move(pHttpResponse)); } @@ -5206,6 +5369,42 @@ namespace Azure { namespace Storage { namespace Blobs { { response.CopyCompletionTime = response_copy_completion_time_iterator->second; } + auto response_object_replication_destination_policy_id_iterator + = httpResponse.GetHeaders().find("x-ms-or-policy-id"); + if (response_object_replication_destination_policy_id_iterator + != httpResponse.GetHeaders().end()) + { + response.ObjectReplicationDestinationPolicyId + = response_object_replication_destination_policy_id_iterator->second; + } + { + std::map> orPropertiesMap; + for (auto i = httpResponse.GetHeaders().lower_bound("x-ms-or-"); + i != httpResponse.GetHeaders().end() && i->first.substr(0, 8) == "x-ms-or-"; + ++i) + { + const std::string& header = i->first; + auto underscorePos = header.find('_', 8); + if (underscorePos == std::string::npos) + { + continue; + } + std::string policyId = std::string(header.begin() + 8, header.begin() + underscorePos); + std::string ruleId = header.substr(underscorePos + 1); + + ObjectReplicationRule rule; + rule.RuleId = std::move(ruleId); + rule.ReplicationStatus = ObjectReplicationStatusFromString(i->second); + orPropertiesMap[policyId].emplace_back(std::move(rule)); + } + for (auto& property : orPropertiesMap) + { + ObjectReplicationPolicy policy; + policy.PolicyId = property.first; + policy.Rules = std::move(property.second); + response.ObjectReplicationSourceProperties.emplace_back(std::move(policy)); + } + } return Azure::Core::Response( std::move(response), std::move(pHttpResponse)); } diff --git a/sdk/storage/test/blobs/blob_container_client_test.cpp b/sdk/storage/test/blobs/blob_container_client_test.cpp index 544ec807a..086cdaf09 100644 --- a/sdk/storage/test/blobs/blob_container_client_test.cpp +++ b/sdk/storage/test/blobs/blob_container_client_test.cpp @@ -701,8 +701,8 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(deletedContainerItem.IsDeleted); EXPECT_TRUE(deletedContainerItem.VersionId.HasValue()); EXPECT_FALSE(deletedContainerItem.VersionId.GetValue().empty()); - EXPECT_TRUE(deletedContainerItem.DeletedOn.HasValue()); - EXPECT_FALSE(deletedContainerItem.DeletedOn.GetValue().empty()); + EXPECT_TRUE(deletedContainerItem.DeletedTime.HasValue()); + EXPECT_FALSE(deletedContainerItem.DeletedTime.GetValue().empty()); EXPECT_TRUE(deletedContainerItem.RemainingRetentionDays.HasValue()); EXPECT_GE(deletedContainerItem.RemainingRetentionDays.GetValue(), 0); diff --git a/sdk/storage/test/blobs/blob_service_client_test.cpp b/sdk/storage/test/blobs/blob_service_client_test.cpp index 4430ae759..36159a287 100644 --- a/sdk/storage/test/blobs/blob_service_client_test.cpp +++ b/sdk/storage/test/blobs/blob_service_client_test.cpp @@ -152,7 +152,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(container.LastModified.empty()); EXPECT_FALSE(container.IsDeleted); EXPECT_FALSE(container.VersionId.HasValue()); - EXPECT_FALSE(container.DeletedOn.HasValue()); + EXPECT_FALSE(container.DeletedTime.HasValue()); EXPECT_FALSE(container.RemainingRetentionDays.HasValue()); EXPECT_EQ(container.DefaultEncryptionScope, c_AccountEncryptionKey); EXPECT_FALSE(container.PreventEncryptionScopeOverride);