From c700b7811c814ceceb1f56330cb45ba7c4d228c5 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Fri, 7 Aug 2020 12:14:26 +0800 Subject: [PATCH] [Storage Blobs Service] Blob Versioning (#409) * blob versioning --- sdk/storage/inc/blobs/append_blob_client.hpp | 10 +++ sdk/storage/inc/blobs/blob_client.hpp | 10 +++ sdk/storage/inc/blobs/block_blob_client.hpp | 10 +++ sdk/storage/inc/blobs/page_blob_client.hpp | 10 +++ .../inc/blobs/protocol/blob_rest_client.hpp | 63 +++++++++++++++- sdk/storage/src/blobs/append_blob_client.cpp | 14 ++++ sdk/storage/src/blobs/blob_client.cpp | 14 ++++ .../src/blobs/blob_container_client.cpp | 20 +++++- sdk/storage/src/blobs/blob_sas_builder.cpp | 34 +++++++-- sdk/storage/src/blobs/block_blob_client.cpp | 14 ++++ sdk/storage/src/blobs/page_blob_client.cpp | 14 ++++ .../test/blobs/append_blob_client_test.cpp | 6 +- .../test/blobs/blob_container_client_test.cpp | 68 ++++++++++++++++++ sdk/storage/test/blobs/blob_sas_test.cpp | 72 ++++++++++++++----- .../test/blobs/block_blob_client_test.cpp | 42 ++++++++--- .../test/blobs/page_blob_client_test.cpp | 14 +++- 16 files changed, 379 insertions(+), 36 deletions(-) diff --git a/sdk/storage/inc/blobs/append_blob_client.hpp b/sdk/storage/inc/blobs/append_blob_client.hpp index b11235ba1..cfdf22b17 100644 --- a/sdk/storage/inc/blobs/append_blob_client.hpp +++ b/sdk/storage/inc/blobs/append_blob_client.hpp @@ -96,6 +96,16 @@ namespace Azure { namespace Storage { namespace Blobs { */ AppendBlobClient WithSnapshot(const std::string& snapshot) const; + /** + * @brief Creates a clone of this instance that references a version ID rather than the base + * blob. + * + * @param versionId The version ID returning a URL to the base blob. + * @return A new AppendBlobClient instance. + * @remarks Pass empty string to remove the version ID returning the base blob. + */ + AppendBlobClient WithVersionId(const std::string& versionId) const; + /** * @brief Creates a new 0-length append blob. The content of any existing blob is * overwritten with the newly initialized append blob. diff --git a/sdk/storage/inc/blobs/blob_client.hpp b/sdk/storage/inc/blobs/blob_client.hpp index 7b1070735..8e5b0fec9 100644 --- a/sdk/storage/inc/blobs/blob_client.hpp +++ b/sdk/storage/inc/blobs/blob_client.hpp @@ -136,6 +136,16 @@ namespace Azure { namespace Storage { namespace Blobs { */ BlobClient WithSnapshot(const std::string& snapshot) const; + /** + * @brief Creates a clone of this instance that references a version ID rather than the base + * blob. + * + * @param versionId The version ID returning a URL to the base blob. + * @return A new BlobClient instance. + * @remarks Pass empty string to remove the version ID returning the base blob. + */ + BlobClient WithVersionId(const std::string& versionId) const; + /** * @brief Returns all user-defined metadata, standard HTTP properties, and system * properties for the blob. It does not return the content of the blob. diff --git a/sdk/storage/inc/blobs/block_blob_client.hpp b/sdk/storage/inc/blobs/block_blob_client.hpp index f8b1f7b1e..bd3a7e843 100644 --- a/sdk/storage/inc/blobs/block_blob_client.hpp +++ b/sdk/storage/inc/blobs/block_blob_client.hpp @@ -106,6 +106,16 @@ namespace Azure { namespace Storage { namespace Blobs { */ BlockBlobClient WithSnapshot(const std::string& snapshot) const; + /** + * @brief Creates a clone of this instance that references a version ID rather than the base + * blob. + * + * @param versionId The version ID returning a URL to the base blob. + * @return A new BlockBlobClient instance. + * @remarks Pass empty string to remove the version ID returning the base blob. + */ + BlockBlobClient WithVersionId(const std::string& versionId) const; + /** * @brief Creates a new block blob, or updates the content of an existing block blob. Updating * an existing block blob overwrites any existing metadata on the blob. diff --git a/sdk/storage/inc/blobs/page_blob_client.hpp b/sdk/storage/inc/blobs/page_blob_client.hpp index eed5125d7..4db77a602 100644 --- a/sdk/storage/inc/blobs/page_blob_client.hpp +++ b/sdk/storage/inc/blobs/page_blob_client.hpp @@ -98,6 +98,16 @@ namespace Azure { namespace Storage { namespace Blobs { */ PageBlobClient WithSnapshot(const std::string& snapshot) const; + /** + * @brief Creates a clone of this instance that references a version ID rather than the base + * blob. + * + * @param versionId The version ID returning a URL to the base blob. + * @return A new PageBlobClient instance. + * @remarks Pass empty string to remove the version ID returning the base blob. + */ + PageBlobClient WithVersionId(const std::string& versionId) const; + /** * @brief Creates a new page blob of the specified size. The content of any existing * blob is overwritten with the newly initialized page blob. diff --git a/sdk/storage/inc/blobs/protocol/blob_rest_client.hpp b/sdk/storage/inc/blobs/protocol/blob_rest_client.hpp index e5852b79c..872ad73eb 100644 --- a/sdk/storage/inc/blobs/protocol/blob_rest_client.hpp +++ b/sdk/storage/inc/blobs/protocol/blob_rest_client.hpp @@ -302,6 +302,7 @@ namespace Azure { namespace Storage { namespace Blobs { std::string LastModified; Azure::Core::Nullable ContentMd5; Azure::Core::Nullable ContentCrc64; + Azure::Core::Nullable VersionId; Azure::Core::Nullable SequenceNumber; Azure::Core::Nullable ServerEncrypted; Azure::Core::Nullable EncryptionKeySha256; @@ -498,6 +499,7 @@ namespace Azure { namespace Storage { namespace Blobs { std::string Snapshot; std::string ETag; std::string LastModified; + Azure::Core::Nullable VersionId; Azure::Core::Nullable ServerEncrypted; Azure::Core::Nullable EncryptionKeySha256; }; // struct BlobSnapshotInfo @@ -811,7 +813,8 @@ namespace Azure { namespace Storage { namespace Blobs { Deleted = 2, Metadata = 4, Snapshots = 8, - UncomittedBlobs = 16, + Versions = 16, + UncomittedBlobs = 32, }; // bitwise enum ListBlobsIncludeItem inline ListBlobsIncludeItem operator|(ListBlobsIncludeItem lhs, ListBlobsIncludeItem rhs) @@ -845,6 +848,7 @@ namespace Azure { namespace Storage { namespace Blobs { ListBlobsIncludeItem::Deleted, ListBlobsIncludeItem::Metadata, ListBlobsIncludeItem::Snapshots, + ListBlobsIncludeItem::Versions, ListBlobsIncludeItem::UncomittedBlobs, }; const char* string_list[] = { @@ -852,6 +856,7 @@ namespace Azure { namespace Storage { namespace Blobs { "deleted", "metadata", "snapshots", + "versions", "uncommittedblobs", }; std::string ret; @@ -1144,6 +1149,7 @@ namespace Azure { namespace Storage { namespace Blobs { std::string LastModified; std::string CopyId; Blobs::CopyStatus CopyStatus = Blobs::CopyStatus::Unknown; + Azure::Core::Nullable VersionId; }; // struct BlobCopyInfo struct BlobDownloadResponse @@ -1177,6 +1183,8 @@ namespace Azure { namespace Storage { namespace Blobs { std::string Name; bool Deleted = false; std::string Snapshot; + Azure::Core::Nullable VersionId; + Azure::Core::Nullable IsCurrentVersion; BlobHttpHeaders HttpHeaders; std::map Metadata; std::string CreationTime; @@ -3800,6 +3808,8 @@ namespace Azure { namespace Storage { namespace Blobs { k_Name, k_Deleted, k_Snapshot, + k_VersionId, + k_IsCurrentVersion, k_Properties, k_ContentType, k_ContentEncoding, @@ -3855,6 +3865,14 @@ namespace Azure { namespace Storage { namespace Blobs { { path.emplace_back(XmlTagName::k_Snapshot); } + else if (std::strcmp(node.Name, "VersionId") == 0) + { + path.emplace_back(XmlTagName::k_VersionId); + } + else if (std::strcmp(node.Name, "IsCurrentVersion") == 0) + { + path.emplace_back(XmlTagName::k_IsCurrentVersion); + } else if (std::strcmp(node.Name, "Properties") == 0) { path.emplace_back(XmlTagName::k_Properties); @@ -3959,6 +3977,14 @@ namespace Azure { namespace Storage { namespace Blobs { { ret.Snapshot = node.Value; } + else if (path.size() == 1 && path[0] == XmlTagName::k_VersionId) + { + ret.VersionId = node.Value; + } + else if (path.size() == 1 && path[0] == XmlTagName::k_IsCurrentVersion) + { + ret.IsCurrentVersion = std::strcmp(node.Value, "true") == 0; + } else if ( path.size() == 2 && path[0] == XmlTagName::k_Properties && path[1] == XmlTagName::k_ContentType) @@ -5119,6 +5145,11 @@ namespace Azure { namespace Storage { namespace Blobs { response.CopyId = httpResponse.GetHeaders().at("x-ms-copy-id"); response.CopyStatus = CopyStatusFromString(httpResponse.GetHeaders().at("x-ms-copy-status")); + auto response_version_id_iterator = httpResponse.GetHeaders().find("x-ms-version-id"); + if (response_version_id_iterator != httpResponse.GetHeaders().end()) + { + response.VersionId = response_version_id_iterator->second; + } return Azure::Core::Response(std::move(response), std::move(pHttpResponse)); } @@ -5264,6 +5295,11 @@ namespace Azure { namespace Storage { namespace Blobs { response.EncryptionKeySha256 = response_encryption_key_sha256_iterator->second; } response.Snapshot = httpResponse.GetHeaders().at("x-ms-snapshot"); + auto response_version_id_iterator = httpResponse.GetHeaders().find("x-ms-version-id"); + if (response_version_id_iterator != httpResponse.GetHeaders().end()) + { + response.VersionId = response_version_id_iterator->second; + } return Azure::Core::Response( std::move(response), std::move(pHttpResponse)); } @@ -5724,6 +5760,11 @@ namespace Azure { namespace Storage { namespace Blobs { { response.ContentCrc64 = response_content_crc64_iterator->second; } + auto response_version_id_iterator = httpResponse.GetHeaders().find("x-ms-version-id"); + if (response_version_id_iterator != httpResponse.GetHeaders().end()) + { + response.VersionId = response_version_id_iterator->second; + } auto response_server_encrypted_iterator = httpResponse.GetHeaders().find("x-ms-server-encrypted"); if (response_server_encrypted_iterator != httpResponse.GetHeaders().end()) @@ -6085,6 +6126,11 @@ namespace Azure { namespace Storage { namespace Blobs { } response.ETag = httpResponse.GetHeaders().at("etag"); response.LastModified = httpResponse.GetHeaders().at("last-modified"); + auto response_version_id_iterator = httpResponse.GetHeaders().find("x-ms-version-id"); + if (response_version_id_iterator != httpResponse.GetHeaders().end()) + { + response.VersionId = response_version_id_iterator->second; + } auto response_server_encrypted_iterator = httpResponse.GetHeaders().find("x-ms-server-encrypted"); if (response_server_encrypted_iterator != httpResponse.GetHeaders().end()) @@ -6442,6 +6488,11 @@ namespace Azure { namespace Storage { namespace Blobs { { response.ContentCrc64 = response_content_crc64_iterator->second; } + auto response_version_id_iterator = httpResponse.GetHeaders().find("x-ms-version-id"); + if (response_version_id_iterator != httpResponse.GetHeaders().end()) + { + response.VersionId = response_version_id_iterator->second; + } auto response_server_encrypted_iterator = httpResponse.GetHeaders().find("x-ms-server-encrypted"); if (response_server_encrypted_iterator != httpResponse.GetHeaders().end()) @@ -7106,6 +7157,11 @@ namespace Azure { namespace Storage { namespace Blobs { response.CopyId = httpResponse.GetHeaders().at("x-ms-copy-id"); response.CopyStatus = CopyStatusFromString(httpResponse.GetHeaders().at("x-ms-copy-status")); + auto response_version_id_iterator = httpResponse.GetHeaders().find("x-ms-version-id"); + if (response_version_id_iterator != httpResponse.GetHeaders().end()) + { + response.VersionId = response_version_id_iterator->second; + } return Azure::Core::Response(std::move(response), std::move(pHttpResponse)); } @@ -7401,6 +7457,11 @@ namespace Azure { namespace Storage { namespace Blobs { { response.ContentCrc64 = response_content_crc64_iterator->second; } + auto response_version_id_iterator = httpResponse.GetHeaders().find("x-ms-version-id"); + if (response_version_id_iterator != httpResponse.GetHeaders().end()) + { + response.VersionId = response_version_id_iterator->second; + } auto response_server_encrypted_iterator = httpResponse.GetHeaders().find("x-ms-server-encrypted"); if (response_server_encrypted_iterator != httpResponse.GetHeaders().end()) diff --git a/sdk/storage/src/blobs/append_blob_client.cpp b/sdk/storage/src/blobs/append_blob_client.cpp index 5e18e869d..976a357b7 100644 --- a/sdk/storage/src/blobs/append_blob_client.cpp +++ b/sdk/storage/src/blobs/append_blob_client.cpp @@ -58,6 +58,20 @@ namespace Azure { namespace Storage { namespace Blobs { return newClient; } + AppendBlobClient AppendBlobClient::WithVersionId(const std::string& versionId) const + { + AppendBlobClient newClient(*this); + if (versionId.empty()) + { + newClient.m_blobUrl.RemoveQuery(Details::c_HttpQueryVersionId); + } + else + { + newClient.m_blobUrl.AppendQuery(Details::c_HttpQueryVersionId, versionId); + } + return newClient; + } + Azure::Core::Response AppendBlobClient::Create( const CreateAppendBlobOptions& options) { diff --git a/sdk/storage/src/blobs/blob_client.cpp b/sdk/storage/src/blobs/blob_client.cpp index ef60714b2..914b63988 100644 --- a/sdk/storage/src/blobs/blob_client.cpp +++ b/sdk/storage/src/blobs/blob_client.cpp @@ -135,6 +135,20 @@ namespace Azure { namespace Storage { namespace Blobs { return newClient; } + BlobClient BlobClient::WithVersionId(const std::string& versionId) const + { + BlobClient newClient(*this); + if (versionId.empty()) + { + newClient.m_blobUrl.RemoveQuery(Details::c_HttpQueryVersionId); + } + else + { + newClient.m_blobUrl.AppendQuery(Details::c_HttpQueryVersionId, versionId); + } + return newClient; + } + Azure::Core::Response BlobClient::Download( const DownloadBlobOptions& options) const { diff --git a/sdk/storage/src/blobs/blob_container_client.cpp b/sdk/storage/src/blobs/blob_container_client.cpp index 4063ee332..d03e4f0fe 100644 --- a/sdk/storage/src/blobs/blob_container_client.cpp +++ b/sdk/storage/src/blobs/blob_container_client.cpp @@ -186,8 +186,16 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.Marker = options.Marker; protocolLayerOptions.MaxResults = options.MaxResults; protocolLayerOptions.Include = options.Include; - return BlobRestClient::Container::ListBlobsFlat( + auto response = BlobRestClient::Container::ListBlobsFlat( options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions); + for (auto& i : response->Items) + { + if (i.VersionId.HasValue() && !i.IsCurrentVersion.HasValue()) + { + i.IsCurrentVersion = false; + } + } + return response; } Azure::Core::Response BlobContainerClient::ListBlobsByHierarchy( @@ -200,8 +208,16 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.Marker = options.Marker; protocolLayerOptions.MaxResults = options.MaxResults; protocolLayerOptions.Include = options.Include; - return BlobRestClient::Container::ListBlobsByHierarchy( + auto response = BlobRestClient::Container::ListBlobsByHierarchy( options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions); + for (auto& i : response->Items) + { + if (i.VersionId.HasValue() && !i.IsCurrentVersion.HasValue()) + { + i.IsCurrentVersion = false; + } + } + return response; } Azure::Core::Response BlobContainerClient::GetAccessPolicy( diff --git a/sdk/storage/src/blobs/blob_sas_builder.cpp b/sdk/storage/src/blobs/blob_sas_builder.cpp index 2cf6d50c7..60f8aed75 100644 --- a/sdk/storage/src/blobs/blob_sas_builder.cpp +++ b/sdk/storage/src/blobs/blob_sas_builder.cpp @@ -110,17 +110,28 @@ namespace Azure { namespace Storage { namespace Blobs { std::string BlobSasBuilder::ToSasQueryParameters(const SharedKeyCredential& credential) { std::string canonicalName = "/blob/" + credential.AccountName + "/" + ContainerName; - if (Resource == BlobSasResource::Blob || Resource == BlobSasResource::BlobSnapshot) + if (Resource == BlobSasResource::Blob || Resource == BlobSasResource::BlobSnapshot + || Resource == BlobSasResource::BlobVersion) { canonicalName += "/" + BlobName; } std::string protocol = SasProtocolToString(Protocol); std::string resource = BlobSasResourceToString(Resource); + std::string snapshotVersion; + if (Resource == BlobSasResource::BlobSnapshot) + { + snapshotVersion = Snapshot; + } + else if (Resource == BlobSasResource::BlobVersion) + { + snapshotVersion = BlobVersionId; + } + std::string stringToSign = Permissions + "\n" + (StartsOn.HasValue() ? StartsOn.GetValue() : "") + "\n" + ExpiresOn + "\n" + canonicalName + "\n" + Identifier + "\n" + (IPRange.HasValue() ? IPRange.GetValue() : "") + "\n" + protocol + "\n" + Version + "\n" - + resource + "\n" + Snapshot + "\n" + CacheControl + "\n" + ContentDisposition + "\n" + + resource + "\n" + snapshotVersion + "\n" + CacheControl + "\n" + ContentDisposition + "\n" + ContentEncoding + "\n" + ContentLanguage + "\n" + ContentType; std::string signature @@ -180,21 +191,32 @@ namespace Azure { namespace Storage { namespace Blobs { const std::string& accountName) { std::string canonicalName = "/blob/" + accountName + "/" + ContainerName; - if (Resource == BlobSasResource::Blob || Resource == BlobSasResource::BlobSnapshot) + if (Resource == BlobSasResource::Blob || Resource == BlobSasResource::BlobSnapshot + || Resource == BlobSasResource::BlobVersion) { canonicalName += "/" + BlobName; } std::string protocol = SasProtocolToString(Protocol); std::string resource = BlobSasResourceToString(Resource); + std::string snapshotVersion; + if (Resource == BlobSasResource::BlobSnapshot) + { + snapshotVersion = Snapshot; + } + else if (Resource == BlobSasResource::BlobVersion) + { + snapshotVersion = BlobVersionId; + } + std::string stringToSign = Permissions + "\n" + (StartsOn.HasValue() ? StartsOn.GetValue() : "") + "\n" + ExpiresOn + "\n" + canonicalName + "\n" + userDelegationKey.SignedObjectId + "\n" + userDelegationKey.SignedTenantId + "\n" + userDelegationKey.SignedStartsOn + "\n" + userDelegationKey.SignedExpiresOn + "\n" + userDelegationKey.SignedService + "\n" + userDelegationKey.SignedVersion + "\n" + (IPRange.HasValue() ? IPRange.GetValue() : "") - + "\n" + protocol + "\n" + Version + "\n" + resource + "\n" + Snapshot + "\n" + CacheControl - + "\n" + ContentDisposition + "\n" + ContentEncoding + "\n" + ContentLanguage + "\n" - + ContentType; + + "\n" + protocol + "\n" + Version + "\n" + resource + "\n" + snapshotVersion + "\n" + + CacheControl + "\n" + ContentDisposition + "\n" + ContentEncoding + "\n" + ContentLanguage + + "\n" + ContentType; std::string signature = Base64Encode(Hmac_Sha256(stringToSign, Base64Decode(userDelegationKey.Value))); diff --git a/sdk/storage/src/blobs/block_blob_client.cpp b/sdk/storage/src/blobs/block_blob_client.cpp index 746337608..88353e2af 100644 --- a/sdk/storage/src/blobs/block_blob_client.cpp +++ b/sdk/storage/src/blobs/block_blob_client.cpp @@ -61,6 +61,20 @@ namespace Azure { namespace Storage { namespace Blobs { return newClient; } + BlockBlobClient BlockBlobClient::WithVersionId(const std::string& versionId) const + { + BlockBlobClient newClient(*this); + if (versionId.empty()) + { + newClient.m_blobUrl.RemoveQuery(Details::c_HttpQueryVersionId); + } + else + { + newClient.m_blobUrl.AppendQuery(Details::c_HttpQueryVersionId, versionId); + } + return newClient; + } + Azure::Core::Response BlockBlobClient::Upload( Azure::Core::Http::BodyStream* content, const UploadBlockBlobOptions& options) const diff --git a/sdk/storage/src/blobs/page_blob_client.cpp b/sdk/storage/src/blobs/page_blob_client.cpp index f52dca3a0..d9bfdf612 100644 --- a/sdk/storage/src/blobs/page_blob_client.cpp +++ b/sdk/storage/src/blobs/page_blob_client.cpp @@ -56,6 +56,20 @@ namespace Azure { namespace Storage { namespace Blobs { return newClient; } + PageBlobClient PageBlobClient::WithVersionId(const std::string& versionId) const + { + PageBlobClient newClient(*this); + if (versionId.empty()) + { + newClient.m_blobUrl.RemoveQuery(Details::c_HttpQueryVersionId); + } + else + { + newClient.m_blobUrl.AppendQuery(Details::c_HttpQueryVersionId, versionId); + } + return newClient; + } + Azure::Core::Response PageBlobClient::Create( int64_t blobContentLength, const CreatePageBlobOptions& options) diff --git a/sdk/storage/test/blobs/append_blob_client_test.cpp b/sdk/storage/test/blobs/append_blob_client_test.cpp index 71b6aab9a..f43d1c35e 100644 --- a/sdk/storage/test/blobs/append_blob_client_test.cpp +++ b/sdk/storage/test/blobs/append_blob_client_test.cpp @@ -42,7 +42,11 @@ namespace Azure { namespace Storage { namespace Test { { auto appendBlobClient = Azure::Storage::Blobs::AppendBlobClient::CreateFromConnectionString( StandardStorageConnectionString(), m_containerName, RandomString()); - appendBlobClient.Create(m_blobUploadOptions); + auto blobContentInfo = appendBlobClient.Create(m_blobUploadOptions); + EXPECT_FALSE(blobContentInfo->ETag.empty()); + EXPECT_FALSE(blobContentInfo->LastModified.empty()); + EXPECT_TRUE(blobContentInfo->VersionId.HasValue()); + EXPECT_FALSE(blobContentInfo->VersionId.GetValue().empty()); auto properties = *appendBlobClient.GetProperties(); EXPECT_TRUE(properties.CommittedBlockCount.HasValue()); diff --git a/sdk/storage/test/blobs/blob_container_client_test.cpp b/sdk/storage/test/blobs/blob_container_client_test.cpp index e95f6242c..d3fdbab13 100644 --- a/sdk/storage/test/blobs/blob_container_client_test.cpp +++ b/sdk/storage/test/blobs/blob_container_client_test.cpp @@ -237,6 +237,74 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(items, blobs); } + TEST_F(BlobContainerClientTest, ListBlobsOtherStuff) + { + std::string blobName = RandomString(); + auto blobClient = m_blobContainerClient->GetAppendBlobClient(blobName); + blobClient.Create(); + blobClient.Delete(); + blobClient.Create(); + blobClient.CreateSnapshot(); + blobClient.SetMetadata({{"k1", "v1"}}); + std::vector content(1); + auto contentStream = Azure::Core::Http::MemoryBodyStream(content.data(), 1); + blobClient.AppendBlock(&contentStream); + + Azure::Storage::Blobs::ListBlobsOptions options; + options.Prefix = blobName; + options.Include = Blobs::ListBlobsIncludeItem::Snapshots | Blobs::ListBlobsIncludeItem::Versions + | Blobs::ListBlobsIncludeItem::Deleted | Blobs::ListBlobsIncludeItem::Metadata; + bool foundSnapshot = false; + bool foundVersions = false; + bool foundCurrentVersion = false; + bool foundNotCurrentVersion = false; + bool foundDeleted = false; + bool foundMetadata = false; + do + { + auto res = m_blobContainerClient->ListBlobsFlat(options); + options.Marker = res->NextMarker; + for (const auto& blob : res->Items) + { + if (!blob.Snapshot.empty()) + { + foundSnapshot = true; + } + if (blob.VersionId.HasValue()) + { + EXPECT_FALSE(blob.VersionId.GetValue().empty()); + foundVersions = true; + } + if (blob.IsCurrentVersion.HasValue()) + { + if (blob.IsCurrentVersion.GetValue()) + { + foundCurrentVersion = true; + } + else + { + foundNotCurrentVersion = true; + } + } + if (blob.Deleted) + { + foundDeleted = true; + } + if (!blob.Metadata.empty()) + { + foundMetadata = true; + } + } + } while (!options.Marker.GetValue().empty()); + EXPECT_TRUE(foundSnapshot); + EXPECT_TRUE(foundVersions); + EXPECT_TRUE(foundCurrentVersion); + EXPECT_TRUE(foundNotCurrentVersion); + // Blobs won't be listed as deleted once versioning is enabled + EXPECT_FALSE(foundDeleted); + EXPECT_TRUE(foundMetadata); + } + TEST_F(BlobContainerClientTest, AccessControlList) { auto container_client = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( diff --git a/sdk/storage/test/blobs/blob_sas_test.cpp b/sdk/storage/test/blobs/blob_sas_test.cpp index 3f7999379..8af16f65b 100644 --- a/sdk/storage/test/blobs/blob_sas_test.cpp +++ b/sdk/storage/test/blobs/blob_sas_test.cpp @@ -109,11 +109,6 @@ namespace Azure { namespace Storage { namespace Test { // TODO: Add test for blob tags }; - auto verify_blob_delete_version = [&](const std::string& sas) { - unused(sas); - // TODO: Add test for versions - }; - for (auto permissions : { AccountSasPermissions::All, AccountSasPermissions::Read, @@ -142,11 +137,6 @@ namespace Azure { namespace Storage { namespace Test { { verify_blob_delete(sasToken); } - if ((permissions & AccountSasPermissions::DeleteVersion) - == AccountSasPermissions::DeleteVersion) - { - verify_blob_delete_version(sasToken); - } if ((permissions & AccountSasPermissions::List) == AccountSasPermissions::List) { verify_blob_list(sasToken); @@ -213,12 +203,6 @@ namespace Azure { namespace Storage { namespace Test { verify_blob_tags(sasToken); verify_blob_tags(sasToken2); } - if ((permissions & Blobs::BlobSasPermissions::DeleteVersion) - == Blobs::BlobSasPermissions::DeleteVersion) - { - verify_blob_delete_version(sasToken); - verify_blob_delete_version(sasToken2); - } } accountSasBuilder.SetPermissions(AccountSasPermissions::All); @@ -482,6 +466,62 @@ namespace Azure { namespace Storage { namespace Test { verify_blob_snapshot_delete(sasToken2); } } + + blobClient0.Create(); + Blobs::BlobSasBuilder BlobVersionSasBuilder = blobSasBuilder; + BlobVersionSasBuilder.Resource = Blobs::BlobSasResource::BlobVersion; + + std::string blobVersionUri; + + auto create_version = [&]() { + std::string versionId = blobClient0.CreateSnapshot()->VersionId.GetValue(); + BlobVersionSasBuilder.BlobVersionId = versionId; + blobVersionUri = blobClient0.WithVersionId(versionId).GetUri(); + blobClient0.SetMetadata({}); + }; + + auto verify_blob_version_read = [&](const std::string sas) { + UriBuilder blobVersionUriWithSas(blobVersionUri); + blobVersionUriWithSas.AppendQueries(sas); + auto blobVersionClient = Blobs::AppendBlobClient(blobVersionUriWithSas.ToString()); + auto downloadedContent = blobVersionClient.Download(); + EXPECT_TRUE(ReadBodyStream(downloadedContent->BodyStream).empty()); + }; + + auto verify_blob_delete_version = [&](const std::string& sas) { + UriBuilder blobVersionUriWithSas(blobVersionUri); + blobVersionUriWithSas.AppendQueries(sas); + auto blobVersionClient = Blobs::AppendBlobClient(blobVersionUriWithSas.ToString()); + blobVersionClient.Delete(); + }; + + for (auto permissions : { + Blobs::BlobSasPermissions::Read | Blobs::BlobSasPermissions::DeleteVersion, + Blobs::BlobSasPermissions::Read, + Blobs::BlobSasPermissions::DeleteVersion, + }) + { + create_version(); + BlobVersionSasBuilder.SetPermissions(permissions); + auto sasToken = BlobVersionSasBuilder.ToSasQueryParameters(*keyCredential); + auto sasToken2 = BlobVersionSasBuilder.ToSasQueryParameters(userDelegationKey, accountName); + + if ((permissions & Blobs::BlobSasPermissions::Read) == Blobs::BlobSasPermissions::Read) + { + verify_blob_version_read(sasToken); + verify_blob_version_read(sasToken2); + } + if ((permissions & Blobs::BlobSasPermissions::DeleteVersion) + == Blobs::BlobSasPermissions::DeleteVersion) + { + create_version(); + sasToken = BlobVersionSasBuilder.ToSasQueryParameters(*keyCredential); + verify_blob_delete_version(sasToken); + create_version(); + sasToken2 = BlobVersionSasBuilder.ToSasQueryParameters(userDelegationKey, accountName); + verify_blob_delete_version(sasToken2); + } + } } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/test/blobs/block_blob_client_test.cpp b/sdk/storage/test/blobs/block_blob_client_test.cpp index bbafb9db6..2a8dfcaf0 100644 --- a/sdk/storage/test/blobs/block_blob_client_test.cpp +++ b/sdk/storage/test/blobs/block_blob_client_test.cpp @@ -64,13 +64,14 @@ namespace Azure { namespace Storage { namespace Test { StandardStorageConnectionString(), m_containerName, RandomString()); auto blobContent = Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - blockBlobClient.Upload(&blobContent, m_blobUploadOptions); + auto blobContentInfo = blockBlobClient.Upload(&blobContent, m_blobUploadOptions); + EXPECT_FALSE(blobContentInfo->ETag.empty()); + EXPECT_FALSE(blobContentInfo->LastModified.empty()); + EXPECT_TRUE(blobContentInfo->VersionId.HasValue()); + EXPECT_FALSE(blobContentInfo->VersionId.GetValue().empty()); blockBlobClient.Delete(); EXPECT_THROW(blockBlobClient.Delete(), StorageError); - blockBlobClient.Undelete(); - blockBlobClient.Delete(); - EXPECT_THROW(blockBlobClient.Delete(), StorageError); } TEST_F(BlockBlobClientTest, UploadDownload) @@ -138,6 +139,8 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(res->ETag.empty()); EXPECT_FALSE(res->LastModified.empty()); EXPECT_FALSE(res->CopyId.empty()); + EXPECT_TRUE(res->VersionId.HasValue()); + EXPECT_FALSE(res->VersionId.GetValue().empty()); EXPECT_TRUE( res->CopyStatus == Azure::Storage::Blobs::CopyStatus::Pending || res->CopyStatus == Azure::Storage::Blobs::CopyStatus::Success); @@ -154,7 +157,7 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(BlockBlobClientTest, SnapShot) + TEST_F(BlockBlobClientTest, SnapShotVersions) { auto res = m_blockBlobClient->CreateSnapshot(); EXPECT_FALSE(res.GetRawResponse().GetHeaders().at(Details::c_HttpHeaderRequestId).empty()); @@ -163,16 +166,31 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(res->ETag.empty()); EXPECT_FALSE(res->LastModified.empty()); EXPECT_FALSE(res->Snapshot.empty()); + EXPECT_TRUE(res->VersionId.HasValue()); + EXPECT_FALSE(res->VersionId.GetValue().empty()); auto snapshotClient = m_blockBlobClient->WithSnapshot(res->Snapshot); EXPECT_EQ(ReadBodyStream(snapshotClient.Download()->BodyStream), m_blobContent); EXPECT_EQ(snapshotClient.GetProperties()->Metadata, m_blobUploadOptions.Metadata); + auto versionClient = m_blockBlobClient->WithVersionId(res->VersionId.GetValue()); + EXPECT_EQ(ReadBodyStream(versionClient.Download()->BodyStream), m_blobContent); + EXPECT_EQ(versionClient.GetProperties()->Metadata, m_blobUploadOptions.Metadata); auto emptyContent = Azure::Core::Http::MemoryBodyStream(nullptr, 0); EXPECT_THROW(snapshotClient.Upload(&emptyContent), StorageError); EXPECT_THROW(snapshotClient.SetMetadata({}), StorageError); - EXPECT_THROW( - snapshotClient.SetAccessTier(Azure::Storage::Blobs::AccessTier::Cool), StorageError); + /* + This feature isn't GA yet. + EXPECT_NO_THROW(snapshotClient.SetAccessTier(Azure::Storage::Blobs::AccessTier::Cool)); + */ EXPECT_THROW( snapshotClient.SetHttpHeaders(Azure::Storage::Blobs::BlobHttpHeaders()), StorageError); + EXPECT_THROW(versionClient.Upload(&emptyContent), StorageError); + EXPECT_THROW(versionClient.SetMetadata({}), StorageError); + /* + This feature isn't GA yet + EXPECT_NO_THROW(versionClient.SetAccessTier(Azure::Storage::Blobs::AccessTier::Cool)); + */ + EXPECT_THROW( + versionClient.SetHttpHeaders(Azure::Storage::Blobs::BlobHttpHeaders()), StorageError); Azure::Storage::Blobs::CreateSnapshotOptions options; options.Metadata = {{"snapshotkey1", "snapshotvalue1"}, {"snapshotkey2", "SNAPSHOTVALUE2"}}; @@ -180,6 +198,10 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(res->Snapshot.empty()); snapshotClient = m_blockBlobClient->WithSnapshot(res->Snapshot); EXPECT_EQ(snapshotClient.GetProperties()->Metadata, options.Metadata); + + EXPECT_NO_THROW(snapshotClient.Delete()); + EXPECT_NO_THROW(versionClient.Delete()); + EXPECT_NO_THROW(m_blockBlobClient->GetProperties()); } TEST_F(BlockBlobClientTest, Properties) @@ -222,8 +244,12 @@ namespace Azure { namespace Storage { namespace Test { Azure::Storage::Blobs::CommitBlockListOptions options; options.HttpHeaders = m_blobUploadOptions.HttpHeaders; options.Metadata = m_blobUploadOptions.Metadata; - blockBlobClient.CommitBlockList( + auto blobContentInfo = blockBlobClient.CommitBlockList( {{Azure::Storage::Blobs::BlockType::Uncommitted, blockId1}}, options); + EXPECT_FALSE(blobContentInfo->ETag.empty()); + EXPECT_FALSE(blobContentInfo->LastModified.empty()); + EXPECT_TRUE(blobContentInfo->VersionId.HasValue()); + EXPECT_FALSE(blobContentInfo->VersionId.GetValue().empty()); auto res = blockBlobClient.GetBlockList(); EXPECT_FALSE(res.GetRawResponse().GetHeaders().at(Details::c_HttpHeaderRequestId).empty()); EXPECT_FALSE(res.GetRawResponse().GetHeaders().at(Details::c_HttpHeaderDate).empty()); diff --git a/sdk/storage/test/blobs/page_blob_client_test.cpp b/sdk/storage/test/blobs/page_blob_client_test.cpp index 81423df2b..8a88dc873 100644 --- a/sdk/storage/test/blobs/page_blob_client_test.cpp +++ b/sdk/storage/test/blobs/page_blob_client_test.cpp @@ -42,7 +42,11 @@ namespace Azure { namespace Storage { namespace Test { { auto pageBlobClient = Azure::Storage::Blobs::PageBlobClient::CreateFromConnectionString( StandardStorageConnectionString(), m_containerName, RandomString()); - pageBlobClient.Create(0, m_blobUploadOptions); + auto blobContentInfo = pageBlobClient.Create(0, m_blobUploadOptions); + EXPECT_FALSE(blobContentInfo->ETag.empty()); + EXPECT_FALSE(blobContentInfo->LastModified.empty()); + EXPECT_TRUE(blobContentInfo->VersionId.HasValue()); + EXPECT_FALSE(blobContentInfo->VersionId.GetValue().empty()); pageBlobClient.Delete(); EXPECT_THROW(pageBlobClient.Delete(), StorageError); @@ -135,7 +139,13 @@ namespace Azure { namespace Storage { namespace Test { std::string snapshot = m_pageBlobClient->CreateSnapshot()->Snapshot; UriBuilder sourceUri(m_pageBlobClient->WithSnapshot(snapshot).GetUri()); sourceUri.AppendQueries(GetSas()); - pageBlobClient.StartCopyIncremental(sourceUri.ToString()); + auto copyInfo = pageBlobClient.StartCopyIncremental(sourceUri.ToString()); + EXPECT_FALSE(copyInfo->ETag.empty()); + EXPECT_FALSE(copyInfo->LastModified.empty()); + EXPECT_FALSE(copyInfo->CopyId.empty()); + EXPECT_NE(copyInfo->CopyStatus, Blobs::CopyStatus::Unknown); + EXPECT_TRUE(copyInfo->VersionId.HasValue()); + EXPECT_FALSE(copyInfo->VersionId.GetValue().empty()); } TEST_F(PageBlobClientTest, Lease)