From 8c4c4e75a968043e2925b3be631316ec11c1fb39 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Wed, 30 Sep 2020 11:38:25 +0800 Subject: [PATCH] [Storage Blobs Service] STG 74 features (#652) * SetExpiry protocol layer * add test * last access time * upgrade blob sas to 2020-02-10 * datalake sas * fix build error * tag count * enable tag test * fix build error * enable container soft delete test --- .../blobs/protocol/blob_rest_client.hpp | 168 ++++++- .../src/blob_sas_builder.cpp | 8 +- .../test/blob_container_client_test.cpp | 35 +- .../test/blob_sas_test.cpp | 6 - .../test/block_blob_client_test.cpp | 36 ++ .../inc/azure/storage/common/constants.hpp | 2 +- .../storage/common/storage_credential.hpp | 12 +- .../CMakeLists.txt | 3 + .../files/datalake/datalake_responses.hpp | 3 +- .../files/datalake/datalake_sas_builder.hpp | 344 ++++++++++++++ .../src/datalake_sas_builder.cpp | 269 +++++++++++ .../test/datalake_sas_test.cpp | 444 ++++++++++++++++++ 12 files changed, 1304 insertions(+), 26 deletions(-) create mode 100644 sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_sas_builder.hpp create mode 100644 sdk/storage/azure-storage-files-datalake/src/datalake_sas_builder.cpp create mode 100644 sdk/storage/azure-storage-files-datalake/test/datalake_sas_test.cpp diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/protocol/blob_rest_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/protocol/blob_rest_client.hpp index 20770d3d6..6d7157e19 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/protocol/blob_rest_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/protocol/blob_rest_client.hpp @@ -23,7 +23,7 @@ namespace Azure { namespace Storage { namespace Blobs { - constexpr static const char* c_ApiVersion = "2019-12-12"; + constexpr static const char* c_ApiVersion = "2020-02-10"; struct AbortCopyBlobFromUriResult { @@ -1156,6 +1156,62 @@ namespace Azure { namespace Storage { namespace Blobs { int64_t SequenceNumber = 0; }; // struct ResizePageBlobResult + enum class ScheduleBlobExpiryOriginType + { + Unknown, + NeverExpire, + RelativeToCreation, + RelativeToNow, + Absolute, + }; // enum class ScheduleBlobExpiryOriginType + + inline std::string ScheduleBlobExpiryOriginTypeToString( + const ScheduleBlobExpiryOriginType& schedule_blob_expiry_origin_type) + { + switch (schedule_blob_expiry_origin_type) + { + case ScheduleBlobExpiryOriginType::Unknown: + return ""; + case ScheduleBlobExpiryOriginType::NeverExpire: + return "NeverExpire"; + case ScheduleBlobExpiryOriginType::RelativeToCreation: + return "RelativeToCreation"; + case ScheduleBlobExpiryOriginType::RelativeToNow: + return "RelativeToNow"; + case ScheduleBlobExpiryOriginType::Absolute: + return "Absolute"; + default: + return std::string(); + } + } + + inline ScheduleBlobExpiryOriginType ScheduleBlobExpiryOriginTypeFromString( + const std::string& schedule_blob_expiry_origin_type) + { + if (schedule_blob_expiry_origin_type == "") + { + return ScheduleBlobExpiryOriginType::Unknown; + } + if (schedule_blob_expiry_origin_type == "NeverExpire") + { + return ScheduleBlobExpiryOriginType::NeverExpire; + } + if (schedule_blob_expiry_origin_type == "RelativeToCreation") + { + return ScheduleBlobExpiryOriginType::RelativeToCreation; + } + if (schedule_blob_expiry_origin_type == "RelativeToNow") + { + return ScheduleBlobExpiryOriginType::RelativeToNow; + } + if (schedule_blob_expiry_origin_type == "Absolute") + { + return ScheduleBlobExpiryOriginType::Absolute; + } + throw std::runtime_error( + "cannot convert " + schedule_blob_expiry_origin_type + " to ScheduleBlobExpiryOriginType"); + } + struct SealAppendBlobResult { std::string ETag; @@ -1167,6 +1223,10 @@ namespace Azure { namespace Storage { namespace Blobs { { }; // struct SetBlobAccessTierResult + struct SetBlobExpiryResult + { + }; // struct SetBlobExpiryResult + struct SetBlobHttpHeadersResult { std::string ETag; @@ -1515,6 +1575,8 @@ namespace Azure { namespace Storage { namespace Blobs { BlobHttpHeaders HttpHeaders; std::map Metadata; std::string CreationTime; + Azure::Core::Nullable ExpiryTime; + Azure::Core::Nullable LastAccessTime; std::string LastModified; std::string ETag; int64_t ContentLength = 0; @@ -1538,6 +1600,9 @@ namespace Azure { namespace Storage { namespace Blobs { std::unique_ptr BodyStream; std::string ETag; std::string LastModified; + std::string CreationTime; + Azure::Core::Nullable ExpiryTime; + Azure::Core::Nullable LastAccessTime; Azure::Core::Nullable ContentRange; BlobHttpHeaders HttpHeaders; std::map Metadata; @@ -1557,6 +1622,7 @@ namespace Azure { namespace Storage { namespace Blobs { ObjectReplicationDestinationPolicyId; // only valid for replication destination blob std::vector ObjectReplicationSourceProperties; // only valid for replication source blob + Azure::Core::Nullable TagCount; }; // struct DownloadBlobResult struct GetBlobPropertiesResult @@ -1564,6 +1630,8 @@ namespace Azure { namespace Storage { namespace Blobs { std::string ETag; std::string LastModified; std::string CreationTime; + Azure::Core::Nullable ExpiryTime; + Azure::Core::Nullable LastAccessTime; std::map Metadata; Blobs::BlobType BlobType = Blobs::BlobType::Unknown; Azure::Core::Nullable LeaseDuration; @@ -1590,6 +1658,7 @@ namespace Azure { namespace Storage { namespace Blobs { ObjectReplicationDestinationPolicyId; // only valid for replication destination blob std::vector ObjectReplicationSourceProperties; // only valid for replication source blob + Azure::Core::Nullable TagCount; }; // struct GetBlobPropertiesResult struct ListBlobsByHierarchySegmentResult @@ -4468,6 +4537,8 @@ namespace Azure { namespace Storage { namespace Blobs { k_CacheControl, k_ContentDisposition, k_CreationTime, + k_ExpiryTime, + k_LastAccessTime, k_LastModified, k_Etag, k_ContentLength, @@ -4558,6 +4629,14 @@ namespace Azure { namespace Storage { namespace Blobs { { path.emplace_back(XmlTagName::k_CreationTime); } + else if (std::strcmp(node.Name, "Expiry-Time") == 0) + { + path.emplace_back(XmlTagName::k_ExpiryTime); + } + else if (std::strcmp(node.Name, "LastAccessTime") == 0) + { + path.emplace_back(XmlTagName::k_LastAccessTime); + } else if (std::strcmp(node.Name, "Last-Modified") == 0) { path.emplace_back(XmlTagName::k_LastModified); @@ -4698,6 +4777,18 @@ namespace Azure { namespace Storage { namespace Blobs { { ret.CreationTime = node.Value; } + else if ( + path.size() == 2 && path[0] == XmlTagName::k_Properties + && path[1] == XmlTagName::k_ExpiryTime) + { + ret.ExpiryTime = node.Value; + } + else if ( + path.size() == 2 && path[0] == XmlTagName::k_Properties + && path[1] == XmlTagName::k_LastAccessTime) + { + ret.LastAccessTime = node.Value; + } else if ( path.size() == 2 && path[0] == XmlTagName::k_Properties && path[1] == XmlTagName::k_LastModified) @@ -5231,6 +5322,18 @@ namespace Azure { namespace Storage { namespace Blobs { { response.LeaseDuration = response_lease_duration_iterator->second; } + response.CreationTime = httpResponse.GetHeaders().at("x-ms-creation-time"); + auto response_expiry_time_iterator = httpResponse.GetHeaders().find("x-ms-expiry-time"); + if (response_expiry_time_iterator != httpResponse.GetHeaders().end()) + { + response.ExpiryTime = response_expiry_time_iterator->second; + } + auto response_last_access_time_iterator + = httpResponse.GetHeaders().find("x-ms-last-access-time"); + if (response_last_access_time_iterator != httpResponse.GetHeaders().end()) + { + response.LastAccessTime = response_last_access_time_iterator->second; + } auto response_content_range_iterator = httpResponse.GetHeaders().find("content-range"); if (response_content_range_iterator != httpResponse.GetHeaders().end()) { @@ -5291,6 +5394,11 @@ namespace Azure { namespace Storage { namespace Blobs { response.ObjectReplicationSourceProperties.emplace_back(std::move(policy)); } } + auto response_tag_count_iterator = httpResponse.GetHeaders().find("x-ms-tag-count"); + if (response_tag_count_iterator != httpResponse.GetHeaders().end()) + { + response.TagCount = std::stoi(response_tag_count_iterator->second); + } return Azure::Core::Response( std::move(response), std::move(pHttpResponse)); } @@ -5380,6 +5488,48 @@ namespace Azure { namespace Storage { namespace Blobs { return DeleteCreateResponse(context, std::move(pHttpResponse)); } + struct SetBlobExpiryOptions + { + Azure::Core::Nullable Timeout; + ScheduleBlobExpiryOriginType ExpiryOrigin; + Azure::Core::Nullable ExpiryTime; + }; // struct SetBlobExpiryOptions + + static Azure::Core::Response ScheduleDeletion( + const Azure::Core::Context& context, + Azure::Core::Http::HttpPipeline& pipeline, + const Azure::Core::Http::Url& url, + const SetBlobExpiryOptions& options) + { + unused(options); + auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); + request.AddHeader("Content-Length", "0"); + request.AddHeader("x-ms-version", c_ApiVersion); + if (options.Timeout.HasValue()) + { + request.GetUrl().AppendQuery("timeout", std::to_string(options.Timeout.GetValue())); + } + request.GetUrl().AppendQuery("comp", "expiry"); + request.AddHeader( + "x-ms-expiry-option", ScheduleBlobExpiryOriginTypeToString(options.ExpiryOrigin)); + if (options.ExpiryTime.HasValue()) + { + request.AddHeader("x-ms-expiry-time", options.ExpiryTime.GetValue()); + } + auto pHttpResponse = pipeline.Send(context, request); + Azure::Core::Http::RawResponse& httpResponse = *pHttpResponse; + SetBlobExpiryResult response; + auto http_status_code + = static_cast::type>( + httpResponse.GetStatusCode()); + if (!(http_status_code == 200)) + { + throw StorageError::CreateFromResponse(std::move(pHttpResponse)); + } + return Azure::Core::Response( + std::move(response), std::move(pHttpResponse)); + } + struct UndeleteBlobOptions { Azure::Core::Nullable Timeout; @@ -5492,6 +5642,17 @@ namespace Azure { namespace Storage { namespace Blobs { response.ETag = httpResponse.GetHeaders().at("etag"); response.LastModified = httpResponse.GetHeaders().at("last-modified"); response.CreationTime = httpResponse.GetHeaders().at("x-ms-creation-time"); + auto response_expiry_time_iterator = httpResponse.GetHeaders().find("x-ms-expiry-time"); + if (response_expiry_time_iterator != httpResponse.GetHeaders().end()) + { + response.ExpiryTime = response_expiry_time_iterator->second; + } + auto response_last_access_time_iterator + = httpResponse.GetHeaders().find("x-ms-last-access-time"); + if (response_last_access_time_iterator != httpResponse.GetHeaders().end()) + { + response.LastAccessTime = response_last_access_time_iterator->second; + } for (auto i = httpResponse.GetHeaders().lower_bound("x-ms-meta-"); i != httpResponse.GetHeaders().end() && i->first.substr(0, 10) == "x-ms-meta-"; ++i) @@ -5682,6 +5843,11 @@ namespace Azure { namespace Storage { namespace Blobs { response.ObjectReplicationSourceProperties.emplace_back(std::move(policy)); } } + auto response_tag_count_iterator = httpResponse.GetHeaders().find("x-ms-tag-count"); + if (response_tag_count_iterator != httpResponse.GetHeaders().end()) + { + response.TagCount = std::stoi(response_tag_count_iterator->second); + } return Azure::Core::Response( std::move(response), std::move(pHttpResponse)); } diff --git a/sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp b/sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp index 8512068a2..c5ebc9172 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp @@ -213,10 +213,10 @@ namespace Azure { namespace Storage { namespace Blobs { + "\n" + ExpiresOn + "\n" + canonicalName + "\n" + userDelegationKey.SignedObjectId + "\n" + userDelegationKey.SignedTenantId + "\n" + userDelegationKey.SignedStartsOn + "\n" + userDelegationKey.SignedExpiresOn + "\n" + userDelegationKey.SignedService + "\n" - + userDelegationKey.SignedVersion + "\n" + (IPRange.HasValue() ? IPRange.GetValue() : "") - + "\n" + protocol + "\n" + Version + "\n" + resource + "\n" + snapshotVersion + "\n" - + CacheControl + "\n" + ContentDisposition + "\n" + ContentEncoding + "\n" + ContentLanguage - + "\n" + ContentType; + + userDelegationKey.SignedVersion + "\n\n\n\n" + + (IPRange.HasValue() ? IPRange.GetValue() : "") + "\n" + protocol + "\n" + Version + "\n" + + resource + "\n" + snapshotVersion + "\n" + CacheControl + "\n" + ContentDisposition + "\n" + + ContentEncoding + "\n" + ContentLanguage + "\n" + ContentType; std::string signature = Base64Encode(Details::HmacSha256(stringToSign, Base64Decode(userDelegationKey.Value))); diff --git a/sdk/storage/azure-storage-blobs/test/blob_container_client_test.cpp b/sdk/storage/azure-storage-blobs/test/blob_container_client_test.cpp index 5735590d7..81460c340 100644 --- a/sdk/storage/azure-storage-blobs/test/blob_container_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/blob_container_client_test.cpp @@ -669,7 +669,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(containerClient.Delete(options)); } - TEST_F(BlobContainerClientTest, DISABLED_Undelete) + TEST_F(BlobContainerClientTest, Undelete) { auto serviceClient = Azure::Storage::Blobs::BlobServiceClient::CreateFromConnectionString( StandardStorageConnectionString()); @@ -732,19 +732,25 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(containerClient2.GetProperties()); } - TEST_F(BlobContainerClientTest, DISABLED_Tags) + TEST_F(BlobContainerClientTest, Tags) { std::string blobName = RandomString(); auto blobClient = Azure::Storage::Blobs::AppendBlobClient::CreateFromConnectionString( StandardStorageConnectionString(), m_containerName, blobName); blobClient.Create(); + auto properties = *blobClient.GetProperties(); + EXPECT_FALSE(properties.TagCount.HasValue()); + + auto downloadRet = blobClient.Download(); + EXPECT_FALSE(downloadRet->TagCount.HasValue()); + std::map tags; - std::string c1 = RandomString(); + std::string c1 = "k" + RandomString(); std::string v1 = RandomString(); - std::string c2 = RandomString(); + std::string c2 = "k" + RandomString(); std::string v2 = RandomString(); - std::string c3 = RandomString(); + std::string c3 = "k" + RandomString(); std::string v3 = RandomString(); tags[c1] = v1; tags[c2] = v2; @@ -756,9 +762,18 @@ namespace Azure { namespace Storage { namespace Test { downloadedTags = blobClient.GetTags()->Tags; EXPECT_EQ(downloadedTags, tags); + properties = *blobClient.GetProperties(); + EXPECT_TRUE(properties.TagCount.HasValue()); + EXPECT_EQ(properties.TagCount.GetValue(), static_cast(tags.size())); + + downloadRet = blobClient.Download(); + EXPECT_TRUE(downloadRet->TagCount.HasValue()); + EXPECT_EQ(downloadRet->TagCount.GetValue(), static_cast(tags.size())); + auto blobServiceClient = Azure::Storage::Blobs::BlobServiceClient::CreateFromConnectionString( StandardStorageConnectionString()); - std::string whereExpression = c1 + " = '" + v1 + "' AND " + c2 + " >= '" + v2 + "'"; + std::string whereExpression + = c1 + " = '" + v1 + "' AND " + c2 + " >= '" + v2 + "' AND " + c3 + " <= '" + v3 + "'"; std::string marker; std::vector findResults; do @@ -787,15 +802,15 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(findResults[0].TagValue.empty()); } - TEST_F(BlobContainerClientTest, DISABLED_AccessConditionTags) + TEST_F(BlobContainerClientTest, AccessConditionTags) { std::map tags; - std::string c1 = RandomString(); + std::string c1 = "k" + RandomString(); std::string v1 = RandomString(); tags[c1] = v1; - std::string successWhereExpression = "\"" + c1 + "\" = '" + v1 + "'"; - std::string failWhereExpression = "\"" + c1 + "\" != '" + v1 + "'"; + std::string successWhereExpression = c1 + " = '" + v1 + "'"; + std::string failWhereExpression = c1 + " != '" + v1 + "'"; std::vector contentData(512); int64_t contentSize = static_cast(contentData.size()); diff --git a/sdk/storage/azure-storage-blobs/test/blob_sas_test.cpp b/sdk/storage/azure-storage-blobs/test/blob_sas_test.cpp index 64cc6fcae..2b1d896ae 100644 --- a/sdk/storage/azure-storage-blobs/test/blob_sas_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/blob_sas_test.cpp @@ -100,22 +100,16 @@ namespace Azure { namespace Storage { namespace Test { }; auto verify_blob_tags = [&](const std::string& sas) { - unused(sas); - /* blobClient0.Create(); std::map tags = {{"tag_key1", "tag_value1"}}; blobClient0.SetTags(tags); auto blobClient = Blobs::AppendBlobClient(blobUri + sas); EXPECT_NO_THROW(blobClient.GetTags()); - */ }; auto verify_blob_filter = [&](const std::string& sas) { - unused(sas); - /* auto serviceClient = Blobs::BlobServiceClient(serviceUri + sas); EXPECT_NO_THROW(serviceClient.FindBlobsByTags("\"tag_key1\" = 'tag_value1'")); - */ }; for (auto permissions : { diff --git a/sdk/storage/azure-storage-blobs/test/block_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/block_blob_client_test.cpp index c2c4b2074..f688ff02e 100644 --- a/sdk/storage/azure-storage-blobs/test/block_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/block_blob_client_test.cpp @@ -85,6 +85,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(res.GetRawResponse().GetHeaders().at(Details::c_HttpHeaderXMsVersion).empty()); EXPECT_FALSE(res->ETag.empty()); EXPECT_FALSE(res->LastModified.empty()); + EXPECT_FALSE(res->CreationTime.empty()); EXPECT_EQ(res->HttpHeaders, m_blobUploadOptions.HttpHeaders); EXPECT_EQ(res->Metadata, m_blobUploadOptions.Metadata); EXPECT_EQ(res->BlobType, Azure::Storage::Blobs::BlobType::BlockBlob); @@ -101,6 +102,41 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(res->ContentRange.GetValue().empty()); } + TEST_F(BlockBlobClientTest, LastAccessTime) + { + { + auto res = m_blockBlobClient->Download(); + ASSERT_TRUE(res->LastAccessTime.HasValue()); + EXPECT_FALSE(res->LastAccessTime.GetValue().empty()); + } + { + auto res = m_blockBlobClient->GetProperties(); + ASSERT_TRUE(res->LastAccessTime.HasValue()); + EXPECT_FALSE(res->LastAccessTime.GetValue().empty()); + } + { + std::string lastAccessTime; + + Azure::Storage::Blobs::ListBlobsSegmentOptions options; + options.Prefix = m_blobName; + do + { + auto res = m_blobContainerClient->ListBlobsFlatSegment(options); + options.Marker = res->NextMarker; + for (const auto& blob : res->Items) + { + if (blob.Name == m_blobName) + { + lastAccessTime = blob.LastAccessTime.GetValue(); + break; + } + } + } while (!options.Marker.GetValue().empty()); + + EXPECT_FALSE(lastAccessTime.empty()); + } + } + TEST_F(BlockBlobClientTest, DownloadEmpty) { std::vector emptyContent; diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/constants.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/constants.hpp index cb4e89174..06b8f7c5c 100644 --- a/sdk/storage/azure-storage-common/inc/azure/storage/common/constants.hpp +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/constants.hpp @@ -17,7 +17,7 @@ namespace Azure { namespace Storage { namespace Details { constexpr static const char* c_HttpHeaderRequestId = "x-ms-request-id"; constexpr static const char* c_HttpHeaderClientRequestId = "x-ms-client-request-id"; constexpr static const char* c_HttpHeaderContentType = "content-type"; - constexpr static const char* c_defaultSasVersion = "2019-12-12"; + constexpr static const char* c_defaultSasVersion = "2020-02-10"; constexpr int c_reliableStreamRetryCount = 3; }}} // namespace Azure::Storage::Details diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/storage_credential.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/storage_credential.hpp index 7751be1cd..823bc87d6 100644 --- a/sdk/storage/azure-storage-common/inc/azure/storage/common/storage_credential.hpp +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/storage_credential.hpp @@ -16,9 +16,14 @@ namespace Azure { namespace Storage { namespace Blobs { struct BlobSasBuilder; } - namespace Files { namespace Shares { - struct ShareSasBuilder; - }} // namespace Files::Shares + namespace Files { + namespace Shares { + struct ShareSasBuilder; + } + namespace DataLake { + struct DataLakeSasBuilder; + } + } // namespace Files struct SharedKeyCredential { @@ -39,6 +44,7 @@ namespace Azure { namespace Storage { friend class SharedKeyPolicy; friend struct Blobs::BlobSasBuilder; friend struct Files::Shares::ShareSasBuilder; + friend struct Files::DataLake::DataLakeSasBuilder; friend struct AccountSasBuilder; std::string GetAccountKey() const { diff --git a/sdk/storage/azure-storage-files-datalake/CMakeLists.txt b/sdk/storage/azure-storage-files-datalake/CMakeLists.txt index 4415be278..50e387421 100644 --- a/sdk/storage/azure-storage-files-datalake/CMakeLists.txt +++ b/sdk/storage/azure-storage-files-datalake/CMakeLists.txt @@ -11,6 +11,7 @@ set (AZURE_STORAGE_DATALAKE_HEADER inc/azure/storage/files/datalake/datalake_options.hpp inc/azure/storage/files/datalake/datalake_path_client.hpp inc/azure/storage/files/datalake/datalake_responses.hpp + inc/azure/storage/files/datalake/datalake_sas_builder.hpp inc/azure/storage/files/datalake/datalake_service_client.hpp inc/azure/storage/files/datalake/datalake_utilities.hpp inc/azure/storage/files/datalake/protocol/datalake_rest_client.hpp @@ -22,6 +23,7 @@ set (AZURE_STORAGE_DATALAKE_SOURCE src/datalake_file_system_client.cpp src/datalake_path_client.cpp src/datalake_responses.cpp + src/datalake_sas_builder.cpp src/datalake_service_client.cpp src/datalake_utilities.cpp ) @@ -52,6 +54,7 @@ target_sources( test/datalake_file_system_client_test.hpp test/datalake_path_client_test.cpp test/datalake_path_client_test.hpp + test/datalake_sas_test.cpp test/datalake_service_client_test.cpp test/datalake_service_client_test.hpp ) diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index 58b407ab5..4e6be0ec1 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -3,7 +3,7 @@ #pragma once -#include "azure/storage/blobs/protocol/blob_rest_client.hpp" +#include "azure/storage/blobs/blob_responses.hpp" #include "azure/storage/files/datalake/protocol/datalake_rest_client.hpp" namespace Azure { namespace Storage { namespace Files { namespace DataLake { @@ -11,6 +11,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { // ServiceClient models: using GetUserDelegationKeyResult = Blobs::GetUserDelegationKeyResult; + using UserDelegationKey = Blobs::UserDelegationKey; using ListFileSystemsSegmentResult = ServiceListFileSystemsResult; // FileSystemClient models: diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_sas_builder.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_sas_builder.hpp new file mode 100644 index 000000000..eb75aa126 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_sas_builder.hpp @@ -0,0 +1,344 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +#include "azure/core/nullable.hpp" +#include "azure/storage/common/account_sas_builder.hpp" +#include "azure/storage/files/datalake/datalake_responses.hpp" + +namespace Azure { namespace Storage { namespace Files { namespace DataLake { + + /** + * @brief Specifies which resources are accessible via the shared access signature. + */ + enum class DataLakeSasResource + { + /** + * @brief Grants access to the content and metadata of any files and directories in the + * filesystem, and to the list of files and directories in the filesystem. + */ + FileSystem, + + /** + * @brief Grants access to the content and metadata of the file. + */ + File, + + /** + * @brief grants access to the files and subdirectories in the directory and to list the paths + * in the directory. + */ + Directory, + }; + + /** + * @brief The list of permissions that can be set for a filesystem's access policy. + */ + enum class DataLakeFileSystemSasPermissions + { + /** + * @brief Indicates that Read is permitted. + */ + Read = 1, + + /** + * @brief Indicates that Write is permitted. + */ + Write = 2, + + /** + * @brief Indicates that Delete is permitted. + */ + Delete = 4, + + /** + * @brief Indicates that List is permitted. + */ + List = 8, + + /** + * @brief Indicates that Add is permitted. + */ + Add = 16, + + /** + * @brief Indicates that Create is permitted. + */ + Create = 32, + + /** + * @beirf Indicates that all permissions are set. + */ + All = ~0, + }; + + inline DataLakeFileSystemSasPermissions operator|( + DataLakeFileSystemSasPermissions lhs, + DataLakeFileSystemSasPermissions rhs) + { + using type = std::underlying_type_t; + return static_cast( + static_cast(lhs) | static_cast(rhs)); + } + + inline DataLakeFileSystemSasPermissions operator&( + DataLakeFileSystemSasPermissions lhs, + DataLakeFileSystemSasPermissions rhs) + { + using type = std::underlying_type_t; + return static_cast( + static_cast(lhs) & static_cast(rhs)); + } + + std::string DataLakeFileSystemSasPermissionsToString( + DataLakeFileSystemSasPermissions permissions); + + /** + * @brief The list of permissions that can be set for a file or directory's access policy. + */ + enum class DataLakeSasPermissions + { + /** + * @brief Indicates that Read is permitted. + */ + Read = 1, + + /** + * @brief Indicates that Write is permitted. + */ + Write = 2, + + /** + * @brief Indicates that Delete is permitted. + */ + + Delete = 4, + + /** + * @brief Indicates that Add is permitted. + */ + Add = 8, + + /** + * @brief Indicates that Create is permitted. + */ + Create = 16, + + /** + * @brief Indicates that List is permitted. + */ + List = 32, + + /** + * @brief Allows the caller to move a blob (file or directory) to a new location. + */ + Move = 64, + + /** + * @brief Allows the caller to get system properties and POSIX ACLs of a blob (file or + * directory). + */ + Execute = 128, + + /** + * @brief Allows the caller to set owner, owning group, or act as the owner when renaming or + * deleting a blob (file or directory) within a folder that has the sticky bit set. + */ + ManageOwnership = 256, + + /** + * @brief Allows the caller to set permissions and POSIX ACLs on blobs (files and directories). + */ + ManageAccessControl = 512, + + /** + * @beirf Indicates that all permissions are set. + */ + All = ~0, + }; + + inline DataLakeSasPermissions operator|(DataLakeSasPermissions lhs, DataLakeSasPermissions rhs) + { + using type = std::underlying_type_t; + return static_cast(static_cast(lhs) | static_cast(rhs)); + } + + inline DataLakeSasPermissions operator&(DataLakeSasPermissions lhs, DataLakeSasPermissions rhs) + { + using type = std::underlying_type_t; + return static_cast(static_cast(lhs) & static_cast(rhs)); + } + + /** + * @brief DataLakeSasBuilder is used to generate a Shared Access Signature (SAS) for an Azure + * Storage DataLake filesystem or path. + */ + struct DataLakeSasBuilder + { + /** + * @brief The storage service version to use to authenticate requests made with this + * shared access signature, and the service version to use when handling requests made with this + * shared access signature. + */ + std::string Version = Storage::Details::c_defaultSasVersion; + + /** + * @brief The optional signed protocol field specifies the protocol permitted for a + * request made with the SAS. + */ + SasProtocol Protocol; + + /** + * @brief Optionally specify the time at which the shared access signature becomes + * valid. + */ + Azure::Core::Nullable StartsOn; + + /** + * @brief The time at which the shared access signature becomes invalid. This field must + * be omitted if it has been specified in an associated stored access policy. + */ + std::string ExpiresOn; + + /** + * @brief Specifies an IP address or a range of IP addresses from which to accept + * requests. If the IP address from which the request originates does not match the IP address + * or address range specified on the SAS token, the request is not authenticated. When + * specifying a range of IP addresses, note that the range is inclusive. + */ + Azure::Core::Nullable IPRange; + + /** + * @brief An optional unique value up to 64 characters in length that correlates to an + * access policy specified for the filesystem. + */ + std::string Identifier; + + /** + * @brief The name of the filesystem being made accessible. + */ + std::string FileSystemName; + + /** + * @brief The name of the path being made accessible, or empty for a filesystem SAS. + */ + std::string Path; + + /** + * @brief Defines whether ornot the Path is a directory. If this value is set to true, the Path + * is a directory for a directory SAS. If set to false or default, the Path is a file for a file + * SAS. + */ + bool IsDirectory = false; + + /** + * @brief Required when Resource is set to Directory to indicate the depth of the directory + * specified in the canonicalized resource field of the string-to-sign to indicate the depth of + * the directory specified in the canonicalized resource field of the string-to-sign. This is + * only used for user delegation SAS. + */ + Azure::Core::Nullable DirectoryDepth; + + /** + * @brief Specifies which resources are accessible via the shared access signature. + */ + DataLakeSasResource Resource; + + /** + * @brief Override the value returned for Cache-Control response header. + */ + std::string CacheControl; + + /** + * @brief Override the value returned for Content-Disposition response header. + */ + std::string ContentDisposition; + + /** + * @brief Override the value returned for Content-Encoding response header. + */ + std::string ContentEncoding; + + /** + * @brief Override the value returned for Content-Language response header. + */ + std::string ContentLanguage; + + /** + * @brief Override the value returned for Content-Type response header. + */ + std::string ContentType; + + /** + * @brief This value will be used for the AAD Object ID of a user authorized by the owner of the + * User Delegation Key to perform the action granted by the SAS. The Azure Storage service will + * ensure that the owner of the user delegation key has the required permissions before granting + * access. No additional permission check for the user specified in this value will be + * performed. This cannot be used in conjuction with AgentObjectId. This is only used with + * generating User Delegation SAS. + */ + std::string PreauthorizedAgentObjectId; + + /** + * @brief This value will be used for the AAD Object ID of a user authorized by the owner of the + * User Delegation Key to perform the action granted by the SAS. The Azure Storage service will + * ensure that the owner of the user delegation key has the required permissions before granting + * access. The Azure Storage Service will perform an additional POSIX ACL check to determine if + * the user is authorized to perform the requested operation. This cannot be used in conjuction + * with PreauthorizedAgentObjectId. This is only used with generating User Delegation SAS. + */ + std::string AgentObjectId; + + /** + * @brief This value will be used for correlating the storage audit logs with the audit logs + * used by the principal generating and distributing SAS. This is only used for User Delegation + * SAS. + */ + std::string CorrelationId; + + /** + * @brief Sets the permissions for the filesystem SAS. + * + * @param permissions The allowed permissions. + */ + void SetPermissions(DataLakeFileSystemSasPermissions permissions) + { + Permissions = DataLakeFileSystemSasPermissionsToString(permissions); + } + + /** + * @brief Sets the permissions for the file SAS or directory SAS. + * + * @param permissions The allowed permissions. + */ + void SetPermissions(DataLakeSasPermissions permissions); + + /** + * @brief Uses the SharedKeyCredential to sign this shared access signature, to produce + * the proper SAS query parameters for authentication requests. + * + * @param credential The storage account's shared key credential. + * @return The SAS query parameters used for authenticating requests. + */ + std::string ToSasQueryParameters(const SharedKeyCredential& credential); + + /** + * @brief Uses an account's user delegation key to sign this shared access signature, to + * produce the proper SAS query parameters for authentication requests. + * + * @param credential UserDelegationKey retruned from BlobServiceClient.GetUserDelegationKey. + * @param accountName The name of the storage account. + * @return The SAS query parameters used for authenticating requests. + */ + std::string ToSasQueryParameters( + const UserDelegationKey& userDelegationKey, + const std::string& accountName); + + private: + std::string Permissions; + }; + +}}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_sas_builder.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_sas_builder.cpp new file mode 100644 index 000000000..e0030a116 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_sas_builder.cpp @@ -0,0 +1,269 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "azure/storage/files/datalake/datalake_sas_builder.hpp" +#include "azure/core/http/http.hpp" +#include "azure/storage/common/crypt.hpp" + +namespace Azure { namespace Storage { namespace Files { namespace DataLake { + namespace { + std::string DataLakeSasResourceToString(DataLakeSasResource resource) + { + if (resource == DataLakeSasResource::FileSystem) + { + return "c"; + } + else if (resource == DataLakeSasResource::File) + { + return "b"; + } + else if (resource == DataLakeSasResource::Directory) + { + return "d"; + } + else + { + throw std::runtime_error("unknown DataLakeSasResource value"); + } + } + } // namespace + + std::string DataLakeFileSystemSasPermissionsToString(DataLakeFileSystemSasPermissions permissions) + { + std::string permissions_str; + // The order matters + if ((permissions & DataLakeFileSystemSasPermissions::Read) + == DataLakeFileSystemSasPermissions::Read) + { + permissions_str += "r"; + } + if ((permissions & DataLakeFileSystemSasPermissions::Add) + == DataLakeFileSystemSasPermissions::Add) + { + permissions_str += "a"; + } + if ((permissions & DataLakeFileSystemSasPermissions::Create) + == DataLakeFileSystemSasPermissions::Create) + { + permissions_str += "c"; + } + if ((permissions & DataLakeFileSystemSasPermissions::Write) + == DataLakeFileSystemSasPermissions::Write) + { + permissions_str += "w"; + } + if ((permissions & DataLakeFileSystemSasPermissions::Delete) + == DataLakeFileSystemSasPermissions::Delete) + { + permissions_str += "d"; + } + if ((permissions & DataLakeFileSystemSasPermissions::List) + == DataLakeFileSystemSasPermissions::List) + { + permissions_str += "l"; + } + return permissions_str; + } + + void DataLakeSasBuilder::SetPermissions(DataLakeSasPermissions permissions) + { + Permissions.clear(); + // The order matters + if ((permissions & DataLakeSasPermissions::Read) == DataLakeSasPermissions::Read) + { + Permissions += "r"; + } + if ((permissions & DataLakeSasPermissions::Add) == DataLakeSasPermissions::Add) + { + Permissions += "a"; + } + if ((permissions & DataLakeSasPermissions::Create) == DataLakeSasPermissions::Create) + { + Permissions += "c"; + } + if ((permissions & DataLakeSasPermissions::Write) == DataLakeSasPermissions::Write) + { + Permissions += "w"; + } + if ((permissions & DataLakeSasPermissions::Delete) == DataLakeSasPermissions::Delete) + { + Permissions += "d"; + } + if ((permissions & DataLakeSasPermissions::List) == DataLakeSasPermissions::List) + { + Permissions += "l"; + } + if ((permissions & DataLakeSasPermissions::Move) == DataLakeSasPermissions::Move) + { + Permissions += "m"; + } + if ((permissions & DataLakeSasPermissions::Execute) == DataLakeSasPermissions::Execute) + { + Permissions += "e"; + } + if ((permissions & DataLakeSasPermissions::ManageOwnership) + == DataLakeSasPermissions::ManageOwnership) + { + Permissions += "o"; + } + if ((permissions & DataLakeSasPermissions::ManageAccessControl) + == DataLakeSasPermissions::ManageAccessControl) + { + Permissions += "p"; + } + } + + std::string DataLakeSasBuilder::ToSasQueryParameters(const SharedKeyCredential& credential) + { + std::string canonicalName = "/blob/" + credential.AccountName + "/" + FileSystemName; + if (Resource == DataLakeSasResource::File) + { + canonicalName += "/" + Path; + } + std::string protocol = SasProtocolToString(Protocol); + std::string resource = DataLakeSasResourceToString(Resource); + + std::string stringToSign = Permissions + "\n" + (StartsOn.HasValue() ? StartsOn.GetValue() : "") + + "\n" + ExpiresOn + "\n" + canonicalName + "\n" + Identifier + "\n" + + (IPRange.HasValue() ? IPRange.GetValue() : "") + "\n" + protocol + "\n" + Version + "\n" + + resource + "\n" + "\n" + CacheControl + "\n" + ContentDisposition + "\n" + ContentEncoding + + "\n" + ContentLanguage + "\n" + ContentType; + + std::string signature = Base64Encode( + Storage::Details::HmacSha256(stringToSign, Base64Decode(credential.GetAccountKey()))); + + Azure::Core::Http::Url builder; + builder.AppendQuery("sv", Version); + builder.AppendQuery("spr", protocol); + if (StartsOn.HasValue()) + { + builder.AppendQuery("st", StartsOn.GetValue()); + } + if (!ExpiresOn.empty()) + { + builder.AppendQuery("se", ExpiresOn); + } + if (IPRange.HasValue()) + { + builder.AppendQuery("sip", IPRange.GetValue()); + } + if (!Identifier.empty()) + { + builder.AppendQuery("si", Identifier); + } + builder.AppendQuery("sr", resource); + if (!Permissions.empty()) + { + builder.AppendQuery("sp", Permissions); + } + builder.AppendQuery("sig", signature); + if (!CacheControl.empty()) + { + builder.AppendQuery("rscc", CacheControl); + } + if (!ContentDisposition.empty()) + { + builder.AppendQuery("rscd", ContentDisposition); + } + if (!ContentEncoding.empty()) + { + builder.AppendQuery("rsce", ContentEncoding); + } + if (!ContentLanguage.empty()) + { + builder.AppendQuery("rscl", ContentLanguage); + } + if (!ContentType.empty()) + { + builder.AppendQuery("rsct", ContentType); + } + + return builder.GetAbsoluteUrl(); + } + + std::string DataLakeSasBuilder::ToSasQueryParameters( + const UserDelegationKey& userDelegationKey, + const std::string& accountName) + { + std::string canonicalName = "/blob/" + accountName + "/" + FileSystemName; + if (Resource == DataLakeSasResource::File || Resource == DataLakeSasResource::Directory) + { + canonicalName += "/" + Path; + } + std::string protocol = SasProtocolToString(Protocol); + std::string resource = DataLakeSasResourceToString(Resource); + + std::string stringToSign = Permissions + "\n" + (StartsOn.HasValue() ? StartsOn.GetValue() : "") + + "\n" + ExpiresOn + "\n" + canonicalName + "\n" + userDelegationKey.SignedObjectId + "\n" + + userDelegationKey.SignedTenantId + "\n" + userDelegationKey.SignedStartsOn + "\n" + + userDelegationKey.SignedExpiresOn + "\n" + userDelegationKey.SignedService + "\n" + + userDelegationKey.SignedVersion + "\n" + PreauthorizedAgentObjectId + "\n" + AgentObjectId + + "\n" + CorrelationId + "\n" + (IPRange.HasValue() ? IPRange.GetValue() : "") + "\n" + + protocol + "\n" + Version + "\n" + resource + "\n" + "\n" + CacheControl + "\n" + + ContentDisposition + "\n" + ContentEncoding + "\n" + ContentLanguage + "\n" + ContentType; + + std::string signature = Base64Encode( + Storage::Details::HmacSha256(stringToSign, Base64Decode(userDelegationKey.Value))); + + Azure::Core::Http::Url builder; + builder.AppendQuery("sv", Version); + builder.AppendQuery("sr", resource); + if (StartsOn.HasValue()) + { + builder.AppendQuery("st", StartsOn.GetValue()); + } + builder.AppendQuery("se", ExpiresOn); + builder.AppendQuery("sp", Permissions); + if (IPRange.HasValue()) + { + builder.AppendQuery("sip", IPRange.GetValue()); + } + builder.AppendQuery("spr", protocol); + builder.AppendQuery("skoid", userDelegationKey.SignedObjectId); + builder.AppendQuery("sktid", userDelegationKey.SignedTenantId); + builder.AppendQuery("skt", userDelegationKey.SignedStartsOn); + builder.AppendQuery("ske", userDelegationKey.SignedExpiresOn); + builder.AppendQuery("sks", userDelegationKey.SignedService); + builder.AppendQuery("skv", userDelegationKey.SignedVersion); + if (!PreauthorizedAgentObjectId.empty()) + { + builder.AppendQuery("saoid", PreauthorizedAgentObjectId); + } + if (!AgentObjectId.empty()) + { + builder.AppendQuery("suoid", AgentObjectId); + } + if (!CorrelationId.empty()) + { + builder.AppendQuery("scid", CorrelationId); + } + if (DirectoryDepth.HasValue()) + { + builder.AppendQuery("sdd", std::to_string(DirectoryDepth.GetValue())); + } + if (!CacheControl.empty()) + { + builder.AppendQuery("rscc", CacheControl); + } + if (!ContentDisposition.empty()) + { + builder.AppendQuery("rscd", ContentDisposition); + } + if (!ContentEncoding.empty()) + { + builder.AppendQuery("rsce", ContentEncoding); + } + if (!ContentLanguage.empty()) + { + builder.AppendQuery("rscl", ContentLanguage); + } + if (!ContentType.empty()) + { + builder.AppendQuery("rsct", ContentType); + } + builder.AppendQuery("sig", signature); + + return builder.GetAbsoluteUrl(); + } + +}}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/test/datalake_sas_test.cpp b/sdk/storage/azure-storage-files-datalake/test/datalake_sas_test.cpp new file mode 100644 index 000000000..e15e9866a --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/datalake_sas_test.cpp @@ -0,0 +1,444 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "azure/storage/blobs/blob_sas_builder.hpp" +#include "azure/storage/files/datalake/datalake_sas_builder.hpp" +#include "datalake_file_system_client_test.hpp" + +namespace Azure { namespace Storage { namespace Test { + + TEST_F(DataLakeFileSystemClientTest, DataLakeSasTest) + { + std::string directory1Name = RandomString(); + std::string directory2Name = RandomString(); + std::string fileName = RandomString(); + Files::DataLake::DataLakeSasBuilder fileSasBuilder; + fileSasBuilder.Protocol = SasProtocol::HttpsAndHtttp; + fileSasBuilder.StartsOn = ToIso8601(std::chrono::system_clock::now() - std::chrono::minutes(5)); + fileSasBuilder.ExpiresOn + = ToIso8601(std::chrono::system_clock::now() + std::chrono::minutes(60)); + fileSasBuilder.FileSystemName = m_fileSystemName; + fileSasBuilder.Path = directory1Name + "/" + directory2Name + "/" + fileName; + fileSasBuilder.Resource = Files::DataLake::DataLakeSasResource::File; + + Files::DataLake::DataLakeSasBuilder directorySasBuilder = fileSasBuilder; + directorySasBuilder.Path = directory1Name; + directorySasBuilder.IsDirectory = true; + directorySasBuilder.DirectoryDepth = 1; + directorySasBuilder.Resource = Files::DataLake::DataLakeSasResource::Directory; + + Files::DataLake::DataLakeSasBuilder filesystemSasBuilder = fileSasBuilder; + filesystemSasBuilder.Path.clear(); + filesystemSasBuilder.Resource = Files::DataLake::DataLakeSasResource::FileSystem; + + auto keyCredential = Details::ParseConnectionString(AdlsGen2ConnectionString()).KeyCredential; + auto accountName = keyCredential->AccountName; + auto serviceClient0 + = Files::DataLake::ServiceClient::CreateFromConnectionString(AdlsGen2ConnectionString()); + auto filesystemClient0 = serviceClient0.GetFileSystemClient(m_fileSystemName); + auto containerClinet0 = Blobs::BlobContainerClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName); + auto directory1Client0 = filesystemClient0.GetDirectoryClient(directory1Name); + auto directory2Client0 = directory1Client0.GetSubDirectoryClient(directory2Name); + auto fileClient0 = directory2Client0.GetFileClient(fileName); + directory1Client0.Create(); + directory2Client0.Create(); + + auto serviceUri = serviceClient0.GetDfsUri(); + auto filesystemUri = filesystemClient0.GetDfsUri(); + auto directory1Uri = directory1Client0.GetDfsUri(); + auto directory2Uri = directory2Client0.GetDfsUri(); + auto fileUri = fileClient0.GetUri(); + + auto serviceClient1 = Files::DataLake::ServiceClient( + serviceUri, + std::make_shared( + AadTenantId(), AadClientId(), AadClientSecret())); + auto userDelegationKey = *serviceClient1.GetUserDelegationKey( + ToIso8601(std::chrono::system_clock::now() - std::chrono::minutes(5)), + ToIso8601(std::chrono::system_clock::now() + std::chrono::minutes(60))); + + auto verify_file_read = [&](const std::string& sas) { + EXPECT_NO_THROW(fileClient0.Create()); + auto fileClient = Files::DataLake::FileClient(fileUri + sas); + auto downloadedContent = fileClient.Read(); + EXPECT_TRUE(ReadBodyStream(downloadedContent->Body).empty()); + }; + + auto verify_file_write = [&](const std::string& sas) { + auto fileClient = Files::DataLake::FileClient(fileUri + sas); + EXPECT_NO_THROW(fileClient.Create()); + }; + + auto verify_file_delete = [&](const std::string& sas) { + fileClient0.Create(); + auto fileClient = Files::DataLake::FileClient(fileUri + sas); + EXPECT_NO_THROW(fileClient.Delete()); + }; + + auto verify_file_add = [&](const std::string& sas) { + unused(sas); + /* + * Add test for append block when DataLake supports append blobs. + */ + }; + + auto verify_filesystem_list = [&](const std::string& sas) { + auto filesystemClient = Files::DataLake::FileSystemClient(filesystemUri + sas); + EXPECT_NO_THROW(filesystemClient.ListPaths(true)); + }; + + auto verify_directory_list = [&](const std::string& sas) { + auto filesystemClient = Files::DataLake::FileSystemClient(filesystemUri + sas); + Files::DataLake::ListPathsOptions options; + options.Directory = directory1Name; + EXPECT_NO_THROW(filesystemClient.ListPaths(true, options)); + }; + + auto verify_file_create = [&](const std::string& sas) { + try + { + fileClient0.Delete(); + } + catch (StorageError&) + { + } + auto fileClient = Files::DataLake::FileClient(fileUri + sas); + fileClient.Create(); + }; + + auto verify_file_move = [&](const std::string& sas) { + try + { + fileClient0.Delete(); + } + catch (StorageError&) + { + } + std::string newFilename = RandomString(); + auto newFileClient0 = directory2Client0.GetFileClient(newFilename); + newFileClient0.Create(); + auto fileClient = Files::DataLake::FileClient(newFileClient0.GetDfsUri() + sas); + EXPECT_NO_THROW(fileClient.Rename(directory1Name + "/" + directory2Name + "/" + fileName)); + }; + + auto verify_file_execute = [&](const std::string& sas) { + fileClient0.Create(); + auto fileClient = Files::DataLake::FileClient(fileUri + sas); + EXPECT_NO_THROW(fileClient0.GetAccessControls()); + }; + + auto verify_file_ownership = [&](const std::string& sas) { + fileClient0.Create(); + auto fileClient = Files::DataLake::FileClient(fileUri + sas); + EXPECT_NO_THROW(fileClient0.GetAccessControls()); + }; + + auto verify_file_permissions = [&](const std::string& sas) { + fileClient0.Create(); + auto fileClient = Files::DataLake::FileClient(fileUri + sas); + auto acls = fileClient0.GetAccessControls()->Acls; + EXPECT_NO_THROW(fileClient.SetAccessControl(acls)); + }; + + for (auto permissions : { + Files::DataLake::DataLakeSasPermissions::All, + Files::DataLake::DataLakeSasPermissions::Read, + Files::DataLake::DataLakeSasPermissions::Write, + Files::DataLake::DataLakeSasPermissions::Delete, + Files::DataLake::DataLakeSasPermissions::Add, + Files::DataLake::DataLakeSasPermissions::Create, + Files::DataLake::DataLakeSasPermissions::List, + Files::DataLake::DataLakeSasPermissions::Move, + Files::DataLake::DataLakeSasPermissions::Execute, + Files::DataLake::DataLakeSasPermissions::ManageOwnership, + Files::DataLake::DataLakeSasPermissions::ManageAccessControl, + }) + { + fileSasBuilder.SetPermissions(permissions); + auto sasToken = fileSasBuilder.ToSasQueryParameters(*keyCredential); + auto sasToken2 = fileSasBuilder.ToSasQueryParameters(userDelegationKey, accountName); + + if ((permissions & Files::DataLake::DataLakeSasPermissions::Read) + == Files::DataLake::DataLakeSasPermissions::Read) + { + verify_file_read(sasToken); + verify_file_read(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeSasPermissions::Write) + == Files::DataLake::DataLakeSasPermissions::Write) + { + verify_file_write(sasToken); + verify_file_write(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeSasPermissions::Delete) + == Files::DataLake::DataLakeSasPermissions::Delete) + { + verify_file_delete(sasToken); + verify_file_delete(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeSasPermissions::Add) + == Files::DataLake::DataLakeSasPermissions::Add) + { + verify_file_add(sasToken); + verify_file_add(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeSasPermissions::Create) + == Files::DataLake::DataLakeSasPermissions::Create) + { + verify_file_create(sasToken); + verify_file_create(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeSasPermissions::ManageAccessControl) + == Files::DataLake::DataLakeSasPermissions::ManageAccessControl) + { + verify_file_permissions(sasToken); + verify_file_permissions(sasToken2); + } + } + + for (auto permissions : { + Files::DataLake::DataLakeSasPermissions::All, + Files::DataLake::DataLakeSasPermissions::Read, + Files::DataLake::DataLakeSasPermissions::Write, + Files::DataLake::DataLakeSasPermissions::Delete, + Files::DataLake::DataLakeSasPermissions::Add, + Files::DataLake::DataLakeSasPermissions::Create, + Files::DataLake::DataLakeSasPermissions::List, + Files::DataLake::DataLakeSasPermissions::Move, + Files::DataLake::DataLakeSasPermissions::Execute, + Files::DataLake::DataLakeSasPermissions::ManageOwnership, + Files::DataLake::DataLakeSasPermissions::ManageAccessControl, + }) + { + directorySasBuilder.SetPermissions(permissions); + auto sasToken2 = directorySasBuilder.ToSasQueryParameters(userDelegationKey, accountName); + + if ((permissions & Files::DataLake::DataLakeSasPermissions::Read) + == Files::DataLake::DataLakeSasPermissions::Read) + { + verify_file_read(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeSasPermissions::Write) + == Files::DataLake::DataLakeSasPermissions::Write) + { + verify_file_write(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeSasPermissions::Delete) + == Files::DataLake::DataLakeSasPermissions::Delete) + { + verify_file_delete(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeSasPermissions::Add) + == Files::DataLake::DataLakeSasPermissions::Add) + { + verify_file_add(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeSasPermissions::Create) + == Files::DataLake::DataLakeSasPermissions::Create) + { + verify_file_create(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeSasPermissions::List) + == Files::DataLake::DataLakeSasPermissions::List) + { + verify_directory_list(sasToken2); + } + unused(verify_file_move); + /* + don't know why, move doesn't work + if ((permissions & Files::DataLake::DataLakeSasPermissions::Move) + == Files::DataLake::DataLakeSasPermissions::Move) + { + verify_file_move(sasToken2); + } + */ + if ((permissions & Files::DataLake::DataLakeSasPermissions::Execute) + == Files::DataLake::DataLakeSasPermissions::Execute) + { + verify_file_execute(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeSasPermissions::ManageOwnership) + == Files::DataLake::DataLakeSasPermissions::ManageOwnership) + { + verify_file_ownership(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeSasPermissions::ManageAccessControl) + == Files::DataLake::DataLakeSasPermissions::ManageAccessControl) + { + verify_file_permissions(sasToken2); + } + } + + for (auto permissions : { + Files::DataLake::DataLakeFileSystemSasPermissions::All, + Files::DataLake::DataLakeFileSystemSasPermissions::Read, + Files::DataLake::DataLakeFileSystemSasPermissions::Write, + Files::DataLake::DataLakeFileSystemSasPermissions::Delete, + Files::DataLake::DataLakeFileSystemSasPermissions::List, + Files::DataLake::DataLakeFileSystemSasPermissions::Add, + Files::DataLake::DataLakeFileSystemSasPermissions::Create, + }) + { + filesystemSasBuilder.SetPermissions(permissions); + auto sasToken = filesystemSasBuilder.ToSasQueryParameters(*keyCredential); + auto sasToken2 = filesystemSasBuilder.ToSasQueryParameters(userDelegationKey, accountName); + + if ((permissions & Files::DataLake::DataLakeFileSystemSasPermissions::All) + == Files::DataLake::DataLakeFileSystemSasPermissions::All) + { + unused(verify_file_move); + /* + don't know why, move doesn't work + verify_file_move(sasToken); + verify_file_move(sasToken2); + */ + } + if ((permissions & Files::DataLake::DataLakeFileSystemSasPermissions::Read) + == Files::DataLake::DataLakeFileSystemSasPermissions::Read) + { + verify_file_read(sasToken); + verify_file_read(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeFileSystemSasPermissions::Write) + == Files::DataLake::DataLakeFileSystemSasPermissions::Write) + { + verify_file_write(sasToken); + verify_file_write(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeFileSystemSasPermissions::Delete) + == Files::DataLake::DataLakeFileSystemSasPermissions::Delete) + { + verify_file_delete(sasToken); + verify_file_delete(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeFileSystemSasPermissions::List) + == Files::DataLake::DataLakeFileSystemSasPermissions::List) + { + verify_filesystem_list(sasToken); + verify_filesystem_list(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeFileSystemSasPermissions::Add) + == Files::DataLake::DataLakeFileSystemSasPermissions::Add) + { + verify_file_add(sasToken); + verify_file_add(sasToken2); + } + if ((permissions & Files::DataLake::DataLakeFileSystemSasPermissions::Create) + == Files::DataLake::DataLakeFileSystemSasPermissions::Create) + { + verify_file_create(sasToken); + verify_file_create(sasToken2); + } + } + + fileSasBuilder.SetPermissions(Files::DataLake::DataLakeSasPermissions::All); + + // Expires + { + Files::DataLake::DataLakeSasBuilder builder2 = fileSasBuilder; + builder2.StartsOn = ToIso8601(std::chrono::system_clock::now() - std::chrono::minutes(5)); + builder2.ExpiresOn = ToIso8601(std::chrono::system_clock::now() - std::chrono::minutes(1)); + auto sasToken = builder2.ToSasQueryParameters(*keyCredential); + EXPECT_THROW(verify_file_create(sasToken), StorageError); + + auto sasToken2 = builder2.ToSasQueryParameters(userDelegationKey, accountName); + EXPECT_THROW(verify_file_create(sasToken2), StorageError); + } + + // Without start time + { + Files::DataLake::DataLakeSasBuilder builder2 = fileSasBuilder; + builder2.StartsOn.Reset(); + auto sasToken = builder2.ToSasQueryParameters(*keyCredential); + EXPECT_NO_THROW(verify_file_create(sasToken)); + auto sasToken2 = builder2.ToSasQueryParameters(userDelegationKey, accountName); + EXPECT_NO_THROW(verify_file_create(sasToken2)); + } + + // IP + { + Files::DataLake::DataLakeSasBuilder builder2 = fileSasBuilder; + builder2.IPRange = "0.0.0.0-0.0.0.1"; + auto sasToken = builder2.ToSasQueryParameters(*keyCredential); + EXPECT_THROW(verify_file_create(sasToken), StorageError); + auto sasToken2 = builder2.ToSasQueryParameters(userDelegationKey, accountName); + EXPECT_THROW(verify_file_create(sasToken2), StorageError); + + builder2.IPRange = "0.0.0.0-255.255.255.255"; + sasToken = builder2.ToSasQueryParameters(*keyCredential); + EXPECT_NO_THROW(verify_file_create(sasToken)); + sasToken2 = builder2.ToSasQueryParameters(userDelegationKey, accountName); + EXPECT_NO_THROW(verify_file_create(sasToken2)); + } + + // PreauthorizedAgentObjectId + { + Files::DataLake::DataLakeSasBuilder builder2 = fileSasBuilder; + builder2.PreauthorizedAgentObjectId = Azure::Core::Uuid::CreateUuid().GetUuidString(); + builder2.CorrelationId = Azure::Core::Uuid::CreateUuid().GetUuidString(); + auto sasToken2 = builder2.ToSasQueryParameters(userDelegationKey, accountName); + EXPECT_NO_THROW(verify_file_read(sasToken2)); + } + + // Identifier + { + Blobs::SetContainerAccessPolicyOptions options; + options.AccessType = Blobs::PublicAccessType::Blob; + Blobs::BlobSignedIdentifier identifier; + identifier.Id = RandomString(64); + identifier.StartsOn = ToIso8601(std::chrono::system_clock::now() - std::chrono::minutes(5)); + identifier.ExpiresOn = ToIso8601(std::chrono::system_clock::now() + std::chrono::minutes(60)); + identifier.Permissions + = Blobs::BlobContainerSasPermissionsToString(Blobs::BlobContainerSasPermissions::Read); + options.SignedIdentifiers.emplace_back(identifier); + containerClinet0.SetAccessPolicy(options); + + Files::DataLake::DataLakeSasBuilder builder2 = fileSasBuilder; + builder2.StartsOn.Reset(); + builder2.ExpiresOn.clear(); + builder2.SetPermissions(static_cast(0)); + builder2.Identifier = identifier.Id; + + auto sasToken = builder2.ToSasQueryParameters(*keyCredential); + EXPECT_NO_THROW(verify_file_read(sasToken)); + } + + // response headers override + { + Files::DataLake::DataLakeHttpHeaders headers; + headers.ContentType = "application/x-binary"; + headers.ContentLanguage = "en-US"; + headers.ContentDisposition = "attachment"; + headers.CacheControl = "no-cache"; + headers.ContentEncoding = "identify"; + + Files::DataLake::DataLakeSasBuilder builder2 = fileSasBuilder; + builder2.SetPermissions(Files::DataLake::DataLakeSasPermissions::Read); + builder2.ContentType = "application/x-binary"; + builder2.ContentLanguage = "en-US"; + builder2.ContentDisposition = "attachment"; + builder2.CacheControl = "no-cache"; + builder2.ContentEncoding = "identify"; + auto sasToken = builder2.ToSasQueryParameters(*keyCredential); + auto fileClient = Files::DataLake::FileClient(fileUri + sasToken); + fileClient0.Create(); + auto p = fileClient.GetProperties(); + EXPECT_EQ(p->HttpHeaders.ContentType, headers.ContentType); + EXPECT_EQ(p->HttpHeaders.ContentLanguage, headers.ContentLanguage); + EXPECT_EQ(p->HttpHeaders.ContentDisposition, headers.ContentDisposition); + EXPECT_EQ(p->HttpHeaders.CacheControl, headers.CacheControl); + EXPECT_EQ(p->HttpHeaders.ContentEncoding, headers.ContentEncoding); + + auto sasToken2 = builder2.ToSasQueryParameters(userDelegationKey, accountName); + fileClient = Files::DataLake::FileClient(fileUri + sasToken); + p = fileClient.GetProperties(); + EXPECT_EQ(p->HttpHeaders.ContentType, headers.ContentType); + EXPECT_EQ(p->HttpHeaders.ContentLanguage, headers.ContentLanguage); + EXPECT_EQ(p->HttpHeaders.ContentDisposition, headers.ContentDisposition); + EXPECT_EQ(p->HttpHeaders.CacheControl, headers.CacheControl); + EXPECT_EQ(p->HttpHeaders.ContentEncoding, headers.ContentEncoding); + } + } + +}}} // namespace Azure::Storage::Test