From 306913ff1b9e47dc96f1e84d857ecde1e68dd9d1 Mon Sep 17 00:00:00 2001 From: Kan Tang Date: Tue, 9 Feb 2021 10:58:11 +0800 Subject: [PATCH] Refined DataLake APIs (#1627) --- .../azure-storage-files-datalake/CHANGELOG.md | 4 ++ .../files/datalake/datalake_options.hpp | 16 +++---- .../files/datalake/datalake_path_client.hpp | 44 +++++++++---------- .../files/datalake/datalake_responses.hpp | 10 ++--- .../protocol/datalake_rest_client.hpp | 28 ++++++------ .../src/datalake_file_client.cpp | 11 ++--- .../src/datalake_path_client.cpp | 6 +-- .../test/datalake_directory_client_test.cpp | 8 ++-- .../test/datalake_file_client_test.cpp | 9 ++-- 9 files changed, 71 insertions(+), 65 deletions(-) diff --git a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md index f90172920..d7b31577c 100644 --- a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md @@ -10,6 +10,10 @@ - `GetDataLakePathPropertiesResult::AccessTierInferred` was renamed to `IsAccessTierInferred`. - `HttpHeaders` of `DownloadDataLakeFileResult` and `DownloadDataLakeFileToResult` was moved into `Details`, to align with Blob service. - Removed `BreakDataLakeLeaseResult::LeaseTime`. +- Renamed APIs for modifying access list recursively. Used to be with pattern `AccessControlRecursiveList`, now is with pattern `AccessControlListRecursive`. +- Refined options for `ScheduleDeletion`, to be consistent with other APIs. +- Renamed `ContentLength` in `PathItem` to `FileSize`. +- In `PathSetAccessControlRecursiveResult`, `DirectoriesSuccessful` is renamed to `NumberOfSuccessfulDirectories`, `FilesSuccessful` is renamed to `NumberOfSuccessfulFiles`, `FailureCount` is renamed to `NumberOfFailures`. ## 12.0.0-beta.7 (2021-02-03) diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index 745e0c047..09a96d43d 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -653,9 +653,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { }; /** - * @brief Optional parameters for DirectoryClient::SetAccessControlRecursiveListSinglePage + * @brief Optional parameters for DirectoryClient::SetAccessControlListRecursiveSinglePage */ - struct SetDataLakePathAccessControlRecursiveListSinglePageOptions + struct SetDataLakePathAccessControlListRecursiveSinglePageOptions { /** * @brief Context for cancelling long running operations. @@ -689,11 +689,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Core::Nullable ContinueOnFailure; }; - using UpdateDataLakePathAccessControlRecursiveListSinglePageOptions - = SetDataLakePathAccessControlRecursiveListSinglePageOptions; + using UpdateDataLakePathAccessControlListRecursiveSinglePageOptions + = SetDataLakePathAccessControlListRecursiveSinglePageOptions; - using RemoveDataLakePathAccessControlRecursiveListSinglePageOptions - = SetDataLakePathAccessControlRecursiveListSinglePageOptions; + using RemoveDataLakePathAccessControlListRecursiveSinglePageOptions + = SetDataLakePathAccessControlListRecursiveSinglePageOptions; using CreateDataLakeFileOptions = CreateDataLakePathOptions; using CreateDataLakeDirectoryOptions = CreateDataLakePathOptions; @@ -756,13 +756,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * ScheduleFileExpiryOriginType::RelativeToCreation or * ScheduleFileExpiryOriginType::RelativeToNow. */ - Azure::Core::Nullable TimeToExpireInMs; + Azure::Core::Nullable TimeToExpire; /** * @brief The expiry time in RFC1123 format. Only work if ExpiryOrigin is * ScheduleFileExpiryOriginType::Absolute. */ - Azure::Core::Nullable ExpiresOn; + Azure::Core::Nullable ExpiresOn; }; using AcquireDataLakeLeaseOptions = Blobs::AcquireBlobLeaseOptions; diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp index 78704943d..23cd11d98 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp @@ -212,16 +212,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * @param options Optional parameters to set an access control recursively to the resource the * directory points to. * @return - * Azure::Core::Response + * Azure::Core::Response * @remark This request is sent to dfs endpoint. */ - Azure::Core::Response - SetAccessControlRecursiveListSinglePage( + Azure::Core::Response + SetAccessControlListRecursiveSinglePage( const std::vector& acls, - const SetDataLakePathAccessControlRecursiveListSinglePageOptions& options - = SetDataLakePathAccessControlRecursiveListSinglePageOptions()) const + const SetDataLakePathAccessControlListRecursiveSinglePageOptions& options + = SetDataLakePathAccessControlListRecursiveSinglePageOptions()) const { - return SetAccessControlRecursiveListSinglePageInternal( + return SetAccessControlListRecursiveSinglePageInternal( Models::PathSetAccessControlRecursiveMode::Set, acls, options); } @@ -233,16 +233,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * @param options Optional parameters to set an access control recursively to the resource the * directory points to. * @return - * Azure::Core::Response + * Azure::Core::Response * @remark This request is sent to dfs endpoint. */ - Azure::Core::Response - UpdateAccessControlRecursiveListSinglePage( + Azure::Core::Response + UpdateAccessControlListRecursiveSinglePage( const std::vector& acls, - const UpdateDataLakePathAccessControlRecursiveListSinglePageOptions& options - = UpdateDataLakePathAccessControlRecursiveListSinglePageOptions()) const + const UpdateDataLakePathAccessControlListRecursiveSinglePageOptions& options + = UpdateDataLakePathAccessControlListRecursiveSinglePageOptions()) const { - return SetAccessControlRecursiveListSinglePageInternal( + return SetAccessControlListRecursiveSinglePageInternal( Models::PathSetAccessControlRecursiveMode::Modify, acls, options); } @@ -254,16 +254,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * @param options Optional parameters to set an access control recursively to the resource the * directory points to. * @return - * Azure::Core::Response + * Azure::Core::Response * @remark This request is sent to dfs endpoint. */ - Azure::Core::Response - RemoveAccessControlRecursiveListSinglePage( + Azure::Core::Response + RemoveAccessControlListRecursiveSinglePage( const std::vector& acls, - const RemoveDataLakePathAccessControlRecursiveListSinglePageOptions& options - = RemoveDataLakePathAccessControlRecursiveListSinglePageOptions()) const + const RemoveDataLakePathAccessControlListRecursiveSinglePageOptions& options + = RemoveDataLakePathAccessControlListRecursiveSinglePageOptions()) const { - return SetAccessControlRecursiveListSinglePageInternal( + return SetAccessControlListRecursiveSinglePageInternal( Models::PathSetAccessControlRecursiveMode::Remove, acls, options); } @@ -281,12 +281,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { { } - Azure::Core::Response - SetAccessControlRecursiveListSinglePageInternal( + Azure::Core::Response + SetAccessControlListRecursiveSinglePageInternal( Models::PathSetAccessControlRecursiveMode mode, const std::vector& acls, - const SetDataLakePathAccessControlRecursiveListSinglePageOptions& options - = SetDataLakePathAccessControlRecursiveListSinglePageOptions()) const; + const SetDataLakePathAccessControlListRecursiveSinglePageOptions& options + = SetDataLakePathAccessControlListRecursiveSinglePageOptions()) const; friend class DataLakeFileSystemClient; friend class DataLakeLeaseClient; diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index 6851bc04f..7b937132c 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -280,12 +280,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam std::string RequestId; }; - using SetDataLakePathAccessControlRecursiveListSinglePageResult + using SetDataLakePathAccessControlListRecursiveSinglePageResult = Details::PathSetAccessControlRecursiveResult; - using UpdateDataLakePathAccessControlRecursiveListSinglePageResult - = SetDataLakePathAccessControlRecursiveListSinglePageResult; - using RemoveDataLakePathAccessControlRecursiveListSinglePageResult - = SetDataLakePathAccessControlRecursiveListSinglePageResult; + using UpdateDataLakePathAccessControlListRecursiveSinglePageResult + = SetDataLakePathAccessControlListRecursiveSinglePageResult; + using RemoveDataLakePathAccessControlListRecursiveSinglePageResult + = SetDataLakePathAccessControlListRecursiveSinglePageResult; using CreateDataLakeDirectoryResult = CreateDataLakePathResult; using DeleteDataLakeDirectoryResult = DeleteDataLakePathResult; diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/protocol/datalake_rest_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/protocol/datalake_rest_client.hpp index 2582b7869..712762e83 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/protocol/datalake_rest_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/protocol/datalake_rest_client.hpp @@ -113,7 +113,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { bool IsDirectory = bool(); Core::DateTime LastModified; std::string ETag; - int64_t ContentLength = int64_t(); + int64_t FileSize = int64_t(); std::string Owner; std::string Group; std::string Permissions; @@ -311,9 +311,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { struct SetAccessControlRecursiveResponse { - int32_t DirectoriesSuccessful = int32_t(); - int32_t FilesSuccessful = int32_t(); - int32_t FailureCount = int32_t(); + int32_t NumberOfSuccessfulDirectories = int32_t(); + int32_t NumberOfSuccessfulFiles = int32_t(); + int32_t NumberOfFailures = int32_t(); std::vector FailedEntries; }; @@ -370,9 +370,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { struct PathSetAccessControlRecursiveResult { - int32_t DirectoriesSuccessful = int32_t(); - int32_t FilesSuccessful = int32_t(); - int32_t FailureCount = int32_t(); + int32_t NumberOfSuccessfulDirectories = int32_t(); + int32_t NumberOfSuccessfulFiles = int32_t(); + int32_t NumberOfFailures = int32_t(); std::vector FailedEntries; Azure::Core::Nullable ContinuationToken; std::string RequestId; @@ -504,7 +504,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { result.ETag = node["etag"].get(); if (node.contains("contentLength")) { - result.ContentLength = std::stoll(node["contentLength"].get()); + result.FileSize = std::stoll(node["contentLength"].get()); } result.Owner = node["owner"].get(); result.Group = node["group"].get(); @@ -1407,9 +1407,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const Azure::Core::Internal::Json::json& node) { SetAccessControlRecursiveResponse result; - result.DirectoriesSuccessful = node["directoriesSuccessful"].get(); - result.FilesSuccessful = node["filesSuccessful"].get(); - result.FailureCount = node["failureCount"].get(); + result.NumberOfSuccessfulDirectories = node["directoriesSuccessful"].get(); + result.NumberOfSuccessfulFiles = node["filesSuccessful"].get(); + result.NumberOfFailures = node["failureCount"].get(); for (const auto& element : node["failedEntries"]) { result.FailedEntries.emplace_back(AclFailedEntryFromJson(element)); @@ -1422,9 +1422,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { SetAccessControlRecursiveResponse object) { PathSetAccessControlRecursiveResult result; - result.DirectoriesSuccessful = object.DirectoriesSuccessful; - result.FilesSuccessful = object.FilesSuccessful; - result.FailureCount = object.FailureCount; + result.NumberOfSuccessfulDirectories = object.NumberOfSuccessfulDirectories; + result.NumberOfSuccessfulFiles = object.NumberOfSuccessfulFiles; + result.NumberOfFailures = object.NumberOfFailures; result.FailedEntries = std::move(object.FailedEntries); return result; diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp index 5472aef17..74591231b 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp @@ -455,18 +455,19 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { { Blobs::Details::BlobRestClient::Blob::SetBlobExpiryOptions protocolLayerOptions; protocolLayerOptions.ExpiryOrigin = expiryOrigin; - if (options.ExpiresOn.HasValue() && options.TimeToExpireInMs.HasValue()) + if (options.ExpiresOn.HasValue() && options.TimeToExpire.HasValue()) { - // ExpiresOn and TimeToExpireInMs should be mutually exlusive. + // ExpiresOn and TimeToExpire should be mutually exlusive. std::abort(); } if (options.ExpiresOn.HasValue()) { - protocolLayerOptions.ExpiryTime = options.ExpiresOn; + protocolLayerOptions.ExpiryTime + = options.ExpiresOn.GetValue().GetString(Azure::Core::DateTime::DateFormat::Rfc1123); } - else if (options.TimeToExpireInMs.HasValue()) + else if (options.TimeToExpire.HasValue()) { - protocolLayerOptions.ExpiryTime = std::to_string(options.TimeToExpireInMs.GetValue()); + protocolLayerOptions.ExpiryTime = std::to_string(options.TimeToExpire.GetValue().count()); } return Blobs::Details::BlobRestClient::Blob::ScheduleDeletion( options.Context, *m_pipeline, m_blobClient.m_blobUrl, protocolLayerOptions); diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp index 389299887..455b4d3cf 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp @@ -471,11 +471,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::move(ret), result.ExtractRawResponse()); } - Azure::Core::Response - DataLakePathClient::SetAccessControlRecursiveListSinglePageInternal( + Azure::Core::Response + DataLakePathClient::SetAccessControlListRecursiveSinglePageInternal( Models::PathSetAccessControlRecursiveMode mode, const std::vector& acls, - const SetDataLakePathAccessControlRecursiveListSinglePageOptions& options) const + const SetDataLakePathAccessControlListRecursiveSinglePageOptions& options) const { Details::DataLakeRestClient::Path::SetAccessControlRecursiveOptions protocolLayerOptions; protocolLayerOptions.Mode = mode; diff --git a/sdk/storage/azure-storage-files-datalake/test/datalake_directory_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/datalake_directory_client_test.cpp index e1758fb8f..7faa49f4b 100644 --- a/sdk/storage/azure-storage-files-datalake/test/datalake_directory_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/datalake_directory_client_test.cpp @@ -345,7 +345,7 @@ namespace Azure { namespace Storage { namespace Test { { // Set Acls recursive. std::vector acls = GetValidAcls(); - EXPECT_NO_THROW(rootDirectoryClient.SetAccessControlRecursiveListSinglePage(acls)); + EXPECT_NO_THROW(rootDirectoryClient.SetAccessControlListRecursiveSinglePage(acls)); std::vector resultAcls1; std::vector resultAcls2; EXPECT_NO_THROW(resultAcls1 = directoryClient1.GetAccessControlList()->Acls); @@ -372,7 +372,7 @@ namespace Azure { namespace Storage { namespace Test { newAcl.Permissions = "rw-"; std::vector acls; acls.emplace_back(std::move(newAcl)); - EXPECT_NO_THROW(rootDirectoryClient.UpdateAccessControlRecursiveListSinglePage(acls)); + EXPECT_NO_THROW(rootDirectoryClient.UpdateAccessControlListRecursiveSinglePage(acls)); std::vector resultAcls1; std::vector resultAcls2; EXPECT_NO_THROW(resultAcls1 = directoryClient1.GetAccessControlList()->Acls); @@ -449,7 +449,7 @@ namespace Azure { namespace Storage { namespace Test { removeAcl.Id = "72a3f86f-271f-439e-b031-25678907d381"; std::vector acls; acls.emplace_back(std::move(removeAcl)); - EXPECT_NO_THROW(rootDirectoryClient.RemoveAccessControlRecursiveListSinglePage(acls)); + EXPECT_NO_THROW(rootDirectoryClient.RemoveAccessControlListRecursiveSinglePage(acls)); std::vector resultAcls1; std::vector resultAcls2; EXPECT_NO_THROW(resultAcls1 = directoryClient1.GetAccessControlList()->Acls); @@ -525,7 +525,7 @@ namespace Azure { namespace Storage { namespace Test { newAcl.Permissions = "rw-"; acls.emplace_back(std::move(newAcl)); } - (rootDirectoryClient.SetAccessControlRecursiveListSinglePage(acls)); + (rootDirectoryClient.SetAccessControlListRecursiveSinglePage(acls)); std::vector resultAcls1; std::vector resultAcls2; EXPECT_NO_THROW(resultAcls1 = directoryClient1.GetAccessControlList()->Acls); diff --git a/sdk/storage/azure-storage-files-datalake/test/datalake_file_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/datalake_file_client_test.cpp index 211e30c1d..fb49a35c0 100644 --- a/sdk/storage/azure-storage-files-datalake/test/datalake_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/datalake_file_client_test.cpp @@ -429,7 +429,7 @@ namespace Azure { namespace Storage { namespace Test { client.ScheduleDeletion( Files::DataLake::ScheduleDataLakeFileExpiryOriginType::RelativeToNow, options), StorageException); - options.TimeToExpireInMs = 1000; + options.TimeToExpire = std::chrono::milliseconds(1000); EXPECT_NO_THROW(client.ScheduleDeletion( Files::DataLake::ScheduleDataLakeFileExpiryOriginType::RelativeToNow, options)); } @@ -441,13 +441,14 @@ namespace Azure { namespace Storage { namespace Test { client.ScheduleDeletion( Files::DataLake::ScheduleDataLakeFileExpiryOriginType::Absolute, options), StorageException); - options.TimeToExpireInMs = 1000; + options.TimeToExpire = std::chrono::milliseconds(1000); EXPECT_THROW( client.ScheduleDeletion( Files::DataLake::ScheduleDataLakeFileExpiryOriginType::Absolute, options), StorageException); - options.ExpiresOn = "Tue, 29 Sep 2100 09:53:03 GMT"; - options.TimeToExpireInMs = Azure::Core::Nullable(); + options.ExpiresOn = Azure::Core::DateTime::Parse( + "Wed, 29 Sep 2100 09:53:03 GMT", Azure::Core::DateTime::DateFormat::Rfc1123); + options.TimeToExpire = Azure::Core::Nullable(); EXPECT_NO_THROW(client.ScheduleDeletion( Files::DataLake::ScheduleDataLakeFileExpiryOriginType::Absolute, options)); }