github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refined DataLake APIs (#1627)

Browse Source
This commit is contained in:
Kan Tang 2021-02-09 10:58:11 +08:00 committed by GitHub
parent 19ea155f45
commit 306913ff1b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 71 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sdk/storage/azure-storage-files-datalake/CHANGELOG.md
View File

@ -10,6 +10,10 @@
- `GetDataLakePathPropertiesResult::AccessTierInferred` was renamed to `IsAccessTierInferred`.
- `HttpHeaders` of `DownloadDataLakeFileResult` and `DownloadDataLakeFileToResult` was moved into `Details`, to align with Blob service.
- Removed `BreakDataLakeLeaseResult::LeaseTime`.
- Renamed APIs for modifying access list recursively. Used to be with pattern `AccessControlRecursiveList`, now is with pattern `AccessControlListRecursive`.
- Refined options for `ScheduleDeletion`, to be consistent with other APIs.
- Renamed `ContentLength` in `PathItem` to `FileSize`.
- In `PathSetAccessControlRecursiveResult`, `DirectoriesSuccessful` is renamed to `NumberOfSuccessfulDirectories`, `FilesSuccessful` is renamed to `NumberOfSuccessfulFiles`, `FailureCount` is renamed to `NumberOfFailures`.
## 12.0.0-beta.7 (2021-02-03)

16
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp
View File

@ -653,9 +653,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
};
/**
* @brief Optional parameters for DirectoryClient::SetAccessControlRecursiveListSinglePage
* @brief Optional parameters for DirectoryClient::SetAccessControlListRecursiveSinglePage
*/
struct SetDataLakePathAccessControlRecursiveListSinglePageOptions
struct SetDataLakePathAccessControlListRecursiveSinglePageOptions
{
/**
* @brief Context for cancelling long running operations.
@ -689,11 +689,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Azure::Core::Nullable<bool> ContinueOnFailure;
};
using UpdateDataLakePathAccessControlRecursiveListSinglePageOptions
= SetDataLakePathAccessControlRecursiveListSinglePageOptions;
using UpdateDataLakePathAccessControlListRecursiveSinglePageOptions
= SetDataLakePathAccessControlListRecursiveSinglePageOptions;
using RemoveDataLakePathAccessControlRecursiveListSinglePageOptions
= SetDataLakePathAccessControlRecursiveListSinglePageOptions;
using RemoveDataLakePathAccessControlListRecursiveSinglePageOptions
= SetDataLakePathAccessControlListRecursiveSinglePageOptions;
using CreateDataLakeFileOptions = CreateDataLakePathOptions;
using CreateDataLakeDirectoryOptions = CreateDataLakePathOptions;
@ -756,13 +756,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* ScheduleFileExpiryOriginType::RelativeToCreation or
* ScheduleFileExpiryOriginType::RelativeToNow.
*/
Azure::Core::Nullable<int64_t> TimeToExpireInMs;
Azure::Core::Nullable<std::chrono::milliseconds> TimeToExpire;
/**
* @brief The expiry time in RFC1123 format. Only work if ExpiryOrigin is
* ScheduleFileExpiryOriginType::Absolute.
*/
Azure::Core::Nullable<std::string> ExpiresOn;
Azure::Core::Nullable<Core::DateTime> ExpiresOn;
};
using AcquireDataLakeLeaseOptions = Blobs::AcquireBlobLeaseOptions;

44
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp
View File

@ -212,16 +212,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* @param options Optional parameters to set an access control recursively to the resource the
* directory points to.
* @return
* Azure::Core::Response<Models::SetDataLakePathAccessControlRecursiveListSinglePageResult>
* Azure::Core::Response<Models::SetDataLakePathAccessControlListRecursiveSinglePageResult>
* @remark This request is sent to dfs endpoint.
*/
Azure::Core::Response<Models::SetDataLakePathAccessControlRecursiveListSinglePageResult>
SetAccessControlRecursiveListSinglePage(
Azure::Core::Response<Models::SetDataLakePathAccessControlListRecursiveSinglePageResult>
SetAccessControlListRecursiveSinglePage(
const std::vector<Models::Acl>& acls,
const SetDataLakePathAccessControlRecursiveListSinglePageOptions& options
= SetDataLakePathAccessControlRecursiveListSinglePageOptions()) const
const SetDataLakePathAccessControlListRecursiveSinglePageOptions& options
= SetDataLakePathAccessControlListRecursiveSinglePageOptions()) const
{
return SetAccessControlRecursiveListSinglePageInternal(
return SetAccessControlListRecursiveSinglePageInternal(
Models::PathSetAccessControlRecursiveMode::Set, acls, options);
}
@ -233,16 +233,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* @param options Optional parameters to set an access control recursively to the resource the
* directory points to.
* @return
* Azure::Core::Response<Models::UpdateDataLakePathAccessControlRecursiveListSinglePageResult>
* Azure::Core::Response<Models::UpdateDataLakePathAccessControlListRecursiveSinglePageResult>
* @remark This request is sent to dfs endpoint.
*/
Azure::Core::Response<Models::UpdateDataLakePathAccessControlRecursiveListSinglePageResult>
UpdateAccessControlRecursiveListSinglePage(
Azure::Core::Response<Models::UpdateDataLakePathAccessControlListRecursiveSinglePageResult>
UpdateAccessControlListRecursiveSinglePage(
const std::vector<Models::Acl>& acls,
const UpdateDataLakePathAccessControlRecursiveListSinglePageOptions& options
= UpdateDataLakePathAccessControlRecursiveListSinglePageOptions()) const
const UpdateDataLakePathAccessControlListRecursiveSinglePageOptions& options
= UpdateDataLakePathAccessControlListRecursiveSinglePageOptions()) const
{
return SetAccessControlRecursiveListSinglePageInternal(
return SetAccessControlListRecursiveSinglePageInternal(
Models::PathSetAccessControlRecursiveMode::Modify, acls, options);
}
@ -254,16 +254,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* @param options Optional parameters to set an access control recursively to the resource the
* directory points to.
* @return
* Azure::Core::Response<Models::RemoveDataLakePathAccessControlRecursiveListSinglePageResult>
* Azure::Core::Response<Models::RemoveDataLakePathAccessControlListRecursiveSinglePageResult>
* @remark This request is sent to dfs endpoint.
*/
Azure::Core::Response<Models::RemoveDataLakePathAccessControlRecursiveListSinglePageResult>
RemoveAccessControlRecursiveListSinglePage(
Azure::Core::Response<Models::RemoveDataLakePathAccessControlListRecursiveSinglePageResult>
RemoveAccessControlListRecursiveSinglePage(
const std::vector<Models::Acl>& acls,
const RemoveDataLakePathAccessControlRecursiveListSinglePageOptions& options
= RemoveDataLakePathAccessControlRecursiveListSinglePageOptions()) const
const RemoveDataLakePathAccessControlListRecursiveSinglePageOptions& options
= RemoveDataLakePathAccessControlListRecursiveSinglePageOptions()) const
{
return SetAccessControlRecursiveListSinglePageInternal(
return SetAccessControlListRecursiveSinglePageInternal(
Models::PathSetAccessControlRecursiveMode::Remove, acls, options);
}
@ -281,12 +281,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
}
Azure::Core::Response<Models::SetDataLakePathAccessControlRecursiveListSinglePageResult>
SetAccessControlRecursiveListSinglePageInternal(
Azure::Core::Response<Models::SetDataLakePathAccessControlListRecursiveSinglePageResult>
SetAccessControlListRecursiveSinglePageInternal(
Models::PathSetAccessControlRecursiveMode mode,
const std::vector<Models::Acl>& acls,
const SetDataLakePathAccessControlRecursiveListSinglePageOptions& options
= SetDataLakePathAccessControlRecursiveListSinglePageOptions()) const;
const SetDataLakePathAccessControlListRecursiveSinglePageOptions& options
= SetDataLakePathAccessControlListRecursiveSinglePageOptions()) const;
friend class DataLakeFileSystemClient;
friend class DataLakeLeaseClient;

10
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp
View File

@ -280,12 +280,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
std::string RequestId;
};
using SetDataLakePathAccessControlRecursiveListSinglePageResult
using SetDataLakePathAccessControlListRecursiveSinglePageResult
= Details::PathSetAccessControlRecursiveResult;
using UpdateDataLakePathAccessControlRecursiveListSinglePageResult
= SetDataLakePathAccessControlRecursiveListSinglePageResult;
using RemoveDataLakePathAccessControlRecursiveListSinglePageResult
= SetDataLakePathAccessControlRecursiveListSinglePageResult;
using UpdateDataLakePathAccessControlListRecursiveSinglePageResult
= SetDataLakePathAccessControlListRecursiveSinglePageResult;
using RemoveDataLakePathAccessControlListRecursiveSinglePageResult
= SetDataLakePathAccessControlListRecursiveSinglePageResult;
using CreateDataLakeDirectoryResult = CreateDataLakePathResult;
using DeleteDataLakeDirectoryResult = DeleteDataLakePathResult;

28
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/protocol/datalake_rest_client.hpp
View File

@ -113,7 +113,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
bool IsDirectory = bool();
Core::DateTime LastModified;
std::string ETag;
int64_t ContentLength = int64_t();
int64_t FileSize = int64_t();
std::string Owner;
std::string Group;
std::string Permissions;
@ -311,9 +311,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
struct SetAccessControlRecursiveResponse
{
int32_t DirectoriesSuccessful = int32_t();
int32_t FilesSuccessful = int32_t();
int32_t FailureCount = int32_t();
int32_t NumberOfSuccessfulDirectories = int32_t();
int32_t NumberOfSuccessfulFiles = int32_t();
int32_t NumberOfFailures = int32_t();
std::vector<AclFailedEntry> FailedEntries;
};
@ -370,9 +370,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
struct PathSetAccessControlRecursiveResult
{
int32_t DirectoriesSuccessful = int32_t();
int32_t FilesSuccessful = int32_t();
int32_t FailureCount = int32_t();
int32_t NumberOfSuccessfulDirectories = int32_t();
int32_t NumberOfSuccessfulFiles = int32_t();
int32_t NumberOfFailures = int32_t();
std::vector<AclFailedEntry> FailedEntries;
Azure::Core::Nullable<std::string> ContinuationToken;
std::string RequestId;
@ -504,7 +504,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
result.ETag = node["etag"].get<std::string>();
if (node.contains("contentLength"))
{
result.ContentLength = std::stoll(node["contentLength"].get<std::string>());
result.FileSize = std::stoll(node["contentLength"].get<std::string>());
}
result.Owner = node["owner"].get<std::string>();
result.Group = node["group"].get<std::string>();
@ -1407,9 +1407,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
const Azure::Core::Internal::Json::json& node)
{
SetAccessControlRecursiveResponse result;
result.DirectoriesSuccessful = node["directoriesSuccessful"].get<int32_t>();
result.FilesSuccessful = node["filesSuccessful"].get<int32_t>();
result.FailureCount = node["failureCount"].get<int32_t>();
result.NumberOfSuccessfulDirectories = node["directoriesSuccessful"].get<int32_t>();
result.NumberOfSuccessfulFiles = node["filesSuccessful"].get<int32_t>();
result.NumberOfFailures = node["failureCount"].get<int32_t>();
for (const auto& element : node["failedEntries"])
{
result.FailedEntries.emplace_back(AclFailedEntryFromJson(element));
@ -1422,9 +1422,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
SetAccessControlRecursiveResponse object)
{
PathSetAccessControlRecursiveResult result;
result.DirectoriesSuccessful = object.DirectoriesSuccessful;
result.FilesSuccessful = object.FilesSuccessful;
result.FailureCount = object.FailureCount;
result.NumberOfSuccessfulDirectories = object.NumberOfSuccessfulDirectories;
result.NumberOfSuccessfulFiles = object.NumberOfSuccessfulFiles;
result.NumberOfFailures = object.NumberOfFailures;
result.FailedEntries = std::move(object.FailedEntries);
return result;

11
sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp
View File

@ -455,18 +455,19 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
Blobs::Details::BlobRestClient::Blob::SetBlobExpiryOptions protocolLayerOptions;
protocolLayerOptions.ExpiryOrigin = expiryOrigin;
if (options.ExpiresOn.HasValue() && options.TimeToExpireInMs.HasValue())
if (options.ExpiresOn.HasValue() && options.TimeToExpire.HasValue())
{
// ExpiresOn and TimeToExpireInMs should be mutually exlusive.
// ExpiresOn and TimeToExpire should be mutually exlusive.
std::abort();
}
if (options.ExpiresOn.HasValue())
{
protocolLayerOptions.ExpiryTime = options.ExpiresOn;
protocolLayerOptions.ExpiryTime
= options.ExpiresOn.GetValue().GetString(Azure::Core::DateTime::DateFormat::Rfc1123);
}
else if (options.TimeToExpireInMs.HasValue())
else if (options.TimeToExpire.HasValue())
{
protocolLayerOptions.ExpiryTime = std::to_string(options.TimeToExpireInMs.GetValue());
protocolLayerOptions.ExpiryTime = std::to_string(options.TimeToExpire.GetValue().count());
}
return Blobs::Details::BlobRestClient::Blob::ScheduleDeletion(
options.Context, *m_pipeline, m_blobClient.m_blobUrl, protocolLayerOptions);

6
sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp
View File

@ -471,11 +471,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
std::move(ret), result.ExtractRawResponse());
}
Azure::Core::Response<Models::SetDataLakePathAccessControlRecursiveListSinglePageResult>
DataLakePathClient::SetAccessControlRecursiveListSinglePageInternal(
Azure::Core::Response<Models::SetDataLakePathAccessControlListRecursiveSinglePageResult>
DataLakePathClient::SetAccessControlListRecursiveSinglePageInternal(
Models::PathSetAccessControlRecursiveMode mode,
const std::vector<Models::Acl>& acls,
const SetDataLakePathAccessControlRecursiveListSinglePageOptions& options) const
const SetDataLakePathAccessControlListRecursiveSinglePageOptions& options) const
{
Details::DataLakeRestClient::Path::SetAccessControlRecursiveOptions protocolLayerOptions;
protocolLayerOptions.Mode = mode;

8
sdk/storage/azure-storage-files-datalake/test/datalake_directory_client_test.cpp
View File

@ -345,7 +345,7 @@ namespace Azure { namespace Storage { namespace Test {
{
// Set Acls recursive.
std::vector<Files::DataLake::Models::Acl> acls = GetValidAcls();
EXPECT_NO_THROW(rootDirectoryClient.SetAccessControlRecursiveListSinglePage(acls));
EXPECT_NO_THROW(rootDirectoryClient.SetAccessControlListRecursiveSinglePage(acls));
std::vector<Files::DataLake::Models::Acl> resultAcls1;
std::vector<Files::DataLake::Models::Acl> resultAcls2;
EXPECT_NO_THROW(resultAcls1 = directoryClient1.GetAccessControlList()->Acls);
@ -372,7 +372,7 @@ namespace Azure { namespace Storage { namespace Test {
newAcl.Permissions = "rw-";
std::vector<Files::DataLake::Models::Acl> acls;
acls.emplace_back(std::move(newAcl));
EXPECT_NO_THROW(rootDirectoryClient.UpdateAccessControlRecursiveListSinglePage(acls));
EXPECT_NO_THROW(rootDirectoryClient.UpdateAccessControlListRecursiveSinglePage(acls));
std::vector<Files::DataLake::Models::Acl> resultAcls1;
std::vector<Files::DataLake::Models::Acl> resultAcls2;
EXPECT_NO_THROW(resultAcls1 = directoryClient1.GetAccessControlList()->Acls);
@ -449,7 +449,7 @@ namespace Azure { namespace Storage { namespace Test {
removeAcl.Id = "72a3f86f-271f-439e-b031-25678907d381";
std::vector<Files::DataLake::Models::Acl> acls;
acls.emplace_back(std::move(removeAcl));
EXPECT_NO_THROW(rootDirectoryClient.RemoveAccessControlRecursiveListSinglePage(acls));
EXPECT_NO_THROW(rootDirectoryClient.RemoveAccessControlListRecursiveSinglePage(acls));
std::vector<Files::DataLake::Models::Acl> resultAcls1;
std::vector<Files::DataLake::Models::Acl> resultAcls2;
EXPECT_NO_THROW(resultAcls1 = directoryClient1.GetAccessControlList()->Acls);
@ -525,7 +525,7 @@ namespace Azure { namespace Storage { namespace Test {
newAcl.Permissions = "rw-";
acls.emplace_back(std::move(newAcl));
}
(rootDirectoryClient.SetAccessControlRecursiveListSinglePage(acls));
(rootDirectoryClient.SetAccessControlListRecursiveSinglePage(acls));
std::vector<Files::DataLake::Models::Acl> resultAcls1;
std::vector<Files::DataLake::Models::Acl> resultAcls2;
EXPECT_NO_THROW(resultAcls1 = directoryClient1.GetAccessControlList()->Acls);

9
sdk/storage/azure-storage-files-datalake/test/datalake_file_client_test.cpp
View File

@ -429,7 +429,7 @@ namespace Azure { namespace Storage { namespace Test {
client.ScheduleDeletion(
Files::DataLake::ScheduleDataLakeFileExpiryOriginType::RelativeToNow, options),
StorageException);
options.TimeToExpireInMs = 1000;
options.TimeToExpire = std::chrono::milliseconds(1000);
EXPECT_NO_THROW(client.ScheduleDeletion(
Files::DataLake::ScheduleDataLakeFileExpiryOriginType::RelativeToNow, options));
}
@ -441,13 +441,14 @@ namespace Azure { namespace Storage { namespace Test {
client.ScheduleDeletion(
Files::DataLake::ScheduleDataLakeFileExpiryOriginType::Absolute, options),
StorageException);
options.TimeToExpireInMs = 1000;
options.TimeToExpire = std::chrono::milliseconds(1000);
EXPECT_THROW(
client.ScheduleDeletion(
Files::DataLake::ScheduleDataLakeFileExpiryOriginType::Absolute, options),
StorageException);
options.ExpiresOn = "Tue, 29 Sep 2100 09:53:03 GMT";
options.TimeToExpireInMs = Azure::Core::Nullable<int64_t>();
options.ExpiresOn = Azure::Core::DateTime::Parse(
"Wed, 29 Sep 2100 09:53:03 GMT", Azure::Core::DateTime::DateFormat::Rfc1123);
options.TimeToExpire = Azure::Core::Nullable<std::chrono::milliseconds>();
EXPECT_NO_THROW(client.ScheduleDeletion(
Files::DataLake::ScheduleDataLakeFileExpiryOriginType::Absolute, options));
}