github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refined returned properties for list/download API in DataLake. (#1580)

Browse Source 
* Refined returned properties for list/download API in DataLake.

* Refine readme

* Resolve review comments
This commit is contained in:
Kan Tang 2021-02-03 18:23:37 +08:00 committed by GitHub
parent d26e148c86
commit d0a6f7e8f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 123 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sdk/storage/azure-storage-files-datalake/CHANGELOG.md
View File

@ -17,7 +17,6 @@
- Added `RequestId` in each return type for REST API calls, except for concurrent APIs. - Added `RequestId` in each return type for REST API calls, except for concurrent APIs.
- Added `UpdateAccessControlListRecursiveSinglePage` to update the access control recursively for a datalake path. - Added `UpdateAccessControlListRecursiveSinglePage` to update the access control recursively for a datalake path.
- Added `RemoveAccessControlListRecursiveSinglePage` to remove the access control recursively for a datalake path. - Added `RemoveAccessControlListRecursiveSinglePage` to remove the access control recursively for a datalake path.
- Added some new properties in `GetDataLakePathPropertiesResult` and `DownloadDataLakeFileResult`.
### Breaking Changes ### Breaking Changes
@ -49,7 +48,8 @@
- Removed `DataLakeDirectoryClient::Delete` and `DataLakeDirectoryClient::DeleteIfExists`. Added `DataLakeDirectoryClient::DeleteEmpty`, `DataLakeDirectoryClient::DeleteEmptyIfExists`, `DataLakeDirectoryClient::DeleteRecursive` and `DataLakeDirectoryClient::DeleteRecursiveIfExists` instead. - Removed `DataLakeDirectoryClient::Delete` and `DataLakeDirectoryClient::DeleteIfExists`. Added `DataLakeDirectoryClient::DeleteEmpty`, `DataLakeDirectoryClient::DeleteEmptyIfExists`, `DataLakeDirectoryClient::DeleteRecursive` and `DataLakeDirectoryClient::DeleteRecursiveIfExists` instead.
- Removed `ContinuationToken` in `DeleteDataLakePathResult` and `DeleteDataLakeDirectoryResult`, as they will never be returned for HNS enabled accounts. - Removed `ContinuationToken` in `DeleteDataLakePathResult` and `DeleteDataLakeDirectoryResult`, as they will never be returned for HNS enabled accounts.
- Renamed `DataLakeFileClient::Read` to `DataLakeFileClient::Download`. Also changed the member `Azure::Core::Nullable<bool> RangeGetContentMd5` in the option to be `Azure::Core::Nullable<HashAlgorithm> RangeHashAlgorithm` instead. - Renamed `DataLakeFileClient::Read` to `DataLakeFileClient::Download`. Also changed the member `Azure::Core::Nullable<bool> RangeGetContentMd5` in the option to be `Azure::Core::Nullable<HashAlgorithm> RangeHashAlgorithm` instead.
- Moved some less commonly used properties into a details data structure for `Download`, `DownloadTo` and `ListFileSystemsSinglePage` API, and enriched the content of the mentioned details data structure.
### Other Changes and Improvements ### Other Changes and Improvements
- Changed `DataLakeFileClient::Flush`'s `endingOffset` parameter's name to `position`. - Changed `DataLakeFileClient::Flush`'s `endingOffset` parameter's name to `position`.

39
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp
View File

@ -18,9 +18,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
using GetUserDelegationKeyResult = Blobs::Models::GetUserDelegationKeyResult; using GetUserDelegationKeyResult = Blobs::Models::GetUserDelegationKeyResult;
using UserDelegationKey = Blobs::Models::UserDelegationKey; using UserDelegationKey = Blobs::Models::UserDelegationKey;
struct FileSystemItem struct FileSystemItemDetails
{ {
std::string Name;
Azure::Core::ETag ETag; Azure::Core::ETag ETag;
Azure::Core::DateTime LastModified; Azure::Core::DateTime LastModified;
Storage::Metadata Metadata; Storage::Metadata Metadata;
@ -30,6 +29,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
Azure::Core::Nullable<LeaseDurationType> LeaseDuration; Azure::Core::Nullable<LeaseDurationType> LeaseDuration;
LeaseStateType LeaseState = LeaseStateType::Available; LeaseStateType LeaseState = LeaseStateType::Available;
LeaseStatusType LeaseStatus = LeaseStatusType::Unlocked; LeaseStatusType LeaseStatus = LeaseStatusType::Unlocked;
}; // struct FileSystemItemDetails
struct FileSystemItem
{
std::string Name;
FileSystemItemDetails Details;
}; // struct BlobContainerItem }; // struct BlobContainerItem
struct ListFileSystemsSinglePageResult struct ListFileSystemsSinglePageResult
@ -218,13 +223,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
using ScheduleDataLakeFileDeletionResult = Blobs::Models::SetBlobExpiryResult; using ScheduleDataLakeFileDeletionResult = Blobs::Models::SetBlobExpiryResult;
using CopyStatus = Blobs::Models::CopyStatus; using CopyStatus = Blobs::Models::CopyStatus;
struct DownloadDataLakeFileResult struct DownloadDataLakeFileDetails
{ {
std::unique_ptr<Azure::Core::Http::BodyStream> Body;
PathHttpHeaders HttpHeaders;
int64_t FileSize = int64_t();
Azure::Core::Http::Range ContentRange;
Azure::Core::Nullable<Storage::ContentHash> TransactionalContentHash;
Azure::Core::ETag ETag; Azure::Core::ETag ETag;
Core::DateTime LastModified; Core::DateTime LastModified;
Azure::Core::Nullable<LeaseDurationType> LeaseDuration; Azure::Core::Nullable<LeaseDurationType> LeaseDuration;
@ -242,6 +242,19 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
Azure::Core::Nullable<Azure::Core::DateTime> CopyCompletedOn; Azure::Core::Nullable<Azure::Core::DateTime> CopyCompletedOn;
Azure::Core::Nullable<std::string> VersionId; Azure::Core::Nullable<std::string> VersionId;
Azure::Core::Nullable<bool> IsCurrentVersion; Azure::Core::Nullable<bool> IsCurrentVersion;
bool IsServerEncrypted = false;
Azure::Core::Nullable<std::vector<uint8_t>> EncryptionKeySha256;
Azure::Core::Nullable<std::string> EncryptionScope;
};
struct DownloadDataLakeFileResult
{
std::unique_ptr<Azure::Core::Http::BodyStream> Body;
PathHttpHeaders HttpHeaders;
int64_t FileSize = int64_t();
Azure::Core::Http::Range ContentRange;
Azure::Core::Nullable<Storage::ContentHash> TransactionalContentHash;
DownloadDataLakeFileDetails Details;
std::string RequestId; std::string RequestId;
}; };
@ -253,14 +266,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
struct DownloadDataLakeFileToResult struct DownloadDataLakeFileToResult
{ {
Azure::Core::ETag ETag; int64_t FileSize = int64_t();
Core::DateTime LastModified; Azure::Core::Http::Range ContentRange;
int64_t ContentLength = 0;
PathHttpHeaders HttpHeaders; PathHttpHeaders HttpHeaders;
Storage::Metadata Metadata; DownloadDataLakeFileDetails Details;
Azure::Core::Nullable<bool> ServerEncrypted;
Azure::Core::Nullable<std::vector<uint8_t>> EncryptionKeySha256;
std::string RequestId;
}; };
using CreateDataLakeFileResult = CreateDataLakePathResult; using CreateDataLakeFileResult = CreateDataLakePathResult;

116
sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp
View File

@ -305,30 +305,34 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
ret.ContentRange = std::move(result->ContentRange); ret.ContentRange = std::move(result->ContentRange);
ret.FileSize = result->BlobSize; ret.FileSize = result->BlobSize;
ret.TransactionalContentHash = std::move(result->TransactionalContentHash); ret.TransactionalContentHash = std::move(result->TransactionalContentHash);
ret.ETag = std::move(result->Details.ETag); ret.Details.ETag = std::move(result->Details.ETag);
ret.LastModified = std::move(result->Details.LastModified); ret.Details.LastModified = std::move(result->Details.LastModified);
if (result->Details.LeaseDuration.HasValue()) if (result->Details.LeaseDuration.HasValue())
{ {
ret.LeaseDuration = Models::LeaseDurationType(result->Details.LeaseDuration.GetValue().Get()); ret.Details.LeaseDuration
= Models::LeaseDurationType(result->Details.LeaseDuration.GetValue().Get());
} }
ret.LeaseState = result->Details.LeaseState.HasValue() ret.Details.LeaseState = result->Details.LeaseState.HasValue()
? FromBlobLeaseState(result->Details.LeaseState.GetValue()) ? FromBlobLeaseState(result->Details.LeaseState.GetValue())
: ret.LeaseState; : ret.Details.LeaseState;
ret.LeaseStatus = result->Details.LeaseStatus.HasValue() ret.Details.LeaseStatus = result->Details.LeaseStatus.HasValue()
? FromBlobLeaseStatus(result->Details.LeaseStatus.GetValue()) ? FromBlobLeaseStatus(result->Details.LeaseStatus.GetValue())
: ret.LeaseStatus; : ret.Details.LeaseStatus;
ret.Metadata = std::move(result->Details.Metadata); ret.Details.Metadata = std::move(result->Details.Metadata);
ret.CreatedOn = std::move(result->Details.CreatedOn); ret.Details.CreatedOn = std::move(result->Details.CreatedOn);
ret.ExpiresOn = std::move(result->Details.ExpiresOn); ret.Details.ExpiresOn = std::move(result->Details.ExpiresOn);
ret.LastAccessedOn = std::move(result->Details.LastAccessedOn); ret.Details.LastAccessedOn = std::move(result->Details.LastAccessedOn);
ret.CopyId = std::move(result->Details.CopyId); ret.Details.CopyId = std::move(result->Details.CopyId);
ret.CopySource = std::move(result->Details.CopySource); ret.Details.CopySource = std::move(result->Details.CopySource);
ret.CopyStatus = std::move(result->Details.CopyStatus); ret.Details.CopyStatus = std::move(result->Details.CopyStatus);
ret.CopyStatusDescription = std::move(result->Details.CopyStatusDescription); ret.Details.CopyStatusDescription = std::move(result->Details.CopyStatusDescription);
ret.CopyProgress = std::move(result->Details.CopyProgress); ret.Details.CopyProgress = std::move(result->Details.CopyProgress);
ret.CopyCompletedOn = std::move(result->Details.CopyCompletedOn); ret.Details.CopyCompletedOn = std::move(result->Details.CopyCompletedOn);
ret.VersionId = std::move(result->Details.VersionId); ret.Details.VersionId = std::move(result->Details.VersionId);
ret.IsCurrentVersion = std::move(result->Details.IsCurrentVersion); ret.Details.IsCurrentVersion = std::move(result->Details.IsCurrentVersion);
ret.Details.EncryptionKeySha256 = std::move(result->Details.EncryptionKeySha256);
ret.Details.EncryptionScope = std::move(result->Details.EncryptionScope);
ret.Details.IsServerEncrypted = result->Details.IsServerEncrypted;
ret.RequestId = std::move(result->RequestId); ret.RequestId = std::move(result->RequestId);
return Azure::Core::Response<Models::DownloadDataLakeFileResult>( return Azure::Core::Response<Models::DownloadDataLakeFileResult>(
std::move(ret), result.ExtractRawResponse()); std::move(ret), result.ExtractRawResponse());
@ -372,13 +376,38 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{ {
auto result = m_blockBlobClient.DownloadTo(buffer, bufferSize, options); auto result = m_blockBlobClient.DownloadTo(buffer, bufferSize, options);
Models::DownloadDataLakeFileToResult ret; Models::DownloadDataLakeFileToResult ret;
ret.ETag = std::move(result->Details.ETag); ret.ContentRange.Length = result->ContentRange.Length.GetValue();
ret.LastModified = std::move(result->Details.LastModified); ret.ContentRange.Offset = result->ContentRange.Offset;
ret.ContentLength = result->ContentRange.Length.GetValue(); ret.FileSize = result->BlobSize;
ret.HttpHeaders = FromBlobHttpHeaders(std::move(result->Details.HttpHeaders)); ret.HttpHeaders = FromBlobHttpHeaders(std::move(result->Details.HttpHeaders));
ret.Metadata = std::move(result->Details.Metadata); ret.Details.ETag = std::move(result->Details.ETag);
ret.ServerEncrypted = result->Details.IsServerEncrypted; ret.Details.LastModified = std::move(result->Details.LastModified);
ret.EncryptionKeySha256 = std::move(result->Details.EncryptionKeySha256); if (result->Details.LeaseDuration.HasValue())
{
ret.Details.LeaseDuration
= Models::LeaseDurationType(result->Details.LeaseDuration.GetValue().Get());
}
ret.Details.LeaseState = result->Details.LeaseState.HasValue()
? FromBlobLeaseState(result->Details.LeaseState.GetValue())
: ret.Details.LeaseState;
ret.Details.LeaseStatus = result->Details.LeaseStatus.HasValue()
? FromBlobLeaseStatus(result->Details.LeaseStatus.GetValue())
: ret.Details.LeaseStatus;
ret.Details.Metadata = std::move(result->Details.Metadata);
ret.Details.CreatedOn = std::move(result->Details.CreatedOn);
ret.Details.ExpiresOn = std::move(result->Details.ExpiresOn);
ret.Details.LastAccessedOn = std::move(result->Details.LastAccessedOn);
ret.Details.CopyId = std::move(result->Details.CopyId);
ret.Details.CopySource = std::move(result->Details.CopySource);
ret.Details.CopyStatus = std::move(result->Details.CopyStatus);
ret.Details.CopyStatusDescription = std::move(result->Details.CopyStatusDescription);
ret.Details.CopyProgress = std::move(result->Details.CopyProgress);
ret.Details.CopyCompletedOn = std::move(result->Details.CopyCompletedOn);
ret.Details.VersionId = std::move(result->Details.VersionId);
ret.Details.IsCurrentVersion = std::move(result->Details.IsCurrentVersion);
ret.Details.EncryptionKeySha256 = std::move(result->Details.EncryptionKeySha256);
ret.Details.EncryptionScope = std::move(result->Details.EncryptionScope);
ret.Details.IsServerEncrypted = result->Details.IsServerEncrypted;
return Azure::Core::Response<Models::DownloadDataLakeFileToResult>( return Azure::Core::Response<Models::DownloadDataLakeFileToResult>(
std::move(ret), result.ExtractRawResponse()); std::move(ret), result.ExtractRawResponse());
} }
@ -389,13 +418,38 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{ {
auto result = m_blockBlobClient.DownloadTo(fileName, options); auto result = m_blockBlobClient.DownloadTo(fileName, options);
Models::DownloadDataLakeFileToResult ret; Models::DownloadDataLakeFileToResult ret;
ret.ETag = std::move(result->Details.ETag); ret.ContentRange.Length = result->ContentRange.Length.GetValue();
ret.LastModified = std::move(result->Details.LastModified); ret.ContentRange.Offset = result->ContentRange.Offset;
ret.ContentLength = result->ContentRange.Length.GetValue(); ret.FileSize = result->BlobSize;
ret.HttpHeaders = FromBlobHttpHeaders(std::move(result->Details.HttpHeaders)); ret.HttpHeaders = FromBlobHttpHeaders(std::move(result->Details.HttpHeaders));
ret.Metadata = std::move(result->Details.Metadata); ret.Details.ETag = std::move(result->Details.ETag);
ret.ServerEncrypted = result->Details.IsServerEncrypted; ret.Details.LastModified = std::move(result->Details.LastModified);
ret.EncryptionKeySha256 = std::move(result->Details.EncryptionKeySha256); if (result->Details.LeaseDuration.HasValue())
{
ret.Details.LeaseDuration
= Models::LeaseDurationType(result->Details.LeaseDuration.GetValue().Get());
}
ret.Details.LeaseState = result->Details.LeaseState.HasValue()
? FromBlobLeaseState(result->Details.LeaseState.GetValue())
: ret.Details.LeaseState;
ret.Details.LeaseStatus = result->Details.LeaseStatus.HasValue()
? FromBlobLeaseStatus(result->Details.LeaseStatus.GetValue())
: ret.Details.LeaseStatus;
ret.Details.Metadata = std::move(result->Details.Metadata);
ret.Details.CreatedOn = std::move(result->Details.CreatedOn);
ret.Details.ExpiresOn = std::move(result->Details.ExpiresOn);
ret.Details.LastAccessedOn = std::move(result->Details.LastAccessedOn);
ret.Details.CopyId = std::move(result->Details.CopyId);
ret.Details.CopySource = std::move(result->Details.CopySource);
ret.Details.CopyStatus = std::move(result->Details.CopyStatus);
ret.Details.CopyStatusDescription = std::move(result->Details.CopyStatusDescription);
ret.Details.CopyProgress = std::move(result->Details.CopyProgress);
ret.Details.CopyCompletedOn = std::move(result->Details.CopyCompletedOn);
ret.Details.VersionId = std::move(result->Details.VersionId);
ret.Details.IsCurrentVersion = std::move(result->Details.IsCurrentVersion);
ret.Details.EncryptionKeySha256 = std::move(result->Details.EncryptionKeySha256);
ret.Details.EncryptionScope = std::move(result->Details.EncryptionScope);
ret.Details.IsServerEncrypted = result->Details.IsServerEncrypted;
return Azure::Core::Response<Models::DownloadDataLakeFileToResult>( return Azure::Core::Response<Models::DownloadDataLakeFileToResult>(
std::move(ret), result.ExtractRawResponse()); std::move(ret), result.ExtractRawResponse());
} }

24
sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp
View File

@ -43,34 +43,34 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{ {
Models::FileSystemItem fileSystem; Models::FileSystemItem fileSystem;
fileSystem.Name = std::move(item.Name); fileSystem.Name = std::move(item.Name);
fileSystem.ETag = std::move(item.Details.ETag); fileSystem.Details.ETag = std::move(item.Details.ETag);
fileSystem.LastModified = std::move(item.Details.LastModified); fileSystem.Details.LastModified = std::move(item.Details.LastModified);
fileSystem.Metadata = std::move(item.Details.Metadata); fileSystem.Details.Metadata = std::move(item.Details.Metadata);
if (item.Details.AccessType == Blobs::Models::PublicAccessType::BlobContainer) if (item.Details.AccessType == Blobs::Models::PublicAccessType::BlobContainer)
{ {
fileSystem.AccessType = Models::PublicAccessType::FileSystem; fileSystem.Details.AccessType = Models::PublicAccessType::FileSystem;
} }
else if (item.Details.AccessType == Blobs::Models::PublicAccessType::Blob) else if (item.Details.AccessType == Blobs::Models::PublicAccessType::Blob)
{ {
fileSystem.AccessType = Models::PublicAccessType::Path; fileSystem.Details.AccessType = Models::PublicAccessType::Path;
} }
else if (item.Details.AccessType == Blobs::Models::PublicAccessType::None) else if (item.Details.AccessType == Blobs::Models::PublicAccessType::None)
{ {
fileSystem.AccessType = Models::PublicAccessType::None; fileSystem.Details.AccessType = Models::PublicAccessType::None;
} }
else else
{ {
fileSystem.AccessType = Models::PublicAccessType(item.Details.AccessType.Get()); fileSystem.Details.AccessType = Models::PublicAccessType(item.Details.AccessType.Get());
} }
fileSystem.HasImmutabilityPolicy = item.Details.HasImmutabilityPolicy; fileSystem.Details.HasImmutabilityPolicy = item.Details.HasImmutabilityPolicy;
fileSystem.HasLegalHold = item.Details.HasLegalHold; fileSystem.Details.HasLegalHold = item.Details.HasLegalHold;
if (item.Details.LeaseDuration.HasValue()) if (item.Details.LeaseDuration.HasValue())
{ {
fileSystem.LeaseDuration fileSystem.Details.LeaseDuration
= Models::LeaseDurationType((item.Details.LeaseDuration.GetValue().Get())); = Models::LeaseDurationType((item.Details.LeaseDuration.GetValue().Get()));
} }
fileSystem.LeaseState = Models::LeaseStateType(item.Details.LeaseState.Get()); fileSystem.Details.LeaseState = Models::LeaseStateType(item.Details.LeaseState.Get());
fileSystem.LeaseStatus = Models::LeaseStatusType(item.Details.LeaseStatus.Get()); fileSystem.Details.LeaseStatus = Models::LeaseStatusType(item.Details.LeaseStatus.Get());
fileSystems.emplace_back(std::move(fileSystem)); fileSystems.emplace_back(std::move(fileSystem));
} }