Refined returned properties for list/download API in DataLake. (#1580)

* Refined returned properties for list/download API in DataLake.

* Refine readme

* Resolve review comments

sdk/storage/azure-storage-files-datalake/CHANGELOG.md

@@ -17,7 +17,6 @@
 - Added `RequestId` in each return type for REST API calls, except for concurrent APIs.
 - Added `UpdateAccessControlListRecursiveSinglePage` to update the access control recursively for a datalake path.
 - Added `RemoveAccessControlListRecursiveSinglePage` to remove the access control recursively for a datalake path.
-- Added some new properties in `GetDataLakePathPropertiesResult` and `DownloadDataLakeFileResult`.
 
 ### Breaking Changes
 
@@ -49,7 +48,8 @@
 - Removed `DataLakeDirectoryClient::Delete` and `DataLakeDirectoryClient::DeleteIfExists`. Added `DataLakeDirectoryClient::DeleteEmpty`, `DataLakeDirectoryClient::DeleteEmptyIfExists`, `DataLakeDirectoryClient::DeleteRecursive` and `DataLakeDirectoryClient::DeleteRecursiveIfExists` instead.
 - Removed `ContinuationToken` in `DeleteDataLakePathResult` and `DeleteDataLakeDirectoryResult`, as they will never be returned for HNS enabled accounts.
 - Renamed `DataLakeFileClient::Read` to `DataLakeFileClient::Download`. Also changed the member `Azure::Core::Nullable<bool> RangeGetContentMd5` in the option to be `Azure::Core::Nullable<HashAlgorithm> RangeHashAlgorithm` instead.
-
+- Moved some less commonly used properties into a details data structure for `Download`, `DownloadTo` and `ListFileSystemsSinglePage` API, and enriched the content of the mentioned details data structure.
+ 
 ### Other Changes and Improvements
 
 - Changed `DataLakeFileClient::Flush`'s `endingOffset` parameter's name to `position`.

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp

@@ -18,9 +18,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
   using GetUserDelegationKeyResult = Blobs::Models::GetUserDelegationKeyResult;
   using UserDelegationKey = Blobs::Models::UserDelegationKey;
 
-  struct FileSystemItem
+  struct FileSystemItemDetails
   {
-    std::string Name;
     Azure::Core::ETag ETag;
     Azure::Core::DateTime LastModified;
     Storage::Metadata Metadata;
@@ -30,6 +29,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
     Azure::Core::Nullable<LeaseDurationType> LeaseDuration;
     LeaseStateType LeaseState = LeaseStateType::Available;
     LeaseStatusType LeaseStatus = LeaseStatusType::Unlocked;
+  }; // struct FileSystemItemDetails
+
+  struct FileSystemItem
+  {
+    std::string Name;
+    FileSystemItemDetails Details;
   }; // struct BlobContainerItem
 
   struct ListFileSystemsSinglePageResult
@@ -218,13 +223,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
   using ScheduleDataLakeFileDeletionResult = Blobs::Models::SetBlobExpiryResult;
   using CopyStatus = Blobs::Models::CopyStatus;
 
-  struct DownloadDataLakeFileResult
+  struct DownloadDataLakeFileDetails
   {
-    std::unique_ptr<Azure::Core::Http::BodyStream> Body;
-    PathHttpHeaders HttpHeaders;
-    int64_t FileSize = int64_t();
-    Azure::Core::Http::Range ContentRange;
-    Azure::Core::Nullable<Storage::ContentHash> TransactionalContentHash;
     Azure::Core::ETag ETag;
     Core::DateTime LastModified;
     Azure::Core::Nullable<LeaseDurationType> LeaseDuration;
@@ -242,6 +242,19 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
     Azure::Core::Nullable<Azure::Core::DateTime> CopyCompletedOn;
     Azure::Core::Nullable<std::string> VersionId;
     Azure::Core::Nullable<bool> IsCurrentVersion;
+    bool IsServerEncrypted = false;
+    Azure::Core::Nullable<std::vector<uint8_t>> EncryptionKeySha256;
+    Azure::Core::Nullable<std::string> EncryptionScope;
+  };
+
+  struct DownloadDataLakeFileResult
+  {
+    std::unique_ptr<Azure::Core::Http::BodyStream> Body;
+    PathHttpHeaders HttpHeaders;
+    int64_t FileSize = int64_t();
+    Azure::Core::Http::Range ContentRange;
+    Azure::Core::Nullable<Storage::ContentHash> TransactionalContentHash;
+    DownloadDataLakeFileDetails Details;
     std::string RequestId;
   };
 
@@ -253,14 +266,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
 
   struct DownloadDataLakeFileToResult
   {
-    Azure::Core::ETag ETag;
-    Core::DateTime LastModified;
-    int64_t ContentLength = 0;
+    int64_t FileSize = int64_t();
+    Azure::Core::Http::Range ContentRange;
     PathHttpHeaders HttpHeaders;
-    Storage::Metadata Metadata;
-    Azure::Core::Nullable<bool> ServerEncrypted;
-    Azure::Core::Nullable<std::vector<uint8_t>> EncryptionKeySha256;
-    std::string RequestId;
+    DownloadDataLakeFileDetails Details;
   };
 
   using CreateDataLakeFileResult = CreateDataLakePathResult;

sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp

@@ -305,30 +305,34 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     ret.ContentRange = std::move(result->ContentRange);
     ret.FileSize = result->BlobSize;
     ret.TransactionalContentHash = std::move(result->TransactionalContentHash);
-    ret.ETag = std::move(result->Details.ETag);
-    ret.LastModified = std::move(result->Details.LastModified);
+    ret.Details.ETag = std::move(result->Details.ETag);
+    ret.Details.LastModified = std::move(result->Details.LastModified);
     if (result->Details.LeaseDuration.HasValue())
     {
-      ret.LeaseDuration = Models::LeaseDurationType(result->Details.LeaseDuration.GetValue().Get());
+      ret.Details.LeaseDuration
+          = Models::LeaseDurationType(result->Details.LeaseDuration.GetValue().Get());
     }
-    ret.LeaseState = result->Details.LeaseState.HasValue()
+    ret.Details.LeaseState = result->Details.LeaseState.HasValue()
         ? FromBlobLeaseState(result->Details.LeaseState.GetValue())
-        : ret.LeaseState;
-    ret.LeaseStatus = result->Details.LeaseStatus.HasValue()
+        : ret.Details.LeaseState;
+    ret.Details.LeaseStatus = result->Details.LeaseStatus.HasValue()
         ? FromBlobLeaseStatus(result->Details.LeaseStatus.GetValue())
-        : ret.LeaseStatus;
-    ret.Metadata = std::move(result->Details.Metadata);
-    ret.CreatedOn = std::move(result->Details.CreatedOn);
-    ret.ExpiresOn = std::move(result->Details.ExpiresOn);
-    ret.LastAccessedOn = std::move(result->Details.LastAccessedOn);
-    ret.CopyId = std::move(result->Details.CopyId);
-    ret.CopySource = std::move(result->Details.CopySource);
-    ret.CopyStatus = std::move(result->Details.CopyStatus);
-    ret.CopyStatusDescription = std::move(result->Details.CopyStatusDescription);
-    ret.CopyProgress = std::move(result->Details.CopyProgress);
-    ret.CopyCompletedOn = std::move(result->Details.CopyCompletedOn);
-    ret.VersionId = std::move(result->Details.VersionId);
-    ret.IsCurrentVersion = std::move(result->Details.IsCurrentVersion);
+        : ret.Details.LeaseStatus;
+    ret.Details.Metadata = std::move(result->Details.Metadata);
+    ret.Details.CreatedOn = std::move(result->Details.CreatedOn);
+    ret.Details.ExpiresOn = std::move(result->Details.ExpiresOn);
+    ret.Details.LastAccessedOn = std::move(result->Details.LastAccessedOn);
+    ret.Details.CopyId = std::move(result->Details.CopyId);
+    ret.Details.CopySource = std::move(result->Details.CopySource);
+    ret.Details.CopyStatus = std::move(result->Details.CopyStatus);
+    ret.Details.CopyStatusDescription = std::move(result->Details.CopyStatusDescription);
+    ret.Details.CopyProgress = std::move(result->Details.CopyProgress);
+    ret.Details.CopyCompletedOn = std::move(result->Details.CopyCompletedOn);
+    ret.Details.VersionId = std::move(result->Details.VersionId);
+    ret.Details.IsCurrentVersion = std::move(result->Details.IsCurrentVersion);
+    ret.Details.EncryptionKeySha256 = std::move(result->Details.EncryptionKeySha256);
+    ret.Details.EncryptionScope = std::move(result->Details.EncryptionScope);
+    ret.Details.IsServerEncrypted = result->Details.IsServerEncrypted;
     ret.RequestId = std::move(result->RequestId);
     return Azure::Core::Response<Models::DownloadDataLakeFileResult>(
         std::move(ret), result.ExtractRawResponse());
@@ -372,13 +376,38 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
   {
     auto result = m_blockBlobClient.DownloadTo(buffer, bufferSize, options);
     Models::DownloadDataLakeFileToResult ret;
-    ret.ETag = std::move(result->Details.ETag);
-    ret.LastModified = std::move(result->Details.LastModified);
-    ret.ContentLength = result->ContentRange.Length.GetValue();
+    ret.ContentRange.Length = result->ContentRange.Length.GetValue();
+    ret.ContentRange.Offset = result->ContentRange.Offset;
+    ret.FileSize = result->BlobSize;
     ret.HttpHeaders = FromBlobHttpHeaders(std::move(result->Details.HttpHeaders));
-    ret.Metadata = std::move(result->Details.Metadata);
-    ret.ServerEncrypted = result->Details.IsServerEncrypted;
-    ret.EncryptionKeySha256 = std::move(result->Details.EncryptionKeySha256);
+    ret.Details.ETag = std::move(result->Details.ETag);
+    ret.Details.LastModified = std::move(result->Details.LastModified);
+    if (result->Details.LeaseDuration.HasValue())
+    {
+      ret.Details.LeaseDuration
+          = Models::LeaseDurationType(result->Details.LeaseDuration.GetValue().Get());
+    }
+    ret.Details.LeaseState = result->Details.LeaseState.HasValue()
+        ? FromBlobLeaseState(result->Details.LeaseState.GetValue())
+        : ret.Details.LeaseState;
+    ret.Details.LeaseStatus = result->Details.LeaseStatus.HasValue()
+        ? FromBlobLeaseStatus(result->Details.LeaseStatus.GetValue())
+        : ret.Details.LeaseStatus;
+    ret.Details.Metadata = std::move(result->Details.Metadata);
+    ret.Details.CreatedOn = std::move(result->Details.CreatedOn);
+    ret.Details.ExpiresOn = std::move(result->Details.ExpiresOn);
+    ret.Details.LastAccessedOn = std::move(result->Details.LastAccessedOn);
+    ret.Details.CopyId = std::move(result->Details.CopyId);
+    ret.Details.CopySource = std::move(result->Details.CopySource);
+    ret.Details.CopyStatus = std::move(result->Details.CopyStatus);
+    ret.Details.CopyStatusDescription = std::move(result->Details.CopyStatusDescription);
+    ret.Details.CopyProgress = std::move(result->Details.CopyProgress);
+    ret.Details.CopyCompletedOn = std::move(result->Details.CopyCompletedOn);
+    ret.Details.VersionId = std::move(result->Details.VersionId);
+    ret.Details.IsCurrentVersion = std::move(result->Details.IsCurrentVersion);
+    ret.Details.EncryptionKeySha256 = std::move(result->Details.EncryptionKeySha256);
+    ret.Details.EncryptionScope = std::move(result->Details.EncryptionScope);
+    ret.Details.IsServerEncrypted = result->Details.IsServerEncrypted;
     return Azure::Core::Response<Models::DownloadDataLakeFileToResult>(
         std::move(ret), result.ExtractRawResponse());
   }
@@ -389,13 +418,38 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
   {
     auto result = m_blockBlobClient.DownloadTo(fileName, options);
     Models::DownloadDataLakeFileToResult ret;
-    ret.ETag = std::move(result->Details.ETag);
-    ret.LastModified = std::move(result->Details.LastModified);
-    ret.ContentLength = result->ContentRange.Length.GetValue();
+    ret.ContentRange.Length = result->ContentRange.Length.GetValue();
+    ret.ContentRange.Offset = result->ContentRange.Offset;
+    ret.FileSize = result->BlobSize;
     ret.HttpHeaders = FromBlobHttpHeaders(std::move(result->Details.HttpHeaders));
-    ret.Metadata = std::move(result->Details.Metadata);
-    ret.ServerEncrypted = result->Details.IsServerEncrypted;
-    ret.EncryptionKeySha256 = std::move(result->Details.EncryptionKeySha256);
+    ret.Details.ETag = std::move(result->Details.ETag);
+    ret.Details.LastModified = std::move(result->Details.LastModified);
+    if (result->Details.LeaseDuration.HasValue())
+    {
+      ret.Details.LeaseDuration
+          = Models::LeaseDurationType(result->Details.LeaseDuration.GetValue().Get());
+    }
+    ret.Details.LeaseState = result->Details.LeaseState.HasValue()
+        ? FromBlobLeaseState(result->Details.LeaseState.GetValue())
+        : ret.Details.LeaseState;
+    ret.Details.LeaseStatus = result->Details.LeaseStatus.HasValue()
+        ? FromBlobLeaseStatus(result->Details.LeaseStatus.GetValue())
+        : ret.Details.LeaseStatus;
+    ret.Details.Metadata = std::move(result->Details.Metadata);
+    ret.Details.CreatedOn = std::move(result->Details.CreatedOn);
+    ret.Details.ExpiresOn = std::move(result->Details.ExpiresOn);
+    ret.Details.LastAccessedOn = std::move(result->Details.LastAccessedOn);
+    ret.Details.CopyId = std::move(result->Details.CopyId);
+    ret.Details.CopySource = std::move(result->Details.CopySource);
+    ret.Details.CopyStatus = std::move(result->Details.CopyStatus);
+    ret.Details.CopyStatusDescription = std::move(result->Details.CopyStatusDescription);
+    ret.Details.CopyProgress = std::move(result->Details.CopyProgress);
+    ret.Details.CopyCompletedOn = std::move(result->Details.CopyCompletedOn);
+    ret.Details.VersionId = std::move(result->Details.VersionId);
+    ret.Details.IsCurrentVersion = std::move(result->Details.IsCurrentVersion);
+    ret.Details.EncryptionKeySha256 = std::move(result->Details.EncryptionKeySha256);
+    ret.Details.EncryptionScope = std::move(result->Details.EncryptionScope);
+    ret.Details.IsServerEncrypted = result->Details.IsServerEncrypted;
     return Azure::Core::Response<Models::DownloadDataLakeFileToResult>(
         std::move(ret), result.ExtractRawResponse());
   }

sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp

@@ -43,34 +43,34 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
       {
         Models::FileSystemItem fileSystem;
         fileSystem.Name = std::move(item.Name);
-        fileSystem.ETag = std::move(item.Details.ETag);
-        fileSystem.LastModified = std::move(item.Details.LastModified);
-        fileSystem.Metadata = std::move(item.Details.Metadata);
+        fileSystem.Details.ETag = std::move(item.Details.ETag);
+        fileSystem.Details.LastModified = std::move(item.Details.LastModified);
+        fileSystem.Details.Metadata = std::move(item.Details.Metadata);
         if (item.Details.AccessType == Blobs::Models::PublicAccessType::BlobContainer)
         {
-          fileSystem.AccessType = Models::PublicAccessType::FileSystem;
+          fileSystem.Details.AccessType = Models::PublicAccessType::FileSystem;
         }
         else if (item.Details.AccessType == Blobs::Models::PublicAccessType::Blob)
         {
-          fileSystem.AccessType = Models::PublicAccessType::Path;
+          fileSystem.Details.AccessType = Models::PublicAccessType::Path;
         }
         else if (item.Details.AccessType == Blobs::Models::PublicAccessType::None)
         {
-          fileSystem.AccessType = Models::PublicAccessType::None;
+          fileSystem.Details.AccessType = Models::PublicAccessType::None;
         }
         else
         {
-          fileSystem.AccessType = Models::PublicAccessType(item.Details.AccessType.Get());
+          fileSystem.Details.AccessType = Models::PublicAccessType(item.Details.AccessType.Get());
         }
-        fileSystem.HasImmutabilityPolicy = item.Details.HasImmutabilityPolicy;
-        fileSystem.HasLegalHold = item.Details.HasLegalHold;
+        fileSystem.Details.HasImmutabilityPolicy = item.Details.HasImmutabilityPolicy;
+        fileSystem.Details.HasLegalHold = item.Details.HasLegalHold;
         if (item.Details.LeaseDuration.HasValue())
         {
-          fileSystem.LeaseDuration
+          fileSystem.Details.LeaseDuration
               = Models::LeaseDurationType((item.Details.LeaseDuration.GetValue().Get()));
         }
-        fileSystem.LeaseState = Models::LeaseStateType(item.Details.LeaseState.Get());
-        fileSystem.LeaseStatus = Models::LeaseStatusType(item.Details.LeaseStatus.Get());
+        fileSystem.Details.LeaseState = Models::LeaseStateType(item.Details.LeaseState.Get());
+        fileSystem.Details.LeaseStatus = Models::LeaseStatusType(item.Details.LeaseStatus.Get());
 
         fileSystems.emplace_back(std::move(fileSystem));
       }