From 6b71b5416d3b583eab36b8207848ab1cbd5baee8 Mon Sep 17 00:00:00 2001 From: Kan Tang Date: Sun, 24 Jan 2021 18:17:29 -0800 Subject: [PATCH] Refined ListPathsSinglePage (#1446) --- .../azure-storage-files-datalake/CHANGELOG.md | 2 + .../datalake/datalake_directory_client.hpp | 13 ++++++ .../datalake/datalake_file_system_client.hpp | 3 +- .../files/datalake/datalake_options.hpp | 6 --- .../src/datalake_directory_client.cpp | 30 ++++++++++++ .../src/datalake_file_system_client.cpp | 1 - .../test/datalake_file_system_client_test.cpp | 46 +++++++++++++------ .../test/datalake_sas_test.cpp | 5 +- 8 files changed, 80 insertions(+), 26 deletions(-) diff --git a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md index e7d37bf00..0e8765098 100644 --- a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md @@ -10,6 +10,7 @@ - Moved all protocol layer generated result types to `Details` namespace. - Renamed `FileSystem` type returned from `ListDataLakeFileSystems` to be `FileSystemItem`. Member object name `FileSystems` is renamed to `Items`. - Renamed `Path` type returned from `ListDataLakePaths` to be `PathItem`. Member object name `Paths` is renamed to `Items`. +- Added `DataLakeDirectoryClient::ListPathsSinglePage` API to list DataLake paths under certain directory. - Added `Metadata`, `AccessType`, `HasImmutabilityPolicy`, `HasLegalHold`, `LeaseDuration`, `LeaseState` and `LeaseStatus` to `FileSystemItem`. - Added new type `LeaseDurationType` to indicate if a lease duration is fixed or infinite. @@ -24,6 +25,7 @@ - Renamed `GetUri` to `GetUrl`. - Added `DataLakeLeaseClient`, all lease related APIs are moved to `DataLakeLeaseClient`. - Changed lease duration to be `std::chrono::seconds`. +- Removed `Directory` in `ListPathsSinglePageOptions`. - Removed unused type `AccountResourceType` and `PathLeaseAction`. - Changed all previous `LeaseDuration` members to a new type named `LeaseDurationType`. diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp index e607130a2..b8f78f6cc 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp @@ -179,6 +179,19 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const SetDataLakeDirectoryAccessControlRecursiveOptions& options = SetDataLakeDirectoryAccessControlRecursiveOptions()) const; + /** + * @brief List the paths in this file system. + * @param recursive If "true", all paths are listed; otherwise, the list will only + * include paths that share the same root. + * @param options Optional parameters to list the paths in file system. + * @return Azure::Core::Response containing the + * results when listing the paths under a file system. + * @remark This request is sent to dfs endpoint. + */ + Azure::Core::Response ListPathsSinglePage( + bool recursive, + const ListPathsSinglePageOptions& options = ListPathsSinglePageOptions()) const; + private: explicit DataLakeDirectoryClient( Azure::Core::Http::Url dfsUrl, diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp index 9295951f6..973006740 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp @@ -166,8 +166,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { /** * @brief List the paths in this file system. * @param recursive If "true", all paths are listed; otherwise, only paths at the root of the - * filesystem are listed. If "directory" is specified, the list will only - * include paths that share the same root. + * filesystem are listed. * @param options Optional parameters to list the paths in file system. * @return Azure::Core::Response containing the * results when listing the paths under a file system. diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index 0b76babb6..d23005d14 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -207,12 +207,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * include up to 5,000 items. */ Azure::Core::Nullable PageSizeHint; - - /** - * @brief Filters results to paths within the specified directory. An error occurs - * if the directory does not exist. - */ - Azure::Core::Nullable Directory; }; /** diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp index 5b2276068..d66e809e6 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp @@ -223,4 +223,34 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { m_dfsUrl, *m_pipeline, options.Context, protocolLayerOptions); } + Azure::Core::Response DataLakeDirectoryClient:: + ListPathsSinglePage(bool recursive, const ListPathsSinglePageOptions& options) const + { + Details::DataLakeRestClient::FileSystem::ListPathsOptions protocolLayerOptions; + protocolLayerOptions.Resource = Models::FileSystemResourceType::Filesystem; + protocolLayerOptions.Upn = options.UserPrincipalName; + protocolLayerOptions.ContinuationToken = options.ContinuationToken; + protocolLayerOptions.MaxResults = options.PageSizeHint; + protocolLayerOptions.RecursiveRequired = recursive; + auto currentPath = m_dfsUrl.GetPath(); + // Remove the filesystem name and get directory name. + auto firstSlashPos = currentPath.find_first_of("/"); + + if (firstSlashPos == 0 || (firstSlashPos == currentPath.size() + 1U)) + { + return Details::DataLakeRestClient::FileSystem::ListPaths( + m_dfsUrl, *m_pipeline, options.Context, protocolLayerOptions); + } + else + { + protocolLayerOptions.Directory + = currentPath.substr(firstSlashPos + 1U, currentPath.size() - firstSlashPos - 1U); + auto fileSystemUrl = m_dfsUrl; + fileSystemUrl.SetPath(currentPath.substr( + 0U, currentPath.size() - protocolLayerOptions.Directory.GetValue().size() - 1U)); + return Details::DataLakeRestClient::FileSystem::ListPaths( + fileSystemUrl, *m_pipeline, options.Context, protocolLayerOptions); + } + } + }}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index 675ce1ba4..84eca54aa 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -298,7 +298,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptions.Upn = options.UserPrincipalName; protocolLayerOptions.ContinuationToken = options.ContinuationToken; protocolLayerOptions.MaxResults = options.PageSizeHint; - protocolLayerOptions.Directory = options.Directory; protocolLayerOptions.RecursiveRequired = recursive; return Details::DataLakeRestClient::FileSystem::ListPaths( m_dfsUrl, *m_pipeline, options.Context, protocolLayerOptions); diff --git a/sdk/storage/azure-storage-files-datalake/test/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/datalake_file_system_client_test.cpp index 4c1d621d7..3e5961fa3 100644 --- a/sdk/storage/azure-storage-files-datalake/test/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/datalake_file_system_client_test.cpp @@ -57,24 +57,42 @@ namespace Azure { namespace Storage { namespace Test { std::vector result; std::string continuation; Files::DataLake::ListPathsSinglePageOptions options; - if (!directory.empty()) + if (directory.empty()) { - options.Directory = directory; + do + { + auto response = m_fileSystemClient->ListPathsSinglePage(recursive, options); + result.insert(result.end(), response->Items.begin(), response->Items.end()); + if (response->ContinuationToken.HasValue()) + { + continuation = response->ContinuationToken.GetValue(); + options.ContinuationToken = continuation; + } + else + { + continuation.clear(); + } + } while (!continuation.empty()); } - do + else { - auto response = m_fileSystemClient->ListPathsSinglePage(recursive, options); - result.insert(result.end(), response->Items.begin(), response->Items.end()); - if (response->ContinuationToken.HasValue()) + auto directoryClient = m_fileSystemClient->GetDirectoryClient(directory); + do { - continuation = response->ContinuationToken.GetValue(); - options.ContinuationToken = continuation; - } - else - { - continuation.clear(); - } - } while (!continuation.empty()); + auto response = directoryClient.ListPathsSinglePage(recursive, options); + result.insert(result.end(), response->Items.begin(), response->Items.end()); + if (response->ContinuationToken.HasValue()) + { + continuation = response->ContinuationToken.GetValue(); + options.ContinuationToken = continuation; + } + else + { + continuation.clear(); + } + } while (!continuation.empty()); + } + return result; } diff --git a/sdk/storage/azure-storage-files-datalake/test/datalake_sas_test.cpp b/sdk/storage/azure-storage-files-datalake/test/datalake_sas_test.cpp index 68569b95c..7cf80fdfa 100644 --- a/sdk/storage/azure-storage-files-datalake/test/datalake_sas_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/datalake_sas_test.cpp @@ -96,9 +96,8 @@ namespace Azure { namespace Storage { namespace Test { auto verify_directory_list = [&](const std::string& sas) { auto filesystemClient = Files::DataLake::DataLakeFileSystemClient(filesystemUrl + sas); - Files::DataLake::ListPathsSinglePageOptions options; - options.Directory = directory1Name; - EXPECT_NO_THROW(filesystemClient.ListPathsSinglePage(true, options)); + auto directoryClient = filesystemClient.GetDirectoryClient(directory1Name); + EXPECT_NO_THROW(directoryClient.ListPathsSinglePage(true)); }; auto verify_file_create = [&](const std::string& sas) {