github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refined ListPathsSinglePage (#1446)

Browse Source
This commit is contained in:
Kan Tang 2021-01-24 18:17:29 -08:00 committed by GitHub
parent f27c5770d1
commit 6b71b5416d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 80 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sdk/storage/azure-storage-files-datalake/CHANGELOG.md
View File

@ -10,6 +10,7 @@
- Moved all protocol layer generated result types to `Details` namespace.
- Renamed `FileSystem` type returned from `ListDataLakeFileSystems` to be `FileSystemItem`. Member object name `FileSystems` is renamed to `Items`.
- Renamed `Path` type returned from `ListDataLakePaths` to be `PathItem`. Member object name `Paths` is renamed to `Items`.
- Added `DataLakeDirectoryClient::ListPathsSinglePage` API to list DataLake paths under certain directory.
- Added `Metadata`, `AccessType`, `HasImmutabilityPolicy`, `HasLegalHold`, `LeaseDuration`, `LeaseState` and `LeaseStatus` to `FileSystemItem`.
- Added new type `LeaseDurationType` to indicate if a lease duration is fixed or infinite.
@ -24,6 +25,7 @@
- Renamed `GetUri` to `GetUrl`.
- Added `DataLakeLeaseClient`, all lease related APIs are moved to `DataLakeLeaseClient`.
- Changed lease duration to be `std::chrono::seconds`.
- Removed `Directory` in `ListPathsSinglePageOptions`.
- Removed unused type `AccountResourceType` and `PathLeaseAction`.
- Changed all previous `LeaseDuration` members to a new type named `LeaseDurationType`.

13
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp
View File

@ -179,6 +179,19 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
const SetDataLakeDirectoryAccessControlRecursiveOptions& options
= SetDataLakeDirectoryAccessControlRecursiveOptions()) const;
/**
* @brief List the paths in this file system.
* @param recursive If "true", all paths are listed; otherwise, the list will only
* include paths that share the same root.
* @param options Optional parameters to list the paths in file system.
* @return Azure::Core::Response<Models::ListPathsSinglePageResult> containing the
* results when listing the paths under a file system.
* @remark This request is sent to dfs endpoint.
*/
Azure::Core::Response<Models::ListPathsSinglePageResult> ListPathsSinglePage(
bool recursive,
const ListPathsSinglePageOptions& options = ListPathsSinglePageOptions()) const;
private:
explicit DataLakeDirectoryClient(
Azure::Core::Http::Url dfsUrl,

3
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp
View File

@ -166,8 +166,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
/**
* @brief List the paths in this file system.
* @param recursive If "true", all paths are listed; otherwise, only paths at the root of the
* filesystem are listed. If "directory" is specified, the list will only
* include paths that share the same root.
* filesystem are listed.
* @param options Optional parameters to list the paths in file system.
* @return Azure::Core::Response<Models::ListPathsSinglePageResult> containing the
* results when listing the paths under a file system.

6
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp
View File

@ -207,12 +207,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* include up to 5,000 items.
*/
Azure::Core::Nullable<int32_t> PageSizeHint;
/**
* @brief Filters results to paths within the specified directory. An error occurs
* if the directory does not exist.
*/
Azure::Core::Nullable<std::string> Directory;
};
/**

30
sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp
View File

@ -223,4 +223,34 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
m_dfsUrl, *m_pipeline, options.Context, protocolLayerOptions);
}
Azure::Core::Response<Models::ListPathsSinglePageResult> DataLakeDirectoryClient::
ListPathsSinglePage(bool recursive, const ListPathsSinglePageOptions& options) const
{
Details::DataLakeRestClient::FileSystem::ListPathsOptions protocolLayerOptions;
protocolLayerOptions.Resource = Models::FileSystemResourceType::Filesystem;
protocolLayerOptions.Upn = options.UserPrincipalName;
protocolLayerOptions.ContinuationToken = options.ContinuationToken;
protocolLayerOptions.MaxResults = options.PageSizeHint;
protocolLayerOptions.RecursiveRequired = recursive;
auto currentPath = m_dfsUrl.GetPath();
// Remove the filesystem name and get directory name.
auto firstSlashPos = currentPath.find_first_of("/");
if (firstSlashPos == 0 || (firstSlashPos == currentPath.size() + 1U))
{
return Details::DataLakeRestClient::FileSystem::ListPaths(
m_dfsUrl, *m_pipeline, options.Context, protocolLayerOptions);
}
else
{
protocolLayerOptions.Directory
= currentPath.substr(firstSlashPos + 1U, currentPath.size() - firstSlashPos - 1U);
auto fileSystemUrl = m_dfsUrl;
fileSystemUrl.SetPath(currentPath.substr(
0U, currentPath.size() - protocolLayerOptions.Directory.GetValue().size() - 1U));
return Details::DataLakeRestClient::FileSystem::ListPaths(
fileSystemUrl, *m_pipeline, options.Context, protocolLayerOptions);
}
}
}}}} // namespace Azure::Storage::Files::DataLake

1
sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp
View File

@ -298,7 +298,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
protocolLayerOptions.Upn = options.UserPrincipalName;
protocolLayerOptions.ContinuationToken = options.ContinuationToken;
protocolLayerOptions.MaxResults = options.PageSizeHint;
protocolLayerOptions.Directory = options.Directory;
protocolLayerOptions.RecursiveRequired = recursive;
return Details::DataLakeRestClient::FileSystem::ListPaths(
m_dfsUrl, *m_pipeline, options.Context, protocolLayerOptions);

46
sdk/storage/azure-storage-files-datalake/test/datalake_file_system_client_test.cpp
View File

@ -57,24 +57,42 @@ namespace Azure { namespace Storage { namespace Test {
std::vector<Files::DataLake::Models::PathItem> result;
std::string continuation;
Files::DataLake::ListPathsSinglePageOptions options;
if (!directory.empty())
if (directory.empty())
{
options.Directory = directory;
do
{
auto response = m_fileSystemClient->ListPathsSinglePage(recursive, options);
result.insert(result.end(), response->Items.begin(), response->Items.end());
if (response->ContinuationToken.HasValue())
{
continuation = response->ContinuationToken.GetValue();
options.ContinuationToken = continuation;
}
else
{
continuation.clear();
}
} while (!continuation.empty());
}
do
else
{
auto response = m_fileSystemClient->ListPathsSinglePage(recursive, options);
result.insert(result.end(), response->Items.begin(), response->Items.end());
if (response->ContinuationToken.HasValue())
auto directoryClient = m_fileSystemClient->GetDirectoryClient(directory);
do
{
continuation = response->ContinuationToken.GetValue();
options.ContinuationToken = continuation;
}
else
{
continuation.clear();
}
} while (!continuation.empty());
auto response = directoryClient.ListPathsSinglePage(recursive, options);
result.insert(result.end(), response->Items.begin(), response->Items.end());
if (response->ContinuationToken.HasValue())
{
continuation = response->ContinuationToken.GetValue();
options.ContinuationToken = continuation;
}
else
{
continuation.clear();
}
} while (!continuation.empty());
}
return result;
}

5
sdk/storage/azure-storage-files-datalake/test/datalake_sas_test.cpp
View File

@ -96,9 +96,8 @@ namespace Azure { namespace Storage { namespace Test {
auto verify_directory_list = [&](const std::string& sas) {
auto filesystemClient = Files::DataLake::DataLakeFileSystemClient(filesystemUrl + sas);
Files::DataLake::ListPathsSinglePageOptions options;
options.Directory = directory1Name;
EXPECT_NO_THROW(filesystemClient.ListPathsSinglePage(true, options));
auto directoryClient = filesystemClient.GetDirectoryClient(directory1Name);
EXPECT_NO_THROW(directoryClient.ListPathsSinglePage(true));
};
auto verify_file_create = [&](const std::string& sas) {