From ad28d10ee03bdd3748cc2184bf38364f96a8f9b2 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Wed, 13 Sep 2023 13:06:14 +0800 Subject: [PATCH] Memorize filesystemUrl in directoryClient so that ListPaths knows the correct endpoint without guessing (#4923) --- sdk/storage/assets.json | 2 +- .../files/datalake/datalake_options.hpp | 6 +++ .../src/datalake_directory_client.cpp | 41 ++++++++++++++----- .../src/datalake_file_system_client.cpp | 4 +- .../ut/datalake_directory_client_test.cpp | 18 ++++++++ 5 files changed, 58 insertions(+), 13 deletions(-) diff --git a/sdk/storage/assets.json b/sdk/storage/assets.json index 8545b24a6..cfaf0d4b1 100644 --- a/sdk/storage/assets.json +++ b/sdk/storage/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "cpp", "TagPrefix": "cpp/storage", - "Tag": "cpp/storage_129d7f8039" + "Tag": "cpp/storage_a5249cec25" } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index 263b72d84..3b9b86381 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -104,6 +104,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * @brief Holds the customer provided key used when making requests. */ Azure::Nullable CustomerProvidedKey; + + /** + * The filesystem url. This is only non-null for directory clients that are created from a + * filesystem client, so that this directory client knows where to send ListPaths requests. + */ + Azure::Nullable FileSystemUrl; }; } // namespace _detail diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp index 6dc161908..d1324883e 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp @@ -220,21 +220,40 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptions.Recursive = recursive; protocolLayerOptions.ContinuationToken = options.ContinuationToken; - const std::string currentPath = m_pathUrl.GetPath(); - auto firstSlashPos = std::find(currentPath.begin(), currentPath.end(), '/'); - const std::string fileSystemName(currentPath.begin(), firstSlashPos); - if (firstSlashPos != currentPath.end()) + Azure::Core::Url fileSystemUrl; + if (m_clientConfiguration.FileSystemUrl.HasValue()) { - ++firstSlashPos; + fileSystemUrl = m_clientConfiguration.FileSystemUrl.Value(); + const std::string fileSystemPath = fileSystemUrl.GetPath(); + const std::string currentPath = m_pathUrl.GetPath(); + std::string directoryPath = currentPath.substr(fileSystemPath.length()); + if (directoryPath.length() > 0 && directoryPath[0] == '/') + { + directoryPath = directoryPath.substr(1); + } + if (!directoryPath.empty()) + { + protocolLayerOptions.Path = directoryPath; + } } - const std::string directoryPath(firstSlashPos, currentPath.end()); - if (!directoryPath.empty()) + else { - protocolLayerOptions.Path = directoryPath; - } + const std::string currentPath = m_pathUrl.GetPath(); + auto firstSlashPos = std::find(currentPath.begin(), currentPath.end(), '/'); + const std::string fileSystemName(currentPath.begin(), firstSlashPos); + if (firstSlashPos != currentPath.end()) + { + ++firstSlashPos; + } + const std::string directoryPath(firstSlashPos, currentPath.end()); + if (!directoryPath.empty()) + { + protocolLayerOptions.Path = directoryPath; + } - auto fileSystemUrl = m_pathUrl; - fileSystemUrl.SetPath(fileSystemName); + fileSystemUrl = m_pathUrl; + fileSystemUrl.SetPath(fileSystemName); + } auto response = _detail::FileSystemClient::ListPaths( *m_pipeline, fileSystemUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index fa44f0273..17cb1ebb6 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -152,11 +152,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { { auto builder = m_fileSystemUrl; builder.AppendPath(_internal::UrlEncodePath(directoryName)); - return DataLakeDirectoryClient( + auto directoryClient = DataLakeDirectoryClient( std::move(builder), m_blobContainerClient.GetBlobClient(directoryName), m_pipeline, m_clientConfiguration); + directoryClient.m_clientConfiguration.FileSystemUrl = m_fileSystemUrl; + return directoryClient; } Azure::Response DataLakeFileSystemClient::Create( diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp index a5a457acc..e9f37cae9 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp @@ -840,6 +840,24 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(results, paths); } + { + // List without FileSystemUrl in client configuration + auto directoryClient = Files::DataLake::DataLakeDirectoryClient( + Files::DataLake::_detail::GetDfsUrlFromUrl(m_directoryClient->GetUrl()), + _internal::ParseConnectionString(AdlsGen2ConnectionString()).KeyCredential, + InitStorageClientOptions()); + + std::set results; + for (auto page = directoryClient.ListPaths(false); page.HasPage(); page.MoveToNextPage()) + { + for (auto& path : page.Paths) + { + results.insert(path.Name); + } + } + + EXPECT_EQ(results, rootPaths); + } { // non-recursive std::set results;