diff --git a/sdk/storage/assets.json b/sdk/storage/assets.json index f792864de..e2b2114cf 100644 --- a/sdk/storage/assets.json +++ b/sdk/storage/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "cpp", "TagPrefix": "cpp/storage", - "Tag": "cpp/storage_b632cb2101" + "Tag": "cpp/storage_789a3955d8" } diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index 9ccf32230..ccc0d29d0 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -6,6 +6,10 @@ - Added `RehydratePendingToCold` value to `ArchiveStatus` enum. +### Bugs Fixed + +- Fixed a bug where `PageBlobClient::GetPageRangesDiff` and `PageBlobClient::GetManagedDiskPageRangesDiff` crash when getting the second page. + ## 12.8.0 (2023-07-11) - Features in `12.8.0-beta.1` are now generally available. diff --git a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp index 176c4fc37..0a1c08457 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp @@ -75,11 +75,15 @@ namespace Azure { namespace Storage { namespace Blobs { *this = m_blobServiceClient->FindBlobsByTags( m_tagFilterSqlExpression, m_operationOptions, context); } - else + else if (m_blobContainerClient) { *this = m_blobContainerClient->FindBlobsByTags( m_tagFilterSqlExpression, m_operationOptions, context); } + else + { + AZURE_UNREACHABLE_CODE(); + } } void ListBlobsPagedResponse::OnNextPage(const Azure::Core::Context& context) @@ -113,7 +117,10 @@ namespace Azure { namespace Storage { namespace Blobs { *this = m_pageBlobClient->GetManagedDiskPageRangesDiff( m_previousSnapshotUrl.Value(), m_operationOptions, context); } - AZURE_UNREACHABLE_CODE(); + else + { + AZURE_UNREACHABLE_CODE(); + } } }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp index 2f2bc10be..8f65f7d2c 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp @@ -882,16 +882,22 @@ namespace Azure { namespace Storage { namespace Test { = c1 + " = '" + v1 + "' AND " + c2 + " >= '" + v2 + "' AND " + c3 + " <= '" + v3 + "'"; std::vector findResults; std::vector findResults2; + int numPages1 = 0; + int numPages2 = 0; for (int i = 0; i < 30; ++i) { + numPages1 = 0; + numPages2 = 0; findResults.clear(); findResults2.clear(); Blobs::FindBlobsByTagsOptions findOptions; findOptions.PageSizeHint = 2; - for (auto pageResult = containerClient.FindBlobsByTags(whereExpression); pageResult.HasPage(); + for (auto pageResult = containerClient.FindBlobsByTags(whereExpression, findOptions); + pageResult.HasPage(); pageResult.MoveToNextPage()) { + ++numPages1; EXPECT_FALSE(pageResult.ServiceEndpoint.empty()); for (auto& item : pageResult.TaggedBlobs) { @@ -901,10 +907,12 @@ namespace Azure { namespace Storage { namespace Test { findResults2.emplace_back(item.BlobName); } } - for (auto pageResult = m_blobContainerClient->FindBlobsByTags(whereExpression); + + for (auto pageResult = m_blobServiceClient->FindBlobsByTags(whereExpression, findOptions); pageResult.HasPage(); pageResult.MoveToNextPage()) { + ++numPages2; EXPECT_FALSE(pageResult.ServiceEndpoint.empty()); for (auto& item : pageResult.TaggedBlobs) { @@ -914,7 +922,6 @@ namespace Azure { namespace Storage { namespace Test { findResults.emplace_back(item.BlobName); } } - if (findResults.size() != blobNames.size() || findResults2.size() != blobNames.size()) { TestSleep(1s); @@ -924,6 +931,8 @@ namespace Azure { namespace Storage { namespace Test { break; } } + EXPECT_GT(numPages1, 2); + EXPECT_GT(numPages2, 2); EXPECT_EQ(findResults.size(), blobNames.size()); EXPECT_EQ(findResults2.size(), blobNames.size()); std::sort(blobNames.begin(), blobNames.end()); diff --git a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp index 6350f5e65..b346ade3f 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp @@ -181,6 +181,31 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(numRanges, static_cast(3)); } + TEST_F(PageBlobClientTest, GetPageRangesDiffContinuation) + { + auto pageBlobClient = *m_pageBlobClient; + std::vector blobContent = RandomBuffer(512); + + pageBlobClient.Create(8_KB); + auto snapshot = pageBlobClient.CreateSnapshot().Value.Snapshot; + + for (int i = 0; i < 3; ++i) + { + auto pageContent = Azure::Core::IO::MemoryBodyStream(blobContent.data(), blobContent.size()); + pageBlobClient.UploadPages(1024 * i, pageContent); + } + + Blobs::GetPageRangesOptions options; + options.PageSizeHint = 1; + int numPages = 0; + for (auto page = pageBlobClient.GetPageRangesDiff(snapshot, options); page.HasPage(); + page.MoveToNextPage()) + { + ++numPages; + } + EXPECT_GT(numPages, 2); + } + TEST_F(PageBlobClientTest, UploadFromUri) { auto pageBlobClient = *m_pageBlobClient; diff --git a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md index 0cfdf84a8..d5dbb56a5 100644 --- a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md @@ -8,6 +8,9 @@ ### Bugs Fixed +- Fixed a bug where `DataLakeDirectoryClient::ListPaths` and `DataLakeFileSystemClient::ListPaths` cannot list the second page and always fail with `std::bad_function_call`. +- Fixed a bug where `DataLakePathClient::SetAccessControlListRecursive` crashes when operating on the second page. + ### Other Changes ## 12.7.0 (2023-07-11) diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index a0baa74f3..41382beb7 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -16,6 +16,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { class DataLakeServiceClient; class DataLakeFileSystemClient; class DataLakePathClient; + class DataLakeDirectoryClient; namespace Models { @@ -828,8 +829,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { private: void OnNextPage(const Azure::Core::Context& context); - std::function - m_onNextPageFunc; + std::shared_ptr m_fileSystemClient; + std::shared_ptr m_directoryClient; + bool m_recursive = false; + ListPathsOptions m_operationOptions; friend class DataLakeFileSystemClient; friend class DataLakeDirectoryClient; diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp index ea4cdcad4..6dc161908 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp @@ -218,6 +218,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptions.Upn = options.UserPrincipalName; protocolLayerOptions.MaxResults = options.PageSizeHint; protocolLayerOptions.Recursive = recursive; + protocolLayerOptions.ContinuationToken = options.ContinuationToken; const std::string currentPath = m_pathUrl.GetPath(); auto firstSlashPos = std::find(currentPath.begin(), currentPath.end(), '/'); @@ -235,55 +236,42 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto fileSystemUrl = m_pathUrl; fileSystemUrl.SetPath(fileSystemName); - auto clientCopy = *this; - std::function func; - func = [func, clientCopy, protocolLayerOptions, fileSystemUrl]( - std::string continuationToken, const Azure::Core::Context& context) { - auto protocolLayerOptionsCopy = protocolLayerOptions; - if (!continuationToken.empty()) + auto response = _detail::FileSystemClient::ListPaths( + *m_pipeline, fileSystemUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); + + ListPathsPagedResponse pagedResponse; + for (auto& path : response.Value.Paths) + { + Models::PathItem item; + item.Name = std::move(path.Name); + item.IsDirectory = path.IsDirectory; + item.LastModified = std::move(path.LastModified); + item.FileSize = path.FileSize; + item.Owner = std::move(path.Owner); + item.Group = std::move(path.Group); + item.Permissions = std::move(path.Permissions); + item.EncryptionScope = std::move(path.EncryptionScope); + item.ETag = std::move(path.ETag); + if (path.CreatedOn.HasValue()) { - protocolLayerOptionsCopy.ContinuationToken = continuationToken; + item.CreatedOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.CreatedOn.Value())); } - auto response = _detail::FileSystemClient::ListPaths( - *clientCopy.m_pipeline, - fileSystemUrl, - protocolLayerOptionsCopy, - _internal::WithReplicaStatus(context)); - - ListPathsPagedResponse pagedResponse; - for (auto& path : response.Value.Paths) + if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != "0") { - Models::PathItem item; - item.Name = std::move(path.Name); - item.IsDirectory = path.IsDirectory; - item.LastModified = std::move(path.LastModified); - item.FileSize = path.FileSize; - item.Owner = std::move(path.Owner); - item.Group = std::move(path.Group); - item.Permissions = std::move(path.Permissions); - item.EncryptionScope = std::move(path.EncryptionScope); - item.ETag = std::move(path.ETag); - if (path.CreatedOn.HasValue()) - { - item.CreatedOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( - std::stoll(path.CreatedOn.Value())); - } - if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != "0") - { - item.ExpiresOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( - std::stoll(path.ExpiresOn.Value())); - } - pagedResponse.Paths.push_back(std::move(item)); + item.ExpiresOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.ExpiresOn.Value())); } - pagedResponse.m_onNextPageFunc = func; - pagedResponse.CurrentPageToken = continuationToken; - pagedResponse.NextPageToken = response.Value.ContinuationToken; - pagedResponse.RawResponse = std::move(response.RawResponse); + pagedResponse.Paths.push_back(std::move(item)); + } + pagedResponse.m_directoryClient = std::make_shared(*this); + pagedResponse.m_recursive = recursive; + pagedResponse.m_operationOptions = options; + pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); + pagedResponse.NextPageToken = response.Value.ContinuationToken; + pagedResponse.RawResponse = std::move(response.RawResponse); - return pagedResponse; - }; - - return func(options.ContinuationToken.ValueOr(std::string()), context); + return pagedResponse; } }}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index f12c17dbd..e4961ab5b 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -274,57 +274,45 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptions.Upn = options.UserPrincipalName; protocolLayerOptions.MaxResults = options.PageSizeHint; protocolLayerOptions.Recursive = recursive; + protocolLayerOptions.ContinuationToken = options.ContinuationToken; - auto clientCopy = *this; - std::function func; - func = [func, clientCopy, protocolLayerOptions]( - std::string continuationToken, const Azure::Core::Context& context) { - auto protocolLayerOptionsCopy = protocolLayerOptions; - if (!continuationToken.empty()) + auto response = _detail::FileSystemClient::ListPaths( + *m_pipeline, m_fileSystemUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); + + ListPathsPagedResponse pagedResponse; + for (auto& path : response.Value.Paths) + { + Models::PathItem item; + item.Name = std::move(path.Name); + item.IsDirectory = path.IsDirectory; + item.LastModified = std::move(path.LastModified); + item.FileSize = path.FileSize; + item.Owner = std::move(path.Owner); + item.Group = std::move(path.Group); + item.Permissions = std::move(path.Permissions); + item.EncryptionScope = std::move(path.EncryptionScope); + item.EncryptionContext = std::move(path.EncryptionContext); + item.ETag = std::move(path.ETag); + if (path.CreatedOn.HasValue()) { - protocolLayerOptionsCopy.ContinuationToken = continuationToken; + item.CreatedOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.CreatedOn.Value())); } - auto response = _detail::FileSystemClient::ListPaths( - *clientCopy.m_pipeline, - clientCopy.m_fileSystemUrl, - protocolLayerOptionsCopy, - _internal::WithReplicaStatus(context)); - - ListPathsPagedResponse pagedResponse; - for (auto& path : response.Value.Paths) + if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != "0") { - Models::PathItem item; - item.Name = std::move(path.Name); - item.IsDirectory = path.IsDirectory; - item.LastModified = std::move(path.LastModified); - item.FileSize = path.FileSize; - item.Owner = std::move(path.Owner); - item.Group = std::move(path.Group); - item.Permissions = std::move(path.Permissions); - item.EncryptionScope = std::move(path.EncryptionScope); - item.EncryptionContext = std::move(path.EncryptionContext); - item.ETag = std::move(path.ETag); - if (path.CreatedOn.HasValue()) - { - item.CreatedOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( - std::stoll(path.CreatedOn.Value())); - } - if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != "0") - { - item.ExpiresOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( - std::stoll(path.ExpiresOn.Value())); - } - pagedResponse.Paths.push_back(std::move(item)); + item.ExpiresOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.ExpiresOn.Value())); } - pagedResponse.m_onNextPageFunc = func; - pagedResponse.CurrentPageToken = continuationToken; - pagedResponse.NextPageToken = response.Value.ContinuationToken; - pagedResponse.RawResponse = std::move(response.RawResponse); + pagedResponse.Paths.push_back(std::move(item)); + } + pagedResponse.m_fileSystemClient = std::make_shared(*this); + pagedResponse.m_recursive = recursive; + pagedResponse.m_operationOptions = options; + pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); + pagedResponse.NextPageToken = response.Value.ContinuationToken; + pagedResponse.RawResponse = std::move(response.RawResponse); - return pagedResponse; - }; - - return func(options.ContinuationToken.ValueOr(std::string()), context); + return pagedResponse; } Azure::Response DataLakeFileSystemClient::GetAccessPolicy( diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp index b1f4c5d2b..3c36cbb05 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp @@ -297,7 +297,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; protocolLayerOptions.Recursive = options.Recursive; protocolLayerOptions.ContinuationToken = continuationToken; - protocolLayerOptions.Paginated = paginated; + if (options.Recursive.HasValue()) + { + protocolLayerOptions.Paginated = paginated; + } auto response = _detail::PathClient::Delete(*m_pipeline, m_pathUrl, protocolLayerOptions, context); continuationToken = Azure::Core::Http::_internal::HttpShared::GetHeaderOrEmptyString( diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp index 5130f7cc1..c6fa0aa1c 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp @@ -3,6 +3,7 @@ #include "azure/storage/files/datalake/datalake_responses.hpp" +#include "azure/storage/files/datalake/datalake_directory_client.hpp" #include "azure/storage/files/datalake/datalake_path_client.hpp" #include "azure/storage/files/datalake/datalake_service_client.hpp" #include "private/datalake_utilities.hpp" @@ -90,7 +91,19 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { void ListPathsPagedResponse::OnNextPage(const Azure::Core::Context& context) { - *this = m_onNextPageFunc(NextPageToken.Value(), context); + m_operationOptions.ContinuationToken = NextPageToken; + if (m_fileSystemClient) + { + *this = m_fileSystemClient->ListPaths(m_recursive, m_operationOptions, context); + } + else if (m_directoryClient) + { + *this = m_directoryClient->ListPaths(m_recursive, m_operationOptions, context); + } + else + { + AZURE_UNREACHABLE_CODE(); + } } void ListDeletedPathsPagedResponse::OnNextPage(const Azure::Core::Context& context) @@ -118,7 +131,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { *this = m_dataLakePathClient->RemoveAccessControlListRecursive( m_acls, m_operationOptions, context); } - AZURE_UNREACHABLE_CODE(); + else + { + AZURE_UNREACHABLE_CODE(); + } } }}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp index 729a341f6..661bca10b 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp @@ -21,9 +21,22 @@ namespace Azure { namespace Storage { namespace Test { m_directoryName = RandomString(); m_directoryClient = std::make_shared( m_fileSystemClient->GetDirectoryClient(m_directoryName)); - m_fileSystemClient->GetFileClient(m_directoryName).Create(); + m_fileSystemClient->GetDirectoryClient(m_directoryName).Create(); } + namespace { + bool CompareDirectoryMetadata(const Storage::Metadata& lhs, const Storage::Metadata& rhs) + { + /* cspell:disable-next-line */ + const std::string c_hdiIsFolder = "hdi_isfolder"; + std::vector> symmetricDiff; + std::set_symmetric_difference( + lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), std::back_inserter(symmetricDiff)); + return symmetricDiff.empty() + || (symmetricDiff.size() == 1 && symmetricDiff[0].first == c_hdiIsFolder); + } + } // namespace + TEST_F(DataLakeDirectoryClientTest, CreateDeleteDirectory) { const std::string baseName = RandomString(); @@ -344,10 +357,10 @@ namespace Azure { namespace Storage { namespace Test { // Set/Get Metadata works EXPECT_NO_THROW(m_directoryClient->SetMetadata(metadata1)); auto result = m_directoryClient->GetProperties().Value.Metadata; - EXPECT_EQ(metadata1, result); + EXPECT_TRUE(CompareDirectoryMetadata(metadata1, result)); EXPECT_NO_THROW(m_directoryClient->SetMetadata(metadata2)); result = m_directoryClient->GetProperties().Value.Metadata; - EXPECT_EQ(metadata2, result); + EXPECT_TRUE(CompareDirectoryMetadata(metadata2, result)); } { @@ -363,13 +376,9 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(client1.Create(options1)); EXPECT_NO_THROW(client2.Create(options2)); auto result = client1.GetProperties().Value.Metadata; - /* cspell:disable-next-line */ - metadata1["hdi_isfolder"] = "true"; - /* cspell:disable-next-line */ - metadata2["hdi_isfolder"] = "true"; - EXPECT_EQ(metadata1, result); + EXPECT_TRUE(CompareDirectoryMetadata(metadata1, result)); result = client2.GetProperties().Value.Metadata; - EXPECT_EQ(metadata2, result); + EXPECT_TRUE(CompareDirectoryMetadata(metadata2, result)); } } @@ -381,10 +390,10 @@ namespace Azure { namespace Storage { namespace Test { // Get Metadata via properties works EXPECT_NO_THROW(m_directoryClient->SetMetadata(metadata1)); auto result = m_directoryClient->GetProperties(); - EXPECT_EQ(metadata1, result.Value.Metadata); + EXPECT_TRUE(CompareDirectoryMetadata(metadata1, result.Value.Metadata)); EXPECT_NO_THROW(m_directoryClient->SetMetadata(metadata2)); result = m_directoryClient->GetProperties(); - EXPECT_EQ(metadata2, result.Value.Metadata); + EXPECT_TRUE(CompareDirectoryMetadata(metadata2, result.Value.Metadata)); } { @@ -433,6 +442,26 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(DataLakeDirectoryClientTest, DirectoryAccessControlRecursiveMultiPage) + { + auto dirClient = m_fileSystemClient->GetDirectoryClient(RandomString()); + for (int i = 0; i < 5; ++i) + { + auto fileClient = dirClient.GetFileClient(RandomString()); + fileClient.Create(); + } + auto acls = GetAclsForTesting(); + Files::DataLake::SetPathAccessControlListRecursiveOptions options; + options.PageSizeHint = 2; + int numPages = 0; + for (auto page = dirClient.SetAccessControlListRecursive(acls, options); page.HasPage(); + page.MoveToNextPage()) + { + ++numPages; + } + EXPECT_GT(numPages, 2); + } + TEST_F(DataLakeDirectoryClientTest, DirectoryAccessControlRecursive) { // Setup directories. @@ -702,4 +731,105 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(DataLakeDirectoryClientTest, ListPaths) + { + std::set paths; + const std::string dir1 = RandomString(); + const std::string dir2 = RandomString(); + + std::set rootPaths; + rootPaths.emplace(dir1); + rootPaths.emplace(dir2); + paths.emplace(dir1); + paths.emplace(dir2); + + { + // This is to ensure path filter is correctly set for listing, items out of the directory + // won't be listed. + m_fileSystemClient->GetDirectoryClient(RandomString()).Create(); + m_fileSystemClient->GetFileClient(RandomString()).Create(); + } + { + auto dirClient = m_directoryClient->GetSubdirectoryClient(dir1); + + for (int i = 0; i < 3; ++i) + { + std::string filename = RandomString(); + auto fileClient = dirClient.GetFileClient(filename); + fileClient.CreateIfNotExists(); + paths.emplace(dir1 + "/" + filename); + } + + dirClient = m_directoryClient->GetSubdirectoryClient(dir2); + for (int i = 0; i < 4; ++i) + { + std::string filename = RandomString(); + auto fileClient = dirClient.GetFileClient(filename); + fileClient.CreateIfNotExists(); + paths.emplace(dir2 + "/" + filename); + } + std::string filename = RandomString(); + auto fileClient = m_directoryClient->GetFileClient(filename); + fileClient.CreateIfNotExists(); + paths.emplace(filename); + rootPaths.emplace(filename); + } + + { + // append root directory prefix + std::set tmp; + for (const auto& i : rootPaths) + { + tmp.insert(m_directoryName + "/" + i); + } + rootPaths = tmp; + tmp.clear(); + for (const auto& i : paths) + { + tmp.insert(m_directoryName + "/" + i); + } + paths = tmp; + } + + { + // Normal list recursively. + std::set results; + for (auto page = m_directoryClient->ListPaths(true); page.HasPage(); page.MoveToNextPage()) + { + for (auto& path : page.Paths) + { + results.insert(path.Name); + } + } + + EXPECT_EQ(results, paths); + } + { + // non-recursive + std::set results; + for (auto page = m_directoryClient->ListPaths(false); page.HasPage(); page.MoveToNextPage()) + { + for (auto& path : page.Paths) + { + results.insert(path.Name); + } + } + + EXPECT_EQ(results, rootPaths); + } + { + // List max result + Files::DataLake::ListPathsOptions options; + options.PageSizeHint = 2; + int numPages = 0; + for (auto page = m_directoryClient->ListPaths(true, options); page.HasPage(); + page.MoveToNextPage()) + { + EXPECT_LE(page.Paths.size(), 2U); + ++numPages; + } + EXPECT_GT(numPages, 2); + } + } + }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index 474c0f851..1067faa0e 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -171,6 +171,8 @@ namespace Azure { namespace Storage { namespace Test { std::set rootPaths; rootPaths.emplace(dir1); rootPaths.emplace(dir2); + paths.emplace(dir1); + paths.emplace(dir2); { auto dirClient = m_fileSystemClient->GetDirectoryClient(dir1); @@ -208,10 +210,7 @@ namespace Azure { namespace Storage { namespace Test { } } - for (const auto& path : paths) - { - EXPECT_NE(results.find(path), results.end()); - } + EXPECT_EQ(results, paths); } { // non-recursive @@ -224,18 +223,20 @@ namespace Azure { namespace Storage { namespace Test { } } - for (const auto& path : rootPaths) - { - EXPECT_NE(results.find(path), results.end()); - } - EXPECT_LT(results.size(), paths.size()); + EXPECT_EQ(results, rootPaths); } { // List max result Files::DataLake::ListPathsOptions options; options.PageSizeHint = 2; - auto response = m_fileSystemClient->ListPaths(true, options); - EXPECT_LE(2U, response.Paths.size()); + int numPages = 0; + for (auto page = m_fileSystemClient->ListPaths(true, options); page.HasPage(); + page.MoveToNextPage()) + { + EXPECT_LE(page.Paths.size(), 2U); + ++numPages; + } + EXPECT_GT(numPages, 2); } } @@ -771,12 +772,15 @@ namespace Azure { namespace Storage { namespace Test { Files::DataLake::ListDeletedPathsOptions options; options.PageSizeHint = 1; std::vector paths; + int numPages = 0; for (auto pageResult = m_fileSystemClient->ListDeletedPaths(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { + ++numPages; paths.insert(paths.end(), pageResult.DeletedPaths.begin(), pageResult.DeletedPaths.end()); EXPECT_LE(pageResult.DeletedPaths.size(), 1); } + EXPECT_GT(numPages, 1); EXPECT_EQ(2, paths.size()); } // prefix works