github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Storage Blobs Service] ListBlobsByHierarchy (#265)

Browse Source 
* add API ListBlobsByHierarchy

* fix compiler error on GCC 8

* Update Blob API version to 2019-12-12

* use lowercase header key
This commit is contained in:
JinmingHu 2020-07-15 10:45:22 +08:00 committed by GitHub
parent 0eacb230cb
commit 3db59439c9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 524 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
sdk/storage/inc/blobs/blob_container_client.hpp
View File

@ -176,9 +176,7 @@ namespace Azure { namespace Storage { namespace Blobs {
* @brief Returns a single segment of blobs in this container, starting from the
* specified Marker, Use an empty Marker to start enumeration from the beginning and the
* NextMarker if it's not empty to make subsequent calls to ListBlobsFlat to continue
* enumerating the blobs segment by segment. Blobs are ordered lexicographically by name. A
* Delimiter can be used to traverse a virtual hierarchy of blobs as though it were a file
* system.
* enumerating the blobs segment by segment. Blobs are ordered lexicographically by name.
*
* @param options Optional parameters to execute this function.
* @return A
@ -186,6 +184,23 @@ namespace Azure { namespace Storage { namespace Blobs {
*/
BlobsFlatSegment ListBlobsFlat(const ListBlobsOptions& options = ListBlobsOptions()) const;
/**
* @brief Returns a single segment of blobs in this container, starting from the
* specified Marker, Use an empty Marker to start enumeration from the beginning and the
* NextMarker if it's not empty to make subsequent calls to ListBlobsByHierarchy to continue
* enumerating the blobs segment by segment. Blobs are ordered lexicographically by name. A
* Delimiter can be used to traverse a virtual hierarchy of blobs as though it were a file
* system.
*
* @param delimiter This can be used to to traverse a virtual hierarchy of blobs as though it
* were a file system. The delimiter may be a single character or a string.
* @param options Optional parameters to execute this function.
* @return A BlobsFlatSegment describing a segment of the blobs in the container.
*/
BlobsHierarchySegment ListBlobsByHierarchy(
const std::string& delimiter,
const ListBlobsOptions& options = ListBlobsOptions()) const;
private:
UrlBuilder m_containerUrl;
std::shared_ptr<Azure::Core::Http::HttpPipeline> m_pipeline;

6
sdk/storage/inc/blobs/blob_options.hpp
View File

@ -241,12 +241,6 @@ namespace Azure { namespace Storage { namespace Blobs {
*/
Azure::Core::Nullable<std::string> Prefix;
/**
* @brief Used to traverse a virtual hierarchy of blobs as though it were a file
* system.
*/
Azure::Core::Nullable<std::string> Delimiter;
/**
* @brief A string value that identifies the portion of the list of blobs to be
* returned with the next listing operation. The operation returns a non-empty

602
sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp
View File

File diff suppressed because it is too large Load Diff

19
sdk/storage/src/blobs/blob_container_client.cpp
View File

@ -166,13 +166,26 @@ namespace Azure { namespace Storage { namespace Blobs {
BlobsFlatSegment BlobContainerClient::ListBlobsFlat(const ListBlobsOptions& options) const
{
BlobRestClient::Container::ListBlobsOptions protocolLayerOptions;
BlobRestClient::Container::ListBlobsFlatOptions protocolLayerOptions;
protocolLayerOptions.Prefix = options.Prefix;
protocolLayerOptions.Delimiter = options.Delimiter;
protocolLayerOptions.Marker = options.Marker;
protocolLayerOptions.MaxResults = options.MaxResults;
protocolLayerOptions.Include = options.Include;
return BlobRestClient::Container::ListBlobs(
return BlobRestClient::Container::ListBlobsFlat(
options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions);
}
BlobsHierarchySegment BlobContainerClient::ListBlobsByHierarchy(
const std::string& delimiter,
const ListBlobsOptions& options) const
{
BlobRestClient::Container::ListBlobsByHierarchyOptions protocolLayerOptions;
protocolLayerOptions.Prefix = options.Prefix;
protocolLayerOptions.Delimiter = delimiter;
protocolLayerOptions.Marker = options.Marker;
protocolLayerOptions.MaxResults = options.MaxResults;
protocolLayerOptions.Include = options.Include;
return BlobRestClient::Container::ListBlobsByHierarchy(
options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions);
}

2
sdk/storage/src/common/shared_key_policy.cpp
View File

@ -29,7 +29,7 @@ namespace Azure { namespace Storage {
"If-Unmodified-Since",
"Range"})
{
auto ite = headers.find(headerName);
auto ite = headers.find(Azure::Core::Details::ToLower(headerName));
if (ite != headers.end())
{
if (headerName == "Content-Length" && ite->second == "0")

68
sdk/storage/test/blobs/blob_container_client_test.cpp
View File

@ -146,48 +146,70 @@ namespace Azure { namespace Storage { namespace Test {
{
const std::string delimiter = "/";
const std::string prefix = RandomString();
const std::string prefix1 = prefix + "-" + RandomString();
const std::string prefix2 = prefix + "-" + RandomString();
std::set<std::string> blobs;
std::string blobName = prefix;
for (int i = 0; i < 5; ++i)
for (const auto& blobNamePrefix : {prefix1, prefix2})
{
blobName = blobName + delimiter + RandomString();
auto blobClient = m_blobContainerClient->GetBlockBlobClient(blobName);
auto emptyContent = Azure::Core::Http::MemoryBodyStream(nullptr, 0);
blobClient.Upload(emptyContent);
blobs.insert(blobName);
for (int i = 0; i < 3; ++i)
{
std::string blobName = blobNamePrefix + delimiter + RandomString();
auto blobClient = m_blobContainerClient->GetBlockBlobClient(blobName);
auto emptyContent = Azure::Core::Http::MemoryBodyStream(nullptr, 0);
blobClient.Upload(emptyContent);
blobs.insert(blobName);
}
}
Azure::Storage::Blobs::ListBlobsOptions options;
options.Delimiter = delimiter;
options.Prefix = prefix + delimiter;
std::set<std::string> listBlobs;
options.Prefix = prefix;
std::set<std::string> items;
while (true)
{
auto res = m_blobContainerClient->ListBlobsFlat(options);
EXPECT_EQ(res.Delimiter, options.Delimiter.GetValue());
auto res = m_blobContainerClient->ListBlobsByHierarchy(delimiter, options);
EXPECT_EQ(res.Delimiter, delimiter);
EXPECT_EQ(res.Prefix, options.Prefix.GetValue());
for (const auto& blob : res.Items)
EXPECT_TRUE(res.Items.empty());
for (const auto& i : res.BlobPrefixes)
{
listBlobs.insert(blob.Name);
items.emplace(i.Name);
}
if (!res.NextMarker.empty())
{
options.Marker = res.NextMarker;
}
else if (!res.Items.empty())
{
options.Prefix = res.Items[0].Name + delimiter;
if (options.Marker.HasValue())
{
options.Marker.Reset();
}
}
else
{
break;
}
}
EXPECT_EQ(listBlobs, blobs);
EXPECT_EQ(items, (std::set<std::string>{prefix1 + delimiter, prefix2 + delimiter}));
items.clear();
for (const auto& p : {prefix1, prefix2})
{
options.Prefix = p + delimiter;
while (true)
{
auto res = m_blobContainerClient->ListBlobsByHierarchy(delimiter, options);
EXPECT_EQ(res.Delimiter, delimiter);
EXPECT_EQ(res.Prefix, options.Prefix.GetValue());
EXPECT_TRUE(res.BlobPrefixes.empty());
for (const auto& i : res.Items)
{
items.emplace(i.Name);
}
if (!res.NextMarker.empty())
{
options.Marker = res.NextMarker;
}
else
{
break;
}
}
}
EXPECT_EQ(items, blobs);
}
}}} // namespace Azure::Storage::Test