github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Suggested changes in API review (#244)

Browse Source 
* Rename Properties->HttpHeaders

* Rename BlobXxxItems->Items

* Remove MaxResults in response of list operations

* specify time format in comments for GetUSerDelegationKey API

* per operation pipelines and per retry pipelines

* Define some response types and rename ListBlobsFlat

* Add different return types for some APIs

* Rename BlobDownloadInfo->BlobDownloadResponse

* Assign default value to BlobContentLength to supporess compiler warnings

* add concurrent download to buffer

* concurrent upload block blob from buffer
This commit is contained in:
JinmingHu 2020-07-07 10:54:05 +08:00 committed by GitHub
parent 9fb4119ccb
commit 66c7518dce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 1082 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
sdk/storage/CMakeLists.txt
View File

@ -18,6 +18,7 @@ set (AZURE_STORAGE_BLOB_HEADER
inc/common/shared_key_policy.hpp
inc/common/crypt.hpp
inc/common/xml_wrapper.hpp
inc/common/concurrent_transfer.hpp
inc/blobs/blob.hpp
inc/blobs/blob_service_client.hpp
inc/blobs/blob_container_client.hpp
@ -85,11 +86,15 @@ set(AZURE_STORAGE_SOURCE
add_library(azure-storage ${AZURE_STORAGE_HEADER} ${AZURE_STORAGE_SOURCE})
set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
set(THREADS_PREFER_PTHREAD_FLAG TRUE)
find_package(Threads REQUIRED)
find_package(LibXml2 REQUIRED)
target_include_directories(azure-storage PUBLIC ${LIBXML2_INCLUDE_DIR} $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/inc> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/external> $<INSTALL_INTERFACE:include/azure_storage>)
target_link_libraries(azure-storage azure-core ${LIBXML2_LIBRARIES})
target_link_libraries(azure-storage Threads::Threads azure-core ${LIBXML2_LIBRARIES})
if(MSVC)
target_link_libraries(azure-storage bcrypt)

89
sdk/storage/inc/blobs/blob_client.hpp
View File

@ -14,6 +14,18 @@
namespace Azure { namespace Storage { namespace Blobs {
struct BlobDownloadInfo
{
std::string ETag;
std::string LastModified;
int64_t ContentLength = 0;
BlobHttpHeaders HttpHeaders;
std::map<std::string, std::string> Metadata;
Blobs::BlobType BlobType = Blobs::BlobType::Unknown;
Azure::Core::Nullable<bool> ServerEncrypted;
Azure::Core::Nullable<std::string> EncryptionKeySHA256;
};
class BlockBlobClient;
class AppendBlobClient;
class PageBlobClient;
@ -144,11 +156,12 @@ namespace Azure { namespace Storage { namespace Blobs {
/**
* @brief Sets system properties on the blob.
*
* @param options Optional
* parameters to execute this function.
* @return A BlobInfo describing the updated blob.
* @param httpHeaders The standard HTTP header system properties to set.
* @param options Optional parameters to execute this function.
* @return A SetBlobHttpHeadersResponse describing the updated blob.
*/
BlobInfo SetHttpHeaders(
SetBlobHttpHeadersResponse SetHttpHeaders(
BlobHttpHeaders httpHeaders,
const SetBlobHttpHeadersOptions& options = SetBlobHttpHeadersOptions()) const;
/**
@ -158,10 +171,9 @@ namespace Azure { namespace Storage { namespace Blobs {
* @param metadata Custom metadata to set for this blob.
* @param
* options Optional parameters to execute this function.
* @return A BlobInfo describing
* the updated blob.
* @return A SetBlobMetadataResponse describing the updated blob.
*/
BlobInfo SetMetadata(
SetBlobMetadataResponse SetMetadata(
std::map<std::string, std::string> metadata,
const SetBlobMetadataOptions& options = SetBlobMetadataOptions()) const;
@ -172,10 +184,9 @@ namespace Azure { namespace Storage { namespace Blobs {
* @param Tier Indicates the tier to be set on the blob.
* @param options Optional
* parameters to execute this function.
* @return A BasicResponse on successfully setting
* the tier.
* @return A SetAccessTierResponse on successfully setting the tier.
*/
BasicResponse SetAccessTier(
SetAccessTierResponse SetAccessTier(
AccessTier Tier,
const SetAccessTierOptions& options = SetAccessTierOptions()) const;
@ -201,23 +212,50 @@ namespace Azure { namespace Storage { namespace Blobs {
*
* @param copyId ID of the copy operation to abort.
* @param options Optional parameters to execute this function.
* @return A BasicResponse
* on successfully aborting.
* @return A AbortCopyBlobResponse on successfully aborting.
*/
BasicResponse AbortCopyFromUri(
AbortCopyBlobResponse AbortCopyFromUri(
const std::string& copyId,
const AbortCopyFromUriOptions& options = AbortCopyFromUriOptions()) const;
/**
* @brief Downloads a blob from the service, including its metadata and properties.
* @brief Downloads a blob or a blob range from the service, including its metadata and
* properties.
*
* @param options Optional parameters to execute this function.
* @return A BlobDownloadResponse describing the downloaded blob.
* BlobDownloadResponse.BodyStream contains the blob's data.
*/
BlobDownloadResponse Download(const DownloadBlobOptions& options = DownloadBlobOptions()) const;
* *
/**
* @brief Downloads a blob or a blob range from the service to a memory buffer using parallel
* requests.
*
* @param buffer A memory buffer to write the blob content to.
* @param bufferSize Size of the memory buffer. Size must be larger or equal to size of the blob
* or blob range.
* @param options Optional parameters to execute this function.
* @return A
* BlobDownloadInfo describing the downloaded blob.
* BlobDownloadInfo.BodyStream contains the blob's data.
*/
BlobDownloadInfo Download(const DownloadBlobOptions& options = DownloadBlobOptions()) const;
BlobDownloadInfo DownloadToBuffer(
uint8_t* buffer,
std::size_t bufferSize,
const DownloadBlobToBufferOptions& options = DownloadBlobToBufferOptions()) const;
/**
* @brief Downloads a blob or a blob range from the service to a file using parallel
* requests.
*
* @param file A file path to write the downloaded content to.
* @param options Optional parameters to execute this function.
* @return A
* BlobDownloadInfo describing the downloaded blob.
*/
BlobDownloadInfo DownloadToFile(
const std::string& file,
const DownloadBlobToFileOptions& options = DownloadBlobToFileOptions()) const;
/**
* @brief Creates a read-only snapshot of a blob.
@ -236,10 +274,9 @@ namespace Azure { namespace Storage { namespace Blobs {
* snapshots. You can delete both at the same time using DeleteBlobOptions.DeleteSnapshots.
*
* @param options Optional parameters to execute this function.
* @return A
* BasicResponse on successfully deleting.
* @return A DeleteBlobResponse on successfully deleting.
*/
BasicResponse Delete(const DeleteBlobOptions& options = DeleteBlobOptions()) const;
DeleteBlobResponse Delete(const DeleteBlobOptions& options = DeleteBlobOptions()) const;
/**
* @brief Restores the contents and metadata of a soft deleted blob and any associated
@ -247,16 +284,22 @@ namespace Azure { namespace Storage { namespace Blobs {
*
* @param options Optional parameters to execute this
* function.
* @return A BasicResponse on successfully deleting.
* @return A UndeleteBlobResponse on successfully deleting.
*/
BasicResponse Undelete(const UndeleteBlobOptions& options = UndeleteBlobOptions()) const;
UndeleteBlobResponse Undelete(const UndeleteBlobOptions& options = UndeleteBlobOptions()) const;
protected:
UrlBuilder m_blobUrl;
std::shared_ptr<Azure::Core::Http::HttpPipeline> m_pipeline;
private:
BlobClient() = default;
explicit BlobClient(
UrlBuilder blobUri,
std::shared_ptr<Azure::Core::Http::HttpPipeline> pipeline)
: m_blobUrl(std::move(blobUri)), m_pipeline(std::move(pipeline))
{
}
friend class BlobContainerClient;
};
}}} // namespace Azure::Storage::Blobs

33
sdk/storage/inc/blobs/blob_container_client.hpp
View File

@ -119,7 +119,7 @@ namespace Azure { namespace Storage { namespace Blobs {
/**
* @brief Gets the container's primary uri endpoint.
*
*
* @return The
* container's primary uri endpoint.
*/
@ -128,7 +128,7 @@ namespace Azure { namespace Storage { namespace Blobs {
/**
* @brief Creates a new container under the specified account. If the container with the
* same name already exists, the operation fails.
*
*
* @param options Optional
* parameters to execute this function.
* @return A BlobContainerInfo describing the newly
@ -140,13 +140,12 @@ namespace Azure { namespace Storage { namespace Blobs {
/**
* @brief Marks the specified container for deletion. The container and any blobs
* contained within it are later deleted during garbage collection.
*
*
* @param
* options Optional parameters to execute this function.
* @return A BasicResponse if
* successful.
* @return A DeleteContainerResponse if successful.
*/
BasicResponse Delete(
DeleteContainerResponse Delete(
const DeleteBlobContainerOptions& options = DeleteBlobContainerOptions()) const;
/**
@ -167,31 +166,37 @@ namespace Azure { namespace Storage { namespace Blobs {
* @param metadata Custom metadata to set for this container.
* @param options
* Optional parameters to execute this function.
* @return A BlobContainerInfo if
* successful.
* @return A SetContainerMetadataResponse if successful.
*/
BlobContainerInfo SetMetadata(
SetContainerMetadataResponse SetMetadata(
std::map<std::string, std::string> metadata,
SetBlobContainerMetadataOptions options = SetBlobContainerMetadataOptions()) const;
/**
* @brief Returns a single segment of blobs in this container, starting from the
* specified Marker, Use an empty Marker to start enumeration from the beginning and the
* NextMarker if it's not empty to make subsequent calls to ListBlobs to continue enumerating
* the blobs segment by segment. Blobs are ordered lexicographically by name. A Delimiter can be
* used to traverse a virtual hierarchy of blobs as though it were a file system.
* NextMarker if it's not empty to make subsequent calls to ListBlobsFlat to continue
* enumerating the blobs segment by segment. Blobs are ordered lexicographically by name. A
* Delimiter can be used to traverse a virtual hierarchy of blobs as though it were a file
* system.
*
* @param options Optional parameters to execute this function.
* @return A
* BlobsFlatSegment describing a segment of the blobs in the container.
*/
BlobsFlatSegment ListBlobs(const ListBlobsOptions& options = ListBlobsOptions()) const;
BlobsFlatSegment ListBlobsFlat(const ListBlobsOptions& options = ListBlobsOptions()) const;
private:
UrlBuilder m_containerUrl;
std::shared_ptr<Azure::Core::Http::HttpPipeline> m_pipeline;
BlobContainerClient() = default;
explicit BlobContainerClient(
UrlBuilder containerUri,
std::shared_ptr<Azure::Core::Http::HttpPipeline> pipeline)
: m_containerUrl(std::move(containerUri)), m_pipeline(std::move(pipeline))
{
}
friend class BlobServiceClient;
};

156
sdk/storage/inc/blobs/blob_options.hpp
View File

@ -17,10 +17,16 @@ namespace Azure { namespace Storage { namespace Blobs {
struct BlobServiceClientOptions
{
/**
* @brief Transport pipeline policies for authentication, retries, etc., that are
* applied to every request.
* @brief Transport pipeline policies for authentication, additional HTTP headers, etc., that
* are applied to every request.
*/
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> policies;
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> PerOperationPolicies;
/**
* @brief Transport pipeline policies for authentication, additional HTTP headers, etc., that
* are applied to every retrial.
*/
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> PerRetryPolicies;
};
/**
@ -77,10 +83,16 @@ namespace Azure { namespace Storage { namespace Blobs {
struct BlobContainerClientOptions
{
/**
* @brief Transport pipeline policies for authentication, retries, etc., that are
* applied to every request.
* @brief Transport pipeline policies for authentication, additional HTTP headers, etc., that
* are applied to every request.
*/
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> policies;
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> PerOperationPolicies;
/**
* @brief Transport pipeline policies for authentication, additional HTTP headers, etc., that
* are applied to every retrial.
*/
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> PerRetryPolicies;
};
/**
@ -157,7 +169,7 @@ namespace Azure { namespace Storage { namespace Blobs {
};
/**
* @brief Optional parameters for BlobContainerClient::ListBlobs.
* @brief Optional parameters for BlobContainerClient::ListBlobsFlat.
*/
struct ListBlobsOptions
{
@ -205,10 +217,16 @@ namespace Azure { namespace Storage { namespace Blobs {
struct BlobClientOptions
{
/**
* @brief Transport pipeline policies for authentication, retries, etc., that are
* applied to every request.
* @brief Transport pipeline policies for authentication, additional HTTP headers, etc., that
* are applied to every request.
*/
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> policies;
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> PerOperationPolicies;
/**
* @brief Transport pipeline policies for authentication, additional HTTP headers, etc., that
* are applied to every retrial.
*/
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> PerRetryPolicies;
};
/**
@ -278,36 +296,6 @@ namespace Azure { namespace Storage { namespace Blobs {
*/
Azure::Core::Context Context;
/**
* @brief The MIME content type of the blob.
*/
std::string ContentType;
/**
* @brief Specifies which content encodings have been applied to the blob.
*/
std::string ContentEncoding;
/**
* @brief Specifies the natural languages used by this resource.
*/
std::string ContentLanguage;
/**
* @brief Sets the blob’s MD5 hash.
*/
std::string ContentMD5;
/**
* @brief Sets the blob's cache control.
*/
std::string CacheControl;
/**
* @brief Sets the blob’s Content-Disposition header.
*/
std::string ContentDisposition;
/**
* @brief Specify this header to perform the operation only if the resource has been
* modified since the specified time.
@ -545,6 +533,50 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Nullable<std::string> IfNoneMatch;
};
/**
* @brief Optional parameters for BlobClient::DownloadToBuffer.
*/
struct DownloadBlobToBufferOptions
{
/**
* @brief Context for cancelling long running operations.
*/
Azure::Core::Context Context;
/**
* @brief Downloads only the bytes of the blob from this offset.
*/
Azure::Core::Nullable<int64_t> Offset;
/**
* @brief Returns at most this number of bytes of the blob from the offset. Null means
* download until the end.
*/
Azure::Core::Nullable<int64_t> Length;
/**
* @brief The size of the first range request in bytes. Blobs smaller than this limit will be
* downloaded in a single request. Blobs larger than this limit will continue being downloaded
* in chunks of size ChunkSize.
*/
Azure::Core::Nullable<int64_t> InitialChunkSize;
/**
* @brief The maximum number of bytes in a single request.
*/
Azure::Core::Nullable<int64_t> ChunkSize;
/**
* @brief The maximum number of threads that may be used in a parallel transfer.
*/
int Concurrency = 1;
};
/**
* @brief Optional parameters for BlobClient::DownloadToFile.
*/
using DownloadBlobToFileOptions = DownloadBlobToBufferOptions;
/**
* @brief Optional parameters for BlobClient::CreateSnapshot.
*/
@ -676,7 +708,7 @@ namespace Azure { namespace Storage { namespace Blobs {
/**
* @brief The standard HTTP header system properties to set.
*/
BlobHttpHeaders Properties;
BlobHttpHeaders HttpHeaders;
/**
* @brief Name-value pairs associated with the blob as metadata.
@ -714,6 +746,42 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Nullable<std::string> IfNoneMatch;
};
/**
* @brief Optional parameters for BlockBlobClient::UploadFromBuffer.
*/
struct UploadBlobOptions
{
/**
* @brief Context for cancelling long running operations.
*/
Azure::Core::Context Context;
/**
* @brief The standard HTTP header system properties to set.
*/
BlobHttpHeaders HttpHeaders;
/**
* @brief Name-value pairs associated with the blob as metadata.
*/
std::map<std::string, std::string> Metadata;
/**
* @brief Indicates the tier to be set on blob.
*/
Azure::Core::Nullable<AccessTier> Tier;
/**
* @brief The maximum number of bytes in a single request.
*/
Azure::Core::Nullable<int64_t> ChunkSize;
/**
* @brief The maximum number of threads that may be used in a parallel transfer.
*/
int Concurrency = 1;
};
/**
* @brief Optional parameters for BlockBlobClient::StageBlock.
*/
@ -818,7 +886,7 @@ namespace Azure { namespace Storage { namespace Blobs {
/**
* @brief The standard HTTP header system properties to set.
*/
BlobHttpHeaders Properties;
BlobHttpHeaders HttpHeaders;
/**
* @brief Name-value pairs associated with the blob as metadata.
@ -906,7 +974,7 @@ namespace Azure { namespace Storage { namespace Blobs {
/**
* @brief The standard HTTP header system properties to set.
*/
BlobHttpHeaders Properties;
BlobHttpHeaders HttpHeaders;
/**
* @brief Name-value pairs associated with the blob as metadata.
@ -1100,7 +1168,7 @@ namespace Azure { namespace Storage { namespace Blobs {
/**
* @brief The standard HTTP header system properties to set.
*/
BlobHttpHeaders Properties;
BlobHttpHeaders HttpHeaders;
/**
* @brief Name-value pairs associated with the blob as metadata.

7
sdk/storage/inc/blobs/blob_service_client.hpp
View File

@ -110,9 +110,10 @@ namespace Azure { namespace Storage { namespace Blobs {
* @brief Retrieves a key that can be used to delegate Active Directory authorization to
* shared access signatures.
*
* @param startsOn Start time for the key's validity. The time should be specified in UTC.
* @param expiresOn Expiration of the key's validity.
* The time should be specified in UTC.
* @param startsOn Start time for the key's validity, in ISO date format. The time should be
* specified in UTC.
* @param expiresOn Expiration of the key's validity, in ISO date format. The time should be
* specified in UTC.
* @param options Optional parameters to execute
* this function.
* @return A deserialized UserDelegationKey instance.

14
sdk/storage/inc/blobs/block_blob_client.hpp
View File

@ -113,6 +113,20 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Http::BodyStream& content,
const UploadBlockBlobOptions& options = UploadBlockBlobOptions()) const;
/**
* @brief Creates a new block blob, or updates the content of an existing block blob. Updating
* an existing block blob overwrites any existing metadata on the blob.
*
* @param buffer A memory buffer containing the content to upload.
* @param bufferSize Size of the memory buffer.
* @param options Optional parameters to execute this function.
* @return A BlockBlobInfo describing the state of the updated block blob.
*/
BlobContentInfo UploadFromBuffer(
const uint8_t* buffer,
std::size_t bufferSize,
const UploadBlobOptions& options = UploadBlobOptions()) const;
/**
* @brief Creates a new block as part of a block blob's staging area to be eventually
* committed via the CommitBlockList operation.

432
sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp
View File

@ -28,6 +28,14 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Http::BodyStream,
std::function<void(Azure::Core::Http::BodyStream*)>>;
struct AbortCopyBlobResponse
{
std::string RequestId;
std::string Date;
std::string Version;
Azure::Core::Nullable<std::string> ClientRequestId;
}; // struct AbortCopyBlobResponse
enum class AccessTier
{
Unknown,
@ -172,14 +180,6 @@ namespace Azure { namespace Storage { namespace Blobs {
throw std::runtime_error("cannot convert " + access_tier + " to AccessTier");
}
struct BasicResponse
{
std::string RequestId;
std::string Date;
std::string Version;
Azure::Core::Nullable<std::string> ClientRequestId;
}; // struct BasicResponse
struct BlobAppendInfo
{
std::string RequestId;
@ -276,17 +276,6 @@ namespace Azure { namespace Storage { namespace Blobs {
std::string ContentDisposition;
}; // struct BlobHttpHeaders
struct BlobInfo
{
std::string RequestId;
std::string Date;
std::string Version;
Azure::Core::Nullable<std::string> ClientRequestId;
std::string ETag;
std::string LastModified;
Azure::Core::Nullable<int64_t> SequenceNumber;
}; // struct BlobInfo
enum class BlobLeaseState
{
Available,
@ -562,6 +551,22 @@ namespace Azure { namespace Storage { namespace Blobs {
throw std::runtime_error("cannot convert " + copy_status + " to CopyStatus");
}
struct DeleteBlobResponse
{
std::string RequestId;
std::string Date;
std::string Version;
Azure::Core::Nullable<std::string> ClientRequestId;
}; // struct DeleteBlobResponse
struct DeleteContainerResponse
{
std::string RequestId;
std::string Date;
std::string Version;
Azure::Core::Nullable<std::string> ClientRequestId;
}; // struct DeleteContainerResponse
enum class DeleteSnapshotsOption
{
None,
@ -813,6 +818,54 @@ namespace Azure { namespace Storage { namespace Blobs {
throw std::runtime_error("cannot convert " + rehydrate_priority + " to RehydratePriority");
}
struct SetAccessTierResponse
{
std::string RequestId;
std::string Date;
std::string Version;
Azure::Core::Nullable<std::string> ClientRequestId;
}; // struct SetAccessTierResponse
struct SetBlobHttpHeadersResponse
{
std::string RequestId;
std::string Date;
std::string Version;
Azure::Core::Nullable<std::string> ClientRequestId;
std::string ETag;
std::string LastModified;
Azure::Core::Nullable<int64_t> SequenceNumber;
}; // struct SetBlobHttpHeadersResponse
struct SetBlobMetadataResponse
{
std::string RequestId;
std::string Date;
std::string Version;
Azure::Core::Nullable<std::string> ClientRequestId;
std::string ETag;
std::string LastModified;
Azure::Core::Nullable<int64_t> SequenceNumber;
}; // struct SetBlobMetadataResponse
struct SetContainerMetadataResponse
{
std::string RequestId;
std::string Date;
std::string Version;
Azure::Core::Nullable<std::string> ClientRequestId;
std::string ETag;
std::string LastModified;
}; // struct SetContainerMetadataResponse
struct UndeleteBlobResponse
{
std::string RequestId;
std::string Date;
std::string Version;
Azure::Core::Nullable<std::string> ClientRequestId;
}; // struct UndeleteBlobResponse
struct UserDelegationKey
{
std::string RequestId;
@ -885,7 +938,7 @@ namespace Azure { namespace Storage { namespace Blobs {
Blobs::CopyStatus CopyStatus = Blobs::CopyStatus::Unknown;
}; // struct BlobCopyInfo
struct BlobDownloadInfo
struct BlobDownloadResponse
{
std::string RequestId;
std::string Date;
@ -895,7 +948,7 @@ namespace Azure { namespace Storage { namespace Blobs {
std::string ETag;
std::string LastModified;
Azure::Core::Nullable<std::string> ContentRange;
BlobHttpHeaders Properties;
BlobHttpHeaders HttpHeaders;
std::map<std::string, std::string> Metadata;
Azure::Core::Nullable<int64_t> SequenceNumber; // only for page blob
Azure::Core::Nullable<int64_t> CommittedBlockCount; // only for append blob
@ -907,14 +960,14 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Nullable<BlobLeaseStatus> LeaseStatus;
Azure::Core::Nullable<bool> ServerEncrypted;
Azure::Core::Nullable<std::string> EncryptionKeySHA256;
}; // struct BlobDownloadInfo
}; // struct BlobDownloadResponse
struct BlobItem
{
std::string Name;
bool Deleted = false;
std::string Snapshot;
BlobHttpHeaders Properties;
BlobHttpHeaders HttpHeaders;
std::map<std::string, std::string> Metadata;
std::string CreationTime;
std::string LastModified;
@ -945,12 +998,7 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Nullable<BlobLeaseState> LeaseState;
Azure::Core::Nullable<BlobLeaseStatus> LeaseStatus;
int64_t ContentLength = 0;
std::string ContentType;
std::string ContentEncoding;
std::string ContentLanguage;
std::string ContentMD5;
std::string CacheControl;
std::string ContentDisposition;
BlobHttpHeaders HttpHeaders;
Azure::Core::Nullable<int64_t> SequenceNumber; // only for page blob
Azure::Core::Nullable<int32_t> CommittedBlockCount; // only for append blob
Azure::Core::Nullable<bool> ServerEncrypted;
@ -977,9 +1025,8 @@ namespace Azure { namespace Storage { namespace Blobs {
std::string Prefix;
std::string Marker;
std::string NextMarker;
Azure::Core::Nullable<int32_t> MaxResults;
std::string Delimiter;
std::vector<BlobItem> BlobItems;
std::vector<BlobItem> Items;
}; // struct BlobsFlatSegment
struct ListContainersSegment
@ -992,8 +1039,7 @@ namespace Azure { namespace Storage { namespace Blobs {
std::string Prefix;
std::string Marker;
std::string NextMarker;
Azure::Core::Nullable<int32_t> MaxResults;
std::vector<BlobContainerItem> BlobContainerItems;
std::vector<BlobContainerItem> Items;
}; // struct ListContainersSegment
class BlobRestClient {
@ -1201,7 +1247,6 @@ namespace Azure { namespace Storage { namespace Blobs {
k_Prefix,
k_Marker,
k_NextMarker,
k_MaxResults,
k_Containers,
k_Container,
k_Unknown,
@ -1243,10 +1288,6 @@ namespace Azure { namespace Storage { namespace Blobs {
{
path.emplace_back(XmlTagName::k_NextMarker);
}
else if (std::strcmp(node.Name, "MaxResults") == 0)
{
path.emplace_back(XmlTagName::k_MaxResults);
}
else if (std::strcmp(node.Name, "Containers") == 0)
{
path.emplace_back(XmlTagName::k_Containers);
@ -1262,7 +1303,7 @@ namespace Azure { namespace Storage { namespace Blobs {
if (path.size() == 3 && path[0] == XmlTagName::k_EnumerationResults
&& path[1] == XmlTagName::k_Containers && path[2] == XmlTagName::k_Container)
{
ret.BlobContainerItems.emplace_back(BlobContainerItemFromXml(reader));
ret.Items.emplace_back(BlobContainerItemFromXml(reader));
path.pop_back();
}
}
@ -1285,12 +1326,6 @@ namespace Azure { namespace Storage { namespace Blobs {
{
ret.NextMarker = node.Value;
}
else if (
path.size() == 2 && path[0] == XmlTagName::k_EnumerationResults
&& path[1] == XmlTagName::k_MaxResults)
{
ret.MaxResults = std::stoi(node.Value);
}
}
else if (node.Type == XmlNodeType::Attribute)
{
@ -1746,13 +1781,13 @@ namespace Azure { namespace Storage { namespace Blobs {
return request;
}
static BasicResponse DeleteParseResponse(
static DeleteContainerResponse DeleteParseResponse(
Azure::Core::Context context,
std::unique_ptr<Azure::Core::Http::Response> pHttpResponse)
{
unused(context);
Azure::Core::Http::Response& httpResponse = *pHttpResponse;
BasicResponse response;
DeleteContainerResponse response;
auto http_status_code
= static_cast<std::underlying_type<Azure::Core::Http::HttpStatusCode>::type>(
httpResponse.GetStatusCode());
@ -1772,7 +1807,7 @@ namespace Azure { namespace Storage { namespace Blobs {
return response;
}
static BasicResponse Delete(
static DeleteContainerResponse Delete(
Azure::Core::Context context,
Azure::Core::Http::HttpPipeline& pipeline,
const std::string& url,
@ -1931,13 +1966,13 @@ namespace Azure { namespace Storage { namespace Blobs {
return request;
}
static BlobContainerInfo SetMetadataParseResponse(
static SetContainerMetadataResponse SetMetadataParseResponse(
Azure::Core::Context context,
std::unique_ptr<Azure::Core::Http::Response> pHttpResponse)
{
unused(context);
Azure::Core::Http::Response& httpResponse = *pHttpResponse;
BlobContainerInfo response;
SetContainerMetadataResponse response;
auto http_status_code
= static_cast<std::underlying_type<Azure::Core::Http::HttpStatusCode>::type>(
httpResponse.GetStatusCode());
@ -1959,7 +1994,7 @@ namespace Azure { namespace Storage { namespace Blobs {
return response;
}
static BlobContainerInfo SetMetadata(
static SetContainerMetadataResponse SetMetadata(
Azure::Core::Context context,
Azure::Core::Http::HttpPipeline& pipeline,
const std::string& url,
@ -2094,7 +2129,6 @@ namespace Azure { namespace Storage { namespace Blobs {
k_Prefix,
k_Marker,
k_NextMarker,
k_MaxResults,
k_Delimiter,
k_Blobs,
k_Blob,
@ -2137,10 +2171,6 @@ namespace Azure { namespace Storage { namespace Blobs {
{
path.emplace_back(XmlTagName::k_NextMarker);
}
else if (std::strcmp(node.Name, "MaxResults") == 0)
{
path.emplace_back(XmlTagName::k_MaxResults);
}
else if (std::strcmp(node.Name, "Delimiter") == 0)
{
path.emplace_back(XmlTagName::k_Delimiter);
@ -2160,7 +2190,7 @@ namespace Azure { namespace Storage { namespace Blobs {
if (path.size() == 3 && path[0] == XmlTagName::k_EnumerationResults
&& path[1] == XmlTagName::k_Blobs && path[2] == XmlTagName::k_Blob)
{
ret.BlobItems.emplace_back(BlobItemFromXml(reader));
ret.Items.emplace_back(BlobItemFromXml(reader));
path.pop_back();
}
}
@ -2183,12 +2213,6 @@ namespace Azure { namespace Storage { namespace Blobs {
{
ret.NextMarker = node.Value;
}
else if (
path.size() == 2 && path[0] == XmlTagName::k_EnumerationResults
&& path[1] == XmlTagName::k_MaxResults)
{
ret.MaxResults = std::stoi(node.Value);
}
else if (
path.size() == 2 && path[0] == XmlTagName::k_EnumerationResults
&& path[1] == XmlTagName::k_Delimiter)
@ -2385,37 +2409,37 @@ namespace Azure { namespace Storage { namespace Blobs {
path.size() == 2 && path[0] == XmlTagName::k_Properties
&& path[1] == XmlTagName::k_ContentType)
{
ret.Properties.ContentType = node.Value;
ret.HttpHeaders.ContentType = node.Value;
}
else if (
path.size() == 2 && path[0] == XmlTagName::k_Properties
&& path[1] == XmlTagName::k_ContentEncoding)
{
ret.Properties.ContentEncoding = node.Value;
ret.HttpHeaders.ContentEncoding = node.Value;
}
else if (
path.size() == 2 && path[0] == XmlTagName::k_Properties
&& path[1] == XmlTagName::k_ContentLanguage)
{
ret.Properties.ContentLanguage = node.Value;
ret.HttpHeaders.ContentLanguage = node.Value;
}
else if (
path.size() == 2 && path[0] == XmlTagName::k_Properties
&& path[1] == XmlTagName::k_ContentMD5)
{
ret.Properties.ContentMD5 = node.Value;
ret.HttpHeaders.ContentMD5 = node.Value;
}
else if (
path.size() == 2 && path[0] == XmlTagName::k_Properties
&& path[1] == XmlTagName::k_CacheControl)
{
ret.Properties.CacheControl = node.Value;
ret.HttpHeaders.CacheControl = node.Value;
}
else if (
path.size() == 2 && path[0] == XmlTagName::k_Properties
&& path[1] == XmlTagName::k_ContentDisposition)
{
ret.Properties.ContentDisposition = node.Value;
ret.HttpHeaders.ContentDisposition = node.Value;
}
else if (
path.size() == 2 && path[0] == XmlTagName::k_Properties
@ -2604,13 +2628,13 @@ namespace Azure { namespace Storage { namespace Blobs {
return request;
}
static BlobDownloadInfo DownloadParseResponse(
static BlobDownloadResponse DownloadParseResponse(
Azure::Core::Context context,
std::unique_ptr<Azure::Core::Http::Response> pHttpResponse)
{
unused(context);
Azure::Core::Http::Response& httpResponse = *pHttpResponse;
BlobDownloadInfo response;
BlobDownloadResponse response;
auto http_status_code
= static_cast<std::underlying_type<Azure::Core::Http::HttpStatusCode>::type>(
httpResponse.GetStatusCode());
@ -2639,44 +2663,44 @@ namespace Azure { namespace Storage { namespace Blobs {
{
response.ContentCRC64 = response_content_crc64_iterator->second;
}
auto response_properties_content_type_iterator
auto response_http_headers_content_type_iterator
= httpResponse.GetHeaders().find("Content-Type");
if (response_properties_content_type_iterator != httpResponse.GetHeaders().end())
if (response_http_headers_content_type_iterator != httpResponse.GetHeaders().end())
{
response.Properties.ContentType = response_properties_content_type_iterator->second;
response.HttpHeaders.ContentType = response_http_headers_content_type_iterator->second;
}
auto response_properties_content_encoding_iterator
auto response_http_headers_content_encoding_iterator
= httpResponse.GetHeaders().find("Content-Encoding");
if (response_properties_content_encoding_iterator != httpResponse.GetHeaders().end())
if (response_http_headers_content_encoding_iterator != httpResponse.GetHeaders().end())
{
response.Properties.ContentEncoding
= response_properties_content_encoding_iterator->second;
response.HttpHeaders.ContentEncoding
= response_http_headers_content_encoding_iterator->second;
}
auto response_properties_content_language_iterator
auto response_http_headers_content_language_iterator
= httpResponse.GetHeaders().find("Content-Language");
if (response_properties_content_language_iterator != httpResponse.GetHeaders().end())
if (response_http_headers_content_language_iterator != httpResponse.GetHeaders().end())
{
response.Properties.ContentLanguage
= response_properties_content_language_iterator->second;
response.HttpHeaders.ContentLanguage
= response_http_headers_content_language_iterator->second;
}
auto response_properties_cache_control_iterator
auto response_http_headers_cache_control_iterator
= httpResponse.GetHeaders().find("Cache-Control");
if (response_properties_cache_control_iterator != httpResponse.GetHeaders().end())
if (response_http_headers_cache_control_iterator != httpResponse.GetHeaders().end())
{
response.Properties.CacheControl = response_properties_cache_control_iterator->second;
response.HttpHeaders.CacheControl = response_http_headers_cache_control_iterator->second;
}
auto response_properties_content_md5_iterator
auto response_http_headers_content_md5_iterator
= httpResponse.GetHeaders().find("Content-MD5");
if (response_properties_content_md5_iterator != httpResponse.GetHeaders().end())
if (response_http_headers_content_md5_iterator != httpResponse.GetHeaders().end())
{
response.Properties.ContentMD5 = response_properties_content_md5_iterator->second;
response.HttpHeaders.ContentMD5 = response_http_headers_content_md5_iterator->second;
}
auto response_properties_content_disposition_iterator
auto response_http_headers_content_disposition_iterator
= httpResponse.GetHeaders().find("Content-Disposition");
if (response_properties_content_disposition_iterator != httpResponse.GetHeaders().end())
if (response_http_headers_content_disposition_iterator != httpResponse.GetHeaders().end())
{
response.Properties.ContentDisposition
= response_properties_content_disposition_iterator->second;
response.HttpHeaders.ContentDisposition
= response_http_headers_content_disposition_iterator->second;
}
for (auto i = httpResponse.GetHeaders().lower_bound("x-ms-meta-");
i != httpResponse.GetHeaders().end() && i->first.substr(0, 10) == "x-ms-meta-";
@ -2735,7 +2759,7 @@ namespace Azure { namespace Storage { namespace Blobs {
return response;
}
static BlobDownloadInfo Download(
static BlobDownloadResponse Download(
Azure::Core::Context context,
Azure::Core::Http::HttpPipeline& pipeline,
const std::string& url,
@ -2796,13 +2820,13 @@ namespace Azure { namespace Storage { namespace Blobs {
return request;
}
static BasicResponse DeleteParseResponse(
static DeleteBlobResponse DeleteParseResponse(
Azure::Core::Context context,
std::unique_ptr<Azure::Core::Http::Response> pHttpResponse)
{
unused(context);
Azure::Core::Http::Response& httpResponse = *pHttpResponse;
BasicResponse response;
DeleteBlobResponse response;
auto http_status_code
= static_cast<std::underlying_type<Azure::Core::Http::HttpStatusCode>::type>(
httpResponse.GetStatusCode());
@ -2822,7 +2846,7 @@ namespace Azure { namespace Storage { namespace Blobs {
return response;
}
static BasicResponse Delete(
static DeleteBlobResponse Delete(
Azure::Core::Context context,
Azure::Core::Http::HttpPipeline& pipeline,
const std::string& url,
@ -2857,13 +2881,13 @@ namespace Azure { namespace Storage { namespace Blobs {
return request;
}
static BasicResponse UndeleteParseResponse(
static UndeleteBlobResponse UndeleteParseResponse(
Azure::Core::Context context,
std::unique_ptr<Azure::Core::Http::Response> pHttpResponse)
{
unused(context);
Azure::Core::Http::Response& httpResponse = *pHttpResponse;
BasicResponse response;
UndeleteBlobResponse response;
auto http_status_code
= static_cast<std::underlying_type<Azure::Core::Http::HttpStatusCode>::type>(
httpResponse.GetStatusCode());
@ -2883,7 +2907,7 @@ namespace Azure { namespace Storage { namespace Blobs {
return response;
}
static BasicResponse Undelete(
static UndeleteBlobResponse Undelete(
Azure::Core::Context context,
Azure::Core::Http::HttpPipeline& pipeline,
const std::string& url,
@ -2987,38 +3011,44 @@ namespace Azure { namespace Storage { namespace Blobs {
response.LeaseDuration = response_lease_duration_iterator->second;
}
response.ContentLength = std::stoll(httpResponse.GetHeaders().at("Content-Length"));
auto response_content_type_iterator = httpResponse.GetHeaders().find("Content-Type");
if (response_content_type_iterator != httpResponse.GetHeaders().end())
auto response_http_headers_content_type_iterator
= httpResponse.GetHeaders().find("Content-Type");
if (response_http_headers_content_type_iterator != httpResponse.GetHeaders().end())
{
response.ContentType = response_content_type_iterator->second;
response.HttpHeaders.ContentType = response_http_headers_content_type_iterator->second;
}
auto response_content_encoding_iterator
auto response_http_headers_content_encoding_iterator
= httpResponse.GetHeaders().find("Content-Encoding");
if (response_content_encoding_iterator != httpResponse.GetHeaders().end())
if (response_http_headers_content_encoding_iterator != httpResponse.GetHeaders().end())
{
response.ContentEncoding = response_content_encoding_iterator->second;
response.HttpHeaders.ContentEncoding
= response_http_headers_content_encoding_iterator->second;
}
auto response_content_language_iterator
auto response_http_headers_content_language_iterator
= httpResponse.GetHeaders().find("Content-Language");
if (response_content_language_iterator != httpResponse.GetHeaders().end())
if (response_http_headers_content_language_iterator != httpResponse.GetHeaders().end())
{
response.ContentLanguage = response_content_language_iterator->second;
response.HttpHeaders.ContentLanguage
= response_http_headers_content_language_iterator->second;
}
auto response_cache_control_iterator = httpResponse.GetHeaders().find("Cache-Control");
if (response_cache_control_iterator != httpResponse.GetHeaders().end())
auto response_http_headers_cache_control_iterator
= httpResponse.GetHeaders().find("Cache-Control");
if (response_http_headers_cache_control_iterator != httpResponse.GetHeaders().end())
{
response.CacheControl = response_cache_control_iterator->second;
response.HttpHeaders.CacheControl = response_http_headers_cache_control_iterator->second;
}
auto response_content_md5_iterator = httpResponse.GetHeaders().find("Content-MD5");
if (response_content_md5_iterator != httpResponse.GetHeaders().end())
auto response_http_headers_content_md5_iterator
= httpResponse.GetHeaders().find("Content-MD5");
if (response_http_headers_content_md5_iterator != httpResponse.GetHeaders().end())
{
response.ContentMD5 = response_content_md5_iterator->second;
response.HttpHeaders.ContentMD5 = response_http_headers_content_md5_iterator->second;
}
auto response_content_disposition_iterator
auto response_http_headers_content_disposition_iterator
= httpResponse.GetHeaders().find("Content-Disposition");
if (response_content_disposition_iterator != httpResponse.GetHeaders().end())
if (response_http_headers_content_disposition_iterator != httpResponse.GetHeaders().end())
{
response.ContentDisposition = response_content_disposition_iterator->second;
response.HttpHeaders.ContentDisposition
= response_http_headers_content_disposition_iterator->second;
}
auto response_sequence_number_iterator
= httpResponse.GetHeaders().find("x-ms-blob-sequence-number");
@ -3113,12 +3143,7 @@ namespace Azure { namespace Storage { namespace Blobs {
struct SetHttpHeadersOptions
{
Azure::Core::Nullable<int32_t> Timeout;
std::string ContentType;
std::string ContentEncoding;
std::string ContentLanguage;
std::string ContentMD5;
std::string CacheControl;
std::string ContentDisposition;
BlobHttpHeaders HttpHeaders;
Azure::Core::Nullable<std::string> EncryptionKey;
Azure::Core::Nullable<std::string> EncryptionKeySHA256;
Azure::Core::Nullable<std::string> EncryptionAlgorithm;
@ -3142,29 +3167,30 @@ namespace Azure { namespace Storage { namespace Blobs {
{
request.AddQueryParameter("timeout", std::to_string(options.Timeout.GetValue()));
}
if (!options.ContentType.empty())
if (!options.HttpHeaders.ContentType.empty())
{
request.AddHeader("x-ms-blob-content-type", options.ContentType);
request.AddHeader("x-ms-blob-content-type", options.HttpHeaders.ContentType);
}
if (!options.ContentEncoding.empty())
if (!options.HttpHeaders.ContentEncoding.empty())
{
request.AddHeader("x-ms-blob-content-encoding", options.ContentEncoding);
request.AddHeader("x-ms-blob-content-encoding", options.HttpHeaders.ContentEncoding);
}
if (!options.ContentLanguage.empty())
if (!options.HttpHeaders.ContentLanguage.empty())
{
request.AddHeader("x-ms-blob-content-language", options.ContentLanguage);
request.AddHeader("x-ms-blob-content-language", options.HttpHeaders.ContentLanguage);
}
if (!options.CacheControl.empty())
if (!options.HttpHeaders.CacheControl.empty())
{
request.AddHeader("x-ms-blob-cache-control", options.CacheControl);
request.AddHeader("x-ms-blob-cache-control", options.HttpHeaders.CacheControl);
}
if (!options.ContentMD5.empty())
if (!options.HttpHeaders.ContentMD5.empty())
{
request.AddHeader("x-ms-blob-content-md5", options.ContentMD5);
request.AddHeader("x-ms-blob-content-md5", options.HttpHeaders.ContentMD5);
}
if (!options.ContentDisposition.empty())
if (!options.HttpHeaders.ContentDisposition.empty())
{
request.AddHeader("x-ms-blob-content-disposition", options.ContentDisposition);
request.AddHeader(
"x-ms-blob-content-disposition", options.HttpHeaders.ContentDisposition);
}
if (options.EncryptionKey.HasValue())
{
@ -3197,13 +3223,13 @@ namespace Azure { namespace Storage { namespace Blobs {
return request;
}
static BlobInfo SetHttpHeadersParseResponse(
static SetBlobHttpHeadersResponse SetHttpHeadersParseResponse(
Azure::Core::Context context,
std::unique_ptr<Azure::Core::Http::Response> pHttpResponse)
{
unused(context);
Azure::Core::Http::Response& httpResponse = *pHttpResponse;
BlobInfo response;
SetBlobHttpHeadersResponse response;
auto http_status_code
= static_cast<std::underlying_type<Azure::Core::Http::HttpStatusCode>::type>(
httpResponse.GetStatusCode());
@ -3231,7 +3257,7 @@ namespace Azure { namespace Storage { namespace Blobs {
return response;
}
static BlobInfo SetHttpHeaders(
static SetBlobHttpHeadersResponse SetHttpHeaders(
Azure::Core::Context context,
Azure::Core::Http::HttpPipeline& pipeline,
const std::string& url,
@ -3316,13 +3342,13 @@ namespace Azure { namespace Storage { namespace Blobs {
return request;
}
static BlobInfo SetMetadataParseResponse(
static SetBlobMetadataResponse SetMetadataParseResponse(
Azure::Core::Context context,
std::unique_ptr<Azure::Core::Http::Response> pHttpResponse)
{
unused(context);
Azure::Core::Http::Response& httpResponse = *pHttpResponse;
BlobInfo response;
SetBlobMetadataResponse response;
auto http_status_code
= static_cast<std::underlying_type<Azure::Core::Http::HttpStatusCode>::type>(
httpResponse.GetStatusCode());
@ -3344,7 +3370,7 @@ namespace Azure { namespace Storage { namespace Blobs {
return response;
}
static BlobInfo SetMetadata(
static SetBlobMetadataResponse SetMetadata(
Azure::Core::Context context,
Azure::Core::Http::HttpPipeline& pipeline,
const std::string& url,
@ -3388,13 +3414,13 @@ namespace Azure { namespace Storage { namespace Blobs {
return request;
}
static BasicResponse SetAccessTierParseResponse(
static SetAccessTierResponse SetAccessTierParseResponse(
Azure::Core::Context context,
std::unique_ptr<Azure::Core::Http::Response> pHttpResponse)
{
unused(context);
Azure::Core::Http::Response& httpResponse = *pHttpResponse;
BasicResponse response;
SetAccessTierResponse response;
auto http_status_code
= static_cast<std::underlying_type<Azure::Core::Http::HttpStatusCode>::type>(
httpResponse.GetStatusCode());
@ -3414,7 +3440,7 @@ namespace Azure { namespace Storage { namespace Blobs {
return response;
}
static BasicResponse SetAccessTier(
static SetAccessTierResponse SetAccessTier(
Azure::Core::Context context,
Azure::Core::Http::HttpPipeline& pipeline,
const std::string& url,
@ -3603,13 +3629,13 @@ namespace Azure { namespace Storage { namespace Blobs {
return request;
}
static BasicResponse AbortCopyFromUriParseResponse(
static AbortCopyBlobResponse AbortCopyFromUriParseResponse(
Azure::Core::Context context,
std::unique_ptr<Azure::Core::Http::Response> pHttpResponse)
{
unused(context);
Azure::Core::Http::Response& httpResponse = *pHttpResponse;
BasicResponse response;
AbortCopyBlobResponse response;
auto http_status_code
= static_cast<std::underlying_type<Azure::Core::Http::HttpStatusCode>::type>(
httpResponse.GetStatusCode());
@ -3629,7 +3655,7 @@ namespace Azure { namespace Storage { namespace Blobs {
return response;
}
static BasicResponse AbortCopyFromUri(
static AbortCopyBlobResponse AbortCopyFromUri(
Azure::Core::Context context,
Azure::Core::Http::HttpPipeline& pipeline,
const std::string& url,
@ -3783,7 +3809,7 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Nullable<int32_t> Timeout;
Azure::Core::Nullable<std::string> ContentMD5;
Azure::Core::Nullable<std::string> ContentCRC64;
BlobHttpHeaders Properties;
BlobHttpHeaders HttpHeaders;
std::map<std::string, std::string> Metadata;
Azure::Core::Nullable<std::string> LeaseId;
Azure::Core::Nullable<AccessTier> Tier;
@ -3829,29 +3855,30 @@ namespace Azure { namespace Storage { namespace Blobs {
{
request.AddHeader("x-ms-content-crc64", options.ContentCRC64.GetValue());
}
if (!options.Properties.ContentType.empty())
if (!options.HttpHeaders.ContentType.empty())
{
request.AddHeader("x-ms-blob-content-type", options.Properties.ContentType);
request.AddHeader("x-ms-blob-content-type", options.HttpHeaders.ContentType);
}
if (!options.Properties.ContentEncoding.empty())
if (!options.HttpHeaders.ContentEncoding.empty())
{
request.AddHeader("x-ms-blob-content-encoding", options.Properties.ContentEncoding);
request.AddHeader("x-ms-blob-content-encoding", options.HttpHeaders.ContentEncoding);
}
if (!options.Properties.ContentLanguage.empty())
if (!options.HttpHeaders.ContentLanguage.empty())
{
request.AddHeader("x-ms-blob-content-language", options.Properties.ContentLanguage);
request.AddHeader("x-ms-blob-content-language", options.HttpHeaders.ContentLanguage);
}
if (!options.Properties.CacheControl.empty())
if (!options.HttpHeaders.CacheControl.empty())
{
request.AddHeader("x-ms-blob-cache-control", options.Properties.CacheControl);
request.AddHeader("x-ms-blob-cache-control", options.HttpHeaders.CacheControl);
}
if (!options.Properties.ContentMD5.empty())
if (!options.HttpHeaders.ContentMD5.empty())
{
request.AddHeader("x-ms-blob-content-md5", options.Properties.ContentMD5);
request.AddHeader("x-ms-blob-content-md5", options.HttpHeaders.ContentMD5);
}
if (!options.Properties.ContentDisposition.empty())
if (!options.HttpHeaders.ContentDisposition.empty())
{
request.AddHeader("x-ms-blob-content-disposition", options.Properties.ContentDisposition);
request.AddHeader(
"x-ms-blob-content-disposition", options.HttpHeaders.ContentDisposition);
}
std::set<std::string> metadataKeys;
for (const auto& pair : options.Metadata)
@ -4236,7 +4263,7 @@ namespace Azure { namespace Storage { namespace Blobs {
{
Azure::Core::Nullable<int32_t> Timeout;
std::vector<std::pair<BlockType, std::string>> BlockList;
BlobHttpHeaders Properties;
BlobHttpHeaders HttpHeaders;
std::map<std::string, std::string> Metadata;
Azure::Core::Nullable<std::string> LeaseId;
Azure::Core::Nullable<std::string> EncryptionKey;
@ -4275,29 +4302,30 @@ namespace Azure { namespace Storage { namespace Blobs {
{
request.AddQueryParameter("timeout", std::to_string(options.Timeout.GetValue()));
}
if (!options.Properties.ContentType.empty())
if (!options.HttpHeaders.ContentType.empty())
{
request.AddHeader("x-ms-blob-content-type", options.Properties.ContentType);
request.AddHeader("x-ms-blob-content-type", options.HttpHeaders.ContentType);
}
if (!options.Properties.ContentEncoding.empty())
if (!options.HttpHeaders.ContentEncoding.empty())
{
request.AddHeader("x-ms-blob-content-encoding", options.Properties.ContentEncoding);
request.AddHeader("x-ms-blob-content-encoding", options.HttpHeaders.ContentEncoding);
}
if (!options.Properties.ContentLanguage.empty())
if (!options.HttpHeaders.ContentLanguage.empty())
{
request.AddHeader("x-ms-blob-content-language", options.Properties.ContentLanguage);
request.AddHeader("x-ms-blob-content-language", options.HttpHeaders.ContentLanguage);
}
if (!options.Properties.CacheControl.empty())
if (!options.HttpHeaders.CacheControl.empty())
{
request.AddHeader("x-ms-blob-cache-control", options.Properties.CacheControl);
request.AddHeader("x-ms-blob-cache-control", options.HttpHeaders.CacheControl);
}
if (!options.Properties.ContentMD5.empty())
if (!options.HttpHeaders.ContentMD5.empty())
{
request.AddHeader("x-ms-blob-content-md5", options.Properties.ContentMD5);
request.AddHeader("x-ms-blob-content-md5", options.HttpHeaders.ContentMD5);
}
if (!options.Properties.ContentDisposition.empty())
if (!options.HttpHeaders.ContentDisposition.empty())
{
request.AddHeader("x-ms-blob-content-disposition", options.Properties.ContentDisposition);
request.AddHeader(
"x-ms-blob-content-disposition", options.HttpHeaders.ContentDisposition);
}
std::set<std::string> metadataKeys;
for (const auto& pair : options.Metadata)
@ -4667,9 +4695,9 @@ namespace Azure { namespace Storage { namespace Blobs {
struct CreateOptions
{
Azure::Core::Nullable<int32_t> Timeout;
int64_t BlobContentLength;
int64_t BlobContentLength = -1;
Azure::Core::Nullable<int64_t> SequenceNumber;
BlobHttpHeaders Properties;
BlobHttpHeaders HttpHeaders;
std::map<std::string, std::string> Metadata;
Azure::Core::Nullable<std::string> LeaseId;
Azure::Core::Nullable<AccessTier> Tier;
@ -4695,29 +4723,30 @@ namespace Azure { namespace Storage { namespace Blobs {
{
request.AddQueryParameter("timeout", std::to_string(options.Timeout.GetValue()));
}
if (!options.Properties.ContentType.empty())
if (!options.HttpHeaders.ContentType.empty())
{
request.AddHeader("x-ms-blob-content-type", options.Properties.ContentType);
request.AddHeader("x-ms-blob-content-type", options.HttpHeaders.ContentType);
}
if (!options.Properties.ContentEncoding.empty())
if (!options.HttpHeaders.ContentEncoding.empty())
{
request.AddHeader("x-ms-blob-content-encoding", options.Properties.ContentEncoding);
request.AddHeader("x-ms-blob-content-encoding", options.HttpHeaders.ContentEncoding);
}
if (!options.Properties.ContentLanguage.empty())
if (!options.HttpHeaders.ContentLanguage.empty())
{
request.AddHeader("x-ms-blob-content-language", options.Properties.ContentLanguage);
request.AddHeader("x-ms-blob-content-language", options.HttpHeaders.ContentLanguage);
}
if (!options.Properties.CacheControl.empty())
if (!options.HttpHeaders.CacheControl.empty())
{
request.AddHeader("x-ms-blob-cache-control", options.Properties.CacheControl);
request.AddHeader("x-ms-blob-cache-control", options.HttpHeaders.CacheControl);
}
if (!options.Properties.ContentMD5.empty())
if (!options.HttpHeaders.ContentMD5.empty())
{
request.AddHeader("x-ms-blob-content-md5", options.Properties.ContentMD5);
request.AddHeader("x-ms-blob-content-md5", options.HttpHeaders.ContentMD5);
}
if (!options.Properties.ContentDisposition.empty())
if (!options.HttpHeaders.ContentDisposition.empty())
{
request.AddHeader("x-ms-blob-content-disposition", options.Properties.ContentDisposition);
request.AddHeader(
"x-ms-blob-content-disposition", options.HttpHeaders.ContentDisposition);
}
std::set<std::string> metadataKeys;
for (const auto& pair : options.Metadata)
@ -5326,7 +5355,7 @@ namespace Azure { namespace Storage { namespace Blobs {
struct ResizeOptions
{
Azure::Core::Nullable<int32_t> Timeout;
int64_t BlobContentLength;
int64_t BlobContentLength = -1;
Azure::Core::Nullable<std::string> LeaseId;
Azure::Core::Nullable<int64_t> IfSequenceNumberLessThanOrEqualTo;
Azure::Core::Nullable<int64_t> IfSequenceNumberLessThan;
@ -5843,7 +5872,7 @@ namespace Azure { namespace Storage { namespace Blobs {
struct CreateOptions
{
Azure::Core::Nullable<int32_t> Timeout;
BlobHttpHeaders Properties;
BlobHttpHeaders HttpHeaders;
std::map<std::string, std::string> Metadata;
Azure::Core::Nullable<std::string> LeaseId;
Azure::Core::Nullable<std::string> EncryptionKey;
@ -5868,29 +5897,30 @@ namespace Azure { namespace Storage { namespace Blobs {
{
request.AddQueryParameter("timeout", std::to_string(options.Timeout.GetValue()));
}
if (!options.Properties.ContentType.empty())
if (!options.HttpHeaders.ContentType.empty())
{
request.AddHeader("x-ms-blob-content-type", options.Properties.ContentType);
request.AddHeader("x-ms-blob-content-type", options.HttpHeaders.ContentType);
}
if (!options.Properties.ContentEncoding.empty())
if (!options.HttpHeaders.ContentEncoding.empty())
{
request.AddHeader("x-ms-blob-content-encoding", options.Properties.ContentEncoding);
request.AddHeader("x-ms-blob-content-encoding", options.HttpHeaders.ContentEncoding);
}
if (!options.Properties.ContentLanguage.empty())
if (!options.HttpHeaders.ContentLanguage.empty())
{
request.AddHeader("x-ms-blob-content-language", options.Properties.ContentLanguage);
request.AddHeader("x-ms-blob-content-language", options.HttpHeaders.ContentLanguage);
}
if (!options.Properties.CacheControl.empty())
if (!options.HttpHeaders.CacheControl.empty())
{
request.AddHeader("x-ms-blob-cache-control", options.Properties.CacheControl);
request.AddHeader("x-ms-blob-cache-control", options.HttpHeaders.CacheControl);
}
if (!options.Properties.ContentMD5.empty())
if (!options.HttpHeaders.ContentMD5.empty())
{
request.AddHeader("x-ms-blob-content-md5", options.Properties.ContentMD5);
request.AddHeader("x-ms-blob-content-md5", options.HttpHeaders.ContentMD5);
}
if (!options.Properties.ContentDisposition.empty())
if (!options.HttpHeaders.ContentDisposition.empty())
{
request.AddHeader("x-ms-blob-content-disposition", options.Properties.ContentDisposition);
request.AddHeader(
"x-ms-blob-content-disposition", options.HttpHeaders.ContentDisposition);
}
std::set<std::string> metadataKeys;
for (const auto& pair : options.Metadata)

70
sdk/storage/inc/common/concurrent_transfer.hpp Normal file
View File

@ -0,0 +1,70 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// SPDX-License-Identifier: MIT
#pragma once
#include <atomic>
#include <cstdlib>
#include <functional>
#include <future>
namespace Azure { namespace Storage { namespace Details {
inline void ConcurrentTransfer(
int64_t offset,
int64_t length,
int64_t chunkSize,
int concurrency,
// offset, length, chunk id, number of chunks
std::function<void(int64_t, int64_t, int64_t, int64_t)> transferFunc)
{
std::atomic<int> numWorkingThreads{concurrency};
std::atomic<int> nextChunkId{0};
std::atomic<bool> failed{false};
const auto numChunks = (length + chunkSize - 1) / chunkSize;
auto threadFunc = [&]() {
while (true)
{
int chunkId = nextChunkId.fetch_add(1);
if (chunkId >= numChunks || failed)
{
break;
}
int64_t chunkOffset = offset + chunkSize * chunkId;
int64_t chunkLength = std::min(length - chunkSize * chunkId, chunkSize);
try
{
transferFunc(chunkOffset, chunkLength, chunkId, numChunks);
}
catch (std::exception&)
{
if (failed.exchange(true) == false)
{
numWorkingThreads.fetch_sub(1);
throw;
}
}
}
numWorkingThreads.fetch_sub(1);
};
std::vector<std::future<void>> threadHandles;
for (int i = 0; i < concurrency - 1; ++i)
{
threadHandles.emplace_back(std::async(std::launch::async, threadFunc));
}
threadFunc();
for (auto& handle : threadHandles)
{
handle.get();
}
if (numWorkingThreads != 0)
{
std::abort();
}
}
}}} // namespace Azure::Storage::Details

2
sdk/storage/src/blobs/append_blob_client.cpp
View File

@ -60,7 +60,7 @@ namespace Azure { namespace Storage { namespace Blobs {
BlobContentInfo AppendBlobClient::Create(const CreateAppendBlobOptions& options)
{
BlobRestClient::AppendBlob::CreateOptions protocolLayerOptions;
protocolLayerOptions.Properties = options.Properties;
protocolLayerOptions.HttpHeaders = options.HttpHeaders;
protocolLayerOptions.Metadata = options.Metadata;
protocolLayerOptions.IfModifiedSince = options.IfModifiedSince;
protocolLayerOptions.IfUnmodifiedSince = options.IfUnmodifiedSince;

174
sdk/storage/src/blobs/blob_client.cpp
View File

@ -7,6 +7,7 @@
#include "blobs/block_blob_client.hpp"
#include "blobs/page_blob_client.hpp"
#include "common/common_headers_request_policy.hpp"
#include "common/concurrent_transfer.hpp"
#include "common/shared_key_policy.hpp"
#include "common/storage_common.hpp"
#include "http/curl/curl.hpp"
@ -41,7 +42,12 @@ namespace Azure { namespace Storage { namespace Blobs {
: m_blobUrl(blobUri)
{
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> policies;
for (const auto& p : options.policies)
for (const auto& p : options.PerOperationPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
// TODO: Retry policy goes here
for (const auto& p : options.PerRetryPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
@ -59,7 +65,12 @@ namespace Azure { namespace Storage { namespace Blobs {
: m_blobUrl(blobUri)
{
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> policies;
for (const auto& p : options.policies)
for (const auto& p : options.PerOperationPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
// TODO: Retry policy goes here
for (const auto& p : options.PerRetryPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
@ -75,7 +86,12 @@ namespace Azure { namespace Storage { namespace Blobs {
: m_blobUrl(blobUri)
{
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> policies;
for (const auto& p : options.policies)
for (const auto& p : options.PerOperationPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
// TODO: Retry policy goes here
for (const auto& p : options.PerRetryPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
@ -105,7 +121,7 @@ namespace Azure { namespace Storage { namespace Blobs {
return newClient;
}
BlobDownloadInfo BlobClient::Download(const DownloadBlobOptions& options) const
BlobDownloadResponse BlobClient::Download(const DownloadBlobOptions& options) const
{
BlobRestClient::Blob::DownloadOptions protocolLayerOptions;
if (options.Offset.HasValue() && options.Length.HasValue())
@ -128,6 +144,132 @@ namespace Azure { namespace Storage { namespace Blobs {
options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions);
}
BlobDownloadInfo BlobClient::DownloadToBuffer(
uint8_t* buffer,
std::size_t bufferSize,
const DownloadBlobToBufferOptions& options) const
{
constexpr int64_t c_defaultChunkSize = 8 * 1024 * 1024;
// Just start downloading using an initial chunk. If it's a small blob, we'll get the whole
// thing in one shot. If it's a large blob, we'll get its full size in Content-Range and can
// keep downloading it in chunks.
int64_t firstChunkOffset = options.Offset.HasValue() ? options.Offset.GetValue() : 0;
int64_t firstChunkLength = c_defaultChunkSize;
if (options.InitialChunkSize.HasValue())
{
firstChunkLength = options.InitialChunkSize.GetValue();
}
if (options.Length.HasValue())
{
firstChunkLength = std::min(firstChunkLength, options.Length.GetValue());
}
DownloadBlobOptions firstChunkOptions;
firstChunkOptions.Context = options.Context;
firstChunkOptions.Offset = options.Offset;
if (firstChunkOptions.Offset.HasValue())
{
firstChunkOptions.Length = firstChunkLength;
}
auto firstChunk = Download(firstChunkOptions);
int64_t blobSize;
int64_t blobRangeSize;
if (firstChunkOptions.Offset.HasValue())
{
blobSize = std::stoll(firstChunk.ContentRange.GetValue().substr(
firstChunk.ContentRange.GetValue().find('/') + 1));
blobRangeSize = blobSize - firstChunkOffset;
if (options.Length.HasValue())
{
blobRangeSize = std::min(blobRangeSize, options.Length.GetValue());
}
}
else
{
blobSize = firstChunk.BodyStream->Length();
blobRangeSize = blobSize;
}
firstChunkLength = std::min(firstChunkLength, blobRangeSize);
if (static_cast<std::size_t>(blobRangeSize) > bufferSize)
{
throw std::runtime_error(
"buffer is not big enough, blob range size is " + std::to_string(blobRangeSize));
}
int64_t bytesRead = Azure::Core::Http::BodyStream::ReadToCount(
firstChunkOptions.Context, *firstChunk.BodyStream, buffer, firstChunkLength);
if (bytesRead != firstChunkLength)
{
throw std::runtime_error("error when reading body stream");
}
firstChunk.BodyStream.reset();
auto returnTypeConverter = [](BlobDownloadResponse& response) {
BlobDownloadInfo ret;
ret.ETag = std::move(response.ETag);
ret.LastModified = std::move(response.LastModified);
ret.HttpHeaders = std::move(response.HttpHeaders);
ret.Metadata = std::move(response.Metadata);
ret.BlobType = response.BlobType;
ret.ServerEncrypted = response.ServerEncrypted;
ret.EncryptionKeySHA256 = std::move(response.EncryptionKeySHA256);
return ret;
};
BlobDownloadInfo ret = returnTypeConverter(firstChunk);
// Keep downloading the remaining in parallel
auto downloadChunkFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
DownloadBlobOptions chunkOptions;
chunkOptions.Context = options.Context;
chunkOptions.Offset = offset;
chunkOptions.Length = length;
auto chunk = Download(chunkOptions);
int64_t bytesRead = Azure::Core::Http::BodyStream::ReadToCount(
chunkOptions.Context,
*chunk.BodyStream,
buffer + (offset - firstChunkOffset),
chunkOptions.Length.GetValue());
if (bytesRead != chunkOptions.Length.GetValue())
{
throw std::runtime_error("error when reading body stream");
}
if (chunkId == numChunks - 1)
{
ret = returnTypeConverter(chunk);
}
};
int64_t remainingOffset = firstChunkOffset + firstChunkLength;
int64_t remainingSize = blobRangeSize - firstChunkLength;
int64_t chunkSize;
if (options.ChunkSize.HasValue())
{
chunkSize = options.ChunkSize.GetValue();
}
else
{
int64_t c_grainSize = 4 * 1024;
chunkSize = remainingSize / options.Concurrency;
chunkSize = (std::max(chunkSize, int64_t(1)) + c_grainSize - 1) / c_grainSize * c_grainSize;
chunkSize = std::min(chunkSize, c_defaultChunkSize);
}
Details::ConcurrentTransfer(
remainingOffset,
remainingSize,
chunkSize,
options.Concurrency,
downloadChunkFunc);
ret.ContentLength = blobRangeSize;
return ret;
}
BlobProperties BlobClient::GetProperties(const GetBlobPropertiesOptions& options) const
{
BlobRestClient::Blob::GetPropertiesOptions protocolLayerOptions;
@ -139,15 +281,12 @@ namespace Azure { namespace Storage { namespace Blobs {
options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions);
}
BlobInfo BlobClient::SetHttpHeaders(const SetBlobHttpHeadersOptions& options) const
SetBlobHttpHeadersResponse BlobClient::SetHttpHeaders(
BlobHttpHeaders httpHeaders,
const SetBlobHttpHeadersOptions& options) const
{
BlobRestClient::Blob::SetHttpHeadersOptions protocolLayerOptions;
protocolLayerOptions.ContentType = options.ContentType;
protocolLayerOptions.ContentEncoding = options.ContentEncoding;
protocolLayerOptions.ContentLanguage = options.ContentLanguage;
protocolLayerOptions.ContentMD5 = options.ContentMD5;
protocolLayerOptions.CacheControl = options.CacheControl;
protocolLayerOptions.ContentDisposition = options.ContentDisposition;
protocolLayerOptions.HttpHeaders = std::move(httpHeaders);
protocolLayerOptions.IfModifiedSince = options.IfModifiedSince;
protocolLayerOptions.IfUnmodifiedSince = options.IfUnmodifiedSince;
protocolLayerOptions.IfMatch = options.IfMatch;
@ -156,7 +295,7 @@ namespace Azure { namespace Storage { namespace Blobs {
options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions);
}
BlobInfo BlobClient::SetMetadata(
SetBlobMetadataResponse BlobClient::SetMetadata(
std::map<std::string, std::string> metadata,
const SetBlobMetadataOptions& options) const
{
@ -170,8 +309,9 @@ namespace Azure { namespace Storage { namespace Blobs {
options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions);
}
BasicResponse BlobClient::SetAccessTier(AccessTier Tier, const SetAccessTierOptions& options)
const
SetAccessTierResponse BlobClient::SetAccessTier(
AccessTier Tier,
const SetAccessTierOptions& options) const
{
BlobRestClient::Blob::SetAccessTierOptions protocolLayerOptions;
protocolLayerOptions.Tier = Tier;
@ -203,7 +343,7 @@ namespace Azure { namespace Storage { namespace Blobs {
options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions);
}
BasicResponse BlobClient::AbortCopyFromUri(
AbortCopyBlobResponse BlobClient::AbortCopyFromUri(
const std::string& copyId,
const AbortCopyFromUriOptions& options) const
{
@ -227,7 +367,7 @@ namespace Azure { namespace Storage { namespace Blobs {
options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions);
}
BasicResponse BlobClient::Delete(const DeleteBlobOptions& options) const
DeleteBlobResponse BlobClient::Delete(const DeleteBlobOptions& options) const
{
BlobRestClient::Blob::DeleteOptions protocolLayerOptions;
protocolLayerOptions.DeleteSnapshots = options.DeleteSnapshots;
@ -239,7 +379,7 @@ namespace Azure { namespace Storage { namespace Blobs {
options.Context, *m_pipeline, m_blobUrl.ToString(), protocolLayerOptions);
}
BasicResponse BlobClient::Undelete(const UndeleteBlobOptions& options) const
UndeleteBlobResponse BlobClient::Undelete(const UndeleteBlobOptions& options) const
{
BlobRestClient::Blob::UndeleteOptions protocolLayerOptions;
return BlobRestClient::Blob::Undelete(

33
sdk/storage/src/blobs/blob_container_client.cpp
View File

@ -40,7 +40,12 @@ namespace Azure { namespace Storage { namespace Blobs {
: BlobContainerClient(containerUri, options)
{
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> policies;
for (const auto& p : options.policies)
for (const auto& p : options.PerOperationPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
// TODO: Retry policy goes here
for (const auto& p : options.PerRetryPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
@ -58,7 +63,12 @@ namespace Azure { namespace Storage { namespace Blobs {
: BlobContainerClient(containerUri, options)
{
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> policies;
for (const auto& p : options.policies)
for (const auto& p : options.PerOperationPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
// TODO: Retry policy goes here
for (const auto& p : options.PerRetryPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
@ -76,7 +86,12 @@ namespace Azure { namespace Storage { namespace Blobs {
: m_containerUrl(containerUri)
{
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> policies;
for (const auto& p : options.policies)
for (const auto& p : options.PerOperationPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
// TODO: Retry policy goes here
for (const auto& p : options.PerRetryPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
@ -90,10 +105,7 @@ namespace Azure { namespace Storage { namespace Blobs {
{
auto blobUri = m_containerUrl;
blobUri.AppendPath(blobName);
BlobClient blobClient;
blobClient.m_blobUrl = std::move(blobUri);
blobClient.m_pipeline = m_pipeline;
return blobClient;
return BlobClient(std::move(blobUri), m_pipeline);
}
BlockBlobClient BlobContainerClient::GetBlockBlobClient(const std::string& blobName) const
@ -120,7 +132,8 @@ namespace Azure { namespace Storage { namespace Blobs {
options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions);
}
BasicResponse BlobContainerClient::Delete(const DeleteBlobContainerOptions& options) const
DeleteContainerResponse BlobContainerClient::Delete(
const DeleteBlobContainerOptions& options) const
{
BlobRestClient::Container::DeleteOptions protocolLayerOptions;
protocolLayerOptions.IfModifiedSince = options.IfModifiedSince;
@ -138,7 +151,7 @@ namespace Azure { namespace Storage { namespace Blobs {
options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions);
}
BlobContainerInfo BlobContainerClient::SetMetadata(
SetContainerMetadataResponse BlobContainerClient::SetMetadata(
std::map<std::string, std::string> metadata,
SetBlobContainerMetadataOptions options) const
{
@ -149,7 +162,7 @@ namespace Azure { namespace Storage { namespace Blobs {
options.Context, *m_pipeline, m_containerUrl.ToString(), protocolLayerOptions);
}
BlobsFlatSegment BlobContainerClient::ListBlobs(const ListBlobsOptions& options) const
BlobsFlatSegment BlobContainerClient::ListBlobsFlat(const ListBlobsOptions& options) const
{
BlobRestClient::Container::ListBlobsOptions protocolLayerOptions;
protocolLayerOptions.Prefix = options.Prefix;

26
sdk/storage/src/blobs/blob_service_client.cpp
View File

@ -35,7 +35,12 @@ namespace Azure { namespace Storage { namespace Blobs {
: m_serviceUrl(serviceUri)
{
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> policies;
for (const auto& p : options.policies)
for (const auto& p : options.PerOperationPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
// TODO: Retry policy goes here
for (const auto& p : options.PerRetryPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
@ -53,7 +58,12 @@ namespace Azure { namespace Storage { namespace Blobs {
: m_serviceUrl(serviceUri)
{
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> policies;
for (const auto& p : options.policies)
for (const auto& p : options.PerOperationPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
// TODO: Retry policy goes here
for (const auto& p : options.PerRetryPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
@ -71,7 +81,12 @@ namespace Azure { namespace Storage { namespace Blobs {
: m_serviceUrl(serviceUri)
{
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> policies;
for (const auto& p : options.policies)
for (const auto& p : options.PerOperationPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
// TODO: Retry policy goes here
for (const auto& p : options.PerRetryPolicies)
{
policies.emplace_back(std::unique_ptr<Azure::Core::Http::HttpPolicy>(p->Clone()));
}
@ -86,10 +101,7 @@ namespace Azure { namespace Storage { namespace Blobs {
{
auto containerUri = m_serviceUrl;
containerUri.AppendPath(containerName);
BlobContainerClient containerClient;
containerClient.m_containerUrl = std::move(containerUri);
containerClient.m_pipeline = m_pipeline;
return containerClient;
return BlobContainerClient(std::move(containerUri), m_pipeline);
}
ListContainersSegment BlobServiceClient::ListBlobContainersSegment(

64
sdk/storage/src/blobs/block_blob_client.cpp
View File

@ -3,6 +3,8 @@
#include "blobs/block_blob_client.hpp"
#include "common/concurrent_transfer.hpp"
#include "common/crypt.hpp"
#include "common/storage_common.hpp"
namespace Azure { namespace Storage { namespace Blobs {
@ -64,7 +66,7 @@ namespace Azure { namespace Storage { namespace Blobs {
BlobRestClient::BlockBlob::UploadOptions protocolLayerOptions;
protocolLayerOptions.ContentMD5 = options.ContentMD5;
protocolLayerOptions.ContentCRC64 = options.ContentCRC64;
protocolLayerOptions.Properties = options.Properties;
protocolLayerOptions.HttpHeaders = options.HttpHeaders;
protocolLayerOptions.Metadata = options.Metadata;
protocolLayerOptions.Tier = options.Tier;
protocolLayerOptions.IfModifiedSince = options.IfModifiedSince;
@ -75,6 +77,64 @@ namespace Azure { namespace Storage { namespace Blobs {
options.Context, *m_pipeline, m_blobUrl.ToString(), content, protocolLayerOptions);
}
BlobContentInfo BlockBlobClient::UploadFromBuffer(
const uint8_t* buffer,
std::size_t bufferSize,
const UploadBlobOptions& options) const
{
constexpr int64_t c_defaultBlockSize = 8 * 1024 * 1024;
constexpr int64_t c_maximumNumberBlocks = 50000;
constexpr int64_t c_grainSize = 4 * 1024;
int64_t chunkSize = c_defaultBlockSize;
if (options.ChunkSize.HasValue())
{
chunkSize = options.ChunkSize.GetValue();
}
else
{
int64_t minBlockSize = (bufferSize + c_maximumNumberBlocks - 1) / c_maximumNumberBlocks;
chunkSize = std::max(chunkSize, minBlockSize);
chunkSize = (chunkSize + c_grainSize - 1) / c_grainSize * c_grainSize;
}
std::vector<std::pair<BlockType, std::string>> blockIds;
auto getBlockId = [](int64_t id) {
constexpr std::size_t c_blockIdLength = 64;
std::string blockId = std::to_string(id);
blockId = std::string(c_blockIdLength - blockId.length(), '0') + blockId;
return Base64Encode(blockId);
};
auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
Azure::Core::Http::MemoryBodyStream contentStream(buffer + offset, length);
StageBlockOptions chunkOptions;
chunkOptions.Context = options.Context;
auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions);
if (chunkId == numChunks - 1)
{
blockIds.resize(static_cast<std::size_t>(numChunks));
}
};
Details::ConcurrentTransfer(0, bufferSize, chunkSize, options.Concurrency, uploadBlockFunc);
for (std::size_t i = 0; i < blockIds.size(); ++i)
{
blockIds[i].first = BlockType::Uncommitted;
blockIds[i].second = getBlockId(static_cast<int64_t>(i));
}
CommitBlockListOptions commitBlockListOptions;
commitBlockListOptions.Context = options.Context;
commitBlockListOptions.HttpHeaders = options.HttpHeaders;
commitBlockListOptions.Metadata = options.Metadata;
commitBlockListOptions.Tier = options.Tier;
auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions);
commitBlockListResponse.ContentCRC64.Reset();
commitBlockListResponse.ContentMD5.Reset();
return commitBlockListResponse;
}
BlockInfo BlockBlobClient::StageBlock(
const std::string& blockId,
Azure::Core::Http::BodyStream& content,
@ -126,7 +186,7 @@ namespace Azure { namespace Storage { namespace Blobs {
{
BlobRestClient::BlockBlob::CommitBlockListOptions protocolLayerOptions;
protocolLayerOptions.BlockList = blockIds;
protocolLayerOptions.Properties = options.Properties;
protocolLayerOptions.HttpHeaders = options.HttpHeaders;
protocolLayerOptions.Metadata = options.Metadata;
protocolLayerOptions.Tier = options.Tier;
protocolLayerOptions.IfModifiedSince = options.IfModifiedSince;

2
sdk/storage/src/blobs/page_blob_client.cpp
View File

@ -62,7 +62,7 @@ namespace Azure { namespace Storage { namespace Blobs {
BlobRestClient::PageBlob::CreateOptions protocolLayerOptions;
protocolLayerOptions.BlobContentLength = blobContentLength;
protocolLayerOptions.SequenceNumber = options.SequenceNumber;
protocolLayerOptions.Properties = options.Properties;
protocolLayerOptions.HttpHeaders = options.HttpHeaders;
protocolLayerOptions.Metadata = options.Metadata;
protocolLayerOptions.Tier = options.Tier;
protocolLayerOptions.IfModifiedSince = options.IfModifiedSince;

15
sdk/storage/test/append_blob_client_test.cpp
View File

@ -22,17 +22,18 @@ namespace Azure { namespace Storage { namespace Test {
m_blobContent.resize(100);
RandomBuffer(reinterpret_cast<char*>(&m_blobContent[0]), m_blobContent.size());
m_blobUploadOptions.Metadata = {{"key1", "V1"}, {"KEY2", "Value2"}};
m_blobUploadOptions.Properties.ContentType = "application/x-binary";
m_blobUploadOptions.Properties.ContentLanguage = "en-US";
m_blobUploadOptions.Properties.ContentDisposition = "attachment";
m_blobUploadOptions.Properties.CacheControl = "no-cache";
m_blobUploadOptions.Properties.ContentEncoding = "identify";
m_blobUploadOptions.Properties.ContentMD5 = "";
m_blobUploadOptions.HttpHeaders.ContentType = "application/x-binary";
m_blobUploadOptions.HttpHeaders.ContentLanguage = "en-US";
m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment";
m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache";
m_blobUploadOptions.HttpHeaders.ContentEncoding = "identify";
m_blobUploadOptions.HttpHeaders.ContentMD5 = "";
m_appendBlobClient->Create(m_blobUploadOptions);
auto blockContent
= Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size());
m_appendBlobClient->AppendBlock(blockContent);
m_blobUploadOptions.Properties.ContentMD5 = m_appendBlobClient->GetProperties().ContentMD5;
m_blobUploadOptions.HttpHeaders.ContentMD5
= m_appendBlobClient->GetProperties().HttpHeaders.ContentMD5;
}
void AppendBlobClientTest::TearDownTestSuite() { BlobContainerClientTest::TearDownTestSuite(); }

17
sdk/storage/test/blob_container_client_test.cpp
View File

@ -105,17 +105,16 @@ namespace Azure { namespace Storage { namespace Test {
std::set<std::string> listBlobs;
do
{
auto res = m_blobContainerClient->ListBlobs(options);
auto res = m_blobContainerClient->ListBlobsFlat(options);
EXPECT_FALSE(res.RequestId.empty());
EXPECT_FALSE(res.Date.empty());
;
EXPECT_FALSE(res.Version.empty());
EXPECT_FALSE(res.ServiceEndpoint.empty());
EXPECT_EQ(res.MaxResults.GetValue(), options.MaxResults.GetValue());
EXPECT_EQ(res.Container, m_containerName);
options.Marker = res.NextMarker;
for (const auto& blob : res.BlobItems)
for (const auto& blob : res.Items)
{
EXPECT_FALSE(blob.Name.empty());
EXPECT_FALSE(blob.CreationTime.empty());
@ -133,9 +132,9 @@ namespace Azure { namespace Storage { namespace Test {
listBlobs.clear();
do
{
auto res = m_blobContainerClient->ListBlobs(options);
auto res = m_blobContainerClient->ListBlobsFlat(options);
options.Marker = res.NextMarker;
for (const auto& blob : res.BlobItems)
for (const auto& blob : res.Items)
{
listBlobs.insert(blob.Name);
}
@ -164,10 +163,10 @@ namespace Azure { namespace Storage { namespace Test {
std::set<std::string> listBlobs;
while (true)
{
auto res = m_blobContainerClient->ListBlobs(options);
auto res = m_blobContainerClient->ListBlobsFlat(options);
EXPECT_EQ(res.Delimiter, options.Delimiter.GetValue());
EXPECT_EQ(res.Prefix, options.Prefix.GetValue());
for (const auto& blob : res.BlobItems)
for (const auto& blob : res.Items)
{
listBlobs.insert(blob.Name);
}
@ -175,9 +174,9 @@ namespace Azure { namespace Storage { namespace Test {
{
options.Marker = res.NextMarker;
}
else if (!res.BlobItems.empty())
else if (!res.Items.empty())
{
options.Prefix = res.BlobItems[0].Name + delimiter;
options.Prefix = res.Items[0].Name + delimiter;
if (options.Marker.HasValue())
{
options.Marker.Reset();

6
sdk/storage/test/blob_service_client_test.cpp
View File

@ -53,10 +53,9 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_FALSE(res.Date.empty());
EXPECT_FALSE(res.Version.empty());
EXPECT_FALSE(res.ServiceEndpoint.empty());
EXPECT_EQ(res.MaxResults.GetValue(), options.MaxResults.GetValue());
options.Marker = res.NextMarker;
for (const auto& container : res.BlobContainerItems)
for (const auto& container : res.Items)
{
listContainers.insert(container.Name);
}
@ -76,10 +75,9 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_FALSE(res.Date.empty());
EXPECT_FALSE(res.Version.empty());
EXPECT_FALSE(res.ServiceEndpoint.empty());
EXPECT_EQ(res.MaxResults.GetValue(), options.MaxResults.GetValue());
options.Marker = res.NextMarker;
for (const auto& container : res.BlobContainerItems)
for (const auto& container : res.Items)
{
EXPECT_FALSE(container.Name.empty());
EXPECT_FALSE(container.ETag.empty());

295
sdk/storage/test/block_blob_client_test.cpp
View File

@ -5,6 +5,10 @@
#include "common/crypt.hpp"
#include <future>
#include <random>
#include <vector>
namespace Azure { namespace Storage { namespace Blobs {
bool operator==(
@ -37,17 +41,18 @@ namespace Azure { namespace Storage { namespace Test {
m_blobContent.resize(static_cast<std::size_t>(8_MB));
RandomBuffer(reinterpret_cast<char*>(&m_blobContent[0]), m_blobContent.size());
m_blobUploadOptions.Metadata = {{"key1", "V1"}, {"KEY2", "Value2"}};
m_blobUploadOptions.Properties.ContentType = "application/x-binary";
m_blobUploadOptions.Properties.ContentLanguage = "en-US";
m_blobUploadOptions.Properties.ContentDisposition = "attachment";
m_blobUploadOptions.Properties.CacheControl = "no-cache";
m_blobUploadOptions.Properties.ContentEncoding = "identity";
m_blobUploadOptions.Properties.ContentMD5 = "";
m_blobUploadOptions.HttpHeaders.ContentType = "application/x-binary";
m_blobUploadOptions.HttpHeaders.ContentLanguage = "en-US";
m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment";
m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache";
m_blobUploadOptions.HttpHeaders.ContentEncoding = "identity";
m_blobUploadOptions.HttpHeaders.ContentMD5 = "";
m_blobUploadOptions.Tier = Azure::Storage::Blobs::AccessTier::Hot;
auto blobContent
= Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size());
m_blockBlobClient->Upload(blobContent, m_blobUploadOptions);
m_blobUploadOptions.Properties.ContentMD5 = m_blockBlobClient->GetProperties().ContentMD5;
m_blobUploadOptions.HttpHeaders.ContentMD5
= m_blockBlobClient->GetProperties().HttpHeaders.ContentMD5;
}
void BlockBlobClientTest::TearDownTestSuite() { BlobContainerClientTest::TearDownTestSuite(); }
@ -76,7 +81,7 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_FALSE(res.Version.empty());
EXPECT_FALSE(res.ETag.empty());
EXPECT_FALSE(res.LastModified.empty());
EXPECT_EQ(res.Properties, m_blobUploadOptions.Properties);
EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders);
EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata);
EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob);
Azure::Storage::Blobs::DownloadBlobOptions options;
@ -92,6 +97,35 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_FALSE(res.ContentRange.GetValue().empty());
}
TEST_F(BlockBlobClientTest, DownloadEmpty)
{
std::vector<uint8_t> emptyContent;
auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString(
StandardStorageConnectionString(), m_containerName, RandomString());
auto blobContent
= Azure::Core::Http::MemoryBodyStream(emptyContent.data(), emptyContent.size());
blockBlobClient.Upload(blobContent);
blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders);
blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata);
auto res = blockBlobClient.Download();
EXPECT_EQ(res.BodyStream->Length(), 0);
EXPECT_FALSE(res.RequestId.empty());
EXPECT_FALSE(res.Date.empty());
EXPECT_FALSE(res.Version.empty());
EXPECT_FALSE(res.ETag.empty());
EXPECT_FALSE(res.LastModified.empty());
EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders);
EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata);
EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob);
Azure::Storage::Blobs::DownloadBlobOptions options;
options.Offset = 0;
EXPECT_THROW(blockBlobClient.Download(options), std::runtime_error);
options.Length = 1;
EXPECT_THROW(blockBlobClient.Download(options), std::runtime_error);
}
TEST_F(BlockBlobClientTest, CopyFromUri)
{
auto blobClient = m_blobContainerClient->GetBlobClient(RandomString());
@ -136,7 +170,9 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_THROW(snapshotClient.SetMetadata({}), std::runtime_error);
EXPECT_THROW(
snapshotClient.SetAccessTier(Azure::Storage::Blobs::AccessTier::Cool), std::runtime_error);
EXPECT_THROW(snapshotClient.SetHttpHeaders(), std::runtime_error);
EXPECT_THROW(
snapshotClient.SetHttpHeaders(Azure::Storage::Blobs::BlobHttpHeaders()),
std::runtime_error);
Azure::Storage::Blobs::CreateSnapshotOptions options;
options.Metadata = {{"snapshotkey1", "snapshotvalue1"}, {"snapshotKEY2", "SNAPSHOTVALUE2"}};
@ -155,14 +191,7 @@ namespace Azure { namespace Storage { namespace Test {
blockBlobClient.Upload(blobContent);
blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata);
blockBlobClient.SetAccessTier(Azure::Storage::Blobs::AccessTier::Cool);
Azure::Storage::Blobs::SetBlobHttpHeadersOptions options;
options.ContentType = m_blobUploadOptions.Properties.ContentType;
options.ContentEncoding = m_blobUploadOptions.Properties.ContentEncoding;
options.ContentLanguage = m_blobUploadOptions.Properties.ContentLanguage;
options.ContentMD5 = m_blobUploadOptions.Properties.ContentMD5;
options.CacheControl = m_blobUploadOptions.Properties.CacheControl;
options.ContentDisposition = m_blobUploadOptions.Properties.ContentDisposition;
blockBlobClient.SetHttpHeaders(options);
blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders);
auto res = blockBlobClient.GetProperties();
EXPECT_FALSE(res.RequestId.empty());
@ -173,12 +202,7 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_FALSE(res.CreationTime.empty());
EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata);
EXPECT_EQ(res.ContentLength, static_cast<int64_t>(m_blobContent.size()));
EXPECT_EQ(res.ContentType, options.ContentType);
EXPECT_EQ(res.ContentEncoding, options.ContentEncoding);
EXPECT_EQ(res.ContentLanguage, options.ContentLanguage);
EXPECT_EQ(res.ContentMD5, options.ContentMD5);
EXPECT_EQ(res.CacheControl, options.CacheControl);
EXPECT_EQ(res.ContentDisposition, options.ContentDisposition);
EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders);
EXPECT_EQ(res.Tier.GetValue(), Azure::Storage::Blobs::AccessTier::Cool);
EXPECT_FALSE(res.AccessTierChangeTime.GetValue().empty());
}
@ -196,7 +220,7 @@ namespace Azure { namespace Storage { namespace Test {
= Azure::Core::Http::MemoryBodyStream(block1Content.data(), block1Content.size());
blockBlobClient.StageBlock(blockId1, blockContent);
Azure::Storage::Blobs::CommitBlockListOptions options;
options.Properties = m_blobUploadOptions.Properties;
options.HttpHeaders = m_blobUploadOptions.HttpHeaders;
options.Metadata = m_blobUploadOptions.Metadata;
blockBlobClient.CommitBlockList(
{{Azure::Storage::Blobs::BlockType::Uncommitted, blockId1}}, options);
@ -230,4 +254,227 @@ namespace Azure { namespace Storage { namespace Test {
*/
}
TEST_F(BlockBlobClientTest, ConcurrentDownload)
{
std::vector<uint8_t> downloadBuffer = m_blobContent;
for (int c : {1, 2, 4})
{
Azure::Storage::Blobs::DownloadBlobToBufferOptions options;
options.Concurrency = c;
// download whole blob
downloadBuffer.assign(downloadBuffer.size(), '\x00');
auto res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size());
EXPECT_EQ(downloadBuffer, m_blobContent);
EXPECT_EQ(static_cast<std::size_t>(res.ContentLength), downloadBuffer.size());
// download whole blob
downloadBuffer.assign(downloadBuffer.size(), '\x00');
options.Offset = 0;
res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size());
EXPECT_EQ(downloadBuffer, m_blobContent);
EXPECT_EQ(static_cast<std::size_t>(res.ContentLength), downloadBuffer.size());
// download whole blob
downloadBuffer.assign(downloadBuffer.size(), '\x00');
options.Offset = 0;
options.Length = downloadBuffer.size();
res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size());
EXPECT_EQ(downloadBuffer, m_blobContent);
EXPECT_EQ(static_cast<std::size_t>(res.ContentLength), downloadBuffer.size());
// download whole blob
downloadBuffer.assign(downloadBuffer.size(), '\x00');
options.Offset = 0;
options.Length = downloadBuffer.size() * 2;
res = m_blockBlobClient->DownloadToBuffer(downloadBuffer.data(), downloadBuffer.size() * 2);
EXPECT_EQ(downloadBuffer, m_blobContent);
EXPECT_EQ(static_cast<std::size_t>(res.ContentLength), downloadBuffer.size());
options.InitialChunkSize = 4_KB;
options.ChunkSize = 4_KB;
auto downloadRange = [&](int64_t offset, int64_t length) {
int64_t actualLength
= std::min(length, static_cast<int64_t>(m_blobContent.size()) - offset);
auto optionsCopy = options;
optionsCopy.Offset = offset;
optionsCopy.Length = length;
if (actualLength > 0)
{
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(actualLength), '\x00');
auto res = m_blockBlobClient->DownloadToBuffer(
downloadContent.data(), static_cast<std::size_t>(actualLength), optionsCopy);
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
m_blobContent.begin() + static_cast<std::size_t>(offset),
m_blobContent.begin() + static_cast<std::size_t>(offset)
+ downloadContent.size()));
EXPECT_EQ(res.ContentLength, actualLength);
}
else
{
EXPECT_THROW(
m_blockBlobClient->DownloadToBuffer(nullptr, 8 * 1024 * 1024, optionsCopy),
std::runtime_error);
}
};
// random range
std::vector<std::future<void>> downloadRangeTasks;
std::mt19937_64 random_generator(std::random_device{}());
for (int i = 0; i < 16; ++i)
{
std::uniform_int_distribution<int64_t> offsetDistribution(0, m_blobContent.size() - 1);
int64_t offset = offsetDistribution(random_generator);
std::uniform_int_distribution<int64_t> lengthDistribution(1, 64_KB);
int64_t length = lengthDistribution(random_generator);
downloadRangeTasks.emplace_back(
std::async(std::launch::async, downloadRange, offset, length));
}
downloadRangeTasks.emplace_back(std::async(std::launch::async, downloadRange, 0, 1));
downloadRangeTasks.emplace_back(std::async(std::launch::async, downloadRange, 1, 1));
downloadRangeTasks.emplace_back(
std::async(std::launch::async, downloadRange, m_blobContent.size() - 1, 1));
downloadRangeTasks.emplace_back(
std::async(std::launch::async, downloadRange, m_blobContent.size() - 1, 2));
downloadRangeTasks.emplace_back(
std::async(std::launch::async, downloadRange, m_blobContent.size(), 1));
downloadRangeTasks.emplace_back(
std::async(std::launch::async, downloadRange, m_blobContent.size() + 1, 2));
for (auto& task : downloadRangeTasks)
{
task.get();
}
// buffer not big enough
options.Offset = 1;
for (int64_t length : {1ULL, 2ULL, 4_KB, 5_KB, 8_KB, 11_KB, 20_KB})
{
options.Length = length;
EXPECT_THROW(
m_blockBlobClient->DownloadToBuffer(
downloadBuffer.data(), static_cast<std::size_t>(length - 1), options),
std::runtime_error);
}
}
}
TEST_F(BlockBlobClientTest, ConcurrentDownloadEmptyBlob)
{
std::vector<uint8_t> emptyContent;
auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString(
StandardStorageConnectionString(), m_containerName, RandomString());
auto blobContent
= Azure::Core::Http::MemoryBodyStream(emptyContent.data(), emptyContent.size());
blockBlobClient.Upload(blobContent);
blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders);
blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata);
auto res = blockBlobClient.DownloadToBuffer(emptyContent.data(), 0);
EXPECT_EQ(res.ContentLength, 0);
EXPECT_FALSE(res.ETag.empty());
EXPECT_FALSE(res.LastModified.empty());
EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders);
EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata);
EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob);
res = blockBlobClient.DownloadToBuffer(emptyContent.data(), static_cast<std::size_t>(8_MB));
EXPECT_EQ(res.ContentLength, 0);
EXPECT_FALSE(res.ETag.empty());
EXPECT_FALSE(res.LastModified.empty());
EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders);
EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata);
EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob);
for (int c : {1, 2})
{
Azure::Storage::Blobs::DownloadBlobToBufferOptions options;
options.InitialChunkSize = 10;
options.ChunkSize = 10;
options.Concurrency = c;
res = blockBlobClient.DownloadToBuffer(
emptyContent.data(), static_cast<std::size_t>(8_MB), options);
EXPECT_EQ(res.ContentLength, 0);
EXPECT_FALSE(res.ETag.empty());
EXPECT_FALSE(res.LastModified.empty());
EXPECT_EQ(res.HttpHeaders, m_blobUploadOptions.HttpHeaders);
EXPECT_EQ(res.Metadata, m_blobUploadOptions.Metadata);
EXPECT_EQ(res.BlobType, Azure::Storage::Blobs::BlobType::BlockBlob);
options.Offset = 0;
EXPECT_THROW(
blockBlobClient.DownloadToBuffer(
emptyContent.data(), static_cast<std::size_t>(8_MB), options),
std::runtime_error);
options.Offset = 1;
EXPECT_THROW(
blockBlobClient.DownloadToBuffer(
emptyContent.data(), static_cast<std::size_t>(8_MB), options),
std::runtime_error);
options.Offset = 0;
options.Length = 1;
EXPECT_THROW(
blockBlobClient.DownloadToBuffer(
emptyContent.data(), static_cast<std::size_t>(8_MB), options),
std::runtime_error);
options.Offset = 100;
options.Length = 100;
EXPECT_THROW(
blockBlobClient.DownloadToBuffer(
emptyContent.data(), static_cast<std::size_t>(8_MB), options),
std::runtime_error);
}
}
TEST_F(BlockBlobClientTest, ConcurrentUpload)
{
auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString(
StandardStorageConnectionString(), m_containerName, RandomString());
for (int c : {1, 2, 5})
{
for (int64_t length :
{0ULL, 1ULL, 2ULL, 2_KB, 4_KB, 999_KB, 1_MB, 2_MB - 1, 3_MB, 5_MB, 8_MB - 1234, 8_MB})
{
Azure::Storage::Blobs::UploadBlobOptions options;
options.ChunkSize = 1_MB;
options.Concurrency = c;
options.HttpHeaders = m_blobUploadOptions.HttpHeaders;
options.Metadata = m_blobUploadOptions.Metadata;
options.Tier = m_blobUploadOptions.Tier;
auto res = blockBlobClient.UploadFromBuffer(
m_blobContent.data(), static_cast<std::size_t>(length), options);
EXPECT_FALSE(res.RequestId.empty());
EXPECT_FALSE(res.Version.empty());
EXPECT_FALSE(res.Date.empty());
EXPECT_FALSE(res.ETag.empty());
EXPECT_FALSE(res.LastModified.empty());
EXPECT_FALSE(res.SequenceNumber.HasValue());
EXPECT_FALSE(res.ContentCRC64.HasValue());
EXPECT_FALSE(res.ContentMD5.HasValue());
auto properties = blockBlobClient.GetProperties();
EXPECT_EQ(properties.ContentLength, length);
EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders);
EXPECT_EQ(properties.Metadata, options.Metadata);
EXPECT_EQ(properties.Tier.GetValue(), options.Tier.GetValue());
EXPECT_EQ(properties.ETag, res.ETag);
EXPECT_EQ(properties.LastModified, res.LastModified);
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(length), '\x00');
blockBlobClient.DownloadToBuffer(downloadContent.data(), static_cast<std::size_t>(length));
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
m_blobContent.begin(), m_blobContent.begin() + static_cast<std::size_t>(length)));
}
}
}
}}} // namespace Azure::Storage::Test

15
sdk/storage/test/page_blob_client_test.cpp
View File

@ -22,17 +22,18 @@ namespace Azure { namespace Storage { namespace Test {
m_blobContent.resize(static_cast<std::size_t>(1_KB));
RandomBuffer(reinterpret_cast<char*>(&m_blobContent[0]), m_blobContent.size());
m_blobUploadOptions.Metadata = {{"key1", "V1"}, {"KEY2", "Value2"}};
m_blobUploadOptions.Properties.ContentType = "application/x-binary";
m_blobUploadOptions.Properties.ContentLanguage = "en-US";
m_blobUploadOptions.Properties.ContentDisposition = "attachment";
m_blobUploadOptions.Properties.CacheControl = "no-cache";
m_blobUploadOptions.Properties.ContentEncoding = "identity";
m_blobUploadOptions.Properties.ContentMD5 = "";
m_blobUploadOptions.HttpHeaders.ContentType = "application/x-binary";
m_blobUploadOptions.HttpHeaders.ContentLanguage = "en-US";
m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment";
m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache";
m_blobUploadOptions.HttpHeaders.ContentEncoding = "identity";
m_blobUploadOptions.HttpHeaders.ContentMD5 = "";
m_pageBlobClient->Create(m_blobContent.size(), m_blobUploadOptions);
auto pageContent
= Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size());
m_pageBlobClient->UploadPages(pageContent, 0);
m_blobUploadOptions.Properties.ContentMD5 = m_pageBlobClient->GetProperties().ContentMD5;
m_blobUploadOptions.HttpHeaders.ContentMD5
= m_pageBlobClient->GetProperties().HttpHeaders.ContentMD5;
}
void PageBlobClientTest::TearDownTestSuite() { BlobContainerClientTest::TearDownTestSuite(); }