github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add ContentHash struct, replace TransactionalContentMD5 and TransactionContentCrc64 with TransactionalContentHash (#1212)

Browse Source 
* use std::vector<uint8_t> for binary input/output

* struct ContentHash

* ContentHash in Blob Service

* Update sdk/storage/azure-storage-common/src/crypt.cpp

* Update sdk/storage/azure-storage-common/src/crypt.cpp

* add doc

* changelog

* EncryptionKeySha256 and ContentMd5 are changed to binary

* changelog

* fix build error

* fix crash issue

* FIX BUILD ERROR on linux

* Fix bug on Linux

* fix bug
This commit is contained in:
JinmingHu 2020-12-18 16:09:35 +08:00 committed by GitHub
parent 60e70a3695
commit b59b65bb87
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 546 additions and 413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sdk/storage/azure-storage-blobs/CHANGELOG.md
View File

@ -47,6 +47,8 @@
- API signature for CommitBlockList has changed. `BlockType` doesn't need to be specified anymore.
- `PageBlobClient::GetPageRanges` doesn't support getting difference between current blob and a snapshot anymore. Use `PageBlobClient::GetPageRangesDiff` instead.
- Move Blob SAS into `Azure::Storage::Sas` namespace.
- Replace all transactional content MD5/CRC64 with `ContentHash` struct.
- `ContentMd5` HTTP header and `EncrytionKeySha256` are changed to binary(`std::vector<uint8_t>`).
## 12.0.0-beta.5 (2020-11-13)

81
sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp
View File

@ -106,9 +106,9 @@ namespace Azure { namespace Storage { namespace Blobs {
std::string Key;
/**
* @brief Base64 encoded string of the AES256 encryption key's SHA256 hash.
* @brief SHA256 hash of the AES256 encryption key.
*/
std::string KeyHash;
std::vector<uint8_t> KeyHash;
/**
* @brief The algorithm for Azure Blob Storage to encrypt with.
@ -917,18 +917,11 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Context Context;
/**
* @brief An MD5 hash of the blob content. This hash is used to verify the integrity of
* @brief Hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentMd5;
/**
* @brief A CRC64 hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentCrc64;
Azure::Core::Nullable<ContentHash> TransactionalContentHash;
/**
* @brief The standard HTTP header system properties to set.
@ -998,18 +991,11 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Context Context;
/**
* @brief An MD5 hash of the blob content. This hash is used to verify the integrity of
* @brief Hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentMd5;
/**
* @brief A CRC64 hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentCrc64;
Azure::Core::Nullable<ContentHash> TransactionalContentHash;
/**
* @brief Optional conditions that must be met to perform this operation.
@ -1039,18 +1025,11 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Nullable<int64_t> SourceLength;
/**
* @brief An MD5 hash of the blob content. This hash is used to verify the integrity of
* @brief Hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentMd5;
/**
* @brief A CRC64 hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentCrc64;
Azure::Core::Nullable<ContentHash> TransactionalContentHash;
/**
* @brief Optional conditions that must be met to perform this operation.
@ -1157,18 +1136,11 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Context Context;
/**
* @brief An MD5 hash of the blob content. This hash is used to verify the integrity of
* @brief Hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentMd5;
/**
* @brief A CRC64 hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentCrc64;
Azure::Core::Nullable<ContentHash> TransactionalContentHash;
/**
* @brief Optional conditions that must be met to perform this operation.
@ -1198,18 +1170,11 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Nullable<int64_t> SourceLength;
/**
* @brief An MD5 hash of the blob content. This hash is used to verify the integrity of
* @brief Hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentMd5;
/**
* @brief A CRC64 hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentCrc64;
Azure::Core::Nullable<ContentHash> TransactionalContentHash;
/**
* @brief Optional conditions that must be met to perform this operation.
@ -1281,18 +1246,11 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Context Context;
/**
* @brief An MD5 hash of the blob content. This hash is used to verify the integrity of
* @brief Hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentMd5;
/**
* @brief A CRC64 hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentCrc64;
Azure::Core::Nullable<ContentHash> TransactionalContentHash;
/**
* @brief Optional conditions that must be met to perform this operation.
@ -1311,18 +1269,11 @@ namespace Azure { namespace Storage { namespace Blobs {
Azure::Core::Context Context;
/**
* @brief An MD5 hash of the blob content. This hash is used to verify the integrity of
* @brief Hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentMd5;
/**
* @brief A CRC64 hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
* that has arrived with the one that was sent.
*/
Azure::Core::Nullable<std::string> TransactionalContentCrc64;
Azure::Core::Nullable<ContentHash> TransactionalContentHash;
/**
* @brief Optional conditions that must be met to perform this operation.

2
sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp
View File

@ -23,7 +23,7 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Models {
Storage::Metadata Metadata;
Models::BlobType BlobType = Models::BlobType::Unknown;
Azure::Core::Nullable<bool> ServerEncrypted;
Azure::Core::Nullable<std::string> EncryptionKeySha256;
Azure::Core::Nullable<std::vector<uint8_t>> EncryptionKeySha256;
};
using UploadBlockBlobFromResult = UploadBlockBlobResult;

600
sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/protocol/blob_rest_client.hpp
View File

File diff suppressed because it is too large Load Diff

6
sdk/storage/azure-storage-blobs/src/append_blob_client.cpp
View File

@ -121,8 +121,7 @@ namespace Azure { namespace Storage { namespace Blobs {
const AppendBlockOptions& options) const
{
Details::BlobRestClient::AppendBlob::AppendBlockOptions protocolLayerOptions;
protocolLayerOptions.TransactionalContentMd5 = options.TransactionalContentMd5;
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentCrc64;
protocolLayerOptions.TransactionalContentHash = options.TransactionalContentHash;
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.MaxSize = options.AccessConditions.IfMaxSizeLessThanOrEqual;
protocolLayerOptions.AppendPosition = options.AccessConditions.IfAppendPositionEqual;
@ -161,8 +160,7 @@ namespace Azure { namespace Storage { namespace Blobs {
std::numeric_limits<
std::remove_reference_t<decltype(options.SourceOffset.GetValue())>>::max());
}
protocolLayerOptions.TransactionalContentMd5 = options.TransactionalContentMd5;
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentCrc64;
protocolLayerOptions.TransactionalContentHash = options.TransactionalContentHash;
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.MaxSize = options.AccessConditions.IfMaxSizeLessThanOrEqual;
protocolLayerOptions.AppendPosition = options.AccessConditions.IfAppendPositionEqual;

10
sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp
View File

@ -134,8 +134,9 @@ namespace Azure { namespace Storage { namespace Sas {
+ CacheControl + "\n" + ContentDisposition + "\n" + ContentEncoding + "\n" + ContentLanguage
+ "\n" + ContentType;
std::string signature = Base64Encode(
Storage::Details::HmacSha256(stringToSign, Base64Decode(credential.GetAccountKey())));
std::string signature = Base64Encode(Storage::Details::HmacSha256(
std::vector<uint8_t>(stringToSign.begin(), stringToSign.end()),
Base64Decode(credential.GetAccountKey())));
Azure::Core::Http::Url builder;
builder.AppendQueryParameter(
@ -225,8 +226,9 @@ namespace Azure { namespace Storage { namespace Sas {
+ CacheControl + "\n" + ContentDisposition + "\n" + ContentEncoding + "\n" + ContentLanguage
+ "\n" + ContentType;
std::string signature = Base64Encode(
Storage::Details::HmacSha256(stringToSign, Base64Decode(userDelegationKey.Value)));
std::string signature = Base64Encode(Storage::Details::HmacSha256(
std::vector<uint8_t>(stringToSign.begin(), stringToSign.end()),
Base64Decode(userDelegationKey.Value)));
Azure::Core::Http::Url builder;
builder.AppendQueryParameter(

9
sdk/storage/azure-storage-blobs/src/block_blob_client.cpp
View File

@ -81,8 +81,7 @@ namespace Azure { namespace Storage { namespace Blobs {
const UploadBlockBlobOptions& options) const
{
Details::BlobRestClient::BlockBlob::UploadBlockBlobOptions protocolLayerOptions;
protocolLayerOptions.TransactionalContentMd5 = options.TransactionalContentMd5;
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentCrc64;
protocolLayerOptions.TransactionalContentHash = options.TransactionalContentHash;
protocolLayerOptions.HttpHeaders = options.HttpHeaders;
protocolLayerOptions.Metadata = options.Metadata;
protocolLayerOptions.Tier = options.Tier;
@ -269,8 +268,7 @@ namespace Azure { namespace Storage { namespace Blobs {
{
Details::BlobRestClient::BlockBlob::StageBlockOptions protocolLayerOptions;
protocolLayerOptions.BlockId = blockId;
protocolLayerOptions.TransactionalContentMd5 = options.TransactionalContentMd5;
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentCrc64;
protocolLayerOptions.TransactionalContentHash = options.TransactionalContentHash;
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
if (m_customerProvidedKey.HasValue())
{
@ -304,8 +302,7 @@ namespace Azure { namespace Storage { namespace Blobs {
std::numeric_limits<
std::remove_reference_t<decltype(options.SourceOffset.GetValue())>>::max());
}
protocolLayerOptions.TransactionalContentMd5 = options.TransactionalContentMd5;
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentCrc64;
protocolLayerOptions.TransactionalContentHash = options.TransactionalContentHash;
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.SourceIfModifiedSince = options.SourceConditions.IfModifiedSince;
protocolLayerOptions.SourceIfUnmodifiedSince = options.SourceConditions.IfUnmodifiedSince;

6
sdk/storage/azure-storage-blobs/src/page_blob_client.cpp
View File

@ -130,8 +130,7 @@ namespace Azure { namespace Storage { namespace Blobs {
{
Details::BlobRestClient::PageBlob::UploadPageBlobPagesOptions protocolLayerOptions;
protocolLayerOptions.Range = std::make_pair(offset, offset + content->Length() - 1);
protocolLayerOptions.TransactionalContentMd5 = options.TransactionalContentMd5;
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentCrc64;
protocolLayerOptions.TransactionalContentHash = options.TransactionalContentHash;
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
@ -163,8 +162,7 @@ namespace Azure { namespace Storage { namespace Blobs {
= std::make_pair(sourceOffset, sourceOffset + sourceLength - 1);
protocolLayerOptions.Range
= std::make_pair(destinationOffset, destinationOffset + sourceLength - 1);
protocolLayerOptions.TransactionalContentMd5 = options.TransactionalContentMd5;
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentCrc64;
protocolLayerOptions.TransactionalContentHash = options.TransactionalContentHash;
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;

2
sdk/storage/azure-storage-blobs/test/append_blob_client_test.cpp
View File

@ -27,7 +27,7 @@ namespace Azure { namespace Storage { namespace Test {
m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment";
m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache";
m_blobUploadOptions.HttpHeaders.ContentEncoding = "identify";
m_blobUploadOptions.HttpHeaders.ContentMd5 = "";
m_blobUploadOptions.HttpHeaders.ContentMd5.clear();
m_appendBlobClient->Create(m_blobUploadOptions);
auto blockContent
= Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size());

4
sdk/storage/azure-storage-blobs/test/blob_container_client_test.cpp
View File

@ -519,11 +519,11 @@ namespace Azure { namespace Storage { namespace Test {
{
auto getRandomCustomerProvidedKey = []() {
Blobs::EncryptionKey key;
std::string aes256Key;
std::vector<uint8_t> aes256Key;
aes256Key.resize(32);
RandomBuffer(&aes256Key[0], aes256Key.size());
key.Key = Base64Encode(aes256Key);
key.KeyHash = Base64Encode(Details::Sha256(aes256Key));
key.KeyHash = Details::Sha256(aes256Key);
key.Algorithm = Blobs::Models::EncryptionAlgorithmType::Aes256;
return key;
};

2
sdk/storage/azure-storage-blobs/test/block_blob_client_test.cpp
View File

@ -45,7 +45,7 @@ namespace Azure { namespace Storage { namespace Test {
m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment";
m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache";
m_blobUploadOptions.HttpHeaders.ContentEncoding = "identity";
m_blobUploadOptions.HttpHeaders.ContentMd5 = "";
m_blobUploadOptions.HttpHeaders.ContentMd5.clear();
m_blobUploadOptions.Tier = Azure::Storage::Blobs::Models::AccessTier::Hot;
auto blobContent
= Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size());

20
sdk/storage/azure-storage-blobs/test/page_blob_client_test.cpp
View File

@ -33,7 +33,7 @@ namespace Azure { namespace Storage { namespace Test {
m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment";
m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache";
m_blobUploadOptions.HttpHeaders.ContentEncoding = "identity";
m_blobUploadOptions.HttpHeaders.ContentMd5 = "";
m_blobUploadOptions.HttpHeaders.ContentMd5.clear();
m_pageBlobClient->Create(m_blobContent.size(), m_blobUploadOptions);
auto pageContent
= Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size());
@ -219,12 +219,15 @@ namespace Azure { namespace Storage { namespace Test {
auto pageContent = Azure::Core::Http::MemoryBodyStream(blobContent.data(), blobContent.size());
Blobs::UploadPageBlobPagesOptions options;
options.TransactionalContentMd5
= Base64Encode(Md5::Hash(blobContent.data(), blobContent.size()));
ContentHash hash;
hash.Algorithm = HashAlgorithm::Md5;
hash.Value = Md5::Hash(blobContent.data(), blobContent.size());
options.TransactionalContentHash = hash;
EXPECT_NO_THROW(pageBlobClient.UploadPages(0, &pageContent, options));
pageContent.Rewind();
options.TransactionalContentMd5 = DummyMd5;
hash.Value = Base64Decode(DummyMd5);
options.TransactionalContentHash = hash;
EXPECT_THROW(pageBlobClient.UploadPages(0, &pageContent, options), StorageException);
}
@ -240,12 +243,15 @@ namespace Azure { namespace Storage { namespace Test {
auto pageContent = Azure::Core::Http::MemoryBodyStream(blobContent.data(), blobContent.size());
Blobs::UploadPageBlobPagesOptions options;
options.TransactionalContentCrc64
= Base64Encode(Crc64::Hash(blobContent.data(), blobContent.size()));
ContentHash hash;
hash.Algorithm = HashAlgorithm::Crc64;
hash.Value = Crc64::Hash(blobContent.data(), blobContent.size());
options.TransactionalContentHash = hash;
EXPECT_NO_THROW(pageBlobClient.UploadPages(0, &pageContent, options));
pageContent.Rewind();
options.TransactionalContentCrc64 = DummyCrc64;
hash.Value = Base64Decode(DummyCrc64);
options.TransactionalContentHash = hash;
EXPECT_THROW(pageBlobClient.UploadPages(0, &pageContent, options), StorageException);
}

1
sdk/storage/azure-storage-common/CHANGELOG.md
View File

@ -7,6 +7,7 @@
- Move `StorageRetryPolicy`, `StoragePerRetryPolicy` and `SharedKeyPolicy` to `Details` namespace.
- Remove `StorageRetryOptions`, use `Azure::Core::Http::RetryOptions` instead.
- Move Account SAS into `Azure::Storage::Sas` namespace.
- Add new type `ContentHash`.
## 12.0.0-beta.5 (2020-11-13)

5
sdk/storage/azure-storage-common/CMakeLists.txt
View File

@ -75,17 +75,18 @@ target_include_directories(
$<INSTALL_INTERFACE:include>
)
# libxml2 headers and libraries will be changed to PRIVATE in the future.
target_include_directories(azure-storage-common PUBLIC ${LIBXML2_INCLUDE_DIRS})
target_link_libraries(azure-storage-common PUBLIC Azure::azure-core)
target_link_libraries(azure-storage-common INTERFACE Threads::Threads ${LIBXML2_LIBRARIES})
if(MSVC)
target_link_libraries(azure-storage-common INTERFACE bcrypt)
target_link_libraries(azure-storage-common PRIVATE bcrypt)
# C28020 and C28204 are introduced by nlohmann/json
target_compile_options(azure-storage-common PUBLIC /wd28204 /wd28020)
else()
find_package(OpenSSL REQUIRED)
target_link_libraries(azure-storage-common INTERFACE OpenSSL::SSL OpenSSL::Crypto)
target_link_libraries(azure-storage-common PRIVATE OpenSSL::SSL OpenSSL::Crypto)
endif()
get_az_version("${CMAKE_CURRENT_SOURCE_DIR}/inc/azure/storage/common/version.hpp")

27
sdk/storage/azure-storage-common/inc/azure/storage/common/crypt.hpp
View File

@ -5,11 +5,16 @@
#include <cstdint>
#include <string>
#include <vector>
namespace Azure { namespace Storage {
std::string Base64Encode(const std::string& text);
std::string Base64Decode(const std::string& text);
std::string Base64Encode(const std::vector<uint8_t>& data);
inline std::string Base64Encode(const std::string& text)
{
return Base64Encode(std::vector<uint8_t>(text.begin(), text.end()));
}
std::vector<uint8_t> Base64Decode(const std::string& text);
class Md5 {
public:
@ -18,16 +23,16 @@ namespace Azure { namespace Storage {
void Update(const uint8_t* data, std::size_t length);
std::string Digest() const;
std::vector<uint8_t> Digest() const;
static std::string Hash(const uint8_t* data, std::size_t length)
static std::vector<uint8_t> Hash(const uint8_t* data, std::size_t length)
{
Md5 instance;
instance.Update(data, length);
return instance.Digest();
}
static std::string Hash(const std::string& data)
static std::vector<uint8_t> Hash(const std::string& data)
{
return Hash(reinterpret_cast<const uint8_t*>(data.data()), data.length());
}
@ -41,16 +46,16 @@ namespace Azure { namespace Storage {
void Update(const uint8_t* data, std::size_t length);
void Concatenate(const Crc64& other);
std::string Digest() const;
std::vector<uint8_t> Digest() const;
static std::string Hash(const uint8_t* data, std::size_t length)
static std::vector<uint8_t> Hash(const uint8_t* data, std::size_t length)
{
Crc64 instance;
instance.Update(data, length);
return instance.Digest();
}
static std::string Hash(const std::string& data)
static std::vector<uint8_t> Hash(const std::string& data)
{
return Hash(reinterpret_cast<const uint8_t*>(data.data()), data.length());
}
@ -61,8 +66,10 @@ namespace Azure { namespace Storage {
};
namespace Details {
std::string Sha256(const std::string& text);
std::string HmacSha256(const std::string& text, const std::string& key);
std::vector<uint8_t> Sha256(const std::vector<uint8_t>& data);
std::vector<uint8_t> HmacSha256(
const std::vector<uint8_t>& data,
const std::vector<uint8_t>& key);
std::string UrlEncodeQueryParameter(const std::string& value);
std::string UrlEncodePath(const std::string& value);
} // namespace Details

33
sdk/storage/azure-storage-common/inc/azure/storage/common/storage_common.hpp
View File

@ -7,6 +7,7 @@
#include <cstdint>
#include <map>
#include <string>
#include <vector>
#include "azure/core/strings.hpp"
@ -34,4 +35,36 @@ namespace Azure { namespace Storage {
} // namespace Details
using Metadata = std::map<std::string, std::string, Details::CaseInsensitiveComparator>;
/**
* @brief The algorithm used for hash.
*/
enum class HashAlgorithm
{
/**
* @brief MD5 message digest algorithm.
*/
Md5,
/**
* @brief Cyclic redundancy check.
*/
Crc64,
};
/**
* @brief Hash used to check content integrity.
*/
struct ContentHash
{
/**
* @brief Binary hash value.
*/
std::vector<uint8_t> Value;
/**
* @brief The algorithm used for hash.
*/
HashAlgorithm Algorithm;
};
}} // namespace Azure::Storage

5
sdk/storage/azure-storage-common/src/account_sas_builder.cpp
View File

@ -94,8 +94,9 @@ namespace Azure { namespace Storage { namespace Sas {
+ "\n" + (IPRange.HasValue() ? IPRange.GetValue() : "") + "\n" + protocol + "\n"
+ Storage::Details::DefaultSasVersion + "\n";
std::string signature = Base64Encode(
Storage::Details::HmacSha256(stringToSign, Base64Decode(credential.GetAccountKey())));
std::string signature = Base64Encode(Storage::Details::HmacSha256(
std::vector<uint8_t>(stringToSign.begin(), stringToSign.end()),
Base64Decode(credential.GetAccountKey())));
Azure::Core::Http::Url builder;
builder.AppendQueryParameter(

86
sdk/storage/azure-storage-common/src/crypt.cpp
View File

@ -1,10 +1,10 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// SPDX-License-Identifier: MIT
#include <azure/core/platform.hpp>
#include "azure/storage/common/crypt.hpp"
#include "azure/core/platform.hpp"
#if defined(AZ_PLATFORM_WINDOWS)
#if !defined(NOMINMAX)
#define NOMINMAX
@ -154,7 +154,7 @@ namespace Azure { namespace Storage {
~AlgorithmProviderInstance() { BCryptCloseAlgorithmProvider(Handle, 0); }
};
std::string Sha256(const std::string& text)
std::vector<uint8_t> Sha256(const std::vector<uint8_t>& data)
{
static AlgorithmProviderInstance AlgorithmProvider(AlgorithmType::Sha256);
@ -177,18 +177,18 @@ namespace Azure { namespace Storage {
status = BCryptHashData(
hashHandle,
reinterpret_cast<PBYTE>(const_cast<char*>(&text[0])),
static_cast<ULONG>(text.length()),
reinterpret_cast<PBYTE>(const_cast<uint8_t*>(data.data())),
static_cast<ULONG>(data.size()),
0);
if (!BCRYPT_SUCCESS(status))
{
throw std::runtime_error("BCryptHashData failed");
}
std::string hash;
std::vector<uint8_t> hash;
hash.resize(AlgorithmProvider.HashLength);
status = BCryptFinishHash(
hashHandle, reinterpret_cast<PUCHAR>(&hash[0]), static_cast<ULONG>(hash.length()), 0);
hashHandle, reinterpret_cast<PUCHAR>(&hash[0]), static_cast<ULONG>(hash.size()), 0);
if (!BCRYPT_SUCCESS(status))
{
throw std::runtime_error("BCryptFinishHash failed");
@ -199,7 +199,9 @@ namespace Azure { namespace Storage {
return hash;
}
std::string HmacSha256(const std::string& text, const std::string& key)
std::vector<uint8_t> HmacSha256(
const std::vector<uint8_t>& data,
const std::vector<uint8_t>& key)
{
static AlgorithmProviderInstance AlgorithmProvider(AlgorithmType::HmacSha256);
@ -213,8 +215,8 @@ namespace Azure { namespace Storage {
&hashHandle,
reinterpret_cast<PUCHAR>(&context[0]),
static_cast<ULONG>(context.size()),
reinterpret_cast<PUCHAR>(const_cast<char*>(&key[0])),
static_cast<ULONG>(key.length()),
reinterpret_cast<PUCHAR>(const_cast<uint8_t*>(&key[0])),
static_cast<ULONG>(key.size()),
0);
if (!BCRYPT_SUCCESS(status))
{
@ -223,18 +225,18 @@ namespace Azure { namespace Storage {
status = BCryptHashData(
hashHandle,
reinterpret_cast<PBYTE>(const_cast<char*>(&text[0])),
static_cast<ULONG>(text.length()),
reinterpret_cast<PBYTE>(const_cast<uint8_t*>(data.data())),
static_cast<ULONG>(data.size()),
0);
if (!BCRYPT_SUCCESS(status))
{
throw std::runtime_error("BCryptHashData failed");
}
std::string hash;
std::vector<uint8_t> hash;
hash.resize(AlgorithmProvider.HashLength);
status = BCryptFinishHash(
hashHandle, reinterpret_cast<PUCHAR>(&hash[0]), static_cast<ULONG>(hash.length()), 0);
hashHandle, reinterpret_cast<PUCHAR>(&hash[0]), static_cast<ULONG>(hash.size()), 0);
if (!BCRYPT_SUCCESS(status))
{
throw std::runtime_error("BCryptFinishHash failed");
@ -298,15 +300,15 @@ namespace Azure { namespace Storage {
}
}
std::string Md5::Digest() const
std::vector<uint8_t> Md5::Digest() const
{
Md5HashContext* context = static_cast<Md5HashContext*>(m_context);
std::string hash;
std::vector<uint8_t> hash;
hash.resize(context->hashLength);
NTSTATUS status = BCryptFinishHash(
context->hashHandle,
reinterpret_cast<PUCHAR>(&hash[0]),
static_cast<ULONG>(hash.length()),
static_cast<ULONG>(hash.size()),
0);
if (!BCRYPT_SUCCESS(status))
{
@ -315,16 +317,16 @@ namespace Azure { namespace Storage {
return hash;
}
std::string Base64Encode(const std::string& text)
std::string Base64Encode(const std::vector<uint8_t>& data)
{
std::string encoded;
// According to RFC 4648, the encoded length should be ceiling(n / 3) * 4
DWORD encodedLength = static_cast<DWORD>((text.length() + 2) / 3 * 4);
DWORD encodedLength = static_cast<DWORD>((data.size() + 2) / 3 * 4);
encoded.resize(encodedLength);
CryptBinaryToStringA(
reinterpret_cast<const BYTE*>(text.data()),
static_cast<DWORD>(text.length()),
reinterpret_cast<const BYTE*>(data.data()),
static_cast<DWORD>(data.size()),
CRYPT_STRING_BASE64 | CRYPT_STRING_NOCRLF,
static_cast<LPSTR>(&encoded[0]),
&encodedLength);
@ -332,9 +334,9 @@ namespace Azure { namespace Storage {
return encoded;
}
std::string Base64Decode(const std::string& text)
std::vector<uint8_t> Base64Decode(const std::string& text)
{
std::string decoded;
std::vector<uint8_t> decoded;
// According to RFC 4648, the encoded length should be ceiling(n / 3) * 4, so we can infer an
// upper bound here
DWORD decodedLength = DWORD(text.length() / 4 * 3);
@ -344,7 +346,7 @@ namespace Azure { namespace Storage {
text.data(),
static_cast<DWORD>(text.length()),
CRYPT_STRING_BASE64 | CRYPT_STRING_STRICT,
reinterpret_cast<BYTE*>(&decoded[0]),
reinterpret_cast<BYTE*>(decoded.data()),
&decodedLength,
nullptr,
nullptr);
@ -356,30 +358,32 @@ namespace Azure { namespace Storage {
namespace Details {
std::string Sha256(const std::string& text)
std::vector<uint8_t> Sha256(const std::vector<uint8_t>& data)
{
SHA256_CTX context;
SHA256_Init(&context);
SHA256_Update(&context, text.data(), text.length());
SHA256_Update(&context, data.data(), data.size());
unsigned char hash[SHA256_DIGEST_LENGTH];
SHA256_Final(hash, &context);
return std::string(std::begin(hash), std::end(hash));
return std::vector<uint8_t>(std::begin(hash), std::end(hash));
}
std::string HmacSha256(const std::string& text, const std::string& key)
std::vector<uint8_t> HmacSha256(
const std::vector<uint8_t>& data,
const std::vector<uint8_t>& key)
{
char hash[EVP_MAX_MD_SIZE];
uint8_t hash[EVP_MAX_MD_SIZE];
unsigned int hashLength = 0;
HMAC(
EVP_sha256(),
key.data(),
static_cast<int>(key.length()),
reinterpret_cast<const unsigned char*>(text.data()),
text.length(),
static_cast<int>(key.size()),
reinterpret_cast<const unsigned char*>(data.data()),
data.size(),
reinterpret_cast<unsigned char*>(&hash[0]),
&hashLength);
return std::string(hash, hashLength);
return std::vector<uint8_t>(std::begin(hash), std::begin(hash) + hashLength);
}
} // namespace Details
@ -403,20 +407,20 @@ namespace Azure { namespace Storage {
MD5_Update(context, data, length);
}
std::string Md5::Digest() const
std::vector<uint8_t> Md5::Digest() const
{
MD5_CTX* context = static_cast<MD5_CTX*>(m_context);
unsigned char hash[MD5_DIGEST_LENGTH];
MD5_Final(hash, context);
return std::string(std::begin(hash), std::end(hash));
return std::vector<uint8_t>(std::begin(hash), std::end(hash));
}
std::string Base64Encode(const std::string& text)
std::string Base64Encode(const std::vector<uint8_t>& data)
{
BIO* bio = BIO_new(BIO_s_mem());
bio = BIO_push(BIO_new(BIO_f_base64()), bio);
BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL);
BIO_write(bio, text.data(), static_cast<int>(text.length()));
BIO_write(bio, data.data(), static_cast<int>(data.size()));
BIO_flush(bio);
BUF_MEM* bufferPtr;
BIO_get_mem_ptr(bio, &bufferPtr);
@ -426,9 +430,9 @@ namespace Azure { namespace Storage {
return std::string(bufferPtr->data, bufferPtr->length);
}
std::string Base64Decode(const std::string& text)
std::vector<uint8_t> Base64Decode(const std::string& text)
{
std::string decoded;
std::vector<uint8_t> decoded;
// According to RFC 4648, the encoded length should be ceiling(n / 3) * 4, so we can infer an
// upper bound here
std::size_t maxDecodedLength = text.length() / 4 * 3;
@ -1254,9 +1258,9 @@ namespace Azure { namespace Storage {
m_context ^= other.m_context;
}
std::string Crc64::Digest() const
std::vector<uint8_t> Crc64::Digest() const
{
std::string binary;
std::vector<uint8_t> binary;
binary.resize(sizeof(m_context));
for (std::size_t i = 0; i < sizeof(m_context); ++i)
{

5
sdk/storage/azure-storage-common/src/shared_key_policy.cpp
View File

@ -79,7 +79,8 @@ namespace Azure { namespace Storage { namespace Details {
// remove last linebreak
string_to_sign.pop_back();
return Base64Encode(
Details::HmacSha256(string_to_sign, Base64Decode(m_credential->GetAccountKey())));
return Base64Encode(Details::HmacSha256(
std::vector<uint8_t>(string_to_sign.begin(), string_to_sign.end()),
Base64Decode(m_credential->GetAccountKey())));
}
}}} // namespace Azure::Storage::Details

24
sdk/storage/azure-storage-common/test/crypt_functions_test.cpp
View File

@ -1,37 +1,49 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// SPDX-License-Identifier: MIT
#include <cstring>
#include "azure/storage/common/crypt.hpp"
#include "test_base.hpp"
namespace Azure { namespace Storage { namespace Test {
std::vector<uint8_t> ToBinaryVector(const char* text)
{
const uint8_t* start = reinterpret_cast<const uint8_t*>(text);
return std::vector<uint8_t>(start, start + strlen(text));
}
TEST(CryptFunctionsTest, Base64)
{
for (std::size_t len : {0, 10, 100, 1000, 10000})
{
std::string data;
std::vector<uint8_t> data;
data.resize(len);
RandomBuffer(&data[0], data.length());
RandomBuffer(data.data(), data.size());
EXPECT_EQ(Base64Decode(Base64Encode(data)), data);
}
}
TEST(CryptFunctionsTest, Sha256)
{
EXPECT_EQ(Base64Encode(Details::Sha256("")), "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=");
EXPECT_EQ(
Base64Encode(Details::Sha256("Hello Azure!")),
Base64Encode(Details::Sha256(ToBinaryVector(""))),
"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=");
EXPECT_EQ(
Base64Encode(Details::Sha256(ToBinaryVector("Hello Azure!"))),
"Mjzwx2mqGHb9FSgjm33ShNmXYndkgvwA6tQmEiskOHg=");
}
TEST(CryptFunctionsTest, HmacSha256)
{
std::string key = "8CwtGFF1mGR4bPEP9eZ0x1fxKiQ3Ca5N";
std::vector<uint8_t> binaryKey(key.begin(), key.end());
EXPECT_EQ(
Base64Encode(Details::HmacSha256("", key)), "fFy2T+EuCvAgouw/vB/RAJ75z7jwTj+uiURebkFKF5M=");
Base64Encode(Details::HmacSha256(ToBinaryVector(""), binaryKey)),
"fFy2T+EuCvAgouw/vB/RAJ75z7jwTj+uiURebkFKF5M=");
EXPECT_EQ(
Base64Encode(Details::HmacSha256("Hello Azure!", key)),
Base64Encode(Details::HmacSha256(ToBinaryVector("Hello Azure!"), binaryKey)),
"+SBESxQVhI53mSEdZJcCBpdBkaqwzfPaVYZMAf5LP3c=");
}

4
sdk/storage/azure-storage-common/test/test_base.hpp
View File

@ -55,6 +55,10 @@ namespace Azure { namespace Storage { namespace Test {
Storage::Metadata RandomMetadata(size_t size = 5);
void RandomBuffer(char* buffer, std::size_t length);
inline void RandomBuffer(uint8_t* buffer, std::size_t length)
{
RandomBuffer(reinterpret_cast<char*>(buffer), length);
}
std::vector<uint8_t> RandomBuffer(std::size_t length);
inline std::vector<uint8_t> ReadBodyStream(std::unique_ptr<Azure::Core::Http::BodyStream>& stream)

1
sdk/storage/azure-storage-files-datalake/CHANGELOG.md
View File

@ -5,6 +5,7 @@
### Breaking Changes
- Move DataLake SAS into `Azure::Storage::Sas` namespace.
- `EncrytionKeySha256` are changed to binary(`std::vector<uint8_t>`).
## 12.0.0-beta.5 (2020-11-13)

4
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp
View File

@ -86,7 +86,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
Azure::Core::Nullable<LeaseStatusType> LeaseStatus;
DataLakeHttpHeaders HttpHeaders;
Azure::Core::Nullable<bool> ServerEncrypted;
Azure::Core::Nullable<std::string> EncryptionKeySha256;
Azure::Core::Nullable<std::vector<uint8_t>> EncryptionKeySha256;
Azure::Core::Nullable<bool> AccessTierInferred;
Azure::Core::Nullable<std::string> AccessTierChangeTime;
Azure::Core::Nullable<std::string> CopyId;
@ -168,7 +168,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
DataLakeHttpHeaders HttpHeaders;
Storage::Metadata Metadata;
Azure::Core::Nullable<bool> ServerEncrypted;
Azure::Core::Nullable<std::string> EncryptionKeySha256;
Azure::Core::Nullable<std::vector<uint8_t>> EncryptionKeySha256;
};
using CreateFileResult = CreatePathResult;

5
sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp
View File

@ -322,7 +322,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
}
ret.RangeOffset = RangeOffset;
ret.RangeLength = RangeLength;
ret.TransactionalMd5 = std::move(result->TransactionalContentMd5);
if (result->TransactionalContentHash.HasValue())
{
ret.TransactionalMd5 = Base64Encode(result->TransactionalContentHash.GetValue().Value);
}
ret.ETag = std::move(result->ETag);
ret.LastModified = std::move(result->LastModified);
ret.LeaseDuration = std::move(result->LeaseDuration);

10
sdk/storage/azure-storage-files-datalake/src/datalake_sas_builder.cpp
View File

@ -128,8 +128,9 @@ namespace Azure { namespace Storage { namespace Sas {
+ Storage::Details::DefaultSasVersion + "\n" + resource + "\n" + "\n" + CacheControl + "\n"
+ ContentDisposition + "\n" + ContentEncoding + "\n" + ContentLanguage + "\n" + ContentType;
std::string signature = Base64Encode(
Storage::Details::HmacSha256(stringToSign, Base64Decode(credential.GetAccountKey())));
std::string signature = Base64Encode(Storage::Details::HmacSha256(
std::vector<uint8_t>(stringToSign.begin(), stringToSign.end()),
Base64Decode(credential.GetAccountKey())));
Azure::Core::Http::Url builder;
builder.AppendQueryParameter(
@ -208,8 +209,9 @@ namespace Azure { namespace Storage { namespace Sas {
+ CacheControl + "\n" + ContentDisposition + "\n" + ContentEncoding + "\n" + ContentLanguage
+ "\n" + ContentType;
std::string signature = Base64Encode(
Storage::Details::HmacSha256(stringToSign, Base64Decode(userDelegationKey.Value)));
std::string signature = Base64Encode(Storage::Details::HmacSha256(
std::vector<uint8_t>(stringToSign.begin(), stringToSign.end()),
Base64Decode(userDelegationKey.Value)));
Azure::Core::Http::Url builder;
builder.AppendQueryParameter(

5
sdk/storage/azure-storage-files-shares/src/share_sas_builder.cpp
View File

@ -89,8 +89,9 @@ namespace Azure { namespace Storage { namespace Sas {
+ Storage::Details::DefaultSasVersion + "\n" + CacheControl + "\n" + ContentDisposition
+ "\n" + ContentEncoding + "\n" + ContentLanguage + "\n" + ContentType;
std::string signature = Base64Encode(
Storage::Details::HmacSha256(stringToSign, Base64Decode(credential.GetAccountKey())));
std::string signature = Base64Encode(Storage::Details::HmacSha256(
std::vector<uint8_t>(stringToSign.begin(), stringToSign.end()),
Base64Decode(credential.GetAccountKey())));
Azure::Core::Http::Url builder;
builder.AppendQueryParameter(