github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Storage Blobs Service] Transactional MD5 and CRC64 (#456)

Browse Source 
* MD5

* Crc64

* crc64 concatenate
This commit is contained in:
JinmingHu 2020-08-16 00:30:49 +08:00 committed by GitHub
parent 0b2b688e72
commit 86a22901e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 1233 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
sdk/storage/inc/common/crypt.hpp
View File

@ -3,13 +3,65 @@
#pragma once
#include <cstdint>
#include <string>
namespace Azure { namespace Storage {
std::string Sha256(const std::string& text);
std::string Hmac_Sha256(const std::string& text, const std::string& key);
std::string Base64Encode(const std::string& text);
std::string Base64Decode(const std::string& text);
class Md5 {
public:
Md5();
~Md5();
void Update(const uint8_t* data, std::size_t length);
std::string Digest() const;
static std::string Hash(const uint8_t* data, std::size_t length)
{
Md5 instance;
instance.Update(data, length);
return instance.Digest();
}
static std::string Hash(const std::string& data)
{
return Hash(reinterpret_cast<const uint8_t*>(data.data()), data.length());
}
private:
void* m_context;
};
class Crc64 {
public:
void Update(const uint8_t* data, std::size_t length);
void Concatenate(const Crc64& other);
std::string Digest() const;
static std::string Hash(const uint8_t* data, std::size_t length)
{
Crc64 instance;
instance.Update(data, length);
return instance.Digest();
}
static std::string Hash(const std::string& data)
{
return Hash(reinterpret_cast<const uint8_t*>(data.data()), data.length());
}
private:
uint64_t m_context = 0ULL;
uint64_t m_length = 0ULL;
};
namespace Details {
std::string Sha256(const std::string& text);
std::string HmacSha256(const std::string& text, const std::string& key);
} // namespace Details
}} // namespace Azure::Storage

4
sdk/storage/src/blobs/blob_sas_builder.cpp
View File

@ -135,7 +135,7 @@ namespace Azure { namespace Storage { namespace Blobs {
+ ContentEncoding + "\n" + ContentLanguage + "\n" + ContentType;
std::string signature
= Base64Encode(Hmac_Sha256(stringToSign, Base64Decode(credential.GetAccountKey())));
= Base64Encode(Details::HmacSha256(stringToSign, Base64Decode(credential.GetAccountKey())));
UriBuilder builder;
builder.AppendQuery("sv", Version);
@ -219,7 +219,7 @@ namespace Azure { namespace Storage { namespace Blobs {
+ "\n" + ContentType;
std::string signature
= Base64Encode(Hmac_Sha256(stringToSign, Base64Decode(userDelegationKey.Value)));
= Base64Encode(Details::HmacSha256(stringToSign, Base64Decode(userDelegationKey.Value)));
UriBuilder builder;
builder.AppendQuery("sv", Version);

2
sdk/storage/src/common/account_sas_builder.cpp
View File

@ -102,7 +102,7 @@ namespace Azure { namespace Storage {
+ "\n";
std::string signature
= Base64Encode(Hmac_Sha256(stringToSign, Base64Decode(credential.GetAccountKey())));
= Base64Encode(Details::HmacSha256(stringToSign, Base64Decode(credential.GetAccountKey())));
UriBuilder builder;
builder.AppendQuery("sv", Version);

1105
sdk/storage/src/common/crypt.cpp
View File

File diff suppressed because it is too large Load Diff

3
sdk/storage/src/common/shared_key_policy.cpp
View File

@ -84,6 +84,7 @@ namespace Azure { namespace Storage {
// remove last linebreak
string_to_sign.pop_back();
return Base64Encode(Hmac_Sha256(string_to_sign, Base64Decode(m_credential->GetAccountKey())));
return Base64Encode(
Details::HmacSha256(string_to_sign, Base64Decode(m_credential->GetAccountKey())));
}
}} // namespace Azure::Storage

2
sdk/storage/test/blobs/blob_container_client_test.cpp
View File

@ -452,7 +452,7 @@ namespace Azure { namespace Storage { namespace Test {
aes256Key.resize(32);
RandomBuffer(&aes256Key[0], aes256Key.size());
key.Key = Base64Encode(aes256Key);
key.KeyHash = Base64Encode(Sha256(aes256Key));
key.KeyHash = Base64Encode(Details::Sha256(aes256Key));
key.Algorithm = Blobs::EncryptionAlgorithmType::Aes256;
return key;
};

41
sdk/storage/test/blobs/page_blob_client_test.cpp
View File

@ -2,6 +2,7 @@
// SPDX-License-Identifier: MIT
#include "page_blob_client_test.hpp"
#include "common/crypt.hpp"
namespace Azure { namespace Storage { namespace Test {
@ -203,4 +204,44 @@ namespace Azure { namespace Storage { namespace Test {
m_pageBlobClient->BreakLease(options);
}
TEST_F(PageBlobClientTest, ContentMd5)
{
std::vector<uint8_t> blobContent;
blobContent.resize(static_cast<std::size_t>(4_KB));
RandomBuffer(reinterpret_cast<char*>(&blobContent[0]), blobContent.size());
auto pageBlobClient = Azure::Storage::Blobs::PageBlobClient::CreateFromConnectionString(
StandardStorageConnectionString(), m_containerName, RandomString());
pageBlobClient.Create(blobContent.size(), m_blobUploadOptions);
auto pageContent = Azure::Core::Http::MemoryBodyStream(blobContent.data(), blobContent.size());
Blobs::UploadPageBlobPagesOptions options;
options.ContentMd5 = Base64Encode(Md5::Hash(blobContent.data(), blobContent.size()));
EXPECT_NO_THROW(pageBlobClient.UploadPages(&pageContent, 0, options));
pageContent.Rewind();
options.ContentMd5 = c_dummyMd5;
EXPECT_THROW(pageBlobClient.UploadPages(&pageContent, 0, options), StorageError);
}
TEST_F(PageBlobClientTest, ContentCrc64)
{
std::vector<uint8_t> blobContent;
blobContent.resize(static_cast<std::size_t>(4_KB));
RandomBuffer(reinterpret_cast<char*>(&blobContent[0]), blobContent.size());
auto pageBlobClient = Azure::Storage::Blobs::PageBlobClient::CreateFromConnectionString(
StandardStorageConnectionString(), m_containerName, RandomString());
pageBlobClient.Create(blobContent.size(), m_blobUploadOptions);
auto pageContent = Azure::Core::Http::MemoryBodyStream(blobContent.data(), blobContent.size());
Blobs::UploadPageBlobPagesOptions options;
options.ContentCrc64 = Base64Encode(Crc64::Hash(blobContent.data(), blobContent.size()));
EXPECT_NO_THROW(pageBlobClient.UploadPages(&pageContent, 0, options));
pageContent.Rewind();
options.ContentCrc64 = c_dummyCrc64;
EXPECT_THROW(pageBlobClient.UploadPages(&pageContent, 0, options), StorageError);
}
}}} // namespace Azure::Storage::Test

100
sdk/storage/test/common/crypt_functions_test.cpp
View File

@ -19,17 +19,109 @@ namespace Azure { namespace Storage { namespace Test {
TEST(CryptFunctionsTest, Sha256)
{
EXPECT_EQ(Base64Encode(Sha256("")), "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=");
EXPECT_EQ(Base64Encode(Sha256("Hello Azure!")), "Mjzwx2mqGHb9FSgjm33ShNmXYndkgvwA6tQmEiskOHg=");
EXPECT_EQ(Base64Encode(Details::Sha256("")), "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=");
EXPECT_EQ(
Base64Encode(Details::Sha256("Hello Azure!")),
"Mjzwx2mqGHb9FSgjm33ShNmXYndkgvwA6tQmEiskOHg=");
}
TEST(CryptFunctionsTest, HmacSha256)
{
std::string key = "8CwtGFF1mGR4bPEP9eZ0x1fxKiQ3Ca5N";
EXPECT_EQ(Base64Encode(Hmac_Sha256("", key)), "fFy2T+EuCvAgouw/vB/RAJ75z7jwTj+uiURebkFKF5M=");
EXPECT_EQ(
Base64Encode(Hmac_Sha256("Hello Azure!", key)),
Base64Encode(Details::HmacSha256("", key)), "fFy2T+EuCvAgouw/vB/RAJ75z7jwTj+uiURebkFKF5M=");
EXPECT_EQ(
Base64Encode(Details::HmacSha256("Hello Azure!", key)),
"+SBESxQVhI53mSEdZJcCBpdBkaqwzfPaVYZMAf5LP3c=");
}
TEST(CryptFunctionsTest, Md5)
{
EXPECT_EQ(Base64Encode(Md5::Hash("")), "1B2M2Y8AsgTpgAmY7PhCfg==");
EXPECT_EQ(Base64Encode(Md5::Hash("Hello Azure!")), "Pz8543xut4RVSbb2g52Mww==");
auto data = RandomBuffer(static_cast<std::size_t>(16_MB));
Md5 md5Instance;
std::size_t length = 0;
while (length < data.size())
{
std::size_t s = static_cast<std::size_t>(RandomInt(0, 4_MB));
s = std::min(s, data.size() - length);
md5Instance.Update(&data[length], s);
md5Instance.Update(&data[length], 0);
length += s;
}
EXPECT_EQ(md5Instance.Digest(), Md5::Hash(data.data(), data.size()));
}
TEST(CryptFunctionsTest, Crc64)
{
EXPECT_EQ(Base64Encode(Crc64::Hash("")), "AAAAAAAAAAA=");
EXPECT_EQ(Base64Encode(Crc64::Hash("Hello Azure!")), "DtjZpL9/o8c=");
auto data = RandomBuffer(static_cast<std::size_t>(16_MB));
Crc64 crc64Instance;
std::size_t length = 0;
while (length < data.size())
{
std::size_t s = static_cast<std::size_t>(RandomInt(0, 4_MB));
s = std::min(s, data.size() - length);
crc64Instance.Update(&data[length], s);
crc64Instance.Update(&data[length], 0);
length += s;
}
EXPECT_EQ(crc64Instance.Digest(), Crc64::Hash(data.data(), data.size()));
// Test concatenate
crc64Instance = Crc64();
std::string allData;
while (allData.length() < 16_MB)
{
{
Crc64 instance2;
for (auto i = RandomInt(0, 5); i > 0; --i)
{
std::size_t s = static_cast<std::size_t>(RandomInt(0, 512_KB));
std::string data2;
data2.resize(s);
RandomBuffer(&data2[0], s);
instance2.Update(reinterpret_cast<const uint8_t*>(data2.data()), data2.length());
allData += data2;
}
crc64Instance.Concatenate(instance2);
}
switch (RandomInt(0, 2))
{
case 0: {
std::string data2;
crc64Instance.Update(reinterpret_cast<const uint8_t*>(data2.data()), data2.length());
break;
}
case 1: {
Crc64 instance2;
crc64Instance.Concatenate(instance2);
break;
}
case 2: {
std::size_t s = static_cast<std::size_t>(RandomInt(0, 512_KB));
std::string data2;
data2.resize(s);
RandomBuffer(&data2[0], s);
crc64Instance.Update(reinterpret_cast<const uint8_t*>(data2.data()), data2.length());
allData += data2;
break;
}
default:
break;
}
}
EXPECT_EQ(
crc64Instance.Digest(),
Crc64::Hash(reinterpret_cast<const uint8_t*>(allData.data()), allData.size()));
}
}}} // namespace Azure::Storage::Test

14
sdk/storage/test/test_base.cpp
View File

@ -127,7 +127,13 @@ namespace Azure { namespace Storage { namespace Test {
static thread_local std::mt19937_64 random_generator(std::random_device{}());
static char random_char()
uint64_t RandomInt(uint64_t minNumber, uint64_t maxNumber)
{
std::uniform_int_distribution<uint64_t> distribution(minNumber, maxNumber);
return distribution(random_generator);
}
static char RandomChar()
{
const char charset[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
std::uniform_int_distribution<std::size_t> distribution(0, sizeof(charset) - 2);
@ -138,7 +144,7 @@ namespace Azure { namespace Storage { namespace Test {
{
std::string str;
str.resize(size);
std::generate(str.begin(), str.end(), random_char);
std::generate(str.begin(), str.end(), RandomChar);
return str;
}
@ -171,7 +177,7 @@ namespace Azure { namespace Storage { namespace Test {
while (uintptr_t(start_addr) % rand_int_size != 0 && start_addr < end_addr)
{
*(start_addr++) = random_char();
*(start_addr++) = RandomChar();
}
std::uniform_int_distribution<uint64_t> distribution(
@ -183,7 +189,7 @@ namespace Azure { namespace Storage { namespace Test {
}
while (start_addr < end_addr)
{
*(start_addr++) = random_char();
*(start_addr++) = RandomChar();
}
}

7
sdk/storage/test/test_base.hpp
View File

@ -10,6 +10,7 @@
#include "gtest/gtest.h"
#include <chrono>
#include <limits>
namespace Azure { namespace Storage { namespace Test {
@ -39,6 +40,12 @@ namespace Azure { namespace Storage { namespace Test {
}
constexpr static const char* c_dummyETag = "0x8D83B58BDF51D75";
constexpr static const char* c_dummyMd5 = "tQbD1aMPeB+LiPffUwFQJQ==";
constexpr static const char* c_dummyCrc64 = "+DNR5PON4EM=";
uint64_t RandomInt(
uint64_t minNumber = std::numeric_limits<uint64_t>::min(),
uint64_t maxNumber = std::numeric_limits<uint64_t>::max());
std::string RandomString(size_t size = 10);