diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 9f965e1e3..5f563bb3d 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -9,7 +9,7 @@ jobs: matrix: Linux_x64: vm.image: 'ubuntu-18.04' - vcpkg.deps: 'curl[ssl] libxml2' + vcpkg.deps: 'curl[ssl] libxml2 openssl' VCPKG_DEFAULT_TRIPLET: 'x64-linux' Win_x86: vm.image: 'windows-2019' @@ -25,13 +25,13 @@ jobs: CMAKE_GENERATOR_PLATFORM: x64 MacOS_x64: vm.image: 'macOS-10.14' - vcpkg.deps: 'curl[ssl] libxml2' + vcpkg.deps: 'curl[ssl] libxml2 openssl' VCPKG_DEFAULT_TRIPLET: 'x64-osx' # Unit testing ON Linux_x64_with_unit_test: vm.image: 'ubuntu-18.04' - vcpkg.deps: 'curl[ssl] libxml2' + vcpkg.deps: 'curl[ssl] libxml2 openssl' VCPKG_DEFAULT_TRIPLET: 'x64-linux' build.args: ' -DBUILD_TESTING=ON' Win_x86_with_unit_test: @@ -50,7 +50,7 @@ jobs: build.args: ' -DBUILD_TESTING=ON' MacOS_x64_with_unit_test: vm.image: 'macOS-10.14' - vcpkg.deps: 'curl[ssl] libxml2' + vcpkg.deps: 'curl[ssl] libxml2 openssl' VCPKG_DEFAULT_TRIPLET: 'x64-osx' build.args: ' -DBUILD_TESTING=ON' pool: diff --git a/sdk/storage/CMakeLists.txt b/sdk/storage/CMakeLists.txt index 72e93aa4a..30e8d27ef 100644 --- a/sdk/storage/CMakeLists.txt +++ b/sdk/storage/CMakeLists.txt @@ -13,6 +13,9 @@ set(AZURE_STORAGE_HEADER inc/common/storage_common.hpp inc/common/storage_credential.hpp inc/common/storage_url_builder.hpp + inc/common/common_headers_request_policy.hpp + inc/common/shared_key_policy.hpp + inc/common/crypt.hpp inc/blobs/blob.hpp inc/blobs/blob_client.hpp inc/blobs/block_blob_client.hpp @@ -28,6 +31,9 @@ set(AZURE_STORAGE_SOURCE src/blobs/blob_container_client.cpp src/common/storage_credential.cpp src/common/storage_url_builder.cpp + src/common/common_headers_request_policy.cpp + src/common/shared_key_policy.cpp + src/common/crypt.cpp ) add_library(azure-storage ${AZURE_STORAGE_HEADER} ${AZURE_STORAGE_SOURCE}) @@ -39,7 +45,11 @@ target_include_directories(azure-storage PUBLIC ${LIBXML2_INCLUDE_DIR} $ #include #include -#include - -#include "common/storage_url_builder.hpp" -#include "common/storage_credential.hpp" -#include "blob_client_options.hpp" -#include "internal/protocol/blob_rest_client.hpp" namespace Azure { namespace Storage { namespace Blobs { diff --git a/sdk/storage/inc/blobs/blob_client_options.hpp b/sdk/storage/inc/blobs/blob_client_options.hpp index 6ba55d330..bba2e2822 100644 --- a/sdk/storage/inc/blobs/blob_client_options.hpp +++ b/sdk/storage/inc/blobs/blob_client_options.hpp @@ -3,12 +3,12 @@ #pragma once -#include -#include -#include - #include "internal/protocol/blob_rest_client.hpp" +#include +#include +#include + namespace Azure { namespace Storage { namespace Blobs { struct BlobClientOptions @@ -18,7 +18,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct BlockBlobClientOptions : public BlobClientOptions { - }; struct GetBlobPropertiesOptions diff --git a/sdk/storage/inc/blobs/blob_container_client.hpp b/sdk/storage/inc/blobs/blob_container_client.hpp index 1d89d93d0..5c71fc9ad 100644 --- a/sdk/storage/inc/blobs/blob_container_client.hpp +++ b/sdk/storage/inc/blobs/blob_container_client.hpp @@ -3,15 +3,15 @@ #pragma once -#include +#include "blob_container_client_options.hpp" +#include "blobs/blob_client.hpp" +#include "common/storage_credential.hpp" +#include "common/storage_url_builder.hpp" +#include "internal/protocol/blob_rest_client.hpp" + #include #include - -#include "blobs/blob_client.hpp" -#include "common/storage_url_builder.hpp" -#include "common/storage_credential.hpp" -#include "blob_container_client_options.hpp" -#include "internal/protocol/blob_rest_client.hpp" +#include namespace Azure { namespace Storage { namespace Blobs { diff --git a/sdk/storage/inc/blobs/blob_container_client_options.hpp b/sdk/storage/inc/blobs/blob_container_client_options.hpp index 654b28cae..1fb85a732 100644 --- a/sdk/storage/inc/blobs/blob_container_client_options.hpp +++ b/sdk/storage/inc/blobs/blob_container_client_options.hpp @@ -3,11 +3,11 @@ #pragma once +#include "internal/protocol/blob_rest_client.hpp" + +#include #include #include -#include - -#include "internal/protocol/blob_rest_client.hpp" namespace Azure { namespace Storage { namespace Blobs { diff --git a/sdk/storage/inc/blobs/block_blob_client.hpp b/sdk/storage/inc/blobs/block_blob_client.hpp index e722d8e80..5b91cc25d 100644 --- a/sdk/storage/inc/blobs/block_blob_client.hpp +++ b/sdk/storage/inc/blobs/block_blob_client.hpp @@ -3,14 +3,14 @@ #pragma once -#include -#include - +#include "blob_client_options.hpp" #include "blobs/blob_client.hpp" #include "common/storage_credential.hpp" -#include "blob_client_options.hpp" #include "internal/protocol/blob_rest_client.hpp" +#include +#include + namespace Azure { namespace Storage { namespace Blobs { class BlockBlobClient : public BlobClient { diff --git a/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp b/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp index 75704aa48..3afd2f50a 100644 --- a/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp +++ b/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp @@ -818,7 +818,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct ListBlobContainersOptions { std::string Version; - std::string Date; std::string Prefix; std::string Marker; int MaxResults = 0; @@ -831,8 +830,14 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); request.AddHeader("Content-Length", "0"); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } request.AddQueryParameter("comp", "list"); if (!options.Prefix.empty()) { @@ -1004,7 +1009,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct CreateOptions { std::string Version; - std::string Date; PublicAccessType AccessType = PublicAccessType::Private; std::map Metadata; }; // struct CreateOptions @@ -1016,8 +1020,14 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); request.AddQueryParameter("restype", "container"); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } for (const auto& pair : options.Metadata) { request.AddHeader("x-ms-meta-" + pair.first, pair.second); @@ -1068,7 +1078,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct DeleteOptions { std::string Version; - std::string Date; }; // struct DeleteOptions static Azure::Core::Http::Request DeleteConstructRequest( @@ -1078,8 +1087,14 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Delete, url); request.AddHeader("Content-Length", "0"); request.AddQueryParameter("restype", "container"); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } return request; } @@ -1119,7 +1134,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct GetPropertiesOptions { std::string Version; - std::string Date; std::string EncryptionKey; std::string EncryptionKeySHA256; std::string EncryptionAlgorithm; @@ -1132,8 +1146,14 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Head, url); request.AddHeader("Content-Length", "0"); request.AddQueryParameter("restype", "container"); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } if (!options.EncryptionKey.empty()) { request.AddHeader("x-ms-encryption-key", options.EncryptionKey); @@ -1213,7 +1233,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct SetMetadataOptions { std::string Version; - std::string Date; std::map Metadata; }; // struct SetMetadataOptions @@ -1225,8 +1244,14 @@ namespace Azure { namespace Storage { namespace Blobs { request.AddHeader("Content-Length", "0"); request.AddQueryParameter("restype", "container"); request.AddQueryParameter("comp", "metadata"); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } for (const auto& pair : options.Metadata) { request.AddHeader("x-ms-meta-" + pair.first, pair.second); @@ -1272,7 +1297,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct ListBlobsOptions { std::string Version; - std::string Date; std::string Prefix; std::string Delimiter; std::string Marker; @@ -1286,8 +1310,14 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); request.AddHeader("Content-Length", "0"); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } request.AddQueryParameter("restype", "container"); request.AddQueryParameter("comp", "list"); if (!options.Prefix.empty()) @@ -1498,7 +1528,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct DownloadOptions { std::string Version; - std::string Date; std::pair Range; std::string EncryptionKey; std::string EncryptionKeySHA256; @@ -1511,8 +1540,14 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); request.AddHeader("Content-Length", "0"); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } if (options.Range.first <= options.Range.second) { request.AddHeader( @@ -1670,7 +1705,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct DeleteOptions { std::string Version; - std::string Date; DeleteSnapshotsOption DeleteSnapshots = DeleteSnapshotsOption::None; }; // struct DeleteOptions @@ -1680,8 +1714,14 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Delete, url); request.AddHeader("Content-Length", "0"); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } return request; } @@ -1721,7 +1761,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct GetPropertiesOptions { std::string Version; - std::string Date; }; // struct GetPropertiesOptions static Azure::Core::Http::Request GetPropertiesConstructRequest( @@ -1730,8 +1769,14 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Head, url); request.AddHeader("Content-Length", "0"); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } return request; } @@ -1873,7 +1918,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct SetHttpHeadersOptions { std::string Version; - std::string Date; std::string ContentType; std::string ContentEncoding; std::string ContentLanguage; @@ -1892,8 +1936,14 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); request.AddQueryParameter("comp", "properties"); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } if (!options.ContentType.empty()) { request.AddHeader("x-ms-blob-content-type", options.ContentType); @@ -1977,7 +2027,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct SetMetadataOptions { std::string Version; - std::string Date; std::map Metadata; std::string EncryptionKey; std::string EncryptionKeySHA256; @@ -1991,8 +2040,14 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); request.AddQueryParameter("comp", "metadata"); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } for (const auto& pair : options.Metadata) { request.AddHeader("x-ms-meta-" + pair.first, pair.second); @@ -2054,7 +2109,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct UploadOptions { std::string Version; - std::string Date; std::vector* BodyBuffer = nullptr; Azure::Core::Http::BodyStream* BodyStream = nullptr; std::string ContentMD5; @@ -2076,8 +2130,14 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Azure::Core::Http::Request( Azure::Core::Http::HttpMethod::Put, url, *options.BodyBuffer); request.AddHeader("Content-Length", std::to_string(options.BodyBuffer->size())); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } if (!options.EncryptionKey.empty()) { request.AddHeader("x-ms-encryption-key", options.EncryptionKey); @@ -2199,7 +2259,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct StageBlockOptions { std::string Version; - std::string Date; std::vector* BodyBuffer = nullptr; Azure::Core::Http::BodyStream* BodyStream = nullptr; std::string BlockId; @@ -2220,8 +2279,14 @@ namespace Azure { namespace Storage { namespace Blobs { request.AddHeader("Content-Length", std::to_string(options.BodyBuffer->size())); request.AddQueryParameter("comp", "block"); request.AddQueryParameter("blockid", options.BlockId); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } if (!options.ContentMD5.empty()) { request.AddHeader("Content-MD5", options.ContentMD5); @@ -2307,7 +2372,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct CommitBlockListOptions { std::string Version; - std::string Date; std::vector> BlockList; BlobHttpHeaders Properties; std::map Metadata; @@ -2369,8 +2433,14 @@ namespace Azure { namespace Storage { namespace Blobs { = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url, body_buffer); request.AddHeader("Content-Length", std::to_string(body_buffer.size())); request.AddQueryParameter("comp", "blocklist"); - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } if (!options.Properties.ContentType.empty()) { request.AddHeader("x-ms-blob-content-type", options.Properties.ContentType); @@ -2474,7 +2544,6 @@ namespace Azure { namespace Storage { namespace Blobs { struct GetBlockListOptions { std::string Version; - std::string Date; BlockListTypeOption ListType = BlockListTypeOption::All; }; // struct GetBlockListOptions @@ -2490,8 +2559,14 @@ namespace Azure { namespace Storage { namespace Blobs { { request.AddQueryParameter("blocklisttype", block_list_type_option); } - request.AddHeader("x-ms-version", options.Version); - request.AddHeader("x-ms-date", options.Date); + if (!options.Version.empty()) + { + request.AddHeader("x-ms-version", options.Version); + } + else + { + request.AddHeader("x-ms-version", "2019-07-07"); + } return request; } diff --git a/sdk/storage/inc/common/common_headers_request_policy.hpp b/sdk/storage/inc/common/common_headers_request_policy.hpp new file mode 100644 index 000000000..42bf6e022 --- /dev/null +++ b/sdk/storage/inc/common/common_headers_request_policy.hpp @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#pragma once + +#include "http/policy.hpp" + +namespace Azure { namespace Storage { + + class CommonHeadersRequestPolicy : public Core::Http::HttpPolicy { + public: + explicit CommonHeadersRequestPolicy() {} + ~CommonHeadersRequestPolicy() override {} + + HttpPolicy* Clone() const override { return new CommonHeadersRequestPolicy(*this); } + + std::unique_ptr Send( + Core::Context& ctx, + Core::Http::Request& request, + Core::Http::NextHttpPolicy nextHttpPolicy) const override; + }; + +}} // namespace Azure::Storage diff --git a/sdk/storage/inc/common/crypt.hpp b/sdk/storage/inc/common/crypt.hpp new file mode 100644 index 000000000..c84f1057a --- /dev/null +++ b/sdk/storage/inc/common/crypt.hpp @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace Azure { namespace Storage { + + std::string HMAC_SHA256(const std::string& text, const std::string& key); + std::string Base64Encode(const std::string& text); + std::string Base64Decode(const std::string& text); + std::string MD5(const std::string& text); + std::string CRC64(const std::string& text); + +}} // namespace Azure::Storage diff --git a/sdk/storage/inc/common/shared_key_policy.hpp b/sdk/storage/inc/common/shared_key_policy.hpp new file mode 100644 index 000000000..b8ca3d665 --- /dev/null +++ b/sdk/storage/inc/common/shared_key_policy.hpp @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#pragma once + +#include "common/storage_credential.hpp" +#include "http/policy.hpp" + +namespace Azure { namespace Storage { + + class SharedKeyPolicy : public Core::Http::HttpPolicy { + public: + explicit SharedKeyPolicy(std::shared_ptr credential) + : m_credential(std::move(credential)) + { + } + + ~SharedKeyPolicy() override {} + + HttpPolicy* Clone() const override { return new SharedKeyPolicy(m_credential); } + + std::unique_ptr Send( + Core::Context& ctx, + Core::Http::Request& request, + Core::Http::NextHttpPolicy nextHttpPolicy) const override + { + request.AddHeader( + "Authorization", "SharedKey " + m_credential->AccountName + ":" + GetSignature(request)); + return nextHttpPolicy.Send(ctx, request); + } + + private: + std::string GetSignature(const Core::Http::Request& request) const; + + std::shared_ptr m_credential; + }; + +}} // namespace Azure::Storage diff --git a/sdk/storage/inc/common/storage_common.hpp b/sdk/storage/inc/common/storage_common.hpp index a2f79dc74..fefaab168 100644 --- a/sdk/storage/inc/common/storage_common.hpp +++ b/sdk/storage/inc/common/storage_common.hpp @@ -5,8 +5,6 @@ namespace Azure { namespace Storage { - template void unused(T&&...) - { - } + template void unused(T&&...) {} }} // namespace Azure::Storage diff --git a/sdk/storage/inc/common/storage_credential.hpp b/sdk/storage/inc/common/storage_credential.hpp index b353ecb40..e25aa3584 100644 --- a/sdk/storage/inc/common/storage_credential.hpp +++ b/sdk/storage/inc/common/storage_credential.hpp @@ -11,9 +11,7 @@ namespace Azure { namespace Storage { struct TokenCredential { - explicit TokenCredential(std::string token) : Token(std::move(token)) - { - } + explicit TokenCredential(std::string token) : Token(std::move(token)) {} void SetToken(std::string token) { @@ -47,6 +45,7 @@ namespace Azure { namespace Storage { std::string AccountName; private: + friend class SharedKeyPolicy; std::string GetAccountKey() { std::lock_guard guard(Mutex); diff --git a/sdk/storage/inc/common/storage_url_builder.hpp b/sdk/storage/inc/common/storage_url_builder.hpp index 99e4ee897..bd1f71d73 100644 --- a/sdk/storage/inc/common/storage_url_builder.hpp +++ b/sdk/storage/inc/common/storage_url_builder.hpp @@ -3,41 +3,35 @@ #pragma once +#include #include #include -#include namespace Azure { namespace Storage { class UrlBuilder { public: - UrlBuilder() - { - } + UrlBuilder() {} // url must be url-encoded explicit UrlBuilder(const std::string& url); - void SetScheme(const std::string& scheme) - { - m_scheme = scheme; - } + void SetScheme(const std::string& scheme) { m_scheme = scheme; } void SetHost(const std::string& host, bool do_encoding = false) { m_host = do_encoding ? EncodeHost(host) : host; } - void SetPort(uint16_t port) - { - m_port = port; - } + void SetPort(uint16_t port) { m_port = port; } void SetPath(const std::string& path, bool do_encoding = false) { m_path = do_encoding ? EncodePath(path) : path; } + const std::string& GetPath() const { return m_path; } + void AppendPath(const std::string& path, bool do_encoding = false) { if (!m_path.empty() && m_path.back() != '/') @@ -62,10 +56,9 @@ namespace Azure { namespace Storage { } } - void RemoveQuery(const std::string& key) - { - m_query.erase(key); - } + void RemoveQuery(const std::string& key) { m_query.erase(key); } + + const std::map& GetQuery() const { return m_query; } void SetFragment(const std::string& fragment, bool do_encoding = false) { @@ -79,7 +72,9 @@ namespace Azure { namespace Storage { static std::string EncodePath(const std::string& path); static std::string EncodeQuery(const std::string& query); static std::string EncodeFragment(const std::string& fragment); - static std::string EncodeImpl(const std::string& source, const std::function& should_encode); + static std::string EncodeImpl( + const std::string& source, + const std::function& should_encode); std::string m_scheme; std::string m_host; diff --git a/sdk/storage/src/blobs/blob_client.cpp b/sdk/storage/src/blobs/blob_client.cpp index 28483ed9f..4cf723f3a 100644 --- a/sdk/storage/src/blobs/blob_client.cpp +++ b/sdk/storage/src/blobs/blob_client.cpp @@ -3,6 +3,8 @@ #include "blobs/blob_client.hpp" +#include "common/common_headers_request_policy.hpp" +#include "common/shared_key_policy.hpp" #include "common/storage_common.hpp" #include "http/curl/curl.hpp" @@ -56,7 +58,7 @@ namespace Azure { namespace Storage { namespace Blobs { } else if (!accountName.empty()) { - builder.SetHost(accountName + ".blob" + defaultEndpointsProtocol); + builder.SetHost(accountName + ".blob." + EndpointSuffix); } else { @@ -74,20 +76,37 @@ namespace Azure { namespace Storage { namespace Blobs { const std::string& blobUri, std::shared_ptr credential, const BlobClientOptions& options) - : BlobClient(blobUri, options) + : m_blobUrl(blobUri) { - // not implemented yet - unused(credential, options); + std::vector> policies; + for (const auto& p : options.policies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + policies.emplace_back(std::make_unique()); + policies.emplace_back(std::make_unique(credential)); + policies.emplace_back(std::make_unique( + std::make_shared())); + m_pipeline = std::make_shared(policies); } BlobClient::BlobClient( const std::string& blobUri, std::shared_ptr credential, const BlobClientOptions& options) - : BlobClient(blobUri, options) + : m_blobUrl(blobUri) { + std::vector> policies; + for (const auto& p : options.policies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + policies.emplace_back(std::make_unique()); // not implemented yet unused(credential); + policies.emplace_back(std::make_unique( + std::make_shared())); + m_pipeline = std::make_shared(policies); } BlobClient::BlobClient(const std::string& blobUri, const BlobClientOptions& options) @@ -98,6 +117,7 @@ namespace Azure { namespace Storage { namespace Blobs { { policies.emplace_back(std::unique_ptr(p->Clone())); } + policies.emplace_back(std::make_unique()); policies.emplace_back(std::make_unique( std::make_shared())); m_pipeline = std::make_shared(policies); diff --git a/sdk/storage/src/blobs/blob_container_client.cpp b/sdk/storage/src/blobs/blob_container_client.cpp index 2d3f15ef3..0c3191384 100644 --- a/sdk/storage/src/blobs/blob_container_client.cpp +++ b/sdk/storage/src/blobs/blob_container_client.cpp @@ -3,6 +3,8 @@ #include "blobs/blob_container_client.hpp" +#include "common/common_headers_request_policy.hpp" +#include "common/shared_key_policy.hpp" #include "common/storage_common.hpp" #include "http/curl/curl.hpp" @@ -55,7 +57,7 @@ namespace Azure { namespace Storage { namespace Blobs { } else if (!accountName.empty()) { - builder.SetHost(accountName + ".blob" + defaultEndpointsProtocol); + builder.SetHost(accountName + ".blob." + EndpointSuffix); } else { @@ -74,8 +76,16 @@ namespace Azure { namespace Storage { namespace Blobs { const BlobContainerClientOptions& options) : BlobContainerClient(containerUri, options) { - // not implemented yet - unused(credential); + std::vector> policies; + for (const auto& p : options.policies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + policies.emplace_back(std::make_unique()); + policies.emplace_back(std::make_unique(credential)); + policies.emplace_back(std::make_unique( + std::make_shared())); + m_pipeline = std::make_shared(policies); } BlobContainerClient::BlobContainerClient( @@ -84,8 +94,17 @@ namespace Azure { namespace Storage { namespace Blobs { const BlobContainerClientOptions& options) : BlobContainerClient(containerUri, options) { + std::vector> policies; + for (const auto& p : options.policies) + { + policies.emplace_back(std::unique_ptr(p->Clone())); + } + policies.emplace_back(std::make_unique()); // not implemented yet unused(credential); + policies.emplace_back(std::make_unique( + std::make_shared())); + m_pipeline = std::make_shared(policies); } BlobContainerClient::BlobContainerClient( @@ -98,6 +117,7 @@ namespace Azure { namespace Storage { namespace Blobs { { policies.emplace_back(std::unique_ptr(p->Clone())); } + policies.emplace_back(std::make_unique()); policies.emplace_back(std::make_unique( std::make_shared())); m_pipeline = std::make_shared(policies); diff --git a/sdk/storage/src/blobs/block_blob_client.cpp b/sdk/storage/src/blobs/block_blob_client.cpp index f146c71c8..d5f2ddcc3 100644 --- a/sdk/storage/src/blobs/block_blob_client.cpp +++ b/sdk/storage/src/blobs/block_blob_client.cpp @@ -41,9 +41,7 @@ namespace Azure { namespace Storage { namespace Blobs { { } - BlockBlobClient::BlockBlobClient(BlobClient blobClient) : BlobClient(std::move(blobClient)) - { - } + BlockBlobClient::BlockBlobClient(BlobClient blobClient) : BlobClient(std::move(blobClient)) {} BlockBlobClient BlockBlobClient::WithSnapshot(const std::string& snapshot) { diff --git a/sdk/storage/src/common/common_headers_request_policy.cpp b/sdk/storage/src/common/common_headers_request_policy.cpp new file mode 100644 index 000000000..5d81fd5a3 --- /dev/null +++ b/sdk/storage/src/common/common_headers_request_policy.cpp @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "common/common_headers_request_policy.hpp" + +#include + +namespace Azure { namespace Storage { + + std::unique_ptr CommonHeadersRequestPolicy::Send( + Core::Context& ctx, + Core::Http::Request& request, + Core::Http::NextHttpPolicy nextHttpPolicy) const + { + + const auto& headers = request.GetHeaders(); + if (headers.find("Date") == headers.end() && headers.find("x-ms-date") == headers.end()) + { + // add x-ms-date header in RFC1123 format + // TODO: call helper function provided by Azure Core when they provide one. + time_t t = std::time(nullptr); + struct tm ct; +#ifdef _WIN32 + gmtime_s(&ct, &t); +#else + gmtime_r(&t, &ct); +#endif + char dateString[128]; + strftime(dateString, sizeof(dateString), "%a, %d %b %Y %H:%M:%S GMT", &ct); + request.AddHeader("x-ms-date", dateString); + } + + return nextHttpPolicy.Send(ctx, request); + } + +}} // namespace Azure::Storage diff --git a/sdk/storage/src/common/crypt.cpp b/sdk/storage/src/common/crypt.cpp new file mode 100644 index 000000000..da0141d2b --- /dev/null +++ b/sdk/storage/src/common/crypt.cpp @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "common/crypt.hpp" + +#include "common/storage_common.hpp" + +#ifdef _WIN32 +#include +#include +#else +#include +#include +#include +#include +#endif + +#include + +namespace Azure { namespace Storage { + +#ifdef _WIN32 + std::string HMAC_SHA256(const std::string& text, const std::string& key) + { + struct AlgorithmProviderInstance + { + BCRYPT_ALG_HANDLE Handle; + std::size_t ContextSize; + std::size_t HashLength; + + AlgorithmProviderInstance() + { + NTSTATUS status = BCryptOpenAlgorithmProvider( + &Handle, BCRYPT_SHA256_ALGORITHM, NULL, BCRYPT_ALG_HANDLE_HMAC_FLAG); + if (!BCRYPT_SUCCESS(status)) + { + throw std::runtime_error("BCryptOpenAlgorithmProvider failed"); + } + DWORD objectLength = 0; + DWORD dataLength = 0; + status = BCryptGetProperty( + Handle, + BCRYPT_OBJECT_LENGTH, + (PBYTE)&objectLength, + sizeof(objectLength), + &dataLength, + 0); + if (!BCRYPT_SUCCESS(status)) + { + throw std::runtime_error("BCryptGetProperty failed"); + } + ContextSize = objectLength; + DWORD hashLength = 0; + status = BCryptGetProperty( + Handle, BCRYPT_HASH_LENGTH, (PBYTE)&hashLength, sizeof(hashLength), &dataLength, 0); + if (!BCRYPT_SUCCESS(status)) + { + throw std::runtime_error("BCryptGetProperty failed"); + } + HashLength = hashLength; + } + + ~AlgorithmProviderInstance() { BCryptCloseAlgorithmProvider(Handle, 0); } + }; + + static AlgorithmProviderInstance AlgorithmProvider; + + std::string context; + context.resize(AlgorithmProvider.ContextSize); + + BCRYPT_HASH_HANDLE hashHandle; + NTSTATUS status = BCryptCreateHash( + AlgorithmProvider.Handle, + &hashHandle, + (PUCHAR)context.data(), + (ULONG)context.size(), + (PUCHAR)key.data(), + (ULONG)key.length(), + 0); + if (!BCRYPT_SUCCESS(status)) + { + throw std::runtime_error("BCryptCreateHash failed"); + } + + status = BCryptHashData(hashHandle, (PBYTE)text.data(), (ULONG)text.length(), 0); + if (!BCRYPT_SUCCESS(status)) + { + throw std::runtime_error("BCryptHashData failed"); + } + + std::string hash; + hash.resize(AlgorithmProvider.HashLength); + status = BCryptFinishHash(hashHandle, (PUCHAR)hash.data(), (ULONG)hash.length(), 0); + if (!BCRYPT_SUCCESS(status)) + { + throw std::runtime_error("BCryptFinishHash failed"); + } + + BCryptDestroyHash(hashHandle); + + return hash; + } + + std::string Base64Encode(const std::string& text) + { + std::string encoded; + // According to RFC 4648, the encoded length should be ceiling(n / 3) * 4 + DWORD encodedLength = DWORD((text.length() + 2) / 3 * 4); + encoded.resize(encodedLength); + + CryptBinaryToStringA( + (BYTE*)text.data(), + (DWORD)text.length(), + CRYPT_STRING_BASE64 | CRYPT_STRING_NOCRLF, + (LPSTR)encoded.data(), + (DWORD*)&encodedLength); + + return encoded; + } + + std::string Base64Decode(const std::string& text) + { + std::string decoded; + // According to RFC 4648, the encoded length should be ceiling(n / 3) * 4, so we can infer an + // upper bound here + DWORD decodedLength = DWORD(text.length() / 4 * 3); + decoded.resize(decodedLength); + + CryptStringToBinaryA( + text.data(), + (DWORD)text.length(), + CRYPT_STRING_BASE64 | CRYPT_STRING_STRICT, + (BYTE*)decoded.data(), + &decodedLength, + NULL, + NULL); + decoded.resize(decodedLength); + return decoded; + } + +#else + + std::string HMAC_SHA256(const std::string& text, const std::string& key) + { + char hash[EVP_MAX_MD_SIZE]; + unsigned int hashLength = 0; + HMAC( + EVP_sha256(), + key.data(), + (int)key.length(), + (const unsigned char*)text.data(), + text.length(), + (unsigned char*)&hash[0], + &hashLength); + + return std::string(hash, hashLength); + } + + std::string Base64Encode(const std::string& text) + { + BIO* bio = BIO_new(BIO_s_mem()); + bio = BIO_push(BIO_new(BIO_f_base64()), bio); + BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); + BIO_write(bio, text.data(), (int)text.length()); + BIO_flush(bio); + BUF_MEM* bufferPtr; + BIO_get_mem_ptr(bio, &bufferPtr); + BIO_set_close(bio, BIO_NOCLOSE); + BIO_free_all(bio); + + return std::string(bufferPtr->data, bufferPtr->length); + } + + std::string Base64Decode(const std::string& text) + { + std::string decoded; + // According to RFC 4648, the encoded length should be ceiling(n / 3) * 4, so we can infer an + // upper bound here + std::size_t maxDecodedLength = text.length() / 4 * 3; + decoded.resize(maxDecodedLength); + + BIO* bio = BIO_new_mem_buf(text.data(), -1); + bio = BIO_push(BIO_new(BIO_f_base64()), bio); + BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); + int decodedLength = BIO_read(bio, &decoded[0], (int)text.length()); + BIO_free_all(bio); + + decoded.resize(decodedLength); + return decoded; + } +#endif + +}} // namespace Azure::Storage diff --git a/sdk/storage/src/common/shared_key_policy.cpp b/sdk/storage/src/common/shared_key_policy.cpp new file mode 100644 index 000000000..e257f5220 --- /dev/null +++ b/sdk/storage/src/common/shared_key_policy.cpp @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "common/shared_key_policy.hpp" + +#include "common/crypt.hpp" +#include "common/storage_url_builder.hpp" + +#include +#include + +namespace Azure { namespace Storage { + std::string SharedKeyPolicy::GetSignature(const Core::Http::Request& request) const + { + std::string string_to_sign; + string_to_sign += Azure::Core::Http::HttpMethodToString(request.GetMethod()) + "\n"; + + const auto& headers = request.GetHeaders(); + for (std::string headerName : + {"Content-Encoding", + "Content-Language", + "Content-Length", + "Content-MD5", + "Content-Type", + "Date", + "If-Modified-Since", + "If-Match", + "If-None-Match", + "If-Unmodified-Since", + "Range"}) + { + auto ite = headers.find(headerName); + if (ite != headers.end()) + { + if (headerName == "Content-Length" && ite->second == "0") + { + // do nothing + } + else + { + string_to_sign += ite->second; + } + } + string_to_sign += "\n"; + } + + // canonicalized headers + const std::string prefix = "x-ms-"; + for (auto ite = headers.lower_bound(prefix); + ite != headers.end() && ite->first.substr(0, prefix.length()) == prefix; + ++ite) + { + std::string key = ite->first; + std::string value = ite->second; + std::transform(key.begin(), key.end(), key.begin(), [](char c) { + return static_cast(std::tolower(static_cast(c))); + }); + string_to_sign += key + ":" + value + "\n"; + } + + // canonicalized resource + UrlBuilder resourceUrl(request.GetEncodedUrl()); + string_to_sign += "/" + m_credential->AccountName + "/" + resourceUrl.GetPath() + "\n"; + for (const auto& query : resourceUrl.GetQuery()) + { + std::string key = query.first; + std::transform(key.begin(), key.end(), key.begin(), [](char c) { + return static_cast(std::tolower(static_cast(c))); + }); + string_to_sign += key + ":" + query.second + "\n"; + } + + // remove last linebreak + string_to_sign.pop_back(); + + return Base64Encode(HMAC_SHA256(string_to_sign, Base64Decode(m_credential->GetAccountKey()))); + } +}} // namespace Azure::Storage diff --git a/sdk/storage/src/common/storage_credential.cpp b/sdk/storage/src/common/storage_credential.cpp index 244eda766..0e020c33a 100644 --- a/sdk/storage/src/common/storage_credential.cpp +++ b/sdk/storage/src/common/storage_credential.cpp @@ -1,10 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // SPDX-License-Identifier: MIT -#include - #include "common/storage_credential.hpp" +#include + namespace Azure { namespace Storage { std::map ParseConnectionString(const std::string& connectionString) diff --git a/sdk/storage/src/common/storage_url_builder.cpp b/sdk/storage/src/common/storage_url_builder.cpp index 9bb921c50..05051a87f 100644 --- a/sdk/storage/src/common/storage_url_builder.cpp +++ b/sdk/storage/src/common/storage_url_builder.cpp @@ -3,10 +3,9 @@ #include "common/storage_url_builder.hpp" -#include -#include #include #include +#include namespace Azure { namespace Storage { @@ -18,19 +17,22 @@ namespace Azure { namespace Storage { auto schemeIter = url.find(schemeEnd); if (schemeIter != std::string::npos) { - std::transform(url.begin(), url.begin() + schemeIter, std::back_inserter(m_scheme), [](char c) { - return static_cast(std::tolower(static_cast(c))); - }); + std::transform( + url.begin(), url.begin() + schemeIter, std::back_inserter(m_scheme), [](char c) { + return static_cast(std::tolower(static_cast(c))); + }); pos = url.begin() + schemeIter + schemeEnd.length(); } - auto hostIter = std::find_if(pos, url.end(), [](char c) { return c == '/' || c == '?' || c == ':'; }); + auto hostIter + = std::find_if(pos, url.end(), [](char c) { return c == '/' || c == '?' || c == ':'; }); m_host = std::string(pos, hostIter); pos = hostIter; if (pos != url.end() && *pos == ':') { - auto port_ite = std::find_if_not(pos + 1, url.end(), [](char c) { return std::isdigit(static_cast(c)); }); + auto port_ite = std::find_if_not( + pos + 1, url.end(), [](char c) { return std::isdigit(static_cast(c)); }); m_port = std::stoi(std::string(pos + 1, port_ite)); pos = port_ite; }