From 3be793d558b19a8397dd225cafbe292965e7aa2b Mon Sep 17 00:00:00 2001 From: Victor Vazquez Date: Wed, 27 Oct 2021 16:20:34 -0700 Subject: [PATCH] [Core] - Support azure-core version without OpenSSL dependency (#2839) * perf test uuid * use win impl for uuid on linux * use c99 base64 encode-decode --- sdk/core/azure-core/CHANGELOG.md | 3 + sdk/core/azure-core/CMakeLists.txt | 1 + sdk/core/azure-core/src/base64.cpp | 533 +++++++++++++++--- sdk/core/azure-core/src/uuid.cpp | 31 +- .../perf/inc/azure/core/test/uuid_test.hpp | 70 +++ .../test/perf/src/azure_core_perf_test.cpp | 5 +- 6 files changed, 552 insertions(+), 91 deletions(-) create mode 100644 sdk/core/azure-core/test/perf/inc/azure/core/test/uuid_test.hpp diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 6c080f72b..80884e3ed 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -16,6 +16,9 @@ ### Other Changes +- Updated `base64` implementation to remove external dependency. +- Updated `Uuid` implementation for Linux to remove external dependency. + ## 1.2.1 (2021-09-02) ### Bugs Fixed diff --git a/sdk/core/azure-core/CMakeLists.txt b/sdk/core/azure-core/CMakeLists.txt index f47afa75d..d7def404d 100644 --- a/sdk/core/azure-core/CMakeLists.txt +++ b/sdk/core/azure-core/CMakeLists.txt @@ -148,6 +148,7 @@ target_link_libraries(azure-core INTERFACE Threads::Threads) if(WIN32) target_link_libraries(azure-core PRIVATE bcrypt crypt32) else() + # Required for Hashing ( md5 and sha ). find_package(OpenSSL REQUIRED) target_link_libraries(azure-core PRIVATE OpenSSL::SSL) endif() diff --git a/sdk/core/azure-core/src/base64.cpp b/sdk/core/azure-core/src/base64.cpp index 784f4c3f9..c2aa06d67 100644 --- a/sdk/core/azure-core/src/base64.cpp +++ b/sdk/core/azure-core/src/base64.cpp @@ -4,93 +4,474 @@ #include "azure/core/base64.hpp" #include "azure/core/platform.hpp" -#if defined(AZ_PLATFORM_WINDOWS) -#include -#elif defined(AZ_PLATFORM_POSIX) -#include -#include -#include -#endif - #include #include +namespace { + +static char const Base64EncodeArray[65] + = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +static char const EncodingPad = '='; +static int8_t const Base64DecodeArray[256] = { + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + 62, + -1, + -1, + -1, + 63, // 62 is placed at index 43 (for +), 63 at index 47 (for /) + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + -1, + -1, + -1, + -1, + -1, + -1, // 52-61 are placed at index 48-57 (for 0-9), 64 at index 61 (for =) + -1, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + -1, + -1, + -1, + -1, + -1, // 0-25 are placed at index 65-90 (for A-Z) + -1, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + -1, + -1, + -1, + -1, + -1, // 26-51 are placed at index 97-122 (for a-z) + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, // Bytes over 122 ('z') are invalid and cannot be decoded. Hence, padding the map with -1, + // which indicates invalid input + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, +}; + +static int32_t Base64Encode(const uint8_t* threeBytes) +{ + int32_t i = (threeBytes[0] << 16) | (threeBytes[1] << 8) | threeBytes[2]; + + int32_t i0 = Base64EncodeArray[i >> 18]; + int32_t i1 = Base64EncodeArray[(i >> 12) & 0x3F]; + int32_t i2 = Base64EncodeArray[(i >> 6) & 0x3F]; + int32_t i3 = Base64EncodeArray[i & 0x3F]; + + return i0 | (i1 << 8) | (i2 << 16) | (i3 << 24); +} + +static int32_t Base64EncodeAndPadOne(const uint8_t* twoBytes) +{ + int32_t i = twoBytes[0] << 16 | (twoBytes[1] << 8); + + int32_t i0 = Base64EncodeArray[i >> 18]; + int32_t i1 = Base64EncodeArray[(i >> 12) & 0x3F]; + int32_t i2 = Base64EncodeArray[(i >> 6) & 0x3F]; + + return i0 | (i1 << 8) | (i2 << 16) | (EncodingPad << 24); +} + +static int32_t Base64EncodeAndPadTwo(const uint8_t* oneByte) +{ + int32_t i = oneByte[0] << 8; + + int32_t i0 = Base64EncodeArray[i >> 10]; + int32_t i1 = Base64EncodeArray[(i >> 4) & 0x3F]; + + return i0 | (i1 << 8) | (EncodingPad << 16) | (EncodingPad << 24); +} + +static void Base64WriteIntAsFourBytes(char* destination, int32_t value) +{ + destination[3] = static_cast((value >> 24) & 0xFF); + destination[2] = static_cast((value >> 16) & 0xFF); + destination[1] = static_cast((value >> 8) & 0xFF); + destination[0] = static_cast(value & 0xFF); +} + +std::string Base64Encode(const std::vector& data) +{ + size_t sourceIndex = 0; + auto inputSize = data.size(); + auto maxEncodedSize = ((inputSize + 2) / 3) * 4; + // Use a string with size to the max possible result + std::string encodedResult(maxEncodedSize, '0'); + // Removing const from the string to update the placeholder string + auto destination = const_cast(encodedResult.data()); + + while (sourceIndex + 3 <= inputSize) + { + int32_t result = Base64Encode(&data[sourceIndex]); + Base64WriteIntAsFourBytes(destination, result); + destination += 4; + sourceIndex += 3; + } + + if (sourceIndex + 1 == inputSize) + { + int32_t result = Base64EncodeAndPadTwo(&data[sourceIndex]); + Base64WriteIntAsFourBytes(destination, result); + destination += 4; + sourceIndex += 1; + } + else if (sourceIndex + 2 == inputSize) + { + int32_t result = Base64EncodeAndPadOne(&data[sourceIndex]); + Base64WriteIntAsFourBytes(destination, result); + destination += 4; + sourceIndex += 2; + } + + return encodedResult; +} + +static int32_t Base64Decode(const char* encodedBytes) +{ + int32_t i0 = encodedBytes[0]; + int32_t i1 = encodedBytes[1]; + int32_t i2 = encodedBytes[2]; + int32_t i3 = encodedBytes[3]; + + i0 = Base64DecodeArray[i0]; + i1 = Base64DecodeArray[i1]; + i2 = Base64DecodeArray[i2]; + i3 = Base64DecodeArray[i3]; + + i0 <<= 18; + i1 <<= 12; + i2 <<= 6; + + i0 |= i3; + i1 |= i2; + + i0 |= i1; + return i0; +} + +static void Base64WriteThreeLowOrderBytes(std::vector::iterator destination, int64_t value) +{ + destination[0] = static_cast(value >> 16); + destination[1] = static_cast(value >> 8); + destination[2] = static_cast(value); +} + +std::vector Base64Decode(const std::string& text) +{ + auto inputSize = text.size(); + if (inputSize < 4) + { + return std::vector(0); + } + + size_t sourceIndex = 0; + auto inputPtr = text.data(); + auto decodedSize = (inputSize / 4) * 3; + + if (inputPtr[inputSize - 2] == EncodingPad) + { + decodedSize -= 2; + } + else if (inputPtr[inputSize - 1] == EncodingPad) + { + decodedSize -= 1; + } + + std::vector destination(decodedSize); + auto destinationPtr = destination.begin(); + + while (sourceIndex + 4 < inputSize) + { + int64_t result = Base64Decode(inputPtr + sourceIndex); + Base64WriteThreeLowOrderBytes(destinationPtr, result); + destinationPtr += 3; + sourceIndex += 4; + } + + // We are guaranteed to have an input with at least 4 bytes at this point, with a size that is a + // multiple of 4. + int64_t i0 = inputPtr[inputSize - 4]; + int64_t i1 = inputPtr[inputSize - 3]; + int64_t i2 = inputPtr[inputSize - 2]; + int64_t i3 = inputPtr[inputSize - 1]; + + i0 = Base64DecodeArray[i0]; + i1 = Base64DecodeArray[i1]; + + i0 <<= 18; + i1 <<= 12; + + i0 |= i1; + + if (i3 != EncodingPad) + { + i2 = Base64DecodeArray[i2]; + i3 = Base64DecodeArray[i3]; + + i2 <<= 6; + + i0 |= i3; + i0 |= i2; + + Base64WriteThreeLowOrderBytes(destinationPtr, i0); + destinationPtr += 3; + } + else if (i2 != EncodingPad) + { + i2 = Base64DecodeArray[i2]; + + i2 <<= 6; + + i0 |= i2; + + destinationPtr[1] = static_cast(i0 >> 8); + destinationPtr[0] = static_cast(i0 >> 16); + destinationPtr += 2; + } + else + { + destinationPtr[0] = static_cast(i0 >> 16); + destinationPtr += 1; + } + + return destination; +} + +} // namespace + namespace Azure { namespace Core { -#if defined(AZ_PLATFORM_WINDOWS) - std::string Convert::Base64Encode(const std::vector& data) { - std::string encoded; - // According to RFC 4648, the encoded length should be ceiling(n / 3) * 4 - DWORD encodedLength = static_cast((data.size() + 2) / 3 * 4); - encoded.resize(encodedLength); - - CryptBinaryToStringA( - reinterpret_cast(data.data()), - static_cast(data.size()), - CRYPT_STRING_BASE64 | CRYPT_STRING_NOCRLF, - static_cast(&encoded[0]), - &encodedLength); - - return encoded; + return ::Base64Encode(data); } std::vector Convert::Base64Decode(const std::string& text) { - std::vector decoded; - // According to RFC 4648, the encoded length should be ceiling(n / 3) * 4, so we can infer an - // upper bound here - DWORD decodedLength = DWORD(text.length() / 4 * 3); - decoded.resize(decodedLength); - - CryptStringToBinaryA( - text.data(), - static_cast(text.length()), - CRYPT_STRING_BASE64 | CRYPT_STRING_STRICT, - reinterpret_cast(decoded.data()), - &decodedLength, - nullptr, - nullptr); - decoded.resize(decodedLength); - return decoded; + return ::Base64Decode(text); } -#elif defined(AZ_PLATFORM_POSIX) - - std::string Convert::Base64Encode(const std::vector& data) - { - BIO* bio = BIO_new(BIO_s_mem()); - bio = BIO_push(BIO_new(BIO_f_base64()), bio); - BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); - BIO_write(bio, data.data(), static_cast(data.size())); - (void)BIO_flush(bio); - BUF_MEM* bufferPtr; - BIO_get_mem_ptr(bio, &bufferPtr); - std::string toReturn(bufferPtr->data, bufferPtr->length); - BIO_free_all(bio); - - return toReturn; - } - - std::vector Convert::Base64Decode(const std::string& text) - { - std::vector decoded; - // According to RFC 4648, the encoded length should be ceiling(n / 3) * 4, so we can infer an - // upper bound here - size_t maxDecodedLength = text.length() / 4 * 3; - decoded.resize(maxDecodedLength); - - BIO* bio = BIO_new_mem_buf(text.data(), -1); - bio = BIO_push(BIO_new(BIO_f_base64()), bio); - BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); - int decodedLength = BIO_read(bio, &decoded[0], static_cast(text.length())); - BIO_free_all(bio); - - decoded.resize(decodedLength); - return decoded; - } - -#endif - }} // namespace Azure::Core diff --git a/sdk/core/azure-core/src/uuid.cpp b/sdk/core/azure-core/src/uuid.cpp index 864e330d0..e0cfecf5a 100644 --- a/sdk/core/azure-core/src/uuid.cpp +++ b/sdk/core/azure-core/src/uuid.cpp @@ -3,13 +3,19 @@ #include "azure/core/uuid.hpp" -#if defined(AZ_PLATFORM_POSIX) -#include //for RAND_bytes -#endif - #include #include +#if defined(AZ_PLATFORM_POSIX) +#include +namespace { +// 64-bit Mersenne Twister by Matsumoto and Nishimura, 2000 +// Used to generate the random numbers for the Uuid. +// The seed is generated with std::random_device. +static thread_local std::mt19937_64 randomGenerator(std::random_device{}()); +} // namespace +#endif + namespace Azure { namespace Core { std::string Uuid::ToString() { @@ -47,22 +53,19 @@ namespace Azure { namespace Core { #if defined(AZ_PLATFORM_WINDOWS) std::random_device rd; +#else + std::uniform_int_distribution distribution; +#endif for (size_t i = 0; i < UuidSize; i += 4) { +#if defined(AZ_PLATFORM_WINDOWS) const uint32_t x = rd(); +#else + const uint32_t x = distribution(randomGenerator); +#endif std::memcpy(uuid + i, &x, 4); } -#elif defined(AZ_PLATFORM_POSIX) - // This static cast is safe since we know Uuid size, which is a const, will always fit an int. - int ret = RAND_bytes(uuid, static_cast(UuidSize)); - if (ret <= 0) - { - abort(); - } -#else - abort(); -#endif // SetVariant to ReservedRFC4122 uuid[8] = (uuid[8] | ReservedRFC4122) & 0x7F; diff --git a/sdk/core/azure-core/test/perf/inc/azure/core/test/uuid_test.hpp b/sdk/core/azure-core/test/perf/inc/azure/core/test/uuid_test.hpp new file mode 100644 index 000000000..eb21a9226 --- /dev/null +++ b/sdk/core/azure-core/test/perf/inc/azure/core/test/uuid_test.hpp @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @file + * @brief Test the Uuid component performance. + * + */ + +#pragma once + +#include +#include + +#include + +namespace Azure { namespace Core { namespace Test { + + /** + * @brief Measure the Uuid object performance. + */ + class UuidTest : public Azure::Perf::PerfTest { + public: + /** + * @brief Construct a new Uuid test. + * + * @param options The test options. + */ + UuidTest(Azure::Perf::TestOptions options) : PerfTest(options) {} + + /** + * @brief Use Uuid to assign and read. + * + */ + void Run(Azure::Core::Context const&) override + { + auto const total = m_options.GetMandatoryOption("count"); + for (auto count = 0; count < total; count++) + { + Azure::Core::Uuid::CreateUuid(); + } + } + + /** + * @brief Define the test options for the test. + * + * @return The list of test options. + */ + std::vector GetTestOptions() override + { + return {{"count", {"-c"}, "The number of uuid objects to be created.", 1, true}}; + } + + /** + * @brief Get the static Test Metadata for the test. + * + * @return Azure::Perf::TestMetadata describing the test. + */ + static Azure::Perf::TestMetadata GetTestMetadata() + { + return { + "UuidTest", + "Measures the overhead of using Uuid objects", + [](Azure::Perf::TestOptions options) { + return std::make_unique(options); + }}; + } + }; + +}}} // namespace Azure::Core::Test diff --git a/sdk/core/azure-core/test/perf/src/azure_core_perf_test.cpp b/sdk/core/azure-core/test/perf/src/azure_core_perf_test.cpp index 7884b0968..880eab258 100644 --- a/sdk/core/azure-core/test/perf/src/azure_core_perf_test.cpp +++ b/sdk/core/azure-core/test/perf/src/azure_core_perf_test.cpp @@ -4,6 +4,7 @@ #include #include "azure/core/test/nullable_test.hpp" +#include "azure/core/test/uuid_test.hpp" #include @@ -11,7 +12,9 @@ int main(int argc, char** argv) { // Create the test list - std::vector tests{Azure::Core::Test::NullableTest::GetTestMetadata()}; + std::vector tests{ + Azure::Core::Test::NullableTest::GetTestMetadata(), + Azure::Core::Test::UuidTest::GetTestMetadata()}; Azure::Perf::Program::Run(Azure::Core::Context::ApplicationContext, tests, argc, argv);