github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fully functioning eventhubs processor. (#5018)

Browse Source 
* Functioning processor; reliability changes to azure core

* Eventhubs test fixes

* Functioning processor; cleaned up some memory leaks

* 

---------

Co-authored-by: Rick Winter <rick.winter@microsoft.com>
This commit is contained in:
Larry Osterman 2023-10-24 10:45:43 -07:00 committed by GitHub
parent 92364dcee1
commit ccddc7f3ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 2559 additions and 867 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

215
CMakePresets.json
View File

@ -18,7 +18,9 @@
},
"vendor": {
"microsoft.com/VisualStudioSettings/CMake/1.0": {
"hostOS": [ "Windows" ]
"hostOS": [
"Windows"
]
}
},
"condition": {
@ -39,7 +41,9 @@
},
"vendor": {
"microsoft.com/VisualStudioSettings/CMake/1.0": {
"hostOS": [ "Windows" ]
"hostOS": [
"Windows"
]
}
},
"condition": {
@ -105,7 +109,6 @@
"strategy": "external"
}
},
{
"name": "x64",
"displayName": "Windows x64",
@ -211,27 +214,29 @@
"description": "Enable Address Sanitizer (Hidden). Note: ASAN can be extremely disruptive, when enabling ASAN, make sure you run your application under the debugger.",
"hidden": true,
"environment": {
"CFLAGS": "/fsanitize=address",
"CXXFLAGS": "/fsanitize=address"
},
"cacheVariables": { "DISABLE_AZURE_CORE_OPENTELEMETRY": true },
"condition": {
"type": "equals",
"lhs": "${hostSystemName}",
"rhs": "Windows"
"CXXFLAGS": "-fsanitize=address",
"CFLAGS": "-fsanitize=address"
}
},
{
"name": "x86-static-debug-tests",
"displayName": "x86 Debug With Tests, static",
"description": "Windows x86 Debug build with Tests configured",
"inherits": [ "x86-static", "debug-build", "enable-tests" ]
"inherits": [
"x86-static",
"debug-build",
"enable-tests"
]
},
{
"name": "x86-static-release-tests",
"displayName": "x86 Release With Tests, static, libcurl",
"description": "Windows x86 Release build with Tests configured",
"inherits": [ "x86-static", "release-build", "enable-tests" ],
"inherits": [
"x86-static",
"release-build",
"enable-tests"
],
"cacheVariables": {
"RUN_LONG_UNIT_TESTS": true
}
@ -240,38 +245,63 @@
"name": "x64-debug-tests",
"displayName": "x64 Debug With Tests",
"description": "Windows x64 Debug build with Tests configured",
"inherits": [ "x64", "debug-build", "enable-tests" ]
"inherits": [
"x64",
"debug-build",
"enable-tests"
]
},
{
"name": "x86-msvc-static-debug-tests",
"displayName": "x86 MSVC Debug With Tests, static",
"description": "Windows x86 MSVC Debug build with Tests configured",
"inherits": [ "x86-msvc-static", "debug-build", "enable-tests" ]
"inherits": [
"x86-msvc-static",
"debug-build",
"enable-tests"
]
},
{
"name": "x64-static-debug-tests",
"displayName": "x64 Debug With Tests, static",
"description": "Windows x64 Debug build with Tests configured",
"inherits": [ "x64-static", "debug-build", "enable-tests" ]
"inherits": [
"x64-static",
"debug-build",
"enable-tests"
]
},
{
"name": "x64-static-debug-tests-curl",
"displayName": "x64 Debug With Tests, static, libcurl",
"description": "Windows x64 Debug build with Tests configured",
"inherits": [ "x64-static", "debug-build", "enable-tests", "curl-transport" ]
"inherits": [
"x64-static",
"debug-build",
"enable-tests",
"curl-transport"
]
},
{
"name": "x64-static-debug-tests-winhttp",
"displayName": "x64 Debug With Tests, static, winhttp",
"description": "Windows x64 Debug build with Tests configured",
"inherits": [ "x64-static", "debug-build", "enable-tests", "winhttp-transport" ]
"inherits": [
"x64-static",
"debug-build",
"enable-tests",
"winhttp-transport"
]
},
{
"name": "x64-static-debug-tests-OpenSSL111",
"displayName": "x64 Debug With Tests, static, OpenSSL 1.1.1, libcurl",
"description": "Windows x64 Debug build with Tests configured on OpenSSL 1.1.1",
"inherits": [ "x64-static", "debug-build", "enable-tests" ],
"inherits": [
"x64-static",
"debug-build",
"enable-tests"
],
"cacheVariables": {
"VCPKG_OVERLAY_PORTS": "${sourceDir}\\vcpkg-custom-ports"
}
@ -290,7 +320,13 @@
{
"name": "x86-release-tests",
"displayName": "x86 Release With Tests (Note: Does not link because it sets BUILD_TRANSPORT_CUSTOM)",
"inherits": [ "x86", "release-build", "enable-tests", "curl-transport", "winhttp-transport" ],
"inherits": [
"x86",
"release-build",
"enable-tests",
"curl-transport",
"winhttp-transport"
],
"cacheVariables": {
"BUILD_TRANSPORT_CUSTOM": true
}
@ -298,7 +334,13 @@
{
"name": "x64-release-tests",
"displayName": "x64 Release With Tests (Note: Does not link because it sets BUILD_TRANSPORT_CUSTOM)",
"inherits": [ "x64", "release-build", "enable-tests", "curl-transport", "winhttp-transport" ],
"inherits": [
"x64",
"release-build",
"enable-tests",
"curl-transport",
"winhttp-transport"
],
"cacheVariables": {
"BUILD_TRANSPORT_CUSTOM": true
}
@ -319,22 +361,50 @@
{
"name": "x86-msvc-static-debug-perftests",
"displayName": "x86 MSVC Debug static With Perf Tests and samples",
"inherits": [ "x86-msvc-static", "debug-build", "enable-tests", "enable-perf", "enable-samples", "curl-transport", "winhttp-transport" ]
"inherits": [
"x86-msvc-static",
"debug-build",
"enable-tests",
"enable-perf",
"enable-samples",
"curl-transport",
"winhttp-transport"
]
},
{
"name": "x64-msvc-static-debug-perftests",
"displayName": "x64 MSVC Debug static With Perf Tests and samples",
"inherits": [ "x64-msvc-static", "debug-build", "enable-tests", "enable-perf", "enable-samples", "curl-transport", "winhttp-transport" ]
"inherits": [
"x64-msvc-static",
"debug-build",
"enable-tests",
"enable-perf",
"enable-samples",
"curl-transport",
"winhttp-transport"
]
},
{
"name": "x64-msvc-static-release-perftests",
"displayName": "x64 MSVC Release static With Perf Tests and samples",
"inherits": [ "x64-msvc-static", "release-build", "enable-tests", "enable-perf", "enable-samples", "curl-transport", "winhttp-transport" ]
"inherits": [
"x64-msvc-static",
"release-build",
"enable-tests",
"enable-perf",
"enable-samples",
"curl-transport",
"winhttp-transport"
]
},
{
"name": "x64-static-release-perftests",
"displayName": "x64 Release With Perf Tests, static",
"inherits": [ "x64-static", "release-build", "enable-perf" ]
"inherits": [
"x64-static",
"release-build",
"enable-perf"
]
},
{
"name": "linux-basic-gcc9",
@ -342,6 +412,7 @@
"description": "Using compilers: C = /usr/bin/gcc-9, CXX = /usr/bin/g++-9",
"binaryDir": "${sourceDir}/out/build/${presetName}",
"generator": "Ninja",
"hidden": true,
"cacheVariables": {
"CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}",
"CMAKE_C_COMPILER": "/usr/bin/gcc-9",
@ -359,6 +430,7 @@
"description": "Using compilers: C = /usr/bin/clang-11, CXX = /usr/bin/clang++-11",
"binaryDir": "${sourceDir}/out/build/${presetName}",
"generator": "Ninja",
"hidden": true,
"cacheVariables": {
"CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}",
"CMAKE_C_COMPILER": "/usr/bin/clang-11",
@ -394,6 +466,7 @@
"description": "Using compilers: C = /usr/bin/cc, CXX = /usr/bin/c++",
"binaryDir": "${sourceDir}/out/build/${presetName}",
"generator": "Ninja",
"hidden": true,
"cacheVariables": {
"CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}",
"CMAKE_C_COMPILER": "/usr/bin/cc",
@ -405,14 +478,39 @@
"rhs": "Linux"
}
},
{
"name": "linux-basic-g++-11",
"displayName": "Linux G++ 11",
"description": "Using compilers: C = /usr/bin/gcc-11, CXX = /usr/bin/g++-11",
"binaryDir": "${sourceDir}/out/build/${presetName}",
"generator": "Ninja",
"hidden": true,
"cacheVariables": {
"CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}",
"CMAKE_C_COMPILER": "/usr/bin/gcc-11",
"CMAKE_CXX_COMPILER": "/usr/bin/g++-11"
},
"condition": {
"type": "equals",
"lhs": "${hostSystemName}",
"rhs": "Linux"
}
},
{
"name": "linux-gcc9-debug",
"displayName": "Linux GCC 9 Debug",
"inherits": [ "linux-basic-gcc9", "debug-build" ]
"inherits": [
"linux-basic-gcc9",
"debug-build"
]
},
{
"name": "linux-gcc9-debug-tests",
"inherits": [ "linux-basic-gcc9", "debug-build", "enable-tests" ],
"inherits": [
"linux-basic-gcc9",
"debug-build",
"enable-tests"
],
"displayName": "Linux GCC 9 Debug+Tests"
},
{
@ -432,22 +530,73 @@
},
{
"name": "linux-clang-13-debug-tests",
"inherits": [ "linux-basic-clang-13", "debug-build", "enable-tests" ],
"inherits": [
"linux-basic-clang-13",
"debug-build",
"enable-tests"
],
"displayName": "Linux clang 13 Debug+Tests"
},
{
"name": "linux-g++-debug",
"displayName": "Linux c++ Debug",
"inherits": [ "linux-basic-g++", "debug-build" ]
"inherits": [
"linux-basic-g++",
"debug-build"
]
},
{
"name": "linux-g++-debug-tests",
"inherits": [ "linux-basic-g++", "debug-build", "enable-tests" ],
"displayName": "Linux c++ Debug+Tests"
"inherits": [
"linux-basic-g++",
"debug-build",
"enable-tests"
],
"displayName": "Linux g++, Debug+Tests"
},
{
"name": "g++-debug-asan-tests",
"inherits": [
"linux-g++-debug-tests",
"enable-address-sanitizer"
],
"displayName": "Linux g++, ASAN+Debug+Tests"
},
{
"name": "linux-g++-debug-tests-samples",
"inherits": [ "linux-basic-g++", "debug-build", "enable-tests", "enable-samples" ],
"inherits": [
"linux-basic-g++",
"debug-build",
"enable-tests",
"enable-samples"
],
"displayName": "Linux c++ Debug+Tests, samples"
},
{
"name": "linux-g++-11-debug",
"displayName": "Linux g++-11 Debug",
"inherits": [
"linux-basic-g++-11",
"debug-build"
]
},
{
"name": "linux-g++-11-debug-tests",
"inherits": [
"linux-basic-g++-11",
"debug-build",
"enable-tests"
],
"displayName": "Linux g++-11 Debug+Tests"
},
{
"name": "linux-g++-11-debug-tests-samples",
"inherits": [
"linux-basic-g++-11",
"debug-build",
"enable-tests",
"enable-samples"
],
"displayName": "Linux c++ Debug+Tests, samples"
},
{

2
sdk/core/azure-core-amqp/inc/azure/core/amqp/claims_based_security.hpp
View File

@ -18,6 +18,7 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
Failed,
InstanceClosed,
};
std::ostream& operator<<(std::ostream& os, CbsOperationResult const& operationResult);
enum class CbsOpenResult
{
@ -26,6 +27,7 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
Error,
Cancelled,
};
std::ostream& operator<<(std::ostream& os, CbsOpenResult const& operationResult);
enum class CbsTokenType
{

12
sdk/core/azure-core-amqp/inc/azure/core/amqp/common/global_state.hpp
View File

@ -3,6 +3,8 @@
#pragma once
#include <azure/core/azure_assert.hpp>
#include <list>
#include <memory>
#include <mutex>
@ -50,5 +52,15 @@ namespace Azure { namespace Core { namespace Amqp { namespace Common { namespace
std::lock_guard<std::mutex> lock(m_pollablesMutex);
m_pollables.remove(pollable);
}
void AssertIdle()
{
std::lock_guard<std::mutex> lock(m_pollablesMutex);
AZURE_ASSERT(m_pollables.empty());
if (!m_pollables.empty())
{
Azure::Core::_internal::AzureNoReturnPath("Global state is not idle.");
}
}
};
}}}}} // namespace Azure::Core::Amqp::Common::_detail

84
sdk/core/azure-core-amqp/src/amqp/claim_based_security.cpp
View File

@ -15,10 +15,6 @@
using namespace Azure::Core::Diagnostics::_internal;
using namespace Azure::Core::Diagnostics;
namespace Azure { namespace Core { namespace _internal {
void UniqueHandleHelper<CBS_INSTANCE_TAG>::FreeAmqpCbs(CBS_HANDLE value) { cbs_destroy(value); }
}}} // namespace Azure::Core::_internal
namespace Azure { namespace Core { namespace Amqp { namespace _detail {
using namespace Azure::Core::Amqp::_internal;
@ -68,21 +64,21 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
auto rv{m_management->Open(context)};
switch (rv)
{
case ManagementOpenStatus::Invalid:
return CbsOpenResult::Invalid;
case ManagementOpenStatus::Invalid: // LCOV_EXCL_LINE
return CbsOpenResult::Invalid; // LCOV_EXCL_LINE
case ManagementOpenStatus::Ok:
return CbsOpenResult::Ok;
case ManagementOpenStatus::Error:
return CbsOpenResult::Error;
case ManagementOpenStatus::Cancelled:
return CbsOpenResult::Cancelled;
case ManagementOpenStatus::Cancelled: // LCOV_EXCL_LINE
return CbsOpenResult::Cancelled; // LCOV_EXCL_LINE
default:
throw std::runtime_error("Unknown return value from Management::Open()");
}
}
else
{
return CbsOpenResult::Error;
return CbsOpenResult::Error; // LCOV_EXCL_LINE
}
}
@ -98,6 +94,7 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
message.SetBody(static_cast<Models::AmqpValue>(token));
message.ApplicationProperties["name"] = static_cast<Models::AmqpValue>(audience);
auto result = m_management->ExecuteOperation(
"put-token",
(tokenType == CbsTokenType::Jwt ? "jwt" : "servicebus.windows.net:sastoken"),
@ -116,27 +113,78 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
CbsOperationResult cbsResult;
switch (result.Status)
{
case ManagementOperationStatus::Invalid:
cbsResult = CbsOperationResult::Invalid;
case ManagementOperationStatus::Invalid: // LCOV_EXCL_LINE
cbsResult = CbsOperationResult::Invalid; // LCOV_EXCL_LINE
break;
case ManagementOperationStatus::Ok:
cbsResult = CbsOperationResult::Ok;
break;
case ManagementOperationStatus::Error:
cbsResult = CbsOperationResult::Error;
case ManagementOperationStatus::Error: // LCOV_EXCL_LINE
cbsResult = CbsOperationResult::Error; // LCOV_EXCL_LINE
break;
case ManagementOperationStatus::FailedBadStatus:
cbsResult = CbsOperationResult::Failed;
case ManagementOperationStatus::FailedBadStatus: // LCOV_EXCL_LINE
cbsResult = CbsOperationResult::Failed; // LCOV_EXCL_LINE
break;
case ManagementOperationStatus::InstanceClosed:
cbsResult = CbsOperationResult::InstanceClosed;
case ManagementOperationStatus::InstanceClosed: // LCOV_EXCL_LINE
cbsResult = CbsOperationResult::InstanceClosed; // LCOV_EXCL_LINE
break;
default:
throw std::runtime_error("Unknown management operation status.");
throw std::runtime_error("Unknown management operation status."); // LCOV_EXCL_LINE
}
Log::Stream(Logger::Level::Error)
<< "CBS PutToken result: " << cbsResult << " status code: " << result.StatusCode
<< " Error: " << result.Error.Description << ".";
return std::make_tuple(cbsResult, result.StatusCode, result.Error.Description);
}
}
std::ostream& operator<<(std::ostream& os, CbsOperationResult const& operationResult)
{
switch (operationResult)
{
case CbsOperationResult::Invalid:
os << "Invalid";
break;
case CbsOperationResult::Ok:
os << "Ok";
break;
case CbsOperationResult::Error:
os << "Error";
break;
case CbsOperationResult::Failed:
os << "Failed";
break;
case CbsOperationResult::InstanceClosed:
os << "InstanceClosed";
break;
default:
os << "Unknown CbsOperationResult."
<< static_cast<std::underlying_type<CbsOperationResult>::type>(operationResult);
}
return os;
}
std::ostream& operator<<(std::ostream& os, CbsOpenResult const& openResult)
{
switch (openResult)
{
case CbsOpenResult::Invalid:
os << "Invalid";
break;
case CbsOpenResult::Ok:
os << "Ok";
break;
case CbsOpenResult::Error:
os << "Error";
break;
case CbsOpenResult::Cancelled:
os << "Cancelled";
break;
default:
os << "Unknown CbsOpenResult."
<< static_cast<std::underlying_type<CbsOpenResult>::type>(openResult);
}
return os;
}
void ClaimsBasedSecurityImpl::OnError(Models::_internal::AmqpError const& error)
{

171
sdk/core/azure-core-amqp/src/amqp/connection.cpp
View File

@ -333,6 +333,21 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
{CONNECTION_STATE_ERROR, "CONNECTION_STATE_ERROR"},
};
std::ostream& operator<<(std::ostream& os, CONNECTION_STATE state)
{
auto val{UamqpConnectionStateToStringMap.find(state)};
if (val == UamqpConnectionStateToStringMap.end())
{
os << "Unknown connection state: "
<< static_cast<std::underlying_type<decltype(state)>::type>(state);
}
else
{
os << val->second << "(" << static_cast<std::underlying_type<CONNECTION_STATE>::type>(state)
<< ")";
}
return os;
}
} // namespace
_internal::ConnectionState ConnectionStateFromCONNECTION_STATE(CONNECTION_STATE state)
@ -352,6 +367,12 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
{
ConnectionImpl* connection = static_cast<ConnectionImpl*>(context);
if (connection->m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "Connection " << connection->m_containerId << " state changed from " << oldState
<< " to " << newState;
}
if (connection->m_eventHandler)
{
if (!connection->m_isClosing)
@ -366,9 +387,7 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
{
// When the connection transitions into the error or end state, it is no longer pollable.
Log::Stream(Logger::Level::Verbose)
<< "Connection " << connection->m_containerId << " state changed to "
<< UamqpConnectionStateToStringMap.at(newState) << " : " << static_cast<int>(newState)
<< std::endl;
<< "Connection " << connection->m_containerId << " state changed to " << newState;
}
connection->SetState(ConnectionStateFromCONNECTION_STATE(newState));
}
@ -405,26 +424,38 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
m_enableAsyncOperation = enable;
if (enable)
{
Log::Stream(Logger::Level::Verbose)
<< "Try to enable async operation on connection: " << this << " ID: " << m_containerId
<< " count: " << m_openCount.load();
if (m_openCount++ == 0)
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "Enabled async operation on connection: " << this << " ID: " << m_containerId;
<< "Try to enable async operation on connection: " << this << " ID: " << m_containerId
<< " count: " << m_openCount.load();
}
if (m_openCount++ == 0)
{
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "Enabled async operation on connection: " << this << " ID: " << m_containerId;
}
Common::_detail::GlobalStateHolder::GlobalStateInstance()->AddPollable(shared_from_this());
}
}
else
{
AZURE_ASSERT_MSG(m_openCount.load() > 0, "Closing async without opening it first.");
Log::Stream(Logger::Level::Verbose)
<< "Try to disable async operation on connection: " << this << " ID: " << m_containerId
<< " count: " << m_openCount.load();
if (--m_openCount == 0)
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "Disabled async operation on connection: " << this << " ID: " << m_containerId;
<< "Try to disable async operation on connection: " << this << " ID: " << m_containerId
<< " count: " << m_openCount.load();
}
if (--m_openCount == 0)
{
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "Disabled async operation on connection: " << this << " ID: " << m_containerId;
}
Common::_detail::GlobalStateHolder::GlobalStateInstance()->RemovePollable(
shared_from_this());
}
@ -433,8 +464,11 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
void ConnectionImpl::Open()
{
Log::Stream(Logger::Level::Verbose)
<< "ConnectionImpl::Open: " << this << " ID: " << m_containerId;
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "ConnectionImpl::Open: " << this << " ID: " << m_containerId;
}
if (connection_open(m_connection.get()))
{
throw std::runtime_error("Could not open connection."); // LCOV_EXCL_LINE
@ -556,13 +590,74 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
return false;
}
std::string ConnectionImpl::GetSecurityToken(
// Ensure that we have a token for the provided audience.
// If we don't, authenticate the audience with the service using the provided session.
// Note that the granularity of
Credentials::AccessToken ConnectionImpl::AuthenticateAudience(
std::shared_ptr<SessionImpl> session,
std::string const& audience,
Azure::Core::Context const& context) const
Azure::Core::Context const& context)
{
if (GetCredential())
{
if (m_tokenStore.find(audience) == m_tokenStore.end())
std::string audienceUrl = audience;
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose) << "Authenticate connection for audience " << audience;
}
// If the audience looks like a URL for AMQP, AMQPS, or SB, we can use the URL as provided.
if ((audience.find("amqps://") != 0) && (audience.find("amqp://") != 0)
&& (audience.find("sb://") != 0))
{
audienceUrl = "amqps://" + GetHost();
// The provided audience may begin with a /, if not, we need to add the separator.
if (audience.front() != '/')
{
audienceUrl += "/";
}
audienceUrl += audience;
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "Initial audience is not URL, using " << audienceUrl;
}
}
std::unique_lock<std::mutex> lock(m_tokenMutex);
// If we have authenticated this audience, we're done and can return success.
auto token = m_tokenStore.find(audienceUrl);
if (token != m_tokenStore.end())
{
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose) << "Using cached token for " << audienceUrl;
}
return token->second;
}
// We've not authenticated this audience.
// Authenticate it with the server
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "No cached token for " << audienceUrl << ", Authenticating.";
}
// Azure::Core::Amqp::_internal::SessionOptions sessionOptions;
// sessionOptions.InitialIncomingWindowSize = std::numeric_limits<int32_t>::max();
// sessionOptions.InitialOutgoingWindowSize = std::numeric_limits<uint16_t>::max();
// auto authenticationSession{std::make_shared<SessionImpl>(
// shared_from_this(), sessionOptions, nullptr)};
auto claimsBasedSecurity
= std::make_shared<ClaimsBasedSecurityImpl>(session /*authenticationSession*/);
auto cbsOpenStatus = claimsBasedSecurity->Open(context);
if (cbsOpenStatus != CbsOpenResult::Ok)
{
throw std::runtime_error("Could not open Claims Based Security object."); // LCOV_EXCL_LINE
}
try
{
Credentials::TokenRequestContext requestContext;
bool isSasToken = IsSasCredential();
@ -571,11 +666,39 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
requestContext.MinimumExpiration = std::chrono::minutes(60);
}
requestContext.Scopes = m_options.AuthenticationScopes;
return GetCredential()->GetToken(requestContext, context).Token;
}
return m_tokenStore.at(audience).Token;
}
return "";
}
auto accessToken{GetCredential()->GetToken(requestContext, context)};
auto result = claimsBasedSecurity->PutToken(
(IsSasCredential() ? CbsTokenType::Sas : CbsTokenType::Jwt),
audienceUrl,
accessToken.Token,
context);
if (std::get<0>(result) != CbsOperationResult::Ok)
{
throw std::runtime_error("Could not put Claims Based Security token."); // LCOV_EXCL_LINE
}
claimsBasedSecurity->Close();
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "Authenticated connection for audience " << audienceUrl << " successfully.";
}
m_tokenStore.emplace(audienceUrl, accessToken);
return accessToken;
}
catch (std::runtime_error const&)
{
// Ensure that the claims based security object is closed before we leave this scope.
claimsBasedSecurity->Close();
throw;
}
}
else
{
Log::Stream(Logger::Level::Verbose) << "No credential, returning empty token.";
// If the connection is unauthenticated, then just return an empty access token.
return {};
}
}
}}}} // namespace Azure::Core::Amqp::_detail

47
sdk/core/azure-core-amqp/src/amqp/management.cpp
View File

@ -84,8 +84,8 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
*/
if (m_options.ManagementNodeName == "$management")
{
m_session->AuthenticateIfNeeded(
m_managementEntityPath + "/" + m_options.ManagementNodeName, context);
m_accessToken = m_session->GetConnection()->AuthenticateAudience(
m_session, m_managementEntityPath + "/" + m_options.ManagementNodeName, context);
}
{
_internal::MessageSenderOptions messageSenderOptions;
@ -152,10 +152,11 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
Models::AmqpMessage messageToSend,
Context const& context)
{
auto token = m_session->GetConnection()->GetSecurityToken(m_managementNodeName, context);
if (!token.empty())
// If the connection is authenticated, include the token in the message.
if (!m_accessToken.Token.empty())
{
messageToSend.ApplicationProperties["security_token"] = Models::AmqpValue{token};
messageToSend.ApplicationProperties["security_token"]
= Models::AmqpValue{m_accessToken.Token};
}
messageToSend.ApplicationProperties.emplace("operation", operationToPerform);
messageToSend.ApplicationProperties.emplace("type", typeOfOperation);
@ -202,10 +203,12 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
if (m_messageSender && m_messageSenderOpen)
{
m_messageSender->Close();
m_messageSenderOpen = false;
}
if (m_messageReceiver && m_messageReceiverOpen)
{
m_messageReceiver->Close();
m_messageReceiverOpen = false;
}
m_isOpen = false;
}
@ -252,9 +255,8 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
case _internal::MessageSenderState::Idle:
case _internal::MessageSenderState::Closing:
case _internal::MessageSenderState::Error:
Log::Stream(Logger::Level::Error)
<< "Message Sender Changed State to " << static_cast<int>(newState)
<< " while management client is opening";
Log::Stream(Logger::Level::Error) << "Message Sender Changed State to " << newState
<< " while management client is opening";
SetState(ManagementState::Closing);
m_openCompleteQueue.CompleteOperation(_internal::ManagementOpenStatus::Error);
break;
@ -269,9 +271,8 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
case _internal::MessageSenderState::Idle:
case _internal::MessageSenderState::Closing:
case _internal::MessageSenderState::Error:
Log::Stream(Logger::Level::Error)
<< "Message Sender Changed State to " << static_cast<int>(newState)
<< " while management client is open";
Log::Stream(Logger::Level::Error) << "Message Sender Changed State to " << newState
<< " while management client is open";
SetState(ManagementState::Closing);
if (m_eventHandler)
{
@ -293,9 +294,8 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
case _internal::MessageSenderState::Opening:
case _internal::MessageSenderState::Error:
// LCOV_EXCL_START
Log::Stream(Logger::Level::Error)
<< "Message Sender Changed State to " << static_cast<int>(newState)
<< " while management client is closing";
Log::Stream(Logger::Level::Error) << "Message Sender Changed State to " << newState
<< " while management client is closing";
SetState(ManagementState::Closing);
if (m_eventHandler)
{
@ -313,7 +313,7 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
case ManagementState::Idle:
case ManagementState::Error:
Log::Stream(Logger::Level::Error)
<< "Message sender state changed to " << static_cast<int>(newState)
<< "Message sender state changed to " << newState
<< " when management client is in the error state, ignoring.";
break;
// LCOV_EXCL_STOP
@ -378,7 +378,8 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
case _internal::MessageReceiverState::Closing:
case _internal::MessageReceiverState::Error:
Log::Stream(Logger::Level::Error)
<< "Message Receiver Changed State to " << static_cast<int>(newState)
<< "Message Receiver Changed State to "
<< static_cast<std::underlying_type<decltype(newState)>::type>(newState)
<< " while management client is opening";
SetState(ManagementState::Closing);
m_openCompleteQueue.CompleteOperation(_internal::ManagementOpenStatus::Error);
@ -396,7 +397,8 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
case _internal::MessageReceiverState::Closing:
case _internal::MessageReceiverState::Error:
Log::Stream(Logger::Level::Error)
<< "Message Sender Changed State to " << static_cast<int>(newState)
<< "Message Sender Changed State to "
<< static_cast<std::underlying_type<decltype(newState)>::type>(newState)
<< " while management client is open";
SetState(ManagementState::Closing);
if (m_eventHandler)
@ -419,7 +421,8 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
case _internal::MessageReceiverState::Opening:
case _internal::MessageReceiverState::Error:
Log::Stream(Logger::Level::Error)
<< "Message Sender Changed State to " << static_cast<int>(newState)
<< "Message Sender Changed State to "
<< static_cast<std::underlying_type<decltype(newState)>::type>(newState)
<< " while management client is closing";
SetState(ManagementState::Closing);
if (m_eventHandler)
@ -437,7 +440,8 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
case ManagementState::Idle:
case ManagementState::Error:
Log::Stream(Logger::Level::Error)
<< "Message sender state changed to " << static_cast<int>(newState)
<< "Message sender state changed to "
<< static_cast<std::underlying_type<decltype(newState)>::type>(newState)
<< " when management client is in the error state, ignoring.";
break;
// LCOV_EXCL_STOP
@ -531,7 +535,10 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
}
if (!m_sendCompleted)
{
Log::Stream(Logger::Level::Error) << "Received message before send completed.";
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Informational) << "Received message before send completed.";
}
}
Models::_internal::AmqpError messageError;

147
sdk/core/azure-core-amqp/src/amqp/message_receiver.cpp
View File

@ -41,19 +41,55 @@ namespace Azure { namespace Core { namespace Amqp { namespace _internal {
MessageReceiver::~MessageReceiver() noexcept {}
void MessageReceiver::Open(Azure::Core::Context const& context) { m_impl->Open(context); }
void MessageReceiver::Close() { m_impl->Close(); }
void MessageReceiver::Open(Azure::Core::Context const& context)
{
if (m_impl)
{
m_impl->Open(context);
}
else
{
AZURE_ASSERT_FALSE("MessageReceiver::Open called on moved message receiver.");
}
}
void MessageReceiver::Close()
{
if (m_impl)
{
m_impl->Close();
}
}
std::string MessageReceiver::GetSourceName() const { return m_impl->GetSourceName(); }
std::pair<Azure::Nullable<Models::AmqpMessage>, Models::_internal::AmqpError>
MessageReceiver::WaitForIncomingMessage(Azure::Core::Context const& context)
{
return m_impl->WaitForIncomingMessage(context);
if (m_impl)
{
return m_impl->WaitForIncomingMessage(context);
}
else
{
AZURE_ASSERT_FALSE(
"MessageReceiver::WaitForIncomingMessage called on moved message receiver.");
Azure::Core::_internal::AzureNoReturnPath(
"MessageReceiver::WaitForIncomingMessage called on moved message receiver.");
}
}
std::pair<Azure::Nullable<Models::AmqpMessage>, Models::_internal::AmqpError>
MessageReceiver::TryWaitForIncomingMessage()
{
return m_impl->TryWaitForIncomingMessage();
if (m_impl)
{
return m_impl->TryWaitForIncomingMessage();
}
else
{
AZURE_ASSERT_FALSE(
"MessageReceiver::TryWaitForIncomingMessage called on moved message receiver.");
Azure::Core::_internal::AzureNoReturnPath(
"MessageReceiver::TryWaitForIncomingMessage called on moved message receiver.");
}
}
std::string MessageReceiver::GetLinkName() const { return m_impl->GetLinkName(); }
@ -150,8 +186,8 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
<< "Message receiver link detached: " + error.Condition.ToString() << ": "
<< error.Description;
// Cache the error we received in the OnDetach notification so we can return it to the user
// on the next send which fails.
// Cache the error we received in the OnDetach notification so we can return it to the
// user on the next receive which fails.
m_savedMessageError = error;
}
});
@ -334,8 +370,15 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
MESSAGE_RECEIVER_STATE oldState)
{
auto receiver = static_cast<MessageReceiverImpl*>(const_cast<void*>(context));
// If the message receiver isn't open, or if it's in the process of being destroyed, ignore this
// notification.
receiver->m_currentState = MessageReceiverStateFromLowLevel(newState);
if (receiver->m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "Message receiver state change " << oldState << " -> " << newState;
}
// If the message receiver isn't open, or if it's in the process of being destroyed, ignore
// this notification.
if (receiver->m_receiverOpen)
{
if (receiver->m_eventHandler)
@ -370,6 +413,17 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
receiver->m_messageQueue.CompleteOperation(nullptr, error);
}
}
#if RECEIVER_SYNCHRONOUS_CLOSE
// When we transition from the closing to idle state, we can return from the close
// operation.
if (oldState == MESSAGE_RECEIVER_STATE_CLOSING && newState == MESSAGE_RECEIVER_STATE_IDLE)
{
Log::Stream(Logger::Level::Informational)
<< "Message receiver state changed from closing to idle. Receiver closed.";
receiver->m_closeQueue.CompleteOperation(Models::_internal::AmqpError{});
}
#endif
}
}
@ -377,9 +431,12 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
{
if (m_options.AuthenticationRequired)
{
m_session->AuthenticateIfNeeded(static_cast<std::string>(m_source.GetAddress()), context);
m_session->GetConnection()->AuthenticateAudience(
m_session, static_cast<std::string>(m_source.GetAddress()), context);
}
auto lock{m_session->GetConnection()->Lock()};
// Once we've authenticated the connection, establish the link and receiver.
// We cannot do this before authenticating the client.
if (!m_link)
@ -411,7 +468,10 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
}
m_receiverOpen = true;
Log::Stream(Logger::Level::Verbose) << "Opening message receiver. Start async";
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose) << "Opening message receiver. Start async";
}
// Mark the connection as async so that we can use the async APIs.
m_session->GetConnection()->EnableAsyncOperation(true);
}
@ -420,15 +480,23 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
{
if (m_receiverOpen)
{
Log::Stream(Logger::Level::Verbose) << "Lock for Closing message receiver.";
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose) << "Lock for Closing message receiver.";
}
auto lock{m_session->GetConnection()->Lock()};
AZURE_ASSERT(m_link);
Log::Stream(Logger::Level::Verbose) << "Receiver unsubscribe from link detach event.";
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose) << "Receiver unsubscribe from link detach event.";
}
m_link->UnsubscribeFromDetachEvent();
Log::Stream(Logger::Level::Verbose) << "Closing message receiver. Stop async";
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose) << "Closing message receiver. Stop async";
}
m_session->GetConnection()->EnableAsyncOperation(false);
// Clear messages from the queue.
@ -437,18 +505,43 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
{
throw std::runtime_error("Could not close message receiver"); // LCOV_EXCL_LINE
}
m_receiverOpen = false;
}
}
std::string MessageReceiverImpl::GetLinkName() const
{
const char* linkName;
if (messagereceiver_get_link_name(m_messageReceiver.get(), &linkName))
{
throw std::runtime_error("Could not get link name");
}
return std::string(linkName);
}
// Release the lock so that the polling thread can make forward progress delivering the
// detach notification.
#if RECEIVER_SYNCHRONOUS_CLOSE
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "Wait for receiver detach to complete. Current state: " << m_currentState;
}
if (m_currentState == MessageReceiverState::Open
|| m_currentState == MessageReceiverState::Closing)
{
lock.unlock();
// At this point, the underlying link is in the "half closed" state.
// We need to wait for the link to be fully closed before we can destroy it.
auto closeResult = m_closeQueue.WaitForResult(context);
if (!closeResult)
{
throw Azure::Core::OperationCancelledException(
"MessageReceiver close operation was cancelled.");
}
}
#endif
m_receiverOpen = false;
}
}
std::string MessageReceiverImpl::GetLinkName() const
{
const char* linkName;
if (messagereceiver_get_link_name(m_messageReceiver.get(), &linkName))
{
throw std::runtime_error("Could not get link name");
}
return std::string(linkName);
}
}}}} // namespace Azure::Core::Amqp::_detail

81
sdk/core/azure-core-amqp/src/amqp/message_sender.cpp
View File

@ -220,6 +220,13 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
}
}
std::ostream& operator<<(std::ostream& os, MESSAGE_SENDER_STATE state)
{
os << MessageSenderStateFromLowLevel(state) << "("
<< static_cast<std::underlying_type<MESSAGE_SENDER_STATE>::type>(state) << ")";
return os;
}
void MessageSenderImpl::OnMessageSenderStateChangedFn(
void* context,
MESSAGE_SENDER_STATE newState,
@ -230,6 +237,12 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
if (newState != oldState)
{
auto sender = static_cast<MessageSenderImpl*>(const_cast<void*>(context));
sender->m_currentState = MessageSenderStateFromLowLevel(newState);
if (sender->m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "Message sender state changed from " << oldState << " to " << newState << ".";
}
if (sender->m_events)
{
sender->m_events->OnMessageSenderStateChanged(
@ -245,17 +258,29 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
"Message Sender unexpectedly entered the Error State.",
{}});
}
#if SENDER_SYNCHRONOUS_CLOSE
if (oldState == MESSAGE_SENDER_STATE_CLOSING && newState == MESSAGE_SENDER_STATE_IDLE)
{
sender->m_closeQueue.CompleteOperation(Models::_internal::AmqpError{});
}
#endif
}
}
void MessageSenderImpl::Open(Context const& context)
{
Log::Stream(Logger::Level::Verbose) << "Opening message sender. Authenticate if needed.";
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "Opening message sender. Authenticate if needed with audience: " << m_target;
}
if (m_options.AuthenticationRequired)
{
// If we need to authenticate with either ServiceBus or BearerToken, now is the time to do
// it.
m_session->AuthenticateIfNeeded(static_cast<std::string>(m_target.GetAddress()), context);
m_session->GetConnection()->AuthenticateAudience(
m_session, static_cast<std::string>(m_target.GetAddress()), context);
}
auto lock{m_session->GetConnection()->Lock()};
@ -285,7 +310,10 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
}
// Mark the connection as async so that we can use the async APIs.
Log::Stream(Logger::Level::Verbose) << "Opening message sender. Enable async operation.";
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose) << "Opening message sender. Enable async operation.";
}
m_session->GetConnection()->EnableAsyncOperation(true);
m_senderOpen = true;
}
@ -294,19 +322,56 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
{
if (m_senderOpen)
{
Log::Stream(Logger::Level::Verbose) << "Lock for Closing message sender.";
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose) << "Lock for Closing message sender.";
}
auto lock{m_session->GetConnection()->Lock()};
Log::Stream(Logger::Level::Verbose) << "Closing message sender.";
m_session->GetConnection()->EnableAsyncOperation(false);
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose) << "Closing message sender.";
Log::Stream(Logger::Level::Verbose) << "Unsubscribe from link detach event.";
}
m_link->UnsubscribeFromDetachEvent();
#if SENDER_SYNCHRONOUS_CLOSE
bool shouldWaitForClose = m_currentState == _internal::MessageSenderState::Closing
|| m_currentState == _internal::MessageSenderState::Open;
#endif
// Log::Stream(Logger::Level::Verbose) << "Unsubscribe from link detach event.";
// m_link->UnsubscribeFromDetachEvent();
m_session->GetConnection()->EnableAsyncOperation(false);
if (messagesender_close(m_messageSender.get()))
{
throw std::runtime_error("Could not close message sender"); // LCOV_EXCL_LINE
}
#if SENDER_SYNCHRONOUS_CLOSE
if (m_options.EnableTrace)
{
Log::Stream(Logger::Level::Verbose)
<< "Wait for sender detach to complete. Current state: " << m_currentState;
}
// The message sender (and it's underlying link) is in the half open state. Wait until the
// link has fully closed.
if (shouldWaitForClose && false)
{
lock.unlock();
auto result = m_closeQueue.WaitForResult(context);
if (!result)
{
throw Azure::Core::OperationCancelledException(
"Message sender close operation cancelled.");
}
if (std::get<0>(*result))
{
throw std::runtime_error("Error closing message sender"); // LCOV_EXCL_LINE
}
}
#endif
m_senderOpen = false;
}
}

22
sdk/core/azure-core-amqp/src/amqp/private/connection_impl.hpp
View File

@ -114,8 +114,17 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
bool IsAsyncOperation() { return m_enableAsyncOperation; }
bool IsTraceEnabled() { return m_options.EnableTrace; }
bool IsSasCredential() const;
std::string GetSecurityToken(std::string const& audience, Azure::Core::Context const& context)
const;
// Authenticate the audience on this connection using the provided session.
Azure::Core::Credentials::AccessToken AuthenticateAudience(
std::shared_ptr<SessionImpl> session,
std::string const& audience,
Azure::Core::Context const& context);
// Credentials::AccessToken GetSecurityToken(std::string const& audience,
// Azure::Core::Context const& context)
// const;
// void SetToken(std::string const& audience, Credentials::AccessToken const& token);
using LockType = std::recursive_mutex;
@ -135,14 +144,19 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
m_newSessionQueue;
_internal::ConnectionEvents* m_eventHandler{};
_internal::ConnectionState m_connectionState = _internal::ConnectionState::Start;
std::shared_ptr<Credentials::TokenCredential> m_credential{};
std::map<std::string, Credentials::AccessToken> m_tokenStore;
LockType m_amqpMutex;
bool m_enableAsyncOperation = false;
bool m_isClosing = false;
bool m_connectionOpened = false;
std::atomic<uint32_t> m_openCount{0};
// mutex protecting the token acquisition process.
std::mutex m_tokenMutex;
std::shared_ptr<Credentials::TokenCredential> m_credential{};
std::map<std::string, Credentials::AccessToken> m_tokenStore;
ConnectionImpl(
_internal::ConnectionEvents* eventHandler,
_internal::ConnectionOptions const& options);

2
sdk/core/azure-core-amqp/src/amqp/private/management_impl.hpp
View File

@ -99,12 +99,12 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
std::string const& errorCondition,
std::string const& errorDescription);
std::string m_managementNodeName;
_internal::ManagementClientOptions m_options;
std::string m_source;
std::shared_ptr<SessionImpl> m_session;
_internal::ManagementClientEvents* m_eventHandler{};
std::string m_managementEntityPath;
Azure::Core::Credentials::AccessToken m_accessToken;
Azure::Core::Amqp::Common::_internal::AsyncOperationQueue<
_internal::ManagementOperationStatus,

11
sdk/core/azure-core-amqp/src/amqp/private/message_receiver_impl.hpp
View File

@ -19,6 +19,8 @@
#include <memory>
#include <vector>
#define RECEIVER_SYNCHRONOUS_CLOSE 0
namespace Azure { namespace Core { namespace _internal {
template <> struct UniqueHandleHelper<MESSAGE_RECEIVER_INSTANCE_TAG>
{
@ -82,11 +84,20 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
Models::_internal::MessageSource m_source;
std::shared_ptr<_detail::SessionImpl> m_session;
Models::_internal::AmqpError m_savedMessageError{};
_internal::MessageReceiverState m_currentState{};
Azure::Core::Amqp::Common::_internal::
AsyncOperationQueue<Models::AmqpMessage, Models::_internal::AmqpError>
m_messageQueue;
#if RECEIVER_SYNCHRONOUS_CLOSE
// When we close a uAMQP messagereceiver, the link is left in the half closed state. We need to
// wait for the link to be fully closed before we can close the session. This queue will hold
// the close operation until the link is fully closed.
Azure::Core::Amqp::Common::_internal::AsyncOperationQueue<Models::_internal::AmqpError>
m_closeQueue;
#endif
_internal::MessageReceiverEvents* m_eventHandler{};
static AMQP_VALUE OnMessageReceivedFn(const void* context, MESSAGE_HANDLE message);

8
sdk/core/azure-core-amqp/src/amqp/private/message_sender_impl.hpp
View File

@ -11,6 +11,8 @@
#include <tuple>
#define SENDER_SYNCHRONOUS_CLOSE 0
namespace Azure { namespace Core { namespace _internal {
template <> struct UniqueHandleHelper<MESSAGE_SENDER_INSTANCE_TAG>
{
@ -84,6 +86,12 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
Models::_internal::AmqpError>
m_sendCompleteQueue;
#if SENDER_SYNCHRONOUS_CLOSE
Azure::Core::Amqp::Common::_internal::AsyncOperationQueue<Models::_internal::AmqpError>
m_closeQueue;
#endif
_internal::MessageSenderState m_currentState{};
std::shared_ptr<_detail::SessionImpl> m_session;
Models::_internal::MessageTarget m_target;
_internal::MessageSenderOptions m_options;

4
sdk/core/azure-core-amqp/src/amqp/private/session_impl.hpp
View File

@ -67,8 +67,6 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
void Begin();
void End(std::string const& condition_value, std::string const& description);
void AuthenticateIfNeeded(std::string const& audience, Context const& context);
private:
SessionImpl();
std::shared_ptr<_detail::ConnectionImpl> m_connectionToPoll;
@ -76,8 +74,6 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
_internal::SessionOptions m_options;
_internal::SessionEvents* m_eventHandler{};
void Authenticate(std::string const& audience, Context const& context);
static bool OnLinkAttachedFn(
void* context,
LINK_ENDPOINT_HANDLE newLinkEndpoint,

36
sdk/core/azure-core-amqp/src/amqp/session.cpp
View File

@ -222,42 +222,6 @@ namespace Azure { namespace Core { namespace Amqp { namespace _detail {
}
}
void SessionImpl::AuthenticateIfNeeded(std::string const& audience, Context const& context)
{
if (GetConnection()->GetCredential())
{
std::string tokenAudience = audience;
// If the caller provided a URL, use that url, otherwise build the URL to be used.
if ((audience.find("amqps://") != 0) && (audience.find("amqp://") != 0))
{
tokenAudience = "amqps://" + m_connectionToPoll->GetHost() + "/" + audience;
}
Authenticate(tokenAudience, context);
}
}
void SessionImpl::Authenticate(std::string const& audience, Context const& context)
{
auto claimsBasedSecurity = std::make_shared<ClaimsBasedSecurityImpl>(shared_from_this());
auto accessToken = GetConnection()->GetSecurityToken(audience, context);
auto cbsOpenStatus = claimsBasedSecurity->Open(context);
if (cbsOpenStatus != CbsOpenResult::Ok)
{
throw std::runtime_error("Could not open Claims Based Security object."); // LCOV_EXCL_LINE
}
auto result = claimsBasedSecurity->PutToken(
(GetConnection()->IsSasCredential() ? CbsTokenType::Sas : CbsTokenType::Jwt),
audience,
accessToken,
context);
if (std::get<0>(result) != CbsOperationResult::Ok)
{
throw std::runtime_error("Could not put Claims Based Security token."); // LCOV_EXCL_LINE
}
claimsBasedSecurity->Close();
}
bool SessionImpl::OnLinkAttachedFn(
void* context,
LINK_ENDPOINT_INSTANCE_TAG* newLinkEndpoint,

6
sdk/core/azure-core-amqp/src/models/amqp_value.cpp
View File

@ -7,6 +7,8 @@
#include "azure/core/amqp/models/amqp_properties.hpp"
#include "azure/core/amqp/models/amqp_protocol.hpp"
#include <azure/core/internal/diagnostics/log.hpp>
#include <azure_uamqp_c/amqp_definitions_milliseconds.h>
#include <azure_uamqp_c/amqp_definitions_sequence_no.h>
@ -20,6 +22,8 @@
#include <sstream>
using namespace Azure::Core::Amqp::Models::_detail;
using namespace Azure::Core::Diagnostics::_internal;
using namespace Azure::Core::Diagnostics;
namespace Azure { namespace Core { namespace _internal {
void UniqueHandleHelper<AMQP_VALUE_DATA_TAG>::FreeAmqpValue(AMQP_VALUE value)
@ -511,7 +515,7 @@ namespace Azure { namespace Core { namespace Amqp { namespace Models {
if (initializer.size())
{
AmqpValueType expectedType = initializer.begin()->GetType();
for (auto v : initializer)
for (auto const& v : initializer)
{
if (v.GetType() != expectedType)
{

51
sdk/core/azure-core-amqp/test/ut/amqp_value_tests.cpp
View File

@ -340,28 +340,41 @@ TEST_F(TestValues, TestMap)
TEST_F(TestValues, TestArray)
{
{
AmqpArray array1{1, 3, 5, 4, 553991123};
AmqpArray array1{1, 3, 5, 4, 553991123};
EXPECT_EQ(5, array1.size());
EXPECT_EQ(5, array1.size());
AmqpValue value = static_cast<AmqpValue>(array1);
EXPECT_EQ(AmqpValueType::Array, value.GetType());
AmqpValue value = static_cast<AmqpValue>(array1);
EXPECT_EQ(AmqpValueType::Array, value.GetType());
const AmqpArray array2 = value.AsArray();
EXPECT_EQ(5, array2.size());
EXPECT_EQ(1, static_cast<std::int32_t>(array2.at(0)));
EXPECT_EQ(3, static_cast<std::int32_t>(array2.at(1)));
EXPECT_EQ(5, static_cast<std::int32_t>(array2.at(2)));
EXPECT_FALSE(array1 < array2);
EXPECT_FALSE(value < AmqpValue(static_cast<_detail::UniqueAmqpValueHandle>(array2).get()));
}
{
// Because EXPECT_ANY_THROW is a macro, the commas in the lambda below confuse the
// preprocessor. So explicitly capture the lambda and then execute it in the EXPECT_ANY_THROW.
auto v = []() { AmqpArray testArray{3.1, 2.9, 14}; };
EXPECT_ANY_THROW(v());
}
const AmqpArray array2 = value.AsArray();
EXPECT_EQ(5, array2.size());
EXPECT_EQ(1, static_cast<std::int32_t>(array2.at(0)));
EXPECT_EQ(3, static_cast<std::int32_t>(array2.at(1)));
EXPECT_EQ(5, static_cast<std::int32_t>(array2.at(2)));
EXPECT_FALSE(array1 < array2);
EXPECT_FALSE(value < AmqpValue(static_cast<_detail::UniqueAmqpValueHandle>(array2).get()));
}
TEST_F(TestValues, TestArray1)
{
AmqpArray array1{1};
EXPECT_EQ(1, array1.size());
AmqpValue value = static_cast<AmqpValue>(array1);
EXPECT_EQ(AmqpValueType::Array, value.GetType());
GTEST_LOG_(INFO) << "Copy AMQP value as array to a new value";
const AmqpArray array2 = value.AsArray();
}
TEST_F(TestValues, TestArrayDifferentTypes)
{
// Because EXPECT_ANY_THROW is a macro, the commas in the lambda below confuse the
// preprocessor. So explicitly capture the lambda and then execute it in the EXPECT_ANY_THROW.
auto v = []() { AmqpArray testArray{3.1, 2.9, 14}; };
EXPECT_ANY_THROW(v());
}
TEST_F(TestValues, TestChar)

14
sdk/core/azure-core-amqp/test/ut/async_operation_queue_tests.cpp
View File

@ -51,7 +51,6 @@ TEST_F(TestAsyncQueue, TryReadFromQueue)
{
AsyncOperationQueue<int> queue;
std::unique_ptr<std::tuple<int>> item;
Azure::Core::Context context;
item = queue.TryWaitForResult();
EXPECT_FALSE(item);
}
@ -61,9 +60,20 @@ TEST_F(TestAsyncQueue, TryReadFromQueue)
AsyncOperationQueue<int> queue;
queue.CompleteOperation(25);
std::unique_ptr<std::tuple<int>> item;
Azure::Core::Context context;
item = queue.TryWaitForResult();
EXPECT_TRUE(item);
EXPECT_EQ(25, std::get<0>(*item));
}
}
TEST_F(TestAsyncQueue, ReadCanceled)
{
{
AsyncOperationQueue<int> queue;
Azure::Core::Context context;
context.Cancel();
auto item = queue.WaitForResult(context);
EXPECT_FALSE(item);
}
}

15
sdk/core/azure-core-amqp/test/ut/claim_based_security_tests.cpp
View File

@ -46,6 +46,21 @@ namespace Azure { namespace Core { namespace Amqp { namespace Tests {
ClaimsBasedSecurity cbs1(session);
ClaimsBasedSecurity cbs2(session);
}
{
GTEST_LOG_(INFO) << "CbsOperations" << CbsOperationResult::Error;
GTEST_LOG_(INFO) << "CbsOperations" << CbsOperationResult::Invalid;
GTEST_LOG_(INFO) << "CbsOperations" << CbsOperationResult::Failed;
GTEST_LOG_(INFO) << "CbsOperations" << CbsOperationResult::InstanceClosed;
GTEST_LOG_(INFO) << "CbsOperations" << static_cast<CbsOperationResult>(32768);
}
{
GTEST_LOG_(INFO) << "CbsOpens" << CbsOpenResult::Cancelled;
GTEST_LOG_(INFO) << "CbsOpens" << CbsOpenResult::Error;
GTEST_LOG_(INFO) << "CbsOpens" << CbsOpenResult::Ok;
GTEST_LOG_(INFO) << "CbsOpens" << CbsOpenResult::Invalid;
GTEST_LOG_(INFO) << "CbsOpens" << static_cast<CbsOpenResult>(32768);
}
}
#endif // !defined(AZ_PLATFORM_MAC)

12
sdk/core/azure-core-amqp/test/ut/transport_tests.cpp
View File

@ -65,12 +65,12 @@ namespace Azure { namespace Core { namespace Amqp { namespace Tests {
{
{
class TestTransportEvents : public TransportEvents {
AsyncOperationQueue<size_t, std::unique_ptr<uint8_t>> receiveBytesQueue;
AsyncOperationQueue<size_t, std::unique_ptr<uint8_t[]>> receiveBytesQueue;
AsyncOperationQueue<bool> errorQueue;
void OnBytesReceived(Transport const&, uint8_t const* bytes, size_t size) override
{
GTEST_LOG_(INFO) << "On bytes received: " << size;
std::unique_ptr<uint8_t> val(new uint8_t[size]);
std::unique_ptr<uint8_t[]> val(new uint8_t[size]);
memcpy(val.get(), bytes, size);
receiveBytesQueue.CompleteOperation(size, std::move(val));
}
@ -81,7 +81,7 @@ namespace Azure { namespace Core { namespace Amqp { namespace Tests {
}
public:
std::tuple<size_t, std::unique_ptr<uint8_t>> WaitForReceive(
std::tuple<size_t, std::unique_ptr<uint8_t[]>> WaitForReceive(
Transport const& transport,
Azure::Core::Context const& context)
{
@ -176,12 +176,12 @@ Accept: */*
{
{
class TestTransportEvents : public TransportEvents {
AsyncOperationQueue<size_t, std::unique_ptr<uint8_t>> receiveBytesQueue;
AsyncOperationQueue<size_t, std::unique_ptr<uint8_t[]>> receiveBytesQueue;
AsyncOperationQueue<bool> errorQueue;
void OnBytesReceived(Transport const&, uint8_t const* bytes, size_t size) override
{
GTEST_LOG_(INFO) << "On bytes received: " << size;
std::unique_ptr<uint8_t> val(new uint8_t[size]);
std::unique_ptr<uint8_t[]> val(new uint8_t[size]);
memcpy(val.get(), bytes, size);
receiveBytesQueue.CompleteOperation(size, std::move(val));
}
@ -192,7 +192,7 @@ Accept: */*
}
public:
std::tuple<size_t, std::unique_ptr<uint8_t>> WaitForReceive(
std::tuple<size_t, std::unique_ptr<uint8_t[]>> WaitForReceive(
Transport const& transport,
Azure::Core::Context const& context)
{

3
sdk/core/azure-core/src/environment_log_level_listener.cpp
View File

@ -125,7 +125,8 @@ EnvironmentLogLevelListener::GetLogListener()
<< Azure::DateTime(std::chrono::system_clock::now())
.ToString(
DateTime::DateFormat::Rfc3339, DateTime::TimeFractionFormat::AllDigits)
<< "] " << LogLevelToConsoleString(level) << " : " << message;
<< " T: " << std::hex << std::this_thread::get_id() << std::dec << "] "
<< LogLevelToConsoleString(level) << " : " << message;
// If the message ends with a new line, flush the stream otherwise insert a new line to
// terminate the message.

2
sdk/eventhubs/azure-messaging-eventhubs/CMakeLists.txt
View File

@ -68,7 +68,6 @@ set(
inc/azure/messaging/eventhubs/models/processor_models.hpp
inc/azure/messaging/eventhubs/partition_client.hpp
inc/azure/messaging/eventhubs/processor.hpp
inc/azure/messaging/eventhubs/processor_load_balancer.hpp
inc/azure/messaging/eventhubs/processor_partition_client.hpp
inc/azure/messaging/eventhubs/producer_client.hpp
inc/azure/messaging/eventhubs/rtti.hpp
@ -85,6 +84,7 @@ set(
src/private/package_version.hpp
src/private/eventhubs_constants.hpp
src/private/eventhubs_utilities.hpp
src/processor.cpp
src/processor_load_balancer.cpp
src/processor_partition_client.cpp
src/producer_client.cpp

1
sdk/eventhubs/azure-messaging-eventhubs/inc/azure/messaging/eventhubs.hpp
View File

@ -21,6 +21,5 @@
#include "azure/messaging/eventhubs/models/processor_models.hpp"
#include "azure/messaging/eventhubs/partition_client.hpp"
#include "azure/messaging/eventhubs/processor.hpp"
#include "azure/messaging/eventhubs/processor_load_balancer.hpp"
#include "azure/messaging/eventhubs/producer_client.hpp"
#include "azure/messaging/eventhubs/rtti.hpp"

29
sdk/eventhubs/azure-messaging-eventhubs/inc/azure/messaging/eventhubs/consumer_client.hpp
View File

@ -47,11 +47,19 @@ namespace Azure { namespace Messaging { namespace EventHubs {
*/
class ConsumerClient final {
public:
/** Create a new ConsumerClient from an existing one. */
ConsumerClient(ConsumerClient const& other) = default;
/** Copy a new ConsumerClient from an existing one. */
ConsumerClient(ConsumerClient const& other) = delete;
/** Move a consumer client */
ConsumerClient(ConsumerClient&& other) = delete;
/** Assign a ConsumerClient to an existing one. */
ConsumerClient& operator=(ConsumerClient const& other) = default;
ConsumerClient& operator=(ConsumerClient const& other) = delete;
/** Move a consumer client */
ConsumerClient& operator=(ConsumerClient&& other) = delete;
~ConsumerClient();
/** @brief Getter for event hub name
*
@ -99,7 +107,7 @@ namespace Azure { namespace Messaging { namespace EventHubs {
* parameter cannot be empty and should contain the name of your event hub.
* Endpoint=sb://\<your-namespace\>.servicebus.windows.net/;SharedAccessKeyName=\<key-name\>;SharedAccessKey=\<key\>
* When the connection string DOES have an entity path, as shown below, the eventHub parameter
* will be ignored.
* must match the entity path.
* Endpoint=sb://\<your-namespace\>.servicebus.windows.net/;
* SharedAccessKeyName=\<key-name\>;SharedAccessKey=\<key\>;EntityPath=\<entitypath\>;
*/
@ -155,9 +163,6 @@ namespace Azure { namespace Messaging { namespace EventHubs {
Core::Context const& context = {});
private:
void EnsureSession(std::string const& partitionId = {});
Azure::Core::Amqp::_internal::Session GetSession(std::string const& partitionId = {});
/// The connection string for the Event Hubs namespace
std::string m_connectionString;
@ -177,11 +182,21 @@ namespace Azure { namespace Messaging { namespace EventHubs {
std::string m_hostUrl;
/// @brief The message receivers used to receive messages for a given partition.
std::mutex m_receiversLock;
std::map<std::string, Azure::Core::Amqp::_internal::MessageReceiver> m_receivers;
/// @brief The AMQP Sessions used to receive messages for a given partition.
std::mutex m_sessionsLock;
std::map<std::string, Azure::Core::Amqp::_internal::Session> m_sessions;
std::map<std::string, Azure::Core::Amqp::_internal::Connection> m_connections;
/// @brief The options used to configure the consumer client.
ConsumerClientOptions m_consumerClientOptions;
void EnsureConnection(std::string const& partitionId);
void EnsureSession(std::string const& partitionId);
Azure::Core::Amqp::_internal::Connection CreateConnection(std::string const& partitionId);
Azure::Core::Amqp::_internal::Session CreateSession(std::string const& partitionId);
Azure::Core::Amqp::_internal::Session GetSession(std::string const& partitionId);
};
}}} // namespace Azure::Messaging::EventHubs

2
sdk/eventhubs/azure-messaging-eventhubs/inc/azure/messaging/eventhubs/models/checkpoint_store_models.hpp
View File

@ -38,6 +38,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Models {
/// @brief The blob storage name prefix for this ownership.
std::string GetOwnershipPrefixName() const;
};
std::ostream& operator<<(std::ostream& os, Ownership const& value);
/**@brief Checkpoint tracks the last successfully processed event in a partition.
*/
@ -62,4 +63,5 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Models {
/// @brief Returns the name of the blob that stores the checkpoint.
std::string GetCheckpointBlobName() const;
};
std::ostream& operator<<(std::ostream& os, Checkpoint const& value);
}}}} // namespace Azure::Messaging::EventHubs::Models

29
sdk/eventhubs/azure-messaging-eventhubs/inc/azure/messaging/eventhubs/models/processor_load_balancer_models.hpp
View File

@ -20,33 +20,4 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Models {
ProcessorStrategyGreedy
};
/**
* @brief COnfiguration options for the load balancer.
*/
struct LoadBalancerInfo final
{
/// current are the partitions that _we_ own
std::vector<Ownership> Current;
/// unownedOrExpired partitions either had no claim _ever_ or were once
/// owned but the ownership claim has expired.
std::vector<Ownership> UnownedOrExpired;
/// aboveMax are ownerships where the specific owner has too many partitions
/// it contains _all_ the partitions for that particular consumer.
std::vector<Ownership> AboveMax;
/// maxAllowed is the maximum number of partitions a consumer should have
/// If partitions do not divide evenly this will be the "theoretical" max
/// with the assumption that this particular consumer will get an extra
/// partition.
size_t MaxAllowed;
/// extraPartitionPossible is true if the partitions cannot split up evenly
/// amongst all the known consumers.
bool ExtraPartitionPossible;
/// Raw ownerships are the raw ownerships from the checkpoint store.
std::vector<Ownership> Raw;
};
}}}} // namespace Azure::Messaging::EventHubs::Models

16
sdk/eventhubs/azure-messaging-eventhubs/inc/azure/messaging/eventhubs/partition_client.hpp
View File

@ -58,10 +58,14 @@ namespace Azure { namespace Messaging { namespace EventHubs {
public:
/// Create a PartitionClient from another PartitionClient
PartitionClient(PartitionClient const& other) = default;
PartitionClient(PartitionClient const& other) = delete;
/// Create a PartitionClient moving from another PartitionClient
PartitionClient(PartitionClient&& other) = default;
/// Assign a PartitionClient to another PartitionClient
PartitionClient& operator=(PartitionClient const& other) = default;
PartitionClient& operator=(PartitionClient const& other) = delete;
/// Move a PartitionClient to another PartitionClient
PartitionClient& operator=(PartitionClient&& other) = default;
/** Destroy this partition client.
*/
@ -87,9 +91,6 @@ namespace Azure { namespace Messaging { namespace EventHubs {
/// The message receiver used to receive events from the partition.
Azure::Core::Amqp::_internal::MessageReceiver m_receiver;
/// The name of the offset to start receiving events from.
// std::string m_offsetExpression;
/// The options used to create the PartitionClient.
PartitionClientOptions m_partitionOptions;
@ -101,11 +102,6 @@ namespace Azure { namespace Messaging { namespace EventHubs {
*/
Azure::Core::Http::Policies::RetryOptions m_retryOptions{};
// Azure::Core::Amqp::Common::_internal::AsyncOperationQueue<
// Azure::Core::Amqp::Models::AmqpMessage,
// Azure::Core::Amqp::Models::_internal::AmqpError>
// m_receivedMessageQueue;
/** Creates a new PartitionClient
*
* @param messageReceiver Message Receiver for the partition client.

331
sdk/eventhubs/azure-messaging-eventhubs/inc/azure/messaging/eventhubs/processor.hpp
View File

@ -3,15 +3,16 @@
#pragma once
#include "checkpoint_store.hpp"
#include "consumer_client.hpp"
#include "models/processor_load_balancer_models.hpp"
#include "models/processor_models.hpp"
#include "processor_load_balancer.hpp"
#include "processor_partition_client.hpp"
#include <azure/core/context.hpp>
#include <chrono>
#include <thread>
#ifdef TESTING_BUILD_AMQP
#ifdef azure_TESTING_BUILD_AMQP
namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
class ProcessorTest_LoadBalancing_Test;
}}}} // namespace Azure::Messaging::EventHubs::Test
@ -26,17 +27,18 @@ namespace Azure { namespace Messaging { namespace EventHubs {
* ownership of partitions between them.
* The default strategy is ProcessorStrategyBalanced.
*/
Models::ProcessorStrategy LoadBalancingStrategy;
Models::ProcessorStrategy LoadBalancingStrategy{
Models::ProcessorStrategy::ProcessorStrategyBalanced};
/**@brief UpdateInterval controls how often attempt to claim partitions.
* The default value is 10 seconds.
*/
Azure::DateTime::duration UpdateInterval;
Azure::DateTime::duration UpdateInterval{std::chrono::seconds(10)};
/**@brief PartitionExpirationDuration is the amount of time before a partition is
* considered unowned. The default value is 60 seconds.
*/
Azure::DateTime::duration PartitionExpirationDuration;
Azure::DateTime::duration PartitionExpirationDuration{std::chrono::seconds(60)};
/**@brief StartPositions are the default start positions (configurable per
* partition, or with an overall default value) if a checkpoint is not found
@ -54,31 +56,34 @@ namespace Azure { namespace Messaging { namespace EventHubs {
* Defaults to 300 events.
* Disabled if Prefetch < 0.
*/
int32_t Prefetch = 300;
int32_t Prefetch{300};
/** @brief Specifies the maximum number of partitions to process.
*
* By default, the processor will process all available partitions. If a client desires limiting
* the number of partitions to a restricted set, set the MaximumNumberOfPartitions to the number
* of partitions to process.
*/
int32_t MaximumNumberOfPartitions{0};
};
/**@brief Processor uses a [ConsumerClient] and [CheckpointStore] to provide automatic
* load balancing between multiple Processor instances, even in separate
*processes or on separate machines.
*/
namespace _detail {
class ProcessorLoadBalancer;
}
/** @brief Processor uses a ConsumerClient and CheckpointStore to provide automatic load balancing
* between multiple Processor instances, even in separate processes or on separate machines.
*/
class Processor final {
#ifdef TESTING_BUILD_AMQP
#ifdef azure_TESTING_BUILD_AMQP
friend class Test::ProcessorTest_LoadBalancing_Test;
#endif
Azure::DateTime::duration m_ownershipUpdateInterval;
Models::StartPositions m_defaultStartPositions;
std::shared_ptr<CheckpointStore> m_checkpointStore;
int32_t m_prefetch;
std::shared_ptr<ConsumerClient> m_consumerClient;
std::vector<std::shared_ptr<ProcessorPartitionClient>> m_nextPartitionClients;
uint32_t m_currentPartitionClient;
Models::ConsumerClientDetails m_consumerClientDetails;
std::shared_ptr<ProcessorLoadBalancer> m_loadBalancer;
int64_t m_processorOwnerLevel = 0;
typedef std::map<std::string, std::shared_ptr<ProcessorPartitionClient>> ConsumersType;
public:
/** @brief Construct a new Processor object.
*
@ -89,65 +94,189 @@ namespace Azure { namespace Messaging { namespace EventHubs {
Processor(
std::shared_ptr<ConsumerClient> consumerClient,
std::shared_ptr<CheckpointStore> checkpointStore,
ProcessorOptions const& options = {})
: m_defaultStartPositions(options.StartPositions), m_checkpointStore(checkpointStore),
m_prefetch(options.Prefetch), m_consumerClient(consumerClient)
{
m_ownershipUpdateInterval = options.UpdateInterval == Azure::DateTime::duration::zero()
? std::chrono::seconds(10)
: options.UpdateInterval;
ProcessorOptions const& options = {});
m_consumerClientDetails = m_consumerClient->GetDetails();
m_loadBalancer = std::make_shared<ProcessorLoadBalancer>(
m_checkpointStore,
m_consumerClientDetails,
options.LoadBalancingStrategy,
options.PartitionExpirationDuration == Azure::DateTime::duration::zero()
? std::chrono::minutes(1)
: std::chrono::duration_cast<std::chrono::minutes>(
options.PartitionExpirationDuration));
}
~Processor();
/** Construct a Processor from another Processor. */
Processor(Processor const& other) = default;
Processor(Processor const& other) = delete;
/** Assign a Processor to another Processor. */
Processor& operator=(Processor const& other) = default;
Processor& operator=(Processor const& other) = delete;
/** Move to the next partition client */
std::shared_ptr<ProcessorPartitionClient> NextPartitionClient()
{
uint32_t currentPartition = m_currentPartitionClient;
if (currentPartition > m_nextPartitionClients.size() - 1)
{
currentPartition = 0;
}
m_currentPartitionClient = currentPartition + 1;
return m_nextPartitionClients[currentPartition];
}
/** Move to the next partition client
*
* @param context The context to control whether this function is canceled or not.
*
* @return A shared pointer to the next partition client.
*
* NextPartitionClient will retrieve the next ProcessorPartitionClient if one is acquired or
* will block until a new one arrives, or the processor is stopped.
*/
std::shared_ptr<ProcessorPartitionClient> NextPartitionClient(
Azure::Core::Context const& context = {});
/** @brief Executes the processor.
*
* @param context The context to control the request lifetime.
*
* @remark This function will block until Stop() is called. It is intended for customers who
* would prefer to manage the call to Run from their own threads.
*
*/
void Run(Core::Context const& context);
/** @brief Starts the processor.
*
* @param context The context to control the request lifetime.
* @param context The context to control the request lifetime of the processor. Cancelling this
* context will stop the processor from running.
*
* @remark This function starts the processor running in a new thread.
*/
void Run(Core::Context const& context)
{
Models::EventHubProperties eventHubProperties
= m_consumerClient->GetEventHubProperties(context);
ConsumersType consumers;
Dispatch(eventHubProperties, consumers, context);
// time_t timeNowSeconds
// = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
// const auto current = std::chrono::system_clock::from_time_t(timeNowSeconds);
void Start(Azure::Core::Context const& context = {});
//// TODO : this is where we re load balance on the update interval
// while (!context.IsCancelled())
//{
// std::this_thread::sleep_for(m_ownershipUpdateInterval);
// Dispatch(eventHubProperties, consumers, context);
//}
/** @brief Stops a running processor.
*
* @remark This function stops the processor. If the Start method has been called, it will wait
* for the thread to complete.
*/
void Stop();
/** @brief Closes the processor and cancels any current operations.
*
*/
void Close(Core::Context const& context = {})
{
if (m_isRunning)
{
throw std::runtime_error("cannot close a processor that is running");
}
// Drain the partition clients queue.
for (;;)
{
auto client = m_nextPartitionClients.TryRemove();
if (client)
{
client->Close();
}
else
{
break;
}
}
(void)context;
}
private:
/** Representation of Go channel construct.
*
* A channel represents a size limited queue, where items can be inserted and removed. If there
* are no items in the queue, the caller will block until an item is inserted into the queue.
*
* @tparam T The type of the items in the queue.
*
*/
template <class T> class Channel {
public:
Channel() : m_maximumDepth{0} {}
~Channel()
{
Azure::Core::Diagnostics::_internal::Log::Stream(
Azure::Core::Diagnostics::Logger::Level::Verbose)
<< "~Channel. Currently depth is " << m_channelDepth << " and maximum depth is "
<< m_maximumDepth;
Azure::Core::Diagnostics::_internal::Log::Stream(
Azure::Core::Diagnostics::Logger::Level::Verbose)
<< "Clear channel queue.";
while (m_channelDepth > 0)
{
auto value = m_channelQueue.TryWaitForResult();
if (value)
{
m_channelDepth -= 1;
}
}
}
// Insert an item into the channel, returning true if successful, false if the channel is
// full.
bool Insert(T item)
{
std::lock_guard<std::mutex> lock{m_channelLock};
if ((m_maximumDepth != 0) && (m_channelDepth >= m_maximumDepth))
{
return false;
}
m_channelQueue.CompleteOperation(item);
m_channelDepth += 1;
return true;
}
// Remove an item from the channel.
T Remove(Azure::Core::Context const& context)
{
auto value = m_channelQueue.WaitForResult(context);
if (!value)
{
throw Azure::Core::OperationCancelledException("Operation was cancelled.");
}
std::lock_guard<std::mutex> lock{m_channelLock};
m_channelDepth -= 1;
return std::get<0>(*value);
}
/** Try to remove an item from the channel, returning an item from the channel or a default
* constructed object.
*
* @return An item from the channel or a default constructed object.
*
* @remark The T type should have semantics that can distinguish between a default constructed
* item and a valid item. So for example, if T is a pointer, then a nullptr should be returned
* if the channel is empty. But if T is a value such as a string, it will not be possible to
* distinguish between an empty string inserted into the channel and an empty channel.
*/
T TryRemove()
{
std::lock_guard<std::mutex> lock{m_channelLock};
auto value = m_channelQueue.TryWaitForResult();
if (value)
{
m_channelDepth -= 1;
return std::get<0>(*value);
}
return T{};
}
void SetMaximumDepth(size_t maximumDepth)
{
std::lock_guard<std::mutex> lock{m_channelLock};
m_maximumDepth = maximumDepth;
}
private:
std::mutex m_channelLock;
size_t m_channelDepth{};
size_t m_maximumDepth{};
Core::Amqp::Common::_internal::AsyncOperationQueue<T> m_channelQueue;
};
Azure::DateTime::duration m_ownershipUpdateInterval;
Models::StartPositions m_defaultStartPositions;
int32_t m_maximumNumberOfPartitions;
std::shared_ptr<CheckpointStore> m_checkpointStore;
std::shared_ptr<ConsumerClient> m_consumerClient;
int32_t m_prefetch;
Channel<std::shared_ptr<ProcessorPartitionClient>> m_nextPartitionClients;
Models::ConsumerClientDetails m_consumerClientDetails;
std::shared_ptr<_detail::ProcessorLoadBalancer> m_loadBalancer;
int64_t m_processorOwnerLevel{0};
bool m_isRunning{false};
std::thread m_processorThread;
typedef std::map<std::string, std::shared_ptr<ProcessorPartitionClient>> ConsumersType;
/** @brief Dispatches events to the appropriate partition clients.
*
* @param eventHubProperties The properties of the Event Hub.
@ -156,60 +285,15 @@ namespace Azure { namespace Messaging { namespace EventHubs {
*/
void Dispatch(
Models::EventHubProperties const& eventHubProperties,
ConsumersType& consumers,
Core::Context const& context)
{
std::vector<Models::Ownership> ownerships
= m_loadBalancer->LoadBalance(eventHubProperties.PartitionIds, context);
std::shared_ptr<ConsumersType> consumers,
Core::Context const& context);
std::map<std::string, Models::Checkpoint> checkpoints = GetCheckpointsMap(context);
for (auto const& ownership : ownerships)
{
AddPartitionClient(ownership, checkpoints, consumers);
}
}
/** @brief Closes the processor and cancels any current operations.
*
*
*/
void Close()
{
for (auto& consumer : m_nextPartitionClients)
{
consumer->Close();
}
}
private:
void AddPartitionClient(
Models::Ownership const& ownership,
std::map<std::string, Models::Checkpoint>& checkpoints,
ConsumersType& consumers)
{
Models::StartPosition startPosition = GetStartPosition(ownership, checkpoints);
std::weak_ptr<ConsumersType> consumers);
// The consumers parameter is not stabilized across the lifetime of the partition client. Leak
// the partition for now.
std::shared_ptr<ProcessorPartitionClient> processorPartitionClient
= std::make_shared<ProcessorPartitionClient>(
ownership.PartitionId,
m_consumerClient->CreatePartitionClient(
ownership.PartitionId, {startPosition, m_processorOwnerLevel, m_prefetch}),
m_checkpointStore,
m_consumerClientDetails,
[]() { /*
consumers.erase(ownership.PartitionId);*/
});
if (consumers.find(ownership.PartitionId) == consumers.end())
{
consumers.emplace(ownership.PartitionId, processorPartitionClient);
}
m_nextPartitionClients.push_back(processorPartitionClient);
}
void RunInternal(Core::Context const& context, bool manualRun);
Models::StartPosition GetStartPosition(
Models::Ownership const& ownership,
@ -244,21 +328,6 @@ namespace Azure { namespace Messaging { namespace EventHubs {
return startPosition;
}
std::map<std::string, Models::Checkpoint> GetCheckpointsMap(Core::Context const& context)
{
std::vector<Models::Checkpoint> checkpoints = m_checkpointStore->ListCheckpoints(
m_consumerClientDetails.FullyQualifiedNamespace,
m_consumerClientDetails.EventHubName,
m_consumerClientDetails.ConsumerGroup,
context);
std::map<std::string, Models::Checkpoint> checkpointsMap;
for (auto& checkpoint : checkpoints)
{
checkpointsMap.emplace(checkpoint.PartitionId, checkpoint);
}
return checkpointsMap;
}
std::map<std::string, Models::Checkpoint> GetCheckpointsMap(Core::Context const& context);
};
}}} // namespace Azure::Messaging::EventHubs

87
sdk/eventhubs/azure-messaging-eventhubs/inc/azure/messaging/eventhubs/processor_partition_client.hpp
View File

@ -4,7 +4,6 @@
#include "checkpoint_store.hpp"
#include "consumer_client.hpp"
#include <azure/core/amqp.hpp>
namespace Azure { namespace Messaging { namespace EventHubs {
/**@brief ProcessorPartitionClient allows you to receive events, similar to a [PartitionClient],
@ -16,37 +15,22 @@ namespace Azure { namespace Messaging { namespace EventHubs {
* ownership manually, use the [ConsumerClient] instead.
*/
class ProcessorPartitionClient final {
std::string m_partitionId;
PartitionClient m_partitionClient;
std::shared_ptr<CheckpointStore> m_checkpointStore;
std::function<void()> m_cleanupFunc;
Models::ConsumerClientDetails m_consumerClientDetails;
friend class Processor;
public:
/** Constructs a new instance of the ProcessorPartitionClient.
* @param partitionId The identifier of the partition to connect the client to.
* @param partitionClient The [PartitionClient] to use for receiving events.
* @param checkpointStore The [CheckpointStore] to use for storing checkpoints.
* @param consumerClientDetails The [ConsumerClientDetails] to use for storing checkpoints.
* @param cleanupFunc The function to call when the ProcessorPartitionClient is closed.
*/
ProcessorPartitionClient(
std::string partitionId,
PartitionClient partitionClient,
std::shared_ptr<CheckpointStore> checkpointStore,
Models::ConsumerClientDetails consumerClientDetails,
std::function<void()> cleanupFunc)
: m_partitionId(partitionId), m_partitionClient(partitionClient),
m_checkpointStore(checkpointStore), m_cleanupFunc(cleanupFunc),
m_consumerClientDetails(consumerClientDetails)
{
}
/// Copy a ProcessorPartitionClient to another ProcessorPartitionClient.
ProcessorPartitionClient(ProcessorPartitionClient const& other) = default;
ProcessorPartitionClient(ProcessorPartitionClient const& other) = delete;
/// Move a ProcessorPartitionClient to another.
ProcessorPartitionClient(ProcessorPartitionClient&& other) = default;
/// Assignment operator.
ProcessorPartitionClient& operator=(ProcessorPartitionClient const& other) = default;
ProcessorPartitionClient& operator=(ProcessorPartitionClient const& other) = delete;
/// Move a ProcessorPartitionClient to another.
ProcessorPartitionClient& operator=(ProcessorPartitionClient&& other) = default;
~ProcessorPartitionClient();
/** Receives Events from the partition.
* @param maxBatchSize The maximum number of events to receive in a single call to the service.
@ -56,21 +40,62 @@ namespace Azure { namespace Messaging { namespace EventHubs {
uint32_t maxBatchSize,
Core::Context const& context = {})
{
return m_partitionClient.ReceiveEvents(maxBatchSize, context);
return m_partitionClient->ReceiveEvents(maxBatchSize, context);
}
/** Closes the partition client.
/**
* @brief Updates the checkpoint for this partition using the given event data.
*
* Subsequent partition client reads will start from this event.
*
* @param eventData The event data to use for updating the checkpoint.
* @param context The context to pass to the update checkpoint operation.
*/
void UpdateCheckpoint(
Models::ReceivedEventData const& eventData,
Core::Context const& context = {});
/// Returns the partition ID associated with this ProcessorPartitionClient.
std::string PartitionId() const { return m_partitionId; }
/** Closes the partition client. */
void Close()
{
if (m_cleanupFunc != nullptr)
if (m_cleanupFunc)
{
m_cleanupFunc();
}
m_partitionClient.Close();
m_partitionClient->Close();
}
private:
std::string m_partitionId;
std::unique_ptr<PartitionClient> m_partitionClient{};
std::shared_ptr<CheckpointStore> m_checkpointStore;
std::function<void()> m_cleanupFunc;
Models::ConsumerClientDetails m_consumerClientDetails;
/** Constructs a new instance of the ProcessorPartitionClient.
* @param partitionId The identifier of the partition to connect the client to.
* @param checkpointStore The [CheckpointStore] to use for storing checkpoints.
* @param consumerClientDetails The [ConsumerClientDetails] to use for storing checkpoints.
* @param cleanupFunc The function to call when the ProcessorPartitionClient is closed.
*/
ProcessorPartitionClient(
std::string partitionId,
std::shared_ptr<CheckpointStore> checkpointStore,
Models::ConsumerClientDetails consumerClientDetails,
std::function<void()> cleanupFunc)
: m_partitionId(partitionId), m_checkpointStore(checkpointStore),
m_cleanupFunc(cleanupFunc), m_consumerClientDetails(consumerClientDetails)
{
}
void SetPartitionClient(std::unique_ptr<PartitionClient>& partitionClient)
{
m_partitionClient = std::move(partitionClient);
}
void UpdateCheckpoint(
Azure::Core::Amqp::Models::AmqpMessage const& amqpMessage,
Core::Context const& context = {});

67
sdk/eventhubs/azure-messaging-eventhubs/inc/azure/messaging/eventhubs/producer_client.hpp
View File

@ -42,24 +42,6 @@ namespace Azure { namespace Messaging { namespace EventHubs {
/**@brief ProducerClient can be used to send events to an Event Hub.
*/
class ProducerClient final {
/// The connection string for the Event Hubs namespace
std::string m_connectionString;
/// the Event Hubs namespace name (ex: myeventhub.servicebus.windows.net)
std::string m_fullyQualifiedNamespace;
/// The name of the Event Hub
std::string m_eventHub{};
/// The URL to the Event Hubs namespace
std::string m_targetUrl{};
/// Credentials to be used to authenticate the client.
std::shared_ptr<Core::Credentials::TokenCredential> m_credential{};
ProducerClientOptions m_producerClientOptions{};
std::map<std::string, Azure::Core::Amqp::_internal::MessageSender> m_senders{};
std::map<std::string, Azure::Core::Amqp::_internal::Session> m_sessions{};
public:
/** Get the fully qualified namespace from the connection string */
@ -72,10 +54,13 @@ namespace Azure { namespace Messaging { namespace EventHubs {
}
/** Create a ProducerClient from another ProducerClient. */
ProducerClient(ProducerClient const& other) = default;
ProducerClient(ProducerClient const& other) = delete;
/** Assign a ProducerClient another ProducerClient. */
ProducerClient& operator=(ProducerClient const& other) = default;
ProducerClient& operator=(ProducerClient const& other) = delete;
ProducerClient(ProducerClient&& other) = delete;
ProducerClient& operator=(ProducerClient&& other) = delete;
/** Default Constructor for a ProducerClient */
ProducerClient() = default;
@ -170,12 +155,44 @@ namespace Azure { namespace Messaging { namespace EventHubs {
Core::Context const& context = {});
private:
void EnsureSender(
std::string const& partitionId = "",
Azure::Core::Context const& context = {});
Azure::Core::Amqp::_internal::MessageSender GetSender(std::string const& partitionId = "");
/// The connection string for the Event Hubs namespace
std::string m_connectionString;
/// the Event Hubs namespace name (ex: myeventhub.servicebus.windows.net)
std::string m_fullyQualifiedNamespace;
/// The name of the Event Hub
std::string m_eventHub{};
/// The URL to the Event Hubs namespace
std::string m_targetUrl{};
/// Credentials to be used to authenticate the client.
std::shared_ptr<Core::Credentials::TokenCredential> m_credential{};
ProducerClientOptions m_producerClientOptions{};
// Protects m_senders and m_connection.
std::mutex m_sendersLock;
std::map<std::string, Azure::Core::Amqp::_internal::Connection> m_connections{};
std::map<std::string, Azure::Core::Amqp::_internal::MessageSender> m_senders{};
std::mutex m_sessionsLock;
std::map<std::string, Azure::Core::Amqp::_internal::Session> m_sessions{};
Azure::Core::Amqp::_internal::Connection CreateConnection();
Azure::Core::Amqp::_internal::Session CreateSession(std::string const& partitionId);
// Ensure that the connection for this producer has been established.
void EnsureConnection(const std::string& partitionId);
// Ensure that a session for the specified partition ID has been established.
void EnsureSession(std::string const& partitionId);
Azure::Core::Amqp::_internal::Session GetSession(std::string const& partitionId = "");
// Ensure that a message sender for the specified partition has been created.
void EnsureSender(std::string const& partitionId, Azure::Core::Context const& context = {});
Azure::Core::Amqp::_internal::MessageSender GetSender(std::string const& partitionId);
Azure::Core::Amqp::_internal::Session GetSession(std::string const& partitionId);
};
}}} // namespace Azure::Messaging::EventHubs

116
sdk/eventhubs/azure-messaging-eventhubs/src/checkpoint_store.cpp
View File

@ -7,52 +7,94 @@
#include <stdexcept>
using namespace Azure::Messaging::EventHubs::Models;
namespace Azure { namespace Messaging { namespace EventHubs { namespace Models {
std::string Azure::Messaging::EventHubs::Models::Ownership::GetOwnershipName() const
{
if (PartitionId.empty())
std::string Ownership::GetOwnershipName() const
{
throw std::runtime_error("missing ownership fields");
if (PartitionId.empty())
{
throw std::runtime_error("missing ownership fields");
}
std::stringstream strstr;
strstr << GetOwnershipPrefixName() << PartitionId;
return strstr.str();
}
std::stringstream strstr;
strstr << GetOwnershipPrefixName() << PartitionId;
return strstr.str();
}
std::string Azure::Messaging::EventHubs::Models::Ownership::GetOwnershipPrefixName() const
{
if (FullyQualifiedNamespace.empty() || EventHubName.empty() || ConsumerGroup.empty())
std::string Ownership::GetOwnershipPrefixName() const
{
throw std::runtime_error("missing ownership fields");
if (FullyQualifiedNamespace.empty() || EventHubName.empty() || ConsumerGroup.empty())
{
throw std::runtime_error("missing ownership fields");
}
std::stringstream strstr;
strstr << Azure::Core::_internal::StringExtensions::ToLower(FullyQualifiedNamespace) << "/"
<< Azure::Core::_internal::StringExtensions::ToLower(EventHubName) << "/"
<< Azure::Core::_internal::StringExtensions::ToLower(ConsumerGroup) << "/ownership/";
return strstr.str();
}
std::stringstream strstr;
strstr << Azure::Core::_internal::StringExtensions::ToLower(FullyQualifiedNamespace) << "/"
<< Azure::Core::_internal::StringExtensions::ToLower(EventHubName) << "/"
<< Azure::Core::_internal::StringExtensions::ToLower(ConsumerGroup) << "/ownership/";
return strstr.str();
}
std::string Azure::Messaging::EventHubs::Models::Checkpoint::GetCheckpointBlobPrefixName() const
{
if (FullyQualifiedNamespaceName.empty() || EventHubName.empty() || ConsumerGroup.empty())
std::string Azure::Messaging::EventHubs::Models::Checkpoint::GetCheckpointBlobPrefixName() const
{
throw std::runtime_error("missing checkpoint fields");
if (FullyQualifiedNamespaceName.empty() || EventHubName.empty() || ConsumerGroup.empty())
{
throw std::runtime_error("missing checkpoint fields");
}
std::stringstream strstr;
strstr << Azure::Core::_internal::StringExtensions::ToLower(FullyQualifiedNamespaceName) << "/"
<< Azure::Core::_internal::StringExtensions::ToLower(EventHubName) << "/"
<< Azure::Core::_internal::StringExtensions::ToLower(ConsumerGroup) << "/checkpoint/";
return strstr.str();
}
std::stringstream strstr;
strstr << Azure::Core::_internal::StringExtensions::ToLower(FullyQualifiedNamespaceName) << "/"
<< Azure::Core::_internal::StringExtensions::ToLower(EventHubName) << "/"
<< Azure::Core::_internal::StringExtensions::ToLower(ConsumerGroup) << "/checkpoint/";
return strstr.str();
}
std::string Azure::Messaging::EventHubs::Models::Checkpoint::GetCheckpointBlobName() const
{
if (PartitionId.empty())
std::string Checkpoint::GetCheckpointBlobName() const
{
throw std::runtime_error("missing checkpoint fields");
if (PartitionId.empty())
{
throw std::runtime_error("missing checkpoint fields");
}
return GetCheckpointBlobPrefixName() + PartitionId;
}
return GetCheckpointBlobPrefixName() + PartitionId;
}
std::ostream& operator<<(std::ostream& os, Ownership const& value)
{
os << "Ownership = (";
os << "ConsumerGroup = " << value.ConsumerGroup << ", ";
os << "EventHubName = " << value.EventHubName << ", ";
os << "FullyQualifiedNamespace = " << value.FullyQualifiedNamespace << ", ";
os << "PartitionId = " << value.PartitionId << ", ";
os << "OwnerId = " << value.OwnerId;
if (value.ETag.HasValue())
{
os << ", ETag = " << value.ETag.Value().ToString();
}
if (value.LastModifiedTime.HasValue())
{
os << ", LastModifiedTime = " << value.LastModifiedTime.Value().ToString();
}
os << ")";
return os;
}
std::ostream& operator<<(std::ostream& os, Checkpoint const& value)
{
os << "Checkpoint = (";
os << "ConsumerGroup = " << value.ConsumerGroup << ", ";
os << "EventHubName = " << value.EventHubName << ", ";
os << "FullyQualifiedNamespaceName = " << value.FullyQualifiedNamespaceName << ", ";
os << "PartitionId = " << value.PartitionId;
if (value.Offset.HasValue())
{
os << ", Offset = " << value.Offset.Value();
}
if (value.SequenceNumber.HasValue())
{
os << ", SequenceNumber = " << value.SequenceNumber.Value();
}
os << ")";
return os;
}
}}}} // namespace Azure::Messaging::EventHubs::Models

96
sdk/eventhubs/azure-messaging-eventhubs/src/consumer_client.cpp
View File

@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
#include "private/eventhubs_constants.hpp"
#include "private/eventhubs_utilities.hpp"
#include "private/package_version.hpp"
@ -23,7 +24,7 @@ namespace Azure { namespace Messaging { namespace EventHubs {
m_consumerClientOptions(options)
{
auto sasCredential
= std::make_shared<ServiceBusSasConnectionStringCredential>(m_connectionString);
= std::make_shared<ServiceBusSasConnectionStringCredential>(m_connectionString, eventHub);
m_credential = sasCredential;
if (!sasCredential->GetEntityPath().empty())
@ -31,8 +32,8 @@ namespace Azure { namespace Messaging { namespace EventHubs {
m_eventHub = sasCredential->GetEntityPath();
}
m_fullyQualifiedNamespace = sasCredential->GetHostName();
m_hostUrl = "amqps://" + m_fullyQualifiedNamespace + "/" + m_eventHub + "/ConsumerGroups/"
+ m_consumerGroup;
m_hostUrl = _detail::EventHubsServiceScheme + m_fullyQualifiedNamespace + "/" + m_eventHub
+ _detail::EventHubsConsumerGroupsPath + m_consumerGroup;
}
ConsumerClient::ConsumerClient(
@ -44,36 +45,79 @@ namespace Azure { namespace Messaging { namespace EventHubs {
: m_fullyQualifiedNamespace{fullyQualifiedNamespace}, m_eventHub{eventHub},
m_consumerGroup{consumerGroup}, m_credential{credential}, m_consumerClientOptions(options)
{
m_hostUrl = "amqps://" + m_fullyQualifiedNamespace + "/" + m_eventHub + "/ConsumerGroups/"
+ m_consumerGroup;
m_hostUrl = _detail::EventHubsServiceScheme + m_fullyQualifiedNamespace + "/" + m_eventHub
+ _detail::EventHubsConsumerGroupsPath + m_consumerGroup;
}
ConsumerClient::~ConsumerClient()
{
Log::Stream(Logger::Level::Informational) << "Destroy consumer client.";
// Tear down the sessions and then the connections, in that order.
for (auto& sender : m_receivers)
{
sender.second.Close();
}
while (!m_sessions.empty())
{
m_sessions.erase(m_sessions.begin());
}
while (!m_connections.empty())
{
m_connections.erase(m_connections.begin());
};
}
Azure::Core::Amqp::_internal::Connection ConsumerClient::CreateConnection(
std::string const& partitionId)
{
ConnectionOptions connectOptions;
connectOptions.ContainerId
= "Consumer for " + m_consumerClientOptions.ApplicationID + " on " + partitionId;
connectOptions.EnableTrace = _detail::EnableAmqpTrace;
connectOptions.AuthenticationScopes = {"https://eventhubs.azure.net/.default"};
// Set the user agent related properties in the connectOptions based on the package
// information and application ID.
_detail::EventHubsUtilities::SetUserAgent(
connectOptions, m_consumerClientOptions.ApplicationID);
return Azure::Core::Amqp::_internal::Connection{
m_fullyQualifiedNamespace, m_credential, connectOptions};
}
void ConsumerClient::EnsureConnection(std::string const& partitionId)
{
std::unique_lock<std::mutex> lock(m_sessionsLock);
if (m_connections.find(partitionId) == m_connections.end())
{
m_connections.emplace(partitionId, CreateConnection(partitionId));
}
}
Azure::Core::Amqp::_internal::Session ConsumerClient::CreateSession(
std::string const& partitionId)
{
SessionOptions sessionOptions;
sessionOptions.InitialIncomingWindowSize
= static_cast<uint32_t>(std::numeric_limits<int32_t>::max());
return m_connections.at(partitionId).CreateSession(sessionOptions);
}
void ConsumerClient::EnsureSession(std::string const& partitionId)
{
EnsureConnection(partitionId);
std::unique_lock<std::mutex> lock(m_sessionsLock);
if (m_sessions.find(partitionId) == m_sessions.end())
{
ConnectionOptions connectOptions;
connectOptions.ContainerId = m_consumerClientOptions.ApplicationID;
connectOptions.EnableTrace = true;
connectOptions.AuthenticationScopes = {"https://eventhubs.azure.net/.default"};
// Set the user agent related properties in the connectOptions based on the package
// information and application ID.
_detail::EventHubsUtilities::SetUserAgent(
connectOptions, m_consumerClientOptions.ApplicationID);
Connection connection(m_fullyQualifiedNamespace, m_credential, connectOptions);
SessionOptions sessionOptions;
sessionOptions.InitialIncomingWindowSize
= static_cast<uint32_t>(std::numeric_limits<int32_t>::max());
Session session{connection.CreateSession(sessionOptions)};
m_sessions.emplace(partitionId, session);
m_sessions.emplace(partitionId, CreateSession(partitionId));
}
}
Azure::Core::Amqp::_internal::Session ConsumerClient::GetSession(std::string const& partitionId)
Azure::Core::Amqp::_internal::Session ConsumerClient::GetSession(
std::string const& partitionId = {})
{
std::unique_lock<std::mutex> lock(m_sessionsLock);
return m_sessions.at(partitionId);
}
@ -99,7 +143,7 @@ namespace Azure { namespace Messaging { namespace EventHubs {
Models::EventHubProperties ConsumerClient::GetEventHubProperties(Core::Context const& context)
{
// Since EventHub properties are not tied to a partition, we don't specify a partition ID.
EnsureSession();
EnsureSession({});
return _detail::EventHubsUtilities::GetEventHubsProperties(GetSession(), m_eventHub, context);
}
@ -108,9 +152,9 @@ namespace Azure { namespace Messaging { namespace EventHubs {
std::string const& partitionId,
Core::Context const& context)
{
EnsureSession(partitionId);
EnsureSession({});
return _detail::EventHubsUtilities::GetEventHubsPartitionProperties(
GetSession(partitionId), m_eventHub, partitionId, context);
GetSession({}), m_eventHub, partitionId, context);
}
}}} // namespace Azure::Messaging::EventHubs

4
sdk/eventhubs/azure-messaging-eventhubs/src/event_data_batch.cpp
View File

@ -37,9 +37,6 @@ namespace Azure { namespace Messaging { namespace EventHubs {
std::vector<Azure::Core::Amqp::Models::AmqpBinaryData> messageList;
for (auto const& marshalledMessage : m_marshalledMessages)
{
std::stringstream ss;
_detail::EventHubsUtilities::LogRawBuffer(ss, marshalledMessage);
Log::Stream(Logger::Level::Informational) << "Add marshalled AMQP message:" << ss.str();
Azure::Core::Amqp::Models::AmqpBinaryData data(marshalledMessage);
messageList.push_back(data);
}
@ -64,7 +61,6 @@ namespace Azure { namespace Messaging { namespace EventHubs {
_detail::PartitionKeyAnnotation, Azure::Core::Amqp::Models::AmqpValue(m_partitionKey));
}
Log::Stream(Logger::Level::Informational) << "Insert AMQP message: " << message;
auto serializedMessage = Azure::Core::Amqp::Models::AmqpMessage::Serialize(message);
if (m_marshalledMessages.size() == 0)

1
sdk/eventhubs/azure-messaging-eventhubs/src/eventhubs_utilities.cpp
View File

@ -106,4 +106,5 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace _detail
}
return isTransient;
}
}}}} // namespace Azure::Messaging::EventHubs::_detail

15
sdk/eventhubs/azure-messaging-eventhubs/src/partition_client.cpp
View File

@ -34,7 +34,7 @@ namespace Azure { namespace Messaging { namespace EventHubs {
std::string GetStartExpression(Models::StartPosition const& startPosition)
{
Log::Stream(Logger::Level::Verbose)
<< "Get Start Expression, startPosition: " << startPosition;
<< "Get Start Expression for StartPosition: " << startPosition;
std::string greaterThan = ">";
if (startPosition.Inclusive)
@ -94,7 +94,7 @@ namespace Azure { namespace Messaging { namespace EventHubs {
// If we don't have a filter value, then default to the start.
if (returnValue.empty())
{
Log::Stream(Logger::Level::Verbose) << "No return value, use default.";
Log::Stream(Logger::Level::Verbose) << "No start position set, use default.";
return "amqp.annotation.x-opt-offset > '@latest'";
}
else
@ -123,7 +123,7 @@ namespace Azure { namespace Messaging { namespace EventHubs {
Azure::Core::Amqp::Models::_internal::MessageSource messageSource(sourceOptions);
Azure::Core::Amqp::_internal::MessageReceiverOptions receiverOptions;
receiverOptions.EnableTrace = true;
receiverOptions.EnableTrace = _detail::EnableAmqpTrace;
// Set the link credit to the prefetch count. If the user has not set a prefetch count, then
// we will use the default value.
if (options.Prefetch >= 0)
@ -163,7 +163,12 @@ namespace Azure { namespace Messaging { namespace EventHubs {
{
}
PartitionClient::~PartitionClient() { m_receiver.Close(); }
PartitionClient::~PartitionClient()
{
Log::Stream(Logger::Level::Verbose) << "~PartitionClient() "
<< "Close Receiver.";
m_receiver.Close();
}
/** Receive events from the partition.
*
@ -190,8 +195,6 @@ namespace Azure { namespace Messaging { namespace EventHubs {
if (result.first.HasValue())
{
messages.push_back(Models::ReceivedEventData{result.first.Value()});
Log::Stream(Logger::Level::Verbose)
<< "Peeked message. Message count now " << messages.size();
}
else if (result.second)
{

10
sdk/eventhubs/azure-messaging-eventhubs/src/partition_client_models.cpp
View File

@ -14,25 +14,25 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Models {
os << "StartPosition:[";
if (sp.Offset.HasValue())
{
os << "Off: " << sp.Offset.Value() << std::endl;
os << "Off: " << sp.Offset.Value();
}
if (sp.SequenceNumber.HasValue())
{
os << "Seq: " << sp.SequenceNumber.Value() << std::endl;
os << "Seq: " << sp.SequenceNumber.Value();
}
if (sp.EnqueuedTime.HasValue())
{
os << "Enq: " << sp.EnqueuedTime.Value().ToString();
}
os << " Inclusive: " << std::boolalpha << sp.Inclusive;
if (sp.Earliest.HasValue())
{
os << " Earliest: " << std::boolalpha << sp.Earliest.Value() << std::endl;
os << " Earliest: " << std::boolalpha << sp.Earliest.Value();
}
if (sp.Latest.HasValue())
{
os << "Latest: " << std::boolalpha << sp.Latest.Value() << std::endl;
os << "Latest: " << std::boolalpha << sp.Latest.Value();
}
os << " Inclusive: " << std::boolalpha << sp.Inclusive;
os << "]";
return os;

3
sdk/eventhubs/azure-messaging-eventhubs/src/private/eventhubs_constants.hpp
View File

@ -15,4 +15,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace _detail
constexpr const char* SequenceNumberAnnotation = "x-opt-sequence-number";
constexpr const char* OffsetNumberAnnotation = "x-opt-offset";
constexpr const char* EnqueuedTimeAnnotation = "x-opt-enqueued-time";
constexpr const char* EventHubsServiceScheme = "amqps://";
constexpr const char* EventHubsConsumerGroupsPath = "/ConsumerGroups/";
}}}} // namespace Azure::Messaging::EventHubs::_detail

6
sdk/eventhubs/azure-messaging-eventhubs/src/private/eventhubs_utilities.hpp
View File

@ -20,6 +20,8 @@
namespace Azure { namespace Messaging { namespace EventHubs { namespace _detail {
constexpr bool EnableAmqpTrace = true;
class EventHubsExceptionFactory {
public:
/**
@ -112,7 +114,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace _detail
// Create a management client off the session.
// Eventhubs management APIs return a status code in the "status-code" application properties.
Azure::Core::Amqp::_internal::ManagementClientOptions managementClientOptions;
managementClientOptions.EnableTrace = true;
managementClientOptions.EnableTrace = EnableAmqpTrace;
managementClientOptions.ExpectedStatusCodeKeyName = "status-code";
Azure::Core::Amqp::_internal::ManagementClient managementClient{
session.CreateManagementClient(eventHubName, managementClientOptions)};
@ -179,7 +181,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace _detail
// Create a management client off the session.
// Eventhubs management APIs return a status code in the "status-code" application properties.
Azure::Core::Amqp::_internal::ManagementClientOptions managementClientOptions;
managementClientOptions.EnableTrace = true;
managementClientOptions.EnableTrace = EnableAmqpTrace;
managementClientOptions.ExpectedStatusCodeKeyName = "status-code";
Azure::Core::Amqp::_internal::ManagementClient managementClient{
session.CreateManagementClient(eventHubName, managementClientOptions)};

54
sdk/eventhubs/azure-messaging-eventhubs/inc/azure/messaging/eventhubs/processor_load_balancer.hpp → sdk/eventhubs/azure-messaging-eventhubs/src/private/processor_load_balancer.hpp
View File

@ -3,16 +3,16 @@
// cspell: words lbinfo
#pragma once
#include "checkpoint_store.hpp"
#include "models/consumer_client_models.hpp"
#include "models/partition_client_models.hpp"
#include "models/processor_load_balancer_models.hpp"
#include "azure/messaging/eventhubs/checkpoint_store.hpp"
#include "azure/messaging/eventhubs/models/consumer_client_models.hpp"
#include "azure/messaging/eventhubs/models/partition_client_models.hpp"
#include "azure/messaging/eventhubs/models/processor_load_balancer_models.hpp"
#include <azure/core/context.hpp>
#include <chrono>
#ifdef TESTING_BUILD_AMQP
#ifdef azure_TESTING_BUILD_AMQP
namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
class ProcessorLoadBalancerTest_Greedy_EnoughUnownedPartitions_Test;
class ProcessorLoadBalancerTest_Balanced_UnownedPartitions_Test;
@ -25,13 +25,45 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
}}}} // namespace Azure::Messaging::EventHubs::Test
#endif
namespace Azure { namespace Messaging { namespace EventHubs {
namespace Azure { namespace Messaging { namespace EventHubs { namespace Models { namespace _detail {
/**
* @brief Configuration options for the load balancer.
*/
struct LoadBalancerInfo final
{
/// current are the partitions that _we_ own
std::vector<Ownership> Current;
/// unownedOrExpired partitions either had no claim _ever_ or were once
/// owned but the ownership claim has expired.
std::vector<Ownership> UnownedOrExpired;
/// aboveMax are ownerships where the specific owner has too many partitions
/// it contains _all_ the partitions for that particular consumer.
std::vector<Ownership> AboveMax;
/// maxAllowed is the maximum number of partitions a consumer should have
/// If partitions do not divide evenly this will be the "theoretical" max
/// with the assumption that this particular consumer will get an extra
/// partition.
size_t MaxAllowed;
/// extraPartitionPossible is true if the partitions cannot split up evenly
/// amongst all the known consumers.
bool ExtraPartitionPossible;
/// Raw ownerships are the raw ownerships from the checkpoint store.
std::vector<Ownership> Raw;
};
}}}}} // namespace Azure::Messaging::EventHubs::Models::_detail
namespace Azure { namespace Messaging { namespace EventHubs { namespace _detail {
/**@brief ProcessorLoadBalancer is used by [Processor] to provide automatic load balancing
* between multiple Processor instances, even in separate processes or on separate machines.
*/
class ProcessorLoadBalancer final {
#ifdef TESTING_BUILD_AMQP
#ifdef azure_TESTING_BUILD_AMQP
friend class Test::ProcessorLoadBalancerTest_Greedy_EnoughUnownedPartitions_Test;
friend class Test::ProcessorLoadBalancerTest_Balanced_UnownedPartitions_Test;
friend class Test::ProcessorLoadBalancerTest_Greedy_ForcedToSteal_Test;
@ -49,7 +81,7 @@ namespace Azure { namespace Messaging { namespace EventHubs {
/**@brief GetAvailablePartitions finds all partitions that are either completely unowned _or_
* their ownership is stale.
*/
Models::LoadBalancerInfo GetAvailablePartitions(
Models::_detail::LoadBalancerInfo GetAvailablePartitions(
std::vector<std::string> const& partitionIDs,
Core::Context const& context);
@ -69,11 +101,11 @@ namespace Azure { namespace Messaging { namespace EventHubs {
*know it exists until then.
*/
std::vector<Models::Ownership> BalancedLoadBalancer(
Models::LoadBalancerInfo const& lbinfo,
Models::_detail::LoadBalancerInfo const& lbinfo,
Core::Context const& context);
std::vector<Models::Ownership> GreedyLoadBalancer(
Models::LoadBalancerInfo const& lbInfo,
Models::_detail::LoadBalancerInfo const& lbInfo,
Core::Context const& context);
public:
@ -126,4 +158,4 @@ namespace Azure { namespace Messaging { namespace EventHubs {
std::vector<std::string> const& partitionIDs,
Core::Context const& context = {});
};
}}} // namespace Azure::Messaging::EventHubs
}}}} // namespace Azure::Messaging::EventHubs::_detail

4
sdk/eventhubs/azure-messaging-eventhubs/src/private/retry_operation.hpp
View File

@ -7,7 +7,7 @@
#include <chrono>
#include <functional>
#if defined(TESTING_BUILD_AMQP)
#if defined(azure_TESTING_BUILD_AMQP)
// Define the class used from tests to validate retry enabled
namespace Azure { namespace Messaging { namespace EventHubs { namespace _internal { namespace Test {
class RetryOperationTest_ShouldRetryTrue1_Test;
@ -18,7 +18,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace _interna
#endif
namespace Azure { namespace Messaging { namespace EventHubs { namespace _detail {
class RetryOperation {
#if defined(TESTING_BUILD_AMQP)
#if defined(azure_TESTING_BUILD_AMQP)
// make tests classes friends to validate set Retry
friend class Azure::Messaging::EventHubs::_internal::Test::
RetryOperationTest_ShouldRetryTrue1_Test;

213
sdk/eventhubs/azure-messaging-eventhubs/src/processor.cpp Normal file
View File

@ -0,0 +1,213 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
#include "azure/messaging/eventhubs/processor.hpp"
#include "azure/messaging/eventhubs/models/management_models.hpp"
#include "azure/messaging/eventhubs/models/partition_client_models.hpp"
#include "private/processor_load_balancer.hpp"
#include <azure/core/diagnostics/logger.hpp>
#include <azure/core/internal/diagnostics/log.hpp>
#include <iomanip>
using namespace Azure::Core::Diagnostics::_internal;
using namespace Azure::Core::Diagnostics;
namespace Azure { namespace Messaging { namespace EventHubs {
Processor::Processor(
std::shared_ptr<ConsumerClient> consumerClient,
std::shared_ptr<CheckpointStore> checkpointStore,
ProcessorOptions const& options)
: m_defaultStartPositions(options.StartPositions),
m_maximumNumberOfPartitions{options.MaximumNumberOfPartitions},
m_checkpointStore(checkpointStore), m_consumerClient(consumerClient),
m_prefetch(options.Prefetch), m_nextPartitionClients{}
{
m_ownershipUpdateInterval = options.UpdateInterval == Azure::DateTime::duration::zero()
? std::chrono::seconds(10)
: options.UpdateInterval;
m_consumerClientDetails = m_consumerClient->GetDetails();
m_loadBalancer = std::make_shared<_detail::ProcessorLoadBalancer>(
m_checkpointStore,
m_consumerClientDetails,
options.LoadBalancingStrategy,
options.PartitionExpirationDuration == Azure::DateTime::duration::zero()
? std::chrono::minutes(1)
: std::chrono::duration_cast<std::chrono::minutes>(
options.PartitionExpirationDuration));
}
Processor::~Processor()
{
Log::Stream(Logger::Level::Verbose) << "~Processor.";
Stop();
m_consumerClient.reset();
}
void Processor::Start(Azure::Core::Context const& context)
{
m_processorThread = std::thread([this, context]() {
try
{
this->RunInternal(context, false);
}
catch (std::exception& ex)
{
Log::Stream(Logger::Level::Error) << "Exception caught running processor: " << ex.what();
}
});
m_isRunning = true;
}
// Stop the running processor, waiting for the processor to terminate.
void Processor::Stop()
{
Log::Stream(Logger::Level::Verbose) << "Stop processor.";
m_isRunning = false;
if (m_processorThread.joinable())
{
m_processorThread.join();
}
}
void Processor::Run(Core::Context const& context) { RunInternal(context, true); }
void Processor::RunInternal(Core::Context const& context, bool publicInvocation)
{
Models::EventHubProperties eventHubProperties
= m_consumerClient->GetEventHubProperties(context);
if (m_maximumNumberOfPartitions != 0)
{
eventHubProperties.PartitionIds.resize(m_maximumNumberOfPartitions);
}
// Establish the maximum depth of the partition clients channel.
m_nextPartitionClients.SetMaximumDepth(eventHubProperties.PartitionIds.size());
auto consumers = std::make_shared<ConsumersType>();
try
{
// If this is a public invocation (the caller directly called "Run"), then we want to ignore
// the m_isRunning boolean.
while (!context.IsCancelled() && (publicInvocation ? true : m_isRunning))
{
Dispatch(eventHubProperties, consumers, context);
Log::Stream(Logger::Level::Verbose)
<< "Processor Sleeping for "
<< std::chrono::duration_cast<std::chrono::milliseconds>(m_ownershipUpdateInterval)
.count()
<< " milliseconds. ";
std::this_thread::sleep_for(m_ownershipUpdateInterval);
}
}
catch (std::exception& ex)
{
Log::Stream(Logger::Level::Error) << "Exception caught running processor: " << ex.what();
}
}
void Processor::Dispatch(
Models::EventHubProperties const& eventHubProperties,
std::shared_ptr<Processor::ConsumersType> consumers,
Core::Context const& context)
{
std::vector<Models::Ownership> ownerships
= m_loadBalancer->LoadBalance(eventHubProperties.PartitionIds, context);
std::map<std::string, Models::Checkpoint> checkpoints = GetCheckpointsMap(context);
for (auto const& ownership : ownerships)
{
AddPartitionClient(ownership, checkpoints, consumers);
}
}
std::map<std::string, Models::Checkpoint> Processor::GetCheckpointsMap(
Core::Context const& context)
{
std::vector<Models::Checkpoint> checkpoints = m_checkpointStore->ListCheckpoints(
m_consumerClientDetails.FullyQualifiedNamespace,
m_consumerClientDetails.EventHubName,
m_consumerClientDetails.ConsumerGroup,
context);
std::map<std::string, Models::Checkpoint> checkpointsMap;
for (auto& checkpoint : checkpoints)
{
checkpointsMap.emplace(checkpoint.PartitionId, checkpoint);
}
return checkpointsMap;
}
void Processor::AddPartitionClient(
Models::Ownership const& ownership,
std::map<std::string, Models::Checkpoint>& checkpoints,
std::weak_ptr<ConsumersType> consumers)
{
Log::Stream(Logger::Level::Verbose) << "Add partition client for " << ownership;
std::shared_ptr<ProcessorPartitionClient> processorPartitionClient
= std::make_shared<ProcessorPartitionClient>(ProcessorPartitionClient(
ownership.PartitionId,
m_checkpointStore,
m_consumerClientDetails,
[consumers, ownership]() {
if (auto strongConsumers = consumers.lock())
{
strongConsumers->erase(ownership.PartitionId);
}
}));
// Try to add the partition client to the map. If it's already there, we discard the one we just
// created in favor of the existing processor partition client.
if (auto strongConsumers = consumers.lock())
{
auto added = strongConsumers->emplace(ownership.PartitionId, processorPartitionClient);
if (!added.second)
{
Log::Stream(Logger::Level::Verbose)
<< "Partition client already in consumers map, ignoring.";
return;
}
}
// Now that we've verified there is no active processor partition client for this partition, we
// can create a partition client to make the processor partition client fully functional.
Models::StartPosition startPosition = GetStartPosition(ownership, checkpoints);
PartitionClientOptions partitionClientOptions;
partitionClientOptions.StartPosition = startPosition;
partitionClientOptions.Prefetch = m_prefetch;
partitionClientOptions.OwnerLevel = m_processorOwnerLevel;
auto partitionClient{std::make_unique<PartitionClient>(
m_consumerClient->CreatePartitionClient(ownership.PartitionId, partitionClientOptions))};
processorPartitionClient->SetPartitionClient(partitionClient);
// Add the new processor partition client to the next partitions client queue. If the queue
// is full, discard the client.
if (!m_nextPartitionClients.Insert(processorPartitionClient))
{
Log::Stream(Logger::Level::Verbose)
<< "nextPartitionClients is full, discarding partition client..";
processorPartitionClient->Close();
}
}
std::shared_ptr<ProcessorPartitionClient> Processor::NextPartitionClient(
Azure::Core::Context const& context)
{
Log::Stream(Logger::Level::Verbose) << "NextPartitionClient: Retrieve next client";
auto nextClient = m_nextPartitionClients.Remove(context);
return nextClient;
}
}}} // namespace Azure::Messaging::EventHubs

175
sdk/eventhubs/azure-messaging-eventhubs/src/processor_load_balancer.cpp
View File

@ -1,43 +1,58 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
#include "azure/messaging/eventhubs.hpp"
#include "private/processor_load_balancer.hpp"
#include "azure/messaging/eventhubs/checkpoint_store.hpp"
#include "azure/messaging/eventhubs/models/processor_load_balancer_models.hpp"
#include <azure/core/diagnostics/logger.hpp>
#include <azure/core/internal/diagnostics/log.hpp>
#include <iomanip>
#include <set>
#include <stdexcept>
// cspell: words lbinfo
using namespace Azure::Core::Diagnostics::_internal;
using namespace Azure::Core::Diagnostics;
using namespace Azure::Messaging::EventHubs::Models;
using namespace Azure::Messaging::EventHubs::_detail;
using namespace Azure::Messaging::EventHubs::Models::_detail;
Azure::Messaging::EventHubs::Models::LoadBalancerInfo
Azure::Messaging::EventHubs::ProcessorLoadBalancer::GetAvailablePartitions(
LoadBalancerInfo ProcessorLoadBalancer::GetAvailablePartitions(
std::vector<std::string> const& partitionIDs,
Core::Context const& context)
{
Log::Stream(Logger::Level::Verbose)
<< "[" << m_consumerClientDetails.ClientId
<< "] Get Available Partitions for: " << m_consumerClientDetails.FullyQualifiedNamespace
<< "/" << m_consumerClientDetails.EventHubName << "/"
<< m_consumerClientDetails.ConsumerGroup;
std::vector<Models::Ownership> ownerships = m_checkpointStore->ListOwnership(
m_consumerClientDetails.FullyQualifiedNamespace,
m_consumerClientDetails.EventHubName,
m_consumerClientDetails.ConsumerGroup,
m_consumerClientDetails.ClientId,
context);
std::vector<Models::Ownership> unownedOrExpired;
std::map<std::string, bool> alreadyProcessed;
std::set<std::string> alreadyProcessed;
std::map<std::string, std::vector<Models::Ownership>> groupedByOwner;
groupedByOwner[m_consumerClientDetails.ClientId] = std::vector<Models::Ownership>();
groupedByOwner.emplace(m_consumerClientDetails.ClientId, std::vector<Models::Ownership>{});
for (auto& ownership : ownerships)
{
if (alreadyProcessed.find(ownership.PartitionId) != alreadyProcessed.end())
{
continue;
}
alreadyProcessed[ownership.PartitionId] = true;
Azure::DateTime now(Azure::_detail::Clock::now());
if (ownership.OwnerId.empty()
|| std::chrono::duration_cast<std::chrono::minutes>(
now - ownership.LastModifiedTime.Value())
> m_duration)
alreadyProcessed.emplace(ownership.PartitionId);
if (std::chrono::duration_cast<std::chrono::minutes>(
Azure::_detail::Clock::now() - ownership.LastModifiedTime.Value())
> m_duration)
{
unownedOrExpired.push_back(ownership);
continue;
@ -45,6 +60,9 @@ Azure::Messaging::EventHubs::ProcessorLoadBalancer::GetAvailablePartitions(
groupedByOwner[ownership.OwnerId].push_back(ownership);
}
Log::Stream(Logger::Level::Verbose)
<< "Number of expired partitions: " << unownedOrExpired.size();
for (auto& partitionID : partitionIDs)
{
if (alreadyProcessed.find(partitionID) != alreadyProcessed.end())
@ -52,15 +70,18 @@ Azure::Messaging::EventHubs::ProcessorLoadBalancer::GetAvailablePartitions(
continue;
}
unownedOrExpired.push_back(Ownership{
m_consumerClientDetails.ConsumerGroup,
m_consumerClientDetails.EventHubName,
m_consumerClientDetails.FullyQualifiedNamespace,
partitionID,
m_consumerClientDetails.ClientId,
});
Ownership newOwnership;
newOwnership.ConsumerGroup = m_consumerClientDetails.ConsumerGroup;
newOwnership.EventHubName = m_consumerClientDetails.EventHubName;
newOwnership.FullyQualifiedNamespace = m_consumerClientDetails.FullyQualifiedNamespace;
newOwnership.PartitionId = partitionID;
newOwnership.OwnerId = m_consumerClientDetails.ClientId;
unownedOrExpired.push_back(newOwnership);
}
Log::Stream(Logger::Level::Verbose)
<< "Number of unowned partitions: " << unownedOrExpired.size();
size_t maxAllowed = partitionIDs.size() / groupedByOwner.size();
bool hasRemainder = (partitionIDs.size() % groupedByOwner.size()) > 0;
if (hasRemainder)
@ -72,6 +93,10 @@ Azure::Messaging::EventHubs::ProcessorLoadBalancer::GetAvailablePartitions(
for (auto& entry : groupedByOwner)
{
if (entry.first == m_consumerClientDetails.ClientId)
{
continue;
}
if (entry.second.size() > maxAllowed)
{
for (auto& ownership : entry.second)
@ -81,19 +106,18 @@ Azure::Messaging::EventHubs::ProcessorLoadBalancer::GetAvailablePartitions(
}
}
return Models::LoadBalancerInfo{
groupedByOwner[m_consumerClientDetails.ClientId],
unownedOrExpired,
aboveMax,
maxAllowed,
hasRemainder,
ownerships};
LoadBalancerInfo rv;
rv.Current = groupedByOwner[m_consumerClientDetails.ClientId];
rv.UnownedOrExpired = unownedOrExpired;
rv.AboveMax = aboveMax;
rv.MaxAllowed = maxAllowed;
rv.ExtraPartitionPossible = hasRemainder;
rv.Raw = ownerships;
return rv;
}
std::vector<Azure::Messaging::EventHubs::Models::Ownership>
Azure::Messaging::EventHubs::ProcessorLoadBalancer::GetRandomOwnerships(
std::vector<Models::Ownership> const& ownerships,
size_t const count)
std::vector<Azure::Messaging::EventHubs::Models::Ownership> ProcessorLoadBalancer::
GetRandomOwnerships(std::vector<Models::Ownership> const& ownerships, size_t const count)
{
std::vector<Models::Ownership> randomOwnerships;
std::vector<Models::Ownership> remainingOwnerships = ownerships;
@ -110,53 +134,49 @@ Azure::Messaging::EventHubs::ProcessorLoadBalancer::GetRandomOwnerships(
return randomOwnerships;
}
Azure::Messaging::EventHubs::Models::Ownership
Azure::Messaging::EventHubs::ProcessorLoadBalancer::ResetOwnership(Models::Ownership ownership)
Ownership ProcessorLoadBalancer::ResetOwnership(Models::Ownership ownership)
{
ownership.OwnerId = m_consumerClientDetails.ClientId;
return ownership;
}
std::vector<Azure::Messaging::EventHubs::Models::Ownership>
Azure::Messaging::EventHubs::ProcessorLoadBalancer::BalancedLoadBalancer(
Models::LoadBalancerInfo const& lbinfo,
Core::Context const& context)
std::vector<Azure::Messaging::EventHubs::Models::Ownership> ProcessorLoadBalancer::
BalancedLoadBalancer(LoadBalancerInfo const& loadBalancerInfo, Core::Context const& context)
{
(void)context;
std::vector<Models::Ownership> ours;
if (lbinfo.UnownedOrExpired.size() > 0)
if (loadBalancerInfo.UnownedOrExpired.size() > 0)
{
size_t index = std::rand() % lbinfo.UnownedOrExpired.size();
Models::Ownership ownership = ResetOwnership(lbinfo.UnownedOrExpired[index]);
size_t index = std::rand() % loadBalancerInfo.UnownedOrExpired.size();
Models::Ownership ownership = ResetOwnership(loadBalancerInfo.UnownedOrExpired[index]);
ours.push_back(ownership);
}
if (lbinfo.AboveMax.size() > 0)
if (loadBalancerInfo.AboveMax.size() > 0)
{
size_t index = std::rand() % lbinfo.AboveMax.size();
Models::Ownership ownership = ResetOwnership(lbinfo.AboveMax[index]);
size_t index = std::rand() % loadBalancerInfo.AboveMax.size();
Models::Ownership ownership = ResetOwnership(loadBalancerInfo.AboveMax[index]);
ours.push_back(ownership);
}
return ours;
}
std::vector<Azure::Messaging::EventHubs::Models::Ownership>
Azure::Messaging::EventHubs::ProcessorLoadBalancer::GreedyLoadBalancer(
Models::LoadBalancerInfo const& lbInfo,
std::vector<Ownership> ProcessorLoadBalancer::GreedyLoadBalancer(
LoadBalancerInfo const& loadBalancerInfo,
Core::Context const& context)
{
(void)context;
std::vector<Models::Ownership> ours = lbInfo.Current;
std::vector<Models::Ownership> ours = loadBalancerInfo.Current;
// try claiming from the completely unowned or expires ownerships _first_
std::vector<Models::Ownership> randomOwneships
= GetRandomOwnerships(lbInfo.UnownedOrExpired, lbInfo.MaxAllowed - ours.size());
std::vector<Models::Ownership> randomOwneships = GetRandomOwnerships(
loadBalancerInfo.UnownedOrExpired, loadBalancerInfo.MaxAllowed - ours.size());
ours.insert(ours.end(), randomOwneships.begin(), randomOwneships.end());
if (ours.size() < lbInfo.MaxAllowed)
if (ours.size() < loadBalancerInfo.MaxAllowed)
{ // try claiming from the completely unowned or expires ownerships _first_
std::vector<Models::Ownership> randomOwnerships
= GetRandomOwnerships(lbInfo.AboveMax, lbInfo.MaxAllowed - ours.size());
= GetRandomOwnerships(loadBalancerInfo.AboveMax, loadBalancerInfo.MaxAllowed - ours.size());
ours.insert(ours.end(), randomOwnerships.begin(), randomOwnerships.end());
}
for (Models::Ownership& ownership : ours)
@ -167,24 +187,49 @@ Azure::Messaging::EventHubs::ProcessorLoadBalancer::GreedyLoadBalancer(
return ours;
}
std::vector<Azure::Messaging::EventHubs::Models::Ownership>
Azure::Messaging::EventHubs::ProcessorLoadBalancer::LoadBalance(
namespace {
std::string partitionsForOwnerships(std::vector<Ownership> const& ownerships)
{
std::stringstream ss;
ss << "[";
bool first = true;
for (auto& ownership : ownerships)
{
if (!first)
{
ss << ", ";
}
first = false;
ss << ownership.PartitionId;
}
ss << "]";
return ss.str();
}
} // namespace
std::vector<Ownership> ProcessorLoadBalancer::LoadBalance(
std::vector<std::string> const& partitionIDs,
Core::Context const& context)
{
Models::LoadBalancerInfo lbInfo = GetAvailablePartitions(partitionIDs, context);
LoadBalancerInfo loadBalancerInfo = GetAvailablePartitions(partitionIDs, context);
bool claimMore = true;
if (lbInfo.Current.size() >= lbInfo.MaxAllowed)
if (loadBalancerInfo.Current.size() >= loadBalancerInfo.MaxAllowed)
{
// - I have _exactly_ the right amount
// or
// - I have too many. We expect to have some stolen from us, but we'll maintain
// ownership for now.
claimMore = false;
Log::Stream(Logger::Level::Verbose)
<< "Owns " << loadBalancerInfo.Current.size() << " of " << partitionIDs.size()
<< " partitions. Max allowed is " << loadBalancerInfo.MaxAllowed << std::endl;
}
else if (lbInfo.ExtraPartitionPossible && lbInfo.Current.size() == lbInfo.MaxAllowed - 1)
else if (
loadBalancerInfo.ExtraPartitionPossible
&& loadBalancerInfo.Current.size() == loadBalancerInfo.MaxAllowed - 1)
{
// In the 'extraPartitionPossible' scenario, some consumers will have an extra partition
// since things don't divide up evenly. We're one under the max, which means we _might_
@ -193,22 +238,28 @@ Azure::Messaging::EventHubs::ProcessorLoadBalancer::LoadBalance(
// We will attempt to grab _one_ more but only if there are free partitions available
// or if one of the consumers has more than the max allowed.
claimMore = lbInfo.UnownedOrExpired.size() > 0 || lbInfo.AboveMax.size() > 0;
claimMore
= loadBalancerInfo.UnownedOrExpired.size() > 0 || loadBalancerInfo.AboveMax.size() > 0;
Log::Stream(Logger::Level::Verbose)
<< "Unowned/expired: " << loadBalancerInfo.UnownedOrExpired.size()
<< " Above max: " << loadBalancerInfo.AboveMax.size()
<< "Need to claim more: " << std::boolalpha << claimMore;
}
std::vector<Models::Ownership> ownerships = lbInfo.Current;
std::vector<Models::Ownership> ownerships = loadBalancerInfo.Current;
if (claimMore)
{
switch (m_strategy)
{
case Models::ProcessorStrategy::ProcessorStrategyGreedy: {
ownerships = GreedyLoadBalancer(lbInfo, context);
ownerships = GreedyLoadBalancer(loadBalancerInfo, context);
}
break;
case Models::ProcessorStrategy::ProcessorStrategyBalanced: {
std::vector<Models::Ownership> newOwnership = BalancedLoadBalancer(lbInfo, context);
std::vector<Models::Ownership> newOwnership
= BalancedLoadBalancer(loadBalancerInfo, context);
ownerships.insert(ownerships.end(), newOwnership.begin(), newOwnership.end());
}
break;
@ -219,5 +270,9 @@ Azure::Messaging::EventHubs::ProcessorLoadBalancer::LoadBalance(
}
std::vector<Models::Ownership> actual = m_checkpointStore->ClaimOwnership(ownerships, context);
Log::Stream(Logger::Level::Verbose)
<< "[" << m_consumerClientDetails.ClientId << "] Asked for "
<< partitionsForOwnerships(ownerships) << ", got " << partitionsForOwnerships(actual);
return actual;
}

49
sdk/eventhubs/azure-messaging-eventhubs/src/processor_partition_client.cpp
View File

@ -4,8 +4,27 @@
#include "private/eventhubs_constants.hpp"
#include <azure/core/diagnostics/logger.hpp>
#include <azure/core/internal/diagnostics/log.hpp>
#include <iomanip>
using namespace Azure::Core::Diagnostics::_internal;
using namespace Azure::Core::Diagnostics;
namespace Azure { namespace Messaging { namespace EventHubs {
ProcessorPartitionClient::~ProcessorPartitionClient()
{
// Only log the destructor if the partition client has a value.
if (!m_partitionId.empty())
{
Log::Stream(Logger::Level::Verbose) << "~ProcessorPartitionClient() for " << m_partitionId;
Log::Stream(Logger::Level::Verbose)
<< "PartitionClient is " << (m_partitionClient ? "not " : "") << "null.";
}
}
void ProcessorPartitionClient::UpdateCheckpoint(
Azure::Core::Amqp::Models::AmqpMessage const& amqpMessage,
Core::Context const& context)
@ -44,4 +63,34 @@ namespace Azure { namespace Messaging { namespace EventHubs {
m_checkpointStore->UpdateCheckpoint(checkpoint, context);
}
void ProcessorPartitionClient::UpdateCheckpoint(
Models::ReceivedEventData const& eventData,
Core::Context const& context)
{
uint64_t sequenceNumber{};
if (!eventData.SequenceNumber.HasValue())
{
throw std::runtime_error("Event does not have a sequence number.");
}
if (eventData.SequenceNumber.HasValue())
{
sequenceNumber = eventData.SequenceNumber.Value();
}
uint64_t offset{};
if (!eventData.Offset.HasValue())
{
offset = eventData.Offset.Value();
}
Models::Checkpoint checkpoint;
checkpoint.ConsumerGroup = m_consumerClientDetails.ConsumerGroup;
checkpoint.FullyQualifiedNamespaceName = m_consumerClientDetails.FullyQualifiedNamespace;
checkpoint.PartitionId = m_partitionId;
checkpoint.EventHubName = m_consumerClientDetails.EventHubName;
checkpoint.SequenceNumber = sequenceNumber;
checkpoint.Offset = offset;
m_checkpointStore->UpdateCheckpoint(checkpoint, context);
}
}}} // namespace Azure::Messaging::EventHubs

166
sdk/eventhubs/azure-messaging-eventhubs/src/producer_client.cpp
View File

@ -5,6 +5,7 @@
#include "azure/messaging/eventhubs/event_data_batch.hpp"
#include "azure/messaging/eventhubs/eventhubs_exception.hpp"
#include "private/eventhubs_constants.hpp"
#include "private/eventhubs_utilities.hpp"
#include "private/retry_operation.hpp"
@ -24,13 +25,14 @@ namespace Azure { namespace Messaging { namespace EventHubs {
{
auto sasCredential
= std::make_shared<Azure::Core::Amqp::_internal::ServiceBusSasConnectionStringCredential>(
connectionString);
connectionString, eventHub);
m_credential = sasCredential;
m_eventHub
= (sasCredential->GetEntityPath().empty() ? eventHub : sasCredential->GetEntityPath());
m_fullyQualifiedNamespace = sasCredential->GetHostName();
m_targetUrl = _detail::EventHubsServiceScheme + m_fullyQualifiedNamespace + "/" + m_eventHub;
}
ProducerClient::ProducerClient(
@ -39,76 +41,11 @@ namespace Azure { namespace Messaging { namespace EventHubs {
std::shared_ptr<Azure::Core::Credentials::TokenCredential> credential,
Azure::Messaging::EventHubs::ProducerClientOptions options)
: m_fullyQualifiedNamespace{fullyQualifiedNamespace}, m_eventHub{eventHub},
m_targetUrl{_detail::EventHubsServiceScheme + m_fullyQualifiedNamespace + "/" + m_eventHub},
m_credential{credential}, m_producerClientOptions(options)
{
}
void ProducerClient::EnsureSession(std::string const& partitionId = {})
{
if (m_sessions.find(partitionId) == m_sessions.end())
{
Azure::Core::Amqp::_internal::ConnectionOptions connectOptions;
connectOptions.ContainerId = m_producerClientOptions.ApplicationID;
connectOptions.EnableTrace = true;
connectOptions.AuthenticationScopes = {"https://eventhubs.azure.net/.default"};
// Set the UserAgent related properties on this message sender.
_detail::EventHubsUtilities::SetUserAgent(
connectOptions, m_producerClientOptions.ApplicationID);
std::string fullyQualifiedNamespace{m_fullyQualifiedNamespace};
Azure::Core::Amqp::_internal::Connection connection(
fullyQualifiedNamespace, m_credential, connectOptions);
Azure::Core::Amqp::_internal::SessionOptions sessionOptions;
sessionOptions.InitialIncomingWindowSize = std::numeric_limits<int32_t>::max();
sessionOptions.InitialOutgoingWindowSize = std::numeric_limits<uint16_t>::max();
Azure::Core::Amqp::_internal::Session session{connection.CreateSession(sessionOptions)};
m_sessions.emplace(partitionId, session);
}
}
Azure::Core::Amqp::_internal::Session ProducerClient::GetSession(std::string const& partitionId)
{
return m_sessions.at(partitionId);
}
void ProducerClient::EnsureSender(
std::string const& partitionId,
Azure::Core::Context const& context)
{
if (m_senders.find(partitionId) == m_senders.end())
{
m_targetUrl = "amqps://" + m_fullyQualifiedNamespace + "/" + m_eventHub;
EnsureSession(partitionId);
std::string targetUrl = m_targetUrl;
if (!partitionId.empty())
{
targetUrl += "/Partitions/" + partitionId;
}
Azure::Core::Amqp::_internal::MessageSenderOptions senderOptions;
senderOptions.Name = m_producerClientOptions.Name;
senderOptions.EnableTrace = true;
senderOptions.MaxMessageSize = m_producerClientOptions.MaxMessageSize;
Azure::Core::Amqp::_internal::MessageSender sender
= GetSession(partitionId).CreateMessageSender(targetUrl, senderOptions, nullptr);
sender.Open(context);
m_senders.emplace(partitionId, sender);
}
}
Azure::Core::Amqp::_internal::MessageSender ProducerClient::GetSender(
std::string const& partitionId)
{
return m_senders.at(partitionId);
}
EventDataBatch ProducerClient::CreateBatch(
EventDataBatchOptions const& options,
Core::Context const& context)
@ -169,22 +106,107 @@ namespace Azure { namespace Messaging { namespace EventHubs {
Send(batch, context);
}
Azure::Core::Amqp::_internal::Connection ProducerClient::CreateConnection()
{
Azure::Core::Amqp::_internal::ConnectionOptions connectOptions;
connectOptions.ContainerId = m_producerClientOptions.ApplicationID;
connectOptions.EnableTrace = _detail::EnableAmqpTrace;
connectOptions.AuthenticationScopes = {"https://eventhubs.azure.net/.default"};
// Set the UserAgent related properties on this message sender.
_detail::EventHubsUtilities::SetUserAgent(
connectOptions, m_producerClientOptions.ApplicationID);
return Azure::Core::Amqp::_internal::Connection{
m_fullyQualifiedNamespace, m_credential, connectOptions};
}
void ProducerClient::EnsureConnection(std::string const& partitionId)
{
std::unique_lock<std::mutex> lock(m_sessionsLock);
if (m_connections.find(partitionId) == m_connections.end())
{
m_connections.emplace(partitionId, CreateConnection());
}
}
void ProducerClient::EnsureSession(std::string const& partitionId)
{
// Ensure that a connection has been created for this producer.
EnsureConnection(partitionId);
// Ensure that a session has been created for this partition.
std::unique_lock<std::mutex> lock(m_sessionsLock);
if (m_sessions.find(partitionId) == m_sessions.end())
{
m_sessions.emplace(partitionId, CreateSession(partitionId));
}
}
Azure::Core::Amqp::_internal::Session ProducerClient::GetSession(std::string const& partitionId)
{
std::unique_lock<std::mutex> lock(m_sessionsLock);
return m_sessions.at(partitionId);
}
void ProducerClient::EnsureSender(
std::string const& partitionId,
Azure::Core::Context const& context)
{
std::unique_lock<std::mutex> lock(m_sendersLock);
if (m_senders.find(partitionId) == m_senders.end())
{
EnsureSession(partitionId);
std::string targetUrl{m_targetUrl};
if (!partitionId.empty())
{
targetUrl += "/Partitions/" + partitionId;
}
Azure::Core::Amqp::_internal::MessageSenderOptions senderOptions;
senderOptions.Name = m_producerClientOptions.Name;
senderOptions.EnableTrace = _detail::EnableAmqpTrace;
senderOptions.MaxMessageSize = m_producerClientOptions.MaxMessageSize;
Azure::Core::Amqp::_internal::MessageSender sender
= GetSession(partitionId).CreateMessageSender(targetUrl, senderOptions, nullptr);
sender.Open(context);
m_senders.emplace(partitionId, std::move(sender));
}
}
Azure::Core::Amqp::_internal::MessageSender ProducerClient::GetSender(
std::string const& partitionId)
{
return m_senders.at(partitionId);
}
Azure::Core::Amqp::_internal::Session ProducerClient::CreateSession(
std::string const& partitionId)
{
Azure::Core::Amqp::_internal::SessionOptions sessionOptions;
sessionOptions.InitialIncomingWindowSize = std::numeric_limits<int32_t>::max();
sessionOptions.InitialOutgoingWindowSize = std::numeric_limits<uint16_t>::max();
return m_connections.at(partitionId).CreateSession(sessionOptions);
}
Models::EventHubProperties ProducerClient::GetEventHubProperties(Core::Context const& context)
{
// EventHub properties are not associated with a particular partition, so create a message
// sender on the empty partition.
EnsureSession();
EnsureConnection({});
return _detail::EventHubsUtilities::GetEventHubsProperties(GetSession(), m_eventHub, context);
EnsureSession({});
auto session{GetSession({})};
return _detail::EventHubsUtilities::GetEventHubsProperties(session, m_eventHub, context);
}
Models::EventHubPartitionProperties ProducerClient::GetPartitionProperties(
std::string const& partitionId,
Core::Context const& context)
{
EnsureConnection(partitionId);
EnsureSession(partitionId);
Azure::Core::Amqp::_internal::Session session{GetSession(partitionId)};
return _detail::EventHubsUtilities::GetEventHubsPartitionProperties(
GetSession(partitionId), m_eventHub, partitionId, context);
session, m_eventHub, partitionId, context);
}
}}} // namespace Azure::Messaging::EventHubs

2
sdk/eventhubs/azure-messaging-eventhubs/test/CMakeLists.txt
View File

@ -7,7 +7,7 @@
cmake_minimum_required (VERSION 3.13)
if(BUILD_TESTING)
add_compile_definitions(TESTING_BUILD_AMQP)
add_compile_definitions(azure_TESTING_BUILD_AMQP)
if (NOT AZ_ALL_LIBRARIES OR FETCH_SOURCE_DEPS)
include(AddGoogleTest)
enable_testing ()

33
sdk/eventhubs/azure-messaging-eventhubs/test/ut/checkpoint_store_test.cpp
View File

@ -35,14 +35,16 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{
std::string const testName = GetRandomName();
std::string consumerGroup = GetEnv("EVENTHUB_CONSUMER_GROUP");
Azure::Messaging::EventHubs::Test::TestCheckpointStore checkpointStore;
auto checkpoints = checkpointStore.ListCheckpoints(
std::shared_ptr<CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
auto checkpoints = checkpointStore->ListCheckpoints(
"fully-qualified-namespace", "event-hub-name", "consumer-group");
EXPECT_EQ(0ul, checkpoints.size());
checkpointStore.UpdateCheckpoint(Azure::Messaging::EventHubs::Models::Checkpoint{
checkpointStore->UpdateCheckpoint(Azure::Messaging::EventHubs::Models::Checkpoint{
consumerGroup,
"event-hub-name",
"ns.servicebus.windows.net",
@ -51,7 +53,14 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
202,
});
checkpoints = checkpointStore.ListCheckpoints(
{
// There still should be no checkpoints in the partition we first queried.
checkpoints = checkpointStore->ListCheckpoints(
"fully-qualified-namespace", "event-hub-name", "consumer-group");
EXPECT_EQ(0ul, checkpoints.size());
}
checkpoints = checkpointStore->ListCheckpoints(
"ns.servicebus.windows.net", "event-hub-name", consumerGroup);
EXPECT_EQ(checkpoints.size(), 1ul);
EXPECT_EQ(consumerGroup, checkpoints[0].ConsumerGroup);
@ -61,7 +70,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
EXPECT_EQ(202, checkpoints[0].SequenceNumber.Value());
EXPECT_EQ(101, checkpoints[0].Offset.Value());
checkpointStore.UpdateCheckpoint(Azure::Messaging::EventHubs::Models::Checkpoint{
checkpointStore->UpdateCheckpoint(Azure::Messaging::EventHubs::Models::Checkpoint{
consumerGroup,
"event-hub-name",
"ns.servicebus.windows.net",
@ -70,7 +79,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
203,
});
checkpoints = checkpointStore.ListCheckpoints(
checkpoints = checkpointStore->ListCheckpoints(
"ns.servicebus.windows.net", "event-hub-name", consumerGroup);
EXPECT_EQ(checkpoints.size(), 1ul);
EXPECT_EQ(consumerGroup, checkpoints[0].ConsumerGroup);
@ -85,17 +94,17 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{
std::string const testName = GetRandomName();
TestCheckpointStore checkpointStore;
std::unique_ptr<CheckpointStore> checkpointStore = std::make_unique<TestCheckpointStore>();
auto ownerships = checkpointStore.ListOwnership(
auto ownerships = checkpointStore->ListOwnership(
"fully-qualified-namespace", "event-hub-name", "consumer-group");
EXPECT_EQ(0ul, ownerships.size());
ownerships = checkpointStore.ClaimOwnership(
ownerships = checkpointStore->ClaimOwnership(
std::vector<Azure::Messaging::EventHubs::Models::Ownership>{});
EXPECT_EQ(0ul, ownerships.size());
ownerships = checkpointStore.ClaimOwnership(
ownerships = checkpointStore->ClaimOwnership(
std::vector<Azure::Messaging::EventHubs::Models::Ownership>{
Azure::Messaging::EventHubs::Models::Ownership{
"$Default",
@ -118,7 +127,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
//
// This ownership should NOT take precedence over the previous ownership, so the set of
// ownerships returned should be empty.
ownerships = checkpointStore.ClaimOwnership(
ownerships = checkpointStore->ClaimOwnership(
std::vector<Azure::Messaging::EventHubs::Models::Ownership>{
Azure::Messaging::EventHubs::Models::Ownership{
"$Default",
@ -129,7 +138,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
Azure::ETag("randomETAG")}});
EXPECT_EQ(0ul, ownerships.size());
ownerships = checkpointStore.ClaimOwnership(
ownerships = checkpointStore->ClaimOwnership(
std::vector<Azure::Messaging::EventHubs::Models::Ownership>{
Azure::Messaging::EventHubs::Models::Ownership{
"$Default",

66
sdk/eventhubs/azure-messaging-eventhubs/test/ut/consumer_client_test.cpp
View File

@ -47,31 +47,30 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
std::string consumerGroup = GetEnv("EVENTHUB_CONSUMER_GROUP");
std::string eventHubName = GetEnv("EVENTHUB_NAME");
auto client = Azure::Messaging::EventHubs::ConsumerClient(
Azure::Messaging::EventHubs::ConsumerClient client(
connStringNoEntityPath, eventHubName, consumerGroup);
EXPECT_EQ(eventHubName, client.GetEventHubName());
}
TEST_F(ConsumerClientTest, ConnectionStringEntityPath_LIVEONLY_)
{
std::string const connStringNoEntityPath
std::string const connStringWithEntityPath
= GetEnv("EVENTHUB_CONNECTION_STRING") + ";EntityPath=hehe";
std::string consumerGroup = GetEnv("EVENTHUB_CONSUMER_GROUP");
std::string eventHubName = GetEnv("EVENTHUB_NAME");
// The eventHubName parameter is ignored because the eventhub name is in the connection string.
auto client = Azure::Messaging::EventHubs::ConsumerClient(
connStringNoEntityPath, eventHubName, "$DefaultZ");
EXPECT_EQ("hehe", client.GetEventHubName());
EXPECT_EQ("$DefaultZ", client.GetConsumerGroup());
// The eventHubName parameter must match the name in the connection string.because the eventhub
// name is in the connection string.
EXPECT_ANY_THROW(Azure::Messaging::EventHubs::ConsumerClient client(
connStringWithEntityPath, eventHubName, "$DefaultZ"));
}
TEST_F(ConsumerClientTest, ConnectionStringEntityPathNoConsumerGroup_LIVEONLY_)
{
std::string const connStringNoEntityPath = GetEnv("EVENTHUB_CONNECTION_STRING");
std::string eventHubName = GetEnv("EVENTHUB_NAME");
auto client = Azure::Messaging::EventHubs::ConsumerClient(connStringNoEntityPath, eventHubName);
Azure::Messaging::EventHubs::ConsumerClient client(connStringNoEntityPath, eventHubName);
EXPECT_EQ(eventHubName, client.GetEventHubName());
EXPECT_EQ("$Default", client.GetConsumerGroup());
}
@ -80,7 +79,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{
std::string const connStringNoEntityPath
= GetEnv("EVENTHUB_CONNECTION_STRING") + ";EntityPath=hehe";
auto client = Azure::Messaging::EventHubs::ConsumerClient(connStringNoEntityPath);
Azure::Messaging::EventHubs::ConsumerClient client(connStringNoEntityPath);
EXPECT_EQ("hehe", client.GetEventHubName());
EXPECT_EQ("$Default", client.GetConsumerGroup());
}
@ -88,8 +87,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
TEST_F(ConsumerClientTest, ConnectToPartition_LIVEONLY_)
{
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string const connStringNoEntityPath
= GetEnv("EVENTHUB_CONNECTION_STRING") + ";EntityPath=" + eventHubName;
std::string const connString = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ConsumerClientOptions options;
options.ApplicationID
= std::string(testing::UnitTest::GetInstance()->current_test_info()->name())
@ -97,8 +95,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
options.Name = testing::UnitTest::GetInstance()->current_test_case()->name();
auto client = Azure::Messaging::EventHubs::ConsumerClient(
connStringNoEntityPath, eventHubName, "$Default", options);
Azure::Messaging::EventHubs::ConsumerClient client(
connString, eventHubName, "$Default", options);
Azure::Messaging::EventHubs::PartitionClientOptions partitionOptions;
partitionOptions.StartPosition.Inclusive = true;
// We want to consume all messages from the earliest.
@ -117,21 +115,21 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
TEST_F(ConsumerClientTest, GetEventHubProperties_LIVEONLY_)
{
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string const connStringEntityPath
= GetEnv("EVENTHUB_CONNECTION_STRING") + ";EntityPath=" + eventHubName;
std::string const connString = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ConsumerClientOptions options;
options.ApplicationID = testing::UnitTest::GetInstance()->current_test_info()->name();
options.Name = testing::UnitTest::GetInstance()->current_test_case()->name();
auto client = Azure::Messaging::EventHubs::ConsumerClient(connStringEntityPath);
Azure::Messaging::EventHubs::ConsumerClient client(connString, eventHubName);
Azure::Messaging::EventHubs::PartitionClientOptions partitionOptions;
partitionOptions.StartPosition.Inclusive = true;
Azure::Messaging::EventHubs::PartitionClient partitionClient
= client.CreatePartitionClient("0", partitionOptions);
auto result = client.GetEventHubProperties();
Azure::Messaging::EventHubs::Models::EventHubProperties result;
ASSERT_NO_THROW(result = client.GetEventHubProperties());
EXPECT_EQ(result.Name, eventHubName);
EXPECT_TRUE(result.PartitionIds.size() > 0);
}
@ -139,15 +137,14 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
TEST_F(ConsumerClientTest, GetPartitionProperties_LIVEONLY_)
{
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string const connStringEntityPath
= GetEnv("EVENTHUB_CONNECTION_STRING") + ";EntityPath=" + eventHubName;
std::string const connString = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ConsumerClientOptions options;
options.ApplicationID = testing::UnitTest::GetInstance()->current_test_info()->name();
options.Name = testing::UnitTest::GetInstance()->current_test_case()->name();
auto client = Azure::Messaging::EventHubs::ConsumerClient(connStringEntityPath);
Azure::Messaging::EventHubs::ConsumerClient client(connString, eventHubName);
Azure::Messaging::EventHubs::PartitionClientOptions partitionOptions;
partitionOptions.StartPosition.Inclusive = true;
@ -173,19 +170,20 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
std::string eventHubName{GetRandomName("eventhub")};
auto eventHub{eventhubNamespace.CreateEventHub(eventHubName)};
std::string const connStringEntityPath
= GetEnv("EVENTHUB_CONNECTION_STRING") + ";EntityPath=" + eventHubName;
std::string const connString = GetEnv("EVENTHUB_CONNECTION_STRING");
// Populate the eventhub instance with 50 messages.
constexpr size_t numberOfEvents = 50;
GTEST_LOG_(INFO) << "Populate eventhubs instance.";
{
Azure::Messaging::EventHubs::ProducerClientOptions producerOptions;
producerOptions.ApplicationID = testing::UnitTest::GetInstance()->current_test_info()->name();
producerOptions.Name = testing::UnitTest::GetInstance()->current_test_info()->name();
Azure::Messaging::EventHubs::ProducerClient producer{connStringEntityPath, eventHubName};
Azure::Messaging::EventHubs::ProducerClient producer{connString, eventHubName};
EventDataBatchOptions eventBatchOptions;
eventBatchOptions.PartitionId = "0";
EventDataBatch batch{producer.CreateBatch(eventBatchOptions)};
for (int i = 0; i < 50; ++i)
for (size_t i = 0; i < numberOfEvents; ++i)
{
EXPECT_TRUE(batch.TryAddMessage(Models::EventData{"Test"}));
}
@ -198,10 +196,9 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{
Azure::Messaging::EventHubs::ConsumerClientOptions options;
options.ApplicationID = testing::UnitTest::GetInstance()->current_test_info()->name();
options.Name = testing::UnitTest::GetInstance()->current_test_case()->name();
auto client = Azure::Messaging::EventHubs::ConsumerClient(connStringEntityPath);
Azure::Messaging::EventHubs::ConsumerClient client(connString, eventHubName);
Azure::Messaging::EventHubs::PartitionClientOptions partitionOptions;
partitionOptions.StartPosition.Earliest = true;
partitionOptions.StartPosition.Inclusive = true;
@ -211,28 +208,35 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
// Sleep for a bit for the messages to be received.
GTEST_LOG_(INFO) << "Sleep until messages received.";
std::this_thread::sleep_for(std::chrono::milliseconds(500));
std::this_thread::sleep_for(std::chrono::seconds(2));
size_t totalReceived{0};
{
std::chrono::system_clock::time_point start = std::chrono::system_clock::now();
auto messages = partitionClient.ReceiveEvents(5);
std::chrono::system_clock::time_point end = std::chrono::system_clock::now();
std::chrono::duration<double> elapsed_seconds = end - start;
EXPECT_EQ(messages.size(), 5ul);
EXPECT_GE(messages.size(), 1ul);
EXPECT_LE(messages.size(), 5ul);
EXPECT_TRUE(elapsed_seconds.count() < 1);
totalReceived += messages.size();
}
// We should have 45 messages left, which we should get immediately.
do
{
std::chrono::system_clock::time_point start = std::chrono::system_clock::now();
auto messages = partitionClient.ReceiveEvents(50);
std::chrono::system_clock::time_point end = std::chrono::system_clock::now();
std::chrono::duration<double> elapsed_seconds = end - start;
EXPECT_EQ(messages.size(), 45ul);
EXPECT_LE(messages.size(), 45ul);
totalReceived += messages.size();
EXPECT_TRUE(elapsed_seconds.count() < 1);
}
} while (totalReceived < numberOfEvents);
// Now when we wait, we should block.
EXPECT_EQ(totalReceived, numberOfEvents);
// We have consumed all the events. Attempting to consume one more should block.
{
Azure::Core::Context timeout = Azure::Core::Context::ApplicationContext.WithDeadline(
Azure::DateTime::clock::now() + std::chrono::seconds(3));

26
sdk/eventhubs/azure-messaging-eventhubs/test/ut/eventhubs_admin_client.cpp
View File

@ -129,7 +129,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
throw Azure::Core::RequestFailedException(result);
}
Azure::Core::Json::_internal::json jsonOutput = ParseAzureCliOutput(bodyAsString);
Azure::Core::Json::_internal::json jsonOutput
= Azure::Core::Json::_internal::json::parse(bodyAsString);
if (jsonOutput.is_null())
{
throw std::runtime_error("JSON output is null!");
@ -343,7 +344,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
throw Azure::Core::RequestFailedException(result);
}
Azure::Core::Json::_internal::json jsonOutput = ParseAzureCliOutput(bodyAsString);
Azure::Core::Json::_internal::json jsonOutput
= Azure::Core::Json::_internal::json::parse(bodyAsString);
if (jsonOutput.is_null())
{
throw std::runtime_error("JSON output is null!");
@ -396,7 +398,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
throw Azure::Core::RequestFailedException(result);
}
Azure::Core::Json::_internal::json jsonOutput = ParseAzureCliOutput(bodyAsString);
Azure::Core::Json::_internal::json jsonOutput
= Azure::Core::Json::_internal::json::parse(bodyAsString);
if (jsonOutput.is_null())
{
throw std::runtime_error("JSON output is null!");
@ -452,7 +455,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
throw Azure::Core::RequestFailedException(result);
}
Azure::Core::Json::_internal::json jsonOutput = ParseAzureCliOutput(bodyAsString);
Azure::Core::Json::_internal::json jsonOutput
= Azure::Core::Json::_internal::json::parse(bodyAsString);
if (jsonOutput.is_null())
{
throw std::runtime_error("JSON output is null!");
@ -482,7 +486,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
throw Azure::Core::RequestFailedException(result);
}
Azure::Core::Json::_internal::json jsonOutput = ParseAzureCliOutput(bodyAsString);
Azure::Core::Json::_internal::json jsonOutput
= Azure::Core::Json::_internal::json::parse(bodyAsString);
if (jsonOutput.is_null())
{
throw std::runtime_error("JSON output is null!");
@ -544,7 +549,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
throw Azure::Core::RequestFailedException(result);
}
Azure::Core::Json::_internal::json jsonOutput = ParseAzureCliOutput(bodyAsString);
Azure::Core::Json::_internal::json jsonOutput
= Azure::Core::Json::_internal::json::parse(bodyAsString);
return EventHub(eventHubName, m_resourceGroup, m_subscriptionId);
}
@ -564,14 +570,14 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
request.SetHeader("Accept", "application/json");
auto result = m_pipeline->Send(request, context);
auto& val = result->GetBody();
std::string bodyAsString{reinterpret_cast<const char*>(val.data()), val.size()};
if (result->GetStatusCode() != Azure::Core::Http::HttpStatusCode::Ok)
if (result->GetStatusCode() != Azure::Core::Http::HttpStatusCode::Ok
&& result->GetStatusCode() != Azure::Core::Http::HttpStatusCode::NoContent)
{
throw Azure::Core::RequestFailedException(result);
}
Azure::Core::Json::_internal::json jsonOutput = ParseAzureCliOutput(bodyAsString);
// There is no expected body on a delete eventhub response.
return true;
}

113
sdk/eventhubs/azure-messaging-eventhubs/test/ut/processor_load_balancer_test.cpp
View File

@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
#include "../src/private/processor_load_balancer.hpp"
#include "eventhubs_test_base.hpp"
#include "test_checkpoint_store.hpp"
@ -110,13 +111,14 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
} // namespace
TEST_F(ProcessorLoadBalancerTest, Greedy_EnoughUnownedPartitions)
{
Azure::Messaging::EventHubs::Test::TestCheckpointStore checkpointStore;
std::shared_ptr<CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
checkpointStore.ClaimOwnership(std::vector<Azure::Messaging::EventHubs::Models::Ownership>{
checkpointStore->ClaimOwnership(std::vector<Azure::Messaging::EventHubs::Models::Ownership>{
TestOwnership("0", "some-client"), TestOwnership("3", "some-client")});
Azure::Messaging::EventHubs::ProcessorLoadBalancer loadBalancer(
std::make_shared<TestCheckpointStore>(checkpointStore),
Azure::Messaging::EventHubs::_detail::ProcessorLoadBalancer loadBalancer(
checkpointStore,
TestConsumerDetails("new-client"),
Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyGreedy,
std::chrono::minutes(2));
@ -133,13 +135,14 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
TEST_F(ProcessorLoadBalancerTest, Balanced_UnownedPartitions)
{
Azure::Messaging::EventHubs::Test::TestCheckpointStore checkpointStore;
std::shared_ptr<CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
checkpointStore.ClaimOwnership(std::vector<Azure::Messaging::EventHubs::Models::Ownership>{
checkpointStore->ClaimOwnership(std::vector<Azure::Messaging::EventHubs::Models::Ownership>{
TestOwnership("0", "some-client"), TestOwnership("3", "some-client")});
Azure::Messaging::EventHubs::ProcessorLoadBalancer loadBalancer(
std::make_shared<TestCheckpointStore>(checkpointStore),
Azure::Messaging::EventHubs::_detail::ProcessorLoadBalancer loadBalancer(
checkpointStore,
TestConsumerDetails("new-client"),
Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyBalanced,
std::chrono::minutes(2));
@ -161,17 +164,18 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
TEST_F(ProcessorLoadBalancerTest, Greedy_ForcedToSteal)
{
Azure::Messaging::EventHubs::Test::TestCheckpointStore checkpointStore;
std::shared_ptr<CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
checkpointStore.ClaimOwnership(std::vector<Azure::Messaging::EventHubs::Models::Ownership>{
checkpointStore->ClaimOwnership(std::vector<Azure::Messaging::EventHubs::Models::Ownership>{
TestOwnership("0", "some-client"),
TestOwnership("1", "some-client"),
TestOwnership("2", "some-client"),
TestOwnership("3", "some-client"),
});
Azure::Messaging::EventHubs::ProcessorLoadBalancer loadBalancer(
std::make_shared<TestCheckpointStore>(checkpointStore),
Azure::Messaging::EventHubs::_detail::ProcessorLoadBalancer loadBalancer(
checkpointStore,
TestConsumerDetails("new-client"),
Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyGreedy,
std::chrono::minutes(2));
@ -192,27 +196,26 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyBalanced,
Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyGreedy})
{
Azure::Messaging::EventHubs::Test::TestCheckpointStore checkpointStore;
std::shared_ptr<TestCheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
const std::string clientA = "clientA";
const std::string clientB = "clientB";
Models::Ownership midOwner = TestOwnership("2", "clientC");
checkpointStore.ClaimOwnership(
checkpointStore->ClaimOwnership(
{TestOwnership("0", clientA),
TestOwnership("1", clientA),
midOwner,
TestOwnership("3", clientB),
TestOwnership("4", clientB)});
checkpointStore.ExpireOwnership(midOwner);
// NOTE: TEST HOOK FOR ExpireOwnership.
checkpointStore->ExpireOwnership(midOwner);
Azure::Messaging::EventHubs::ProcessorLoadBalancer loadBalancer(
std::make_shared<TestCheckpointStore>(checkpointStore),
TestConsumerDetails(clientB),
strategy,
std::chrono::minutes(2));
Azure::Messaging::EventHubs::_detail::ProcessorLoadBalancer loadBalancer(
checkpointStore, TestConsumerDetails(clientB), strategy, std::chrono::minutes(2));
auto ownerships = loadBalancer.LoadBalance({"0", "1", "2", "3", "4"});
@ -231,12 +234,13 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyBalanced,
Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyGreedy})
{
Azure::Messaging::EventHubs::Test::TestCheckpointStore checkpointStore;
std::shared_ptr<CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
const std::string clientA = "clientA";
const std::string clientB = "clientB";
checkpointStore.ClaimOwnership(
checkpointStore->ClaimOwnership(
{TestOwnership("0", clientA),
TestOwnership("1", clientA),
TestOwnership("2", clientA),
@ -245,11 +249,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{
Azure::Messaging::EventHubs::ProcessorLoadBalancer loadBalancer(
std::make_shared<TestCheckpointStore>(checkpointStore),
TestConsumerDetails(clientB),
strategy,
std::chrono::minutes(2));
Azure::Messaging::EventHubs::_detail::ProcessorLoadBalancer loadBalancer(
checkpointStore, TestConsumerDetails(clientB), strategy, std::chrono::minutes(2));
auto ownerships = loadBalancer.LoadBalance({"0", "1", "2", "3", "4"});
@ -266,11 +267,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{
Azure::Messaging::EventHubs::ProcessorLoadBalancer loadBalancer(
std::make_shared<TestCheckpointStore>(checkpointStore),
TestConsumerDetails(clientA),
strategy,
std::chrono::minutes(2));
Azure::Messaging::EventHubs::_detail::ProcessorLoadBalancer loadBalancer(
checkpointStore, TestConsumerDetails(clientA), strategy, std::chrono::minutes(2));
auto ownerships = loadBalancer.LoadBalance({"0", "1", "2", "3", "4"});
@ -294,12 +292,13 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyBalanced,
Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyGreedy})
{
Azure::Messaging::EventHubs::Test::TestCheckpointStore checkpointStore;
std::shared_ptr<CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
const std::string clientA = "clientA";
const std::string clientB = "clientB";
checkpointStore.ClaimOwnership(
checkpointStore->ClaimOwnership(
{TestOwnership("0", clientA),
TestOwnership("1", clientA),
TestOwnership("2", clientB),
@ -307,11 +306,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{
Azure::Messaging::EventHubs::ProcessorLoadBalancer loadBalancer(
std::make_shared<TestCheckpointStore>(checkpointStore),
TestConsumerDetails(clientB),
strategy,
std::chrono::minutes(2));
Azure::Messaging::EventHubs::_detail::ProcessorLoadBalancer loadBalancer(
checkpointStore, TestConsumerDetails(clientB), strategy, std::chrono::minutes(2));
auto ownerships = loadBalancer.LoadBalance({"0", "1", "2", "3"});
@ -328,11 +324,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{
Azure::Messaging::EventHubs::ProcessorLoadBalancer loadBalancer(
std::make_shared<TestCheckpointStore>(checkpointStore),
TestConsumerDetails(clientA),
strategy,
std::chrono::minutes(2));
Azure::Messaging::EventHubs::_detail::ProcessorLoadBalancer loadBalancer(
checkpointStore, TestConsumerDetails(clientA), strategy, std::chrono::minutes(2));
auto ownerships = loadBalancer.LoadBalance({"0", "1", "2", "3"});
@ -355,23 +348,20 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyBalanced,
Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyGreedy})
{
Azure::Messaging::EventHubs::Test::TestCheckpointStore checkpointStore;
std::shared_ptr<CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
const std::string clientA = "clientA";
const std::string clientB = "clientB";
checkpointStore.ClaimOwnership(
checkpointStore->ClaimOwnership(
{TestOwnership("0", clientA),
TestOwnership("1", clientA),
TestOwnership("2", ""),
TestOwnership("3", clientB),
TestOwnership("4", clientB)});
Azure::Messaging::EventHubs::ProcessorLoadBalancer loadBalancer(
std::make_shared<TestCheckpointStore>(checkpointStore),
TestConsumerDetails(clientB),
strategy,
std::chrono::minutes(2));
Azure::Messaging::EventHubs::_detail::ProcessorLoadBalancer loadBalancer(
checkpointStore, TestConsumerDetails(clientB), strategy, std::chrono::minutes(2));
auto ownerships = loadBalancer.LoadBalance({"0", "1", "2", "3", "4"});
auto clientOwned = GroupByOwner(ownerships)[clientB];
@ -395,12 +385,13 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyBalanced,
Azure::Messaging::EventHubs::Models::ProcessorStrategy::ProcessorStrategyGreedy})
{
Azure::Messaging::EventHubs::Test::TestCheckpointStore checkpointStore;
std::shared_ptr<CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
const std::string clientA = "clientA";
const std::string clientB = "clientB";
checkpointStore.ClaimOwnership(
checkpointStore->ClaimOwnership(
{TestOwnership("0", clientA),
TestOwnership("1", clientA),
TestOwnership("2", clientA),
@ -408,11 +399,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{
Azure::Messaging::EventHubs::ProcessorLoadBalancer loadBalancer(
std::make_shared<TestCheckpointStore>(checkpointStore),
TestConsumerDetails(clientA),
strategy,
std::chrono::minutes(2));
Azure::Messaging::EventHubs::_detail::ProcessorLoadBalancer loadBalancer(
checkpointStore, TestConsumerDetails(clientA), strategy, std::chrono::minutes(2));
auto ownerships = loadBalancer.LoadBalance({"0", "1", "2", "3"});
@ -426,11 +414,8 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
{
Azure::Messaging::EventHubs::ProcessorLoadBalancer loadBalancer(
std::make_shared<TestCheckpointStore>(checkpointStore),
TestConsumerDetails(clientB),
strategy,
std::chrono::minutes(2));
Azure::Messaging::EventHubs::_detail::ProcessorLoadBalancer loadBalancer(
checkpointStore, TestConsumerDetails(clientB), strategy, std::chrono::minutes(2));
auto ownerships = loadBalancer.LoadBalance({"0", "1", "2", "3"});

593
sdk/eventhubs/azure-messaging-eventhubs/test/ut/processor_test.cpp
View File

@ -4,26 +4,457 @@
#include "./test_checkpoint_store.hpp"
#include "eventhubs_test_base.hpp"
#include <azure/core/amqp/common/global_state.hpp>
#include <azure/core/context.hpp>
#include <azure/core/platform.hpp>
#include <azure/core/test/test_base.hpp>
#include <azure/identity.hpp>
#include <azure/messaging/eventhubs.hpp>
#include <azure/messaging/eventhubs/consumer_client.hpp>
#include <azure/messaging/eventhubs/processor.hpp>
#include <azure/messaging/eventhubs/producer_client.hpp>
#include <set>
#include <gtest/gtest.h>
// cspell: words azeventhubs
namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
class ProcessorTest : public EventHubsTestBase {
};
namespace {
std::string GetRandomName()
std::string GetRandomName(const char* baseName = "checkpoint")
{
std::string name = "checkpoint";
std::string name = baseName;
name.append(Azure::Core::Uuid::CreateUuid().ToString());
return name;
}
class scope_guard {
std::function<void()> m_callback;
public:
scope_guard(std::function<void()>&& callback) : m_callback(std::move(callback)) {}
~scope_guard() noexcept { m_callback(); }
};
// C++ implementation of Go "WaitGroup" construct.
class WaitGroup {
public:
void Wait()
{
std::unique_lock<std::mutex> lock(m_stateLock);
if (m_waitables == 0)
{
return;
}
m_waitComplete.wait(lock, [this]() -> bool { return m_waitables == 0; });
}
void AddWaiter(int32_t count = 1)
{
std::unique_lock<std::mutex> lock(m_stateLock);
m_waitables += count;
}
void CompleteWaiter()
{
std::unique_lock<std::mutex> lock(m_stateLock);
m_waitables -= 1;
if (m_waitables == 0)
{
m_waitComplete.notify_all();
}
}
private:
int32_t m_waitables{};
std::mutex m_stateLock;
std::condition_variable m_waitComplete;
};
} // namespace
class ProcessorTest : public EventHubsTestBase {
protected:
static void SetUpTestSuite() { EventHubsTestBase::SetUpTestSuite(); }
static void TearDownTestSuite() { EventHubsTestBase::TearDownTestSuite(); }
void TearDown() override
{
EventHubsTestBase::TearDown();
// When the test is torn down, the global state MUST be idle. If it is not, something leaked.
Azure::Core::Amqp::Common::_detail::GlobalStateHolder::GlobalStateInstance()->AssertIdle();
}
void TestWithLoadBalancer(Models::ProcessorStrategy processorStrategy)
{
Azure::Core::Context context = Azure::Core::Context::ApplicationContext.WithDeadline(
Azure::DateTime::clock::now() + std::chrono::minutes(5));
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string consumerGroup = GetEnv("EVENTHUB_CONSUMER_GROUP");
std::string const connectionString = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ConsumerClientOptions consumerClientOptions;
consumerClientOptions.ApplicationID
= testing::UnitTest::GetInstance()->current_test_info()->name();
consumerClientOptions.Name = testing::UnitTest::GetInstance()->current_test_info()->name();
std::string containerName{GetRandomName("proctest")};
// Create the checkpoint store
std::shared_ptr<CheckpointStore> checkpointStore{std::make_shared<TestCheckpointStore>()};
GTEST_LOG_(INFO) << "Checkpoint store created";
std::shared_ptr<ConsumerClient> consumerClient{std::make_shared<ConsumerClient>(
connectionString, eventHubName, consumerGroup, consumerClientOptions)};
GTEST_LOG_(INFO) << "Consumer Client created";
ProcessorOptions processorOptions;
processorOptions.LoadBalancingStrategy = processorStrategy;
processorOptions.UpdateInterval = std::chrono::milliseconds(1000);
processorOptions.Prefetch = 1500; // Set the initial link credits to 1500.
Processor processor{consumerClient, checkpointStore, processorOptions};
// Warm up the consumer client - establish connection to the server, etc.
auto eventHubProperties = consumerClient->GetEventHubProperties(context);
ProducerClientOptions producerOptions;
producerOptions.Name = "Producer for LoadBalancerTest";
ProducerClient producerClient{connectionString, eventHubName, producerOptions};
#if PROCESSOR_ON_TEST_THREAD
std::thread processEventsThread([&]() {
std::set<std::string> partitionsAcquired;
std::vector<std::thread> processEventsThreads;
// When we exit the process thread, cancel the context to unblock the processor.
scope_guard onExit([&context] { context.Cancel(); });
WaitGroup waitGroup;
for (auto const& partitionId : eventHubProperties.PartitionIds)
{
std::shared_ptr<ProcessorPartitionClient> partitionClient
= processor.NextPartitionClient(context);
waitGroup.AddWaiter();
ASSERT_EQ(partitionsAcquired.find(partitionId), partitionsAcquired.end())
<< "No previous client for " << partitionClient->PartitionId();
processEventsThreads.push_back(
std::thread([&waitGroup, &producerClient, partitionClient, &context, this] {
scope_guard onExit([&] { waitGroup.CompleteWaiter(); });
ProcessEventsForLoadBalancerTest(producerClient, partitionClient, context);
}));
}
// Block until all the events have been processed.
waitGroup.Wait();
// And wait until all the threads have completed.
for (auto& thread : processEventsThreads)
{
if (thread.joinable())
{
thread.join();
}
}
// Stop the processor, we're done with the test.
processor.Stop();
});
processor.Run(context);
processEventsThread.join();
#else
// Run the processor on a background thread and the test on the foreground.
processor.Start(context);
#if ASYNC_PROCESS_EVENTS
std::set<std::string> partitionsAcquired;
std::vector<std::thread> processEventsThreads;
// When we exit the process thread, cancel the context to unblock the processor.
// scope_guard onExit([&context] { context.Cancel(); });
WaitGroup waitGroup;
for (auto const& partitionId : eventHubProperties.PartitionIds)
{
std::shared_ptr<ProcessorPartitionClient> partitionClient
= processor.NextPartitionClient(context);
waitGroup.AddWaiter();
ASSERT_EQ(partitionsAcquired.find(partitionId), partitionsAcquired.end())
<< "No previous client for " << partitionClient->PartitionId();
processEventsThreads.push_back(
std::thread([&waitGroup, &producerClient, partitionClient, &context, this] {
scope_guard onExit([&] { waitGroup.CompleteWaiter(); });
ProcessEventsForLoadBalancerTest(producerClient, partitionClient, context);
}));
}
// Block until all the events have been processed.
waitGroup.Wait();
// And wait until all the threads have completed.
for (auto& thread : processEventsThreads)
{
if (thread.joinable())
{
thread.join();
}
}
#else
std::set<std::string> partitionsAcquired;
for (auto const& partitionId : eventHubProperties.PartitionIds)
{
std::shared_ptr<ProcessorPartitionClient> partitionClient
= processor.NextPartitionClient(context);
ASSERT_EQ(partitionsAcquired.find(partitionId), partitionsAcquired.end())
<< "No previous client for " << partitionClient->PartitionId();
ProcessEventsForLoadBalancerTest(producerClient, partitionClient, context);
}
#endif
// Stop the processor, we're done with the test.
processor.Stop();
#endif
}
void TestPartitionAcquisition(Models::ProcessorStrategy processorStrategy)
{
Azure::Core::Context context = Azure::Core::Context::ApplicationContext.WithDeadline(
Azure::DateTime::clock::now() + std::chrono::minutes(5));
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string consumerGroup = GetEnv("EVENTHUB_CONSUMER_GROUP");
std::string const connString = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ConsumerClientOptions consumerClientOptions;
consumerClientOptions.ApplicationID
= testing::UnitTest::GetInstance()->current_test_info()->name();
consumerClientOptions.Name = testing::UnitTest::GetInstance()->current_test_info()->name();
std::string containerName{GetRandomName("proctest")};
// Create the checkpoint store
std::shared_ptr<CheckpointStore> checkpointStore{std::make_shared<TestCheckpointStore>()};
GTEST_LOG_(INFO) << "Checkpoint store created";
std::shared_ptr<ConsumerClient> consumerClient{std::make_shared<ConsumerClient>(
connString, eventHubName, consumerGroup, consumerClientOptions)};
GTEST_LOG_(INFO) << "Consumer Client created";
ProcessorOptions processorOptions;
processorOptions.LoadBalancingStrategy = processorStrategy;
processorOptions.UpdateInterval = std::chrono::milliseconds(1);
Processor processor{consumerClient, checkpointStore, processorOptions};
Azure::Core::Context runContext;
processor.Start(runContext);
scope_guard onExit{[&processor]() { processor.Stop(); }};
auto eventhubProperties{consumerClient->GetEventHubProperties(runContext)};
std::set<std::string> partitionsAcquired;
for (auto& partitionId : eventhubProperties.PartitionIds)
{
GTEST_LOG_(INFO) << "Waiting for next partition client. Might be " << partitionId;
auto partitionClient = processor.NextPartitionClient(runContext);
EXPECT_EQ(partitionsAcquired.find(partitionClient->PartitionId()), partitionsAcquired.end())
<< "No previous client for partition " << partitionClient->PartitionId();
partitionsAcquired.emplace(partitionClient->PartitionId());
}
}
void ProcessEventsForLoadBalancerTest(
ProducerClient& producerClient,
std::shared_ptr<ProcessorPartitionClient> partitionClient,
Azure::Core::Context const& context)
{
Azure::Core::Context producerContext{context};
try
{
// initialize any resources needed to process the partition
// This is the equivalent to PartitionOpen
GTEST_LOG_(INFO) << "Started processing partition " << partitionClient->PartitionId();
const int32_t expectedEventsCount = 1000;
const int32_t batchSize = 1000;
EXPECT_EQ(0, expectedEventsCount % batchSize)
<< "Keep the math simple - even # of messages for each batch";
#if ASYNC_GENERATE_EVENTS
std::thread produceEvents([&, partitionClient]() {
// Wait for 10 seconds for all of the consumer clients to be spun up.
GTEST_LOG_(INFO)
<< "Produce Events thread: Wait for 10 seconds for processor to create receivers.";
std::this_thread::sleep_for(std::chrono::seconds(10));
try
{
GTEST_LOG_(INFO) << "Generate " << std::dec << expectedEventsCount << " events in "
<< std::dec << (expectedEventsCount / batchSize) << " batch messages.";
for (int i = 0; i < expectedEventsCount / batchSize; i++)
{
producerContext.ThrowIfCancelled();
EventDataBatchOptions batchOptions;
batchOptions.PartitionId = partitionClient->PartitionId();
auto batch = producerClient.CreateBatch(batchOptions, producerContext);
for (int j = 0; j < batchSize; j++)
{
std::stringstream ss;
ss << "[" << partitionClient->PartitionId() << ":[" << i << ":" << j
<< "]] Message";
batch.TryAddMessage(Models::EventData{ss.str()});
}
GTEST_LOG_(INFO) << "Send batch " << i << ", targeting partition "
<< partitionClient->PartitionId();
producerClient.Send(batch, producerContext);
}
}
catch (std::runtime_error& ex)
{
GTEST_LOG_(FATAL) << "Exception thrown sending messages" << ex.what();
}
});
#else
try
{
GTEST_LOG_(INFO) << "Generate " << std::dec << expectedEventsCount << " events in "
<< std::dec << (expectedEventsCount / batchSize)
<< " batch messages on partition " << partitionClient->PartitionId();
for (int i = 0; i < expectedEventsCount / batchSize; i++)
{
producerContext.ThrowIfCancelled();
EventDataBatchOptions batchOptions;
batchOptions.PartitionId = partitionClient->PartitionId();
auto batch = producerClient.CreateBatch(batchOptions, producerContext);
for (int j = 0; j < batchSize; j++)
{
std::stringstream ss;
ss << "[" << partitionClient->PartitionId() << ":[" << i << ":" << j << "]] Message";
batch.TryAddMessage(Models::EventData{ss.str()});
}
GTEST_LOG_(INFO) << "Send batch " << i << ", targeting partition "
<< partitionClient->PartitionId();
producerClient.Send(batch, producerContext);
}
}
catch (std::runtime_error& ex)
{
GTEST_LOG_(FATAL) << "Exception thrown sending messages" << ex.what();
}
#endif
std::vector<Models::ReceivedEventData> allEvents;
while (!context.IsCancelled())
{
auto receiveContext
= context.WithDeadline(Azure::DateTime::clock::now() + std::chrono::seconds(50));
GTEST_LOG_(INFO) << "Receive up to 100 events with a 50 second timeout on partition "
<< partitionClient->PartitionId();
auto events = partitionClient->ReceiveEvents(100, receiveContext);
if (events.size() != 0)
{
GTEST_LOG_(INFO) << "Processing " << events.size() << " events for partition "
<< partitionClient->PartitionId();
allEvents.insert(allEvents.end(), events.begin(), events.end());
GTEST_LOG_(INFO) << "Updating checkpoint for partition "
<< partitionClient->PartitionId();
partitionClient->UpdateCheckpoint(events.back(), context);
if (allEvents.size() == expectedEventsCount)
{
GTEST_LOG_(INFO) << "Received all expected events; returning.";
#if ASYNC_GENERATE_EVENTS
produceEvents.join();
#endif
return;
}
}
}
}
catch (std::runtime_error const& ex)
{
GTEST_LOG_(FATAL) << "Exception thrown receiving messages." << ex.what();
producerContext.Cancel();
}
}
};
TEST_F(ProcessorTest, BasicTest)
{
std::shared_ptr<Azure::Messaging::EventHubs::CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string consumerGroup = GetEnv("EVENTHUB_CONSUMER_GROUP");
std::string const connStringNoEntityPath = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ConsumerClientOptions consumerClientOptions;
consumerClientOptions.ApplicationID
= testing::UnitTest::GetInstance()->current_test_info()->name();
consumerClientOptions.Name = testing::UnitTest::GetInstance()->current_test_info()->name();
ProcessorOptions processorOptions;
processorOptions.LoadBalancingStrategy = Models::ProcessorStrategy::ProcessorStrategyBalanced;
processorOptions.UpdateInterval = std::chrono::seconds(2);
Processor processor(
std::make_shared<ConsumerClient>(
connStringNoEntityPath, eventHubName, consumerGroup, consumerClientOptions),
checkpointStore,
processorOptions);
}
TEST_F(ProcessorTest, StartStop_LIVEONLY_)
{
std::shared_ptr<Azure::Messaging::EventHubs::CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string consumerGroup = GetEnv("EVENTHUB_CONSUMER_GROUP");
std::string const connStringNoEntityPath = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ConsumerClientOptions consumerClientOptions;
consumerClientOptions.ApplicationID
= testing::UnitTest::GetInstance()->current_test_info()->name();
consumerClientOptions.Name = testing::UnitTest::GetInstance()->current_test_info()->name();
ProcessorOptions processorOptions;
processorOptions.LoadBalancingStrategy = Models::ProcessorStrategy::ProcessorStrategyBalanced;
processorOptions.UpdateInterval = std::chrono::seconds(2);
Processor processor(
std::make_shared<ConsumerClient>(
connStringNoEntityPath, eventHubName, consumerGroup, consumerClientOptions),
checkpointStore,
processorOptions);
processor.Start();
processor.Stop();
processor.Close();
}
TEST_F(ProcessorTest, JustStop_LIVEONLY_)
{
std::shared_ptr<Azure::Messaging::EventHubs::CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string consumerGroup = GetEnv("EVENTHUB_CONSUMER_GROUP");
std::string const connStringNoEntityPath = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ConsumerClientOptions consumerClientOptions;
consumerClientOptions.ApplicationID
= testing::UnitTest::GetInstance()->current_test_info()->name();
consumerClientOptions.Name = testing::UnitTest::GetInstance()->current_test_info()->name();
ProcessorOptions processorOptions;
processorOptions.LoadBalancingStrategy = Models::ProcessorStrategy::ProcessorStrategyBalanced;
processorOptions.UpdateInterval = std::chrono::seconds(2);
Processor processor(
std::make_shared<ConsumerClient>(
connStringNoEntityPath, eventHubName, consumerGroup, consumerClientOptions),
checkpointStore,
processorOptions);
processor.Stop();
processor.Close();
}
TEST_F(ProcessorTest, LoadBalancing_LIVEONLY_)
{
std::string const testName = GetRandomName();
@ -33,27 +464,157 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string consumerGroup = GetEnv("EVENTHUB_CONSUMER_GROUP");
std::string const connStringNoEntityPath
= GetEnv("EVENTHUB_CONNECTION_STRING") + ";EntityPath=" + eventHubName;
Azure::Messaging::EventHubs::ConsumerClientOptions options;
options.ApplicationID = testing::UnitTest::GetInstance()->current_test_info()->name();
options.Name = testing::UnitTest::GetInstance()->current_test_info()->name();
std::string const connStringNoEntityPath = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ConsumerClientOptions consumerClientOptions;
consumerClientOptions.ApplicationID
= testing::UnitTest::GetInstance()->current_test_info()->name();
consumerClientOptions.Name = testing::UnitTest::GetInstance()->current_test_info()->name();
auto client = Azure::Messaging::EventHubs::ConsumerClient(
connStringNoEntityPath, eventHubName, consumerGroup, options);
ProcessorOptions processorOptions;
processorOptions.LoadBalancingStrategy = Models::ProcessorStrategy::ProcessorStrategyBalanced;
processorOptions.UpdateInterval = std::chrono::seconds(2);
Processor processor(
std::make_shared<ConsumerClient>(client), checkpointStore, processorOptions);
std::make_shared<ConsumerClient>(
connStringNoEntityPath, eventHubName, consumerGroup, consumerClientOptions),
checkpointStore,
processorOptions);
processor.Run({});
Azure::Core::Context context;
std::thread workerThread([&processor, &context]() { processor.Run(context); });
GTEST_LOG_(INFO) << "Sleep for 5 seconds to allow the processor to stabilize.";
std::this_thread::sleep_for(std::chrono::seconds(5));
GTEST_LOG_(INFO) << "Sleep for 2 seconds to allow the processor to stabilize.";
std::this_thread::sleep_for(std::chrono::seconds(2));
context.Cancel();
// Now wait for the worker thread to finish.
workerThread.join();
processor.Close();
}
TEST_F(ProcessorTest, LoadBalancing_Cancel_LIVEONLY_)
{
std::string const testName = GetRandomName();
std::shared_ptr<Azure::Messaging::EventHubs::CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string consumerGroup = GetEnv("EVENTHUB_CONSUMER_GROUP");
std::string const connStringNoEntityPath = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ConsumerClientOptions consumerClientOptions;
consumerClientOptions.ApplicationID
= testing::UnitTest::GetInstance()->current_test_info()->name();
consumerClientOptions.Name = testing::UnitTest::GetInstance()->current_test_info()->name();
ProcessorOptions processorOptions;
processorOptions.LoadBalancingStrategy = Models::ProcessorStrategy::ProcessorStrategyBalanced;
processorOptions.UpdateInterval = std::chrono::seconds(2);
Processor processor(
std::make_shared<ConsumerClient>(
connStringNoEntityPath, eventHubName, consumerGroup, consumerClientOptions),
checkpointStore,
processorOptions);
Azure::Core::Context runContext;
std::thread workerThread([runContext, &processor]() { processor.Run(runContext); });
GTEST_LOG_(INFO) << "Sleep for 10 seconds to allow the processor to stabilize.";
std::this_thread::sleep_for(std::chrono::seconds(10));
runContext.Cancel();
// Now wait for the worker thread to finish.
workerThread.join();
processor.Close();
}
TEST_F(ProcessorTest, Processor_ClientUniquePartitionClients_LIVEONLY_)
{
std::string const testName = GetRandomName();
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string consumerGroup = GetEnv("EVENTHUB_CONSUMER_GROUP");
std::string const connStringNoEntityPath = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ConsumerClientOptions consumerClientOptions;
consumerClientOptions.ApplicationID
= testing::UnitTest::GetInstance()->current_test_info()->name();
consumerClientOptions.Name = testing::UnitTest::GetInstance()->current_test_info()->name();
auto consumerClient = std::make_shared<ConsumerClient>(
connStringNoEntityPath, eventHubName, consumerGroup, consumerClientOptions);
auto eventhubInfo = consumerClient->GetEventHubProperties();
std::shared_ptr<Azure::Messaging::EventHubs::CheckpointStore> checkpointStore{
std::make_shared<Azure::Messaging::EventHubs::Test::TestCheckpointStore>()};
ProcessorOptions processorOptions;
processorOptions.LoadBalancingStrategy = Models::ProcessorStrategy::ProcessorStrategyBalanced;
processorOptions.UpdateInterval = std::chrono::seconds(2);
Processor processor(consumerClient, checkpointStore, processorOptions);
// Start the processor running.
processor.Start();
std::set<std::string> clientIds;
std::map<std::string, std::shared_ptr<ProcessorPartitionClient>> partitionClients;
for (size_t i = 0; i < eventhubInfo.PartitionIds.size(); i += 1)
{
auto partitionClient = processor.NextPartitionClient();
GTEST_LOG_(INFO) << "Received partition client for partition "
<< partitionClient->PartitionId();
ASSERT_EQ(clientIds.find(partitionClient->PartitionId()), clientIds.end())
<< "Received duplicate partition client for partition " << partitionClient->PartitionId();
clientIds.emplace(partitionClient->PartitionId());
partitionClients.emplace(partitionClient->PartitionId(), partitionClient);
}
// Attempts to retrieve a partition client should throw because there are no clients available.
auto context = Azure::Core::Context::ApplicationContext.WithDeadline(
Azure::DateTime::clock::now() + std::chrono::milliseconds(50));
EXPECT_ANY_THROW(processor.NextPartitionClient(context));
while (!partitionClients.empty())
{
auto partitionClientIterator = partitionClients.begin();
auto partitionClient = partitionClientIterator->second;
if (partitionClientIterator != partitionClients.end())
{
partitionClients.erase(partitionClientIterator);
}
partitionClient->Close();
}
processor.Stop();
}
// The processor balanced and greedy tests fail when run on Linux or Mac. The tests run fine on
// Windows. For now, disable the tests on Linux and Mac.
#if !defined(AZ_PLATFORM_LINUX) && !defined(AZ_PLATFORM_MAC)
TEST_F(ProcessorTest, Processor_Balanced_LIVEONLY_)
{
TestWithLoadBalancer(Models::ProcessorStrategy::ProcessorStrategyBalanced);
}
TEST_F(ProcessorTest, Processor_Greedy_LIVEONLY_)
{
TestWithLoadBalancer(Models::ProcessorStrategy::ProcessorStrategyGreedy);
}
#endif
#if 0
TEST_F(ProcessorTest, Processor_Balanced_AcquisitionOnly_LIVEONLY_)
{
TestPartitionAcquisition(Models::ProcessorStrategy::ProcessorStrategyBalanced);
}
TEST_F(ProcessorTest, Processor_Greedy_AcquisitionOnly_LIVEONLY_)
{
TestPartitionAcquisition(Models::ProcessorStrategy::ProcessorStrategyGreedy);
}
#endif
}}}} // namespace Azure::Messaging::EventHubs::Test

43
sdk/eventhubs/azure-messaging-eventhubs/test/ut/producer_client_test.cpp
View File

@ -21,17 +21,16 @@ TEST_F(ProducerClientTest, ConnectionStringNoEntityPath)
std::string const connStringNoEntityPath = GetEnv("EVENTHUB_CONNECTION_STRING");
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
auto client = Azure::Messaging::EventHubs::ProducerClient(connStringNoEntityPath, eventHubName);
Azure::Messaging::EventHubs::ProducerClient client{connStringNoEntityPath, eventHubName};
EXPECT_EQ(eventHubName, client.GetEventHubName());
}
TEST_F(ProducerClientTest, ConnectionStringEntityPath)
{
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string const connStringEntityPath
= GetEnv("EVENTHUB_CONNECTION_STRING") + ";EntityPath=" + eventHubName;
std::string const connString = GetEnv("EVENTHUB_CONNECTION_STRING");
auto client = Azure::Messaging::EventHubs::ProducerClient(connStringEntityPath, eventHubName);
Azure::Messaging::EventHubs::ProducerClient client{connString, eventHubName};
EXPECT_EQ(eventHubName, client.GetEventHubName());
}
@ -44,23 +43,21 @@ TEST_F(ProducerClientTest, TokenCredential_LIVEONLY_)
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
Azure::Messaging::EventHubs::ProducerClientOptions producerOptions;
producerOptions.ApplicationID = "appId";
auto client = Azure::Messaging::EventHubs::ProducerClient(
"gearamaeh1.servicebus.windows.net", eventHubName, credential);
Azure::Messaging::EventHubs::ProducerClient client{
"gearamaeh1.servicebus.windows.net", eventHubName, credential};
EXPECT_EQ(eventHubName, client.GetEventHubName());
}
TEST_F(ProducerClientTest, SendMessage_LIVEONLY_)
{
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string const connStringEntityPath
= GetEnv("EVENTHUB_CONNECTION_STRING") + ";EntityPath=" + eventHubName;
std::string const connString = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ProducerClientOptions producerOptions;
producerOptions.Name = "sender-link";
producerOptions.ApplicationID = "some";
auto client = Azure::Messaging::EventHubs::ProducerClient(
connStringEntityPath, eventHubName, producerOptions);
Azure::Messaging::EventHubs::ProducerClient client{connString, eventHubName, producerOptions};
Azure::Core::Amqp::Models::AmqpMessage message2;
Azure::Messaging::EventHubs::Models::EventData message1;
@ -97,15 +94,13 @@ TEST_F(ProducerClientTest, SendMessage_LIVEONLY_)
TEST_F(ProducerClientTest, EventHubRawMessageSend_LIVEONLY_)
{
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string const connStringEntityPath
= GetEnv("EVENTHUB_CONNECTION_STRING") + ";EntityPath=" + eventHubName;
std::string const connString = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ProducerClientOptions producerOptions;
producerOptions.Name = "sender-link";
producerOptions.ApplicationID = "some";
auto client = Azure::Messaging::EventHubs::ProducerClient(
connStringEntityPath, eventHubName, producerOptions);
Azure::Messaging::EventHubs::ProducerClient client{connString, eventHubName, producerOptions};
client.Send(Azure::Messaging::EventHubs::Models::EventData{"This is a test message"});
@ -119,15 +114,13 @@ TEST_F(ProducerClientTest, EventHubRawMessageSend_LIVEONLY_)
TEST_F(ProducerClientTest, GetEventHubProperties_LIVEONLY_)
{
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string const connStringEntityPath
= GetEnv("EVENTHUB_CONNECTION_STRING") + ";EntityPath=" + eventHubName;
std::string const connString = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ProducerClientOptions producerOptions;
producerOptions.Name = "sender-link";
producerOptions.ApplicationID = "some";
auto client = Azure::Messaging::EventHubs::ProducerClient(
connStringEntityPath, eventHubName, producerOptions);
Azure::Messaging::EventHubs::ProducerClient client{connString, eventHubName, producerOptions};
auto result = client.GetEventHubProperties();
EXPECT_EQ(result.Name, eventHubName);
@ -137,17 +130,17 @@ TEST_F(ProducerClientTest, GetEventHubProperties_LIVEONLY_)
TEST_F(ProducerClientTest, GetPartitionProperties_LIVEONLY_)
{
std::string eventHubName{GetEnv("EVENTHUB_NAME")};
std::string const connStringEntityPath
= GetEnv("EVENTHUB_CONNECTION_STRING") + ";EntityPath=" + eventHubName;
std::string const connString = GetEnv("EVENTHUB_CONNECTION_STRING");
Azure::Messaging::EventHubs::ProducerClientOptions producerOptions;
producerOptions.Name = "sender-link";
producerOptions.ApplicationID = "some";
auto client = Azure::Messaging::EventHubs::ProducerClient(
connStringEntityPath, eventHubName, producerOptions);
Azure::Messaging::EventHubs::ProducerClient client{connString, eventHubName, producerOptions};
auto result = client.GetPartitionProperties("0");
EXPECT_EQ(result.Name, eventHubName);
EXPECT_EQ(result.PartitionId, "0");
ASSERT_NO_THROW([&]() {
auto result = client.GetPartitionProperties("0");
EXPECT_EQ(result.Name, eventHubName);
EXPECT_EQ(result.PartitionId, "0");
}());
}

12
sdk/eventhubs/azure-messaging-eventhubs/test/ut/round_trip_test.cpp
View File

@ -23,7 +23,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
int64_t startSequenceNumber = 0;
{
auto producer = Azure::Messaging::EventHubs::ProducerClient(connectionString, eventHubName);
Azure::Messaging::EventHubs::ProducerClient producer{connectionString, eventHubName};
auto partitionProperties = producer.GetPartitionProperties("1");
startSequenceNumber = partitionProperties.LastEnqueuedSequenceNumber;
@ -38,7 +38,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
Azure::Messaging::EventHubs::PartitionClientOptions partitionOptions;
partitionOptions.StartPosition.SequenceNumber = startSequenceNumber;
auto consumer = Azure::Messaging::EventHubs::ConsumerClient(
Azure::Messaging::EventHubs::ConsumerClient consumer(
connectionString, eventHubName, consumerGroup);
auto receiver = consumer.CreatePartitionClient("1", partitionOptions);
@ -59,7 +59,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
int64_t startOffset = 0;
{
auto producer = Azure::Messaging::EventHubs::ProducerClient(connectionString, eventHubName);
Azure::Messaging::EventHubs::ProducerClient producer(connectionString, eventHubName);
auto partitionProperties = producer.GetPartitionProperties("1");
startOffset = partitionProperties.LastEnqueuedOffset;
@ -71,7 +71,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
}
{
auto consumer = Azure::Messaging::EventHubs::ConsumerClient(
Azure::Messaging::EventHubs::ConsumerClient consumer(
connectionString, eventHubName, consumerGroup);
Azure::Messaging::EventHubs::PartitionClientOptions partitionOptions;
@ -104,7 +104,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
Azure::DateTime startTime;
{
auto producer = Azure::Messaging::EventHubs::ProducerClient(connectionString, eventHubName);
Azure::Messaging::EventHubs::ProducerClient producer(connectionString, eventHubName);
auto partitionProperties = producer.GetPartitionProperties("1");
GTEST_LOG_(INFO) << "Partition Properties: " << partitionProperties;
startTime = partitionProperties.LastEnqueuedTimeUtc + std::chrono::seconds(1);
@ -124,7 +124,7 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
}
{
auto consumer = Azure::Messaging::EventHubs::ConsumerClient(
Azure::Messaging::EventHubs::ConsumerClient consumer(
connectionString, eventHubName, consumerGroup);
Azure::Messaging::EventHubs::PartitionClientOptions partitionOptions;

119
sdk/eventhubs/azure-messaging-eventhubs/test/ut/test_checkpoint_store.hpp
View File

@ -7,13 +7,12 @@
#include <azure/identity.hpp>
#include <azure/messaging/eventhubs.hpp>
#include <mutex>
#include <gtest/gtest.h>
namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
class TestCheckpointStore : public Azure::Messaging::EventHubs::CheckpointStore {
std::map<std::string, Azure::Messaging::EventHubs::Models::Checkpoint> m_checkpoints;
std::map<std::string, Azure::Messaging::EventHubs::Models::Ownership> m_ownerships;
public:
TestCheckpointStore()
{
@ -25,17 +24,22 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
std::string const& fullyQualifiedNamespace,
std::string const& eventHubName,
std::string const& consumerGroup,
Core::Context const& context = {}) override
Core::Context const& = {}) override
{
(void)fullyQualifiedNamespace;
(void)eventHubName;
(void)consumerGroup;
(void)context;
std::unique_lock<std::mutex> lock(m_mutex);
std::string prefix = Models::Checkpoint{consumerGroup, eventHubName, fullyQualifiedNamespace}
.GetCheckpointBlobPrefixName();
GTEST_LOG_(INFO) << "List checkpoints: List checkpoints for prefix " << prefix;
std::vector<Azure::Messaging::EventHubs::Models::Checkpoint> checkpoints;
for (auto const& checkpoint : m_checkpoints)
{
checkpoints.push_back(checkpoint.second);
// If the key to the checkpoint matches the prefix, add that checkpoint.
if (checkpoint.first.find(prefix) == 0)
{
checkpoints.push_back(checkpoint.second);
}
}
GTEST_LOG_(INFO) << "List checkpoints: " << checkpoints.size() << " checkpoints found";
return checkpoints;
}
@ -43,25 +47,49 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
std::string const& fullyQualifiedNamespace,
std::string const& eventHubName,
std::string const& consumerGroup,
Core::Context const& context = {}) override
Core::Context const& = {}) override
{
(void)fullyQualifiedNamespace;
(void)eventHubName;
(void)consumerGroup;
(void)context;
std::string prefix = Models::Ownership{consumerGroup, eventHubName, fullyQualifiedNamespace}
.GetOwnershipPrefixName();
GTEST_LOG_(INFO) << "ListOwnership: List ownership for prefix " << prefix;
if (prefix.find("testeventhub") == 0)
{
GTEST_LOG_(ERROR) << "Fully qualified namespace is not valid.";
}
std::unique_lock<std::mutex> lock(m_mutex);
std::vector<Azure::Messaging::EventHubs::Models::Ownership> ownerships;
for (auto const& ownership : m_ownerships)
{
ownerships.push_back(ownership.second);
GTEST_LOG_(INFO) << "Check ownership " << ownership.first << " for prefix " << prefix
<< ".";
// If the key to the ownership matches the prefix, add that ownership.
if (ownership.first.find(prefix) == 0)
{
ownerships.push_back(ownership.second);
}
}
GTEST_LOG_(INFO) << "ListOwnership: " << ownerships.size() << " ownerships found";
if (ownerships.size() != 0)
{
for (auto const& ownership : ownerships)
{
GTEST_LOG_(INFO) << "ListOwnership: Ownership found: " << ownership;
}
}
return ownerships;
}
std::vector<Azure::Messaging::EventHubs::Models::Ownership> ClaimOwnership(
std::vector<Models::Ownership> const& partitionOwnership,
Core::Context const& context = {}) override
Core::Context const& = {}) override
{
(void)context;
GTEST_LOG_(INFO) << "Claim Ownership for: " << partitionOwnership.size() << " partitions";
for (auto const& ownership : partitionOwnership)
{
GTEST_LOG_(INFO) << "Claim Ownership for: " << ownership;
}
std::unique_lock<std::mutex> lock(m_mutex);
std::vector<Models::Ownership> owned;
for (auto const& ownership : partitionOwnership)
{
@ -71,9 +99,28 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
owned.push_back(newOwnership);
}
}
GTEST_LOG_(INFO) << "Claim Ownership: " << owned.size() << " ownerships claimed";
return owned;
}
// *** TEST HOOK *** */
void ExpireOwnership(Azure::Messaging::EventHubs::Models::Ownership const& o)
{
std::unique_lock<std::mutex> lock(m_mutex);
Models::Ownership temp = o;
temp.LastModifiedTime
= temp.LastModifiedTime.ValueOr(std::chrono::system_clock::now()) - std::chrono::hours(6);
std::string key = temp.GetOwnershipPrefixName() + temp.PartitionId;
m_ownerships[key] = temp;
}
private:
std::map<std::string, Azure::Messaging::EventHubs::Models::Checkpoint> m_checkpoints;
std::map<std::string, Azure::Messaging::EventHubs::Models::Ownership> m_ownerships;
std::mutex m_mutex;
Azure::Messaging::EventHubs::Models::Ownership UpdateOwnership(
Azure::Messaging::EventHubs::Models::Ownership const& ownership)
{
@ -83,19 +130,24 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
throw std::runtime_error("Invalid ownership");
}
std::string key = ownership.FullyQualifiedNamespace + "/" + ownership.EventHubName + "/"
+ ownership.ConsumerGroup + "/" + ownership.PartitionId;
std::string key = ownership.GetOwnershipPrefixName() + ownership.PartitionId;
GTEST_LOG_(INFO) << "Update Ownership for key " << key;
if (m_ownerships.find(key) != m_ownerships.end())
// If the ownership exists, check the ETag to see if it matches.
{
if (ownership.ETag.HasValue() == false)
auto foundOwnership = m_ownerships.find(key);
if (foundOwnership != m_ownerships.end())
{
throw std::runtime_error("ETag is required for claiming ownership");
}
if (ownership.ETag.HasValue() && !foundOwnership->second.ETag.HasValue())
{
throw std::runtime_error("ETag mismatch in partition ownership.");
}
if (ownership.ETag.Value() != m_ownerships[key].ETag.Value())
{
return Models::Ownership{};
if (ownership.ETag.HasValue()
&& (ownership.ETag.Value() != foundOwnership->second.ETag.Value()))
{
return Models::Ownership{};
}
}
}
Azure::Messaging::EventHubs::Models::Ownership newOwnership = ownership;
@ -106,28 +158,19 @@ namespace Azure { namespace Messaging { namespace EventHubs { namespace Test {
return newOwnership;
}
void ExpireOwnership(Azure::Messaging::EventHubs::Models::Ownership& o)
{
Models::Ownership temp = o;
temp.LastModifiedTime
= temp.LastModifiedTime.ValueOr(std::chrono::system_clock::now()) - std::chrono::hours(6);
std::string key = temp.FullyQualifiedNamespace + "/" + temp.EventHubName + "/"
+ temp.ConsumerGroup + "/" + temp.PartitionId;
m_ownerships[key] = temp;
}
void UpdateCheckpoint(
Azure::Messaging::EventHubs::Models::Checkpoint const& checkpoint,
Core::Context const& context = {}) override
{
std::unique_lock<std::mutex> lock(m_mutex);
GTEST_LOG_(INFO) << "UpdateCheckpoint for " << checkpoint;
(void)context;
if (checkpoint.ConsumerGroup.empty() || checkpoint.EventHubName.empty()
|| checkpoint.FullyQualifiedNamespaceName.empty() || checkpoint.PartitionId.empty())
{
throw std::runtime_error("Invalid checkpoint");
}
std::string key = checkpoint.FullyQualifiedNamespaceName + "/" + checkpoint.EventHubName + "/"
+ checkpoint.ConsumerGroup + "/" + checkpoint.PartitionId;
std::string key = checkpoint.GetCheckpointBlobPrefixName() + checkpoint.PartitionId;
m_checkpoints[key] = checkpoint;
}
};