Test proxy (#4118)

* start

* recording

* playback

* refactor, playback and record

* recordings for kv

* klklk

* open telemetry, identity

* attest

* all tests recorded except storage

* STORAGE RECORDINGDS

* some cleanup

* ignore result

* testproxy remade

* fiiine , do something with the result , goooosh

* install test proxy

* clang

* clang

* certs maybe

* cmake generate fix,

* start test proxy script

* start test proxy on env

* clang, move TP after build

* reregen

* certs

* sda

* dss

* allow insecure connections

* put back

* debug

* iuy

* try again

* ewew

* chmod

* try again

* update sanitizer

* output testproxy log

* sjhgasjgdajh

* folder

* worxy

* fix tests and log file

* format files

* clang format

* clang

* sa

* sa

* cleanup

* cspell

* oops

* remove redundant method

* tests

* put back original

* restore to 933486385a

* recordings

* remove storage values

* storage recordings

* disable non functioning tests

* remove core install of test proxy

* _LIVEONLY_ tests

* clang install when needed

* logs on condition

* skip tests

* revert cpp to original version

* quick test

* right that one

* one skip

* identity pushed

* maybe

* override

* clang

* clang

* attestetion

* keyvault

* reenable 20+ tests

* 5 tests left , lease related

* a bit of cleanup

* try now

* snitizers

* some fixes

* capitalization

* clang , cover, peakA

* WEIRD NAME THING ON WINDOWS

* storage recordings

* one more livee

* createappenddelete_liveonly

* CreateWithTags_LIVEONLY_

* try capitalization

* another onw

* maybe now

* all liveonly

* try restore before test

* typo

* condition

* clang and cc

* azure core ci

* qwqwq

* dsasdas

* cleanup1

* typo

* spaces

* cleanup2

* cleanup 3

* remove start proxy

* cleanup +1

* Update cmake-modules/TestProxyPrep.cmake

Co-authored-by: Rick Winter <rick.winter@microsoft.com>

* Update eng/scripts/Start-TestProxy.ps1

Co-authored-by: Rick Winter <rick.winter@microsoft.com>

* Update eng/scripts/Stop-TestProxy.ps1

Co-authored-by: Rick Winter <rick.winter@microsoft.com>

* PR comments

* clangs

* Update sdk/core/azure-core-test/src/test_proxy_policy.cpp

* build

Co-authored-by: Rick Winter <rick.winter@microsoft.com>

cmake-modules/TestProxyPrep.cmake

@@ -0,0 +1,10 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: MIT
+#
+
+macro(CopyTestProxyScripts)
+    file(COPY ${CMAKE_SOURCE_DIR}/eng/Scripts/Start-TestProxy.ps1
+        DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+    file(COPY ${CMAKE_SOURCE_DIR}/eng/Scripts/Stop-TestProxy.ps1
+        DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+endmacro()

eng/pipelines/templates/jobs/ci.tests.yml

@@ -147,8 +147,21 @@ jobs:
           VcpkgArgs: "$(VcpkgArgs)"
           Env: "$(CmakeEnvArg)"
 
+      - template: /eng/common/testproxy/test-proxy-tool.yml
+        parameters:
+          runProxy: true
+          rootFolder: '$(Build.SourcesDirectory)/sdk/${{parameters.ServiceDirectory}}'
+          templateFolder: '$(Build.SourcesDirectory)/sdk/${{parameters.ServiceDirectory}}'
+          condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON'))
+        
       - ${{ parameters.PreTestSteps }}
       
+      - pwsh: |
+          test-proxy restore -a $(Build.SourcesDirectory)/sdk/${{parameters.ServiceDirectory}}
+        workingDirectory: '$(Build.SourcesDirectory)/sdk/${{parameters.ServiceDirectory}}'
+        displayName: Restore Recordings
+        condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON'), ne('${{parameters.ServiceDirectory}}', 'core'))
+
       - pwsh: |
           ctest `
             -C Debug `
@@ -161,6 +174,13 @@ jobs:
         displayName: Test
         
       - ${{ parameters.PostTestSteps }}
+            
+      - pwsh: |
+          get-content test-proxy.log
+        displayName: TestProxy Log
+        condition: and(succeededOrFailed(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON'))
+        workingDirectory: $(Build.SourcesDirectory)
+        continueOnError: true
 
       - task: PublishTestResults@2
         inputs:

eng/pipelines/templates/jobs/live.tests.yml

@@ -135,6 +135,10 @@ jobs:
       Location: ${{ coalesce(parameters.Location, parameters.CloudConfig.Location) }}
       SubscriptionConfiguration: $(SubscriptionConfiguration)
 
+  - template: /eng/common/testproxy/test-proxy-tool.yml
+    parameters:
+      runProxy: false
+
   - ${{ parameters.PreTestSteps }}
 
     # For non multi-config generator use the same build configuration to run tests

eng/scripts/Start-TestProxy.ps1

@@ -0,0 +1,42 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: MIT
+
+param(
+[Parameter(Mandatory=$True)]
+[string] $AssetsPath
+)
+# check is there is another test-proxy running
+$running = Get-Process -Name test-proxy
+echo $AssetsPath
+if($running)
+{
+    echo "test-proxy running, no need for new instance"
+    exit 0
+}
+
+# make sure errors collection is empty
+$error.clear()
+
+#check if we have a test-proxy available
+$CurrentVersion = (Get-Command -Name "test-proxy" -ErrorAction SilentlyContinue).Version
+
+if($error){
+    echo "Will install testproxy"
+
+    dotnet tool update azure.sdk.tools.testproxy --global --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json --version "1.0.0-dev*"
+    # clear the errors again
+    $error.clear()
+
+    #check again for test proxy presence
+    $CurrentVersion = (Get-Command -Name "test-proxy" -ErrorAction SilentlyContinue).Version
+
+    # if we have errors this means we had issues installing it , needs to be done by hand
+    if($error){
+        echo "Unable to install testproxy. Try installing manually."
+        exit 1
+    }
+}
+
+echo "Start test proxy with argument list --storage-location $AssetsPath"
+#starts it in a separate process that will outlive pwsh in order to serve requests.
+Start-Process 'test-proxy' -ArgumentList "--storage-location $AssetsPath" 

eng/scripts/Stop-TestProxy.ps1

@@ -0,0 +1,4 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: MIT
+
+Stop-Process -Name "test-proxy"

sdk/attestation/assets.json

@@ -0,0 +1,6 @@
+{
+  "AssetsRepo": "Azure/azure-sdk-assets",
+  "AssetsRepoPrefixPath": "cpp",
+  "TagPrefix": "cpp/attestation",
+  "Tag": "cpp/attestation_b384d96f95"
+}

sdk/attestation/azure-security-attestation/test/ut/policygetset_test.cpp

@@ -405,6 +405,7 @@ namespace Azure { namespace Security { namespace Attestation { namespace Test {
     std::string GetTestName(testing::TestParamInfo<PolicyTests::ParamType> const& testInfo)
     {
       std::string testName;
+      int suffixVotes = 0;
       switch (testInfo.param.TestType)
       {
         case TestCaseType::GetPolicy:
@@ -415,6 +416,7 @@ namespace Azure { namespace Security { namespace Attestation { namespace Test {
           break;
         case TestCaseType::ModifyPolicySecured:
           testName += "ModifyGeneratedKey";
+          suffixVotes++;
           break;
         case TestCaseType::ModifyPolicyUnsecured:
           testName += "ModifyUnsecured";
@@ -427,6 +429,7 @@ namespace Azure { namespace Security { namespace Attestation { namespace Test {
       {
         case ServiceInstanceType::AAD:
           testName += "AAD";
+          suffixVotes++;
           break;
         case ServiceInstanceType::Isolated:
           testName += "Isolated";
@@ -440,6 +443,11 @@ namespace Azure { namespace Security { namespace Attestation { namespace Test {
 
       testName += "_";
       testName += testInfo.param.TeeType.ToString();
+      if (suffixVotes == 2)
+      {
+        testName += "_LIVEONLY_";
+      };
+      //+"_LIVEONLY_";
       return testName;
     }
   } // namespace

sdk/attestation/azure-security-attestation/test/ut/tpmattestation_test.cpp

@@ -29,6 +29,7 @@ namespace Azure { namespace Security { namespace Attestation { namespace Test {
   protected:
     std::shared_ptr<Azure::Core::Credentials::TokenCredential> m_credential;
     std::unique_ptr<AttestationAdministrationClient> m_adminClient;
+
     // Create
     virtual void SetUp() override
     {

sdk/core/azure-core-test/CMakeLists.txt

@@ -10,21 +10,18 @@ set(CMAKE_CXX_STANDARD_REQUIRED True)
 
 set(
   AZURE_CORE_TEST_HEADER
-  inc/azure/core/test/interceptor_manager.hpp
   inc/azure/core/test/network_models.hpp
-  inc/azure/core/test/playback_http_client.hpp
-  inc/azure/core/test/record_network_call_policy.hpp
   inc/azure/core/test/test_base.hpp
   inc/azure/core/test/test_context_manager.hpp
+  inc/azure/core/test/test_proxy_manager.hpp
 )
 
 set(
   AZURE_CORE_TEST_SOURCE
   src/private/package_version.hpp
-  src/interceptor_manager.cpp
-  src/playback_http_transport.cpp
-  src/record_policy.cpp
+  src/test_proxy_policy.cpp
   src/test_base.cpp
+  src/test_proxy_manager.cpp
 )
 
 add_library (
@@ -46,4 +43,4 @@ target_include_directories (azure-core-test-fw
 # make sure that users can consume the project as a library.
 add_library (Azure::Core::Test ALIAS azure-core-test-fw)
 target_link_libraries(azure-core-test-fw PRIVATE azure-core Azure::azure-identity gtest)
-create_map_file(azure-core-test-fw azure-core-test-fw.map)
+create_map_file(azure-core-test-fw azure-core-test-fw.map)

sdk/core/azure-core-test/README.md

@@ -51,3 +51,4 @@ Azure SDK for C++ is licensed under the [MIT](https://github.com/Azure/azure-sdk
 [c_compiler]: https://visualstudio.microsoft.com/vs/features/cplusplus/
 [cloud_shell]: https://docs.microsoft.com/azure/cloud-shell/overview
 [cloud_shell_bash]: https://shell.azure.com/bash
+ 

sdk/core/azure-core-test/inc/azure/core/test/interceptor_manager.hpp

@@ -1,160 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-/**
- * @brief Keep the state of the playback-record-live tests.
- *
- * @remark The interceptor is a singleton that is init during the test configuration.
- * Depending on the test mode, the interceptor will handle the recorder data.
- *
- * - If test mode is LIVE, the Interceptor will not affect the test behavior.
- * - If test mode is RECORD, the Interceptor will init the `record data` to be written after
- * capturing each request going out to the network and also recording the server response for that
- * request.
- * - If test mode is PLAYBACK, the interceptor will load the `record data` and use it to answer HTTP
- * client request without sending the request to the network.
- *
- * @remark The interceptor handle the `recorded data, provides the HTTP transport adapter and the
- * record policy. However, adding the policy and adapter to a pipeline is done by the user.
- */
-
-#pragma once
-
-#include <azure/core/credentials/credentials.hpp>
-#include <azure/core/http/http.hpp>
-#include <azure/core/http/policies/policy.hpp>
-#include <memory>
-#include <string>
-
-#include "azure/core/test/network_models.hpp"
-#include "azure/core/test/playback_http_client.hpp"
-#include "azure/core/test/record_network_call_policy.hpp"
-#include "azure/core/test/test_context_manager.hpp"
-
-// Used by recordPolicy and playback transport adapter.
-#if !defined(RECORDING_BODY_STREAM_SENTINEL)
-#define RECORDING_BODY_STREAM_SENTINEL "__bodyStream__"
-#endif
-namespace Azure { namespace Core { namespace Test {
-
-  /**
-   * @brief TestNonExpiringCredential Credential authenticates with the Azure services using a
-   * Tenant ID, Client ID and a client secret.
-   *
-   */
-  class TestNonExpiringCredential final : public Core::Credentials::TokenCredential {
-  public:
-    Core::Credentials::AccessToken GetToken(
-        Core::Credentials::TokenRequestContext const& tokenRequestContext,
-        Core::Context const& context) const override
-    {
-      Core::Credentials::AccessToken accessToken;
-      accessToken.Token = "magicToken";
-      accessToken.ExpiresOn = DateTime::max();
-
-      if (context.IsCancelled() || tokenRequestContext.Scopes.size() == 0)
-      {
-        accessToken.ExpiresOn = DateTime::min();
-      }
-
-      return accessToken;
-    }
-  };
-
-  /**
-   * @brief A class that keeps track of network calls by either reading the data from an existing
-   * test session record or recording the network calls in memory.
-   *
-   */
-  class InterceptorManager {
-  private:
-    Azure::Core::Test::RecordedData m_recordedData;
-    // Using a reference because the context lives in the test_base class and we don't want to make
-    // a copy.
-    Azure::Core::Test::TestContextManager& m_testContext;
-
-  public:
-    /**
-     * @brief Enables to init an interceptor with empty values.
-     *
-     */
-    InterceptorManager(Azure::Core::Test::TestContextManager& testContext)
-        : m_testContext(testContext)
-    {
-    }
-
-    /**
-     * Gets the recorded data reference that InterceptorManager is keeping track of.
-     *
-     * @return The recorded data reference managed by InterceptorManager.
-     */
-    Azure::Core::Test::RecordedData& GetRecordedData() { return m_recordedData; }
-
-    /**
-     * Gets HTTP pipeline policy that records network calls and its data is managed by the
-     * InterceptorManager.
-     *
-     * @return HttpPipelinePolicy to record network calls.
-     */
-    std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy> GetRecordPolicy()
-    {
-      return std::make_unique<Azure::Core::Test::RecordNetworkCallPolicy>(this);
-    }
-
-    /**
-     * @brief Get a non-expiring token credential. This is a test utility for use in playback
-     * scenarios where the token is not relevant.
-     *
-     * @return std::shared_ptr<Core::Credentials::TokenCredential>
-     */
-    std::shared_ptr<Core::Credentials::TokenCredential> GetTestCredential()
-    {
-      return std::make_shared<TestNonExpiringCredential>();
-    }
-
-    /**
-     * Gets a new HTTP transport adapter that plays back test session records managed by the
-     * InterceptorManager.
-     *
-     * @return An HTTP transport adapter that plays back network calls from its recorded data.
-     */
-    std::unique_ptr<Azure::Core::Http::HttpTransport> GetPlaybackTransport()
-    {
-      return std::make_unique<Azure::Core::Test::PlaybackClient>(this);
-    }
-
-    /**
-     * @brief Get the Test Context object.
-     *
-     * @return Azure::Core::Test::TestContextManager const&
-     */
-    Azure::Core::Test::TestContextManager const& GetTestContext() const { return m_testContext; }
-
-    /**
-     * @brief Read from environment and parse the a test mode.
-     *
-     * @remark If the AZURE_TEST_MODE variable is not found, default test mode is LIVE mode.
-     *
-     * @return TestMode
-     */
-    static TestMode GetTestMode();
-
-    /**
-     * @brief This function is expected to be called by the playback transport adapter.
-     *
-     * @remark The name of the test is known and set when the test is actually started. That's why
-     * the recorded data can't be loaded until the test is already running (Can't load on SetUp).
-     *
-     */
-    void LoadTestData();
-
-    /**
-     * @brief Removes sensitive info from a request Url.
-     *
-     * @param url The request Url.
-     * @return Azure::Core::Url
-     */
-    Azure::Core::Url RedactUrl(Azure::Core::Url const& url);
-  };
-
-}}} // namespace Azure::Core::Test

sdk/core/azure-core-test/inc/azure/core/test/playback_http_client.hpp

@@ -1,57 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-/**
- * @brief HTTP client that plays back NetworkCallRecord NetworkCallRecords.
- */
-
-#pragma once
-
-#include <string>
-
-#include <azure/core/context.hpp>
-#include <azure/core/http/http.hpp>
-#include <azure/core/http/policies/policy.hpp>
-#include <azure/core/io/body_stream.hpp>
-#include <azure/core/response.hpp>
-
-#include "azure/core/test/network_models.hpp"
-
-namespace Azure { namespace Core { namespace Test {
-
-  // Partial class. Required to reference the Interceptor that is defined in the implementation.
-  class InterceptorManager;
-
-  /**
-   * @brief Creates an HTTP Transport adapter that answer to requests using recorded data.
-   *
-   */
-  class PlaybackClient : public Azure::Core::Http::HttpTransport {
-  private:
-    Azure::Core::Test::InterceptorManager* m_interceptorManager;
-
-  public:
-    /**
-     * @brief Construct a new Playback Client that uses \p recordedData to answer to the HTTP
-     * request.
-     *
-     * @param interceptorManager A reference to the interceptor manager holding the recorded data.
-     */
-    PlaybackClient(Azure::Core::Test::InterceptorManager* interceptorManager)
-        : m_interceptorManager(interceptorManager)
-    {
-    }
-
-    /**
-     * @brief Override the HTTPTransport `send` contract.
-     *
-     * @param context The context that can cancel the request.
-     * @param request The HTTP request details.
-     * @return The HTTP raw response containing code, headers and payload.
-     */
-    std::unique_ptr<Azure::Core::Http::RawResponse> Send(
-        Azure::Core::Http::Request& request,
-        Azure::Core::Context const& context) override;
-  };
-
-}}} // namespace Azure::Core::Test

sdk/core/azure-core-test/inc/azure/core/test/test_base.hpp

@@ -7,9 +7,9 @@
 
 #include <gtest/gtest.h>
 
-#include "azure/core/test/interceptor_manager.hpp"
 #include "azure/core/test/network_models.hpp"
 #include "azure/core/test/test_context_manager.hpp"
+#include "azure/core/test/test_proxy_manager.hpp"
 #include <azure/core/credentials/credentials.hpp>
 #include <azure/core/credentials/token_credential_options.hpp>
 #include <azure/core/internal/client_options.hpp>
@@ -34,7 +34,8 @@ using namespace std::chrono_literals;
 namespace Azure { namespace Core { namespace Test {
 
   /**
-   * @brief The base class provides the tools for a test to use the Record&PlayBack functionalities.
+   * @brief The base class provides the tools for a test to use the Record&Playback
+   * functionalities.
    *
    */
   class TestBase : public ::testing::Test {
@@ -48,12 +49,19 @@ namespace Azure { namespace Core { namespace Test {
 
     void PrepareOptions(Azure::Core::_internal::ClientOptions& options)
     {
+      if (m_wasSkipped)
+      {
+        return;
+      }
+
       // Set up client options depending on the test-mode
       if (m_testContext.IsPlaybackMode())
       {
         // Playback mode uses:
         //  - playback transport adapter to read and return payload from json files
-        options.Transport.Transport = m_interceptor->GetPlaybackTransport();
+        m_testProxy->StartPlaybackRecord(TestMode::PLAYBACK);
+        m_testProxy->ConfigureInsecureConnection(options);
+        options.PerRetryPolicies.push_back(m_testProxy->GetTestProxyPolicy());
       }
       else if (!m_testContext.IsLiveMode())
       {
@@ -61,7 +69,9 @@ namespace Azure { namespace Core { namespace Test {
         //  - curl or winhttp transport adapter
         //  - Recording policy. Intercept server responses to create json files
         // AZURE_TEST_RECORDING_DIR is exported by CMAKE
-        options.PerRetryPolicies.push_back(m_interceptor->GetRecordPolicy());
+        m_testProxy->StartPlaybackRecord(TestMode::RECORD);
+        m_testProxy->ConfigureInsecureConnection(options);
+        options.PerRetryPolicies.push_back(m_testProxy->GetTestProxyPolicy());
       }
     }
 
@@ -71,14 +81,14 @@ namespace Azure { namespace Core { namespace Test {
       {
         // Playback mode uses:
         //  - never-expiring test credential to never require a token
-        credential = m_interceptor->GetTestCredential();
+        credential = m_testProxy->GetTestCredential();
       }
     }
 
     // Call this method to update client options with the required configuration to
     // support Record & Playback.
-    // If Playback or Record is not set, no changes will be done to the clientOptions or credential.
-    // Call this before creating the SDK client
+    // If Playback or Record is not set, no changes will be done to the clientOptions or
+    // credential. Call this before creating the SDK client
     void PrepareClientOptions(
         std::shared_ptr<Core::Credentials::TokenCredential>& credential,
         Azure::Core::_internal::ClientOptions& options)
@@ -97,8 +107,11 @@ namespace Azure { namespace Core { namespace Test {
 
     void SkipTest()
     {
-      m_wasSkipped = true;
-      GTEST_SKIP();
+      if (!m_wasSkipped)
+      {
+        m_wasSkipped = true;
+        GTEST_SKIP();
+      }
     }
 
     std::string RemovePreffix(std::string const& src)
@@ -126,7 +139,7 @@ namespace Azure { namespace Core { namespace Test {
 
   protected:
     Azure::Core::Test::TestContextManager m_testContext;
-    std::unique_ptr<Azure::Core::Test::InterceptorManager> m_interceptor;
+    std::unique_ptr<Azure::Core::Test::TestProxyManager> m_testProxy;
 
     bool shouldSkipTest() { return m_wasSkipped; }
 
@@ -241,7 +254,7 @@ namespace Azure { namespace Core { namespace Test {
       {
         // Playback mode uses:
         //  - never-expiring test credential to never require a token
-        return m_interceptor->GetTestCredential();
+        return m_testProxy->GetTestCredential();
       }
       else
       {
@@ -299,8 +312,8 @@ namespace Azure { namespace Core { namespace Test {
      * with the values emitted by the New-TestResources.ps1 script.
      *
      * @note The Azure CI pipeline upper cases all environment variables defined in the pipeline.
-     * Since some operating systems have case sensitive environment variables, on debug builds, this
-     * function ensures that the environment variable being retrieved is all upper case.
+     * Since some operating systems have case sensitive environment variables, on debug builds,
+     * this function ensures that the environment variable being retrieved is all upper case.
      *
      */
     std::string GetEnv(std::string const& name)
@@ -354,8 +367,8 @@ namespace Azure { namespace Core { namespace Test {
     /**
      * @brief Run before each test.
      *
-     * @param baseRecordingPath - the base recording path to be used for this test. Normally this is
-     * `AZURE_TEST_RECORDING_DIR`.
+     * @param baseRecordingPath - the base recording path to be used for this test. Normally this
+     * is `AZURE_TEST_RECORDING_DIR`.
      *
      * For example:
      *
@@ -370,14 +383,19 @@ namespace Azure { namespace Core { namespace Test {
       std::string recordingPath(baseRecordingPath);
       recordingPath.append("/recordings");
 
-      m_testContext.TestMode = Azure::Core::Test::InterceptorManager::GetTestMode();
+      m_testContext.TestMode = Azure::Core::Test::TestProxyManager::GetTestMode();
       // Use the test info to init the test context and interceptor.
       auto testNameInfo = ::testing::UnitTest::GetInstance()->current_test_info();
       // set the interceptor for the current test
       m_testContext.RenameTest(
           Sanitize(testNameInfo->test_suite_name()), Sanitize(testNameInfo->name()));
       m_testContext.RecordingPath = recordingPath;
-      m_interceptor = std::make_unique<Azure::Core::Test::InterceptorManager>(m_testContext);
+      m_testContext.AssetsPath = GetAssetsPath();
+
+      if (!m_wasSkipped)
+      {
+        m_testProxy = std::make_unique<Azure::Core::Test::TestProxyManager>(m_testContext);
+      }
     }
 
     /**
@@ -388,5 +406,10 @@ namespace Azure { namespace Core { namespace Test {
      *
      */
     void TearDown() override;
+
+    /**
+     * Returns the assets.json file path used when invoking the test-proxy playback/record
+     */
+    virtual std::string GetAssetsPath() { return "assets.json"; }
   };
 }}} // namespace Azure::Core::Test

sdk/core/azure-core-test/inc/azure/core/test/test_context_manager.hpp

@@ -20,6 +20,12 @@ namespace Azure { namespace Core { namespace Test {
    */
   class TestContextManager {
   public:
+    /**
+     * @brief The path where the asset.json for the current test exists, will be passed as part of
+     * the playback request to the test-proxy via the manager.
+     *
+     */
+    std::string AssetsPath;
     /**
      * @brief The path where the tests recordings are written.
      *
@@ -84,6 +90,16 @@ namespace Azure { namespace Core { namespace Test {
       return fullName;
     }
 
+    std::string GetTestRecordingPathName() const
+    {
+      std::string fullName(RecordingPath);
+      fullName.append("/");
+      fullName.append(GetTestPlaybackRecordingName());
+      fullName.append(".json");
+
+      return fullName;
+    }
+
     std::string GetTestName() const { return m_testName; }
     std::string GetTestSuiteName() const { return m_testSuite; }
 
@@ -120,6 +136,8 @@ namespace Azure { namespace Core { namespace Test {
 
     constexpr static const char* LiveOnlyToken = "_LIVEONLY_";
 
+    std::string RecordingId;
+
   private:
     std::string m_testName;
     std::string m_testSuite;

sdk/core/azure-core-test/inc/azure/core/test/test_proxy_manager.hpp

@@ -0,0 +1,170 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+/**
+ * Test proxy mamager class
+ */
+
+#pragma once
+
+#include <azure/core/credentials/credentials.hpp>
+#include <azure/core/http/http.hpp>
+#include <azure/core/http/policies/policy.hpp>
+#include <memory>
+#include <string>
+
+#include "azure/core/test/network_models.hpp"
+#include "azure/core/test/test_context_manager.hpp"
+#include "azure/core/test/test_proxy_policy.hpp"
+#if defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER)
+#include <azure/core/http/curl_transport.hpp>
+#endif
+#if defined(BUILD_TRANSPORT_WINHTTP_ADAPTER)
+#include <azure/core/http/win_http_transport.hpp>
+#endif
+#include <azure/core/http/policies/policy.hpp>
+#include <azure/core/internal/http/pipeline.hpp>
+
+using namespace Azure::Core::Http::_internal;
+
+namespace Azure { namespace Core { namespace Test {
+
+  class TestNonExpiringCredential final : public Core::Credentials::TokenCredential {
+  public:
+    Core::Credentials::AccessToken GetToken(
+        Core::Credentials::TokenRequestContext const& tokenRequestContext,
+        Core::Context const& context) const override
+    {
+      Core::Credentials::AccessToken accessToken;
+      accessToken.Token = "magicToken";
+      accessToken.ExpiresOn = DateTime::max();
+
+      if (context.IsCancelled() || tokenRequestContext.Scopes.size() == 0)
+      {
+        accessToken.ExpiresOn = DateTime::min();
+      }
+
+      return accessToken;
+    }
+  };
+
+  class TestProxyManager {
+  private:
+    // Using a reference because the context lives in the test_base class and we don't want to make
+    // a copy.
+    Azure::Core::Test::TestContextManager& m_testContext;
+    const std::string m_proxy = "https://localhost:5001";
+    bool m_isInsecureEnabled = true;
+    TestMode m_currentMode = TestMode::LIVE;
+    std::unique_ptr<Azure::Core::Http::_internal::HttpPipeline> m_privatePipeline;
+
+  public:
+    /**
+     * @brief Configures the transport to ignore certificate validation
+     *
+     */
+    void ConfigureInsecureConnection(Azure::Core::_internal::ClientOptions& clientOptions);
+
+    /**
+     * @brief Enables to init TestProxyManager with empty values.
+     *
+     */
+    TestProxyManager(Azure::Core::Test::TestContextManager& testContext)
+        : m_testContext(testContext)
+    {
+      Azure::Core::_internal::ClientOptions clientOp;
+      clientOp.Retry.MaxRetries = 0;
+      ConfigureInsecureConnection(clientOp);
+      std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> policiesOp;
+      std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> policiesRe;
+      Azure::Core::Http::_internal::HttpPipeline pipeline(
+          clientOp, "PerfFw", "na", std::move(policiesRe), std::move(policiesOp));
+      m_privatePipeline = std::make_unique<Azure::Core::Http::_internal::HttpPipeline>(pipeline);
+      SetProxySanitizer();
+    }
+
+    /**
+     * Are we in RECORD mode
+     *
+     * @return bool indicating RECORD mode
+     */
+    bool IsRecordMode() { return m_currentMode == TestMode::RECORD; }
+
+    /**
+     * Are we in PLAYBACK mode
+     *
+     * @return bool indicating PLAYBACK mode
+     */
+    bool IsPlaybackMode() { return m_currentMode == TestMode::PLAYBACK; }
+
+    /**
+     * Gets the proxy https url
+     *
+     * @return string containing the https url of the proxy (e.g. "https://localhost:xyz")
+     */
+    std::string GetTestProxy() { return m_proxy; };
+
+    /**
+     * Gets a ref to the test context
+     *
+     * @return Test context ref
+     */
+    Azure::Core::Test::TestContextManager& GetTestContext() { return m_testContext; }
+
+    /**
+     * Gets HTTP pipeline policy that records network calls and its data is managed by the
+     * TestProxy.
+     *
+     * @return HttpPipelinePolicy to record network calls.
+     */
+    std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy> GetTestProxyPolicy();
+
+    /**
+     * @brief Read from environment and parse the a test mode.
+     *
+     * @remark If the AZURE_TEST_MODE variable is not found, default test mode is LIVE mode.
+     *
+     * @return TestMode
+     */
+    static TestMode GetTestMode();
+
+    /**
+     * @brief Get a non-expiring token credential. This is a test utility for use in playback
+     * scenarios where the token is not relevant.
+     *
+     * @return std::shared_ptr<Core::Credentials::TokenCredential>
+     */
+    std::shared_ptr<Core::Credentials::TokenCredential> GetTestCredential()
+    {
+      return std::make_shared<TestNonExpiringCredential>();
+    }
+
+    /**
+     * Sets proxy to stop RECORD test and save the recording file.
+     *
+     */
+    void SetStopRecordMode();
+
+    /**
+     * Sets proxy to stop PLAYBACK test.
+     *
+     */
+    void SetStopPlaybackMode();
+
+    /**
+     * Gets the test recording ID
+     *
+     * @returns recording ID
+     */
+    std::string GetRecordingId() { return m_testContext.RecordingId; }
+
+    void StartPlaybackRecord(TestMode testMode);
+    void StopPlaybackRecord(TestMode testMode);
+
+  private:
+    std::string PrepareRequestBody();
+    void SetProxySanitizer();
+    bool CheckSanitizers();
+  };
+
+}}} // namespace Azure::Core::Test

sdk/core/azure-core-test/inc/azure/core/test/test_proxy_policy.hpp

@@ -11,61 +11,50 @@
 #include <memory>
 #include <string>
 
+#include "azure/core/test/network_models.hpp"
+#include "azure/core/test/test_proxy_manager.hpp"
+
 #include <azure/core/context.hpp>
 #include <azure/core/http/http.hpp>
 #include <azure/core/http/policies/policy.hpp>
 #include <azure/core/response.hpp>
 
-#include "azure/core/test/network_models.hpp"
-
 namespace Azure { namespace Core { namespace Test {
 
   // Partial class. Required to reference the Interceptor that is defined in the implementation.
-  class InterceptorManager;
+  class TestProxyManager;
 
   /**
    * @brief Creates a policy that records network calls into recordedData.
    *
    */
-  class RecordNetworkCallPolicy : public Azure::Core::Http::Policies::HttpPolicy {
+  class TestProxyPolicy : public Azure::Core::Http::Policies::HttpPolicy {
   private:
-    Azure::Core::Test::InterceptorManager* m_interceptorManager;
-    // Used to save the first byte from request payloads.
-    // Then, get a subsequent request ask for a bodyStream response, the symbol is used to generate
-    // a bodyStream from it.
-    // This feature is helpful to let a storage tests ( for example ) to upload a big payload (more
-    // than 10Kb) and download it later.
-    // The request for upload will contain the payload to upload.
-    // Then the request for download will use the symbol to generate a bodyStream.
-    std::unique_ptr<uint8_t> m_symbol;
+    Azure::Core::Test::TestProxyManager* m_testProxy;
 
   public:
     /**
      * @brief Disable default constructor.
      *
      */
-    RecordNetworkCallPolicy() = delete;
+    TestProxyPolicy() = delete;
 
     /**
-     * @brief Construct the record network policy which will save the HTTP request and response to
+     * @brief Construct the TestProxyPolicy which will save the HTTP request and response to
      * the \p recordedData.
      *
-     * @param interceptorManager A reference to the interceptor manager which holds the recorded
-     * data.
+     * @param testProxy A reference to the test proxy manager
      */
-    RecordNetworkCallPolicy(Azure::Core::Test::InterceptorManager* interceptorManager)
-        : m_interceptorManager(interceptorManager), m_symbol(std::make_unique<uint8_t>('x'))
-    {
-    }
+    TestProxyPolicy(Azure::Core::Test::TestProxyManager* testProxy) : m_testProxy(testProxy) {}
 
     /**
-     * @brief Cronstructs a new record network policy with the same recorded data.
+     * @brief Cronstructs a new TestProxyPolicy with the same recorded data.
      *
-     * @return A record network policy with the same recorded data.
+     * @return A TestProxyPolicy with the same recorded data.
      */
     std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy> Clone() const override
     {
-      return std::make_unique<RecordNetworkCallPolicy>(m_interceptorManager);
+      return std::make_unique<TestProxyPolicy>(m_testProxy);
     }
 
     /**

sdk/core/azure-core-test/src/interceptor_manager.cpp

@@ -1,109 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-#include <azure/core/internal/environment.hpp>
-#include <azure/core/internal/json/json.hpp>
-#include <azure/core/internal/strings.hpp>
-
-#include "azure/core/test/interceptor_manager.hpp"
-
-#include <fstream>
-#include <iostream>
-#include <regex>
-#include <stdexcept>
-#include <string>
-
-using namespace Azure::Core::Test;
-using namespace Azure::Core::Json::_internal;
-using namespace Azure::Core;
-using Azure::Core::_internal::Environment;
-
-TestMode InterceptorManager::GetTestMode()
-{
-  auto value = Environment::GetVariable("AZURE_TEST_MODE");
-  if (value.empty())
-  {
-    return Azure::Core::Test::TestMode::LIVE;
-  }
-
-  if (Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual(
-          value, "RECORD"))
-  {
-    return Azure::Core::Test::TestMode::RECORD;
-  }
-  else if (Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual(
-               value, "PLAYBACK"))
-  {
-    return Azure::Core::Test::TestMode::PLAYBACK;
-  }
-  else if (Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual(
-               value, "LIVE"))
-  {
-    return Azure::Core::Test::TestMode::LIVE;
-  }
-
-  // unexpected variable value
-  throw std::runtime_error("Invalid environment variable: " + value);
-}
-
-void InterceptorManager::LoadTestData()
-{
-  if (m_recordedData.NetworkCallRecords.size() > 0)
-  {
-    // TestData was loaded it before.
-    return;
-  }
-
-  std::string const recordingName(
-      m_testContext.RecordingPath + "/" + m_testContext.GetTestPlaybackRecordingName() + ".json");
-  std::ifstream readFile(recordingName);
-  if (!readFile.is_open())
-  {
-    throw std::runtime_error("Can't open recording: " + recordingName);
-  }
-
-  std::string recordingContent(
-      (std::istreambuf_iterator<char>(readFile)), std::istreambuf_iterator<char>());
-
-  auto const jsonRecord = json::parse(recordingContent);
-  auto const networkRecords = jsonRecord["networkCallRecords"];
-  for (auto const& record : networkRecords)
-  {
-    NetworkCallRecord modelRecord;
-    modelRecord.Method = record["Method"];
-    modelRecord.Url = record["Url"];
-    modelRecord.Headers = record["Headers"].get<std::map<std::string, std::string>>();
-    modelRecord.Response = record["Response"].get<std::map<std::string, std::string>>();
-    m_recordedData.NetworkCallRecords.push_back(modelRecord);
-  }
-  readFile.close();
-}
-
-Url InterceptorManager::RedactUrl(Url const& url)
-{
-  Azure::Core::Url redactedUrl;
-  redactedUrl.SetScheme(url.GetScheme());
-  auto host = url.GetHost();
-  auto hostWithNoAccount = std::find(host.begin(), host.end(), '.');
-  redactedUrl.SetHost("REDACTED" + std::string(hostWithNoAccount, host.end()));
-  // replace any uniqueID from the path for a hardcoded id
-  // For the regex, we should not assume anything about the version of UUID format being used. So,
-  // using the most general regex to get any uuid version.
-  redactedUrl.SetPath(std::regex_replace(
-      url.GetPath(),
-      std::regex("[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"),
-      "33333333-3333-3333-3333-333333333333"));
-  // Query parameters
-  for (auto const& qp : url.GetQueryParameters())
-  {
-    if (qp.first == "sig")
-    {
-      redactedUrl.AppendQueryParameter("sig", "REDACTED");
-    }
-    else
-    {
-      redactedUrl.AppendQueryParameter(qp.first, qp.second);
-    }
-  }
-  return redactedUrl;
-}

sdk/core/azure-core-test/src/playback_http_transport.cpp

@@ -1,202 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-#include <azure/core/http/http.hpp>
-
-#include "azure/core/test/interceptor_manager.hpp"
-#include "azure/core/test/playback_http_client.hpp"
-
-#include <cstdlib>
-#include <mutex>
-#include <stdexcept>
-#include <string>
-#include <vector>
-
-using namespace Azure::Core::Http;
-using namespace Azure::Core::Test;
-
-/**
- * @brief Infomation about special behavior headers.
- *
- * @details This structure helps to describe how to handle the playback response when there are
- * special headers. For example, for unique id headers, the playback transport adapter can take one
- * unique id comming from the request and use it as part of the playback response.
- *
- */
-struct UniqueIdInfo
-{
-  /**
-   * @brief If this header key is found in the request, the response should override the header
-   * defined by `ReplaceResponseHeader` in the response.
-   *
-   */
-  std::string RequestHeader;
-
-  /**
-   * @brief This field can be used to condition the replacement of the response header to only when
-   * the request header is equal to this value.
-   *
-   */
-  std::string RequestHeaderOnlyIfValue;
-
-  /**
-   * @brief This is the header in the response to be replaced.
-   *
-   */
-  std::string ReplaceResponseHeader;
-
-  /**
-   * @brief Use this field to override the response value with another header value from the
-   * request. Leave it empty to use the value from `RequestHeader`.
-   *
-   */
-  std::string ReplacedValueWithHeader;
-};
-
-/**
- * @brief Define the special headers
- *
- * @details Current rules:
- *
- * - If header x-ms-proposed-lease-id is in the request, then use its value for the header
- * x-ms-lease-id in the response.
- *
- * - If header x-ms-lease-action is in the request and its value is equals to renew, then use the
- * value from request header x-ms-lease-id for the response header value of x-ms-lease-id
- *
- */
-std::vector<UniqueIdInfo> UniqueHeaders(
-    {{"x-ms-proposed-lease-id", "", "x-ms-lease-id", ""},
-     {"x-ms-lease-action", "renew", "x-ms-lease-id", "x-ms-lease-id"}});
-
-std::mutex PlaybackClientMutex;
-
-std::unique_ptr<RawResponse> PlaybackClient::Send(
-    Request& request,
-    Azure::Core::Context const& context)
-{
-  context.ThrowIfCancelled();
-
-  {
-    // This mutex obligates the playbackClient to run `Send` method from just one thread at a time.
-    // PlaybackClient forces program to distach one request at a time
-    // This is how playback client can support concurrency, if multiple threads uses the same
-    // pipeline to perform a request, the playback client will dispatch one at a time.
-    std::unique_lock<std::mutex> lock(PlaybackClientMutex);
-
-    // The test name can't be known before the test is started. That's why the test data is loaded
-    // up to this point instead of loading it on test SetUp. The test data will be loaded just one
-    // time.
-    m_interceptorManager->LoadTestData();
-
-    auto& recordedData = m_interceptorManager->GetRecordedData();
-    Azure::Core::Url const redactedUrl = m_interceptorManager->RedactUrl(request.GetUrl());
-
-    std::map<std::string, std::string> uniqueIds;
-    auto const& requestHeaders = request.GetHeaders();
-    for (auto const& requestHeader : requestHeaders)
-    {
-      auto const uniqueHeaderInRequest = std::find_if(
-          UniqueHeaders.begin(),
-          UniqueHeaders.end(),
-          [requestHeader](UniqueIdInfo const& uniqueHeaderInfo) {
-            if (uniqueHeaderInfo.RequestHeaderOnlyIfValue.empty())
-            {
-              return requestHeader.first == uniqueHeaderInfo.RequestHeader;
-            }
-            else
-            {
-              return requestHeader.first == uniqueHeaderInfo.RequestHeader
-                  && requestHeader.second == uniqueHeaderInfo.RequestHeaderOnlyIfValue;
-            }
-          });
-      if (uniqueHeaderInRequest != UniqueHeaders.end())
-      {
-        // header is a uniqueHeader, save the value to use in the response.
-        auto const headerForReplacing = uniqueHeaderInRequest->ReplacedValueWithHeader.empty()
-            ? requestHeader.second
-            : requestHeaders.at(uniqueHeaderInRequest->ReplacedValueWithHeader);
-
-        uniqueIds.emplace(uniqueHeaderInRequest->ReplaceResponseHeader, headerForReplacing);
-      }
-    }
-
-    for (auto record = recordedData.NetworkCallRecords.begin();
-         record != recordedData.NetworkCallRecords.end();)
-    {
-      auto url = redactedUrl.GetAbsoluteUrl();
-      auto m = request.GetMethod().ToString();
-      // Use the first occurrence and remove it from the recording.
-      if (m == record->Method && url == record->Url)
-      {
-        // StatusCode
-        auto const statusCode
-            = HttpStatusCode(std::stoi(record->Response.find("STATUS_CODE")->second));
-        auto rpIt = record->Response.find("REASON_PHRASE");
-        auto rp = rpIt != record->Response.end() ? rpIt->second : "recorded response";
-        auto response = std::make_unique<RawResponse>(1, 1, statusCode, rp);
-
-        // Headers
-        for (auto const& header : record->Response)
-        {
-          if (header.first != "STATUS_CODE" && header.first != "BODY"
-              && header.first != "REASON_PHRASE")
-          {
-            auto const replaceWithUnique = std::find_if(
-                uniqueIds.begin(),
-                uniqueIds.end(),
-                [header](std::pair<std::string, std::string> const& uniqueHeaderInfo) {
-                  return header.first == uniqueHeaderInfo.first;
-                });
-
-            if (replaceWithUnique == uniqueIds.end())
-            {
-              response->SetHeader(header.first, header.second);
-            }
-            else
-            {
-              response->SetHeader(header.first, uniqueIds[header.first]);
-            }
-          }
-        }
-
-        // Body
-        auto const body = record->Response.find("BODY")->second;
-        {
-          std::string const bodyStreamSentinel(RECORDING_BODY_STREAM_SENTINEL);
-          auto const bodyStreamSentinelLen = bodyStreamSentinel.length();
-          if (body.length() > bodyStreamSentinelLen
-              && bodyStreamSentinel
-                  == std::string(body.begin(), body.begin() + bodyStreamSentinelLen))
-          {
-            // Sentinel found. Generate bodyStream
-            std::string bodyStreamSettings(body.begin() + bodyStreamSentinelLen, body.end());
-            auto const separator = bodyStreamSettings.find('_');
-            auto const bodyStreamSize = std::atoi(
-                std::string(bodyStreamSettings.begin(), bodyStreamSettings.begin() + separator)
-                    .data());
-            auto const bodyStreamFillWith = std::atoi(
-                std::string(bodyStreamSettings.begin() + separator + 1, bodyStreamSettings.end())
-                    .data());
-
-            response->SetBodyStream(std::make_unique<CircularBodyStream>(
-                bodyStreamSize, static_cast<uint8_t>(bodyStreamFillWith)));
-          }
-          else
-          {
-            // No special sentinel. Use the entire body from recording as the response.
-            std::vector<uint8_t> bodyVector(body.begin(), body.end());
-            response->SetBodyStream(std::make_unique<WithMemoryBodyStream>(bodyVector));
-          }
-        }
-
-        // take the record out of the recording
-        record = recordedData.NetworkCallRecords.erase(record);
-
-        return response;
-      }
-      ++record;
-    }
-  }
-  throw std::runtime_error("Did not found a response for the request in the recordings.");
-}

sdk/core/azure-core-test/src/record_policy.cpp

@@ -1,130 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-#include "azure/core/test/interceptor_manager.hpp"
-#include "azure/core/test/network_models.hpp"
-#include "azure/core/test/record_network_call_policy.hpp"
-
-#include <azure/core/internal/strings.hpp>
-
-#include <string>
-#include <vector>
-
-using namespace Azure::Core::Http::Policies;
-using namespace Azure::Core::Http;
-using namespace Azure::Core::Test;
-using namespace Azure::Core::_internal;
-
-// 2 Kb max
-#define MAX_SUPPORTED_BODYSTREAM_SIZE 1024 * 2
-
-/**
- * @brief Records network request and response into RecordedData.
- *
- * @param ctx The context for canceling the request.
- * @param request The HTTP request that is sent.
- * @param nextHttpPolicy The next policy in the pipeline.
- * @return The HTTP raw response.
- */
-std::unique_ptr<RawResponse> RecordNetworkCallPolicy::Send(
-    Request& request,
-    NextHttpPolicy nextHttpPolicy,
-    Context const& ctx) const
-{
-  if (m_interceptorManager->GetTestMode() != TestMode::RECORD
-      || m_interceptorManager->GetTestContext().LiveOnly)
-  {
-    return nextHttpPolicy.Send(request, ctx);
-  }
-
-  // Init recordedRecord
-  NetworkCallRecord record;
-
-  record.Method = request.GetMethod().ToString();
-  // Capture headers
-  {
-    std::vector<std::string> headersToBeCaptured
-        = {"x-ms-client-request-id", "Content-Type", "x-ms-version", "User-Agent"};
-
-    for (auto const& header : request.GetHeaders())
-    {
-      if (std::find_if(
-              headersToBeCaptured.begin(),
-              headersToBeCaptured.end(),
-              [header](std::string const& compare) {
-                return StringExtensions::LocaleInvariantCaseInsensitiveEqual(compare, header.first);
-              })
-          != headersToBeCaptured.end())
-        record.Headers.emplace(header.first, header.second);
-    }
-  }
-
-  // Remove sensitive information such as SAS token signatures from the recording.
-  {
-    Azure::Core::Url const redactedUrl = m_interceptorManager->RedactUrl(request.GetUrl());
-    record.Url = redactedUrl.GetAbsoluteUrl();
-  }
-
-  // At this point, the request has been recorded. Send it to capture the response.
-  auto response = nextHttpPolicy.Send(request, ctx);
-
-  record.Response.emplace(
-      "STATUS_CODE",
-      std::to_string(static_cast<typename std::underlying_type<Http::HttpStatusCode>::type>(
-          response->GetStatusCode())));
-
-  record.Response.emplace("REASON_PHRASE", response->GetReasonPhrase());
-
-  for (auto const& header : response->GetHeaders())
-  {
-    if (header.first == "x-ms-encryption-key-sha256")
-    {
-      record.Response.emplace(header.first, "REDACTED");
-    }
-    else
-    {
-      auto headerValue = header.second;
-      record.Response.emplace(header.first, headerValue);
-    }
-  }
-
-  // BodyStreams are currently supported ony up to MAX_SUPPORTED_BODYSTREAM_SIZE
-  // The content is downloaded to the response body and the returned body stream from playback will
-  // stream from the memory buffer instead of the network.
-  // When bigger than MAX_SUPPORTED_BODYSTREAM_SIZE, the recording will use the symbol captured from
-  // the last request with a bodyStream as a sentinel to tell the playback transport adapter how to
-  // generate a bodyStream
-  auto bodyStream = response->ExtractBodyStream();
-  if (bodyStream != nullptr)
-  {
-    auto const bodyStreamLen = bodyStream->Length();
-    if (bodyStreamLen > MAX_SUPPORTED_BODYSTREAM_SIZE)
-    {
-      // Avoid recording a long stream response, instead, let's use the first byte from the payload
-      // and record the size of the expected payload. This will work for Upload/Download big data
-      // Write body for recording
-      std::string bodyResponseStr(
-          RECORDING_BODY_STREAM_SENTINEL + std::to_string(bodyStreamLen) + "_"
-          + std::to_string(*m_symbol));
-      std::vector<uint8_t> bodyResponseBytes(bodyResponseStr.begin(), bodyResponseStr.end());
-      response->SetBody(bodyResponseBytes);
-
-      // Let the response to own the body stream again
-      response->SetBodyStream(std::move(bodyStream));
-    }
-    else
-    {
-      // SelfMemoryBodyStream would copy the response to memory
-      response->SetBody(bodyStream->ReadToEnd());
-      response->SetBodyStream(std::make_unique<WithMemoryBodyStream>(response->GetBody()));
-    }
-  }
-
-  // Capture response
-  auto const& body = response->GetBody();
-  std::string bodystr(body.begin(), body.end());
-  record.Response.emplace("BODY", bodystr);
-  m_interceptorManager->GetRecordedData().NetworkCallRecords.push_back(record);
-
-  return response;
-}

sdk/core/azure-core-test/src/test_base.cpp

@@ -3,7 +3,6 @@
 
 #include <azure/core/internal/json/json.hpp>
 
-#include "azure/core/test/interceptor_manager.hpp"
 #include "azure/core/test/network_models.hpp"
 #include "azure/core/test/test_base.hpp"
 
@@ -14,39 +13,16 @@ using namespace Azure::Core::Json::_internal;
 
 void Azure::Core::Test::TestBase::TearDown()
 {
-
-  if (m_testContext.IsLiveMode() || m_testContext.IsPlaybackMode())
+  if (m_wasSkipped)
   {
-    // Don't want to record here
     return;
   }
-
-  json root;
-  json records;
-  auto const& recordData = m_interceptor->GetRecordedData();
-
-  if (recordData.NetworkCallRecords.size() == 0)
+  if (m_testProxy->IsRecordMode())
   {
-    // Don't make empty recordings
-    return;
+    m_testProxy->StopPlaybackRecord(TestMode::RECORD);
   }
-
-  for (auto const& record : recordData.NetworkCallRecords)
+  if (m_testProxy->IsPlaybackMode())
   {
-    json recordJson;
-    recordJson["Headers"] = json(record.Headers);
-    recordJson["Response"] = json(record.Response);
-    recordJson["Method"] = record.Method;
-    recordJson["Url"] = record.Url;
-    records.push_back(recordJson);
+    m_testProxy->StopPlaybackRecord(TestMode::PLAYBACK);
   }
-  root["networkCallRecords"] = records;
-
-  // Write json to file
-  std::ofstream outFile;
-  // AZURE_TEST_RECORDING_DIR is exported from CMAKE
-  std::string testPath(m_testContext.RecordingPath);
-  outFile.open(testPath + "/" + m_testContext.GetTestPlaybackRecordingName() + ".json");
-  outFile << root.dump(2) << std::endl;
-  outFile.close();
 }

sdk/core/azure-core-test/src/test_proxy_manager.cpp

@@ -0,0 +1,280 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+#include <azure/core/internal/environment.hpp>
+#include <azure/core/internal/json/json.hpp>
+#include <azure/core/internal/strings.hpp>
+
+#include "azure/core/test/test_proxy_manager.hpp"
+
+#include <fstream>
+#include <iostream>
+#include <regex>
+#include <stdexcept>
+#include <string>
+
+using namespace Azure::Core::Test;
+using namespace Azure::Core;
+using Azure::Core::_internal::Environment;
+
+TestMode TestProxyManager::GetTestMode()
+{
+  auto value = Environment::GetVariable("AZURE_TEST_MODE");
+  if (value.empty())
+  {
+    return Azure::Core::Test::TestMode::LIVE;
+  }
+
+  if (Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual(
+          value, "RECORD"))
+  {
+    return Azure::Core::Test::TestMode::RECORD;
+  }
+  else if (Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual(
+               value, "PLAYBACK"))
+  {
+    return Azure::Core::Test::TestMode::PLAYBACK;
+  }
+  else if (Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual(
+               value, "LIVE"))
+  {
+    return Azure::Core::Test::TestMode::LIVE;
+  }
+
+  // unexpected variable value
+  throw std::runtime_error("Invalid environment variable value for AZURE_TEST_MODE: " + value);
+}
+
+void TestProxyManager::ConfigureInsecureConnection(
+    Azure::Core::_internal::ClientOptions& clientOptions)
+{
+  // NOTE: perf-fm is injecting the SSL config and transport here for the client options
+  //       If the test overrides the options/transport, this can be undone.
+  if (m_isInsecureEnabled)
+  {
+#if defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER)
+    Azure::Core::Http::CurlTransportOptions curlOptions;
+    curlOptions.SslVerifyPeer = false;
+    curlOptions.SslOptions.AllowFailedCrlRetrieval = true;
+    clientOptions.Transport.Transport
+        = std::make_shared<Azure::Core::Http::CurlTransport>(curlOptions);
+#elif defined(BUILD_TRANSPORT_WINHTTP_ADAPTER)
+    Azure::Core::Http::WinHttpTransportOptions winHttpOptions;
+    winHttpOptions.IgnoreUnknownCertificateAuthority = true;
+    clientOptions.Transport.Transport
+        = std::make_shared<Azure::Core::Http::WinHttpTransport>(winHttpOptions);
+#else
+    // avoid the variable not used warning
+    (void)clientOptions;
+#endif
+  }
+}
+
+std::string TestProxyManager::PrepareRequestBody()
+{
+  std::string recordingPath = m_testContext.GetTestRecordingPathName();
+  std::string body;
+
+  auto sdkPos = recordingPath.rfind("sdk");
+  recordingPath = recordingPath.substr(sdkPos, recordingPath.size() - sdkPos);
+  body = "{\"x-recording-file\":\"";
+  body.append(recordingPath);
+  body.append("\"");
+  body.append(",");
+  body.append("\"x-recording-assets-file\":\"");
+  body.append(m_testContext.AssetsPath);
+  body.append("\"");
+
+  body.append("}");
+
+  return body;
+}
+
+void TestProxyManager::StartPlaybackRecord(TestMode testMode)
+{
+  if (IsPlaybackMode() || IsRecordMode())
+  {
+    std::string mode = (IsPlaybackMode() ? "playback" : "record");
+    std::cout << "TestProxy already in " + mode + " mode.";
+    return;
+  }
+
+  m_currentMode = testMode;
+
+  Azure::Core::Url startRequest(m_proxy);
+  if (testMode == TestMode::PLAYBACK)
+  {
+    startRequest.AppendPath("playback");
+  }
+  else if (testMode == TestMode::RECORD)
+  {
+    startRequest.AppendPath("record");
+  }
+
+  startRequest.AppendPath("start");
+  std::string body = PrepareRequestBody();
+
+  Azure::Core::IO::MemoryBodyStream payloadStream(
+      reinterpret_cast<const uint8_t*>(body.data()), body.size());
+  Azure::Core::Http::Request request(
+      Azure::Core::Http::HttpMethod::Post, startRequest, &payloadStream);
+
+  Azure::Core::Context ctx;
+  auto response = m_privatePipeline->Send(request, ctx);
+
+  auto const& headers = response->GetHeaders();
+  auto findHeader = std::find_if(
+      headers.begin(), headers.end(), [](std::pair<std::string const&, std::string const&> h) {
+        return h.first == "x-recording-id";
+      });
+  m_testContext.RecordingId = findHeader->second;
+}
+
+void TestProxyManager::StopPlaybackRecord(TestMode testMode)
+{
+  if (testMode == TestMode::PLAYBACK && !IsPlaybackMode())
+  {
+    throw std::runtime_error("TestProxy not in playback mode.");
+  }
+  if (testMode == TestMode::RECORD && !IsRecordMode())
+  {
+    throw std::runtime_error("TestProxy not in record mode");
+  }
+
+  Azure::Core::Url stopRequest(m_proxy);
+
+  if (m_currentMode == TestMode::PLAYBACK)
+  {
+    stopRequest.AppendPath("playback");
+  }
+  else if (m_currentMode == TestMode::RECORD)
+  {
+    stopRequest.AppendPath("record");
+  }
+
+  stopRequest.AppendPath("stop");
+
+  Azure::Core::Http::Request request(Azure::Core::Http::HttpMethod::Post, stopRequest);
+  request.SetHeader("x-recording-id", m_testContext.RecordingId);
+  Azure::Core::Context ctx;
+
+  m_privatePipeline->Send(request, ctx);
+
+  m_testContext.RecordingId.clear();
+  m_currentMode = TestMode::LIVE;
+}
+std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy> TestProxyManager::GetTestProxyPolicy()
+{
+  return std::make_unique<Azure::Core::Test::TestProxyPolicy>(this);
+}
+
+bool TestProxyManager::CheckSanitizers()
+{
+  Azure::Core::Url checkRequest(m_proxy);
+
+  checkRequest.AppendPath("Info");
+  checkRequest.AppendPath("Active");
+
+  {
+    Azure::Core::Http::Request request(Azure::Core::Http::HttpMethod::Get, checkRequest);
+    Azure::Core::Context ctx;
+    auto response = m_privatePipeline->Send(request, ctx);
+
+    auto rawResponse = response->GetBody();
+    std::string stringBody(rawResponse.begin(), rawResponse.end());
+    std::string regex = "\"https://(?<account>[a-zA-Z0-9\\-]+).\"";
+    std::vector<std::string> stringsInOrder
+        = {"UriRegexSanitizer",
+           regex,
+           "BodyRegexSanitizer",
+           regex,
+           "HeaderRegexSanitizer",
+           regex,
+           "GeneralRegexSanitizer",
+           regex,
+           "CustomDefaultMatcher"};
+
+    size_t position = 0;
+
+    for (auto& part : stringsInOrder)
+    {
+      position = stringBody.find(part, position);
+      if (position == std::string::npos)
+      {
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
+void TestProxyManager::SetProxySanitizer()
+{
+  if (CheckSanitizers())
+  {
+    return;
+  }
+  Azure::Core::Url sanitizerRequest(m_proxy);
+  sanitizerRequest.AppendPath("Admin");
+  sanitizerRequest.AppendPath("AddSanitizer");
+  const std::string regexBody
+      = "{\"key\" : \"Location\",\"value\" : \"REDACTED\",\"regex\": "
+        "\"https://(?<account>[a-zA-Z0-9\\\\-]+).\",\"groupForReplace\" : \"account\"}";
+
+  Azure::Core::Url matcherRequest(m_proxy);
+  matcherRequest.AppendPath("Admin");
+  matcherRequest.AppendPath("SetMatcher");
+  const std::string matcherBody
+      = "{\"compareBodies\": false ,\"ignoreQueryOrdering\": true , \"ignoredHeaders\": "
+        "\"x-ms-copy-source,x-ms-proposed-lease-id,x-ms-lease-id,x-ms-file-change-"
+        "time,x-ms-file-creation-time,x-ms-file-last-write-time,x-ms-destination-"
+        "lease-id,x-ms-source-lease-id,x-ms-source-content-crc64,x-ms-content-crc64,x-ms-source-"
+        "content-md5,x-ms-content-md5,Content-MD5\"}";
+
+  {
+    Azure::Core::IO::MemoryBodyStream payloadStream(
+        reinterpret_cast<const uint8_t*>(regexBody.data()), regexBody.size());
+    Azure::Core::Http::Request request(
+        Azure::Core::Http::HttpMethod::Post, sanitizerRequest, &payloadStream);
+    request.SetHeader("x-abstraction-identifier", "UriRegexSanitizer");
+    Azure::Core::Context ctx;
+    auto response = m_privatePipeline->Send(request, ctx);
+  }
+  {
+    Azure::Core::IO::MemoryBodyStream payloadStream(
+        reinterpret_cast<const uint8_t*>(regexBody.data()), regexBody.size());
+    Azure::Core::Http::Request request(
+        Azure::Core::Http::HttpMethod::Post, sanitizerRequest, &payloadStream);
+    request.SetHeader("x-abstraction-identifier", "BodyRegexSanitizer");
+    Azure::Core::Context ctx;
+    auto response = m_privatePipeline->Send(request, ctx);
+  }
+  {
+    Azure::Core::IO::MemoryBodyStream payloadStream(
+        reinterpret_cast<const uint8_t*>(regexBody.data()), regexBody.size());
+    Azure::Core::Http::Request request(
+        Azure::Core::Http::HttpMethod::Post, sanitizerRequest, &payloadStream);
+    request.SetHeader("x-abstraction-identifier", "HeaderRegexSanitizer");
+    Azure::Core::Context ctx;
+    auto response = m_privatePipeline->Send(request, ctx);
+  }
+  {
+    Azure::Core::IO::MemoryBodyStream payloadStream(
+        reinterpret_cast<const uint8_t*>(regexBody.data()), regexBody.size());
+    Azure::Core::Http::Request request(
+        Azure::Core::Http::HttpMethod::Post, sanitizerRequest, &payloadStream);
+    request.SetHeader("x-abstraction-identifier", "GeneralRegexSanitizer");
+    Azure::Core::Context ctx;
+    auto response = m_privatePipeline->Send(request, ctx);
+  }
+  {
+    Azure::Core::IO::MemoryBodyStream payloadStream(
+        reinterpret_cast<const uint8_t*>(matcherBody.data()), matcherBody.size());
+    Azure::Core::Http::Request request(
+        Azure::Core::Http::HttpMethod::Post, matcherRequest, &payloadStream);
+    request.SetHeader("x-abstraction-identifier", "CustomDefaultMatcher");
+    Azure::Core::Context ctx;
+    auto response = m_privatePipeline->Send(request, ctx);
+  }
+}

sdk/core/azure-core-test/src/test_proxy_policy.cpp

@@ -0,0 +1,83 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+#include "azure/core/test/test_proxy_policy.hpp"
+#include "azure/core/test/network_models.hpp"
+#include "azure/core/test/test_context_manager.hpp"
+
+#include <azure/core/internal/strings.hpp>
+
+#include <string>
+#include <vector>
+
+using namespace Azure::Core::Http::Policies;
+using namespace Azure::Core::Http;
+using namespace Azure::Core::Test;
+using namespace Azure::Core::_internal;
+
+/**
+ * @brief Records network request and response into RecordedData.
+ *
+ * @param ctx The context for canceling the request.
+ * @param request The HTTP request that is sent.
+ * @param nextHttpPolicy The next policy in the pipeline.
+ * @return The HTTP raw response.
+ */
+std::unique_ptr<RawResponse> TestProxyPolicy::Send(
+    Request& request,
+    NextHttpPolicy nextHttpPolicy,
+    Context const& ctx) const
+{
+  std::string const recordId(m_testProxy->GetRecordingId());
+  if (recordId.empty() || m_testProxy->GetTestContext().LiveOnly
+      || m_testProxy->GetTestMode() == TestMode::LIVE)
+  {
+    return nextHttpPolicy.Send(request, ctx);
+  }
+
+  // Use a new request to redirect
+  auto redirectRequest = Azure::Core::Http::Request(
+      request.GetMethod(), Azure::Core::Url(m_testProxy->GetTestProxy()), request.GetBodyStream());
+  if (!request.ShouldBufferResponse())
+  {
+    // This is a download with keep connection open. Let's switch the request
+    redirectRequest = Azure::Core::Http::Request(
+        request.GetMethod(), Azure::Core::Url(m_testProxy->GetTestProxy()), false);
+  }
+
+  redirectRequest.GetUrl().SetPath(request.GetUrl().GetPath());
+
+  // Copy all headers
+  for (auto& header : request.GetHeaders())
+  {
+    redirectRequest.SetHeader(header.first, header.second);
+  }
+  // QP
+  for (auto const& qp : request.GetUrl().GetQueryParameters())
+  {
+    redirectRequest.GetUrl().AppendQueryParameter(qp.first, qp.second);
+  }
+  // Set x-recording-upstream-base-uri
+  {
+    auto const& url = request.GetUrl();
+    auto const port = url.GetPort();
+    auto const host
+        = url.GetScheme() + "://" + url.GetHost() + (port != 0 ? ":" + std::to_string(port) : "");
+    redirectRequest.SetHeader("x-recording-upstream-base-uri", host);
+  }
+  // Set recording-id
+  redirectRequest.SetHeader("x-recording-id", recordId);
+
+  if (m_testProxy->IsRecordMode())
+  {
+    // RECORDING mode
+    redirectRequest.SetHeader("x-recording-mode", "record");
+  }
+  else if (m_testProxy->IsPlaybackMode())
+  {
+    // Playback mode
+    redirectRequest.SetHeader("x-recording-mode", "playback");
+  }
+
+  return nextHttpPolicy.Send(redirectRequest, ctx);
+}