diff --git a/CMakeSettings.json b/CMakeSettings.json index b1a94d21b..055478cac 100644 --- a/CMakeSettings.json +++ b/CMakeSettings.json @@ -1,16 +1,28 @@ { - "configurations": [ - { - "name": "x64-Debug", - "generator": "Ninja", - "configurationType": "Debug", - "inheritEnvironments": [ "msvc_x64_x64" ], - "buildRoot": "${projectDir}\\out\\build\\${name}", - "installRoot": "${projectDir}\\out\\install\\${name}", - "cmakeCommandArgs": "-DINSTALL_GTEST=OFF -DBUILD_TESTING=OFF -DBUILD_CURL_TRANSPORT=ON", - "buildCommandArgs": "-v", - "ctestCommandArgs": "", - "variables": [] - } - ] -} + "configurations": [ + { + "name": "x64-Debug", + "generator": "Ninja", + "configurationType": "Debug", + "inheritEnvironments": [ "msvc_x64_x64" ], + "buildRoot": "${projectDir}\\out\\build\\${name}", + "installRoot": "${projectDir}\\out\\install\\${name}", + "cmakeCommandArgs": "-DINSTALL_GTEST=OFF -DBUILD_TESTING=OFF -DBUILD_CURL_TRANSPORT=ON", + "buildCommandArgs": "-v", + "ctestCommandArgs": "", + "variables": [] + }, + { + "name": "x86-Debug", + "generator": "Ninja", + "configurationType": "Debug", + "buildRoot": "${projectDir}\\out\\build\\${name}", + "installRoot": "${projectDir}\\out\\install\\${name}", + "cmakeCommandArgs": "", + "buildCommandArgs": "", + "ctestCommandArgs": "", + "inheritEnvironments": [ "msvc_x86" ], + "variables": [] + } + ] +} \ No newline at end of file diff --git a/sdk/core/azure-core/CMakeLists.txt b/sdk/core/azure-core/CMakeLists.txt index 7002f1d62..9a667da45 100644 --- a/sdk/core/azure-core/CMakeLists.txt +++ b/sdk/core/azure-core/CMakeLists.txt @@ -8,9 +8,10 @@ project(${TARGET_NAME} LANGUAGES CXX) set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED True) -find_package(CURL CONFIG) +set(CURL_MIN_REQUIRED_VERSION 7.4) +find_package(CURL ${CURL_MIN_REQUIRED_VERSION} CONFIG) if(NOT CURL_FOUND) - find_package(CURL REQUIRED) + find_package(CURL ${CURL_MIN_REQUIRED_VERSION} REQUIRED) endif() add_library ( @@ -21,7 +22,9 @@ add_library ( src/http/policy.cpp src/http/request.cpp src/http/response.cpp - src/http/curl/curl_transport.cpp + src/http/stream.cpp + src/http/url.cpp + src/http/curl/curl.cpp src/http/winhttp/win_http_transport.cpp ) diff --git a/sdk/core/azure-core/inc/http/curl/curl.hpp b/sdk/core/azure-core/inc/http/curl/curl.hpp index 943af009e..d50a078de 100644 --- a/sdk/core/azure-core/inc/http/curl/curl.hpp +++ b/sdk/core/azure-core/inc/http/curl/curl.hpp @@ -14,77 +14,459 @@ namespace Azure { namespace Core { namespace Http { - class CurlTransport : public HttpTransport { + constexpr auto UploadSstreamPageSize = 1024 * 64; + constexpr auto LibcurlReaderSize = 100; + + /** + * @brief Statefull component that controls sending an HTTP Request with libcurl thru the wire and + * parsing and building an HTTP Response. + * This session supports the classic libcurl easy interface to send and receive bytes from network + * using callbacks. + * This session also supports working with the custom HTTP protocol option from libcurl to + * manually upload and download bytes using a network socket. This implementation is used when + * working with streams so customers can lazily pull data from netwok using an stream abstraction. + * + * @remarks This component is expected to be used by an HTTP Transporter to ensure that + * transporter to be re usuable in multiple pipelines while every call to network is unique. + */ + class CurlSession { private: - CurlTransport(const CurlTransport&) = delete; - CurlTransport& operator=(const CurlTransport&) = delete; + /** + * @brief Enum used by ResponseBufferParser to control the parsing internal state while building + * the HTTP Response + * + */ + enum class ResponseParserState + { + StatusLine, + Headers, + EndOfHeaders, + }; - // for every client instance, create a default response - std::unique_ptr m_response; - bool m_firstHeader; + /** + * @brief stateful component used to read and parse a buffer to construct a valid HTTP Response. + * + * It uses an internal string as buffers to accumulate a response token (version, code, header, + * etc) until the next delimiter is found. Then it uses this string to keep building the HTTP + * Response. + * + * @remark Only status line and headers are parsed and built. Body is ignored by this component. + * A libcurl session will use this component to build and return the HTTP Response with a body + * stream to the pipeline. + */ + class ResponseBufferParser { + private: + /** + * @brief Controls what the parser is expecting during the reading process + * + */ + ResponseParserState state; + /** + * @brief Unique prt to a response. Parser will create an Initial-valid HTTP Response and then + * it will append headers to it. This response is moved to a different owner once parsing is + * completed. + * + */ + std::unique_ptr m_response; + /** + * @brief Indicates if parser has found the end of the headers and there is nothing left for + * the HTTP Response. + * + */ + bool m_parseCompleted; + /** + * @brief This buffer is used when the parsed buffer doesn't contain a completed token. The + * content from the buffer will be appended to this buffer. Once that a delimiter is found, + * the token for the HTTP Response is taken from this internal sting if it contains data. + * + * @remark This buffer allows a libcurl session to use any size of buffer to read from a + * socket while constructing an initial valid HTTP Response. No matter if the response from + * wire contains hundreds of headers, we can use only one fixed size buffer to parse it all. + * + */ + std::string m_internalBuffer; + + /** + * @brief This method is invoked by the Parsing process if the internal state is set to status + * code. Function will get the status-line expected tokens until finding the end of status + * line delimiter. + * + * @remark When the end of status line delimiter is found, this method will create the HTTP + * Response. The HTTP Response is constructed by default with body type as Stream. + * + * @param buffer Points to a memory address with all or some part of a HTTP status line. + * @param bufferSize Indicates the size of the buffer. + * @return Returns the index of the last parsed position from buffer. + */ + size_t BuildStatusCode(uint8_t const* const buffer, size_t const bufferSize); + + /** + * @brief This method is invoked by the Parsing process if the internal state is set to + * headers. Function will keep adding headers to the HTTP Response created before while + * parsing an status line. + * + * @param buffer Points to a memory address with all or some part of a HTTP header. + * @param bufferSize Indicates the size of the buffer. + * @return Returns the index of the last parsed position from buffer. When the returned value + * is smaller than the body size, means there is part of the body response in the buffer. + */ + size_t BuildHeader(uint8_t const* const buffer, size_t const bufferSize); + + public: + /** + * @brief Construct a new Response Buffer Parser object. + * Set the initial state and parsing completion. + * + */ + ResponseBufferParser() + { + state = ResponseParserState::StatusLine; + this->m_parseCompleted = false; + } + + // Parse contents of buffer to construct HttpResponse. Returns the index of the last parsed + // possition. Return bufferSize when all buffer was used to parse + /** + * @brief Parses the content of a buffer to constuct a valid HTTP Response. This method is + * expected to be called over and over until it returns 0, indicating there is nothing more to + * parse to build the HTTP Response. + * + * @param buffer points to a memory area that contains, all or some part of an HTTP response. + * @param bufferSize Indicates the size of the buffer. + * @return Returns the index of the last parsed position. Returning a 0 means nothing was + * parsed and it is likely that the HTTP Response is completed. Returning the same value as + * the buffer size means all buffer was parsed and the HTTP might be completed or not. + * Returning a value smaller than the buffer size will likely indicate that the HTTP Response + * is completed and that the rest of the buffer contains part of the response body. + */ + size_t Parse(uint8_t const* const buffer, size_t const bufferSize); + + /** + * @brief Indicates when the parser has completed parsing and building the HTTP Response. + * + * @return true if parsing is completed. Otherwise false. + */ + bool IsParseCompleted() const { return this->m_parseCompleted; } + + /** + * @brief Moves the internal response to a different owner. + * + * @return Will move the response only if parsing is completed and if the HTTP Response was + * not moved before. + */ + std::unique_ptr GetResponse() + { + if (this->m_parseCompleted && this->m_response != nullptr) + { + return std::move(this->m_response); + } + return nullptr; // parse is not completed or response has been moved already. + } + }; + + /** + * @brief libcurl handle to be used in the session. + * + */ CURL* m_pCurl; - static size_t WriteHeadersCallBack(void* contents, size_t size, size_t nmemb, void* userp); - static size_t WriteBodyCallBack(void* contents, size_t size, size_t nmemb, void* userp); + /** + * @brief libcurl socket abstraction used when working with streams. + * + */ + curl_socket_t m_curlSocket; - // setHeaders() - CURLcode SetUrl(Request& request) - { - return curl_easy_setopt(m_pCurl, CURLOPT_URL, request.GetEncodedUrl().c_str()); - } + /** + * @brief unique ptr for the HTTP Response. The session is responsable for creating the response + * once that an HTTP status line is received. + * + */ + std::unique_ptr m_response; - CURLcode SetHeaders(Request& request) - { - auto headers = request.GetHeaders(); - if (headers.size() == 0) - { - return CURLE_OK; - } + /** + * @brief The HTTP Request for to be used by the session. + * + */ + Request& m_request; - // creates a slist for bulding curl headers - struct curl_slist* headerList = NULL; + /** + * @brief Controls the progress of a body buffer upload when using libcurl callbacks. Woks like + * an offset to move the pointer to read the body from the HTTP Request on each callback. + * + */ + size_t uploadedBytes; - // insert headers - for (auto header : headers) - { - // TODO: check result is not null or trow - headerList = curl_slist_append(headerList, (header.first + ":" + header.second).c_str()); - } + /** + * @brief Control field that gets true as soon as there is no more data to read from network. A + * network socket will return 0 once we got the entire reponse. + * + */ + bool m_rawResponseEOF; - // set all headers from slist - return curl_easy_setopt(m_pCurl, CURLOPT_HTTPHEADER, headerList); - } + /** + * @brief Control field to handle the case when part of HTTP response body was copied to the + * inner buffer. When a libcurl stream tries to read part of the body, this field will help to + * decide how much data to take from the inner buffer before pulling more data from network. + * + */ + size_t m_bodyStartInBuffer; - CURLcode SetWriteResponse() - { - auto settingUp = curl_easy_setopt(m_pCurl, CURLOPT_HEADERFUNCTION, WriteHeadersCallBack); - if (settingUp != CURLE_OK) - { - return settingUp; - } - settingUp = curl_easy_setopt(m_pCurl, CURLOPT_HEADERDATA, (void*)this); - if (settingUp != CURLE_OK) - { - return settingUp; - } - // TODO: set up cache size. user should be able to set it up - settingUp = curl_easy_setopt(m_pCurl, CURLOPT_WRITEFUNCTION, WriteBodyCallBack); - if (settingUp != CURLE_OK) - { - return settingUp; - } + /** + * @brief This is a copy of the value of an HTTP response header `content-length`. The value is + * received as string and parsed to size_t. This field avoid parsing the string header everytime + * from HTTP Response. + * + * @remark This value is also used to avoid trying to read more data from network than what we + * are expecting to. + * + */ + uint64_t m_contentLength; - return curl_easy_setopt(m_pCurl, CURLOPT_WRITEDATA, (void*)this); - } + /** + * @brief Internal buffer from a session used to read bytes from a socket. This buffer is only + * used while constructing an HTTP Response without adding a body to it. Customers would + * provide their own buffer to copy from socket when reading the HTTP body using streams. + * + */ + uint8_t m_readBuffer[LibcurlReaderSize]; // to work with libcurl custom read. - CURLcode Perform(Context& context, Request& request); + /** + * @brief convenient function that indicates when the HTTP Request will need to upload a payload + * or not. + * + * @return true if the HTTP Request will need to upload bytes to wire. + * + */ + bool isUploadRequest(); + + /** + * @brief Set up libcurl handle with a value for CURLOPT_URL. + * + * @return returns the libcurl result after setting up. + */ + CURLcode SetUrl(); + + /** + * @brief Set up libcurl handle with a value for CURLOPT_CONNECT_ONLY. + * + * @remark This configuration is required to enabled the custom upload/download from libcurl + * easy interface. + * + * @return returns the libcurl result after setting up. + */ + CURLcode SetConnectOnly(); + + /** + * @brief Set up libcurl handle to behave as an specific HTTP Method. + * + * @return returns the libcurl result after setting up. + */ + CURLcode SetMethod(); + + /** + * @brief Creates a list of libcurl headers and set it up to CURLOPT_HTTPHEADER. + * + * @remark For an HTTP Request that requires uploading bytes to network, this method will set + * the content-length header and will also set libcurl to avoid sending an expect; header to + * only ask server if it is OK to upload the body. + * + * @return returns the libcurl result after setting up. + */ + CURLcode SetHeaders(); + + /** + * @brief Set up libcurl callback functions for writing and user data. User data ptr for all + * callbacks is set to reference the session object. + * + * @return returns the libcurl result after setting up. + */ + CURLcode SetWriteResponse(); + + /** + * @brief Set up libcurl callback functions for reading and user data. User data ptr for all + * callbacks is set to reference the session object. + * + * @return returns the libcurl result after setting up. + */ + CURLcode SetReadRequest(); + + /** + * @brief Function used when working with Streams to manually write from the HTTP Request to the + * wire. + * + * @return CURL_OK when response is sent successfully. + */ + CURLcode HttpRawSend(); + + /** + * @brief This method will use libcurl socket to write all the bytes from buffer. + * + * @remarks Hardcoded timeout is used in case a socket stop responding. + * + * @param buffer ptr to the data to be sent to wire. + * @param bufferSize size of the buffer to send. + * @return CURL_OK when response is sent successfully. + */ + CURLcode SendBuffer(uint8_t* buffer, size_t bufferSize); + + /** + * @brief This function is used after sending an HTTP request to the server to read the HTTP + * Response from wire until the end of headers only. + * + * @return CURL_OK when an HTTP response is created. + */ + CURLcode ReadStatusLineAndHeadersFromRawResponse(); + + /** + * @brief This function is used when working with streams to pull more data from the wire. + * Function will try to keep pulling data from socket until the buffer is all written or until + * there is no more data to get from the socket. + * + * @param buffer prt to buffer where to copy bytes from socket. + * @param bufferSize size of the buffer and the requested bytes to be pulled from wire. + * @return return the numbers of bytes pulled from socket. It can be less than what it was + * requested. + */ + uint64_t ReadSocketToBuffer(uint8_t* buffer, size_t bufferSize); public: - CurlTransport(); - ~CurlTransport(); + /** + * @brief Construct a new Curl Session object. Init internal libcurl handler. + * + * @param request reference to an HTTP Request. + */ + CurlSession(Request& request) : m_request(request) + { + this->m_pCurl = curl_easy_init(); + this->m_bodyStartInBuffer = 0; + } + /** + * @brief Function will use the HTTP request received in constutor to perform a network call + * based on the HTTP request configuration. + * + * @param context TBD + * @return CURLE_OK when the network call is completed successfully. + */ + CURLcode Perform(Context& context); + + /** + * @brief Moved the ownership of the HTTP Response out of the session. + * + * @return the unique ptr to the HTTP Response or null if the HTTP Response is not yet created + * or was moved before. + */ + std::unique_ptr GetResponse(); + + /** + * @brief Helper method for reading with a Stream. Function will figure it out where to get + * bytes from (either the libcurl socket of the internal buffer from session). The offset is + * how stream controls how much it was already read. + * + * @param buffer ptr to a buffer where to write bytes from HTTP Response body. + * @param bufferSize size of the buffer. + * @param offset the number of bytes previously read. + * @return the number of bytes read. + */ + uint64_t ReadWithOffset(uint8_t* buffer, uint64_t bufferSize, uint64_t offset); + }; + + /** + * @brief Concrete implementation of an HTTP Transport that uses libcurl. + * + */ + class CurlTransport : public HttpTransport { + public: + /** + * @brief Implements interface to send an HTTP Request and produce an HTTP Response + * + * @param context TBD + * @param request an HTTP Request to be send. + * @return unique ptr to an HTTP Response. + */ std::unique_ptr Send(Context& context, Request& request) override; }; + /** + * @brief concrete implementation of a body stream to read bytes for the HTTP body using libcurl + * handler. + */ + class CurlBodyStream : public Azure::Core::Http::BodyStream { + private: + /** + * @brief length of the entire HTTP Response body. + * + */ + uint64_t m_length; + + /** + * @brief reference to a Curl Session with all the configuration to be used to read from wire. + * + */ + CurlSession* m_curlSession; + + /** + * @brief Numbers of bytes already read. + * + */ + uint64_t m_offset; + + public: + /** + * @brief Construct a new Curl Body Stream object. + * + * @param length size of the HTTP Response body. + * @param curlSession reference to a libcurl session that contains the libcurl handler to be + * used. + */ + CurlBodyStream(uint64_t length, CurlSession* curlSession) + : m_length(length), m_curlSession(curlSession), m_offset(0) + { + } + + ~CurlBodyStream() + { + if (this->m_curlSession != nullptr) + { + delete this->m_curlSession; // Session Destructor will cleanup libcurl handle + } + } + + /** + * @brief Gets the length of the HTTP Response body. + * + * @return uint64_t + */ + uint64_t Length() const override { return this->m_length; } + + /** + * @brief Gets the number of bytes received on count from netwok. Copies the bytes to the + * buffer. + * + * @param buffer ptr to a buffer where to copy bytes from network. + * @param count number of bytes to copy from network into buffer. + * @return the number of read and copied bytes from network to buffer. + */ + uint64_t Read(uint8_t* buffer, uint64_t count) override + { + if (this->m_length == this->m_offset) + { + return 0; + } + // Read bytes from curl into buffer. As max as the length of Stream is allowed + auto readCount = this->m_curlSession->ReadWithOffset(buffer, count, this->m_offset); + this->m_offset += readCount; + return readCount; + } + + /** + * @brief clean up heap. Removes the libcurl session and stream from the heap. + * + * @remark calling this method deletes the stream. + * + */ + void Close() override{}; + }; + }}} // namespace Azure::Core::Http diff --git a/sdk/core/azure-core/inc/http/http.hpp b/sdk/core/azure-core/inc/http/http.hpp index 1752cc69f..160ac7cd1 100644 --- a/sdk/core/azure-core/inc/http/http.hpp +++ b/sdk/core/azure-core/inc/http/http.hpp @@ -3,6 +3,8 @@ #pragma once +#include "stream.hpp" + #include #include #include @@ -14,33 +16,9 @@ namespace Azure { namespace Core { namespace Http { enum class TransportKind { - //TODO move this to Factory - Curl, - WinHttp - }; - - // BodyStream is used to read data to/from a service - class BodyStream { - public: - static BodyStream* null; - - // Returns the length of the data; used with the HTTP Content-Length header - virtual uint64_t Length() = 0; - - // Resets the stream back to the beginning (for retries) - // Derived classes that send data in an HTTP request MUST override this and implement it - // properly. - virtual void Rewind() - { - throw "Not Implemented"; // TODO: Replace with best practice as defined by guideline - }; - - // Reads more data; EOF if return < count; throws if error/canceled - virtual uint64_t Read(/*Context& context, */ uint8_t* buffer, uint64_t offset, uint64_t count) - = 0; - - // Closes the stream; typically called after all data read or if an error occurs. - virtual void Close() = 0; + // TODO move this to Factory + Curl, + WinHttp }; enum class HttpStatusCode @@ -150,40 +128,28 @@ namespace Azure { namespace Core { namespace Http { } } - class Request { + enum class BodyType + { + Buffer, + Stream, + }; + // parses full url into protocol, host, port, path and query. + // Authority is not currently supported. + class URL { private: - // query needs to be first or at least before url, since url might update it + std::string m_scheme; + std::string m_host; + std::string m_port; + std::string m_path; std::map m_queryParameters; - HttpMethod _method; - std::string _url; - std::map m_headers; - std::map m_retryHeaders; - std::map m_retryQueryParameters; - // Request can contain no body, or either of next bodies (_bodyBuffer plus size or bodyStream) - BodyStream* m_bodyStream; - std::vector m_bodyBuffer; - - // flag to know where to insert header - bool m_retryModeEnabled; - - // returns left map plus all items in right - // when duplicates, left items are preferred - static std::map MergeMaps( - std::map left, - std::map const& right) - { - left.insert(right.begin(), right.end()); - return left; - } - /** * Will check if there are any query parameter in url looking for symbol '?' * If it is found, it will insert query parameters to m_queryParameters internal field * and remove it from url */ - const std::string parseUrl(std::string const& url) + const std::string SaveAndRemoveQueryParameter(std::string const& url) { const auto firstPosition = std::find(url.begin(), url.end(), '?'); @@ -206,7 +172,7 @@ namespace Azure { namespace Core { namespace Http { // Note: if there is another = symbol before nextPosition, it will be part of the // paramenter value. And if there is not a ? symbol, we add empty string as value - m_queryParameters.insert(std::pair( + this->m_queryParameters.insert(std::pair( std::string(position, equalChar), std::string(valueStart, nextPosition))); position = nextPosition; @@ -215,45 +181,87 @@ namespace Azure { namespace Core { namespace Http { return std::string(url.begin(), firstPosition); } - Request( - HttpMethod httpMethod, - std::string const& url, - BodyStream* bodyStream, - std::vector bodyBuffer) - : _method(std::move(httpMethod)), _url(parseUrl(url)), m_bodyStream(bodyStream), - m_bodyBuffer(std::move(bodyBuffer)), m_retryModeEnabled(false) + public: + URL(std::string const& url); + void AppendPath(std::string const& path) { - // TODO: parse url + this->m_path += "/" + path; // TODO: check if the path already finish with a / } + std::string ToString() const + { + auto port = this->m_port.size() > 0 ? ":" + this->m_port : ""; + return this->m_scheme + "://" + this->m_host + port + this->m_path; // TODO: add query params + } + std::string GetPath() const { return this->m_path; } + std::map GetQueryParameters() const + { + return this->m_queryParameters; + } + void AddQueryParameter(std::string const& name, std::string const& value) + { + this->m_queryParameters.insert(std::pair(name, value)); + } + }; + + class Request { + + private: + HttpMethod m_method; + URL m_url; + std::map m_headers; + std::map m_retryHeaders; + std::map m_retryQueryParameters; + // Work only with streams + BodyStream* m_bodyStream; + + // flag to know where to insert header + bool m_retryModeEnabled; + + // returns left map plus all items in right + // when duplicates, left items are preferred + static std::map MergeMaps( + std::map left, + std::map const& right) + { + left.insert(right.begin(), right.end()); + return left; + } + + std::string GetQueryString() const; public: - Request(HttpMethod httpMethod, std::string const& url) - : Request(httpMethod, url, BodyStream::null, std::vector()) - { - } - - Request(HttpMethod httpMethod, std::string const& url, std::vector bodyBuffer) - : Request(httpMethod, url, BodyStream::null, std::move(bodyBuffer)) - { - } - Request(HttpMethod httpMethod, std::string const& url, BodyStream* bodyStream) - : Request(httpMethod, url, bodyStream, std::vector()) + : m_method(std::move(httpMethod)), m_url(url), m_bodyStream(bodyStream), + m_retryModeEnabled(false) { } + // Typically used for GET with no request body. + Request(HttpMethod httpMethod, std::string const& url) + : Request(httpMethod, url, BodyStream::null) + { + } + + ~Request() + { + if (this->m_bodyStream != BodyStream::null) + { + delete this->m_bodyStream; + } + } + // Methods used to build HTTP request - void AddPath(std::string const& path); + void AppendPath(std::string const& path); void AddQueryParameter(std::string const& name, std::string const& value); void AddHeader(std::string const& name, std::string const& value); void StartRetry(); // only called by retry policy // Methods used by transport layer (and logger) to send request HttpMethod GetMethod() const; - std::string GetEncodedUrl() const; // should return encoded url + std::string GetEncodedUrl() const; // should call URL encode std::map GetHeaders() const; BodyStream* GetBodyStream(); - std::vector const& GetBodyBuffer(); + std::string GetHTTPMessagePreBody() const; }; /* @@ -278,60 +286,62 @@ namespace Azure { namespace Core { namespace Http { class Response { private: - uint16_t m_majorVersion; - uint16_t m_minorVersion; + int32_t m_majorVersion; + int32_t m_minorVersion; HttpStatusCode m_statusCode; std::string m_reasonPhrase; std::map m_headers; - // Response can contain no body, or either of next bodies (m_bodyBuffer or - // bodyStream) - std::vector m_bodyBuffer; - Http::BodyStream* m_bodyStream; + BodyStream* m_bodyStream; Response( - int16_t majorVersion, - uint16_t minorVersion, + int32_t majorVersion, + int32_t minorVersion, HttpStatusCode statusCode, std::string const& reasonPhrase, - std::vector const& bodyBuffer, BodyStream* const BodyStream) : m_majorVersion(majorVersion), m_minorVersion(minorVersion), m_statusCode(statusCode), - m_reasonPhrase(reasonPhrase), m_bodyBuffer(bodyBuffer), m_bodyStream(BodyStream) + m_reasonPhrase(reasonPhrase), m_bodyStream(BodyStream) { } public: Response( - uint16_t majorVersion, - uint16_t minorVersion, + int32_t majorVersion, + int32_t minorVersion, HttpStatusCode statusCode, std::string const& reasonPhrase) - : Response( - majorVersion, - minorVersion, - statusCode, - reasonPhrase, - std::vector(), - Http::BodyStream::null) + : Response(majorVersion, minorVersion, statusCode, reasonPhrase, BodyStream::null) { } + ~Response() + { + if (this->m_bodyStream != BodyStream::null) + { + delete this->m_bodyStream; + } + } + // Methods used to build HTTP response void AddHeader(std::string const& name, std::string const& value); - void AppendBody(uint8_t* ptr, uint64_t size); - - // Util methods for customers to read response/parse an http response - HttpStatusCode GetStatusCode(); - std::string const& GetReasonPhrase(); - std::map const& GetHeaders(); - std::vector& GetBodyBuffer(); + // rfc form header-name: OWS header-value OWS + void AddHeader(std::string const& header); + void SetBodyStream(BodyStream* stream); // adding getters for version and stream body. Clang will complain on Mac if we have unused // fields in a class - uint16_t GetmajorVersion() { return m_majorVersion; } - uint16_t GetMinorVersion() { return m_minorVersion; } - Http::BodyStream* GetBodyStream() { return m_bodyStream; } + int32_t GetMajorVersion() const { return this->m_majorVersion; } + int32_t GetMinorVersion() const { return this->m_minorVersion; } + HttpStatusCode GetStatusCode() const; + std::string const& GetReasonPhrase(); + std::map const& GetHeaders(); + BodyStream* GetBodyStream() { return this->m_bodyStream; } + + // Allocates a buffer in heap and reads and copy stream content into it. + // util for any API that needs to get the content from stream as a buffer + static std::unique_ptr> ConstructBodyBufferFromStream( + BodyStream* const stream); }; }}} // namespace Azure::Core::Http diff --git a/sdk/core/azure-core/inc/http/stream.hpp b/sdk/core/azure-core/inc/http/stream.hpp new file mode 100644 index 000000000..4e8bae457 --- /dev/null +++ b/sdk/core/azure-core/inc/http/stream.hpp @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include +#include +#include + +namespace Azure { namespace Core { namespace Http { + + // BodyStream is used to read data to/from a service + class BodyStream { + public: + static BodyStream* null; + + // Returns the length of the data; used with the HTTP Content-Length header + virtual uint64_t Length() const = 0; + + // Resets the stream back to the beginning (for retries) + // Derived classes that send data in an HTTP request MUST override this and implement it + // properly. + virtual void Rewind() + { + throw "Not Implemented"; // TODO: Replace with best practice as defined by guideline + }; + + // Reads more data; throws if error/canceled + // return copied size + virtual uint64_t Read(/*Context& context, */ uint8_t* buffer, uint64_t count) = 0; + + // Closes the stream; typically called after all data read or if an error occurs. + virtual void Close() = 0; + + // Desstructor. Enables derived classes to call its destructor + virtual ~BodyStream() = 0; + }; + + class MemoryBodyStream : public BodyStream { + private: + std::vector m_buffer; + uint64_t m_offset = 0; + + public: + MemoryBodyStream(std::vector buffer) : m_buffer(std::move(buffer)) {} + + // cast as vector from ptr and length + MemoryBodyStream(uint8_t* ptr, uint64_t length) : m_buffer(ptr, ptr + length) {} + + uint64_t Length() const override { return this->m_buffer.size(); } + + uint64_t Read(uint8_t* buffer, uint64_t count) override + { + uint64_t copy_length = std::min(count, (this->m_buffer.size() - m_offset)); + // Copy what's left or just the count + std::memcpy(buffer, m_buffer.data() + m_offset, (size_t)copy_length); + // move position + m_offset += copy_length; + + return copy_length; + } + + void Rewind() override { m_offset = 0; } + + void Close() override {} + }; + + /* + TODO: fix file to work multi-platform + class FileBodyStream : public BodyStream { + private: + FILE* stream; + uint64_t length; + + public: + FileBodyStream(FILE* stream) + { + // set internal fields + this->stream = stream; + // calculate size seeking end... + this->length = fseeko64(stream, 0, SEEK_END); + // seek back to beggin + this->Rewind(); + } + + // Rewind seek back to 0 + void Rewind() { rewind(this->stream); } + + uint64_t Read( uint8_t* buffer, uint64_t count) + { + // do static cast here? + return (uint64_t)fread(buffer, 1, count, this->stream); + } + + // close does nothing opp + void Close() { fclose(this->stream); } + }; */ + + class LimitBodyStream : public BodyStream { + BodyStream* m_inner; + uint64_t m_length; + uint64_t m_bytesRead = 0; + + LimitBodyStream(BodyStream* inner, uint64_t max_length) + : m_inner(inner), m_length(std::min(inner->Length(), max_length)) + { + } + + uint64_t Length() const override { return this->m_length; } + void Rewind() override + { + this->m_inner->Rewind(); + this->m_bytesRead = 0; + } + uint64_t Read(uint8_t* buffer, uint64_t count) override + { + // Read up to count or whatever length is remaining; whichever is less + uint64_t bytesRead + = m_inner->Read(buffer, std::min(count, this->m_length - this->m_bytesRead)); + this->m_bytesRead += bytesRead; + return bytesRead; + } + void Close() override { this->m_inner->Close(); } + }; + +}}} // namespace Azure::Core::Http diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp new file mode 100644 index 000000000..c5ca9a1d6 --- /dev/null +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -0,0 +1,521 @@ + +#include "http/curl/curl.hpp" + +#include "azure.hpp" +#include "http/http.hpp" + +#include + +using namespace Azure::Core::Http; + +std::unique_ptr CurlTransport::Send(Context& context, Request& request) +{ + // Create CurlSession in heap. We will point to it from response's stream to keep it alive + CurlSession* session = new CurlSession(request); + + auto performing = session->Perform(context); + + if (performing != CURLE_OK) + { + switch (performing) + { + case CURLE_COULDNT_RESOLVE_HOST: { + throw Azure::Core::Http::CouldNotResolveHostException(); + } + case CURLE_WRITE_ERROR: { + throw Azure::Core::Http::ErrorWhileWrittingResponse(); + } + default: { + throw Azure::Core::Http::TransportException(); + } + } + } + + return session->GetResponse(); +} + +CURLcode CurlSession::Perform(Context& context) +{ + AZURE_UNREFERENCED_PARAMETER(context); + + // Working with Body Buffer. let Libcurl use the classic callback to read/write + auto settingUp = SetUrl(); + if (settingUp != CURLE_OK) + { + return settingUp; + } + + settingUp = SetConnectOnly(); + if (settingUp != CURLE_OK) + { + return settingUp; + } + + // stablish connection only (won't send or receive nothing yet) + settingUp = curl_easy_perform(this->m_pCurl); + if (settingUp != CURLE_OK) + { + return settingUp; + } + // Record socket to be used + settingUp = curl_easy_getinfo(this->m_pCurl, CURLINFO_ACTIVESOCKET, &this->m_curlSocket); + if (settingUp != CURLE_OK) + { + return settingUp; + } + + // Send request + settingUp = HttpRawSend(); + if (settingUp != CURLE_OK) + { + return settingUp; + } + this->m_rawResponseEOF = false; // Control EOF for response; + return ReadStatusLineAndHeadersFromRawResponse(); +} + +// Creates an HTTP Response with specific bodyType +static std::unique_ptr CreateHTTPResponse(std::string const& header) +{ + // set response code, http version and reason phrase (i.e. HTTP/1.1 200 OK) + auto start = header.begin() + 5; // HTTP = 4, / = 1, moving to 5th place for version + auto end = std::find(start, header.end(), '.'); + auto majorVersion = std::stoi(std::string(start, end)); + + start = end + 1; // start of minor version + end = std::find(start, header.end(), ' '); + auto minorVersion = std::stoi(std::string(start, end)); + + start = end + 1; // start of status code + end = std::find(start, header.end(), ' '); + auto statusCode = std::stoi(std::string(start, end)); + + start = end + 1; // start of reason phrase + end = std::find(start, header.end(), '\r'); + auto reasonPhrase = std::string(start, end); // remove \r + + // allocate the instance of response to heap with shared ptr + // So this memory gets delegated outside Curl Transport as a shared ptr so memory will be + // eventually released + return std::make_unique( + (uint16_t)majorVersion, (uint16_t)minorVersion, HttpStatusCode(statusCode), reasonPhrase); +} + +// To wait for a socket to be ready to be read/write +static int WaitForSocketReady(curl_socket_t sockfd, int for_recv, long timeout_ms) +{ + struct timeval tv; + fd_set infd, outfd, errfd; + int res; + + tv.tv_sec = timeout_ms / 1000; + tv.tv_usec = (timeout_ms % 1000) * 1000; + + FD_ZERO(&infd); + FD_ZERO(&outfd); + FD_ZERO(&errfd); + + FD_SET(sockfd, &errfd); /* always check for error */ + + if (for_recv) + { + FD_SET(sockfd, &infd); + } + else + { + FD_SET(sockfd, &outfd); + } + + /* select() returns the number of signalled sockets or -1 */ + res = select((int)sockfd + 1, &infd, &outfd, &errfd, &tv); + return res; +} + +bool CurlSession::isUploadRequest() +{ + return this->m_request.GetMethod() == HttpMethod::Put + || this->m_request.GetMethod() == HttpMethod::Post; +} + +CURLcode CurlSession::SetUrl() +{ + return curl_easy_setopt(this->m_pCurl, CURLOPT_URL, this->m_request.GetEncodedUrl().c_str()); +} + +CURLcode CurlSession::SetConnectOnly() +{ + return curl_easy_setopt(this->m_pCurl, CURLOPT_CONNECT_ONLY, 1L); +} + +// Send buffer thru the wire +CURLcode CurlSession::SendBuffer(uint8_t* buffer, size_t bufferSize) +{ + if (bufferSize <= 0) + { + return CURLE_OK; + } + + size_t sentBytesTotal = 0; + CURLcode sendResult; + + do + { + size_t sentBytesPerRequest; + do + { + sentBytesPerRequest = 0; + auto sendFrom = buffer + sentBytesTotal; + auto remainingBytes = bufferSize - sentBytesTotal; + + sendResult = curl_easy_send(this->m_pCurl, sendFrom, remainingBytes, &sentBytesPerRequest); + sentBytesTotal += sentBytesPerRequest; + + if (sendResult == CURLE_AGAIN && !WaitForSocketReady(this->m_curlSocket, 0, 60000L)) + { + throw; + } + } while (sendResult == CURLE_AGAIN); + + if (sendResult != CURLE_OK) + { + return sendResult; + } + + } while (sentBytesTotal < bufferSize); + + return CURLE_OK; +} + +// custom sending to wire an http request +CURLcode CurlSession::HttpRawSend() +{ + auto rawRequest = this->m_request.GetHTTPMessagePreBody(); + uint64_t rawRequestLen = rawRequest.size(); + + CURLcode sendResult = SendBuffer((uint8_t*)rawRequest.data(), (size_t)rawRequestLen); + if (this->m_request.GetMethod() == HttpMethod::Get) + { + uint8_t endOfRequest[] = "0"; + return SendBuffer(endOfRequest, 1); // need one more byte to end request + } + + auto streamBody = this->m_request.GetBodyStream(); + // Send body 64k at a time (libcurl default) + // NOTE: if stream is on top a contiguous memory, we can avoid allocating this copying buffer + std::unique_ptr unique_buffer(new uint8_t[UploadSstreamPageSize]); + auto buffer = unique_buffer.get(); + while (rawRequestLen > 0) + { + rawRequestLen = streamBody->Read(buffer, UploadSstreamPageSize); + sendResult = SendBuffer(buffer, (size_t)rawRequestLen); + } + return sendResult; +} + +// Read status line plus headers to create a response with no body +CURLcode CurlSession::ReadStatusLineAndHeadersFromRawResponse() +{ + auto parser = ResponseBufferParser(); + + // Keep reading until all headers were read + while (!parser.IsParseCompleted()) + { + // Try to fill internal buffer from socket. + // If response is smaller than buffer, we will get back the size of the response + auto bufferSize = ReadSocketToBuffer(this->m_readBuffer, LibcurlReaderSize); + + // parse from buffer to create response + auto bytesParsed = parser.Parse(this->m_readBuffer, (size_t)bufferSize); + // if end of headers is reach before the end of response, that's where body start + if (bytesParsed < bufferSize) + { + this->m_bodyStartInBuffer = bytesParsed + 1; // Set the start of body (skip \n) + } + } + + this->m_response = parser.GetResponse(); + // TODO: tolower ContentLength + auto headers = this->m_response->GetHeaders(); + auto bodySize = atoi(headers.at("Content-Length").data()); + // content-length is used later by session and session won't have access to the response any more + // (unique_ptr), so we save this value + this->m_contentLength = bodySize; + // Move session to live inside the stream from response. + this->m_response->SetBodyStream(new CurlBodyStream(bodySize, this)); + + return CURLE_OK; +} + +uint64_t CurlSession::ReadWithOffset(uint8_t* buffer, uint64_t bufferSize, uint64_t offset) +{ + if (bufferSize <= 0) + { + return 0; + } + + // calculate where to start reading from inner buffer + auto fixedOffset + = offset == 0 ? offset + 1 : offset; // advance the last '\n' from headers end on first read + auto innerBufferStart = this->m_bodyStartInBuffer + fixedOffset; + // total size from content-length header less any bytes already read + auto remainingBodySize = this->m_contentLength - fixedOffset; + + // set ptr for writting + auto writePosition = buffer; + // Set the max to be written as the size of remaining body size + auto bytesToWrite = std::min<>(bufferSize, remainingBodySize); + + // If bodyStartInBuffer is set and while innerBufferStart is less than the buffer, it means there + // is still data at innerbuffer for the body that is not yet read + if (this->m_bodyStartInBuffer > 0 && LibcurlReaderSize > innerBufferStart) + { + // Calculate the right size of innerBuffer: + // It can be smaller than the total body, in that case we will take as much as the size of + // buffer + // It can be bugger than the total body, in that case we will take as much as the size of the + // body + auto innerBufferWithBodyContent = LibcurlReaderSize - innerBufferStart; + auto innerbufferSize = remainingBodySize < innerBufferWithBodyContent + ? remainingBodySize + : innerBufferWithBodyContent; + + // Requested less data than what we have at inner buffer, take it from innerBuffer + if (bufferSize <= innerbufferSize) + { + std::memcpy(writePosition, this->m_readBuffer + innerBufferStart, (size_t)bytesToWrite); + return bytesToWrite; + } + // Requested more data than what we have at innerbuffer. Take all from inner buffer and continue + std::memcpy(writePosition, this->m_readBuffer + innerBufferStart, (size_t)innerbufferSize); + + // Return if all body was read and theres not need to read socket + if (innerbufferSize == remainingBodySize) + { + // libcurl handle can be clean now. We won't request more data + curl_easy_cleanup(this->m_pCurl); + return innerbufferSize; + } + + // next write will be done after reading from socket, move ptr to where to write and how many + // to write + writePosition += innerbufferSize; + bytesToWrite -= innerbufferSize; + } + + // read from socket the remaining requested bytes + auto bytesRead = ReadSocketToBuffer(writePosition, (size_t)bytesToWrite); + if (remainingBodySize - bytesRead == 0) + { + // No more to read from socket + curl_easy_cleanup(this->m_pCurl); + } + + return bytesRead; +} + +// Read from socket until buffer is full or until socket has no more data +uint64_t CurlSession::ReadSocketToBuffer(uint8_t* buffer, size_t bufferSize) +{ + CURLcode readResult; + size_t readBytes = 0; + size_t totalRead = 0; + auto pendingToRead = bufferSize; + + while (!this->m_rawResponseEOF && pendingToRead > 0) + { + do // try to read from socket until response is OK + { + readResult = curl_easy_recv(this->m_pCurl, buffer + totalRead, pendingToRead, &readBytes); + + // socket not ready. Wait or fail on timeout + if (readResult == CURLE_AGAIN && !WaitForSocketReady(this->m_curlSocket, 1, 60000L)) + { + throw; + } + } while (readResult == CURLE_AGAIN); // Keep trying to read until result is not CURLE_AGAIN + + // At this point we read, update counters + totalRead += readBytes; + pendingToRead -= readBytes; + + if (readBytes == 0) + { + // set socket as nothing more to read + this->m_rawResponseEOF = true; + break; + } + } + + return totalRead; +} + +std::unique_ptr CurlSession::GetResponse() +{ + if (this->m_response != nullptr) + { + return std::move(this->m_response); + } + return nullptr; +} + +size_t CurlSession::ResponseBufferParser::Parse( + uint8_t const* const buffer, + size_t const bufferSize) +{ + if (this->m_parseCompleted) + { + return 0; + } + + switch (this->state) + { + case ResponseParserState::StatusLine: { + auto parsedBytes = BuildStatusCode(buffer, bufferSize); + if (parsedBytes < bufferSize) // status code is built and buffer can be still parsed + { + // Can keep parsing, Control have moved to headers + return parsedBytes + Parse(buffer + parsedBytes, bufferSize - parsedBytes); + } + return parsedBytes; + } + case ResponseParserState::Headers: { + auto parsedBytes = BuildHeader(buffer, bufferSize); + if (!this->m_parseCompleted + && parsedBytes < bufferSize) // status code is built and buffer can be still parsed + { + // Can keep parsing, Control have moved to headers + return parsedBytes + Parse(buffer + parsedBytes, bufferSize - parsedBytes); + } + return parsedBytes; + } + case ResponseParserState::EndOfHeaders: + default: { + return 0; + } + } +} + +// Finds delimiter '\r' as the end of the +size_t CurlSession::ResponseBufferParser::BuildStatusCode( + uint8_t const* const buffer, + size_t const bufferSize) +{ + if (this->state != ResponseParserState::StatusLine) + { + return 0; // Wrong internal state to call this method. + } + + uint8_t endOfStatusLine = '\r'; + auto endOfBuffer = buffer + bufferSize; + + // Look for the end of status line in buffer + auto indexOfEndOfStatusLine = std::find(buffer, endOfBuffer, endOfStatusLine); + + if (indexOfEndOfStatusLine == endOfBuffer) + { + // did not find the delimiter yet, copy to internal buffer + this->m_internalBuffer.append(buffer, endOfBuffer); + return bufferSize; // all buffer read and requesting for more + } + + // Delimiter found, check if there is data in the internal buffer + if (this->m_internalBuffer.size() > 0) + { + // If the index is same as buffer it means delimiter is at position 0, meaning that + // internalBuffer containst the status line and we don't need to add anything else + if (indexOfEndOfStatusLine > buffer) + { + // Append and build response minus the delimiter + this->m_internalBuffer.append(buffer, indexOfEndOfStatusLine); + } + this->m_response = CreateHTTPResponse(this->m_internalBuffer); + } + else + { + // Internal Buffer was not required, create response directly from buffer + this->m_response = CreateHTTPResponse(std::string(buffer, indexOfEndOfStatusLine)); + } + + // update control + this->state = ResponseParserState::Headers; + this->m_internalBuffer.clear(); + + // Return the index of the next char to read after delimiter + // No need to advance one more char ('\n') (since we might be at the end of the array) + // Parsing Headers will make sure to move one possition + return indexOfEndOfStatusLine + 1 - buffer; +} + +// Finds delimiter '\r' as the end of the +size_t CurlSession::ResponseBufferParser::BuildHeader( + uint8_t const* const buffer, + size_t const bufferSize) +{ + if (this->state != ResponseParserState::Headers) + { + return 0; // can't run this if state is not Headers. + } + + uint8_t delimiter = '\r'; + auto start = buffer; + auto endOfBuffer = buffer + bufferSize; + + if (bufferSize == 1 && buffer[0] == '\n') + { + // rare case of using buffer of size 1 to read. In this case, if the char is next value after + // headers or previous header, just consider it as read and return + return bufferSize; + } + else if (bufferSize > 1 && this->m_internalBuffer.size() == 0) // only if nothing in buffer, + // advance + { + // move offset one possition. This is because readStatusLine and readHeader will read up to + // '\r' then next delimeter is '\n' and we don't care + start = buffer + 1; + } + + // Look for the end of status line in buffer + auto indexOfEndOfStatusLine = std::find(start, endOfBuffer, delimiter); + + if (indexOfEndOfStatusLine == start && this->m_internalBuffer.size() == 0) + { + // \r found at the start means the end of headers + this->m_internalBuffer.clear(); + this->m_parseCompleted = true; + return 1; // can't return more than the found delimiter. On read remaining we need to also + // remove first char + } + + if (indexOfEndOfStatusLine == endOfBuffer) + { + // did not find the delimiter yet, copy to internal buffer + this->m_internalBuffer.append(start, endOfBuffer); + return bufferSize; // all buffer read and requesting for more + } + + // Delimiter found, check if there is data in the internal buffer + if (this->m_internalBuffer.size() > 0) + { + // If the index is same as buffer it means delimiter is at position 0, meaning that + // internalBuffer containst the status line and we don't need to add anything else + if (indexOfEndOfStatusLine > buffer) + { + // Append and build response minus the delimiter + this->m_internalBuffer.append(start, indexOfEndOfStatusLine); + } + this->m_response->AddHeader(this->m_internalBuffer); + } + else + { + // Internal Buffer was not required, create response directly from buffer + this->m_response->AddHeader(std::string(start, indexOfEndOfStatusLine)); + } + + // reuse buffer + this->m_internalBuffer.clear(); + + // Return the index of the next char to read after delimiter + // No need to advance one more char ('\n') (since we might be at the end of the array) + // Parsing Headers will make sure to move one possition + return indexOfEndOfStatusLine + 1 - buffer; +} diff --git a/sdk/core/azure-core/src/http/curl/curl_transport.cpp b/sdk/core/azure-core/src/http/curl/curl_transport.cpp deleted file mode 100644 index 3d5f1bc04..000000000 --- a/sdk/core/azure-core/src/http/curl/curl_transport.cpp +++ /dev/null @@ -1,185 +0,0 @@ - -#include "azure.hpp" -#include "http/curl/curl.hpp" -#include "http/http.hpp" - -#include - -using namespace Azure::Core::Http; - -CurlTransport::CurlTransport() : HttpTransport() { m_pCurl = curl_easy_init(); } - -CurlTransport::~CurlTransport() { curl_easy_cleanup(m_pCurl); } - -std::unique_ptr CurlTransport::Send(Context& context, Request& request) -{ - auto performing = Perform(context, request); - - if (performing != CURLE_OK) - { - switch (performing) - { - case CURLE_COULDNT_RESOLVE_HOST: { - throw Azure::Core::Http::CouldNotResolveHostException(); - } - case CURLE_WRITE_ERROR: { - throw Azure::Core::Http::ErrorWhileWrittingResponse(); - } - default: { - throw Azure::Core::Http::TransportException(); - } - } - } - return std::move(m_response); -} - -CURLcode CurlTransport::Perform(Context& context, Request& request) -{ - AZURE_UNREFERENCED_PARAMETER(context); - - m_firstHeader = true; - - auto settingUp = SetUrl(request); - if (settingUp != CURLE_OK) - { - return settingUp; - } - - if (request.GetMethod() == HttpMethod::Get) - { - settingUp = curl_easy_setopt(m_pCurl, CURLOPT_HTTPGET, 1L); - } - else if (request.GetMethod() == HttpMethod::Put) - { - settingUp = curl_easy_setopt(m_pCurl, CURLOPT_PUT, 1L); - } - else - { - settingUp - = curl_easy_setopt(m_pCurl, CURLOPT_CUSTOMREQUEST, HttpMethodToString(request.GetMethod()).data()); - } - if (settingUp != CURLE_OK) - { - return settingUp; - } - - if (request.GetMethod() == HttpMethod::Head) - { - settingUp = curl_easy_setopt(m_pCurl, CURLOPT_NOBODY, 1L); - if (settingUp != CURLE_OK) - { - return settingUp; - } - } - - settingUp = SetHeaders(request); - if (settingUp != CURLE_OK) - { - return settingUp; - } - - settingUp = SetWriteResponse(); - if (settingUp != CURLE_OK) - { - return settingUp; - } - - return curl_easy_perform(m_pCurl); -} - -static std::unique_ptr ParseAndSetFirstHeader(std::string const& header) -{ - // set response code, http version and reason phrase (i.e. HTTP/1.1 200 OK) - auto start = header.begin() + 5; // HTTP = 4, / = 1, moving to 5th place for version - auto end = std::find(start, header.end(), '.'); - auto majorVersion = std::stoi(std::string(start, end)); - - start = end + 1; // start of minor version - end = std::find(start, header.end(), ' '); - auto minorVersion = std::stoi(std::string(start, end)); - - start = end + 1; // start of status code - end = std::find(start, header.end(), ' '); - auto statusCode = std::stoi(std::string(start, end)); - - start = end + 1; // start of reason phrase - auto reasonPhrase = std::string(start, header.end() - 2); // remove \r and \n from the end - - // allocate the instance of response to heap with shared ptr - // So this memory gets delegated outside Curl Transport as a shared ptr so memory will be - // eventually released - return std::make_unique( - (uint16_t)majorVersion, (uint16_t)minorVersion, HttpStatusCode(statusCode), reasonPhrase); -} - -static void ParseHeader(std::string const& header, std::unique_ptr& response) -{ - // get name and value from header - auto start = header.begin(); - auto end = std::find(start, header.end(), ':'); - - if (end == header.end()) - { - return; // not a valid header or end of headers symbol reached - } - - auto headerName = std::string(start, end); - start = end + 1; // start value - while (start < header.end() && (*start == ' ' || *start == '\t')) - { - ++start; - } - - auto headerValue = std::string(start, header.end() - 2); // remove \r and \n from the end - - response->AddHeader(headerName, headerValue); -} - -// Callback function for curl. This is called for every header that curl get from network -size_t CurlTransport::WriteHeadersCallBack(void* contents, size_t size, size_t nmemb, void* userp) -{ - // No need to check for overflow, Curl already allocated this size internally for contents - size_t const expected_size = size * nmemb; - - // cast client - CurlTransport* client = static_cast(userp); - // convert response to standard string - std::string const& response = std::string((char*)contents, expected_size); - - if (client->m_firstHeader) - { - // first header is expected to be the status code, version and reasonPhrase - client->m_response = ParseAndSetFirstHeader(response); - client->m_firstHeader = false; - return expected_size; - } - - if (client->m_response != nullptr) // only if a response has been created - { - // parse all next headers and add them - ParseHeader(response, client->m_response); - } - - // This callback needs to return the response size or curl will consider it as it failed - return expected_size; -} - -// callback function for libcurl. It would be called as many times as need to ready a body from -// network -size_t CurlTransport::WriteBodyCallBack(void* contents, size_t size, size_t nmemb, void* userp) -{ - // No need to check for overflow, Curl already allocated this size internally for contents - size_t const expected_size = size * nmemb; - - // cast client - CurlTransport* client = static_cast(userp); - - if (client->m_response != nullptr) // only if a response has been created - { - // TODO: check if response is to be written to buffer or to Stream - client->m_response->AppendBody((uint8_t*)contents, expected_size); - } - - // This callback needs to return the response size or curl will consider it as it failed - return expected_size; -} diff --git a/sdk/core/azure-core/src/http/request.cpp b/sdk/core/azure-core/src/http/request.cpp index 7ceb2f079..a9fde7f00 100644 --- a/sdk/core/azure-core/src/http/request.cpp +++ b/sdk/core/azure-core/src/http/request.cpp @@ -8,7 +8,7 @@ using namespace Azure::Core::Http; -void Request::AddPath(std::string const& path) { this->_url += "/" + path; } +void Request::AppendPath(std::string const& path) { this->m_url.AppendPath(path); } void Request::AddQueryParameter(std::string const& name, std::string const& value) { @@ -19,7 +19,7 @@ void Request::AddQueryParameter(std::string const& name, std::string const& valu } else { - this->m_queryParameters.insert(std::pair(name, value)); + this->m_url.AddQueryParameter(name, value); } } @@ -42,17 +42,13 @@ void Request::StartRetry() this->m_retryHeaders.clear(); } -HttpMethod Request::GetMethod() const { return this->_method; } +HttpMethod Request::GetMethod() const { return this->m_method; } -std::string Request::GetEncodedUrl() const +std::string Request::GetQueryString() const { - if (this->m_queryParameters.size() == 0 && this->m_retryQueryParameters.size() == 0) - { - return _url; // no query parameters to add - } - // remove query duplicates - auto queryParameters = Request::MergeMaps(this->m_retryQueryParameters, this->m_queryParameters); + auto queryParameters + = Request::MergeMaps(this->m_retryQueryParameters, this->m_url.GetQueryParameters()); // build url auto queryString = std::string(""); for (auto pair : queryParameters) @@ -60,7 +56,17 @@ std::string Request::GetEncodedUrl() const queryString += (queryString.empty() ? "?" : "&") + pair.first + "=" + pair.second; } - return _url + queryString; + return queryString; +} + +std::string Request::GetEncodedUrl() const +{ + if (this->m_url.GetQueryParameters().size() == 0 && this->m_retryQueryParameters.size() == 0) + { + return m_url.ToString(); // no query parameters to add + } + + return m_url.ToString() + GetQueryString(); } std::map Request::GetHeaders() const @@ -71,3 +77,25 @@ std::map Request::GetHeaders() const } BodyStream* Request::GetBodyStream() { return m_bodyStream; } + +// Writes an HTTP request with RFC2730 without the body (head line and headers) +// https://tools.ietf.org/html/rfc7230#section-3.1.1 +std::string Request::GetHTTPMessagePreBody() const +{ + std::string httpRequest(HttpMethodToString(this->m_method)); + // origin-form. TODO: parse URL to split host from path and use it here instead of empty + // HTTP version harcoded to 1.0 + httpRequest += " " + this->m_url.GetPath() + GetQueryString() + " HTTP/1.1\r\n"; + // headers + for (auto header : this->GetHeaders()) + { + httpRequest += header.first; + httpRequest += ": "; + httpRequest += header.second; + httpRequest += "\r\n"; + } + // end of headers + httpRequest += "\r\n"; + + return httpRequest; +} diff --git a/sdk/core/azure-core/src/http/response.cpp b/sdk/core/azure-core/src/http/response.cpp index 93d00379b..06dc208d9 100644 --- a/sdk/core/azure-core/src/http/response.cpp +++ b/sdk/core/azure-core/src/http/response.cpp @@ -9,49 +9,59 @@ using namespace Azure::Core::Http; -HttpStatusCode Response::GetStatusCode() { return m_statusCode; } +HttpStatusCode Response::GetStatusCode() const { return m_statusCode; } std::string const& Response::GetReasonPhrase() { return m_reasonPhrase; } std::map const& Response::GetHeaders() { return this->m_headers; } -std::vector& Response::GetBodyBuffer() { return m_bodyBuffer; } - -namespace { -inline bool IsStringEqualsIgnoreCase(std::string const& a, std::string const& b) +void Response::AddHeader(std::string const& header) { - auto alen = a.length(); - auto blen = b.length(); + // get name and value from header + auto start = header.begin(); + auto end = std::find(start, header.end(), ':'); - if (alen != blen) + if (end == header.end()) { - return false; + return; // not a valid header or end of headers symbol reached } - for (size_t index = 0; index < alen; index++) + auto headerName = std::string(start, end); + start = end + 1; // start value + while (start < header.end() && (*start == ' ' || *start == '\t')) { - // TODO: tolower is bad for some charsets, see if this can be enhanced - if (std::tolower(a.at(index)) != std::tolower(b.at(index))) - { - return false; - } + ++start; } - return true; + + end = std::find(start, header.end(), '\r'); + auto headerValue = std::string(start, end); // remove \r + + AddHeader(headerName, headerValue); } -} // namespace void Response::AddHeader(std::string const& name, std::string const& value) { - if (IsStringEqualsIgnoreCase("Content-Length", name)) - { - // whenever this header is found, we reserve the value of it to be pre-allocated to write - // response - m_bodyBuffer.reserve(std::stol(value)); - } + // TODO: make sure the Content-Length header is insterted as "Content-Length" no mather the case + // We currently assume we receive it like it and expected to be there from all HTTP + // Responses. this->m_headers.insert(std::pair(name, value)); } -void Response::AppendBody(uint8_t* ptr, uint64_t size) +void Response::SetBodyStream(BodyStream* stream) { this->m_bodyStream = stream; } + +std::unique_ptr> Response::ConstructBodyBufferFromStream( + BodyStream* const stream) { - m_bodyBuffer.insert(m_bodyBuffer.end(), ptr, ptr + size); -} \ No newline at end of file + if (stream == nullptr) + { + return nullptr; + } + + uint8_t const bodySize = (uint8_t)stream->Length(); + std::unique_ptr> unique_buffer(new std::vector(bodySize)); + + auto buffer = unique_buffer.get()->data(); + stream->Read(buffer, bodySize); + + return unique_buffer; +} diff --git a/sdk/core/azure-core/src/http/stream.cpp b/sdk/core/azure-core/src/http/stream.cpp new file mode 100644 index 000000000..593b3492e --- /dev/null +++ b/sdk/core/azure-core/src/http/stream.cpp @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include + +using namespace Azure::Core::Http; + +BodyStream::~BodyStream() {} diff --git a/sdk/core/azure-core/src/http/url.cpp b/sdk/core/azure-core/src/http/url.cpp new file mode 100644 index 000000000..120670ce1 --- /dev/null +++ b/sdk/core/azure-core/src/http/url.cpp @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include + +using namespace Azure::Core::Http; + +URL::URL(std::string const& url) +{ + if (url.size() == 0) + { + return; // nothing to set + } + + // Remove Query Parameters from url + auto noQueryParamsUrl = SaveAndRemoveQueryParameter(url); + + auto endOfUrl = noQueryParamsUrl.end(); + auto start = noQueryParamsUrl.begin(); + // Protocol + auto protocolEnd = std::find(start, endOfUrl, ':'); + if (protocolEnd != endOfUrl) + { + auto protocolDelimiter = std::string(protocolEnd, endOfUrl); + // Check protocol delimiter is there ://, otherwise it can be a port + if (protocolDelimiter.size() >= 3 && protocolDelimiter[1] == '/' && protocolDelimiter[2] == '/') + { + this->m_scheme = std::string(start, protocolEnd); + start = protocolEnd + 3; + } + } + + // Host + auto endOfHost = std::find(start, endOfUrl, '/'); + auto startOfPort = std::find(start, endOfUrl, ':'); + if (startOfPort < endOfHost) + { + this->m_port = std::string(startOfPort + 1, endOfHost); + } + this->m_host = std::string(start, std::min(startOfPort, endOfHost)); + + // finish if there is nothing more ahead + if (endOfHost == endOfUrl) + { + return; + } + + // Advance to path + start = endOfHost + 1; + + // Path + this->m_path = std::string(start, endOfUrl); + if (this->m_path.size() > 0) + { + this->m_path = "/" + this->m_path; + } +} diff --git a/sdk/core/azure-core/test/main.cpp b/sdk/core/azure-core/test/main.cpp index 131e588d9..af40883c4 100644 --- a/sdk/core/azure-core/test/main.cpp +++ b/sdk/core/azure-core/test/main.cpp @@ -94,7 +94,7 @@ TEST(Http_Request, add_path) std::string url = "http://test.com"; Http::Request req(httpMethod, url); - EXPECT_NO_THROW(req.AddPath("path")); + EXPECT_NO_THROW(req.AppendPath("path")); EXPECT_PRED2( [](std::string a, std::string b) { return a == b; }, req.GetEncodedUrl(), url + "/path"); @@ -104,13 +104,13 @@ TEST(Http_Request, add_path) req.GetEncodedUrl(), url + "/path?query=value"); - EXPECT_NO_THROW(req.AddPath("path2")); + EXPECT_NO_THROW(req.AppendPath("path2")); EXPECT_PRED2( [](std::string a, std::string b) { return a == b; }, req.GetEncodedUrl(), url + "/path/path2?query=value"); - EXPECT_NO_THROW(req.AddPath("path3")); + EXPECT_NO_THROW(req.AppendPath("path3")); EXPECT_PRED2( [](std::string a, std::string b) { return a == b; }, req.GetEncodedUrl(), diff --git a/sdk/samples/http_client/curl/CMakeLists.txt b/sdk/samples/http_client/curl/CMakeLists.txt index 92db8d9a4..494363e65 100644 --- a/sdk/samples/http_client/curl/CMakeLists.txt +++ b/sdk/samples/http_client/curl/CMakeLists.txt @@ -5,6 +5,7 @@ if (BUILD_CURL_TRANSPORT) cmake_minimum_required (VERSION 3.12) set(TARGET_NAME "azure_core_with_curl") +set(TARGET_NAME_STREAM "azure_core_with_curl_stream") project(${TARGET_NAME} LANGUAGES CXX) set(CMAKE_CXX_STANDARD 14) @@ -12,9 +13,15 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) add_executable ( ${TARGET_NAME} - src/azure_core_with_curl + src/azure_core_with_curl_bodyBuffer +) + +add_executable ( + ${TARGET_NAME_STREAM} + src/azure_core_with_curl_bodyStream ) target_link_libraries(${TARGET_NAME} PRIVATE azure-core) +target_link_libraries(${TARGET_NAME_STREAM} PRIVATE azure-core) endif() diff --git a/sdk/samples/http_client/curl/src/azure_core_with_curl.cpp b/sdk/samples/http_client/curl/src/azure_core_with_curl.cpp deleted file mode 100644 index 20ea30c7a..000000000 --- a/sdk/samples/http_client/curl/src/azure_core_with_curl.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: MIT - -/** - * @brief Simulates customer application that is linked with azure-core and azure-transport-curl - * - */ - -#include -#include "http/pipeline.hpp" -#include - -#include -#include - -using namespace Azure::Core; -using namespace Azure::Core::Http; -using namespace std; - -int main() -{ - string host("https://httpbin.org/get"); - cout << "testing curl from transport" << endl << "Host: " << host << endl; - - try - { - auto request = Http::Request(Http::HttpMethod::Get, host); - request.AddHeader("one", "header"); - request.AddHeader("other", "header2"); - request.AddHeader("header", "value"); - - //Create the Transport - std::shared_ptr transport = std::make_unique(); - - std::vector> policies; - policies.push_back(std::make_unique()); - - RetryOptions retryOptions; - policies.push_back(std::make_unique(retryOptions)); - - // Add the transport policy - policies.push_back(std::make_unique(std::move(transport))); - - auto httpPipeline = Http::HttpPipeline(policies); - - auto context = Context(); - std::shared_ptr response = httpPipeline.Send(context, request); - - if (response == nullptr) - { - cout << "Error. Response returned as null"; - return 0; - } - - cout << static_cast::type>( - response->GetStatusCode()) - << endl; - cout << response->GetReasonPhrase() << endl; - cout << "headers:" << endl; - for (auto header : response->GetHeaders()) - { - cout << header.first << " : " << header.second << endl; - } - cout << "Body (buffer):" << endl; - auto bodyVector = response->GetBodyBuffer(); - cout << std::string(bodyVector.begin(), bodyVector.end()); - } - catch (Http::CouldNotResolveHostException& e) - { - cout << e.what() << endl; - } - catch (Http::TransportException& e) - { - cout << e.what() << endl; - } - - return 0; -} diff --git a/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyBuffer.cpp b/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyBuffer.cpp new file mode 100644 index 000000000..177088c59 --- /dev/null +++ b/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyBuffer.cpp @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @brief Simulates customer application that is linked with azure-core and azure-transport-curl + * + */ + +#include "http/pipeline.hpp" + +#include +#include +#include +#include + +using namespace Azure::Core; +using namespace Azure::Core::Http; +using namespace std; + +constexpr auto BufferSize = 50; + +std::vector buffer(BufferSize); +Http::Request createGetRequest(); +Http::Request createPutRequest(); +void printRespose(std::unique_ptr response); + +int main() +{ + try + { + // Both requests uses a body buffer to be uploaded that would produce responses with bodyBuffer + auto getRequest = createGetRequest(); + auto putRequest = createPutRequest(); + + // Create the Transport + std::shared_ptr transport = std::make_unique(); + std::vector> policies; + policies.push_back(std::make_unique()); + RetryOptions retryOptions; + policies.push_back(std::make_unique(retryOptions)); + // Add the transport policy + policies.push_back(std::make_unique(std::move(transport))); + auto httpPipeline = Http::HttpPipeline(policies); + + auto context = Context(); + std::unique_ptr getResponse = httpPipeline.Send(context, getRequest); + std::unique_ptr putResponse = httpPipeline.Send(context, putRequest); + + printRespose(std::move(getResponse)); + printRespose(std::move(putResponse)); + } + catch (Http::CouldNotResolveHostException& e) + { + cout << e.what() << endl; + } + catch (Http::TransportException& e) + { + cout << e.what() << endl; + } + + return 0; +} + +Http::Request createGetRequest() +{ + string host("https://httpbin.org/get"); + cout << "Creating a GET request to" << endl << "Host: " << host << endl; + + auto request = Http::Request(Http::HttpMethod::Get, host, new Http::MemoryBodyStream(buffer)); + request.AddHeader("one", "header"); + request.AddHeader("other", "header2"); + request.AddHeader("header", "value"); + + request.AddHeader("Host", "httpbin.org"); + + return request; +} + +Http::Request createPutRequest() +{ + string host("https://httpbin.org/put"); + cout << "Creating a PUT request to" << endl << "Host: " << host << endl; + + std::fill(buffer.begin(), buffer.end(), 'x'); + buffer[0] = '{'; + buffer[1] = '\"'; + buffer[3] = '\"'; + buffer[4] = ':'; + buffer[5] = '\"'; + buffer[BufferSize - 2] = '\"'; + buffer[BufferSize - 1] = '}'; // set buffer to look like a Json `{"x":"xxx...xxx"}` + + auto request + = Http::Request(Http::HttpMethod::Put, host, new Http::MemoryBodyStream(std::move(buffer))); + request.AddHeader("one", "header"); + request.AddHeader("other", "header2"); + request.AddHeader("header", "value"); + + request.AddHeader("Host", "httpbin.org"); + request.AddHeader("Content-Length", std::to_string(BufferSize)); + + return request; +} + +void printRespose(std::unique_ptr response) +{ + if (response == nullptr) + { + cout << "Error. Response returned as null"; + throw; + } + + cout << static_cast::type>( + response->GetStatusCode()) + << endl; + cout << response->GetReasonPhrase() << endl; + cout << "headers:" << endl; + for (auto header : response->GetHeaders()) + { + cout << header.first << " : " << header.second << endl; + } + cout << "Body (buffer):" << endl; + auto bodyVector = *Http::Response::ConstructBodyBufferFromStream(response->GetBodyStream()).get(); + cout << std::string(bodyVector.begin(), bodyVector.end()); + + return; +} diff --git a/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyStream.cpp b/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyStream.cpp new file mode 100644 index 000000000..c04ed0317 --- /dev/null +++ b/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyStream.cpp @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @brief Simulates customer application that is linked with azure-core and azure-transport-curl + * with Stream Body + * + */ + +#include "http/pipeline.hpp" + +#include +#include +#include +#include +#include +#include + +using namespace Azure::Core; +using namespace Azure::Core::Http; +using namespace std; + +// For bodyBuffer +constexpr auto BufferSize = 50; +std::vector buffer(BufferSize); + +// For StreamBody +constexpr auto StreamSize = 200; +std::array bufferStream; + +Http::Request createGetRequest(); +Http::Request createPutRequest(); +Http::Request createPutStreamRequest(); +void printStream(std::unique_ptr response); + +int main() +{ + try + { + // GetRequest. No body, produces stream + auto getRequest = createGetRequest(); + // PutRequest. buffer body, produces stream + auto putRequest = createPutRequest(); + // PutRequest. Stream body, produces stream + auto putStreamRequest = createPutStreamRequest(); + + // Create the Transport + std::shared_ptr transport = std::make_unique(); + + std::vector> policies; + policies.push_back(std::make_unique()); + + RetryOptions retryOptions; + policies.push_back(std::make_unique(retryOptions)); + + // Add the transport policy + policies.push_back(std::make_unique(std::move(transport))); + + auto httpPipeline = Http::HttpPipeline(policies); + + std::unique_ptr response; + auto context = Context(); + + response = httpPipeline.Send(context, getRequest); + printStream(std::move(response)); + + response = httpPipeline.Send(context, putRequest); + printStream(std::move(response)); + + response = httpPipeline.Send(context, putStreamRequest); + printStream(std::move(response)); + } + catch (Http::CouldNotResolveHostException& e) + { + cout << e.what() << endl; + } + catch (Http::TransportException& e) + { + cout << e.what() << endl; + } + + return 0; +} + +// Request GET with no body that produces stream response +Http::Request createGetRequest() +{ + string host("https://httpbin.org/get?arg=1&arg2=2"); + cout << "Creating a GET request to" << endl << "Host: " << host << endl; + + auto request = Http::Request(Http::HttpMethod::Get, host); + request.AddHeader("one", "header"); + request.AddHeader("other", "header2"); + request.AddHeader("header", "value"); + + request.AddHeader("Host", "httpbin.org"); + + request.AddQueryParameter("dinamicArg", "3"); + request.AddQueryParameter("dinamicArg2", "4"); + + return request; +} + +// Put Request with bodyBufferBody that produces stream +Http::Request createPutRequest() +{ + string host("https://httpbin.org/put?a=1"); + cout << "Creating a PUT request to" << endl << "Host: " << host << endl; + + std::fill(buffer.begin(), buffer.end(), 'x'); + buffer[0] = '{'; + buffer[1] = '\"'; + buffer[3] = '\"'; + buffer[4] = ':'; + buffer[5] = '\"'; + buffer[BufferSize - 2] = '\"'; + buffer[BufferSize - 1] = '}'; // set buffer to look like a Json `{"x":"xxx...xxx"}` + + auto request = Http::Request(Http::HttpMethod::Put, host, new Http::MemoryBodyStream(buffer)); + request.AddHeader("one", "header"); + request.AddHeader("other", "header2"); + request.AddHeader("header", "value"); + + request.AddHeader("Host", "httpbin.org"); + request.AddHeader("Content-Length", std::to_string(BufferSize)); + + return request; +} + +// Put Request with stream body that produces stream +Http::Request createPutStreamRequest() +{ + string host("https://httpbin.org/put"); + cout << "Creating a PUT request to" << endl << "Host: " << host << endl; + + bufferStream.fill('1'); + bufferStream[0] = '{'; + bufferStream[1] = '\"'; + bufferStream[3] = '\"'; + bufferStream[4] = ':'; + bufferStream[5] = '\"'; + bufferStream[StreamSize - 2] = '\"'; + bufferStream[StreamSize - 1] = '}'; // set buffer to look like a Json `{"1":"111...111"}` + + auto request = Http::Request( + Http::HttpMethod::Put, host, new MemoryBodyStream(bufferStream.data(), StreamSize)); + request.AddHeader("one", "header"); + request.AddHeader("other", "header2"); + request.AddHeader("header", "value"); + + request.AddHeader("Host", "httpbin.org"); + request.AddHeader("Content-Length", std::to_string(StreamSize)); + + request.AddQueryParameter("dinamicArg", "1"); + request.AddQueryParameter("dinamicArg2", "1"); + request.AddQueryParameter("dinamicArg3", "1"); + + return request; +} + +void printStream(std::unique_ptr response) +{ + if (response == nullptr) + { + cout << "Error. Response returned as null"; + std::cin.ignore(); + return; + } + + cout << static_cast::type>( + response->GetStatusCode()) + << endl; + cout << response->GetReasonPhrase() << endl; + cout << "headers:" << endl; + for (auto header : response->GetHeaders()) + { + cout << header.first << " : " << header.second << endl; + } + cout << "Body (stream):" << endl; + + uint8_t b[100]; + auto bodyStream = response->GetBodyStream(); + uint64_t readCount; + do + { + readCount = bodyStream->Read(b, 10); + cout << std::string(b, b + readCount); + + } while (readCount > 0); + + cout << endl << "Press any key to continue..." << endl; + std::cin.ignore(); + + bodyStream->Close(); + return; +} diff --git a/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp b/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp index 87b00de68..2c06dbf26 100644 --- a/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp +++ b/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp @@ -1036,9 +1036,11 @@ namespace Azure { namespace Storage { namespace Blobs { { throw std::runtime_error("HTTP status code " + std::to_string(http_status_code)); } - XmlReader reader( - reinterpret_cast(http_response.GetBodyBuffer().data()), - http_response.GetBodyBuffer().size()); + + auto stream = http_response.GetBodyStream(); + auto unique_buffer = Core::Http::Response::ConstructBodyBufferFromStream(stream); + + XmlReader reader(reinterpret_cast(unique_buffer.get()), (size_t)stream->Length()); response = ListContainersSegmentFromXml(reader); response.Version = http_response.GetHeaders().at("x-ms-version"); response.Date = http_response.GetHeaders().at("Date"); @@ -1076,11 +1078,14 @@ namespace Azure { namespace Storage { namespace Blobs { XmlWriter writer; GetUserDelegationKeyOptionsToXml(writer, options); std::string xml_body = writer.GetDocument(); - std::vector body_buffer(xml_body.begin(), xml_body.end()); - uint64_t body_buffer_length = body_buffer.size(); + auto bodyBuffer = std::vector(xml_body.begin(), xml_body.end()); + auto request = Azure::Core::Http::Request( - Azure::Core::Http::HttpMethod::Post, url, std::move(body_buffer)); - request.AddHeader("Content-Length", std::to_string(body_buffer_length)); + Azure::Core::Http::HttpMethod::Post, + url, + new Azure::Core::Http::MemoryBodyStream(std::move(bodyBuffer))); + + request.AddHeader("Content-Length", std::to_string(bodyBuffer.size())); request.AddQueryParameter("restype", "service"); request.AddQueryParameter("comp", "userdelegationkey"); request.AddHeader("x-ms-version", "2019-07-07"); @@ -1099,9 +1104,11 @@ namespace Azure { namespace Storage { namespace Blobs { { throw std::runtime_error("HTTP status code " + std::to_string(http_status_code)); } - XmlReader reader( - reinterpret_cast(http_response.GetBodyBuffer().data()), - http_response.GetBodyBuffer().size()); + + auto stream = http_response.GetBodyStream(); + auto unique_buffer = Core::Http::Response::ConstructBodyBufferFromStream(stream); + + XmlReader reader(reinterpret_cast(unique_buffer.get()), (size_t)stream->Length()); response = UserDelegationKeyFromXml(reader); response.Version = http_response.GetHeaders().at("x-ms-version"); response.Date = http_response.GetHeaders().at("Date"); @@ -1896,9 +1903,11 @@ namespace Azure { namespace Storage { namespace Blobs { { throw std::runtime_error("HTTP status code " + std::to_string(http_status_code)); } - XmlReader reader( - reinterpret_cast(http_response.GetBodyBuffer().data()), - http_response.GetBodyBuffer().size()); + + auto stream = http_response.GetBodyStream(); + auto unique_buffer = Core::Http::Response::ConstructBodyBufferFromStream(stream); + + XmlReader reader(reinterpret_cast(unique_buffer.get()), (size_t)stream->Length()); response = BlobsFlatSegmentFromXml(reader); response.Version = http_response.GetHeaders().at("x-ms-version"); response.Date = http_response.GetHeaders().at("Date"); @@ -2553,7 +2562,11 @@ namespace Azure { namespace Storage { namespace Blobs { response.CommittedBlockCount = std::stoull(response_committedblockcount_iterator->second); } response.BlobType = BlobTypeFromString(http_response.GetHeaders().at("x-ms-blob-type")); - response.BodyBuffer = http_response.GetBodyBuffer(); + + auto stream = http_response.GetBodyStream(); + + response.BodyBuffer + = std::move(*Core::Http::Response::ConstructBodyBufferFromStream(stream).get()); return response; } @@ -3441,7 +3454,9 @@ namespace Azure { namespace Storage { namespace Blobs { const UploadOptions& options) { auto request = Azure::Core::Http::Request( - Azure::Core::Http::HttpMethod::Put, url, *options.BodyBuffer); + Azure::Core::Http::HttpMethod::Put, + url, + new Azure::Core::Http::MemoryBodyStream(*options.BodyBuffer)); request.AddHeader("Content-Length", std::to_string(options.BodyBuffer->size())); request.AddHeader("x-ms-version", "2019-07-07"); if (!options.EncryptionKey.empty()) @@ -3596,7 +3611,9 @@ namespace Azure { namespace Storage { namespace Blobs { const StageBlockOptions& options) { auto request = Azure::Core::Http::Request( - Azure::Core::Http::HttpMethod::Put, url, *options.BodyBuffer); + Azure::Core::Http::HttpMethod::Put, + url, + new Azure::Core::Http::MemoryBodyStream(*options.BodyBuffer)); request.AddHeader("Content-Length", std::to_string(options.BodyBuffer->size())); request.AddQueryParameter("comp", "block"); request.AddQueryParameter("blockid", options.BlockId); @@ -3853,7 +3870,9 @@ namespace Azure { namespace Storage { namespace Blobs { std::vector body_buffer(xml_body.begin(), xml_body.end()); uint64_t body_buffer_length = body_buffer.size(); auto request = Azure::Core::Http::Request( - Azure::Core::Http::HttpMethod::Put, url, std::move(body_buffer)); + Azure::Core::Http::HttpMethod::Put, + url, + new Azure::Core::Http::MemoryBodyStream(std::move(body_buffer))); request.AddHeader("Content-Length", std::to_string(body_buffer_length)); request.AddQueryParameter("comp", "blocklist"); request.AddHeader("x-ms-version", "2019-07-07"); @@ -4024,9 +4043,11 @@ namespace Azure { namespace Storage { namespace Blobs { { throw std::runtime_error("HTTP status code " + std::to_string(http_status_code)); } + auto stream = http_response.GetBodyStream(); + auto unique_buffer = Core::Http::Response::ConstructBodyBufferFromStream(stream); + XmlReader reader( - reinterpret_cast(http_response.GetBodyBuffer().data()), - http_response.GetBodyBuffer().size()); + reinterpret_cast(unique_buffer.get()), (size_t)stream->Length()); response = BlobBlockListInfoFromXml(reader); response.Version = http_response.GetHeaders().at("x-ms-version"); response.Date = http_response.GetHeaders().at("Date"); @@ -4380,7 +4401,9 @@ namespace Azure { namespace Storage { namespace Blobs { const UploadPagesOptions& options) { auto request = Azure::Core::Http::Request( - Azure::Core::Http::HttpMethod::Put, url, *options.BodyBuffer); + Azure::Core::Http::HttpMethod::Put, + url, + new Azure::Core::Http::MemoryBodyStream(*options.BodyBuffer)); request.AddHeader("Content-Length", std::to_string(options.BodyBuffer->size())); request.AddQueryParameter("comp", "page"); request.AddHeader("x-ms-version", "2019-07-07"); @@ -4967,9 +4990,11 @@ namespace Azure { namespace Storage { namespace Blobs { { throw std::runtime_error("HTTP status code " + std::to_string(http_status_code)); } + auto stream = http_response.GetBodyStream(); + auto unique_buffer = Core::Http::Response::ConstructBodyBufferFromStream(stream); + XmlReader reader( - reinterpret_cast(http_response.GetBodyBuffer().data()), - http_response.GetBodyBuffer().size()); + reinterpret_cast(unique_buffer.get()), (size_t)stream->Length()); response = PageRangesInfoFromXml(reader); response.Version = http_response.GetHeaders().at("x-ms-version"); response.Date = http_response.GetHeaders().at("Date"); @@ -5409,7 +5434,9 @@ namespace Azure { namespace Storage { namespace Blobs { const AppendBlockOptions& options) { auto request = Azure::Core::Http::Request( - Azure::Core::Http::HttpMethod::Put, url, *options.BodyBuffer); + Azure::Core::Http::HttpMethod::Put, + url, + new Azure::Core::Http::MemoryBodyStream(*options.BodyBuffer)); request.AddHeader("Content-Length", std::to_string(options.BodyBuffer->size())); request.AddQueryParameter("comp", "appendblock"); request.AddHeader("x-ms-version", "2019-07-07");