From 517b8e6245b7a08757bc31032e41512fcc8258e3 Mon Sep 17 00:00:00 2001 From: Kan Tang Date: Mon, 1 Feb 2021 14:05:59 +0800 Subject: [PATCH] PutRangeFromUrl support in file service. (#1530) * PutRangeFromUrl support * Update share_file_client.cpp --- .../azure-storage-files-shares/CHANGELOG.md | 1 + .../files/shares/share_file_client.hpp | 15 +++ .../storage/files/shares/share_options.hpp | 7 +- .../storage/files/shares/share_responses.hpp | 2 +- .../src/share_file_client.cpp | 56 ++++++++++ .../test/share_file_client_test.cpp | 100 ++++++++++++++++++ 6 files changed, 174 insertions(+), 7 deletions(-) diff --git a/sdk/storage/azure-storage-files-shares/CHANGELOG.md b/sdk/storage/azure-storage-files-shares/CHANGELOG.md index 210be1e91..d54a21a92 100644 --- a/sdk/storage/azure-storage-files-shares/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-shares/CHANGELOG.md @@ -4,6 +4,7 @@ ### New Features +- Added support for `UploadRangeFromUri` in file client. - Added support for `SetProperties` in share client. This API supports update share tier and adjusting share's quota. - Added support to get share's tier status in `ListSharesSinglePage` and `GetProperties`. diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_file_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_file_client.hpp index 6ac6854fa..cbf6a7abd 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_file_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_file_client.hpp @@ -297,6 +297,21 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ForceCloseAllShareFileHandlesOptions& options = ForceCloseAllShareFileHandlesOptions()) const; + /** + * @brief Upload a range from the source URI to this file's specific range. + * @param sourceUri The source URI of the content to be uploaded. + * @param sourceRange The source URI's range to be uploaded to file. + * @param range The range of the file this source to be uploaded from. + * @param options Optional parameters to upload a range to file. + * @return Azure::Core::Response containing the returned + * information. + */ + Azure::Core::Response UploadRangeFromUri( + const std::string& sourceUri, + const Azure::Core::Http::Range& sourceRange, + const Azure::Core::Http::Range& range, + const UploadFileRangeFromUriOptions& options = UploadFileRangeFromUriOptions()) const; + private: Azure::Core::Http::Url m_shareFileUrl; std::shared_ptr m_pipeline; diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp index c5ed18344..68afed44f 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp @@ -652,18 +652,13 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { LeaseAccessConditions AccessConditions; }; - struct UploadFileRangeFromUrlOptions + struct UploadFileRangeFromUriOptions { /** * @brief Context for cancelling long running operations. */ Azure::Core::Context Context; - /** - * @brief The range of the source file. - */ - Azure::Core::Nullable SourceRange; - /** * @brief Specify the crc64 calculated for the range of bytes that must be read from the copy * source. diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_responses.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_responses.hpp index 0ec2acab0..736c6e3c7 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_responses.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_responses.hpp @@ -154,7 +154,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { names using SetShareFileMetadataResult = Details::FileSetMetadataResult; using UploadShareFileRangeResult = Details::FileUploadRangeResult; using ClearShareFileRangeResult = Details::FileUploadRangeResult; - using UploadFileRangeFromUrlResult = Details::FileUploadRangeFromUrlResult; + using UploadFileRangeFromUriResult = Details::FileUploadRangeFromUrlResult; using GetShareFileRangeListResult = Details::FileGetRangeListResult; using ListShareFileHandlesSinglePageResult = ListShareDirectoryHandlesSinglePageResult; using ForceCloseAllShareFileHandlesResult = Details::FileForceCloseHandlesResult; diff --git a/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp b/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp index ee516607f..b222d58e8 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp @@ -1009,4 +1009,60 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { return Azure::Core::Response( std::move(result), createResult.ExtractRawResponse()); } + + Azure::Core::Response ShareFileClient::UploadRangeFromUri( + const std::string& sourceUri, + const Azure::Core::Http::Range& sourceRange, + const Azure::Core::Http::Range& range, + const UploadFileRangeFromUriOptions& options) const + { + if (!sourceRange.Length.HasValue()) + { + // sourceRange must have length to perform this operation. + std::abort(); + } + int64_t destLength = sourceRange.Length.GetValue(); + if (range.Length.HasValue()) + { + // Let server decide the behavior if the length does not match. + destLength = range.Length.GetValue(); + } + + auto protocolLayerOptions = Details::ShareRestClient::File::UploadRangeFromUrlOptions(); + protocolLayerOptions.TargetRange = std::string("bytes=") + std::to_string(range.Offset) + + std::string("-") + std::to_string(range.Offset + destLength - 1); + protocolLayerOptions.ContentLength = 0; + protocolLayerOptions.CopySource = sourceUri; + protocolLayerOptions.LeaseIdOptional = options.AccessConditions.LeaseId; + if (options.SourceContentHash.HasValue() + && options.SourceContentHash.GetValue().Algorithm == HashAlgorithm::Md5) + { + // SourceContentHash now only supports Crc64 hash algorithm. + std::abort(); + } + protocolLayerOptions.SourceContentCrc64 = options.SourceContentHash; + if (options.SourceAccessCondition.IfMatchContentHash.HasValue() + && options.SourceAccessCondition.IfMatchContentHash.GetValue().Algorithm + == HashAlgorithm::Md5) + { + // IfMatchContentHash now only supports Crc64 hash algorithm. + std::abort(); + } + protocolLayerOptions.SourceIfMatchCrc64 = options.SourceAccessCondition.IfMatchContentHash; + if (options.SourceAccessCondition.IfNoneMatchContentHash.HasValue() + && options.SourceAccessCondition.IfNoneMatchContentHash.GetValue().Algorithm + == HashAlgorithm::Md5) + { + // IfNoneMatchContentHash now only supports Crc64 hash algorithm. + std::abort(); + } + protocolLayerOptions.SourceIfNoneMatchCrc64 + = options.SourceAccessCondition.IfNoneMatchContentHash; + protocolLayerOptions.SourceRange = std::string("bytes=") + std::to_string(sourceRange.Offset) + + std::string("-") + std::to_string(sourceRange.Offset + sourceRange.Length.GetValue() - 1); + protocolLayerOptions.XMsWrite = Models::FileRangeWriteFromUrlType::Update; + + return Details::ShareRestClient::File::UploadRangeFromUrl( + m_shareFileUrl, *m_pipeline, options.Context, protocolLayerOptions); + } }}}} // namespace Azure::Storage::Files::Shares diff --git a/sdk/storage/azure-storage-files-shares/test/share_file_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/share_file_client_test.cpp index 129494063..e5ce65191 100644 --- a/sdk/storage/azure-storage-files-shares/test/share_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/share_file_client_test.cpp @@ -798,4 +798,104 @@ namespace Azure { namespace Storage { namespace Test { FAIL(); } + TEST_F(FileShareFileClientTest, UploadRangeFromUri) + { + size_t fileSize = 1 * 1024 * 1024; + std::string fileName = RandomString(); + auto fileContent = RandomBuffer(fileSize); + auto memBodyStream = Core::Http::MemoryBodyStream(fileContent); + auto sourceFileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(fileName); + sourceFileClient.Create(fileSize); + EXPECT_NO_THROW(sourceFileClient.UploadRange(0, &memBodyStream)); + + auto destFileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString(10)); + destFileClient.Create(fileSize * 4); + Azure::Core::Http::Range sourceRange; + Azure::Core::Http::Range destRange; + sourceRange.Length = fileSize; + destRange.Offset = fileSize; + destRange.Length = fileSize; + + // Get the SAS of the file + Sas::ShareSasBuilder fileSasBuilder; + fileSasBuilder.Protocol = Sas::SasProtocol::HttpsAndHttp; + fileSasBuilder.StartsOn = std::chrono::system_clock::now() - std::chrono::minutes(5); + fileSasBuilder.ExpiresOn = std::chrono::system_clock::now() + std::chrono::minutes(60); + fileSasBuilder.ShareName = m_shareName; + fileSasBuilder.FilePath = fileName; + fileSasBuilder.Resource = Sas::ShareSasResource::File; + fileSasBuilder.SetPermissions(Sas::ShareSasPermissions::Read); + std::string sourceSas = fileSasBuilder.GenerateSasToken( + *Details::ParseConnectionString(StandardStorageConnectionString()).KeyCredential); + + Files::Shares::Models::UploadFileRangeFromUriResult uploadResult; + EXPECT_NO_THROW( + uploadResult = *destFileClient.UploadRangeFromUri( + sourceFileClient.GetUrl() + sourceSas, sourceRange, destRange)); + + Files::Shares::Models::DownloadShareFileResult result; + Files::Shares::DownloadShareFileOptions downloadOptions; + downloadOptions.Range = destRange; + EXPECT_NO_THROW(result = destFileClient.Download(downloadOptions).ExtractValue()); + auto resultBuffer = Core::Http::BodyStream::ReadToEnd(Core::Context(), *(result.BodyStream)); + EXPECT_EQ(fileContent, resultBuffer); + Files::Shares::Models::GetShareFileRangeListResult getRangeResult; + EXPECT_NO_THROW(getRangeResult = destFileClient.GetRangeList().ExtractValue()); + EXPECT_EQ(1U, getRangeResult.Ranges.size()); + EXPECT_EQ(static_cast(fileSize), getRangeResult.Ranges[0].Offset); + EXPECT_TRUE(getRangeResult.Ranges[0].Length.HasValue()); + EXPECT_EQ(static_cast(fileSize), getRangeResult.Ranges[0].Length.GetValue()); + + // source access condition works. + std::vector invalidCrc64( + uploadResult.TransactionalContentHash.Value.begin(), + uploadResult.TransactionalContentHash.Value.begin() + + uploadResult.TransactionalContentHash.Value.size() / 2); + { + Files::Shares::UploadFileRangeFromUriOptions uploadRangeOptions; + uploadRangeOptions.SourceAccessCondition.IfNoneMatchContentHash + = uploadResult.TransactionalContentHash; + EXPECT_THROW( + uploadResult = *destFileClient.UploadRangeFromUri( + sourceFileClient.GetUrl() + sourceSas, sourceRange, destRange, uploadRangeOptions), + StorageException); + // Below code seems to be triggering a server bug. Uncomment when server resolves the issue. + // uploadRangeOptions.SourceAccessCondition.IfNoneMatchContentHash.GetValue().Value + // = invalidCrc64; + // EXPECT_NO_THROW( + // uploadResult = *destFileClient.UploadRangeFromUri( + // sourceFileClient.GetUrl() + sourceSas, sourceRange, destRange, uploadRangeOptions)); + } + { + Files::Shares::UploadFileRangeFromUriOptions uploadRangeOptions; + uploadRangeOptions.SourceAccessCondition.IfMatchContentHash + = uploadResult.TransactionalContentHash; + EXPECT_NO_THROW( + uploadResult = *destFileClient.UploadRangeFromUri( + sourceFileClient.GetUrl() + sourceSas, sourceRange, destRange, uploadRangeOptions)); + // Below code seems to be triggering a server high latency. Uncomment when server resolves the + // issue. + // uploadRangeOptions.SourceAccessCondition.IfMatchContentHash.GetValue().Value = + // invalidCrc64; + // EXPECT_THROW( + // uploadResult = *destFileClient.UploadRangeFromUri( + // sourceFileClient.GetUrl() + sourceSas, sourceRange, destRange, uploadRangeOptions), + // StorageException); + } + { + Files::Shares::UploadFileRangeFromUriOptions uploadRangeOptions; + uploadRangeOptions.SourceContentHash = uploadResult.TransactionalContentHash; + EXPECT_NO_THROW( + uploadResult = *destFileClient.UploadRangeFromUri( + sourceFileClient.GetUrl() + sourceSas, sourceRange, destRange, uploadRangeOptions)); + // Below code seems to be triggering a server high latency. Uncomment when server resolves the + // issue. + // uploadRangeOptions.SourceContentHash.GetValue().Value = invalidCrc64; + // EXPECT_THROW( + // uploadResult = *destFileClient.UploadRangeFromUri( + // sourceFileClient.GetUrl() + sourceSas, sourceRange, destRange, uploadRangeOptions), + // StorageException); + } + } + }}} // namespace Azure::Storage::Test