From 53bd00034b5db539aeabdff06a0b079d3d318661 Mon Sep 17 00:00:00 2001 From: Kan Tang Date: Wed, 26 Aug 2020 19:56:15 -0700 Subject: [PATCH] Added samples and more test cases to file service and resolved some issues. (#539) --- .../inc/shares/protocol/share_rest_client.hpp | 16 ++- sdk/storage/inc/shares/share_constants.hpp | 1 + sdk/storage/inc/shares/share_file_client.hpp | 8 +- sdk/storage/inc/shares/share_options.hpp | 8 +- sdk/storage/inc/shares/share_responses.hpp | 8 +- .../sample/file_share_getting_started.cpp | 39 ++++- sdk/storage/src/shares/share_file_client.cpp | 65 ++++++--- .../test/shares/share_file_client_test.cpp | 133 +++++++++++++++++- 8 files changed, 234 insertions(+), 44 deletions(-) diff --git a/sdk/storage/inc/shares/protocol/share_rest_client.hpp b/sdk/storage/inc/shares/protocol/share_rest_client.hpp index 1c692db62..e817d00f2 100644 --- a/sdk/storage/inc/shares/protocol/share_rest_client.hpp +++ b/sdk/storage/inc/shares/protocol/share_rest_client.hpp @@ -4,14 +4,14 @@ #pragma once +#include "azure/core/http/http.hpp" +#include "azure/core/http/pipeline.hpp" +#include "azure/core/nullable.hpp" +#include "azure/core/response.hpp" #include "common/storage_common.hpp" #include "common/storage_error.hpp" #include "common/xml_wrapper.hpp" -#include "azure/core/http/http.hpp" -#include "azure/core/http/pipeline.hpp" #include "json.hpp" -#include "azure/core/nullable.hpp" -#include "azure/core/response.hpp" #include #include @@ -6148,8 +6148,12 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { result.ContentMd5 = response.GetHeaders().at(Details::c_HeaderContentMd5); } - result.IsServerEncrypted - = response.GetHeaders().at(Details::c_HeaderRequestIsServerEncrypted) == "true"; + if (response.GetHeaders().find(Details::c_HeaderRequestIsServerEncrypted) + != response.GetHeaders().end()) + { + result.IsServerEncrypted + = response.GetHeaders().at(Details::c_HeaderRequestIsServerEncrypted) == "true"; + } return Azure::Core::Response( std::move(result), std::move(responsePtr)); } diff --git a/sdk/storage/inc/shares/share_constants.hpp b/sdk/storage/inc/shares/share_constants.hpp index 6de450d93..182623883 100644 --- a/sdk/storage/inc/shares/share_constants.hpp +++ b/sdk/storage/inc/shares/share_constants.hpp @@ -7,6 +7,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { constexpr static const char* c_FileDefaultTimeValue = "now"; + constexpr static const char* c_FileCopySourceTime = "source"; constexpr static const char* c_FileInheritPermission = "inherit"; constexpr static const char* c_FilePreserveSmbProperties = "preserve"; diff --git a/sdk/storage/inc/shares/share_file_client.hpp b/sdk/storage/inc/shares/share_file_client.hpp index d6f042292..e39d53fe6 100644 --- a/sdk/storage/inc/shares/share_file_client.hpp +++ b/sdk/storage/inc/shares/share_file_client.hpp @@ -3,12 +3,12 @@ #pragma once -#include "common/storage_credential.hpp" -#include "common/storage_uri_builder.hpp" #include "azure/core/credentials/credentials.hpp" #include "azure/core/http/pipeline.hpp" -#include "protocol/share_rest_client.hpp" #include "azure/core/response.hpp" +#include "common/storage_credential.hpp" +#include "common/storage_uri_builder.hpp" +#include "protocol/share_rest_client.hpp" #include "share_client.hpp" #include "share_directory_client.hpp" #include "share_options.hpp" @@ -262,11 +262,13 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { /** * @brief Clears some range of data within the file. * @param offset Specifies the starting offset for the content to be cleared within the file. + * @param length Specifies the length for the content to be cleared within the file. * @return Azure::Core::Response containing the information of the cleared * range returned from the server. */ Azure::Core::Response ClearRange( int64_t offset, + int64_t length, const ClearFileRangeOptions& options = ClearFileRangeOptions()) const; /** diff --git a/sdk/storage/inc/shares/share_options.hpp b/sdk/storage/inc/shares/share_options.hpp index 17bab27dc..0719bd34f 100644 --- a/sdk/storage/inc/shares/share_options.hpp +++ b/sdk/storage/inc/shares/share_options.hpp @@ -3,8 +3,8 @@ #pragma once -#include "common/access_conditions.hpp" #include "azure/core/nullable.hpp" +#include "common/access_conditions.hpp" #include "protocol/share_rest_client.hpp" #include "share_responses.hpp" @@ -579,12 +579,6 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { */ Azure::Core::Context Context; - /** - * @brief Specifies the length to be cleared, if omitted, all content after the offset will be - * cleared. - */ - Azure::Core::Nullable Length; - /** * @brief The operation will only succeed if the access condition is met. */ diff --git a/sdk/storage/inc/shares/share_responses.hpp b/sdk/storage/inc/shares/share_responses.hpp index 5c611fc76..a15e5f02e 100644 --- a/sdk/storage/inc/shares/share_responses.hpp +++ b/sdk/storage/inc/shares/share_responses.hpp @@ -71,15 +71,15 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { FileAttributes Attributes = static_cast(0); /** - * @brief Creation time for the file/directory. Default value: Now. + * @brief Creation time for the file/directory.. */ - Azure::Core::Nullable FileCreationTime = std::string(c_FileDefaultTimeValue); + Azure::Core::Nullable FileCreationTime; /** - * @brief Last write time for the file/directory. Default value: Now. + * @brief Last write time for the file/directory.. */ - Azure::Core::Nullable FileLastWriteTime = std::string(c_FileDefaultTimeValue); + Azure::Core::Nullable FileLastWriteTime; }; // FileClient models: diff --git a/sdk/storage/sample/file_share_getting_started.cpp b/sdk/storage/sample/file_share_getting_started.cpp index 3f51b9298..8e97eb555 100644 --- a/sdk/storage/sample/file_share_getting_started.cpp +++ b/sdk/storage/sample/file_share_getting_started.cpp @@ -2,7 +2,44 @@ // SPDX-License-Identifier: MIT #include "samples_common.hpp" +#include "shares/shares.hpp" #include SAMPLE(FileShareGettingStarted, FileShareGettingStarted) -void FileShareGettingStarted() {} +void FileShareGettingStarted() +{ + using namespace Azure::Storage::Files::Shares; + + std::string shareName = "sample-share"; + std::string fileName = "sample-file"; + std::string fileContent = "Hello Azure!"; + + auto shareClient = ShareClient::CreateFromConnectionString(GetConnectionString(), shareName); + try + { + shareClient.Create(); + } + catch (std::runtime_error& e) + { + // The share may already exist + std::cout << e.what() << std::endl; + } + + FileClient fileClient = shareClient.GetFileClient(fileName); + + fileClient.UploadFrom(reinterpret_cast(fileContent.data()), fileContent.size()); + + std::map fileMetadata = {{"key1", "value1"}, {"key2", "value2"}}; + fileClient.SetMetadata(fileMetadata); + + auto properties = *fileClient.GetProperties(); + for (auto metadata : properties.Metadata) + { + std::cout << metadata.first << ":" << metadata.second << std::endl; + } + fileContent.resize(static_cast(properties.ContentLength)); + + fileClient.DownloadTo(reinterpret_cast(&fileContent[0]), fileContent.size()); + + std::cout << fileContent << std::endl; +} diff --git a/sdk/storage/src/shares/share_file_client.cpp b/sdk/storage/src/shares/share_file_client.cpp index 4989cc491..b2996ae24 100644 --- a/sdk/storage/src/shares/share_file_client.cpp +++ b/sdk/storage/src/shares/share_file_client.cpp @@ -3,6 +3,8 @@ #include "shares/share_file_client.hpp" +#include "azure/core/credentials/policy/policies.hpp" +#include "azure/core/http/curl/curl.hpp" #include "common/concurrent_transfer.hpp" #include "common/constants.hpp" #include "common/crypt.hpp" @@ -12,8 +14,6 @@ #include "common/storage_common.hpp" #include "common/storage_per_retry_policy.hpp" #include "common/storage_version.hpp" -#include "azure/core/credentials/policy/policies.hpp" -#include "azure/core/http/curl/curl.hpp" #include "shares/share_constants.hpp" namespace Azure { namespace Storage { namespace Files { namespace Shares { @@ -275,21 +275,49 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { protocolLayerOptions.CopySource = std::move(copySource); protocolLayerOptions.FileCopyFileAttributes = FileAttributesToString(options.SmbProperties.Attributes); - protocolLayerOptions.FileCopyFileCreationTime = options.SmbProperties.FileCreationTime; - protocolLayerOptions.FileCopyFileLastWriteTime = options.SmbProperties.FileLastWriteTime; - if (options.FilePermission.HasValue()) + if (options.SmbProperties.FileCreationTime.HasValue()) { - protocolLayerOptions.FilePermission = options.FilePermission; - } - else if (options.SmbProperties.FilePermissionKey.HasValue()) - { - protocolLayerOptions.FilePermissionKey = options.SmbProperties.FilePermissionKey; + protocolLayerOptions.FileCopyFileCreationTime + = options.SmbProperties.FileCreationTime.GetValue(); } else { - protocolLayerOptions.FilePermission = std::string(c_FileInheritPermission); + protocolLayerOptions.FileCopyFileCreationTime = std::string(c_FileCopySourceTime); + } + if (options.SmbProperties.FileLastWriteTime.HasValue()) + { + protocolLayerOptions.FileCopyFileLastWriteTime + = options.SmbProperties.FileLastWriteTime.GetValue(); + } + else + { + protocolLayerOptions.FileCopyFileLastWriteTime = std::string(c_FileCopySourceTime); + } + if (options.FilePermissionCopyMode.HasValue()) + { + protocolLayerOptions.XMsFilePermissionCopyMode = options.FilePermissionCopyMode.GetValue(); + if (options.FilePermissionCopyMode.GetValue() == PermissionCopyModeType::Override) + { + if (options.FilePermission.HasValue()) + { + protocolLayerOptions.FilePermission = options.FilePermission; + } + else if (options.SmbProperties.FilePermissionKey.HasValue()) + { + protocolLayerOptions.FilePermissionKey = options.SmbProperties.FilePermissionKey; + } + else + { + throw std::runtime_error( + "FilePermission or FilePermissionKey must be set if FilePermissionCopyMode is set to " + "PermissionCopyModeType::Override."); + } + } + } + else + { + protocolLayerOptions.XMsFilePermissionCopyMode = PermissionCopyModeType::Source; } - protocolLayerOptions.XMsFilePermissionCopyMode = options.FilePermissionCopyMode; protocolLayerOptions.FileCopyIgnoreReadOnly = options.IgnoreReadOnly; protocolLayerOptions.FileCopySetArchiveAttribute = options.SetArchiveAttribute; protocolLayerOptions.LeaseIdOptional = options.AccessConditions.LeaseId; @@ -443,21 +471,14 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { Azure::Core::Response FileClient::ClearRange( int64_t offset, + int64_t length, const ClearFileRangeOptions& options) const { auto protocolLayerOptions = ShareRestClient::File::UploadRangeOptions(); protocolLayerOptions.XMsWrite = FileRangeWriteType::Clear; protocolLayerOptions.ContentLength = 0; - if (options.Length.HasValue()) - { - protocolLayerOptions.XMsRange = std::string("bytes=") + std::to_string(offset) - + std::string("-") + std::to_string(offset + options.Length.GetValue() - 1); - } - else - { - protocolLayerOptions.XMsRange - = std::string("bytes=") + std::to_string(offset) + std::string("-"); - } + protocolLayerOptions.XMsRange = std::string("bytes=") + std::to_string(offset) + + std::string("-") + std::to_string(offset + length - 1); protocolLayerOptions.LeaseIdOptional = options.AccessConditions.LeaseId; return ShareRestClient::File::UploadRange( diff --git a/sdk/storage/test/shares/share_file_client_test.cpp b/sdk/storage/test/shares/share_file_client_test.cpp index fd5ed39ed..756f56022 100644 --- a/sdk/storage/test/shares/share_file_client_test.cpp +++ b/sdk/storage/test/shares/share_file_client_test.cpp @@ -3,6 +3,7 @@ #include "share_file_client_test.hpp" +#include "common/crypt.hpp" #include "common/file_io.hpp" #include "common/storage_common.hpp" @@ -157,7 +158,7 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(FileShareFileClientTest, DirectorySmbProperties) + TEST_F(FileShareFileClientTest, FileSmbProperties) { Files::Shares::FileShareSmbProperties properties; properties.Attributes @@ -518,4 +519,134 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(FileShareFileClientTest, RangeUploadDownload) + { + auto rangeSize = 1 * 1024 * 1024; + auto numOfChunks = 3; + auto rangeContent = RandomBuffer(rangeSize); + auto memBodyStream = Core::Http::MemoryBodyStream(rangeContent); + { + // Simple upload/download. + auto fileClient = m_shareClient->GetFileClient(LowercaseRandomString(10)); + fileClient.Create(static_cast(numOfChunks) * rangeSize); + for (int32_t i = 0; i < numOfChunks; ++i) + { + memBodyStream.Rewind(); + EXPECT_NO_THROW( + fileClient.UploadRange(&memBodyStream, static_cast(rangeSize) * i)); + } + + for (int32_t i = 0; i < numOfChunks; ++i) + { + std::vector resultBuffer; + Files::Shares::DownloadFileOptions downloadOptions; + downloadOptions.Offset = static_cast(rangeSize) * i; + downloadOptions.Length = rangeSize; + EXPECT_NO_THROW( + resultBuffer = Core::Http::BodyStream::ReadToEnd( + Core::Context(), *fileClient.Download(downloadOptions)->BodyStream)); + EXPECT_EQ(rangeContent, resultBuffer); + } + } + + { + // MD5 works. + memBodyStream.Rewind(); + auto md5String = Base64Encode(Md5::Hash(rangeContent.data(), rangeContent.size())); + auto invalidMd5String = Base64Encode(Md5::Hash(std::string("This is garbage."))); + auto fileClient = m_shareClient->GetFileClient(LowercaseRandomString(10)); + Files::Shares::UploadFileRangeOptions uploadOptions; + fileClient.Create(static_cast(numOfChunks) * rangeSize); + uploadOptions.ContentMd5 = md5String; + EXPECT_NO_THROW(fileClient.UploadRange(&memBodyStream, 0, uploadOptions)); + uploadOptions.ContentMd5 = invalidMd5String; + memBodyStream.Rewind(); + EXPECT_THROW(fileClient.UploadRange(&memBodyStream, 0, uploadOptions), StorageError); + } + } + + TEST_F(FileShareFileClientTest, CopyRelated) + { + size_t fileSize = 1 * 1024 * 1024; + auto fileContent = RandomBuffer(fileSize); + auto memBodyStream = Core::Http::MemoryBodyStream(fileContent); + { + // Simple copy works. + auto fileClient = m_shareClient->GetFileClient(LowercaseRandomString(10)); + fileClient.Create(fileSize); + + auto destFileClient = m_shareClient->GetFileClient(LowercaseRandomString(10)); + Files::Shares::StartCopyFileResult result; + EXPECT_NO_THROW(result = destFileClient.StartCopy(fileClient.GetUri()).ExtractValue()); + EXPECT_EQ(Files::Shares::CopyStatusType::Success, result.CopyStatus); + EXPECT_FALSE(result.CopyId.empty()); + } + + { + // Copy mode with override and empty permission throws error.. + auto fileClient = m_shareClient->GetFileClient(LowercaseRandomString(10)); + fileClient.Create(fileSize); + + auto destFileClient = m_shareClient->GetFileClient(LowercaseRandomString(10)); + Files::Shares::StartCopyFileOptions copyOptions; + copyOptions.FilePermissionCopyMode = Files::Shares::PermissionCopyModeType::Override; + EXPECT_THROW(destFileClient.StartCopy(fileClient.GetUri(), copyOptions), std::runtime_error); + } + + // This needs support of SAS to work. + //{ + // // Upload Range from URL works. + // auto fileClient = m_shareClient->GetFileClient(LowercaseRandomString(10)); + // fileClient.Create(fileSize * 2); + + // auto destFileClient = m_shareClient->GetFileClient(LowercaseRandomString(10)); + // destFileClient.Create(fileSize * 2); + // // EXPECT_NO_THROW(fileClient.UploadRange(&memBodyStream, 0)); + // Files::Shares::UploadFileRangeFromUrlResult result; + // Files::Shares::UploadFileRangeFromUrlOptions options; + // options.SourceOffset = 0; + // options.SourceLength = fileSize; + // EXPECT_NO_THROW( + // result + // = destFileClient.UploadRangeFromUrl(fileClient.GetUri(), fileSize, fileSize, options) + // .ExtractValue()); + + // std::vector resultBuffer; + // Files::Shares::DownloadFileOptions downloadOptions; + // downloadOptions.Offset = fileSize; + // downloadOptions.Length = fileSize; + // EXPECT_NO_THROW( + // resultBuffer = Core::Http::BodyStream::ReadToEnd( + // Core::Context(), *fileClient.Download(downloadOptions)->BodyStream)); + // EXPECT_EQ(fileContent, resultBuffer); + //} + } + + TEST_F(FileShareFileClientTest, RangeRelated) + { + size_t fileSize = 1 * 1024 * 1024; + auto fileContent = RandomBuffer(fileSize); + auto memBodyStream = Core::Http::MemoryBodyStream(fileContent); + auto halfContent + = std::vector(fileContent.begin(), fileContent.begin() + fileSize / 2); + halfContent.resize(fileSize); + auto fileClient = m_shareClient->GetFileClient(LowercaseRandomString(10)); + fileClient.Create(fileSize); + EXPECT_NO_THROW(fileClient.UploadRange(&memBodyStream, 0)); + EXPECT_NO_THROW(fileClient.ClearRange(fileSize / 2, fileSize / 2)); + std::vector downloadContent(static_cast(fileSize), '\x00'); + EXPECT_NO_THROW( + fileClient.DownloadTo(downloadContent.data(), static_cast(fileSize))); + EXPECT_EQ(halfContent, downloadContent); + + EXPECT_NO_THROW(fileClient.ClearRange(512, 512)); + Files::Shares::GetFileRangeListResult result; + EXPECT_NO_THROW(result = fileClient.GetRangeList().ExtractValue()); + EXPECT_EQ(2U, result.RangeList.size()); + result.RangeList[0].Start = 0; + result.RangeList[0].End = 511; + result.RangeList[1].Start = 1024; + result.RangeList[1].End = fileSize / 2; + } + }}} // namespace Azure::Storage::Test