diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index dde4bb63e..f06a3cab7 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -79,6 +79,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam std::string ETag; std::string LastModified; std::string CreationTime; + int64_t ContentLength; std::map Metadata; Azure::Core::Nullable LeaseDuration; Azure::Core::Nullable LeaseState; diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_utilities.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_utilities.hpp index 4ef523094..5564c4633 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_utilities.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_utilities.hpp @@ -10,9 +10,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam std::string GetBlobUriFromUri(const std::string& uri); std::string GetDfsUriFromUri(const std::string& uri); - std::map DeserializeMetadata( - const std::string& dataLakePropertiesString); - std::string SerializeMetadata(const std::map& dataLakePropertiesMap); std::string GetSubstringTillDelimiter( diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp index 234f7ace1..a1eac5e86 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp @@ -320,6 +320,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { ret.CopyCompletionTime = std::move(result->CopyCompletionTime); ret.ExpiryTime = std::move(result->ExpiryTime); ret.LastAccessTime = std::move(result->LastAccessTime); + ret.ContentLength = result->ContentLength; return Azure::Core::Response( std::move(ret), result.ExtractRawResponse()); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp index 454d89c82..d4a2c66bc 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp @@ -33,39 +33,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam return result; } - std::map DeserializeMetadata( - const std::string& dataLakePropertiesString) - { - std::map result; - - std::string::const_iterator cur = dataLakePropertiesString.begin(); - - auto getSubstrTillDelimiter - = [](char delimiter, const std::string& string, std::string::const_iterator& cur) { - auto begin = cur; - auto end = std::find(cur, string.end(), delimiter); - cur = end; - if (cur != string.end()) - { - ++cur; - } - return std::string(begin, end); - }; - - while (cur != dataLakePropertiesString.end()) - { - std::string key = getSubstrTillDelimiter('=', dataLakePropertiesString, cur); - std::string val = getSubstrTillDelimiter(',', dataLakePropertiesString, cur); - - if (!key.empty() || !val.empty()) - { - result[std::move(key)] = Base64Decode(val); - } - } - - return result; - } - std::string SerializeMetadata(const std::map& dataLakePropertiesMap) { std::string result; diff --git a/sdk/storage/azure-storage-files-datalake/test/datalake_directory_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/datalake_directory_client_test.cpp index 41bd1ce02..236402565 100644 --- a/sdk/storage/azure-storage-files-datalake/test/datalake_directory_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/datalake_directory_client_test.cpp @@ -3,6 +3,8 @@ #include "datalake_directory_client_test.hpp" +#include "azure/storage/common/shared_key_policy.hpp" + #include namespace Azure { namespace Storage { namespace Test { @@ -335,4 +337,52 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(DataLakeDirectoryClientTest, ConstructorsWorks) + { + { + // Create from connection string validates static creator function and shared key constructor. + auto directoryName = LowercaseRandomString(10); + auto connectionStringClient + = Azure::Storage::Files::DataLake::DirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, directoryName); + EXPECT_NO_THROW(connectionStringClient.Create()); + EXPECT_NO_THROW(connectionStringClient.Delete(true)); + } + + { + // Create from client secret credential. + auto credential = std::make_shared( + AadTenantId(), AadClientId(), AadClientSecret()); + + auto clientSecretClient = Azure::Storage::Files::DataLake::DirectoryClient( + Azure::Storage::Files::DataLake::DirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, LowercaseRandomString(10)) + .GetUri(), + credential); + + EXPECT_NO_THROW(clientSecretClient.Create()); + EXPECT_NO_THROW(clientSecretClient.Delete(true)); + } + + { + // Create from Anonymous credential. + auto objectName = LowercaseRandomString(10); + auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName); + Azure::Storage::Blobs::SetContainerAccessPolicyOptions options; + options.AccessType = Azure::Storage::Blobs::Models::PublicAccessType::Container; + containerClient.SetAccessPolicy(options); + + auto directoryClient + = Azure::Storage::Files::DataLake::DirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, objectName); + EXPECT_NO_THROW(directoryClient.Create()); + + auto anonymousClient + = Azure::Storage::Files::DataLake::DirectoryClient(directoryClient.GetUri()); + + EXPECT_NO_THROW(anonymousClient.GetProperties()); + } + } + }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/datalake_file_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/datalake_file_client_test.cpp index 626de3e1b..4ba1d8318 100644 --- a/sdk/storage/azure-storage-files-datalake/test/datalake_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/datalake_file_client_test.cpp @@ -3,7 +3,25 @@ #include "datalake_file_client_test.hpp" +#include "azure/storage/blobs.hpp" +#include "azure/storage/common/file_io.hpp" +#include "azure/storage/common/shared_key_policy.hpp" + #include +#include +#include +#include + +namespace Azure { namespace Storage { namespace Files { namespace DataLake { namespace Models { + + bool operator==(const DataLakeHttpHeaders& lhs, const DataLakeHttpHeaders& rhs) + { + return lhs.ContentType == rhs.ContentType && lhs.ContentEncoding == rhs.ContentEncoding + && lhs.ContentLanguage == rhs.ContentLanguage && lhs.CacheControl == rhs.CacheControl + && lhs.ContentDisposition == rhs.ContentDisposition; + } + +}}}}} // namespace Azure::Storage::Files::DataLake::Models namespace Azure { namespace Storage { namespace Test { @@ -418,4 +436,140 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(DataLakeFileClientTest, ConcurrentUploadDownload) + { + std::vector fileContent = RandomBuffer(static_cast(8_MB)); + + auto testUploadFromBuffer = [&](int concurrency, int64_t fileSize) { + auto fileClient = m_fileSystemClient->GetFileClient(RandomString()); + + Azure::Storage::Files::DataLake::UploadFileFromOptions options; + options.ChunkSize = 1_MB; + options.Concurrency = concurrency; + options.HttpHeaders = GetInterestingHttpHeaders(); + options.Metadata = RandomMetadata(); + auto res + = fileClient.UploadFrom(fileContent.data(), static_cast(fileSize), options); + EXPECT_FALSE(res->ETag.empty()); + EXPECT_FALSE(res->LastModified.empty()); + auto properties = *fileClient.GetProperties(); + EXPECT_EQ(properties.ContentLength, fileSize); + EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); + EXPECT_EQ(properties.Metadata, options.Metadata); + EXPECT_EQ(properties.ETag, res->ETag); + EXPECT_EQ(properties.LastModified, res->LastModified); + std::vector downloadContent(static_cast(fileSize), '\x00'); + fileClient.DownloadTo(downloadContent.data(), static_cast(fileSize)); + EXPECT_EQ( + downloadContent, + std::vector( + fileContent.begin(), fileContent.begin() + static_cast(fileSize))); + }; + + auto testUploadFromFile = [&](int concurrency, int64_t fileSize) { + auto fileClient = m_fileSystemClient->GetFileClient(RandomString()); + + Azure::Storage::Files::DataLake::UploadFileFromOptions options; + options.ChunkSize = 1_MB; + options.Concurrency = concurrency; + options.HttpHeaders = GetInterestingHttpHeaders(); + options.Metadata = RandomMetadata(); + + std::string tempFilename = RandomString(); + { + Azure::Storage::Details::FileWriter fileWriter(tempFilename); + fileWriter.Write(fileContent.data(), fileSize, 0); + } + auto res = fileClient.UploadFrom(tempFilename, options); + EXPECT_FALSE(res->ETag.empty()); + EXPECT_FALSE(res->LastModified.empty()); + auto properties = *fileClient.GetProperties(); + EXPECT_EQ(properties.ContentLength, fileSize); + EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); + EXPECT_EQ(properties.Metadata, options.Metadata); + EXPECT_EQ(properties.ETag, res->ETag); + EXPECT_EQ(properties.LastModified, res->LastModified); + std::vector downloadContent(static_cast(fileSize), '\x00'); + fileClient.DownloadTo(downloadContent.data(), static_cast(fileSize)); + EXPECT_EQ( + downloadContent, + std::vector( + fileContent.begin(), fileContent.begin() + static_cast(fileSize))); + std::string tempFileDestinationName = RandomString(); + fileClient.DownloadTo(tempFileDestinationName); + Azure::Storage::Details::FileReader fileReader(tempFileDestinationName); + auto size = fileReader.GetFileSize(); + EXPECT_EQ(fileSize, size); + DeleteFile(tempFileDestinationName); + DeleteFile(tempFilename); + }; + + std::vector> futures; + for (int c : {1, 2, 5}) + { + for (int64_t l : + {0ULL, 1ULL, 2ULL, 2_KB, 4_KB, 999_KB, 1_MB, 2_MB - 1, 3_MB, 5_MB, 8_MB - 1234, 8_MB}) + { + ASSERT_GE(fileContent.size(), static_cast(l)); + futures.emplace_back(std::async(std::launch::async, testUploadFromBuffer, c, l)); + futures.emplace_back(std::async(std::launch::async, testUploadFromFile, c, l)); + } + } + for (auto& f : futures) + { + f.get(); + } + } + + TEST_F(DataLakeFileClientTest, ConstructorsWorks) + { + { + // Create from connection string validates static creator function and shared key constructor. + auto fileName = LowercaseRandomString(10); + auto connectionStringClient + = Azure::Storage::Files::DataLake::FileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, fileName); + EXPECT_NO_THROW(connectionStringClient.Create()); + EXPECT_NO_THROW(connectionStringClient.Delete()); + } + + { + // Create from client secret credential. + auto credential = std::make_shared( + AadTenantId(), AadClientId(), AadClientSecret()); + + auto clientSecretClient = Azure::Storage::Files::DataLake::FileClient( + Azure::Storage::Files::DataLake::FileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, LowercaseRandomString(10)) + .GetUri(), + credential); + + EXPECT_NO_THROW(clientSecretClient.Create()); + EXPECT_NO_THROW(clientSecretClient.Delete()); + } + + { + // Create from Anonymous credential. + std::vector blobContent; + blobContent.resize(static_cast(1_MB)); + RandomBuffer(reinterpret_cast(&blobContent[0]), blobContent.size()); + auto objectName = LowercaseRandomString(10); + auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName); + Azure::Storage::Blobs::SetContainerAccessPolicyOptions options; + options.AccessType = Azure::Storage::Blobs::Models::PublicAccessType::Blob; + containerClient.SetAccessPolicy(options); + auto blobClient = containerClient.GetBlockBlobClient(objectName); + auto memoryStream + = Azure::Core::Http::MemoryBodyStream(blobContent.data(), blobContent.size()); + EXPECT_NO_THROW(blobClient.Upload(&memoryStream)); + + auto anonymousClient = Azure::Storage::Files::DataLake::FileClient( + Azure::Storage::Files::DataLake::FileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, objectName) + .GetUri()); + + EXPECT_NO_THROW(anonymousClient.Read()); + } + } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/datalake_file_system_client_test.cpp index b701df924..cddf08af3 100644 --- a/sdk/storage/azure-storage-files-datalake/test/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/datalake_file_system_client_test.cpp @@ -283,4 +283,32 @@ namespace Azure { namespace Storage { namespace Test { fileUrl, m_fileSystemClient->GetUri() + "/" + Storage::Details::UrlEncodePath(fileName)); } } + + TEST_F(DataLakeFileSystemClientTest, ConstructorsWorks) + { + { + // Create from connection string validates static creator function and shared key constructor. + auto fileSystemName = LowercaseRandomString(10); + auto connectionStringClient + = Azure::Storage::Files::DataLake::FileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), fileSystemName); + EXPECT_NO_THROW(connectionStringClient.Create()); + EXPECT_NO_THROW(connectionStringClient.Delete()); + } + + { + // Create from client secret credential. + auto credential = std::make_shared( + AadTenantId(), AadClientId(), AadClientSecret()); + + auto clientSecretClient = Azure::Storage::Files::DataLake::FileSystemClient( + Azure::Storage::Files::DataLake::FileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), LowercaseRandomString(10)) + .GetUri(), + credential); + + EXPECT_NO_THROW(clientSecretClient.Create()); + EXPECT_NO_THROW(clientSecretClient.Delete()); + } + } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/datalake_path_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/datalake_path_client_test.cpp index 91e519b75..8526ad057 100644 --- a/sdk/storage/azure-storage-files-datalake/test/datalake_path_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/datalake_path_client_test.cpp @@ -21,7 +21,7 @@ namespace Azure { namespace Storage { namespace Test { void DataLakePathClientTest::TearDownTestSuite() { - m_fileSystemClient->GetFileClient(m_pathName).Delete(); + m_pathClient->Delete(); DataLakeFileSystemClientTest::TearDownTestSuite(); } @@ -288,4 +288,51 @@ namespace Azure { namespace Storage { namespace Test { options.BreakPeriod = 0; m_pathClient->BreakLease(options); } + + TEST_F(DataLakePathClientTest, ConstructorsWorks) + { + { + // Create from connection string validates static creator function and shared key constructor. + auto pathName = LowercaseRandomString(10); + auto connectionStringClient + = Azure::Storage::Files::DataLake::PathClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, pathName); + EXPECT_NO_THROW( + connectionStringClient.Create(Files::DataLake::Models::PathResourceType::File)); + EXPECT_NO_THROW(connectionStringClient.Delete()); + } + + { + // Create from client secret credential. + auto credential = std::make_shared( + AadTenantId(), AadClientId(), AadClientSecret()); + + auto clientSecretClient = Azure::Storage::Files::DataLake::PathClient( + Azure::Storage::Files::DataLake::PathClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, LowercaseRandomString(10)) + .GetUri(), + credential); + + EXPECT_NO_THROW(clientSecretClient.Create(Files::DataLake::Models::PathResourceType::File)); + EXPECT_NO_THROW(clientSecretClient.Delete()); + } + + { + // Create from Anonymous credential. + auto objectName = LowercaseRandomString(10); + auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName); + Azure::Storage::Blobs::SetContainerAccessPolicyOptions options; + options.AccessType = Azure::Storage::Blobs::Models::PublicAccessType::Container; + containerClient.SetAccessPolicy(options); + + auto pathClient = Azure::Storage::Files::DataLake::PathClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, objectName); + EXPECT_NO_THROW(pathClient.Create(Files::DataLake::Models::PathResourceType::File)); + + auto anonymousClient = Azure::Storage::Files::DataLake::PathClient(pathClient.GetUri()); + + EXPECT_NO_THROW(anonymousClient.GetProperties()); + } + } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/datalake_service_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/datalake_service_client_test.cpp index a2a94ee3c..95cd79a77 100644 --- a/sdk/storage/azure-storage-files-datalake/test/datalake_service_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/datalake_service_client_test.cpp @@ -137,4 +137,29 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(DataLakeServiceClientTest, AnonymousConstructorsWorks) + { + auto keyCredential + = Azure::Storage::Details::ParseConnectionString(AdlsGen2ConnectionString()).KeyCredential; + AccountSasBuilder accountSasBuilder; + accountSasBuilder.Protocol = SasProtocol::HttpsAndHtttp; + accountSasBuilder.StartsOn + = ToIso8601(std::chrono::system_clock::now() - std::chrono::minutes(5)); + accountSasBuilder.ExpiresOn + = ToIso8601(std::chrono::system_clock::now() + std::chrono::minutes(60)); + accountSasBuilder.Services = AccountSasServices::Blobs; + accountSasBuilder.ResourceTypes = AccountSasResource::All; + accountSasBuilder.SetPermissions(AccountSasPermissions::All); + auto sasToken = accountSasBuilder.GenerateSasToken(*keyCredential); + + // Create from Anonymous credential. + auto datalakeServiceUri + = Azure::Storage::Files::DataLake::DataLakeServiceClient::CreateFromConnectionString( + AdlsGen2ConnectionString()) + .GetUri(); + auto datalakeServiceClient + = Azure::Storage::Files::DataLake::DataLakeServiceClient(datalakeServiceUri + sasToken); + EXPECT_NO_THROW(datalakeServiceClient.ListFileSystemsSegement()); + } + }}} // namespace Azure::Storage::Test