github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add more test cases to increase datalake service test coverage. (#1014)

Browse Source 
* Add more test cases to increase datalake service test coverage.

* add coverage for download to file.

* Reformat the include files.
This commit is contained in:
Kan Tang 2020-11-23 23:51:32 -08:00 committed by GitHub
parent fdc06b978d
commit 847e769576
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 307 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp
View File

@ -79,6 +79,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
std::string ETag;
std::string LastModified;
std::string CreationTime;
int64_t ContentLength;
std::map<std::string, std::string> Metadata;
Azure::Core::Nullable<std::string> LeaseDuration;
Azure::Core::Nullable<LeaseStateType> LeaseState;

3
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_utilities.hpp
View File

@ -10,9 +10,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
std::string GetBlobUriFromUri(const std::string& uri);
std::string GetDfsUriFromUri(const std::string& uri);
std::map<std::string, std::string> DeserializeMetadata(
const std::string& dataLakePropertiesString);
std::string SerializeMetadata(const std::map<std::string, std::string>& dataLakePropertiesMap);
std::string GetSubstringTillDelimiter(

1
sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp
View File

@ -320,6 +320,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
ret.CopyCompletionTime = std::move(result->CopyCompletionTime);
ret.ExpiryTime = std::move(result->ExpiryTime);
ret.LastAccessTime = std::move(result->LastAccessTime);
ret.ContentLength = result->ContentLength;
return Azure::Core::Response<Models::GetPathPropertiesResult>(
std::move(ret), result.ExtractRawResponse());
}

33
sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp
View File

@ -33,39 +33,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
return result;
}
std::map<std::string, std::string> DeserializeMetadata(
const std::string& dataLakePropertiesString)
{
std::map<std::string, std::string> result;
std::string::const_iterator cur = dataLakePropertiesString.begin();
auto getSubstrTillDelimiter
= [](char delimiter, const std::string& string, std::string::const_iterator& cur) {
auto begin = cur;
auto end = std::find(cur, string.end(), delimiter);
cur = end;
if (cur != string.end())
{
++cur;
}
return std::string(begin, end);
};
while (cur != dataLakePropertiesString.end())
{
std::string key = getSubstrTillDelimiter('=', dataLakePropertiesString, cur);
std::string val = getSubstrTillDelimiter(',', dataLakePropertiesString, cur);
if (!key.empty() || !val.empty())
{
result[std::move(key)] = Base64Decode(val);
}
}
return result;
}
std::string SerializeMetadata(const std::map<std::string, std::string>& dataLakePropertiesMap)
{
std::string result;

50
sdk/storage/azure-storage-files-datalake/test/datalake_directory_client_test.cpp
View File

@ -3,6 +3,8 @@
#include "datalake_directory_client_test.hpp"
#include "azure/storage/common/shared_key_policy.hpp"
#include <algorithm>
namespace Azure { namespace Storage { namespace Test {
@ -335,4 +337,52 @@ namespace Azure { namespace Storage { namespace Test {
}
}
TEST_F(DataLakeDirectoryClientTest, ConstructorsWorks)
{
{
// Create from connection string validates static creator function and shared key constructor.
auto directoryName = LowercaseRandomString(10);
auto connectionStringClient
= Azure::Storage::Files::DataLake::DirectoryClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), m_fileSystemName, directoryName);
EXPECT_NO_THROW(connectionStringClient.Create());
EXPECT_NO_THROW(connectionStringClient.Delete(true));
}
{
// Create from client secret credential.
auto credential = std::make_shared<Azure::Identity::ClientSecretCredential>(
AadTenantId(), AadClientId(), AadClientSecret());
auto clientSecretClient = Azure::Storage::Files::DataLake::DirectoryClient(
Azure::Storage::Files::DataLake::DirectoryClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), m_fileSystemName, LowercaseRandomString(10))
.GetUri(),
credential);
EXPECT_NO_THROW(clientSecretClient.Create());
EXPECT_NO_THROW(clientSecretClient.Delete(true));
}
{
// Create from Anonymous credential.
auto objectName = LowercaseRandomString(10);
auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), m_fileSystemName);
Azure::Storage::Blobs::SetContainerAccessPolicyOptions options;
options.AccessType = Azure::Storage::Blobs::Models::PublicAccessType::Container;
containerClient.SetAccessPolicy(options);
auto directoryClient
= Azure::Storage::Files::DataLake::DirectoryClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), m_fileSystemName, objectName);
EXPECT_NO_THROW(directoryClient.Create());
auto anonymousClient
= Azure::Storage::Files::DataLake::DirectoryClient(directoryClient.GetUri());
EXPECT_NO_THROW(anonymousClient.GetProperties());
}
}
}}} // namespace Azure::Storage::Test

154
sdk/storage/azure-storage-files-datalake/test/datalake_file_client_test.cpp
View File

@ -3,7 +3,25 @@
#include "datalake_file_client_test.hpp"
#include "azure/storage/blobs.hpp"
#include "azure/storage/common/file_io.hpp"
#include "azure/storage/common/shared_key_policy.hpp"
#include <algorithm>
#include <future>
#include <random>
#include <vector>
namespace Azure { namespace Storage { namespace Files { namespace DataLake { namespace Models {
bool operator==(const DataLakeHttpHeaders& lhs, const DataLakeHttpHeaders& rhs)
{
return lhs.ContentType == rhs.ContentType && lhs.ContentEncoding == rhs.ContentEncoding
&& lhs.ContentLanguage == rhs.ContentLanguage && lhs.CacheControl == rhs.CacheControl
&& lhs.ContentDisposition == rhs.ContentDisposition;
}
}}}}} // namespace Azure::Storage::Files::DataLake::Models
namespace Azure { namespace Storage { namespace Test {
@ -418,4 +436,140 @@ namespace Azure { namespace Storage { namespace Test {
}
}
TEST_F(DataLakeFileClientTest, ConcurrentUploadDownload)
{
std::vector<uint8_t> fileContent = RandomBuffer(static_cast<std::size_t>(8_MB));
auto testUploadFromBuffer = [&](int concurrency, int64_t fileSize) {
auto fileClient = m_fileSystemClient->GetFileClient(RandomString());
Azure::Storage::Files::DataLake::UploadFileFromOptions options;
options.ChunkSize = 1_MB;
options.Concurrency = concurrency;
options.HttpHeaders = GetInterestingHttpHeaders();
options.Metadata = RandomMetadata();
auto res
= fileClient.UploadFrom(fileContent.data(), static_cast<std::size_t>(fileSize), options);
EXPECT_FALSE(res->ETag.empty());
EXPECT_FALSE(res->LastModified.empty());
auto properties = *fileClient.GetProperties();
EXPECT_EQ(properties.ContentLength, fileSize);
EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders);
EXPECT_EQ(properties.Metadata, options.Metadata);
EXPECT_EQ(properties.ETag, res->ETag);
EXPECT_EQ(properties.LastModified, res->LastModified);
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(fileSize), '\x00');
fileClient.DownloadTo(downloadContent.data(), static_cast<std::size_t>(fileSize));
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
fileContent.begin(), fileContent.begin() + static_cast<std::size_t>(fileSize)));
};
auto testUploadFromFile = [&](int concurrency, int64_t fileSize) {
auto fileClient = m_fileSystemClient->GetFileClient(RandomString());
Azure::Storage::Files::DataLake::UploadFileFromOptions options;
options.ChunkSize = 1_MB;
options.Concurrency = concurrency;
options.HttpHeaders = GetInterestingHttpHeaders();
options.Metadata = RandomMetadata();
std::string tempFilename = RandomString();
{
Azure::Storage::Details::FileWriter fileWriter(tempFilename);
fileWriter.Write(fileContent.data(), fileSize, 0);
}
auto res = fileClient.UploadFrom(tempFilename, options);
EXPECT_FALSE(res->ETag.empty());
EXPECT_FALSE(res->LastModified.empty());
auto properties = *fileClient.GetProperties();
EXPECT_EQ(properties.ContentLength, fileSize);
EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders);
EXPECT_EQ(properties.Metadata, options.Metadata);
EXPECT_EQ(properties.ETag, res->ETag);
EXPECT_EQ(properties.LastModified, res->LastModified);
std::vector<uint8_t> downloadContent(static_cast<std::size_t>(fileSize), '\x00');
fileClient.DownloadTo(downloadContent.data(), static_cast<std::size_t>(fileSize));
EXPECT_EQ(
downloadContent,
std::vector<uint8_t>(
fileContent.begin(), fileContent.begin() + static_cast<std::size_t>(fileSize)));
std::string tempFileDestinationName = RandomString();
fileClient.DownloadTo(tempFileDestinationName);
Azure::Storage::Details::FileReader fileReader(tempFileDestinationName);
auto size = fileReader.GetFileSize();
EXPECT_EQ(fileSize, size);
DeleteFile(tempFileDestinationName);
DeleteFile(tempFilename);
};
std::vector<std::future<void>> futures;
for (int c : {1, 2, 5})
{
for (int64_t l :
{0ULL, 1ULL, 2ULL, 2_KB, 4_KB, 999_KB, 1_MB, 2_MB - 1, 3_MB, 5_MB, 8_MB - 1234, 8_MB})
{
ASSERT_GE(fileContent.size(), static_cast<std::size_t>(l));
futures.emplace_back(std::async(std::launch::async, testUploadFromBuffer, c, l));
futures.emplace_back(std::async(std::launch::async, testUploadFromFile, c, l));
}
}
for (auto& f : futures)
{
f.get();
}
}
TEST_F(DataLakeFileClientTest, ConstructorsWorks)
{
{
// Create from connection string validates static creator function and shared key constructor.
auto fileName = LowercaseRandomString(10);
auto connectionStringClient
= Azure::Storage::Files::DataLake::FileClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), m_fileSystemName, fileName);
EXPECT_NO_THROW(connectionStringClient.Create());
EXPECT_NO_THROW(connectionStringClient.Delete());
}
{
// Create from client secret credential.
auto credential = std::make_shared<Azure::Identity::ClientSecretCredential>(
AadTenantId(), AadClientId(), AadClientSecret());
auto clientSecretClient = Azure::Storage::Files::DataLake::FileClient(
Azure::Storage::Files::DataLake::FileClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), m_fileSystemName, LowercaseRandomString(10))
.GetUri(),
credential);
EXPECT_NO_THROW(clientSecretClient.Create());
EXPECT_NO_THROW(clientSecretClient.Delete());
}
{
// Create from Anonymous credential.
std::vector<uint8_t> blobContent;
blobContent.resize(static_cast<std::size_t>(1_MB));
RandomBuffer(reinterpret_cast<char*>(&blobContent[0]), blobContent.size());
auto objectName = LowercaseRandomString(10);
auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), m_fileSystemName);
Azure::Storage::Blobs::SetContainerAccessPolicyOptions options;
options.AccessType = Azure::Storage::Blobs::Models::PublicAccessType::Blob;
containerClient.SetAccessPolicy(options);
auto blobClient = containerClient.GetBlockBlobClient(objectName);
auto memoryStream
= Azure::Core::Http::MemoryBodyStream(blobContent.data(), blobContent.size());
EXPECT_NO_THROW(blobClient.Upload(&memoryStream));
auto anonymousClient = Azure::Storage::Files::DataLake::FileClient(
Azure::Storage::Files::DataLake::FileClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), m_fileSystemName, objectName)
.GetUri());
EXPECT_NO_THROW(anonymousClient.Read());
}
}
}}} // namespace Azure::Storage::Test

28
sdk/storage/azure-storage-files-datalake/test/datalake_file_system_client_test.cpp
View File

@ -283,4 +283,32 @@ namespace Azure { namespace Storage { namespace Test {
fileUrl, m_fileSystemClient->GetUri() + "/" + Storage::Details::UrlEncodePath(fileName));
}
}
TEST_F(DataLakeFileSystemClientTest, ConstructorsWorks)
{
{
// Create from connection string validates static creator function and shared key constructor.
auto fileSystemName = LowercaseRandomString(10);
auto connectionStringClient
= Azure::Storage::Files::DataLake::FileSystemClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), fileSystemName);
EXPECT_NO_THROW(connectionStringClient.Create());
EXPECT_NO_THROW(connectionStringClient.Delete());
}
{
// Create from client secret credential.
auto credential = std::make_shared<Azure::Identity::ClientSecretCredential>(
AadTenantId(), AadClientId(), AadClientSecret());
auto clientSecretClient = Azure::Storage::Files::DataLake::FileSystemClient(
Azure::Storage::Files::DataLake::FileSystemClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), LowercaseRandomString(10))
.GetUri(),
credential);
EXPECT_NO_THROW(clientSecretClient.Create());
EXPECT_NO_THROW(clientSecretClient.Delete());
}
}
}}} // namespace Azure::Storage::Test

49
sdk/storage/azure-storage-files-datalake/test/datalake_path_client_test.cpp
View File

@ -21,7 +21,7 @@ namespace Azure { namespace Storage { namespace Test {
void DataLakePathClientTest::TearDownTestSuite()
{
m_fileSystemClient->GetFileClient(m_pathName).Delete();
m_pathClient->Delete();
DataLakeFileSystemClientTest::TearDownTestSuite();
}
@ -288,4 +288,51 @@ namespace Azure { namespace Storage { namespace Test {
options.BreakPeriod = 0;
m_pathClient->BreakLease(options);
}
TEST_F(DataLakePathClientTest, ConstructorsWorks)
{
{
// Create from connection string validates static creator function and shared key constructor.
auto pathName = LowercaseRandomString(10);
auto connectionStringClient
= Azure::Storage::Files::DataLake::PathClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), m_fileSystemName, pathName);
EXPECT_NO_THROW(
connectionStringClient.Create(Files::DataLake::Models::PathResourceType::File));
EXPECT_NO_THROW(connectionStringClient.Delete());
}
{
// Create from client secret credential.
auto credential = std::make_shared<Azure::Identity::ClientSecretCredential>(
AadTenantId(), AadClientId(), AadClientSecret());
auto clientSecretClient = Azure::Storage::Files::DataLake::PathClient(
Azure::Storage::Files::DataLake::PathClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), m_fileSystemName, LowercaseRandomString(10))
.GetUri(),
credential);
EXPECT_NO_THROW(clientSecretClient.Create(Files::DataLake::Models::PathResourceType::File));
EXPECT_NO_THROW(clientSecretClient.Delete());
}
{
// Create from Anonymous credential.
auto objectName = LowercaseRandomString(10);
auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), m_fileSystemName);
Azure::Storage::Blobs::SetContainerAccessPolicyOptions options;
options.AccessType = Azure::Storage::Blobs::Models::PublicAccessType::Container;
containerClient.SetAccessPolicy(options);
auto pathClient = Azure::Storage::Files::DataLake::PathClient::CreateFromConnectionString(
AdlsGen2ConnectionString(), m_fileSystemName, objectName);
EXPECT_NO_THROW(pathClient.Create(Files::DataLake::Models::PathResourceType::File));
auto anonymousClient = Azure::Storage::Files::DataLake::PathClient(pathClient.GetUri());
EXPECT_NO_THROW(anonymousClient.GetProperties());
}
}
}}} // namespace Azure::Storage::Test

25
sdk/storage/azure-storage-files-datalake/test/datalake_service_client_test.cpp
View File

@ -137,4 +137,29 @@ namespace Azure { namespace Storage { namespace Test {
}
}
TEST_F(DataLakeServiceClientTest, AnonymousConstructorsWorks)
{
auto keyCredential
= Azure::Storage::Details::ParseConnectionString(AdlsGen2ConnectionString()).KeyCredential;
AccountSasBuilder accountSasBuilder;
accountSasBuilder.Protocol = SasProtocol::HttpsAndHtttp;
accountSasBuilder.StartsOn
= ToIso8601(std::chrono::system_clock::now() - std::chrono::minutes(5));
accountSasBuilder.ExpiresOn
= ToIso8601(std::chrono::system_clock::now() + std::chrono::minutes(60));
accountSasBuilder.Services = AccountSasServices::Blobs;
accountSasBuilder.ResourceTypes = AccountSasResource::All;
accountSasBuilder.SetPermissions(AccountSasPermissions::All);
auto sasToken = accountSasBuilder.GenerateSasToken(*keyCredential);
// Create from Anonymous credential.
auto datalakeServiceUri
= Azure::Storage::Files::DataLake::DataLakeServiceClient::CreateFromConnectionString(
AdlsGen2ConnectionString())
.GetUri();
auto datalakeServiceClient
= Azure::Storage::Files::DataLake::DataLakeServiceClient(datalakeServiceUri + sasToken);
EXPECT_NO_THROW(datalakeServiceClient.ListFileSystemsSegement());
}
}}} // namespace Azure::Storage::Test