github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refined DataLake sample (#336)

Browse Source 
* Refined DataLake sample

* Added DirectoryClient::GetFileClient, resolved some comment and test failures.

* Resolved an issue introduced by lowercasing response headers resulting in error code/message not being passed, and refined sample.

* Refined the structure according to comments.

* Changed buffer initialziation to stack allocated.

* Resolved some further comments.
This commit is contained in:
Kan Tang 2020-07-23 15:40:16 +08:00 committed by GitHub
parent 29f5e30ad3
commit 8568327a56
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 176 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sdk/storage/inc/common/constants.hpp
View File

@ -16,4 +16,5 @@ namespace Azure { namespace Storage { namespace Details {
constexpr static const char* c_HttpHeaderXMsVersion = "x-ms-version";
constexpr static const char* c_HttpHeaderRequestId = "x-ms-request-id";
constexpr static const char* c_HttpHeaderClientRequestId = "x-ms-client-request-id";
constexpr static const char* c_HttpHeaderContentType = "content-type";
}}} // namespace Azure::Storage::Details

2
sdk/storage/inc/common/storage_error.hpp
View File

@ -24,6 +24,6 @@ namespace Azure { namespace Storage {
std::unique_ptr<Azure::Core::Http::RawResponse> RawResponse;
static StorageError CreateFromResponse(
/* const */ std::unique_ptr<Azure::Core::Http::RawResponse> response);
std::unique_ptr<Azure::Core::Http::RawResponse> response);
};
}} // namespace Azure::Storage

7
sdk/storage/inc/datalake/directory_client.hpp
View File

@ -65,6 +65,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
const std::string& directoryUri,
const DirectoryClientOptions& options = DirectoryClientOptions());
/**
* @brief Create a FileClient from current DirectoryClient
* @param path Path of the file under the directory.
* @return FileClient
*/
FileClient GetFileClient(const std::string& path) const;
/**
* @brief Gets the directory's primary uri endpoint. This is the endpoint used for blob
* storage available features in DataLake.

1
sdk/storage/inc/datalake/file_client.hpp
View File

@ -231,5 +231,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
}
friend class FileSystemClient;
friend class DirectoryClient;
};
}}}} // namespace Azure::Storage::Files::DataLake

106
sdk/storage/sample/datalake_getting_started.cpp
View File

@ -3,11 +3,113 @@
#include "datalake/datalake.hpp"
#include "samples_common.hpp"
#include <iostream>
SAMPLE(DataLakeGettingStarted, DataLakeGettingStarted)
void DataLakeGettingStarted()
{
// Please note that you can always reference test cases for advanced usages.
using namespace Azure::Storage::Files::DataLake;
auto client = ServiceClient::CreateFromConnectionString(GetConnectionString());
auto response = client.ListFileSystems();
std::string fileSystemName = "sample-file-system";
std::string directoryName = "sample-directory";
std::string fileName = "sample-file";
// Initializing a ServiceClient that can then initialize the FileSystemClient or list file
// systems.
auto serviceClient = ServiceClient::CreateFromConnectionString(GetConnectionString());
// Initializing a FileSystemClient that can then initialize the PathClient, FileClient,
// DirectoryClient.
auto fileSystemClient
= FileSystemClient::CreateFromConnectionString(GetConnectionString(), fileSystemName);
try
{
// Create file systems and ignore the already exist error.
try
{
fileSystemClient.Create();
}
catch (Azure::Storage::StorageError& e)
{
if (e.ErrorCode != "ContainerAlreadyExists")
{
throw;
}
else
{
std::cout << "ErrorCode: " + e.ErrorCode << std::endl;
std::cout << "ReasonPhrase: " + e.ReasonPhrase << std::endl;
}
}
// Create a directory.
auto directoryClient = fileSystemClient.GetDirectoryClient(directoryName);
directoryClient.Create();
// Creates a file under the directory.
auto fileClient = directoryClient.GetFileClient(fileName);
fileClient.Create();
// Append/flush/read data from the client.
// Append data
// Initialize the string that contains the first piece of data to be appended to the file.
std::string str1 = "Hello ";
// Initialize the buffer that represents what contains your data to be appended, please ignore
// how it is constructed here, since the memory copy is not efficient.
std::string str2 = "World!";
std::vector<uint8_t> buffer(str1.begin(), str1.end());
// One way of passing in the buffer, note that the buffer is not copied.
auto bufferStream = Azure::Core::Http::MemoryBodyStream(buffer);
fileClient.AppendData(&bufferStream, 0 /* Offset of the position to be appended.*/);
// Another way of passing in the buffer, note that buffer is also not copied.
bufferStream = Azure::Core::Http::MemoryBodyStream(
reinterpret_cast<const uint8_t*>(str2.data()), str2.size());
fileClient.AppendData(&bufferStream, str1.size());
// Flush
fileClient.FlushData(str1.size() + str2.size());
// Read
auto result = fileClient.Read();
Azure::Core::Context context;
std::vector<uint8_t> downloaded
= Azure::Core::Http::BodyStream::ReadToEnd(context, *(result->Body));
// downloaded contains your downloaded data.
std::cout << "Downloaded data was:\n" + std::string(downloaded.begin(), downloaded.end())
<< std::endl;
// List all file systems.
std::string continuation;
std::vector<FileSystem> fileSystems;
do
{
auto response = serviceClient.ListFileSystems();
if (response->Continuation.HasValue())
{
continuation = response->Continuation.GetValue();
}
fileSystems.insert(
fileSystems.end(), response->Filesystems.begin(), response->Filesystems.end());
} while (!continuation.empty());
// Delete file system.
fileSystemClient.Delete();
std::cout << "Successfully finished sample." << std::endl;
}
catch (const Azure::Storage::StorageError& e)
{
// Deal with the information when storage error is met.
std::cout << "Error encountered when sending the request." << std::endl;
std::cout << "ErrorCode: " + e.ErrorCode << std::endl;
std::cout << "Message: " + e.Message << std::endl;
std::cout << "ReasonPhrase: " + e.ReasonPhrase << std::endl;
std::cout << "RequestId: " + e.RequestId << std::endl;
}
}

15
sdk/storage/src/common/storage_error.cpp
View File

@ -3,6 +3,7 @@
#include "common/storage_error.hpp"
#include "common/constants.hpp"
#include "common/xml_wrapper.hpp"
#include "json.hpp"
@ -38,9 +39,11 @@ namespace Azure { namespace Storage {
std::string errorCode;
std::string message;
if (response->GetHeaders().find("Content-Type") != response->GetHeaders().end())
if (response->GetHeaders().find(Details::c_HttpHeaderContentType)
!= response->GetHeaders().end())
{
if (response->GetHeaders().at("Content-Type").find("xml") != std::string::npos)
if (response->GetHeaders().at(Details::c_HttpHeaderContentType).find("xml")
!= std::string::npos)
{
auto xmlReader
= XmlReader(reinterpret_cast<const char*>(bodyBuffer.data()), bodyBuffer.size());
@ -101,12 +104,16 @@ namespace Azure { namespace Storage {
}
}
}
else if (response->GetHeaders().at("Content-Type").find("html") != std::string::npos)
else if (
response->GetHeaders().at(Details::c_HttpHeaderContentType).find("html")
!= std::string::npos)
{
// TODO: add a refined message parsed from result.
message = std::string(bodyBuffer.begin(), bodyBuffer.end());
}
else if (response->GetHeaders().at("Content-Type").find("json") != std::string::npos)
else if (
response->GetHeaders().at(Details::c_HttpHeaderContentType).find("json")
!= std::string::npos)
{
auto jsonParser = nlohmann::json::parse(bodyBuffer);
errorCode = jsonParser["error"]["code"].get<std::string>();

10
sdk/storage/src/datalake/directory_client.cpp
View File

@ -11,6 +11,7 @@
#include "common/storage_version.hpp"
#include "credentials/policy/policies.hpp"
#include "datalake/datalake_utilities.hpp"
#include "datalake/file_client.hpp"
#include "http/curl/curl.hpp"
#include <limits>
@ -118,6 +119,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
m_pipeline = std::make_shared<Azure::Core::Http::HttpPipeline>(policies);
}
FileClient DirectoryClient::GetFileClient(const std::string& path) const
{
auto builder = m_dfsUri;
builder.AppendPath(path, true);
auto blobClient = m_blobClient;
blobClient.m_blobUrl.AppendPath(path, true);
return FileClient(std::move(builder), std::move(blobClient), m_pipeline);
}
Azure::Core::Response<DirectoryRenameInfo> DirectoryClient::Rename(
const std::string& destinationPath,
const DirectoryRenameOptions& options) const

38
sdk/storage/test/datalake/directory_client_test.cpp
View File

@ -104,25 +104,27 @@ namespace Azure { namespace Storage { namespace Test {
{
{
// Normal create/rename/delete.
std::vector<Files::DataLake::DirectoryClient> directoryClient;
std::vector<Files::DataLake::DirectoryClient> directoryClients;
for (int32_t i = 0; i < 5; ++i)
{
auto client = m_fileSystemClient->GetDirectoryClient(LowercaseRandomString());
EXPECT_NO_THROW(client.Create());
directoryClient.emplace_back(std::move(client));
directoryClients.emplace_back(std::move(client));
}
auto directoryClientClone = directoryClient;
for (auto& client : directoryClient)
std::vector<std::string> newPaths;
for (auto& client : directoryClients)
{
EXPECT_NO_THROW(client.Rename(LowercaseRandomString()));
auto newPath = LowercaseRandomString();
EXPECT_NO_THROW(client.Rename(newPath));
newPaths.push_back(newPath);
}
for (const auto& client : directoryClientClone)
for (const auto& client : directoryClients)
{
EXPECT_THROW(client.Delete(false), StorageError);
}
for (const auto& client : directoryClient)
for (const auto& newPath : newPaths)
{
EXPECT_NO_THROW(client.Delete(false));
EXPECT_NO_THROW(m_fileSystemClient->GetDirectoryClient(newPath).Delete(false));
}
}
{
@ -142,8 +144,9 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_THROW(client.Rename(LowercaseRandomString(), options1), StorageError);
Files::DataLake::DirectoryRenameOptions options2;
options2.SourceAccessConditions.IfUnmodifiedSince = response->LastModified;
EXPECT_NO_THROW(client.Rename(LowercaseRandomString(), options2));
EXPECT_NO_THROW(client.Delete(false));
auto newPath = LowercaseRandomString();
EXPECT_NO_THROW(client.Rename(newPath, options2));
EXPECT_NO_THROW(m_fileSystemClient->GetDirectoryClient(newPath).Delete(false));
}
}
{
@ -163,8 +166,9 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_THROW(client.Rename(LowercaseRandomString(), options1), StorageError);
Files::DataLake::DirectoryRenameOptions options2;
options2.SourceAccessConditions.IfMatch = response->ETag;
EXPECT_NO_THROW(client.Rename(LowercaseRandomString(), options2));
EXPECT_NO_THROW(client.Delete(false));
auto newPath = LowercaseRandomString();
EXPECT_NO_THROW(client.Rename(newPath, options2));
EXPECT_NO_THROW(m_fileSystemClient->GetDirectoryClient(newPath).Delete(false));
}
}
{
@ -177,7 +181,7 @@ namespace Azure { namespace Storage { namespace Test {
directoryClient.emplace_back(std::move(client));
}
{
// Rename to a non-existing file system will fail but will not change URI.
// Rename to a non-existing file system will fail and source is not changed.
Files::DataLake::DirectoryRenameOptions options;
options.DestinationFileSystem = LowercaseRandomString();
for (auto& client : directoryClient)
@ -187,7 +191,7 @@ namespace Azure { namespace Storage { namespace Test {
}
}
{
// Rename to a non-existing file system will succeed and changes URI.
// Rename to an existing file system will succeed and changes URI.
auto newfileSystemName = LowercaseRandomString(10);
auto newfileSystemClient = std::make_shared<Files::DataLake::FileSystemClient>(
Files::DataLake::FileSystemClient::CreateFromConnectionString(
@ -197,9 +201,9 @@ namespace Azure { namespace Storage { namespace Test {
options.DestinationFileSystem = newfileSystemName;
for (auto& client : directoryClient)
{
EXPECT_NO_THROW(client.Rename(LowercaseRandomString(), options));
EXPECT_NO_THROW(client.GetProperties());
EXPECT_NO_THROW(client.Delete(false));
auto newPath = LowercaseRandomString();
EXPECT_NO_THROW(client.Rename(newPath, options));
EXPECT_NO_THROW(newfileSystemClient->GetDirectoryClient(newPath).Delete(false));
}
}
}

36
sdk/storage/test/datalake/file_client_test.cpp
View File

@ -87,25 +87,27 @@ namespace Azure { namespace Storage { namespace Test {
{
{
// Normal create/rename/delete.
std::vector<Files::DataLake::FileClient> fileClient;
std::vector<Files::DataLake::FileClient> fileClients;
for (int32_t i = 0; i < 5; ++i)
{
auto client = m_fileSystemClient->GetFileClient(LowercaseRandomString());
EXPECT_NO_THROW(client.Create());
fileClient.emplace_back(std::move(client));
fileClients.emplace_back(std::move(client));
}
auto fileClientClone = fileClient;
for (auto& client : fileClient)
std::vector<std::string> newPaths;
for (auto& client : fileClients)
{
EXPECT_NO_THROW(client.Rename(LowercaseRandomString()));
auto newPath = LowercaseRandomString();
EXPECT_NO_THROW(client.Rename(newPath));
newPaths.push_back(newPath);
}
for (const auto& client : fileClientClone)
for (const auto& client : fileClients)
{
EXPECT_THROW(client.Delete(), StorageError);
}
for (const auto& client : fileClient)
for (const auto& newPath : newPaths)
{
EXPECT_NO_THROW(client.Delete());
EXPECT_NO_THROW(m_fileSystemClient->GetDirectoryClient(newPath).Delete(false));
}
}
{
@ -125,8 +127,9 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_THROW(client.Rename(LowercaseRandomString(), options1), StorageError);
Files::DataLake::FileRenameOptions options2;
options2.SourceAccessConditions.IfUnmodifiedSince = response->LastModified;
EXPECT_NO_THROW(client.Rename(LowercaseRandomString(), options2));
EXPECT_NO_THROW(client.Delete());
auto newPath = LowercaseRandomString();
EXPECT_NO_THROW(client.Rename(newPath, options2));
EXPECT_NO_THROW(m_fileSystemClient->GetDirectoryClient(newPath).Delete(false));
}
}
{
@ -146,8 +149,9 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_THROW(client.Rename(LowercaseRandomString(), options1), StorageError);
Files::DataLake::FileRenameOptions options2;
options2.SourceAccessConditions.IfMatch = response->ETag;
EXPECT_NO_THROW(client.Rename(LowercaseRandomString(), options2));
EXPECT_NO_THROW(client.Delete());
auto newPath = LowercaseRandomString();
EXPECT_NO_THROW(client.Rename(newPath, options2));
EXPECT_NO_THROW(m_fileSystemClient->GetDirectoryClient(newPath).Delete(false));
}
}
{
@ -170,7 +174,7 @@ namespace Azure { namespace Storage { namespace Test {
}
}
{
// Rename to a non-existing file system will succeed and changes URI.
// Rename to an existing file system will succeed and changes URI.
auto newfileSystemName = LowercaseRandomString(10);
auto newfileSystemClient = std::make_shared<Files::DataLake::FileSystemClient>(
Files::DataLake::FileSystemClient::CreateFromConnectionString(
@ -180,9 +184,9 @@ namespace Azure { namespace Storage { namespace Test {
options.DestinationFileSystem = newfileSystemName;
for (auto& client : fileClient)
{
EXPECT_NO_THROW(client.Rename(LowercaseRandomString(), options));
EXPECT_NO_THROW(client.GetProperties());
EXPECT_NO_THROW(client.Delete());
auto newPath = LowercaseRandomString();
EXPECT_NO_THROW(client.Rename(newPath, options));
EXPECT_NO_THROW(newfileSystemClient->GetDirectoryClient(newPath).Delete(false));
}
}
}