github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Storage/STG90 features (#4827)

Browse Source 
* Storage/STG90-Data Lake Pagination Delete (#4816)

* add pagination delete

* remove id and secret

* fix spell

* add clientConfiguartion to all clients

* fix doxygen doc

* fix conversation

* add unit tests

* fix cspell

* fix clang format

* update test code

* update test case

* update record

* Storage/Archive to Cold Tier Rehydration (#4825)

* add RehydratePendingToCold

* add unit test

* update changelog.md

* update record
This commit is contained in:
microzchang 2023-07-31 21:21:53 +08:00 committed by GitHub
parent 1634d5825e
commit 5f3fe6fa5e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 294 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sdk/storage/assets.json
View File

@ -2,5 +2,5 @@
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "cpp",
"TagPrefix": "cpp/storage",
"Tag": "cpp/storage_67000ec514"
"Tag": "cpp/storage_b632cb2101"
}

6
sdk/storage/azure-storage-blobs/CHANGELOG.md
View File

@ -4,11 +4,7 @@
### Features Added
### Breaking Changes
### Bugs Fixed
### Other Changes
- Added `RehydratePendingToCold` value to `ArchiveStatus` enum.
## 12.8.0 (2023-07-11)

2
sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp
View File

@ -1106,6 +1106,8 @@ namespace Azure { namespace Storage { namespace Blobs {
AZ_STORAGE_BLOBS_DLLEXPORT const static ArchiveStatus RehydratePendingToHot;
/** Constant value of type ArchiveStatus: RehydratePendingToCool */
AZ_STORAGE_BLOBS_DLLEXPORT const static ArchiveStatus RehydratePendingToCool;
/** Constant value of type ArchiveStatus: RehydratePendingToCold */
AZ_STORAGE_BLOBS_DLLEXPORT const static ArchiveStatus RehydratePendingToCold;
private:
std::string m_value;

1
sdk/storage/azure-storage-blobs/src/rest_client.cpp
View File

@ -154,6 +154,7 @@ namespace Azure { namespace Storage { namespace Blobs {
const AccessTier AccessTier::Cold("Cold");
const ArchiveStatus ArchiveStatus::RehydratePendingToHot("rehydrate-pending-to-hot");
const ArchiveStatus ArchiveStatus::RehydratePendingToCool("rehydrate-pending-to-cool");
const ArchiveStatus ArchiveStatus::RehydratePendingToCold("rehydrate-pending-to-cold");
const RehydratePriority RehydratePriority::High("High");
const RehydratePriority RehydratePriority::Standard("Standard");
const ObjectReplicationStatus ObjectReplicationStatus::Complete("complete");

21
sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp
View File

@ -818,6 +818,27 @@ namespace Azure { namespace Storage { namespace Test {
blobItem.Details.RehydratePriority.Value(), Blobs::Models::RehydratePriority::Standard);
}
TEST_F(BlockBlobClientTest, RehydrateTierToCold)
{
m_blockBlobClient->SetAccessTier(Blobs::Models::AccessTier::Archive);
m_blockBlobClient->SetAccessTier(Blobs::Models::AccessTier::Cold);
auto properties = m_blockBlobClient->GetProperties().Value;
ASSERT_TRUE(properties.ArchiveStatus.HasValue());
EXPECT_EQ(
properties.ArchiveStatus.Value(), Blobs::Models::ArchiveStatus::RehydratePendingToCold);
ASSERT_TRUE(properties.RehydratePriority.HasValue());
EXPECT_EQ(properties.RehydratePriority.Value(), Blobs::Models::RehydratePriority::Standard);
auto blobItem = GetBlobItem(m_blobName);
ASSERT_TRUE(blobItem.Details.ArchiveStatus.HasValue());
EXPECT_EQ(
blobItem.Details.ArchiveStatus.Value(),
Blobs::Models::ArchiveStatus::RehydratePendingToCold);
ASSERT_TRUE(blobItem.Details.RehydratePriority.HasValue());
EXPECT_EQ(
blobItem.Details.RehydratePriority.Value(), Blobs::Models::RehydratePriority::Standard);
}
TEST_F(BlockBlobClientTest, SetTierCold)
{
m_blockBlobClient->SetAccessTier(Blobs::Models::AccessTier::Cold);

4
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp
View File

@ -237,12 +237,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Azure::Core::Url directoryUrl,
Blobs::BlobClient blobClient,
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
_detail::DatalakeClientConfiguration clientConfiguration)
: DataLakePathClient(
std::move(directoryUrl),
std::move(blobClient),
pipeline,
std::move(customerProvidedKey))
std::move(clientConfiguration))
{
}

4
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp
View File

@ -286,12 +286,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Azure::Core::Url fileUrl,
Blobs::BlobClient blobClient,
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
_detail::DatalakeClientConfiguration clientConfiguration)
: DataLakePathClient(
std::move(fileUrl),
std::move(blobClient),
pipeline,
std::move(customerProvidedKey))
std::move(clientConfiguration))
{
}

6
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp
View File

@ -273,16 +273,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Azure::Core::Url m_fileSystemUrl;
Blobs::BlobContainerClient m_blobContainerClient;
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> m_pipeline;
Azure::Nullable<EncryptionKey> m_customerProvidedKey;
_detail::DatalakeClientConfiguration m_clientConfiguration;
explicit DataLakeFileSystemClient(
Azure::Core::Url fileSystemUrl,
Blobs::BlobContainerClient blobContainerClient,
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
_detail::DatalakeClientConfiguration clientConfiguration)
: m_fileSystemUrl(std::move(fileSystemUrl)),
m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline)),
m_customerProvidedKey(std::move(customerProvidedKey))
m_clientConfiguration(std::move(clientConfiguration))
{
}
friend class DataLakeLeaseClient;

21
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp
View File

@ -86,6 +86,27 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
using SetServicePropertiesOptions = Blobs::SetServicePropertiesOptions;
using EncryptionKey = Blobs::EncryptionKey;
namespace _detail {
struct DatalakeClientConfiguration
{
/**
* API version used by this client.
*/
std::string ApiVersion;
/**
* @brief The token credential used to initialize the client.
*/
std::shared_ptr<Core::Credentials::TokenCredential> TokenCredential;
/**
* @brief Holds the customer provided key used when making requests.
*/
Azure::Nullable<EncryptionKey> CustomerProvidedKey;
};
} // namespace _detail
/**
* @brief Client options used to initialize all DataLake clients.
*/

11
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp
View File

@ -306,8 +306,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
/** @brief Http Pipeline */
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> m_pipeline;
/** @brief Customer provided encryption key. */
Azure::Nullable<EncryptionKey> m_customerProvidedKey;
/** @brief Client configurations*/
_detail::DatalakeClientConfiguration m_clientConfiguration;
/**
* @brief Construct a new DataLakePathClient
@ -315,16 +316,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* @param pathUrl The URL of the path represented by this client.
* @param blobClient The BlobClient needed for blob operations performed on this path.
* @param pipeline The HTTP pipeline for sending and receiving REST requests and responses.
* @param customerProvidedKey Customer provided key to encrypt the data.
* @param clientConfiguration Client configurations
*
*/
explicit DataLakePathClient(
Azure::Core::Url pathUrl,
Blobs::BlobClient blobClient,
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
_detail::DatalakeClientConfiguration clientConfiguration)
: m_pathUrl(std::move(pathUrl)), m_blobClient(std::move(blobClient)),
m_pipeline(std::move(pipeline)), m_customerProvidedKey(std::move(customerProvidedKey))
m_pipeline(std::move(pipeline)), m_clientConfiguration(std::move(clientConfiguration))
{
}

2
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp
View File

@ -150,6 +150,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Azure::Core::Url m_serviceUrl;
Blobs::BlobServiceClient m_blobServiceClient;
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> m_pipeline;
Azure::Nullable<EncryptionKey> m_customerProvidedKey;
_detail::DatalakeClientConfiguration m_clientConfiguration;
};
}}}} // namespace Azure::Storage::Files::DataLake

3
sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp
View File

@ -26,7 +26,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
/**
* The version used for the operations to Azure storage services.
*/
constexpr static const char* ApiVersion = "2021-06-08";
constexpr static const char* ApiVersion = "2023-08-03";
} // namespace _detail
namespace Models {
namespace _detail {
@ -475,6 +475,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
ETag IfNoneMatch;
Nullable<DateTime> IfModifiedSince;
Nullable<DateTime> IfUnmodifiedSince;
Nullable<bool> Paginated;
};
static Response<Models::DeletePathResult> Delete(
Core::Http::_internal::HttpPipeline& pipeline,

16
sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp
View File

@ -67,7 +67,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
auto blobClient = m_blobClient;
blobClient.m_blobUrl.AppendPath(_internal::UrlEncodePath(fileName));
return DataLakeFileClient(
std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey);
std::move(builder), std::move(blobClient), m_pipeline, m_clientConfiguration);
}
DataLakeDirectoryClient DataLakeDirectoryClient::GetSubdirectoryClient(
@ -78,7 +78,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
auto blobClient = m_blobClient;
blobClient.m_blobUrl.AppendPath(_internal::UrlEncodePath(subdirectoryName));
return DataLakeDirectoryClient(
std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey);
std::move(builder), std::move(blobClient), m_pipeline, m_clientConfiguration);
}
Azure::Response<DataLakeFileClient> DataLakeDirectoryClient::RenameFile(
@ -122,12 +122,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
*m_pipeline, destinationDfsUrl, protocolLayerOptions, context);
auto renamedBlobClient = Blobs::BlobClient(
_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey);
_detail::GetBlobUrlFromUrl(destinationDfsUrl),
m_pipeline,
m_clientConfiguration.CustomerProvidedKey);
auto renamedFileClient = DataLakeFileClient(
std::move(destinationDfsUrl),
std::move(renamedBlobClient),
m_pipeline,
m_customerProvidedKey);
m_clientConfiguration);
return Azure::Response<DataLakeFileClient>(
std::move(renamedFileClient), std::move(response.RawResponse));
}
@ -173,12 +175,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
*m_pipeline, destinationDfsUrl, protocolLayerOptions, context);
auto renamedBlobClient = Blobs::BlobClient(
_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey);
_detail::GetBlobUrlFromUrl(destinationDfsUrl),
m_pipeline,
m_clientConfiguration.CustomerProvidedKey);
auto renamedDirectoryClient = DataLakeDirectoryClient(
std::move(destinationDfsUrl),
std::move(renamedBlobClient),
m_pipeline,
m_customerProvidedKey);
m_clientConfiguration);
return Azure::Response<DataLakeDirectoryClient>(
std::move(renamedDirectoryClient), std::move(response.RawResponse));
}

20
sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp
View File

@ -82,11 +82,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
}
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.Flush = options.Flush;
if (m_customerProvidedKey.HasValue())
if (m_clientConfiguration.CustomerProvidedKey.HasValue())
{
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
protocolLayerOptions.EncryptionKey = m_clientConfiguration.CustomerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256
= m_clientConfiguration.CustomerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm
= m_clientConfiguration.CustomerProvidedKey.Value().Algorithm.ToString();
}
protocolLayerOptions.LeaseAction = options.LeaseAction;
protocolLayerOptions.ProposedLeaseId = options.LeaseId;
@ -124,11 +126,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
if (m_customerProvidedKey.HasValue())
if (m_clientConfiguration.CustomerProvidedKey.HasValue())
{
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
protocolLayerOptions.EncryptionKey = m_clientConfiguration.CustomerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256
= m_clientConfiguration.CustomerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm
= m_clientConfiguration.CustomerProvidedKey.Value().Algorithm.ToString();
}
protocolLayerOptions.LeaseAction = options.LeaseAction;
protocolLayerOptions.ProposedLeaseId = options.LeaseId;

40
sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp
View File

@ -50,9 +50,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
: m_fileSystemUrl(fileSystemUrl), m_blobContainerClient(
_detail::GetBlobUrlFromUrl(fileSystemUrl),
credential,
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
_detail::GetBlobClientOptions(options))
{
m_clientConfiguration.ApiVersion
= options.ApiVersion.empty() ? _detail::ApiVersion : options.ApiVersion;
m_clientConfiguration.CustomerProvidedKey = options.CustomerProvidedKey;
DataLakeClientOptions newOptions = options;
newOptions.PerRetryPolicies.emplace_back(
std::make_unique<_internal::SharedKeyPolicy>(credential));
@ -79,9 +82,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
: m_fileSystemUrl(fileSystemUrl), m_blobContainerClient(
_detail::GetBlobUrlFromUrl(fileSystemUrl),
credential,
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
_detail::GetBlobClientOptions(options))
{
m_clientConfiguration.ApiVersion
= options.ApiVersion.empty() ? _detail::ApiVersion : options.ApiVersion;
m_clientConfiguration.TokenCredential = credential;
m_clientConfiguration.CustomerProvidedKey = options.CustomerProvidedKey;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
perRetryPolicies.emplace_back(std::make_unique<_internal::StorageSwitchToSecondaryPolicy>(
@ -109,9 +116,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
const DataLakeClientOptions& options)
: m_fileSystemUrl(fileSystemUrl), m_blobContainerClient(
_detail::GetBlobUrlFromUrl(fileSystemUrl),
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
_detail::GetBlobClientOptions(options))
{
m_clientConfiguration.ApiVersion
= options.ApiVersion.empty() ? _detail::ApiVersion : options.ApiVersion;
m_clientConfiguration.CustomerProvidedKey = options.CustomerProvidedKey;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
perRetryPolicies.emplace_back(std::make_unique<_internal::StorageSwitchToSecondaryPolicy>(
@ -133,7 +143,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
builder.AppendPath(_internal::UrlEncodePath(fileName));
auto blobClient = m_blobContainerClient.GetBlobClient(fileName);
return DataLakeFileClient(
std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey);
std::move(builder), std::move(blobClient), m_pipeline, m_clientConfiguration);
}
DataLakeDirectoryClient DataLakeFileSystemClient::GetDirectoryClient(
@ -142,10 +152,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
auto builder = m_fileSystemUrl;
builder.AppendPath(_internal::UrlEncodePath(directoryName));
return DataLakeDirectoryClient(
builder,
std::move(builder),
m_blobContainerClient.GetBlobClient(directoryName),
m_pipeline,
m_customerProvidedKey);
m_clientConfiguration);
}
Azure::Response<Models::CreateFileSystemResult> DataLakeFileSystemClient::Create(
@ -392,12 +402,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
*m_pipeline, destinationDfsUrl, protocolLayerOptions, context);
auto renamedBlobClient = Blobs::BlobClient(
_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey);
_detail::GetBlobUrlFromUrl(destinationDfsUrl),
m_pipeline,
m_clientConfiguration.CustomerProvidedKey);
auto renamedFileClient = DataLakeFileClient(
std::move(destinationDfsUrl),
std::move(renamedBlobClient),
m_pipeline,
m_customerProvidedKey);
m_clientConfiguration);
return Azure::Response<DataLakeFileClient>(
std::move(renamedFileClient), std::move(result.RawResponse));
}
@ -443,12 +455,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
*m_pipeline, destinationDfsUrl, protocolLayerOptions, context);
auto renamedBlobClient = Blobs::BlobClient(
_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey);
_detail::GetBlobUrlFromUrl(destinationDfsUrl),
m_pipeline,
m_clientConfiguration.CustomerProvidedKey);
auto renamedDirectoryClient = DataLakeDirectoryClient(
std::move(destinationDfsUrl),
std::move(renamedBlobClient),
m_pipeline,
m_customerProvidedKey);
m_clientConfiguration);
return Azure::Response<DataLakeDirectoryClient>(
std::move(renamedDirectoryClient), std::move(result.RawResponse));
}

64
sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp
View File

@ -48,9 +48,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
: m_pathUrl(pathUrl), m_blobClient(
_detail::GetBlobUrlFromUrl(pathUrl),
credential,
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
_detail::GetBlobClientOptions(options))
{
m_clientConfiguration.ApiVersion
= options.ApiVersion.empty() ? _detail::ApiVersion : options.ApiVersion;
m_clientConfiguration.CustomerProvidedKey = options.CustomerProvidedKey;
DataLakeClientOptions newOptions = options;
newOptions.PerRetryPolicies.emplace_back(
std::make_unique<_internal::SharedKeyPolicy>(credential));
@ -77,9 +80,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
: m_pathUrl(pathUrl), m_blobClient(
_detail::GetBlobUrlFromUrl(pathUrl),
credential,
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
_detail::GetBlobClientOptions(options))
{
m_clientConfiguration.ApiVersion
= options.ApiVersion.empty() ? _detail::ApiVersion : options.ApiVersion;
m_clientConfiguration.TokenCredential = credential;
m_clientConfiguration.CustomerProvidedKey = options.CustomerProvidedKey;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
perRetryPolicies.emplace_back(std::make_unique<_internal::StorageSwitchToSecondaryPolicy>(
@ -106,9 +113,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
const std::string& pathUrl,
const DataLakeClientOptions& options)
: m_pathUrl(pathUrl),
m_blobClient(_detail::GetBlobUrlFromUrl(pathUrl), _detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
m_blobClient(_detail::GetBlobUrlFromUrl(pathUrl), _detail::GetBlobClientOptions(options))
{
m_clientConfiguration.ApiVersion
= options.ApiVersion.empty() ? _detail::ApiVersion : options.ApiVersion;
m_clientConfiguration.CustomerProvidedKey = options.CustomerProvidedKey;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
perRetryPolicies.emplace_back(std::make_unique<_internal::StorageSwitchToSecondaryPolicy>(
@ -235,11 +245,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
protocolLayerOptions.ExpiresOn
= std::to_string(options.ScheduleDeletionOptions.TimeToExpire.Value().count());
}
if (m_customerProvidedKey.HasValue())
if (m_clientConfiguration.CustomerProvidedKey.HasValue())
{
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
protocolLayerOptions.EncryptionKey = m_clientConfiguration.CustomerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256
= m_clientConfiguration.CustomerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm
= m_clientConfiguration.CustomerProvidedKey.Value().Algorithm.ToString();
}
return _detail::PathClient::Create(*m_pipeline, m_pathUrl, protocolLayerOptions, context);
}
@ -271,14 +283,30 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
const DeletePathOptions& options,
const Azure::Core::Context& context) const
{
_detail::PathClient::DeletePathOptions protocolLayerOptions;
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
protocolLayerOptions.Recursive = options.Recursive;
return _detail::PathClient::Delete(*m_pipeline, m_pathUrl, protocolLayerOptions, context);
bool paginated = m_clientConfiguration.ApiVersion >= "2023-08-03"
&& m_clientConfiguration.TokenCredential && options.Recursive.HasValue()
&& options.Recursive.Value();
std::string continuationToken;
while (true)
{
_detail::PathClient::DeletePathOptions protocolLayerOptions;
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
protocolLayerOptions.Recursive = options.Recursive;
protocolLayerOptions.ContinuationToken = continuationToken;
protocolLayerOptions.Paginated = paginated;
auto response
= _detail::PathClient::Delete(*m_pipeline, m_pathUrl, protocolLayerOptions, context);
continuationToken = Azure::Core::Http::_internal::HttpShared::GetHeaderOrEmptyString(
response.RawResponse->GetHeaders(), "x-ms-continuation");
if (continuationToken.empty())
{
return response;
}
}
}
Azure::Response<Models::DeletePathResult> DataLakePathClient::DeleteIfExists(

24
sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp
View File

@ -44,9 +44,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
: m_serviceUrl(serviceUrl), m_blobServiceClient(
_detail::GetBlobUrlFromUrl(serviceUrl),
credential,
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
_detail::GetBlobClientOptions(options))
{
m_clientConfiguration.ApiVersion
= options.ApiVersion.empty() ? _detail::ApiVersion : options.ApiVersion;
m_clientConfiguration.CustomerProvidedKey = options.CustomerProvidedKey;
DataLakeClientOptions newOptions = options;
newOptions.PerRetryPolicies.emplace_back(
std::make_unique<_internal::SharedKeyPolicy>(credential));
@ -73,9 +76,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
: m_serviceUrl(serviceUrl), m_blobServiceClient(
_detail::GetBlobUrlFromUrl(serviceUrl),
credential,
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
_detail::GetBlobClientOptions(options))
{
m_clientConfiguration.ApiVersion
= options.ApiVersion.empty() ? _detail::ApiVersion : options.ApiVersion;
m_clientConfiguration.TokenCredential = credential;
m_clientConfiguration.CustomerProvidedKey = options.CustomerProvidedKey;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
perRetryPolicies.emplace_back(std::make_unique<_internal::StorageSwitchToSecondaryPolicy>(
@ -103,9 +110,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
const DataLakeClientOptions& options)
: m_serviceUrl(serviceUrl), m_blobServiceClient(
_detail::GetBlobUrlFromUrl(serviceUrl),
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
_detail::GetBlobClientOptions(options))
{
m_clientConfiguration.ApiVersion
= options.ApiVersion.empty() ? _detail::ApiVersion : options.ApiVersion;
m_clientConfiguration.CustomerProvidedKey = options.CustomerProvidedKey;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
perRetryPolicies.emplace_back(std::make_unique<_internal::StorageSwitchToSecondaryPolicy>(
@ -130,7 +140,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
builder,
m_blobServiceClient.GetBlobContainerClient(fileSystemName),
m_pipeline,
m_customerProvidedKey);
m_clientConfiguration);
}
ListFileSystemsPagedResponse DataLakeServiceClient::ListFileSystems(

26
sdk/storage/azure-storage-files-datalake/src/rest_client.cpp
View File

@ -61,7 +61,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.GetUrl().AppendQueryParameter("timeout", std::to_string(options.Timeout.Value()));
}
request.SetHeader("x-ms-version", "2021-06-08");
request.SetHeader("x-ms-version", "2023-08-03");
if (options.ContinuationToken.HasValue() && !options.ContinuationToken.Value().empty())
{
request.GetUrl().AppendQueryParameter(
@ -157,7 +157,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.GetUrl().AppendQueryParameter("timeout", std::to_string(options.Timeout.Value()));
}
request.SetHeader("x-ms-version", "2021-06-08");
request.SetHeader("x-ms-version", "2023-08-03");
if (options.Resource.HasValue() && !options.Resource.Value().ToString().empty())
{
request.GetUrl().AppendQueryParameter(
@ -345,7 +345,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.GetUrl().AppendQueryParameter("timeout", std::to_string(options.Timeout.Value()));
}
request.SetHeader("x-ms-version", "2021-06-08");
request.SetHeader("x-ms-version", "2023-08-03");
if (options.Recursive.HasValue())
{
request.GetUrl().AppendQueryParameter(
@ -380,9 +380,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
"If-Unmodified-Since",
options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123));
}
if (options.Paginated.HasValue())
{
request.GetUrl().AppendQueryParameter(
"paginated", options.Paginated.Value() ? "true" : "false");
}
auto pRawResponse = pipeline.Send(request, context);
auto httpStatusCode = pRawResponse->GetStatusCode();
if (httpStatusCode != Core::Http::HttpStatusCode::Ok)
if (!(httpStatusCode == Core::Http::HttpStatusCode::Ok
|| httpStatusCode == Core::Http::HttpStatusCode::Accepted))
{
throw StorageException::CreateFromResponse(std::move(pRawResponse));
}
@ -437,7 +443,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
"If-Unmodified-Since",
options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123));
}
request.SetHeader("x-ms-version", "2021-06-08");
request.SetHeader("x-ms-version", "2023-08-03");
auto pRawResponse = pipeline.Send(request, context);
auto httpStatusCode = pRawResponse->GetStatusCode();
if (httpStatusCode != Core::Http::HttpStatusCode::Ok)
@ -484,7 +490,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.SetHeader("x-ms-acl", options.Acl.Value());
}
request.SetHeader("x-ms-version", "2021-06-08");
request.SetHeader("x-ms-version", "2023-08-03");
auto pRawResponse = pipeline.Send(request, context);
auto httpStatusCode = pRawResponse->GetStatusCode();
if (httpStatusCode != Core::Http::HttpStatusCode::Ok)
@ -537,7 +543,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.SetHeader("x-ms-undelete-source", options.UndeleteSource.Value());
}
request.SetHeader("x-ms-version", "2021-06-08");
request.SetHeader("x-ms-version", "2023-08-03");
auto pRawResponse = pipeline.Send(request, context);
auto httpStatusCode = pRawResponse->GetStatusCode();
if (httpStatusCode != Core::Http::HttpStatusCode::Ok)
@ -588,7 +594,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
"If-Unmodified-Since",
options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123));
}
request.SetHeader("x-ms-version", "2021-06-08");
request.SetHeader("x-ms-version", "2023-08-03");
auto pRawResponse = pipeline.Send(request, context);
auto httpStatusCode = pRawResponse->GetStatusCode();
if (httpStatusCode != Core::Http::HttpStatusCode::Ok)
@ -687,7 +693,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
"If-Unmodified-Since",
options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123));
}
request.SetHeader("x-ms-version", "2021-06-08");
request.SetHeader("x-ms-version", "2023-08-03");
if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty())
{
request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value());
@ -771,7 +777,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value());
}
request.SetHeader("x-ms-version", "2021-06-08");
request.SetHeader("x-ms-version", "2023-08-03");
if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty())
{
request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value());

22
sdk/storage/azure-storage-files-datalake/swagger/README.md
View File

@ -9,7 +9,7 @@ package-name: azure-storage-files-datalake
namespace: Azure::Storage::Files::DataLake
output-folder: generated
clear-output-folder: true
input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/storage/data-plane/Azure.Storage.Files.DataLake/preview/2021-06-08/DataLakeStorage.json
input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/storage/data-plane/Azure.Storage.Files.DataLake/preview/2023-05-03/DataLakeStorage.json
```
## ModelFour Options
@ -88,12 +88,12 @@ directive:
"name": "ApiVersion",
"modelAsString": false
},
"enum": ["2021-06-08"]
"enum": ["2023-08-03"]
};
- from: swagger-document
where: $.parameters
transform: >
$.ApiVersionParameter.enum[0] = "2021-06-08";
$.ApiVersionParameter.enum[0] = "2023-08-03";
```
### Rename Operations
@ -329,16 +329,18 @@ directive:
- from: swagger-document
where: $["x-ms-paths"]["/{filesystem}/{path}"].delete.responses
transform: >
delete $["200"].headers["x-ms-continuation"];
delete $["200"].headers["x-ms-deletion-id"];
$["200"].schema = {
for (const status_code of ["200", "202"]) {
delete $[status_code].headers["x-ms-continuation"];
delete $[status_code].headers["x-ms-deletion-id"];
$[status_code].schema = {
"type": "object",
"x-ms-client-name": "DeletePathResult",
"x-ms-sealed": false,
"properties": {
"Deleted": {"type": "boolean", "x-ms-client-default": true, "x-ms-json": ""}
}
};
"Deleted": {"type": "boolean", "x-ms-client-default": true, "x-ms-json": ""}
}
};
}
```
### RenamePath
@ -513,4 +515,6 @@ directive:
transform: >
$["200"].schema.properties["Deleted"].description = "Indicates if the file or directory was successfully deleted by this operation.";
$["200"].schema.description = "Response type for #Azure::Storage::Files::DataLake::DataLakePathClient::Delete.";
$["202"].schema.properties["Deleted"].description = "Indicates if the file or directory was successfully deleted by this operation.";
$["202"].schema.description = "Response type for #Azure::Storage::Files::DataLake::DataLakePathClient::Delete.";
```

27
sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp
View File

@ -101,6 +101,33 @@ namespace Azure { namespace Storage { namespace Test {
}
}
TEST_F(DataLakeDirectoryClientTest, OAuthDelete)
{
const std::string baseName = RandomString();
auto oauthFileSystemClient = Files::DataLake::DataLakeFileSystemClient(
Files::DataLake::_detail::GetDfsUrlFromUrl(m_fileSystemClient->GetUrl()),
std::make_shared<Azure::Identity::ClientSecretCredential>(
AadTenantId(), AadClientId(), AadClientSecret(), GetTokenCredentialOptions()),
InitStorageClientOptions<Files::DataLake::DataLakeClientOptions>());
// Delete empty
auto emptyDir = baseName + "OAuthEmptyDir";
auto emptyDirClient = oauthFileSystemClient.GetDirectoryClient(emptyDir);
EXPECT_NO_THROW(emptyDirClient.Create());
EXPECT_NO_THROW(emptyDirClient.DeleteEmpty());
// Recursive delete
auto rootDir = baseName + "OAuthRoot";
auto rootDirClient = oauthFileSystemClient.GetDirectoryClient(rootDir);
EXPECT_NO_THROW(rootDirClient.Create());
for (int32_t i = 0; i < 5; ++i)
{
auto client = m_fileSystemClient->GetDirectoryClient(rootDir + "/d" + std::to_string(i));
EXPECT_NO_THROW(client.Create());
}
EXPECT_THROW(rootDirClient.DeleteEmpty(), StorageException);
EXPECT_NO_THROW(rootDirClient.DeleteRecursive());
}
TEST_F(DataLakeDirectoryClientTest, CreateDeleteIfExistsDirectory)
{
std::string const baseName = RandomString();

13
sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp
View File

@ -99,6 +99,19 @@ namespace Azure { namespace Storage { namespace Test {
}
}
TEST_F(DataLakeFileClientTest, OAuthDelete)
{
auto oauthFileSystemClient = Files::DataLake::DataLakeFileSystemClient(
Files::DataLake::_detail::GetDfsUrlFromUrl(m_fileSystemClient->GetUrl()),
std::make_shared<Azure::Identity::ClientSecretCredential>(
AadTenantId(), AadClientId(), AadClientSecret(), GetTokenCredentialOptions()),
InitStorageClientOptions<Files::DataLake::DataLakeClientOptions>());
const std::string oauthFileName = RandomString();
auto oauthFileClient = oauthFileSystemClient.GetFileClient(oauthFileName);
EXPECT_NO_THROW(oauthFileClient.Create());
EXPECT_NO_THROW(oauthFileClient.Delete());
}
TEST_F(DataLakeFileClientTest, CreateDeleteIfExistsFiles)
{
{

50
sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp
View File

@ -6,6 +6,7 @@
#include <azure/identity/client_secret_credential.hpp>
#include <algorithm>
#include <future>
#include <thread>
namespace Azure { namespace Storage { namespace Test {
@ -274,6 +275,55 @@ namespace Azure { namespace Storage { namespace Test {
}
}
TEST_F(DataLakePathClientTest, DISABLED_PaginationDelete)
{
// This test should be tested locally because it needs an AAD app that has no RBAC permissions
// to do the ACL check.
const std::string tenantId = AadTenantId();
const std::string appId = AadClientId();
const std::string appSecret = AadClientSecret();
// Create resource
std::string directoryName = RandomString();
auto directoryClient = m_fileSystemClient->GetDirectoryClient(directoryName);
directoryClient.Create();
// Concurrent create 5000+ files
std::vector<std::future<void>> futures;
for (int i = 0; i < 50; ++i)
{
futures.emplace_back(std::async(std::launch::async, [&]() {
for (int i = 0; i < 101; ++i)
{
directoryClient.GetFileClient(RandomString()).Create();
}
}));
}
for (auto& f : futures)
{
f.get();
}
// Set Acls
auto rootDirClient = m_fileSystemClient->GetDirectoryClient("");
rootDirClient.SetPermissions("rwxrwxrwx");
auto aclResult = rootDirClient.GetAccessControlList();
auto acls = aclResult.Value.Acls;
Files::DataLake::Models::Acl acl;
acl.Permissions = "rwx";
acl.Id = appId;
acl.Type = "user";
acls.emplace_back(acl);
rootDirClient.SetAccessControlListRecursive(acls);
// Pagination delete
Files::DataLake::DataLakePathClient oauthDirectoryClient(
Files::DataLake::_detail::GetDfsUrlFromUrl(directoryClient.GetUrl()),
std::make_shared<Azure::Identity::ClientSecretCredential>(tenantId, appId, appSecret));
Files::DataLake::DeletePathOptions options;
options.Recursive = true;
EXPECT_NO_THROW(oauthDirectoryClient.Delete(options));
}
TEST_F(DataLakePathClientTest, PathAccessControls)
{
{