github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Upload/Create is non-overwriting by default (#745)

Browse Source 
* add overwrite option

* fix failed ut

* add ut
This commit is contained in:
JinmingHu 2020-10-13 10:41:15 +08:00 committed by GitHub
parent b6032bfe54
commit 5d1a8f4cbc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 117 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp
View File

@ -930,6 +930,11 @@ namespace Azure { namespace Storage { namespace Blobs {
*/
Azure::Core::Context Context;
/**
* @brief Whether the upload should overwrite any existing blobs.
*/
bool Overwrite = false;
/**
* @brief An MD5 hash of the blob content. This hash is used to verify the integrity of
* the blob during transport. When this header is specified, the storage service checks the hash
@ -1144,6 +1149,11 @@ namespace Azure { namespace Storage { namespace Blobs {
*/
Azure::Core::Context Context;
/**
* @brief Whether the existing blob should be deleted and recreated.
*/
bool Overwrite = false;
/**
* @brief The standard HTTP header system properties to set.
*/
@ -1257,6 +1267,11 @@ namespace Azure { namespace Storage { namespace Blobs {
*/
Azure::Core::Context Context;
/**
* @brief Whether the existing blob should be deleted and recreated.
*/
bool Overwrite = false;
/**
* @brief The sequence number is a user-controlled value that you can use to track requests. The
* value of the sequence number must be between 0 and 2^63 - 1.

4
sdk/storage/azure-storage-blobs/src/append_blob_client.cpp
View File

@ -85,6 +85,10 @@ namespace Azure { namespace Storage { namespace Blobs {
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
if (!options.Overwrite)
{
protocolLayerOptions.IfNoneMatch = c_ETagWildcard;
}
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
if (m_customerProvidedKey.HasValue())
{

4
sdk/storage/azure-storage-blobs/src/block_blob_client.cpp
View File

@ -92,6 +92,10 @@ namespace Azure { namespace Storage { namespace Blobs {
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
if (!options.Overwrite)
{
protocolLayerOptions.IfNoneMatch = c_ETagWildcard;
}
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
if (m_customerProvidedKey.HasValue())
{

4
sdk/storage/azure-storage-blobs/src/page_blob_client.cpp
View File

@ -89,6 +89,10 @@ namespace Azure { namespace Storage { namespace Blobs {
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
if (!options.Overwrite)
{
protocolLayerOptions.IfNoneMatch = c_ETagWildcard;
}
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
if (m_customerProvidedKey.HasValue())
{

11
sdk/storage/azure-storage-blobs/test/append_blob_client_test.cpp
View File

@ -330,4 +330,15 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_TRUE(getPropertiesResult->IsSealed.GetValue());
}
TEST_F(AppendBlobClientTest, Overwrite)
{
std::string blobName = RandomString();
auto blobClient = m_blobContainerClient->GetAppendBlobClient(blobName);
EXPECT_NO_THROW(blobClient.Create());
EXPECT_THROW(blobClient.Create(), StorageError);
Blobs::CreateAppendBlobOptions options;
options.Overwrite = true;
EXPECT_NO_THROW(blobClient.Create(options));
}
}}} // namespace Azure::Storage::Test

52
sdk/storage/azure-storage-blobs/test/blob_container_client_test.cpp
View File

@ -456,12 +456,14 @@ namespace Azure { namespace Storage { namespace Test {
createOptions.PreventEncryptionScopeOverride.GetValue());
auto appendBlobClient = containerClient.GetAppendBlobClient(blobName);
auto blobContentInfo = appendBlobClient.Create();
appendBlobClient.Delete();
EXPECT_TRUE(blobContentInfo->EncryptionScope.HasValue());
EXPECT_EQ(blobContentInfo->EncryptionScope.GetValue(), c_TestEncryptionScope);
auto appendBlobClientWithoutEncryptionScope
= Azure::Storage::Blobs::AppendBlobClient::CreateFromConnectionString(
StandardStorageConnectionString(), containerName, blobName);
blobContentInfo = appendBlobClientWithoutEncryptionScope.Create();
appendBlobClientWithoutEncryptionScope.Delete();
EXPECT_TRUE(blobContentInfo->EncryptionScope.HasValue());
EXPECT_EQ(blobContentInfo->EncryptionScope.GetValue(), c_TestEncryptionScope);
containerClient.Delete();
@ -488,6 +490,7 @@ namespace Azure { namespace Storage { namespace Test {
StandardStorageConnectionString(), m_containerName, blobName);
EXPECT_THROW(appendBlobClientWithoutEncryptionScope.AppendBlock(&bodyStream), StorageError);
EXPECT_THROW(appendBlobClientWithoutEncryptionScope.CreateSnapshot(), StorageError);
appendBlobClient.Delete();
}
}
@ -774,29 +777,41 @@ namespace Azure { namespace Storage { namespace Test {
StandardStorageConnectionString());
std::string whereExpression
= c1 + " = '" + v1 + "' AND " + c2 + " >= '" + v2 + "' AND " + c3 + " <= '" + v3 + "'";
std::string marker;
std::vector<Blobs::FilterBlobItem> findResults;
do
for (int i = 0; i < 30; ++i)
{
Blobs::FindBlobsByTagsOptions options;
if (!marker.empty())
std::string marker;
do
{
options.Marker = marker;
}
auto findBlobsRet = *blobServiceClient.FindBlobsByTags(whereExpression, options);
EXPECT_FALSE(findBlobsRet.ServiceEndpoint.empty());
EXPECT_EQ(findBlobsRet.Where, whereExpression);
options.Marker = findBlobsRet.NextMarker;
Blobs::FindBlobsByTagsOptions options;
if (!marker.empty())
{
options.Marker = marker;
}
auto findBlobsRet = *blobServiceClient.FindBlobsByTags(whereExpression, options);
EXPECT_FALSE(findBlobsRet.ServiceEndpoint.empty());
EXPECT_EQ(findBlobsRet.Where, whereExpression);
options.Marker = findBlobsRet.NextMarker;
for (auto& i : findBlobsRet.Items)
for (auto& item : findBlobsRet.Items)
{
EXPECT_FALSE(item.BlobName.empty());
EXPECT_FALSE(item.ContainerName.empty());
EXPECT_FALSE(item.TagValue.empty());
findResults.emplace_back(std::move(item));
}
} while (!marker.empty());
if (findResults.empty())
{
EXPECT_FALSE(i.BlobName.empty());
EXPECT_FALSE(i.ContainerName.empty());
EXPECT_FALSE(i.TagValue.empty());
findResults.emplace_back(std::move(i));
std::this_thread::sleep_for(std::chrono::seconds(1));
}
} while (!marker.empty());
EXPECT_FALSE(findResults.empty());
else
{
break;
}
}
ASSERT_FALSE(findResults.empty());
EXPECT_EQ(findResults[0].BlobName, blobName);
EXPECT_EQ(findResults[0].ContainerName, m_containerName);
EXPECT_FALSE(findResults[0].TagValue.empty());
@ -868,6 +883,7 @@ namespace Azure { namespace Storage { namespace Test {
options.AccessConditions.TagConditions = failWhereExpression;
EXPECT_THROW(appendBlobClient.Create(options), StorageError);
options.AccessConditions.TagConditions = successWhereExpression;
options.Overwrite = true;
EXPECT_NO_THROW(appendBlobClient.Create(options));
appendBlobClient.SetTags(tags);
}
@ -943,6 +959,7 @@ namespace Azure { namespace Storage { namespace Test {
options.AccessConditions.TagConditions = failWhereExpression;
EXPECT_THROW(pageBlobClient.Create(contentSize, options), StorageError);
options.AccessConditions.TagConditions = successWhereExpression;
options.Overwrite = true;
EXPECT_NO_THROW(pageBlobClient.Create(contentSize, options));
pageBlobClient.SetTags(tags);
@ -1004,6 +1021,7 @@ namespace Azure { namespace Storage { namespace Test {
content.Rewind();
EXPECT_THROW(blockBlobClient.Upload(&content, options), StorageError);
options.AccessConditions.TagConditions = successWhereExpression;
options.Overwrite = true;
content.Rewind();
EXPECT_NO_THROW(blockBlobClient.Upload(&content, options));
blockBlobClient.SetTags(tags);

16
sdk/storage/azure-storage-blobs/test/blob_sas_test.cpp
View File

@ -56,11 +56,13 @@ namespace Azure { namespace Storage { namespace Test {
auto blobClient = Blobs::AppendBlobClient(blobUri + sas);
auto downloadedContent = blobClient.Download();
EXPECT_TRUE(ReadBodyStream(downloadedContent->BodyStream).empty());
blobClient0.Delete();
};
auto verify_blob_write = [&](const std::string& sas) {
auto blobClient = Blobs::AppendBlobClient(blobUri + sas);
EXPECT_NO_THROW(blobClient.Create());
blobClient0.Delete();
};
auto verify_blob_delete = [&](const std::string& sas) {
@ -76,6 +78,7 @@ namespace Azure { namespace Storage { namespace Test {
reinterpret_cast<const uint8_t*>(content.data()), content.size());
auto blobClient = Blobs::AppendBlobClient(blobUri + sas);
EXPECT_NO_THROW(blobClient.AppendBlock(&blockContent));
blobClient0.Delete();
};
auto verify_blob_list = [&](const std::string& sas) {
@ -105,6 +108,7 @@ namespace Azure { namespace Storage { namespace Test {
blobClient0.SetTags(tags);
auto blobClient = Blobs::AppendBlobClient(blobUri + sas);
EXPECT_NO_THROW(blobClient.GetTags());
blobClient0.Delete();
};
auto verify_blob_filter = [&](const std::string& sas) {
@ -414,6 +418,7 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_EQ(p->HttpHeaders.ContentDisposition, headers.ContentDisposition);
EXPECT_EQ(p->HttpHeaders.CacheControl, headers.CacheControl);
EXPECT_EQ(p->HttpHeaders.ContentEncoding, headers.ContentEncoding);
blobClient0.Delete();
}
blobClient0.Create();
@ -470,6 +475,12 @@ namespace Azure { namespace Storage { namespace Test {
}
}
{
Blobs::DeleteBlobOptions options;
options.DeleteSnapshots = Blobs::DeleteSnapshotsOption::IncludeSnapshots;
blobClient0.Delete(options);
}
blobClient0.Create();
Blobs::BlobSasBuilder BlobVersionSasBuilder = blobSasBuilder;
BlobVersionSasBuilder.Resource = Blobs::BlobSasResource::BlobVersion;
@ -525,6 +536,11 @@ namespace Azure { namespace Storage { namespace Test {
verify_blob_delete_version(sasToken2);
}
}
{
Blobs::DeleteBlobOptions options;
options.DeleteSnapshots = Blobs::DeleteSnapshotsOption::IncludeSnapshots;
blobClient0.Delete(options);
}
}
}}} // namespace Azure::Storage::Test

16
sdk/storage/azure-storage-blobs/test/block_blob_client_test.cpp
View File

@ -759,4 +759,20 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_TRUE(exceptionCaught);
}
TEST_F(BlockBlobClientTest, Overwrite)
{
std::string blobName = RandomString();
std::vector<uint8_t> blobContent(1);
auto blobContentStream
= Azure::Core::Http::MemoryBodyStream(m_blobContent.data(), m_blobContent.size());
auto blobClient = m_blobContainerClient->GetBlockBlobClient(blobName);
EXPECT_NO_THROW(blobClient.Upload(&blobContentStream));
blobContentStream.Rewind();
EXPECT_THROW(blobClient.Upload(&blobContentStream), StorageError);
blobContentStream.Rewind();
Blobs::UploadBlockBlobOptions options;
options.Overwrite = true;
EXPECT_NO_THROW(blobClient.Upload(&blobContentStream, options));
}
}}} // namespace Azure::Storage::Test

11
sdk/storage/azure-storage-blobs/test/page_blob_client_test.cpp
View File

@ -251,4 +251,15 @@ namespace Azure { namespace Storage { namespace Test {
EXPECT_THROW(pageBlobClient.UploadPages(0, &pageContent, options), StorageError);
}
TEST_F(PageBlobClientTest, Overwrite)
{
std::string blobName = RandomString();
auto blobClient = m_blobContainerClient->GetPageBlobClient(blobName);
EXPECT_NO_THROW(blobClient.Create(1024));
EXPECT_THROW(blobClient.Create(1024), StorageError);
Blobs::CreatePageBlobOptions options;
options.Overwrite = true;
EXPECT_NO_THROW(blobClient.Create(1024, options));
}
}}} // namespace Azure::Storage::Test

1
sdk/storage/azure-storage-common/inc/azure/storage/common/storage_common.hpp
View File

@ -12,6 +12,7 @@ namespace Azure { namespace Storage {
constexpr int32_t c_InfiniteLeaseDuration = -1;
constexpr static const char* c_AccountEncryptionKey = "$account-encryption-key";
constexpr static const char* c_ETagWildcard = "*";
std::string CreateUniqueLeaseId();