From 5290690be0b779e0c0c7b22f10cfc4d6c99eb051 Mon Sep 17 00:00:00 2001 From: George Arama <50641385+gearama@users.noreply.github.com> Date: Mon, 7 Feb 2022 11:47:08 -0800 Subject: [PATCH] Storage src (#3304) * before sync * download latest version * blobs and datalake * shares and queues * some cleanup * for PR --- cmake-modules/FolderList.cmake | 47 ++++++++++++++----- .../azure-storage-blobs/CMakeLists.txt | 12 ++++- .../azure-storage-common/CMakeLists.txt | 14 +++++- .../CMakeLists.txt | 12 ++++- .../azure-storage-files-shares/CMakeLists.txt | 12 ++++- .../azure-storage-queues/CMakeLists.txt | 12 ++++- 6 files changed, 92 insertions(+), 17 deletions(-) diff --git a/cmake-modules/FolderList.cmake b/cmake-modules/FolderList.cmake index fd5baa32e..2c7eccccc 100644 --- a/cmake-modules/FolderList.cmake +++ b/cmake-modules/FolderList.cmake @@ -1,23 +1,33 @@ macro(GetFolderList project) message ("project found ${project}") - + message ("FLAG VALUE : ${FETCH_SOURCE_DEPS}") if(${project} STREQUAL CERTIFICATES) DownloadDepVersion(sdk/core azure-core 1.2.0) DownloadDepVersion(sdk/identity azure-identity 1.1.0) elseif(${project} STREQUAL IDENTITY) DownloadDepVersion(sdk/core azure-core 1.2.0) - endif() - - if(${project} STREQUAL SECRETS) + elseif(${project} STREQUAL SECRETS) DownloadDepVersion(sdk/core azure-core 1.2.0) DownloadDepVersion(sdk/identity azure-identity 1.1.0) - endif() - - if(${project} STREQUAL KEYS) + elseif(${project} STREQUAL KEYS) DownloadDepVersion(sdk/core azure-core 1.2.0) DownloadDepVersion(sdk/identity azure-identity 1.1.0) + elseif(${project} STREQUAL STORAGE_COMMON) + DownloadDepVersion(sdk/core azure-core 1.0.0) + elseif(${project} STREQUAL STORAGE_BLOBS) + DownloadDepVersion(sdk/core azure-core 1.0.0) + DownloadDepVersion(sdk/storage/azure-storage-common azure-storage-common 12.2.0) + elseif(${project} STREQUAL STORAGE_DATALAKE) + DownloadDepVersion(sdk/core azure-core 1.0.0) + DownloadDepVersion(sdk/storage/azure-storage-common azure-storage-common 12.2.0) + DownloadDepVersion(sdk/storage/azure-storage-blobs azure-storage-blobs 12.2.0) + elseif(${project} STREQUAL STORAGE_SHARES) + DownloadDepVersion(sdk/core azure-core 1.0.0) + DownloadDepVersion(sdk/storage/azure-storage-common azure-storage-common 12.2.0) + elseif(${project} STREQUAL STORAGE_QUEUES) + DownloadDepVersion(sdk/core azure-core 1.0.0) + DownloadDepVersion(sdk/storage/azure-storage-common azure-storage-common 12.2.0) endif() - list(REMOVE_DUPLICATES BUILD_FOLDERS) endmacro() @@ -39,7 +49,6 @@ endmacro() macro(SetCompileOptions project) message ("setting up compile options for ${project}") - # Compile Options SetGlobalOptions() endmacro() @@ -50,16 +59,30 @@ macro(DownloadDepVersion DEP_FOLDER DEP_NAME DEP_VERSION) set(DOWNLOAD_FOLDER ${CMAKE_SOURCE_DIR}/build/downloads) set(DOWNLOAD_FILE ${DEP_NAME}_${DEP_VERSION}.zip) set(DEP_PREFIX azure-sdk-for-cpp) - # get the zip - file(DOWNLOAD https://github.com/Azure/azure-sdk-for-cpp/archive/refs/tags/${DOWNLOAD_FILE} ${DOWNLOAD_FOLDER}/${DOWNLOAD_FILE}) + + if(FETCH_SOURCE_DEPS STREQUAL "LATEST") + message("Downloading latest version of ${DEP_NAME}") + #get the latest version from main + file(DOWNLOAD http://github.com/Azure/azure-sdk-for-cpp/archive/main.zip ${DOWNLOAD_FOLDER}/${DOWNLOAD_FILE}) + else() + message("Downloading version ${DEP_VERSION} of ${DEP_NAME}") + # get the zip + file(DOWNLOAD https://github.com/Azure/azure-sdk-for-cpp/archive/refs/tags/${DOWNLOAD_FILE} ${DOWNLOAD_FOLDER}/${DOWNLOAD_FILE}) + endif() #extract the zip file(ARCHIVE_EXTRACT INPUT ${DOWNLOAD_FOLDER}/${DOWNLOAD_FILE} DESTINATION ${DOWNLOAD_FOLDER}/${DEP_NAME}) #make target folder file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/build/${DEP_FOLDER}) + # need a nicer way to copy/move folder # i need to archive the folder then extract at new location - execute_process(COMMAND tar -cf ${DOWNLOAD_FOLDER}/${DEP_NAME}.tar -C ${DOWNLOAD_FOLDER}/${DEP_NAME}/${DEP_PREFIX}-${DEP_NAME}_${DEP_VERSION}/${DEP_FOLDER} .) + if(FETCH_SOURCE_DEPS STREQUAL "LATEST") + execute_process(COMMAND tar -cf ${DOWNLOAD_FOLDER}/${DEP_NAME}.tar -C ${DOWNLOAD_FOLDER}/${DEP_NAME}/azure-sdk-for-cpp-main/${DEP_FOLDER} .) + else() + execute_process(COMMAND tar -cf ${DOWNLOAD_FOLDER}/${DEP_NAME}.tar -C ${DOWNLOAD_FOLDER}/${DEP_NAME}/${DEP_PREFIX}-${DEP_NAME}_${DEP_VERSION}/${DEP_FOLDER} .) + endif() + file(ARCHIVE_EXTRACT INPUT ${DOWNLOAD_FOLDER}/${DEP_NAME}.tar DESTINATION ${CMAKE_SOURCE_DIR}/build/${DEP_FOLDER}) #cleanup file(REMOVE_RECURSE ${DOWNLOAD_FOLDER}) diff --git a/sdk/storage/azure-storage-blobs/CMakeLists.txt b/sdk/storage/azure-storage-blobs/CMakeLists.txt index 6ea02fcf7..3e1fe686b 100644 --- a/sdk/storage/azure-storage-blobs/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/CMakeLists.txt @@ -8,6 +8,7 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) +option(FETCH_SOURCE_DEPS "build source dependencies" OFF) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake-modules") include(AzureVcpkg) @@ -21,7 +22,16 @@ include(AzureBuildTargetForCI) az_vcpkg_integrate() -if(NOT AZ_ALL_LIBRARIES) +if(FETCH_SOURCE_DEPS) + set(AZ_ALL_LIBRARIES ON) + include(FolderList) + SetCompileOptions(STORAGE_BLOBS) + GetFolderList(STORAGE_BLOBS) + foreach(oneFolder IN LISTS BUILD_FOLDERS) + message("add folder ${oneFolder}") + add_subdirectory(${oneFolder} EXCLUDE_FROM_ALL) + endforeach() +elseif(NOT AZ_ALL_LIBRARIES) find_package(azure-storage-common-cpp "12.2.0" CONFIG QUIET) if(NOT azure-storage-common-cpp_FOUND) find_package(azure-storage-common-cpp "12.2.0" REQUIRED) diff --git a/sdk/storage/azure-storage-common/CMakeLists.txt b/sdk/storage/azure-storage-common/CMakeLists.txt index 73e0c4bc8..a7ca840a5 100644 --- a/sdk/storage/azure-storage-common/CMakeLists.txt +++ b/sdk/storage/azure-storage-common/CMakeLists.txt @@ -8,6 +8,7 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) +option(FETCH_SOURCE_DEPS "build source dependencies" OFF) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake-modules") include(AzureVcpkg) @@ -21,7 +22,16 @@ include(AzureBuildTargetForCI) az_vcpkg_integrate() -if(NOT AZ_ALL_LIBRARIES) +if(FETCH_SOURCE_DEPS) + set(AZ_ALL_LIBRARIES ON) + include(FolderList) + SetCompileOptions(STORAGE_COMMON) + GetFolderList(STORAGE_COMMON) + foreach(oneFolder IN LISTS BUILD_FOLDERS) + message("add folder ${oneFolder}") + add_subdirectory(${oneFolder} EXCLUDE_FROM_ALL) + endforeach() +elseif(NOT AZ_ALL_LIBRARIES) find_package(azure-core-cpp "1.0.0" CONFIG QUIET) if(NOT azure-core-cpp_FOUND) find_package(azure-core-cpp "1.0.0" REQUIRED) @@ -115,3 +125,5 @@ create_code_coverage(storage azure-storage-common azure-storage-test "tests?/*;s if(BUILD_TESTING) add_subdirectory(test/ut) endif() + +unset(FETCH_SOURCE_DEPS CACHE) diff --git a/sdk/storage/azure-storage-files-datalake/CMakeLists.txt b/sdk/storage/azure-storage-files-datalake/CMakeLists.txt index 46f4d15e1..05dc31310 100644 --- a/sdk/storage/azure-storage-files-datalake/CMakeLists.txt +++ b/sdk/storage/azure-storage-files-datalake/CMakeLists.txt @@ -8,6 +8,7 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) +option(FETCH_SOURCE_DEPS "build source dependencies" OFF) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake-modules") include(AzureVcpkg) @@ -21,7 +22,16 @@ include(AzureBuildTargetForCI) az_vcpkg_integrate() -if(NOT AZ_ALL_LIBRARIES) +if(FETCH_SOURCE_DEPS) + set(AZ_ALL_LIBRARIES ON) + include(FolderList) + SetCompileOptions(STORAGE_DATALAKE) + GetFolderList(STORAGE_DATALAKE) + foreach(oneFolder IN LISTS BUILD_FOLDERS) + message("add folder ${oneFolder}") + add_subdirectory(${oneFolder} EXCLUDE_FROM_ALL) + endforeach() +elseif(NOT AZ_ALL_LIBRARIES) find_package(azure-storage-blobs-cpp "12.2.0" CONFIG QUIET) if(NOT azure-storage-blobs-cpp_FOUND) find_package(azure-storage-blobs-cpp "12.2.0" REQUIRED) diff --git a/sdk/storage/azure-storage-files-shares/CMakeLists.txt b/sdk/storage/azure-storage-files-shares/CMakeLists.txt index 7075ca1a2..f7a4df569 100644 --- a/sdk/storage/azure-storage-files-shares/CMakeLists.txt +++ b/sdk/storage/azure-storage-files-shares/CMakeLists.txt @@ -8,6 +8,7 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) +option(FETCH_SOURCE_DEPS "build source dependencies" OFF) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake-modules") include(AzureVcpkg) @@ -21,7 +22,16 @@ include(AzureBuildTargetForCI) az_vcpkg_integrate() -if(NOT AZ_ALL_LIBRARIES) +if(FETCH_SOURCE_DEPS) + set(AZ_ALL_LIBRARIES ON) + include(FolderList) + SetCompileOptions(STORAGE_SHARES) + GetFolderList(STORAGE_SHARES) + foreach(oneFolder IN LISTS BUILD_FOLDERS) + message("add folder ${oneFolder}") + add_subdirectory(${oneFolder} EXCLUDE_FROM_ALL) + endforeach() +elseif(NOT AZ_ALL_LIBRARIES) find_package(azure-storage-common-cpp "12.2.0" CONFIG QUIET) if(NOT azure-storage-common-cpp_FOUND) find_package(azure-storage-common-cpp "12.2.0" REQUIRED) diff --git a/sdk/storage/azure-storage-queues/CMakeLists.txt b/sdk/storage/azure-storage-queues/CMakeLists.txt index 428dd83c0..b07a84036 100644 --- a/sdk/storage/azure-storage-queues/CMakeLists.txt +++ b/sdk/storage/azure-storage-queues/CMakeLists.txt @@ -8,6 +8,7 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) +option(FETCH_SOURCE_DEPS "build source dependencies" OFF) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake-modules") include(AzureVcpkg) @@ -21,7 +22,16 @@ include(AzureBuildTargetForCI) az_vcpkg_integrate() -if(NOT AZ_ALL_LIBRARIES) +if(FETCH_SOURCE_DEPS) + set(AZ_ALL_LIBRARIES ON) + include(FolderList) + SetCompileOptions(STORAGE_QUEUES) + GetFolderList(STORAGE_QUEUES) + foreach(oneFolder IN LISTS BUILD_FOLDERS) + message("add folder ${oneFolder}") + add_subdirectory(${oneFolder} EXCLUDE_FROM_ALL) + endforeach() +elseif(NOT AZ_ALL_LIBRARIES) find_package(azure-storage-common-cpp "12.2.0" CONFIG QUIET) if(NOT azure-storage-common-cpp_FOUND) find_package(azure-storage-common-cpp "12.2.0" REQUIRED)