diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 983c4ab37..86d13955a 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -39,16 +39,18 @@ jobs: - java: 8 spark: '3.1' spark-hadoop: '2.7' - profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest' + profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest' + - java: 8 + spark: '3.1' + spark-hadoop: '3.2' + profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.0 -Dspark.archive.name=spark-3.2.0-bin-hadoop3.2.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest' - java: 8 spark: '3.1' spark-hadoop: '3.2' - profiles: '-Pkyuubi-extension-spark-3-1' codecov: 'true' - java: 8 spark: '3.2' spark-hadoop: '3.2' - profiles: '-Pkyuubi-extension-spark-3-2' - java: 11 spark: '3.1' spark-hadoop: '3.2' diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index af6f2f4cc..b5a9583bc 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: profiles: - - '-Pspark-master -pl :kyuubi-spark-sql-engine_2.12,:kyuubi-common_2.12,:kyuubi-ha_2.12,:kyuubi-zookeeper_2.12,:kyuubi-spark-monitor_2.12' + - '-Pspark-master -pl externals/kyuubi-spark-sql-engine -am' env: SPARK_LOCAL_IP: localhost steps: diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index 3855281a8..e1e4ff2dc 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -36,4 +36,6 @@ jobs: ASF_PASSWORD: ${{ secrets.NEXUS_PW }} run: | rm -rf ~/.m2/repository/org/apache/kyuubi - ./build/mvn clean deploy -DskipTests -Pkyuubi-extension-spark-3-1,spark-provided -s ./build/release/asf-settings.xml + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,spark-3.1 -pl dev/kyuubi-extension-spark-3-1 -am + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,spark-3.2 -pl dev/kyuubi-extension-spark-3-2 -am diff --git a/build/release/create-package.sh b/build/release/create-package.sh index 9b0bb9f45..f9cf52daf 100755 --- a/build/release/create-package.sh +++ b/build/release/create-package.sh @@ -75,7 +75,7 @@ package_binary() { echo "Creating binary release tarball ${BIN_TGZ_FILE}" - ${KYUUBI_DIR}/build/dist --tgz --spark-provided -Pkyuubi-extension-spark-3-1 + ${KYUUBI_DIR}/build/dist --tgz --spark-provided -Pspark-3.1 cp "${BIN_TGZ_FILE}" "${RELEASE_DIR}" diff --git a/build/release/release.sh b/build/release/release.sh index 0cd691929..00d7fa87e 100755 --- a/build/release/release.sh +++ b/build/release/release.sh @@ -96,6 +96,12 @@ upload_svn_staging() { upload_nexus_staging() { ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided \ -s "${KYUUBI_DIR}/build/release/asf-settings.xml" + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided,spark-3.1 \ + -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ + -pl dev/kyuubi-extension-spark-3-1 -am + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided,spark-3.2 \ + -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ + -pl dev/kyuubi-extension-spark-3-2 -am } finalize_svn() { diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml index 18ca2390a..3a3f0c0a9 100644 --- a/dev/kyuubi-codecov/pom.xml +++ b/dev/kyuubi-codecov/pom.xml @@ -111,6 +111,14 @@ true + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + diff --git a/dev/kyuubi-extension-spark-3-1/pom.xml b/dev/kyuubi-extension-spark-3-1/pom.xml index 1a7d517bf..f43e93333 100644 --- a/dev/kyuubi-extension-spark-3-1/pom.xml +++ b/dev/kyuubi-extension-spark-3-1/pom.xml @@ -155,13 +155,4 @@ - - - - kyuubi-extension-spark-3-1 - - 3.1.2 - - - diff --git a/dev/kyuubi-extension-spark-3-2/pom.xml b/dev/kyuubi-extension-spark-3-2/pom.xml index 5cf2f9b2d..eccf9d68d 100644 --- a/dev/kyuubi-extension-spark-3-2/pom.xml +++ b/dev/kyuubi-extension-spark-3-2/pom.xml @@ -161,14 +161,4 @@ - - - - kyuubi-extension-spark-3-2 - - 3.2.0 - - - - diff --git a/dev/kyuubi-extension-spark-common/pom.xml b/dev/kyuubi-extension-spark-common/pom.xml index aa981e51d..b3fb584ac 100644 --- a/dev/kyuubi-extension-spark-common/pom.xml +++ b/dev/kyuubi-extension-spark-common/pom.xml @@ -135,13 +135,11 @@ - net.alchim31.maven - scala-maven-plugin - - - - org.scalatest - scalatest-maven-plugin + org.apache.maven.plugins + maven-dependency-plugin + + true + @@ -159,24 +157,12 @@ - org.scalastyle - scalastyle-maven-plugin + org.apache.maven.plugins + maven-deploy-plugin + + true + - - - - kyuubi-extension-spark-3-1 - - 3.1.2 - - - - kyuubi-extension-spark-3-2 - - 3.2.0 - - - diff --git a/docs/community/release.md b/docs/community/release.md index d2d34bde7..1a9afe7d1 100644 --- a/docs/community/release.md +++ b/docs/community/release.md @@ -159,7 +159,7 @@ export RELEASE_RC_NO= ```shell build/mvn versions:set -DgenerateBackupPoms=false \ -DnewVersion="${RELEASE_VERSION}" \ - -Pkubernetes,kyuubi-extension-spark-3-1,spark-block-cleaner,tpcds + -Pspark-3.1,spark-block-cleaner git commit -am "[RELEASE] Bump ${RELEASE_VERSION}" ``` diff --git a/docs/sql/rules.md b/docs/sql/rules.md index a1d810bb0..0723ed4f2 100644 --- a/docs/sql/rules.md +++ b/docs/sql/rules.md @@ -15,8 +15,6 @@ - limitations under the License. --> - -
![](../imgs/kyuubi_logo.png) @@ -53,10 +51,18 @@ And don't worry, Kyuubi will support the new Apache Spark version in future. Tha ## How to use Kyuubi SQL extension -1. you need to choose Apache Spark branch-3.1 or higher version with Kyuubi binary tgz. -2. if you want to compile Kyuubi by yourself, the maven opt should add `-Pkyuubi-extension-spark-3-1` -3. move the jar(`kyuubi-extension-spark-*.jar`) which is in `$KYUUBI_HOME/extension` into `$SPARK_HOME/jars` -4. add a config into `spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` +| Kyuubi Spark SQL extension | Supported Spark version(s) | Available since | EOL | Bundled in Binary release tarball | Maven profile +| -------------------------- | -------------------------- | ---------------- | ---------------- | --------------------------------- | ------------- +| kyuubi-extension-spark-3-1 | 3.1.x | 1.3.0-incubating | N/A | 1.3.0-incubating | spark-3.1 +| kyuubi-extension-spark-3-2 | 3.2.x | 1.4.0-incubating | N/A | 1.4.0-incubating | spark-3.2 + +1. Check the matrix that if you are using the supported Spark version, and find the corresponding Kyuubi Spark SQL Extension jar +2. Get the Kyuubi Spark SQL Extension jar + 1. Each Kyuubi binary release tarball only contains one default version of Kyuubi Spark SQL Extension jar, if you are looking for such version, you can find it under `$KYUUBI_HOME/extension` + 2. All supported versions of Kyuubi Spark SQL Extension jar will be deployed to [Maven Central](https://search.maven.org/search?q=kyuubi-extension-spark) + 3. If you like, you can compile Kyuubi Spark SQL Extension jar by yourself, please activate the corresponding Maven's profile on you compile command, i.e. you can get Kyuubi Spark SQL Extension jar for Spark 3.1 under `dev/kyuubi-extension-spark-3-1/target` when compile with `-Pspark-3.1` +3. Put the Kyuubi Spark SQL extension jar `kyuubi-extension-spark-*.jar` into `$SPARK_HOME/jars` +4. Enable `KyuubiSparkSQLExtension`, i.e. add a config into `$SPARK_HOME/conf/spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` Now, you can enjoy the Kyuubi SQL Extension, and also Kyuubi provides some configs to make these feature easy to use. diff --git a/docs/sql/z-order-benchmark.md b/docs/sql/z-order-benchmark.md index d60cb7c39..5beb18058 100644 --- a/docs/sql/z-order-benchmark.md +++ b/docs/sql/z-order-benchmark.md @@ -15,8 +15,6 @@ - limitations under the License. --> - -
![](../imgs/kyuubi_logo.png) diff --git a/externals/kyuubi-download/pom.xml b/externals/kyuubi-download/pom.xml index 795456744..a5613c531 100644 --- a/externals/kyuubi-download/pom.xml +++ b/externals/kyuubi-download/pom.xml @@ -57,6 +57,14 @@ + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + diff --git a/pom.xml b/pom.xml index 88d8f1ce8..d8ffe7f91 100644 --- a/pom.xml +++ b/pom.xml @@ -1775,6 +1775,10 @@ 1.0.0 org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.HudiTest + + dev/kyuubi-extension-spark-common + dev/kyuubi-extension-spark-3-1 + @@ -1783,6 +1787,10 @@ 3.2.0 org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.HudiTest + + dev/kyuubi-extension-spark-common + dev/kyuubi-extension-spark-3-2 + @@ -1826,22 +1834,6 @@ - - kyuubi-extension-spark-3-1 - - dev/kyuubi-extension-spark-common - dev/kyuubi-extension-spark-3-1 - - - - - kyuubi-extension-spark-3-2 - - dev/kyuubi-extension-spark-common - dev/kyuubi-extension-spark-3-2 - - - spark-block-cleaner diff --git a/tools/spark-block-cleaner/pom.xml b/tools/spark-block-cleaner/pom.xml index e186693f4..e09d6aa91 100644 --- a/tools/spark-block-cleaner/pom.xml +++ b/tools/spark-block-cleaner/pom.xml @@ -55,10 +55,5 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - ${project.basedir}/src/main/resources - -