diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 0b271ba7b..faca022b4 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -54,7 +54,7 @@ jobs: - java: 11 spark: '3.1' spark-hadoop: '3.2' - profiles: '-DskipTests -Pspark-provided' + profiles: '-DskipTests -Pflink-provided,spark-provided' env: SPARK_LOCAL_IP: localhost diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index 78c15e0ca..01c819117 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -37,9 +37,9 @@ jobs: ASF_PASSWORD: ${{ secrets.NEXUS_PW }} run: | rm -rf ~/.m2/repository/org/apache/kyuubi - ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided - ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,spark-3.1 -pl dev/kyuubi-extension-spark-3-1 -am - ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,spark-3.2 -pl dev/kyuubi-extension-spark-3-2 -am + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pflink-provided,spark-provided + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pflink-provided,spark-provided,spark-3.1 -pl dev/kyuubi-extension-spark-3-1 -am + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pflink-provided,spark-provided,spark-3.2 -pl dev/kyuubi-extension-spark-3-2 -am - name: Publish snapshot - branch-1.3 if: ${{ matrix.branch == 'branch-1.3' }} env: @@ -47,4 +47,4 @@ jobs: ASF_PASSWORD: ${{ secrets.NEXUS_PW }} run: | rm -rf ~/.m2/repository/org/apache/kyuubi - ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,kyuubi-extension-spark-3-1 + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pflink-provided,spark-provided,kyuubi-extension-spark-3-1 diff --git a/build/dist b/build/dist index 6e8a7dd42..a4ef98203 100755 --- a/build/dist +++ b/build/dist @@ -30,6 +30,7 @@ set -e KYUUBI_HOME="$(cd "`dirname "$0"`/.."; pwd)" DISTDIR="$KYUUBI_HOME/dist" MAKE_TGZ=false +# TODO: add FLINK_PROVIDED option SPARK_PROVIDED=false NAME=none MVN="$KYUUBI_HOME/build/mvn" @@ -40,10 +41,11 @@ function usage { echo "" echo "Usage:" echo "+--------------------------------------------------------------------------------------+" - echo "| ./build/dist [--name ] [--tgz] [--spark-provided] |" + echo "| ./build/dist [--name ] [--tgz] [--flink-provided] [--spark-provided] |" echo "+--------------------------------------------------------------------------------------+" echo "name: - custom binary name, using project version if undefined" echo "tgz: - whether to make a whole bundled package" + echo "flink-provided: - whether to make a package without Flink binary" echo "spark-provided: - whether to make a package without Spark binary" echo "" } diff --git a/build/release/release.sh b/build/release/release.sh index 00d7fa87e..02a5d79bf 100755 --- a/build/release/release.sh +++ b/build/release/release.sh @@ -94,12 +94,12 @@ upload_svn_staging() { } upload_nexus_staging() { - ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided \ + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided \ -s "${KYUUBI_DIR}/build/release/asf-settings.xml" - ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided,spark-3.1 \ + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,spark-3.1 \ -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ -pl dev/kyuubi-extension-spark-3-1 -am - ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided,spark-3.2 \ + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,spark-3.2 \ -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ -pl dev/kyuubi-extension-spark-3-2 -am } diff --git a/docs/develop_tools/distribution.md b/docs/develop_tools/distribution.md index 1bf8feeab..b6153bd3f 100644 --- a/docs/develop_tools/distribution.md +++ b/docs/develop_tools/distribution.md @@ -33,10 +33,11 @@ For more information on usage, run `./build/dist --help` Usage: +--------------------------------------------------------------------------------------+ -| ./build/dist [--name ] [--tgz] [--spark-provided] | +| ./build/dist [--name ] [--tgz] [--flink-provided] [--spark-provided] | +--------------------------------------------------------------------------------------+ name: - custom binary name, using project version if undefined tgz: - whether to make a whole bundled package +flink-provided: - whether to make a package without Flink binary spark-provided: - whether to make a package without Spark binary ``` @@ -48,10 +49,10 @@ For instance, This results in a Kyuubi distribution named `kyuubi-{version}-bin-custom-name.tgz` for you. -If you are planing to deploy Kyuubi where `spark` is provided, in other word, it's not required to bundle spark binary, use +If you are planing to deploy Kyuubi where `spark`/`flink` is provided, in other word, it's not required to bundle spark/flink binary, use ```bash -./build/dist --tgz --spark-provided +./build/dist --tgz --spark-provided --flink-provided ``` Then you will get a Kyuubi distribution without spark binary named `kyuubi-{version}-bin.tgz`. diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala index 41a564839..6cc8456c4 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala @@ -30,12 +30,12 @@ import org.apache.kyuubi.{KyuubiFunSuite, TestUtils} * * To run the entire test suite: * {{{ - * build/mvn clean install -Pspark-provided -DwildcardSuites=org.apache.kyuubi.engine.spark.udf.KyuubiDefinedFunctionSuite + * build/mvn clean install -Pflink-provided,spark-provided -DwildcardSuites=org.apache.kyuubi.engine.spark.udf.KyuubiDefinedFunctionSuite * }}} * * To re-generate golden files for entire suite, run: * {{{ - * KYUUBI_UPDATE=1 build/mvn clean install -Pspark-provided -DwildcardSuites=org.apache.kyuubi.engine.spark.udf.KyuubiDefinedFunctionSuite + * KYUUBI_UPDATE=1 build/mvn clean install -Pflink-provided,spark-provided -DwildcardSuites=org.apache.kyuubi.engine.spark.udf.KyuubiDefinedFunctionSuite * }}} */ // scalastyle:on line.size.limit diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala index 8645fc95c..107cf8743 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala @@ -52,7 +52,8 @@ object TestUtils { } reader.close() val hint = s"$markdown out of date, please update doc with " + - s"KYUUBI_UPDATE=1 build/mvn clean install -Pspark-provided -DwildcardSuites=$agent" + s"KYUUBI_UPDATE=1 build/mvn clean install -Pflink-provided,spark-provided " + + s"-DwildcardSuites=$agent" assert(newOutput.size === expected.size, hint) newOutput.zip(expected).foreach { case (out, in) => assert(out === in, hint) } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala index 2151869e6..1f8359e44 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala @@ -35,12 +35,12 @@ import org.apache.kyuubi.zookeeper.ZookeeperConf * * To run the entire test suite: * {{{ - * build/mvn clean install -Pspark-provided -DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration + * build/mvn clean install -Pflink-provided,spark-provided -DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration * }}} * * To re-generate golden files for entire suite, run: * {{{ - * KYUUBI_UPDATE=1 build/mvn clean install -Pspark-provided -DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration + * KYUUBI_UPDATE=1 build/mvn clean install -Pflink-provided,spark-provided -DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration * }}} */ // scalastyle:on line.size.limit