[KYUUBI #6163] Set default Spark version to 3.5

# 🔍 Description ## Issue References 🔗 Kyuubi fully supports Spark 3.5 now, this pull request aims to set the default Spark to 3.5 in Kyuubi 1.9 ## Types of changes 🔖 - [ ] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan 🧪 Pass GA. --- # Checklist 📝 - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes #6163 from pan3793/spark-3.5-default. Closes #6163 f386aeb7a [Cheng Pan] Set default Spark version to 3.5 Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
2024-03-12 16:22:37 +08:00 · 2024-03-12 16:22:37 +08:00 · 6297651d83
commit 6297651d83
parent 6a4f6f0c88
12 changed files with 41 additions and 39 deletions
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@ -57,25 +57,25 @@ jobs:
        comment: ["normal"]
        include:
          - java: 8
-            spark: '3.4'
+            spark: '3.5'
            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3 -Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz -Pzookeeper-3.6'
            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
            comment: 'verify-on-spark-3.1-binary'
          - java: 8
-            spark: '3.4'
+            spark: '3.5'
            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
            comment: 'verify-on-spark-3.2-binary'
          - java: 8
-            spark: '3.4'
+            spark: '3.5'
            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
            comment: 'verify-on-spark-3.3-binary'
          - java: 8
-            spark: '3.4'
-            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.5.1 -Dspark.archive.name=spark-3.5.1-bin-hadoop3.tgz -Pzookeeper-3.6'
+            spark: '3.5'
+            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.2 -Dspark.archive.name=spark-3.4.2-bin-hadoop3.tgz -Pzookeeper-3.6'
            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
-            comment: 'verify-on-spark-3.5-binary'
+            comment: 'verify-on-spark-3.4-binary'
        exclude:
          # SPARK-33772: Spark supports JDK 17 since 3.3.0
          - java: 17
@ -105,7 +105,7 @@ jobs:
          python-version: '3.9'
      - name: Build and test Kyuubi and Spark with maven w/o linters
        run: |
-          if [[ "${{ matrix.java }}" == "8" && "${{ matrix.spark }}" == "3.4" && "${{ matrix.spark-archive }}" == "" ]]; then
+          if [[ "${{ matrix.java }}" == "8" && "${{ matrix.spark }}" == "3.5" && "${{ matrix.spark-archive }}" == "" ]]; then
            MVN_OPT="${MVN_OPT} -Pcodecov"
          fi
          TEST_MODULES="dev/kyuubi-codecov"
@ -114,7 +114,7 @@ jobs:
      - name: Code coverage
        if: |
          matrix.java == 8 &&
-          matrix.spark == '3.4' &&
+          matrix.spark == '3.5' &&
          matrix.spark-archive == ''
        uses: codecov/codecov-action@v3
        with:
@ -140,7 +140,7 @@ jobs:
        java:
          - '8'
        spark:
-          - '3.4'
+          - '3.5'
    steps:
      - uses: actions/checkout@v4
      - name: Free up disk space
@ -374,8 +374,8 @@ jobs:
          # https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime
          minikube image load apache/kyuubi:latest
          # pre-install spark into minikube
-          docker pull apache/spark:3.4.2
-          minikube image load apache/spark:3.4.2
+          docker pull apache/spark:3.5.1
+          minikube image load apache/spark:3.5.1
      - name: kubectl pre-check
        run: |
          kubectl get nodes
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@ -29,7 +29,7 @@ jobs:
    strategy:
      matrix:
        profiles:
-          - '-Pspark-master -pl externals/kyuubi-spark-sql-engine -am'
+          - '-Pscala-2.13 -Pspark-master -pl externals/kyuubi-spark-sql-engine -am'
    env:
      SPARK_LOCAL_IP: localhost
    steps:
--- a/build/release/release.sh
+++ b/build/release/release.sh
@ -120,18 +120,18 @@ upload_nexus_staging() {
    -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \
    -pl extensions/spark/kyuubi-extension-spark-3-3 -am

-  # Spark Extension Plugin for Spark 3.5
-  ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.5 \
+  # Spark Extension Plugin for Spark 3.4
+  ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.4 \
    -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \
-    -pl extensions/spark/kyuubi-extension-spark-3-5 -am
+    -pl extensions/spark/kyuubi-extension-spark-3-4 -am

-  # Spark Hive/TPC-DS/TPC-H Connector built with default Spark version (3.4) and Scala 2.13
-  ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.4,scala-2.13 \
+  # Spark Hive/TPC-DS/TPC-H Connector built with default Spark version (3.5) and Scala 2.13
+  ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.5,scala-2.13 \
    -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \
    -pl extensions/spark/kyuubi-spark-connector-hive,extensions/spark/kyuubi-spark-connector-tpcds,extensions/spark/kyuubi-spark-connector-tpch -am

-  # All modules including Spark Extension Plugin and Connectors built with default Spark version (3.4) and default Scala version (2.12)
-  ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.4 \
+  # All modules including Spark Extension Plugin and Connectors built with default Spark version (3.5) and default Scala version (2.12)
+  ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.5 \
    -s "${KYUUBI_DIR}/build/release/asf-settings.xml"
 }

--- a/docs/contributing/code/building.md
+++ b/docs/contributing/code/building.md
@ -65,8 +65,8 @@ Since v1.1.0, Kyuubi support building with different Spark profiles,
 |-------------|---------|-------|
 | -Pspark-3.2 |         | 1.4.0 |
 | -Pspark-3.3 |         | 1.6.0 |
-| -Pspark-3.4 | ✓       | 1.8.0 |
-| -Pspark-3.5 |         | 1.8.0 |
+| -Pspark-3.4 |         | 1.8.0 |
+| -Pspark-3.5 | ✓       | 1.8.0 |

 ## Building Kyuubi Against Different Scala Versions

--- a/docs/extensions/engines/spark/lineage.md
+++ b/docs/extensions/engines/spark/lineage.md
@ -117,7 +117,7 @@ Sometimes, it may be incompatible with other Spark distributions, then you may n
 For example,

 ```shell
-build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -am -DskipTests -Dspark.version=3.1.2
+build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -am -DskipTests -Dspark.version=3.5.1
 ```

 The available `spark.version`s are shown in the following table.
@ -125,11 +125,12 @@ The available `spark.version`s are shown in the following table.
 | Spark Version | Supported | Remark |
 |:-------------:|:---------:|:------:|
 |    master     |     √     |   -    |
+|     3.5.x     |     √     |   -    |
 |     3.4.x     |     √     |   -    |
 |     3.3.x     |     √     |   -    |
 |     3.2.x     |     √     |   -    |
-|     3.1.x     |     √     |   -    |
-|     3.0.x     |     √     |   -    |
+|     3.1.x     |     x     |   -    |
+|     3.0.x     |     x     |   -    |
 |     2.4.x     |     x     |   -    |

 Currently, Spark released with Scala 2.12 are supported.
--- a/docs/extensions/engines/spark/rules.md
+++ b/docs/extensions/engines/spark/rules.md
@ -49,7 +49,7 @@ And don't worry, Kyuubi will support the new Apache Spark version in the future.
 | kyuubi-extension-spark-3-2 | 3.2.x                      | 1.4.0-incubating | N/A   | 1.4.0-incubating                  | spark-3.2     |
 | kyuubi-extension-spark-3-3 | 3.3.x                      | 1.6.0-incubating | N/A   | 1.6.0-incubating                  | spark-3.3     |
 | kyuubi-extension-spark-3-4 | 3.4.x                      | 1.8.0            | N/A   | 1.8.0                             | spark-3.4     |
-| kyuubi-extension-spark-3-4 | 3.5.x                      | 1.8.0            | N/A   | N/A                               | spark-3.5     |
+| kyuubi-extension-spark-3-5 | 3.5.x                      | 1.8.0            | N/A   | 1.9.0                             | spark-3.5     |

 1. Check the matrix that if you are using the supported Spark version, and find the corresponding Kyuubi Spark SQL Extension jar
 2. Get the Kyuubi Spark SQL Extension jar
--- a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderCoreBenchmark.scala
+++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderCoreBenchmark.scala
@ -29,8 +29,8 @@ import org.apache.kyuubi.sql.zorder.ZorderBytesUtils
 *
 * {{{
 *   RUN_BENCHMARK=1 ./build/mvn clean test \
- *   -pl extensions/spark/kyuubi-extension-spark-3-4 -am \
- *   -Pspark-3.4,kyuubi-extension-spark-3-4 \
+ *   -pl extensions/spark/kyuubi-extension-spark-3-5 -am \
+ *   -Pspark-3.5,kyuubi-extension-spark-3-5 \
 *   -Dtest=none -DwildcardSuites=org.apache.spark.sql.ZorderCoreBenchmark
 * }}}
 */
--- a/extensions/spark/kyuubi-spark-authz/README.md
+++ b/extensions/spark/kyuubi-spark-authz/README.md
@ -34,8 +34,8 @@ build/mvn clean package -DskipTests -pl :kyuubi-spark-authz_2.12 -am -Dspark.ver
 `-Dspark.version=`

 - [x] master
- [x] 3.5.x
- [x] 3.4.x (default)
+- [x] 3.5.x (default)
+- [x] 3.4.x
 - [x] 3.3.x
 - [x] 3.2.x
 - [x] 3.1.x
--- a/extensions/spark/kyuubi-spark-lineage/README.md
+++ b/extensions/spark/kyuubi-spark-lineage/README.md
@ -34,8 +34,9 @@ build/mvn clean package -DskipTests -pl :kyuubi-spark-lineage_2.12 -am -Dspark.v
 `-Dspark.version=`

 - [x] master
- [ ] 3.4.x
- [x] 3.3.x (default)
+- [x] 3.5.x (default)
+- [x] 3.4.x
+- [x] 3.3.x
 - [x] 3.2.x
 - [x] 3.1.x

--- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
@ -55,7 +55,7 @@ class KyuubiOnKubernetesWithSparkTestsBase extends WithKyuubiServerOnKubernetes
      Map(
        "spark.master" -> s"k8s://$miniKubeApiMaster",
        // We should update spark docker image in ./github/workflows/master.yml at the same time
-        "spark.kubernetes.container.image" -> "apache/spark:3.4.2",
+        "spark.kubernetes.container.image" -> "apache/spark:3.5.1",
        "spark.kubernetes.container.image.pullPolicy" -> "IfNotPresent",
        "spark.executor.memory" -> "512M",
        "spark.driver.memory" -> "1024M",
--- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
@ -50,7 +50,7 @@ abstract class SparkOnKubernetesSuiteBase
    // TODO Support more Spark version
    // Spark official docker image: https://hub.docker.com/r/apache/spark/tags
    KyuubiConf().set("spark.master", s"k8s://$apiServerAddress")
-      .set("spark.kubernetes.container.image", "apache/spark:3.4.2")
+      .set("spark.kubernetes.container.image", "apache/spark:3.5.1")
      .set("spark.kubernetes.container.image.pullPolicy", "IfNotPresent")
      .set("spark.executor.instances", "1")
      .set("spark.executor.memory", "512M")
--- a/pom.xml
+++ b/pom.xml
@ -133,8 +133,8 @@
        <commons-io.version>2.11.0</commons-io.version>
        <commons-lang.version>2.6</commons-lang.version>
        <commons-lang3.version>3.13.0</commons-lang3.version>
-        <delta.artifact>delta-core</delta.artifact>
-        <delta.version>2.4.0</delta.version>
+        <delta.artifact>delta-spark</delta.artifact>
+        <delta.version>3.1.0</delta.version>
        <failsafe.verion>3.3.2</failsafe.verion>
        <fb303.version>0.9.3</fb303.version>
        <flexmark.version>0.62.2</flexmark.version>
@ -198,10 +198,10 @@
        <snakeyaml.version>2.2</snakeyaml.version>
        <!--
          DO NOT forget to change the following properties when change the minor version of Spark:
-          `delta.version`, `maven.plugin.scalatest.exclude.tags`
+          `delta.version`, `delta.artifact`, `maven.plugin.scalatest.exclude.tags`
          -->
-        <spark.version>3.4.2</spark.version>
-        <spark.binary.version>3.4</spark.binary.version>
+        <spark.version>3.5.1</spark.version>
+        <spark.binary.version>3.5</spark.binary.version>
        <spark.archive.scala.suffix></spark.archive.scala.suffix>
        <spark.archive.name>spark-${spark.version}-bin-hadoop3${spark.archive.scala.suffix}.tgz</spark.archive.name>
        <spark.archive.mirror>${apache.archive.dist}/spark/spark-${spark.version}</spark.archive.mirror>
@ -235,7 +235,7 @@
        <maven.plugin.frontend.version>1.12.1</maven.plugin.frontend.version>
        <maven.plugin.scala.version>4.8.0</maven.plugin.scala.version>
        <maven.plugin.scalatest.version>2.2.0</maven.plugin.scalatest.version>
-        <maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest</maven.plugin.scalatest.exclude.tags>
+        <maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow</maven.plugin.scalatest.exclude.tags>
        <maven.plugin.scalatest.include.tags></maven.plugin.scalatest.include.tags>
        <maven.plugin.scalatest.debug.enabled>false</maven.plugin.scalatest.debug.enabled>
        <maven.plugin.spotless.version>2.30.0</maven.plugin.spotless.version>