From b7effd9d3abae951a7ae4eed4ce7890e73181c77 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 14 Aug 2024 20:57:06 +0800 Subject: [PATCH] [KYUUBI #6609] Bump Spark 3.5.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # :mag: Description Spark 3.5.2 was released recently. Release Notes is available at https://spark.apache.org/releases/spark-release-3-5-2.html ## Types of changes :bookmark: - [ ] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan ๐Ÿงช Pass GHA. --- # Checklist ๐Ÿ“ - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes #6609 from pan3793/spark-3.5.2. Closes #6609 587cf1dd3 [Cheng Pan] Bump Spark 3.5.2 Authored-by: Cheng Pan Signed-off-by: Cheng Pan --- .github/workflows/master.yml | 4 ++-- bin/docker-image-tool.sh | 4 ++-- docs/deployment/kyuubi_on_kubernetes.md | 4 ++-- docs/extensions/engines/spark/lineage.md | 2 +- .../test/deployment/KyuubiOnKubernetesTestsSuite.scala | 2 +- .../kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala | 2 +- pom.xml | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d853940d2..9f7db279e 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -434,8 +434,8 @@ jobs: # https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime minikube image load apache/kyuubi:latest # pre-install spark into minikube - docker pull apache/spark:3.5.1 - minikube image load apache/spark:3.5.1 + docker pull apache/spark:3.5.2 + minikube image load apache/spark:3.5.2 - name: kubectl pre-check run: | kubectl get nodes diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh index 2e8e356b7..7f6f1d623 100755 --- a/bin/docker-image-tool.sh +++ b/bin/docker-image-tool.sh @@ -181,8 +181,8 @@ Examples: $0 -r docker.io/myrepo -t v1.8.1 build $0 -r docker.io/myrepo -t v1.8.1 push - - Build and push with tag "v1.8.1" and Spark-3.5.1 as base image to docker.io/myrepo - $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.1 build + - Build and push with tag "v1.8.1" and Spark-3.5.2 as base image to docker.io/myrepo + $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.2 build $0 -r docker.io/myrepo -t v1.8.1 push - Build and push for multiple archs to docker.io/myrepo diff --git a/docs/deployment/kyuubi_on_kubernetes.md b/docs/deployment/kyuubi_on_kubernetes.md index 8efed5980..1d331b567 100644 --- a/docs/deployment/kyuubi_on_kubernetes.md +++ b/docs/deployment/kyuubi_on_kubernetes.md @@ -42,8 +42,8 @@ Examples: $0 -r docker.io/myrepo -t v1.8.1 build $0 -r docker.io/myrepo -t v1.8.1 push - - Build and push with tag "v1.8.1" and Spark-3.5.1 as base image to docker.io/myrepo - $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.1 build + - Build and push with tag "v1.8.1" and Spark-3.5.2 as base image to docker.io/myrepo + $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.2 build $0 -r docker.io/myrepo -t v1.8.1 push - Build and push for multiple archs to docker.io/myrepo diff --git a/docs/extensions/engines/spark/lineage.md b/docs/extensions/engines/spark/lineage.md index b634473d4..26e284df5 100644 --- a/docs/extensions/engines/spark/lineage.md +++ b/docs/extensions/engines/spark/lineage.md @@ -117,7 +117,7 @@ Sometimes, it may be incompatible with other Spark distributions, then you may n For example, ```shell -build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -am -DskipTests -Dspark.version=3.5.1 +build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -am -DskipTests -Dspark.version=3.5.2 ``` The available `spark.version`s are shown in the following table. diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala index 319b1bfe5..ead11b335 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala @@ -56,7 +56,7 @@ class KyuubiOnKubernetesWithSparkTestsBase extends WithKyuubiServerOnKubernetes Map( "spark.master" -> s"k8s://$miniKubeApiMaster", // We should update spark docker image in ./github/workflows/master.yml at the same time - "spark.kubernetes.container.image" -> "apache/spark:3.5.1", + "spark.kubernetes.container.image" -> "apache/spark:3.5.2", "spark.kubernetes.container.image.pullPolicy" -> "IfNotPresent", "spark.executor.memory" -> "512M", "spark.driver.memory" -> "1024M", diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala index 4692581cc..efbe1e2e9 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala @@ -51,7 +51,7 @@ abstract class SparkOnKubernetesSuiteBase // TODO Support more Spark version // Spark official docker image: https://hub.docker.com/r/apache/spark/tags KyuubiConf().set("spark.master", s"k8s://$apiServerAddress") - .set("spark.kubernetes.container.image", "apache/spark:3.5.1") + .set("spark.kubernetes.container.image", "apache/spark:3.5.2") .set("spark.kubernetes.container.image.pullPolicy", "IfNotPresent") .set("spark.executor.instances", "1") .set("spark.executor.memory", "512M") diff --git a/pom.xml b/pom.xml index 5adcb0892..519d5d759 100644 --- a/pom.xml +++ b/pom.xml @@ -198,7 +198,7 @@ DO NOT forget to change the following properties when change the minor version of Spark: `delta.version`, `delta.artifact`, `maven.plugin.scalatest.exclude.tags` --> - 3.5.1 + 3.5.2 3.5 spark-${spark.version}-bin-hadoop3${spark.archive.scala.suffix}.tgz @@ -1973,7 +1973,7 @@ extensions/spark/kyuubi-spark-connector-hive - 3.5.1 + 3.5.2 3.5 3.2.0 delta-spark_${scala.binary.version}