From b3d33cabfa40591754ab3242e7d9a5b1681fb647 Mon Sep 17 00:00:00 2001 From: zml1206 Date: Tue, 19 Dec 2023 10:39:00 +0800 Subject: [PATCH] [KYUUBI #5856] Bump Spark 3.4.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # :mag: Description ## Issue References ๐Ÿ”— This pull request fixes #5856 ## Describe Your Solution ๐Ÿ”ง ## Types of changes :bookmark: - [ ] Bugfix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan ๐Ÿงช #### Behavior Without This Pull Request :coffin: #### Behavior With This Pull Request :tada: #### Related Unit Tests --- # Checklists ## ๐Ÿ“ Author Self Checklist - [x] My code follows the [style guidelines](https://kyuubi.readthedocs.io/en/master/contributing/code/style.html) of this project - [x] I have performed a self-review - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) ## ๐Ÿ“ Committer Pre-Merge Checklist - [x] Pull request title is okay. - [x] No license issues. - [x] Milestone correctly set? - [x] Test coverage is ok - [x] Assignees are selected. - [x] Minimum number of approvals - [x] No changes are requested **Be nice. Be informative.** Closes #5859 from zml1206/KYUUBI-5856. Closes #5856 872fd06d2 [zml1206] Revert changes in SparkProcessBuilderSuite bc4996f90 [zml1206] Bump spark 3.4.2 Authored-by: zml1206 Signed-off-by: Cheng Pan --- .github/workflows/master.yml | 6 +++--- docker/playground/.env | 2 +- integration-tests/kyuubi-gluten-it/pom.xml | 4 ++-- .../test/deployment/KyuubiOnKubernetesTestsSuite.scala | 2 +- .../kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala | 2 +- pom.xml | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 47147a9b9..289e32c14 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -410,7 +410,7 @@ jobs: with: # passthrough CI into build container build-args: | - CI=${CI} + CI=${CI} MVN_ARG=--flink-provided --hive-provided -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -DskipTests context: . file: build/Dockerfile @@ -427,8 +427,8 @@ jobs: # https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime minikube image load apache/kyuubi:latest # pre-install spark into minikube - docker pull apache/spark:3.4.1 - minikube image load apache/spark:3.4.1 + docker pull apache/spark:3.4.2 + minikube image load apache/spark:3.4.2 - name: kubectl pre-check run: | kubectl get nodes diff --git a/docker/playground/.env b/docker/playground/.env index 3e0c3d41b..e8446fd56 100644 --- a/docker/playground/.env +++ b/docker/playground/.env @@ -24,7 +24,7 @@ KYUUBI_HADOOP_VERSION=3.3.6 POSTGRES_VERSION=12 POSTGRES_JDBC_VERSION=42.3.4 SCALA_BINARY_VERSION=2.12 -SPARK_VERSION=3.4.1 +SPARK_VERSION=3.4.2 SPARK_BINARY_VERSION=3.4 SPARK_HADOOP_VERSION=3.3.4 ZOOKEEPER_VERSION=3.6.3 diff --git a/integration-tests/kyuubi-gluten-it/pom.xml b/integration-tests/kyuubi-gluten-it/pom.xml index 5a2c2984c..ac49c286a 100644 --- a/integration-tests/kyuubi-gluten-it/pom.xml +++ b/integration-tests/kyuubi-gluten-it/pom.xml @@ -31,7 +31,7 @@ 1.1.0-SNAPSHOT - 3.4.1 + 3.4.2 3.4 @@ -93,7 +93,7 @@ gluten-spark-3.4 org.apache.kyuubi.tags.GlutenTest - 3.4.1 + 3.4.2 3.4 diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala index 95e15e6eb..9d47ab998 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala @@ -55,7 +55,7 @@ class KyuubiOnKubernetesWithSparkTestsBase extends WithKyuubiServerOnKubernetes Map( "spark.master" -> s"k8s://$miniKubeApiMaster", // We should update spark docker image in ./github/workflows/master.yml at the same time - "spark.kubernetes.container.image" -> "apache/spark:3.4.1", + "spark.kubernetes.container.image" -> "apache/spark:3.4.2", "spark.kubernetes.container.image.pullPolicy" -> "IfNotPresent", "spark.executor.memory" -> "512M", "spark.driver.memory" -> "1024M", diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala index 09532efe3..ea804575e 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala @@ -50,7 +50,7 @@ abstract class SparkOnKubernetesSuiteBase // TODO Support more Spark version // Spark official docker image: https://hub.docker.com/r/apache/spark/tags KyuubiConf().set("spark.master", s"k8s://$apiServerAddress") - .set("spark.kubernetes.container.image", "apache/spark:3.4.1") + .set("spark.kubernetes.container.image", "apache/spark:3.4.2") .set("spark.kubernetes.container.image.pullPolicy", "IfNotPresent") .set("spark.executor.instances", "1") .set("spark.executor.memory", "512M") diff --git a/pom.xml b/pom.xml index 1d7ae4f86..2a9a793e5 100644 --- a/pom.xml +++ b/pom.xml @@ -200,7 +200,7 @@ DO NOT forget to change the following properties when change the minor version of Spark: `delta.version`, `maven.plugin.scalatest.exclude.tags` --> - 3.4.1 + 3.4.2 3.4 spark-${spark.version}-bin-hadoop3${spark.archive.scala.suffix}.tgz @@ -2283,7 +2283,7 @@ delta-core 2.4.0 - 3.4.1 + 3.4.2 3.4 org.scalatest.tags.Slow