From e85207e2c7cb36fbdbb90ba51d18e30d1990c56f Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 4 Mar 2025 22:25:10 +0800 Subject: [PATCH] [CELEBORN-1413][FOLLOWUP] Rename `celeborn-client-spark-3-4` back to `celeborn-client-spark-3` ### What changes were proposed in this pull request? This PR partially reverts the change of https://github.com/apache/celeborn/pull/2813, namely, restores the renaming of `celeborn-client-spark-3` ### Why are the changes needed? The renaming is not necessary, and might cause some confusion, for example, I wrongly interpreted the `spark-3-4` as Spark 3.4, it also increases the backport efforts for branch-0.5 ### Does this PR introduce _any_ user-facing change? No, it's dev only, before/after this change, the end users always use the shaded client ``` celeborn-client-spark-2-shaded_2.11-0.6.0-SNAPSHOT.jar celeborn-client-spark-3-shaded_2.12-0.6.0-SNAPSHOT.jar celeborn-client-spark-4-shaded_2.13-0.6.0-SNAPSHOT.jar ``` ### How was this patch tested? Pass GA. Closes #3133 from pan3793/CELEBORN-1413-followup. Authored-by: Cheng Pan Signed-off-by: mingji --- .github/workflows/maven.yml | 6 +--- client-spark/spark-3-columnar-common/pom.xml | 2 +- client-spark/spark-3-columnar-shuffle/pom.xml | 2 +- client-spark/spark-3-shaded/pom.xml | 2 +- client-spark/{spark-3-4 => spark-3}/pom.xml | 2 +- .../celeborn/CelebornPartitionUtil.java | 0 .../celeborn/CelebornShuffleDataIO.java | 0 .../celeborn/HashBasedShuffleWriter.java | 0 .../celeborn/SortBasedShuffleWriter.java | 0 .../shuffle/celeborn/SparkShuffleManager.java | 0 .../spark/shuffle/celeborn/SparkUtils.java | 0 .../org/apache/spark/SparkVersionUtil.scala | 0 .../spark/celeborn/ExceptionMakerHelper.scala | 0 .../CelebornShuffleFallbackPolicyRunner.scala | 0 .../celeborn/CelebornShuffleHandle.scala | 0 .../celeborn/CelebornShuffleReader.scala | 0 ...eFetchFailureReportTaskCleanListener.scala | 0 .../celeborn/CelebornPartitionUtilSuiteJ.java | 0 .../CelebornShuffleWriterSuiteBase.java | 0 .../HashBasedShuffleWriterSuiteJ.java | 0 .../shuffle/celeborn/ShuffleManagerHook.java | 0 .../SortBasedShuffleWriterSuiteJ.java | 0 .../celeborn/TestCelebornShuffleManager.java | 0 .../src/test/resources/log4j.properties | 0 .../src/test/resources/log4j2-test.xml | 0 .../CelebornShuffleManagerSuite.scala | 0 .../celeborn/CelebornShuffleReaderSuite.scala | 0 client-spark/spark-4-shaded/pom.xml | 2 +- dev/dependencies.sh | 2 +- pom.xml | 14 +++++----- project/CelebornBuild.scala | 14 +++++----- tests/spark-it/pom.xml | 28 +++++++++---------- 32 files changed, 35 insertions(+), 39 deletions(-) rename client-spark/{spark-3-4 => spark-3}/pom.xml (97%) rename client-spark/{spark-3-4 => spark-3}/src/main/java/org/apache/spark/shuffle/celeborn/CelebornPartitionUtil.java (100%) rename client-spark/{spark-3-4 => spark-3}/src/main/java/org/apache/spark/shuffle/celeborn/CelebornShuffleDataIO.java (100%) rename client-spark/{spark-3-4 => spark-3}/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java (100%) rename client-spark/{spark-3-4 => spark-3}/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java (100%) rename client-spark/{spark-3-4 => spark-3}/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java (100%) rename client-spark/{spark-3-4 => spark-3}/src/main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java (100%) rename client-spark/{spark-3-4 => spark-3}/src/main/scala/org/apache/spark/SparkVersionUtil.scala (100%) rename client-spark/{spark-3-4 => spark-3}/src/main/scala/org/apache/spark/celeborn/ExceptionMakerHelper.scala (100%) rename client-spark/{spark-3-4 => spark-3}/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleFallbackPolicyRunner.scala (100%) rename client-spark/{spark-3-4 => spark-3}/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleHandle.scala (100%) rename client-spark/{spark-3-4 => spark-3}/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala (100%) rename client-spark/{spark-3-4 => spark-3}/src/main/scala/org/apache/spark/shuffle/celeborn/ShuffleFetchFailureReportTaskCleanListener.scala (100%) rename client-spark/{spark-3-4 => spark-3}/src/test/java/org/apache/spark/shuffle/celeborn/CelebornPartitionUtilSuiteJ.java (100%) rename client-spark/{spark-3-4 => spark-3}/src/test/java/org/apache/spark/shuffle/celeborn/CelebornShuffleWriterSuiteBase.java (100%) rename client-spark/{spark-3-4 => spark-3}/src/test/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriterSuiteJ.java (100%) rename client-spark/{spark-3-4 => spark-3}/src/test/java/org/apache/spark/shuffle/celeborn/ShuffleManagerHook.java (100%) rename client-spark/{spark-3-4 => spark-3}/src/test/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriterSuiteJ.java (100%) rename client-spark/{spark-3-4 => spark-3}/src/test/java/org/apache/spark/shuffle/celeborn/TestCelebornShuffleManager.java (100%) rename client-spark/{spark-3-4 => spark-3}/src/test/resources/log4j.properties (100%) rename client-spark/{spark-3-4 => spark-3}/src/test/resources/log4j2-test.xml (100%) rename client-spark/{spark-3-4 => spark-3}/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleManagerSuite.scala (100%) rename client-spark/{spark-3-4 => spark-3}/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReaderSuite.scala (100%) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f23b7f399..f1a2818a2 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -133,12 +133,8 @@ jobs: run: | SPARK_BINARY_VERSION=${{ matrix.spark }} SPARK_MAJOR_VERSION=${SPARK_BINARY_VERSION%%.*} - SPARK_MODULE_NAME=$SPARK_MAJOR_VERSION - if [[ $SPARK_MAJOR_VERSION == "3" ]]; then - SPARK_MODULE_NAME="3-4" - fi PROFILES="-Pgoogle-mirror,spark-${{ matrix.spark }}" - TEST_MODULES="client-spark/common,client-spark/spark-${SPARK_MODULE_NAME},client-spark/spark-${SPARK_MAJOR_VERSION}-columnar-common,client-spark/spark-${SPARK_MAJOR_VERSION}-shaded,tests/spark-it" + TEST_MODULES="client-spark/common,client-spark/spark-${SPARK_MAJOR_VERSION},client-spark/spark-${SPARK_MAJOR_VERSION}-columnar-common,client-spark/spark-${SPARK_MAJOR_VERSION}-shaded,tests/spark-it" build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests build/mvn $PROFILES -pl $TEST_MODULES -Dspark.shuffle.sort.io.plugin.class=${{ matrix.shuffle-plugin-class }} test - name: Upload test log diff --git a/client-spark/spark-3-columnar-common/pom.xml b/client-spark/spark-3-columnar-common/pom.xml index 39a36633e..6b4b3c280 100644 --- a/client-spark/spark-3-columnar-common/pom.xml +++ b/client-spark/spark-3-columnar-common/pom.xml @@ -31,7 +31,7 @@ org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} diff --git a/client-spark/spark-3-columnar-shuffle/pom.xml b/client-spark/spark-3-columnar-shuffle/pom.xml index 0a628cc3d..5f0824437 100644 --- a/client-spark/spark-3-columnar-shuffle/pom.xml +++ b/client-spark/spark-3-columnar-shuffle/pom.xml @@ -49,7 +49,7 @@ org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test-jar test diff --git a/client-spark/spark-3-shaded/pom.xml b/client-spark/spark-3-shaded/pom.xml index 9e2c921ab..d3d59cb87 100644 --- a/client-spark/spark-3-shaded/pom.xml +++ b/client-spark/spark-3-shaded/pom.xml @@ -31,7 +31,7 @@ org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} diff --git a/client-spark/spark-3-4/pom.xml b/client-spark/spark-3/pom.xml similarity index 97% rename from client-spark/spark-3-4/pom.xml rename to client-spark/spark-3/pom.xml index 1f89fad9c..47fb034ee 100644 --- a/client-spark/spark-3-4/pom.xml +++ b/client-spark/spark-3/pom.xml @@ -24,7 +24,7 @@ ../../pom.xml - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} jar Celeborn Client for Spark 3 and 4 diff --git a/client-spark/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/CelebornPartitionUtil.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/CelebornPartitionUtil.java similarity index 100% rename from client-spark/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/CelebornPartitionUtil.java rename to client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/CelebornPartitionUtil.java diff --git a/client-spark/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/CelebornShuffleDataIO.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/CelebornShuffleDataIO.java similarity index 100% rename from client-spark/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/CelebornShuffleDataIO.java rename to client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/CelebornShuffleDataIO.java diff --git a/client-spark/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java similarity index 100% rename from client-spark/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java rename to client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java diff --git a/client-spark/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java similarity index 100% rename from client-spark/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java rename to client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java diff --git a/client-spark/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java similarity index 100% rename from client-spark/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java rename to client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java diff --git a/client-spark/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java similarity index 100% rename from client-spark/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java rename to client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java diff --git a/client-spark/spark-3-4/src/main/scala/org/apache/spark/SparkVersionUtil.scala b/client-spark/spark-3/src/main/scala/org/apache/spark/SparkVersionUtil.scala similarity index 100% rename from client-spark/spark-3-4/src/main/scala/org/apache/spark/SparkVersionUtil.scala rename to client-spark/spark-3/src/main/scala/org/apache/spark/SparkVersionUtil.scala diff --git a/client-spark/spark-3-4/src/main/scala/org/apache/spark/celeborn/ExceptionMakerHelper.scala b/client-spark/spark-3/src/main/scala/org/apache/spark/celeborn/ExceptionMakerHelper.scala similarity index 100% rename from client-spark/spark-3-4/src/main/scala/org/apache/spark/celeborn/ExceptionMakerHelper.scala rename to client-spark/spark-3/src/main/scala/org/apache/spark/celeborn/ExceptionMakerHelper.scala diff --git a/client-spark/spark-3-4/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleFallbackPolicyRunner.scala b/client-spark/spark-3/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleFallbackPolicyRunner.scala similarity index 100% rename from client-spark/spark-3-4/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleFallbackPolicyRunner.scala rename to client-spark/spark-3/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleFallbackPolicyRunner.scala diff --git a/client-spark/spark-3-4/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleHandle.scala b/client-spark/spark-3/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleHandle.scala similarity index 100% rename from client-spark/spark-3-4/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleHandle.scala rename to client-spark/spark-3/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleHandle.scala diff --git a/client-spark/spark-3-4/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala b/client-spark/spark-3/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala similarity index 100% rename from client-spark/spark-3-4/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala rename to client-spark/spark-3/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala diff --git a/client-spark/spark-3-4/src/main/scala/org/apache/spark/shuffle/celeborn/ShuffleFetchFailureReportTaskCleanListener.scala b/client-spark/spark-3/src/main/scala/org/apache/spark/shuffle/celeborn/ShuffleFetchFailureReportTaskCleanListener.scala similarity index 100% rename from client-spark/spark-3-4/src/main/scala/org/apache/spark/shuffle/celeborn/ShuffleFetchFailureReportTaskCleanListener.scala rename to client-spark/spark-3/src/main/scala/org/apache/spark/shuffle/celeborn/ShuffleFetchFailureReportTaskCleanListener.scala diff --git a/client-spark/spark-3-4/src/test/java/org/apache/spark/shuffle/celeborn/CelebornPartitionUtilSuiteJ.java b/client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/CelebornPartitionUtilSuiteJ.java similarity index 100% rename from client-spark/spark-3-4/src/test/java/org/apache/spark/shuffle/celeborn/CelebornPartitionUtilSuiteJ.java rename to client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/CelebornPartitionUtilSuiteJ.java diff --git a/client-spark/spark-3-4/src/test/java/org/apache/spark/shuffle/celeborn/CelebornShuffleWriterSuiteBase.java b/client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/CelebornShuffleWriterSuiteBase.java similarity index 100% rename from client-spark/spark-3-4/src/test/java/org/apache/spark/shuffle/celeborn/CelebornShuffleWriterSuiteBase.java rename to client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/CelebornShuffleWriterSuiteBase.java diff --git a/client-spark/spark-3-4/src/test/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriterSuiteJ.java b/client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriterSuiteJ.java similarity index 100% rename from client-spark/spark-3-4/src/test/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriterSuiteJ.java rename to client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriterSuiteJ.java diff --git a/client-spark/spark-3-4/src/test/java/org/apache/spark/shuffle/celeborn/ShuffleManagerHook.java b/client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/ShuffleManagerHook.java similarity index 100% rename from client-spark/spark-3-4/src/test/java/org/apache/spark/shuffle/celeborn/ShuffleManagerHook.java rename to client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/ShuffleManagerHook.java diff --git a/client-spark/spark-3-4/src/test/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriterSuiteJ.java b/client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriterSuiteJ.java similarity index 100% rename from client-spark/spark-3-4/src/test/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriterSuiteJ.java rename to client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriterSuiteJ.java diff --git a/client-spark/spark-3-4/src/test/java/org/apache/spark/shuffle/celeborn/TestCelebornShuffleManager.java b/client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/TestCelebornShuffleManager.java similarity index 100% rename from client-spark/spark-3-4/src/test/java/org/apache/spark/shuffle/celeborn/TestCelebornShuffleManager.java rename to client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/TestCelebornShuffleManager.java diff --git a/client-spark/spark-3-4/src/test/resources/log4j.properties b/client-spark/spark-3/src/test/resources/log4j.properties similarity index 100% rename from client-spark/spark-3-4/src/test/resources/log4j.properties rename to client-spark/spark-3/src/test/resources/log4j.properties diff --git a/client-spark/spark-3-4/src/test/resources/log4j2-test.xml b/client-spark/spark-3/src/test/resources/log4j2-test.xml similarity index 100% rename from client-spark/spark-3-4/src/test/resources/log4j2-test.xml rename to client-spark/spark-3/src/test/resources/log4j2-test.xml diff --git a/client-spark/spark-3-4/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleManagerSuite.scala b/client-spark/spark-3/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleManagerSuite.scala similarity index 100% rename from client-spark/spark-3-4/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleManagerSuite.scala rename to client-spark/spark-3/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleManagerSuite.scala diff --git a/client-spark/spark-3-4/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReaderSuite.scala b/client-spark/spark-3/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReaderSuite.scala similarity index 100% rename from client-spark/spark-3-4/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReaderSuite.scala rename to client-spark/spark-3/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReaderSuite.scala diff --git a/client-spark/spark-4-shaded/pom.xml b/client-spark/spark-4-shaded/pom.xml index 52ce16618..28589c7f8 100644 --- a/client-spark/spark-4-shaded/pom.xml +++ b/client-spark/spark-4-shaded/pom.xml @@ -31,7 +31,7 @@ org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} diff --git a/dev/dependencies.sh b/dev/dependencies.sh index a9d7ab300..6e0ca055a 100755 --- a/dev/dependencies.sh +++ b/dev/dependencies.sh @@ -169,7 +169,7 @@ case "$MODULE" in SBT_PROJECT="celeborn-client-spark-2" ;; "spark-3"*) # Match all versions starting with "spark-3" - MVN_MODULES="client-spark/spark-3-4" + MVN_MODULES="client-spark/spark-3" SBT_PROJECT="celeborn-client-spark-3" ;; "flink-1.16") diff --git a/pom.xml b/pom.xml index 0c3892cd1..fda2da595 100644 --- a/pom.xml +++ b/pom.xml @@ -1372,7 +1372,7 @@ spark-3.0 client-spark/common - client-spark/spark-3-4 + client-spark/spark-3 client-spark/spark-3-columnar-common client-spark/spark-3-columnar-shuffle client-spark/spark-3-shaded @@ -1392,7 +1392,7 @@ spark-3.1 client-spark/common - client-spark/spark-3-4 + client-spark/spark-3 client-spark/spark-3-columnar-common client-spark/spark-3-columnar-shuffle client-spark/spark-3-shaded @@ -1412,7 +1412,7 @@ spark-3.2 client-spark/common - client-spark/spark-3-4 + client-spark/spark-3 client-spark/spark-3-columnar-common client-spark/spark-3-columnar-shuffle client-spark/spark-3-shaded @@ -1431,7 +1431,7 @@ spark-3.3 client-spark/common - client-spark/spark-3-4 + client-spark/spark-3 client-spark/spark-3-columnar-common client-spark/spark-3-columnar-shuffle client-spark/spark-3-shaded @@ -1450,7 +1450,7 @@ spark-3.4 client-spark/common - client-spark/spark-3-4 + client-spark/spark-3 client-spark/spark-3-columnar-common client-spark/spark-3-columnar-shuffle client-spark/spark-3-shaded @@ -1469,7 +1469,7 @@ spark-3.5 client-spark/common - client-spark/spark-3-4 + client-spark/spark-3 client-spark/spark-3-columnar-common client-spark/spark-3.5-columnar-shuffle client-spark/spark-3-shaded @@ -1488,7 +1488,7 @@ spark-4.0 client-spark/common - client-spark/spark-3-4 + client-spark/spark-3 client-spark/spark-3-columnar-common client-spark/spark-4-columnar-shuffle client-spark/spark-4-shaded diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index dae8ed87c..741ad67ca 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -783,7 +783,7 @@ object Spark24 extends SparkClientProjects { object Spark30 extends SparkClientProjects { - val sparkClientProjectPath = "client-spark/spark-3-4" + val sparkClientProjectPath = "client-spark/spark-3" val sparkClientProjectName = "celeborn-client-spark-3" val sparkClientShadedProjectPath = "client-spark/spark-3-shaded" val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded" @@ -797,7 +797,7 @@ object Spark30 extends SparkClientProjects { object Spark31 extends SparkClientProjects { - val sparkClientProjectPath = "client-spark/spark-3-4" + val sparkClientProjectPath = "client-spark/spark-3" val sparkClientProjectName = "celeborn-client-spark-3" val sparkClientShadedProjectPath = "client-spark/spark-3-shaded" val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded" @@ -811,7 +811,7 @@ object Spark31 extends SparkClientProjects { object Spark32 extends SparkClientProjects { - val sparkClientProjectPath = "client-spark/spark-3-4" + val sparkClientProjectPath = "client-spark/spark-3" val sparkClientProjectName = "celeborn-client-spark-3" val sparkClientShadedProjectPath = "client-spark/spark-3-shaded" val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded" @@ -825,7 +825,7 @@ object Spark32 extends SparkClientProjects { object Spark33 extends SparkClientProjects { - val sparkClientProjectPath = "client-spark/spark-3-4" + val sparkClientProjectPath = "client-spark/spark-3" val sparkClientProjectName = "celeborn-client-spark-3" val sparkClientShadedProjectPath = "client-spark/spark-3-shaded" val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded" @@ -842,7 +842,7 @@ object Spark33 extends SparkClientProjects { object Spark34 extends SparkClientProjects { - val sparkClientProjectPath = "client-spark/spark-3-4" + val sparkClientProjectPath = "client-spark/spark-3" val sparkClientProjectName = "celeborn-client-spark-3" val sparkClientShadedProjectPath = "client-spark/spark-3-shaded" val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded" @@ -856,7 +856,7 @@ object Spark34 extends SparkClientProjects { object Spark35 extends SparkClientProjects { - val sparkClientProjectPath = "client-spark/spark-3-4" + val sparkClientProjectPath = "client-spark/spark-3" val sparkClientProjectName = "celeborn-client-spark-3" val sparkClientShadedProjectPath = "client-spark/spark-3-shaded" val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded" @@ -872,7 +872,7 @@ object Spark35 extends SparkClientProjects { object Spark40 extends SparkClientProjects { - val sparkClientProjectPath = "client-spark/spark-3-4" + val sparkClientProjectPath = "client-spark/spark-3" val sparkClientProjectName = "celeborn-client-spark-4" val sparkClientShadedProjectPath = "client-spark/spark-4-shaded" val sparkClientShadedProjectName = "celeborn-client-spark-4-shaded" diff --git a/tests/spark-it/pom.xml b/tests/spark-it/pom.xml index d66aed131..8b15b9c4f 100644 --- a/tests/spark-it/pom.xml +++ b/tests/spark-it/pom.xml @@ -115,13 +115,13 @@ org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test-jar test @@ -133,13 +133,13 @@ org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test-jar test @@ -151,13 +151,13 @@ org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test-jar test @@ -169,13 +169,13 @@ org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test-jar test @@ -187,13 +187,13 @@ org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test-jar test @@ -205,13 +205,13 @@ org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test-jar test @@ -223,13 +223,13 @@ org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test org.apache.celeborn - celeborn-client-spark-3-4_${scala.binary.version} + celeborn-client-spark-3_${scala.binary.version} ${project.version} test-jar test