e85207e2c7
### What changes were proposed in this pull request? This PR partially reverts the change of https://github.com/apache/celeborn/pull/2813, namely, restores the renaming of `celeborn-client-spark-3` ### Why are the changes needed? The renaming is not necessary, and might cause some confusion, for example, I wrongly interpreted the `spark-3-4` as Spark 3.4, it also increases the backport efforts for branch-0.5 ### Does this PR introduce _any_ user-facing change? No, it's dev only, before/after this change, the end users always use the shaded client ``` celeborn-client-spark-2-shaded_2.11-0.6.0-SNAPSHOT.jar celeborn-client-spark-3-shaded_2.12-0.6.0-SNAPSHOT.jar celeborn-client-spark-4-shaded_2.13-0.6.0-SNAPSHOT.jar ``` ### How was this patch tested? Pass GA. Closes #3133 from pan3793/CELEBORN-1413-followup. Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: mingji <fengmingxiao.fmx@alibaba-inc.com>
219 lines
6.7 KiB
YAML
219 lines
6.7 KiB
YAML
# This workflow will build a Java project with Maven
|
|
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
|
|
|
|
name: Celeborn CI
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
- branch-*
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
- branch-*
|
|
|
|
jobs:
|
|
service:
|
|
runs-on: ubuntu-22.04
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
java:
|
|
- 8
|
|
- 11
|
|
- 17
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: zulu
|
|
java-version: ${{ matrix.java }}
|
|
cache: maven
|
|
check-latest: false
|
|
- name: Test Service with Maven
|
|
run: build/mvn -Pgoogle-mirror test
|
|
- name: Upload test log
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: service-java-${{ matrix.java }}-unit-test-log
|
|
path: |
|
|
**/target/test-reports/**
|
|
**/target/unit-tests.log
|
|
|
|
- name: Upload coverage to Codecov
|
|
if: matrix.java == 8
|
|
uses: codecov/codecov-action@v3
|
|
|
|
spark2:
|
|
runs-on: ubuntu-22.04
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
java:
|
|
- 8
|
|
spark:
|
|
- '2.4'
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: zulu
|
|
java-version: ${{ matrix.java }}
|
|
cache: maven
|
|
check-latest: false
|
|
- name: Test with Maven
|
|
run: |
|
|
SPARK_BINARY_VERSION=${{ matrix.spark }}
|
|
SPARK_MAJOR_VERSION=${SPARK_BINARY_VERSION%%.*}
|
|
PROFILES="-Pgoogle-mirror,spark-${{ matrix.spark }}"
|
|
TEST_MODULES="client-spark/common,client-spark/spark-${SPARK_MAJOR_VERSION},client-spark/spark-${SPARK_MAJOR_VERSION}-shaded,tests/spark-it"
|
|
build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
|
|
build/mvn $PROFILES -pl $TEST_MODULES test
|
|
- name: Upload test log
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: spark-${{ matrix.spark }}-java-${{ matrix.java }}-unit-test-log
|
|
path: |
|
|
**/target/test-reports/**
|
|
**/target/unit-tests.log
|
|
|
|
spark3:
|
|
runs-on: ubuntu-22.04
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
java:
|
|
- 8
|
|
- 11
|
|
- 17
|
|
spark:
|
|
- '3.0'
|
|
- '3.1'
|
|
- '3.2'
|
|
- '3.3'
|
|
- '3.4'
|
|
- '3.5'
|
|
shuffle-plugin-class:
|
|
- 'org.apache.spark.shuffle.sort.io.LocalDiskShuffleDataIO'
|
|
- 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
|
|
exclude:
|
|
# SPARK-33772: Spark supports JDK 17 since 3.3.0
|
|
- java: 17
|
|
spark: '3.0'
|
|
- java: 17
|
|
spark: '3.1'
|
|
- java: 17
|
|
spark: '3.2'
|
|
# ShuffleDriverComponents#supportsReliableStorage was introduced in 3.5.0
|
|
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
|
|
spark: '3.0'
|
|
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
|
|
spark: '3.1'
|
|
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
|
|
spark: '3.2'
|
|
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
|
|
spark: '3.3'
|
|
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
|
|
spark: '3.4'
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: zulu
|
|
java-version: ${{ matrix.java }}
|
|
cache: maven
|
|
check-latest: false
|
|
- name: Test with Maven
|
|
run: |
|
|
SPARK_BINARY_VERSION=${{ matrix.spark }}
|
|
SPARK_MAJOR_VERSION=${SPARK_BINARY_VERSION%%.*}
|
|
PROFILES="-Pgoogle-mirror,spark-${{ matrix.spark }}"
|
|
TEST_MODULES="client-spark/common,client-spark/spark-${SPARK_MAJOR_VERSION},client-spark/spark-${SPARK_MAJOR_VERSION}-columnar-common,client-spark/spark-${SPARK_MAJOR_VERSION}-shaded,tests/spark-it"
|
|
build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
|
|
build/mvn $PROFILES -pl $TEST_MODULES -Dspark.shuffle.sort.io.plugin.class=${{ matrix.shuffle-plugin-class }} test
|
|
- name: Upload test log
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: spark-${{ matrix.spark }}-java-${{ matrix.java }}-unit-test-log
|
|
path: |
|
|
**/target/test-reports/**
|
|
**/target/unit-tests.log
|
|
|
|
flink:
|
|
runs-on: ubuntu-22.04
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
java:
|
|
- 8
|
|
- 11
|
|
flink:
|
|
- '1.16'
|
|
- '1.17'
|
|
- '1.18'
|
|
- '1.19'
|
|
- '1.20'
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: zulu
|
|
java-version: ${{ matrix.java }}
|
|
cache: maven
|
|
check-latest: false
|
|
- name: Test with Maven
|
|
run: |
|
|
PROFILES="-Pgoogle-mirror,flink-${{ matrix.flink }}"
|
|
TEST_MODULES="client-flink/common,client-flink/flink-${{ matrix.flink }},client-flink/flink-${{ matrix.flink }}-shaded,tests/flink-it"
|
|
build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
|
|
build/mvn $PROFILES -pl $TEST_MODULES test
|
|
- name: Upload test log
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: flink-${{ matrix.flink }}-java-${{ matrix.java }}-unit-test-log
|
|
path: |
|
|
**/target/test-reports/**
|
|
**/target/unit-tests.log
|
|
|
|
mr:
|
|
runs-on: ubuntu-22.04
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
java:
|
|
- 8
|
|
- 11
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: zulu
|
|
java-version: ${{ matrix.java }}
|
|
cache: maven
|
|
check-latest: false
|
|
- name: Test with Maven
|
|
run: |
|
|
PROFILES="-Pgoogle-mirror,mr"
|
|
TEST_MODULES="client-mr/mr,client-mr/mr-shaded,tests/mr-it"
|
|
build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
|
|
build/mvn $PROFILES -pl $TEST_MODULES test
|
|
- name: Upload test log
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: mr-java-${{ matrix.java }}-unit-test-log
|
|
path: |
|
|
**/target/test/
|
|
**/target/test-reports/**
|
|
**/target/unit-tests.log
|