3054813a0f
### What changes were proposed in this pull request? ### Why are the changes needed? ### Does this PR introduce _any_ user-facing change? ### How was this patch tested? Closes #2073 from cxzl25/CELEBORN-856. Authored-by: sychen <sychen@ctrip.com> Signed-off-by: mingji <fengmingxiao.fmx@alibaba-inc.com>
251 lines
7.0 KiB
YAML
251 lines
7.0 KiB
YAML
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
|
|
name: Celeborn SBT CI
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
- branch-*
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
- branch-*
|
|
|
|
jobs:
|
|
service:
|
|
runs-on: ubuntu-22.04
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
java:
|
|
- 8
|
|
- 11
|
|
- 17
|
|
scala:
|
|
- '2.12.15'
|
|
- '2.13.5'
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v2
|
|
with:
|
|
distribution: zulu
|
|
java-version: ${{ matrix.java }}
|
|
check-latest: false
|
|
- name: Test Service with SBT
|
|
run: |
|
|
build/sbt ++${{ matrix.scala }} "clean; test"
|
|
- name: Upload test log
|
|
if: failure()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: service-java-${{ matrix.java }}-scala-${{ matrix.scala }}-unit-test-log
|
|
path: |
|
|
**/target/test-reports/**
|
|
|
|
spark2:
|
|
runs-on: ubuntu-22.04
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
java:
|
|
- 8
|
|
spark:
|
|
- '2.4'
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v2
|
|
with:
|
|
distribution: zulu
|
|
java-version: ${{ matrix.java }}
|
|
check-latest: false
|
|
- name: Test with SBT
|
|
run: |
|
|
build/sbt -Pspark-${{ matrix.spark }} "clean; celeborn-spark-group/test"
|
|
- name: Upload test log
|
|
if: failure()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: spark-${{ matrix.spark }}-unit-test-log
|
|
path: |
|
|
**/target/test-reports/**
|
|
|
|
spark3:
|
|
runs-on: ubuntu-22.04
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
spark:
|
|
- '3.0'
|
|
- '3.1'
|
|
- '3.2'
|
|
- '3.3'
|
|
- '3.4'
|
|
- '3.5'
|
|
java:
|
|
- 8
|
|
- 11
|
|
- 17
|
|
scala-binary:
|
|
- '2.12'
|
|
- '2.13'
|
|
shuffle-plugin-class:
|
|
- 'org.apache.spark.shuffle.sort.io.LocalDiskShuffleDataIO'
|
|
- 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
|
|
exclude:
|
|
# SPARK-33772: Spark supports JDK 17 since 3.3.0
|
|
- java: 17
|
|
spark: '3.0'
|
|
- java: 17
|
|
spark: '3.1'
|
|
- java: 17
|
|
spark: '3.2'
|
|
# Spark supports scala 2.13 since 3.2.0
|
|
- scala-binary: '2.13'
|
|
spark: '3.0'
|
|
- scala-binary: '2.13'
|
|
spark: '3.1'
|
|
# ShuffleDriverComponents#supportsReliableStorage was introduced in 3.5.0
|
|
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
|
|
spark: '3.0'
|
|
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
|
|
spark: '3.1'
|
|
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
|
|
spark: '3.2'
|
|
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
|
|
spark: '3.3'
|
|
- shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
|
|
spark: '3.4'
|
|
include:
|
|
# Spark 3.0
|
|
- spark: '3.0'
|
|
scala-binary: '2.12'
|
|
scala: '2.12.10'
|
|
# Spark 3.1
|
|
- spark: '3.1'
|
|
scala-binary: '2.12'
|
|
scala: '2.12.10'
|
|
- spark: '3.1'
|
|
# Spark 3.2
|
|
- spark: '3.2'
|
|
scala-binary: '2.12'
|
|
scala: '2.12.15'
|
|
- spark: '3.2'
|
|
scala-binary: '2.13'
|
|
scala: '2.13.5'
|
|
# Spark 3.3
|
|
- spark: '3.3'
|
|
scala-binary: '2.12'
|
|
scala: '2.12.15'
|
|
- spark: '3.3'
|
|
scala-binary: '2.13'
|
|
scala: '2.13.5'
|
|
# Spark 3.4
|
|
- spark: '3.4'
|
|
scala-binary: '2.12'
|
|
scala: '2.12.17'
|
|
- spark: '3.4'
|
|
scala-binary: '2.13'
|
|
scala: '2.13.5'
|
|
# Spark 3.5
|
|
- spark: '3.5'
|
|
scala-binary: '2.12'
|
|
scala: '2.12.18'
|
|
- spark: '3.5'
|
|
scala-binary: '2.13'
|
|
scala: '2.13.5'
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v2
|
|
with:
|
|
distribution: zulu
|
|
java-version: ${{ matrix.java }}
|
|
check-latest: false
|
|
- name: Test with SBT
|
|
run: |
|
|
build/sbt -Dspark.shuffle.plugin.class=${{ matrix.shuffle-plugin-class }} -Pspark-${{ matrix.spark }} ++${{ matrix.scala }} "clean; celeborn-spark-group/test"
|
|
- name: Upload test log
|
|
if: failure()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: spark-${{ matrix.spark }}-scala-${{ matrix.scala }}-unit-test-log
|
|
path: |
|
|
**/target/test-reports/**
|
|
|
|
flink:
|
|
runs-on: ubuntu-22.04
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
java:
|
|
- 8
|
|
- 11
|
|
flink:
|
|
- '1.14'
|
|
- '1.15'
|
|
- '1.17'
|
|
- '1.18'
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v2
|
|
with:
|
|
distribution: zulu
|
|
java-version: ${{ matrix.java }}
|
|
check-latest: false
|
|
- name: Test with SBT
|
|
run: |
|
|
build/sbt -Pflink-${{ matrix.flink }} "clean; celeborn-flink-group/test"
|
|
- name: Upload test log
|
|
if: failure()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: flink-${{ matrix.flink }}-unit-test-log
|
|
path: |
|
|
**/target/test-reports/**
|
|
|
|
mr:
|
|
runs-on: ubuntu-22.04
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
java:
|
|
- 8
|
|
- 11
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v2
|
|
with:
|
|
distribution: zulu
|
|
java-version: ${{ matrix.java }}
|
|
check-latest: false
|
|
- name: Test with SBT
|
|
run: |
|
|
build/sbt -Pmr "clean; celeborn-mr-group/test"
|
|
- name: Upload test log
|
|
if: failure()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: mr-unit-test-log
|
|
path: |
|
|
**/target/test-reports/** |