[CELEBORN-850][INFRA] Add SBT CI
### What changes were proposed in this pull request? This PR adds new GitHub Actions workflows to enable Continuous Integration using SBT based on #1764 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Pass GA Closes #1771 from cfmcgrady/sbt-ci. Authored-by: Fu Chen <cfmcgrady@gmail.com> Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
Fu Chen
Cheng Pan
parent
40e416c95c
commit
6ba4b7e138
163
.github/workflows/sbt.yml
vendored
Normal file
163
.github/workflows/sbt.yml
vendored
Normal file
@ -0,0 +1,163 @@
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
|
||||
name: Celeborn SBT CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- branch-*
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- branch-*
|
||||
|
||||
jobs:
|
||||
service:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
java:
|
||||
- 8
|
||||
- 11
|
||||
- 17
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Setup JDK ${{ matrix.java }}
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
distribution: zulu
|
||||
java-version: ${{ matrix.java }}
|
||||
cache: maven
|
||||
check-latest: false
|
||||
- name: Test Service with SBT
|
||||
run: |
|
||||
build/sbt "clean; test"
|
||||
- name: Upload test log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: service-${{ matrix.java }}-unit-test-log
|
||||
path: |
|
||||
**/target/test-reports/**
|
||||
|
||||
spark2:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
java:
|
||||
- 8
|
||||
spark:
|
||||
- '2.4'
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Setup JDK ${{ matrix.java }}
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
distribution: zulu
|
||||
java-version: ${{ matrix.java }}
|
||||
cache: maven
|
||||
check-latest: false
|
||||
- name: Test with SBT
|
||||
run: |
|
||||
build/sbt -Pspark-${{ matrix.spark }} "clean; celeborn-spark-group/test"
|
||||
- name: Upload test log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: spark-${{ matrix.spark }}-unit-test-log
|
||||
path: |
|
||||
**/target/test-reports/**
|
||||
|
||||
spark3:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
java:
|
||||
- 8
|
||||
- 11
|
||||
- 17
|
||||
spark:
|
||||
- '3.0'
|
||||
- '3.1'
|
||||
- '3.2'
|
||||
- '3.3'
|
||||
- '3.4'
|
||||
exclude:
|
||||
# SPARK-33772: Spark supports JDK 17 since 3.3.0
|
||||
- java: 17
|
||||
spark: '3.0'
|
||||
- java: 17
|
||||
spark: '3.1'
|
||||
- java: 17
|
||||
spark: '3.2'
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Setup JDK ${{ matrix.java }}
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
distribution: zulu
|
||||
java-version: ${{ matrix.java }}
|
||||
cache: maven
|
||||
check-latest: false
|
||||
- name: Test with SBT
|
||||
run: |
|
||||
build/sbt -Pspark-${{ matrix.spark }} "clean; celeborn-spark-group/test"
|
||||
- name: Upload test log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: spark-${{ matrix.spark }}-unit-test-log
|
||||
path: |
|
||||
**/target/test-reports/**
|
||||
|
||||
flink:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
java:
|
||||
- 8
|
||||
- 11
|
||||
flink:
|
||||
- '1.14'
|
||||
- '1.15'
|
||||
- '1.17'
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Setup JDK ${{ matrix.java }}
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
distribution: zulu
|
||||
java-version: ${{ matrix.java }}
|
||||
cache: maven
|
||||
check-latest: false
|
||||
- name: Test with SBT
|
||||
run: |
|
||||
build/sbt -Pflink-${{ matrix.flink }} "clean; celeborn-flink-group/test"
|
||||
- name: Upload test log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: flink-${{ matrix.flink }}-unit-test-log
|
||||
path: |
|
||||
**/target/test-reports/**
|
||||
@ -89,7 +89,23 @@ object CelebornCommonSettings {
|
||||
|
||||
// Make sure any tests in any project that uses Spark is configured for running well locally
|
||||
Test / javaOptions ++= Seq(
|
||||
"-Xmx4g"
|
||||
"-Xmx4g",
|
||||
"-XX:+IgnoreUnrecognizedVMOptions",
|
||||
"--add-opens=java.base/java.lang=ALL-UNNAMED",
|
||||
"--add-opens=java.base/java.lang.invoke=ALL-UNNAMED",
|
||||
"--add-opens=java.base/java.lang.reflect=ALL-UNNAMED",
|
||||
"--add-opens=java.base/java.io=ALL-UNNAMED",
|
||||
"--add-opens=java.base/java.net=ALL-UNNAMED",
|
||||
"--add-opens=java.base/java.nio=ALL-UNNAMED",
|
||||
"--add-opens=java.base/java.util=ALL-UNNAMED",
|
||||
"--add-opens=java.base/java.util.concurrent=ALL-UNNAMED",
|
||||
"--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED",
|
||||
"--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED",
|
||||
"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
|
||||
"--add-opens=java.base/sun.nio.cs=ALL-UNNAMED",
|
||||
"--add-opens=java.base/sun.security.action=ALL-UNNAMED",
|
||||
"--add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
|
||||
"-Dio.netty.tryReflectionSetAccessible=true"
|
||||
),
|
||||
|
||||
testOptions += Tests.Argument("-oF"),
|
||||
@ -125,7 +141,7 @@ object CelebornCommonSettings {
|
||||
lazy val commonUnitTestDependencies = Seq(
|
||||
"org.mockito" % "mockito-core" % "4.11.0" % "test",
|
||||
"org.scalatest" %% "scalatest" % "3.2.16" % "test",
|
||||
"junit" % "junit" % "4.12" % "test",
|
||||
"junit" % "junit" % "4.13.2" % "test",
|
||||
// https://www.scala-sbt.org/1.x/docs/Testing.html
|
||||
"com.github.sbt" % "junit-interface" % "0.13.3" % "test")
|
||||
}
|
||||
@ -489,7 +505,11 @@ trait SparkClientProjects {
|
||||
val sparkVersion: String
|
||||
val zstdJniVersion: String
|
||||
|
||||
def modules: Seq[Project] = Seq(sparkCommon, sparkClient, sparkIt, sparkClientShade)
|
||||
def modules: Seq[Project] = Seq(sparkCommon, sparkClient, sparkIt, sparkGroup, sparkClientShade)
|
||||
|
||||
// for test only, don't use this group for any other projects
|
||||
lazy val sparkGroup = (project withId "celeborn-spark-group")
|
||||
.aggregate(sparkCommon, sparkClient, sparkIt)
|
||||
|
||||
def sparkCommon: Project = {
|
||||
Project("celeborn-spark-common", file("client-spark/common"))
|
||||
@ -553,7 +573,7 @@ trait SparkClientProjects {
|
||||
) ++ commonUnitTestDependencies
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def sparkClientShade: Project = {
|
||||
Project(sparkClientShadedProjectName, file(sparkClientShadedProjectPath))
|
||||
.dependsOn(sparkClient)
|
||||
@ -657,7 +677,11 @@ trait FlinkClientProjects {
|
||||
lazy val flinkClientsDependency: ModuleID = "org.apache.flink" % "flink-clients" % flinkVersion % "test"
|
||||
lazy val flinkRuntimeWebDependency: ModuleID = "org.apache.flink" % "flink-runtime-web" % flinkVersion % "test"
|
||||
|
||||
def modules: Seq[Project] = Seq(flinkCommon, flinkClient, flinkIt, flinkClientShade)
|
||||
def modules: Seq[Project] = Seq(flinkCommon, flinkClient, flinkIt, flinkGroup, flinkClientShade)
|
||||
|
||||
// for test only, don't use this group for any other projects
|
||||
lazy val flinkGroup = (project withId "celeborn-flink-group")
|
||||
.aggregate(flinkCommon, flinkClient, flinkIt)
|
||||
|
||||
// get flink major version. e.g:
|
||||
// 1.17.0 -> 1.17
|
||||
|
||||
Loading…
Reference in New Issue
Block a user