d5b01fa3e2
### Why are the changes needed? Test Spark 3.5.5 Release Notes https://spark.apache.org/releases/spark-release-3-5-5.html ### How was this patch tested? Pass GHA. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #6939 from pan3793/spark-3.5.5. Closes #6939 8c0288ae5 [Cheng Pan] ga 78b0e72db [Cheng Pan] nit 686a7b0a9 [Cheng Pan] fix d40cc5bba [Cheng Pan] Bump Spark 3.5.5 Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
574 lines
23 KiB
YAML
574 lines
23 KiB
YAML
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
name: CI
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- master
|
|
- branch-*
|
|
pull_request:
|
|
branches:
|
|
- master
|
|
- branch-*
|
|
|
|
concurrency:
|
|
group: test-${{ github.head_ref || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
MAVEN_ARGS: -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Pjdbc-shaded,gen-policy -Dmaven.plugin.download.cache.path=/tmp/engine-archives
|
|
KUBERNETES_VERSION: v1.30.0
|
|
MINIKUBE_VERSION: v1.33.1
|
|
|
|
jobs:
|
|
default:
|
|
name: Kyuubi and Spark Test
|
|
runs-on: ubuntu-24.04
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 6
|
|
matrix:
|
|
java:
|
|
- 8
|
|
- 17
|
|
spark:
|
|
- '3.3'
|
|
- '3.4'
|
|
- '3.5'
|
|
spark-archive: [""]
|
|
exclude-tags: [""]
|
|
comment: ["normal"]
|
|
include:
|
|
- java: 21
|
|
spark: '4.0'
|
|
spark-archive: '-Pscala-2.13'
|
|
exclude-tags: ''
|
|
comment: 'normal'
|
|
- java: 8
|
|
spark: '3.5'
|
|
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
|
|
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
|
|
comment: 'verify-on-spark-3.2-binary'
|
|
- java: 8
|
|
spark: '3.5'
|
|
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
|
|
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
|
|
comment: 'verify-on-spark-3.3-binary'
|
|
- java: 8
|
|
spark: '3.5'
|
|
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.3 -Dspark.archive.name=spark-3.4.3-bin-hadoop3.tgz -Pzookeeper-3.6'
|
|
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
|
|
comment: 'verify-on-spark-3.4-binary'
|
|
- java: 17
|
|
spark: '3.5'
|
|
spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-4.0.0-preview2 -Dspark.archive.name=spark-4.0.0-preview2-bin-hadoop3.tgz'
|
|
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
|
|
comment: 'verify-on-spark-4.0-binary'
|
|
env:
|
|
SPARK_LOCAL_IP: localhost
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Free up disk space
|
|
run: ./.github/scripts/free_disk_space.sh
|
|
- name: Tune Runner VM
|
|
uses: ./.github/actions/tune-runner-vm
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: temurin
|
|
java-version: ${{ matrix.java }}
|
|
cache: 'maven'
|
|
- name: Cache Engine Archives
|
|
uses: ./.github/actions/cache-engine-archives
|
|
- name: Setup Maven
|
|
uses: ./.github/actions/setup-maven
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.9'
|
|
- name: Build and test Kyuubi and Spark with maven w/o linters
|
|
run: |
|
|
if [[ "${{ matrix.java }}" == "8" && "${{ matrix.spark }}" == "3.5" && "${{ matrix.spark-archive }}" == "" ]]; then
|
|
export MAVEN_ARGS="${MAVEN_ARGS} -Pcodecov -Dmaven.javadoc.skip=false -Dmaven.scaladoc.skip=false"
|
|
fi
|
|
TEST_MODULES="dev/kyuubi-codecov"
|
|
./build/mvn clean install -pl ${TEST_MODULES} -am \
|
|
-Pjava-${{ matrix.java }} -Pspark-${{ matrix.spark }} ${{ matrix.spark-archive }} ${{ matrix.exclude-tags }}
|
|
- name: Code coverage
|
|
if: |
|
|
matrix.java == 8 &&
|
|
matrix.spark == '3.5' &&
|
|
matrix.spark-archive == ''
|
|
uses: codecov/codecov-action@v3
|
|
with:
|
|
verbose: true
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: unit-tests-log-java-${{ matrix.java }}-spark-${{ matrix.spark }}-${{ matrix.comment }}
|
|
path: |
|
|
**/target/unit-tests.log
|
|
**/kyuubi-spark-sql-engine.log*
|
|
**/kyuubi-spark-batch-submit.log*
|
|
|
|
scala-test:
|
|
name: Scala Test
|
|
runs-on: ubuntu-24.04
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
scala:
|
|
- '2.13'
|
|
java:
|
|
- '8'
|
|
spark:
|
|
- '3.5'
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Free up disk space
|
|
run: ./.github/scripts/free_disk_space.sh
|
|
- name: Tune Runner VM
|
|
uses: ./.github/actions/tune-runner-vm
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: temurin
|
|
java-version: ${{ matrix.java }}
|
|
cache: 'maven'
|
|
- name: Setup Maven
|
|
uses: ./.github/actions/setup-maven
|
|
- name: Cache Engine Archives
|
|
uses: ./.github/actions/cache-engine-archives
|
|
- name: Build on Scala ${{ matrix.scala }}
|
|
run: |
|
|
TEST_MODULES="!externals/kyuubi-flink-sql-engine,!integration-tests/kyuubi-flink-it"
|
|
./build/mvn clean install -pl ${TEST_MODULES} -am \
|
|
-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.SparkLocalClusterTest \
|
|
-Pscala-${{ matrix.scala }} -Pjava-${{ matrix.java }} -Pspark-${{ matrix.spark }}
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: unit-tests-log-scala-${{ matrix.scala }}-java-${{ matrix.java }}-spark-${{ matrix.spark }}
|
|
path: |
|
|
**/target/unit-tests.log
|
|
**/kyuubi-spark-sql-engine.log*
|
|
**/kyuubi-spark-batch-submit.log*
|
|
**/kyuubi-jdbc-engine.log*
|
|
**/kyuubi-hive-sql-engine.log*
|
|
|
|
spark-connector-cross-version-test:
|
|
name: Spark Connector Cross Version Test
|
|
runs-on: ubuntu-24.04
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 1
|
|
matrix:
|
|
java:
|
|
- 17
|
|
scala:
|
|
- '2.12'
|
|
- '2.13'
|
|
spark-compile:
|
|
- '3.5'
|
|
spark-runtime:
|
|
- '3.4'
|
|
- '3.3'
|
|
comment: [ "normal" ]
|
|
include:
|
|
- java: 17
|
|
scala: "2.13"
|
|
spark-compile: "3.5"
|
|
spark-runtime: "4.0"
|
|
comment: "normal"
|
|
env:
|
|
SPARK_LOCAL_IP: localhost
|
|
TEST_MODULES: "extensions/spark/kyuubi-spark-connector-hive,\
|
|
extensions/spark/kyuubi-spark-connector-tpcds,\
|
|
extensions/spark/kyuubi-spark-connector-tpch"
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Free up disk space
|
|
run: ./.github/scripts/free_disk_space.sh
|
|
- name: Tune Runner VM
|
|
uses: ./.github/actions/tune-runner-vm
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: temurin
|
|
java-version: ${{ matrix.java }}
|
|
cache: 'maven'
|
|
- name: Setup Maven
|
|
uses: ./.github/actions/setup-maven
|
|
- name: Build Kyuubi Spark Connectors with Spark-${{ matrix.spark-compile }}
|
|
run: |
|
|
./build/mvn clean install -pl ${TEST_MODULES} -am -Pjava-${{ matrix.java }} \
|
|
-Pscala-${{ matrix.scala }} -Pspark-${{ matrix.spark-compile }} -DskipTests
|
|
- name: Test Kyuubi Spark Connectors with Spark-${{ matrix.spark-runtime }}
|
|
run: |
|
|
./build/mvn test -pl ${TEST_MODULES} -Pjava-${{ matrix.java }} \
|
|
-Pscala-${{ matrix.scala }} -Pspark-${{ matrix.spark-runtime }} -Pcross-version-test
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: "unit-tests-log-java-${{ matrix.java }}-scala-${{ matrix.scala }}\
|
|
-spark-compile-${{ matrix.spark-compile }}-spark-runtime-${{ matrix.spark-runtime }}\
|
|
-${{ matrix.comment }}"
|
|
path: |
|
|
**/target/unit-tests.log
|
|
|
|
flink-it:
|
|
name: Flink Test
|
|
runs-on: ubuntu-24.04
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 2
|
|
matrix:
|
|
java:
|
|
- 8
|
|
flink:
|
|
- '1.17'
|
|
- '1.18'
|
|
- '1.19'
|
|
- '1.20'
|
|
flink-archive: [ "" ]
|
|
comment: [ "normal" ]
|
|
include:
|
|
- java: 8
|
|
flink: '1.20'
|
|
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.17.2 -Dflink.archive.name=flink-1.17.2-bin-scala_2.12.tgz'
|
|
comment: 'verify-on-flink-1.17-binary'
|
|
- java: 8
|
|
flink: '1.20'
|
|
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.18.1 -Dflink.archive.name=flink-1.18.1-bin-scala_2.12.tgz'
|
|
comment: 'verify-on-flink-1.18-binary'
|
|
- java: 8
|
|
flink: '1.20'
|
|
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.19.1 -Dflink.archive.name=flink-1.19.1-bin-scala_2.12.tgz'
|
|
comment: 'verify-on-flink-1.19-binary'
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Free up disk space
|
|
run: ./.github/scripts/free_disk_space.sh
|
|
- name: Tune Runner VM
|
|
uses: ./.github/actions/tune-runner-vm
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: temurin
|
|
java-version: ${{ matrix.java }}
|
|
cache: 'maven'
|
|
- name: Setup Maven
|
|
uses: ./.github/actions/setup-maven
|
|
- name: Cache Engine Archives
|
|
uses: ./.github/actions/cache-engine-archives
|
|
- name: Build Flink with maven w/o linters
|
|
run: |
|
|
TEST_MODULES="externals/kyuubi-flink-sql-engine,integration-tests/kyuubi-flink-it"
|
|
./build/mvn -pl ${TEST_MODULES} -Pflink-${{ matrix.flink }} ${{ matrix.flink-archive }} -am clean install -DskipTests
|
|
- name: Test Flink
|
|
if: matrix.flink-archive == ''
|
|
run: |
|
|
TEST_MODULES="externals/kyuubi-flink-sql-engine,integration-tests/kyuubi-flink-it"
|
|
./build/mvn -pl ${TEST_MODULES} -Pflink-${{ matrix.flink }} ${{ matrix.flink-archive }} test
|
|
- name: Cross-version test Flink
|
|
if: matrix.flink-archive != ''
|
|
run: |
|
|
IT_FLINK=`echo "${{ matrix.flink-archive }}" | grep -E 'flink\-([0-9]+\.[0-9]+.[0-9]+)\-bin' -o | grep -E '[0-9]+\.[0-9]+' -o`
|
|
IT_MODULE="integration-tests/kyuubi-flink-it"
|
|
./build/mvn -pl ${IT_MODULE} -Pflink-${IT_FLINK} ${{ matrix.flink-archive }} test
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: unit-tests-log-java-${{ matrix.java }}-flink-${{ matrix.flink }}-${{ matrix.comment }}
|
|
path: |
|
|
**/target/unit-tests.log
|
|
**/kyuubi-flink-sql-engine.log*
|
|
|
|
hive-it:
|
|
name: Hive Test
|
|
runs-on: ubuntu-24.04
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 1
|
|
matrix:
|
|
java:
|
|
- 8
|
|
hive-archive: [ "" ]
|
|
comment: [ "normal" ]
|
|
include:
|
|
- java: 8
|
|
hive-archive: '-Dhive.archive.mirror=https://archive.apache.org/dist/hive/hive-2.3.10 -Dhive.archive.name=apache-hive-2.3.10-bin.tar.gz'
|
|
comment: 'verify-on-hive-2.3-binary'
|
|
- java: 8
|
|
hive-archive: '-Dhive.archive.mirror=https://github.com/pan3793/cdh-hive/releases/download/cdh6.3.2-release -Dhive.archive.name=apache-hive-2.1.1-cdh6.3.2-bin.tar.gz'
|
|
comment: 'verify-on-hive-2.1-cdh6-binary'
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Free up disk space
|
|
run: ./.github/scripts/free_disk_space.sh
|
|
- name: Tune Runner VM
|
|
uses: ./.github/actions/tune-runner-vm
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: temurin
|
|
java-version: ${{ matrix.java }}
|
|
cache: 'maven'
|
|
- name: Setup Maven
|
|
uses: ./.github/actions/setup-maven
|
|
- name: Cache Engine Archives
|
|
uses: ./.github/actions/cache-engine-archives
|
|
- name: Build and test Hive with maven w/o linters
|
|
run: |
|
|
TEST_MODULES="externals/kyuubi-hive-sql-engine,integration-tests/kyuubi-hive-it"
|
|
./build/mvn ${{ matrix.hive-archive }} -pl ${TEST_MODULES} -am clean install -DskipTests
|
|
# unpack avatica-1.8.0.jar and delete Jackson classes to workaround CALCITE-1224
|
|
if [[ "${{ matrix.hive-archive }}" == *apache-hive-2.3.10-bin.tar.gz* ]]; then
|
|
HIVE_2310_LIB="$PWD/externals/kyuubi-download/target/apache-hive-2.3.10-bin/lib"
|
|
mkdir avatica-1.8.0-unpack
|
|
cd avatica-1.8.0-unpack
|
|
jar xf $HIVE_2310_LIB/avatica-1.8.0.jar
|
|
rm -rf com/fasterxml/jackson org/slf4j
|
|
jar cf avatica-1.8.0-patched.jar .
|
|
rm $HIVE_2310_LIB/avatica-1.8.0.jar
|
|
mv avatica-1.8.0-patched.jar $HIVE_2310_LIB/
|
|
cd ..
|
|
rm -rf avatica-1.8.0-unpack
|
|
fi
|
|
./build/mvn ${{ matrix.hive-archive }} -pl ${TEST_MODULES} test
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: unit-tests-log-java-${{ matrix.java }}-hive-${{ matrix.comment }}
|
|
path: |
|
|
**/target/unit-tests.log
|
|
**/kyuubi-hive-sql-engine.log*
|
|
|
|
jdbc-trino-tpc-it:
|
|
name: JDBC Trino TPC Tests
|
|
runs-on: ubuntu-24.04
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
java: [ 8 ]
|
|
comment: [ "normal" ]
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Free up disk space
|
|
run: ./.github/scripts/free_disk_space.sh
|
|
- name: Tune Runner VM
|
|
uses: ./.github/actions/tune-runner-vm
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: temurin
|
|
java-version: ${{ matrix.java }}
|
|
cache: 'maven'
|
|
- name: Setup Maven
|
|
uses: ./.github/actions/setup-maven
|
|
- name: Cache Engine Archives
|
|
uses: ./.github/actions/cache-engine-archives
|
|
- name: Build and test JDBC and Trino engines, TPC-H/TPC-DS with maven w/o linters
|
|
run: |
|
|
JDBC_TEST_MODULES="externals/kyuubi-jdbc-engine,integration-tests/kyuubi-jdbc-it"
|
|
TRINO_TEST_MODULES="externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it"
|
|
TPC_TEST_MODULES="kyuubi-server,extensions/spark/kyuubi-spark-connector-tpcds,extensions/spark/kyuubi-spark-connector-tpch"
|
|
./build/mvn -pl ${JDBC_TEST_MODULES},${TRINO_TEST_MODULES},${TPC_TEST_MODULES} -am clean install -DskipTests
|
|
./build/mvn -pl ${JDBC_TEST_MODULES} test
|
|
./build/mvn -pl ${TRINO_TEST_MODULES} test
|
|
./build/mvn -pl ${TPC_TEST_MODULES} test \
|
|
-Dmaven.plugin.scalatest.exclude.tags='' \
|
|
-Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds,org.apache.kyuubi.spark.connector.tpcds.TPCDSQuerySuite,org.apache.kyuubi.spark.connector.tpch.TPCHQuerySuite
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: unit-tests-log-java-${{ matrix.java }}-hive-${{ matrix.comment }}
|
|
path: |
|
|
**/target/unit-tests.log
|
|
**/kyuubi-jdbc-engine.log*
|
|
**/kyuubi-trino-engine.log*
|
|
|
|
kyuubi-on-k8s-it:
|
|
name: Kyuubi Server On Kubernetes Integration Test
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Free up disk space
|
|
run: ./.github/scripts/free_disk_space.sh
|
|
# https://github.com/docker/build-push-action
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
with:
|
|
cache-binary: false
|
|
- name: Build Kyuubi Docker Image
|
|
uses: docker/build-push-action@v6
|
|
with:
|
|
# passthrough CI into build container
|
|
build-args: |
|
|
CI=${CI}
|
|
MVN_ARG=--flink-provided --hive-provided
|
|
context: .
|
|
file: build/Dockerfile.CI
|
|
load: true
|
|
tags: apache/kyuubi:latest
|
|
- name: Cache Engine Archives
|
|
uses: ./.github/actions/cache-engine-archives
|
|
- name: Setup Minikube
|
|
run: |
|
|
# https://minikube.sigs.k8s.io/docs/start/
|
|
curl -LO https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64
|
|
sudo install minikube-linux-amd64 /usr/local/bin/minikube
|
|
minikube start --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --force
|
|
# https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime
|
|
minikube image load apache/kyuubi:latest
|
|
# pre-install spark into minikube
|
|
docker pull apache/spark:3.5.5
|
|
minikube image load apache/spark:3.5.5
|
|
- name: kubectl pre-check
|
|
run: |
|
|
kubectl get nodes
|
|
kubectl get serviceaccount
|
|
kubectl create serviceaccount kyuubi
|
|
kubectl create clusterrolebinding kyuubi-role --clusterrole=edit --serviceaccount=default:kyuubi
|
|
kubectl get serviceaccount
|
|
- name: start kyuubi
|
|
run: kubectl apply -f integration-tests/kyuubi-kubernetes-it/src/test/resources/kyuubi-server.yaml
|
|
- name: kyuubi pod check
|
|
run: |
|
|
kubectl get pods
|
|
kubectl describe pods kyuubi-test
|
|
- name: integration tests
|
|
run: >-
|
|
./build/mvn clean install
|
|
-pl integration-tests/kyuubi-kubernetes-it -am
|
|
-Pkubernetes-it
|
|
-Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.deployment,org.apache.kyuubi.kubernetes.test.KubernetesUtilsTest
|
|
- name: Cat kyuubi server log
|
|
if: failure()
|
|
run: kubectl logs kyuubi-test
|
|
- name: Copy spark engine log from kyuubi pod
|
|
if: failure()
|
|
run: kubectl cp kyuubi-test:/opt/kyuubi/work ./target/work
|
|
- name: Cat spark driver log
|
|
if: failure()
|
|
run: |
|
|
kubectl get pods
|
|
kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I {} kubectl logs {}
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: unit-tests-log-kyuubi-on-k8s-it
|
|
path: |
|
|
**/target/unit-tests.log
|
|
**/target/work/**
|
|
|
|
spark-on-k8s-it:
|
|
name: Spark Engine On Kubernetes Integration Test
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Free up disk space
|
|
run: ./.github/scripts/free_disk_space.sh
|
|
- name: Cache Engine Archives
|
|
uses: ./.github/actions/cache-engine-archives
|
|
- name: Setup JDK 11
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: temurin
|
|
java-version: 11
|
|
cache: 'maven'
|
|
- name: Setup Minikube
|
|
run: |
|
|
# https://minikube.sigs.k8s.io/docs/start/
|
|
curl -LO https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64
|
|
sudo install minikube-linux-amd64 /usr/local/bin/minikube
|
|
minikube start --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --force
|
|
# in case: https://spark.apache.org/docs/latest/running-on-kubernetes.html#rbac
|
|
- name: Create Service Account
|
|
run: |
|
|
kubectl create serviceaccount spark
|
|
kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=default:spark --namespace=default
|
|
kubectl get serviceaccount
|
|
# TODO Support more Spark version
|
|
- name: integration tests
|
|
run: >-
|
|
./build/mvn clean install
|
|
-Pflink-provided,hive-provided
|
|
-Pkubernetes-it
|
|
-Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.spark
|
|
- name: Print Driver Pod logs
|
|
if: failure()
|
|
run: |
|
|
kubectl get pods
|
|
kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I {} kubectl logs {}
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: unit-tests-log-spark-on-k8s-it
|
|
path: |
|
|
**/target/unit-tests.log
|
|
**/kyuubi-spark-sql-engine.log*
|
|
**/kyuubi-spark-batch-submit.log*
|
|
|
|
zookeeper-it:
|
|
name: Zookeeper Integration Test
|
|
runs-on: ubuntu-24.04
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 1
|
|
matrix:
|
|
java: [ 8 ]
|
|
zookeeper: ["3.4", "3.5", "3.6", "3.7" ]
|
|
comment: [ "normal" ]
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Tune Runner VM
|
|
uses: ./.github/actions/tune-runner-vm
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
distribution: temurin
|
|
java-version: ${{ matrix.java }}
|
|
cache: 'maven'
|
|
- name: Setup Maven
|
|
uses: ./.github/actions/setup-maven
|
|
- name: Cache Engine Archives
|
|
uses: ./.github/actions/cache-engine-archives
|
|
- name: zookeeper integration tests
|
|
run: |
|
|
export KYUUBI_IT_ZOOKEEPER_VERSION=${{ matrix.zookeeper }}
|
|
TEST_MODULES="integration-tests/kyuubi-zookeeper-it"
|
|
./build/mvn -pl ${TEST_MODULES} -am clean install -DskipTests
|
|
./build/mvn -pl ${TEST_MODULES} test
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: unit-tests-log-java-${{ matrix.java }}-zookeeper-${{ matrix.comment }}
|
|
path: |
|
|
**/target/unit-tests.log
|