# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # name: CI on: push: branches: - master - branch-* pull_request: branches: - master - branch-* concurrency: group: test-${{ github.head_ref || github.run_id }} cancel-in-progress: true env: MAVEN_ARGS: -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Pjdbc-shaded,gen-policy -Dmaven.plugin.download.cache.path=/tmp/engine-archives KUBERNETES_VERSION: v1.30.0 MINIKUBE_VERSION: v1.33.1 jobs: default: name: Kyuubi and Spark Test runs-on: ubuntu-24.04 strategy: fail-fast: false max-parallel: 6 matrix: java: - 8 - 17 spark: - '3.3' - '3.4' - '3.5' spark-archive: [""] exclude-tags: [""] comment: ["normal"] include: - java: 21 spark: '4.0' spark-archive: '-Pscala-2.13' exclude-tags: '' comment: 'normal' - java: 8 spark: '3.5' spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6' exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest' comment: 'verify-on-spark-3.2-binary' - java: 8 spark: '3.5' spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6' exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest' comment: 'verify-on-spark-3.3-binary' - java: 8 spark: '3.5' spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.3 -Dspark.archive.name=spark-3.4.3-bin-hadoop3.tgz -Pzookeeper-3.6' exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest' comment: 'verify-on-spark-3.4-binary' - java: 17 spark: '3.5' spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-4.0.0-preview2 -Dspark.archive.name=spark-4.0.0-preview2-bin-hadoop3.tgz' exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest' comment: 'verify-on-spark-4.0-binary' env: SPARK_LOCAL_IP: localhost steps: - uses: actions/checkout@v4 - name: Free up disk space run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} uses: actions/setup-java@v4 with: distribution: temurin java-version: ${{ matrix.java }} cache: 'maven' - name: Cache Engine Archives uses: ./.github/actions/cache-engine-archives - name: Setup Maven uses: ./.github/actions/setup-maven - name: Setup Python uses: actions/setup-python@v5 with: python-version: '3.9' - name: Build and test Kyuubi and Spark with maven w/o linters run: | if [[ "${{ matrix.java }}" == "8" && "${{ matrix.spark }}" == "3.5" && "${{ matrix.spark-archive }}" == "" ]]; then export MAVEN_ARGS="${MAVEN_ARGS} -Pcodecov -Dmaven.javadoc.skip=false -Dmaven.scaladoc.skip=false" fi TEST_MODULES="dev/kyuubi-codecov" ./build/mvn clean install -pl ${TEST_MODULES} -am \ -Pjava-${{ matrix.java }} -Pspark-${{ matrix.spark }} ${{ matrix.spark-archive }} ${{ matrix.exclude-tags }} - name: Code coverage if: | matrix.java == 8 && matrix.spark == '3.5' && matrix.spark-archive == '' uses: codecov/codecov-action@v3 with: verbose: true - name: Upload test logs if: failure() uses: actions/upload-artifact@v4 with: name: unit-tests-log-java-${{ matrix.java }}-spark-${{ matrix.spark }}-${{ matrix.comment }} path: | **/target/unit-tests.log **/kyuubi-spark-sql-engine.log* **/kyuubi-spark-batch-submit.log* scala-test: name: Scala Test runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: scala: - '2.13' java: - '8' spark: - '3.5' steps: - uses: actions/checkout@v4 - name: Free up disk space run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} uses: actions/setup-java@v4 with: distribution: temurin java-version: ${{ matrix.java }} cache: 'maven' - name: Setup Maven uses: ./.github/actions/setup-maven - name: Cache Engine Archives uses: ./.github/actions/cache-engine-archives - name: Build on Scala ${{ matrix.scala }} run: | TEST_MODULES="!externals/kyuubi-flink-sql-engine,!integration-tests/kyuubi-flink-it" ./build/mvn clean install -pl ${TEST_MODULES} -am \ -Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.SparkLocalClusterTest \ -Pscala-${{ matrix.scala }} -Pjava-${{ matrix.java }} -Pspark-${{ matrix.spark }} - name: Upload test logs if: failure() uses: actions/upload-artifact@v4 with: name: unit-tests-log-scala-${{ matrix.scala }}-java-${{ matrix.java }}-spark-${{ matrix.spark }} path: | **/target/unit-tests.log **/kyuubi-spark-sql-engine.log* **/kyuubi-spark-batch-submit.log* **/kyuubi-jdbc-engine.log* **/kyuubi-hive-sql-engine.log* spark-connector-cross-version-test: name: Spark Connector Cross Version Test runs-on: ubuntu-24.04 strategy: fail-fast: false max-parallel: 1 matrix: java: - 17 scala: - '2.12' - '2.13' spark-compile: - '3.5' spark-runtime: - '3.4' - '3.3' comment: [ "normal" ] include: - java: 17 scala: "2.13" spark-compile: "3.5" spark-runtime: "4.0" comment: "normal" env: SPARK_LOCAL_IP: localhost TEST_MODULES: "extensions/spark/kyuubi-spark-connector-hive,\ extensions/spark/kyuubi-spark-connector-tpcds,\ extensions/spark/kyuubi-spark-connector-tpch" steps: - uses: actions/checkout@v4 - name: Free up disk space run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} uses: actions/setup-java@v4 with: distribution: temurin java-version: ${{ matrix.java }} cache: 'maven' - name: Setup Maven uses: ./.github/actions/setup-maven - name: Build Kyuubi Spark Connectors with Spark-${{ matrix.spark-compile }} run: | ./build/mvn clean install -pl ${TEST_MODULES} -am -Pjava-${{ matrix.java }} \ -Pscala-${{ matrix.scala }} -Pspark-${{ matrix.spark-compile }} -DskipTests - name: Test Kyuubi Spark Connectors with Spark-${{ matrix.spark-runtime }} run: | ./build/mvn test -pl ${TEST_MODULES} -Pjava-${{ matrix.java }} \ -Pscala-${{ matrix.scala }} -Pspark-${{ matrix.spark-runtime }} -Pcross-version-test - name: Upload test logs if: failure() uses: actions/upload-artifact@v4 with: name: "unit-tests-log-java-${{ matrix.java }}-scala-${{ matrix.scala }}\ -spark-compile-${{ matrix.spark-compile }}-spark-runtime-${{ matrix.spark-runtime }}\ -${{ matrix.comment }}" path: | **/target/unit-tests.log flink-it: name: Flink Test runs-on: ubuntu-24.04 strategy: fail-fast: false max-parallel: 2 matrix: java: - 8 flink: - '1.17' - '1.18' - '1.19' - '1.20' flink-archive: [ "" ] comment: [ "normal" ] include: - java: 8 flink: '1.20' flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.17.2 -Dflink.archive.name=flink-1.17.2-bin-scala_2.12.tgz' comment: 'verify-on-flink-1.17-binary' - java: 8 flink: '1.20' flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.18.1 -Dflink.archive.name=flink-1.18.1-bin-scala_2.12.tgz' comment: 'verify-on-flink-1.18-binary' - java: 8 flink: '1.20' flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.19.1 -Dflink.archive.name=flink-1.19.1-bin-scala_2.12.tgz' comment: 'verify-on-flink-1.19-binary' steps: - uses: actions/checkout@v4 - name: Free up disk space run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} uses: actions/setup-java@v4 with: distribution: temurin java-version: ${{ matrix.java }} cache: 'maven' - name: Setup Maven uses: ./.github/actions/setup-maven - name: Cache Engine Archives uses: ./.github/actions/cache-engine-archives - name: Build Flink with maven w/o linters run: | TEST_MODULES="externals/kyuubi-flink-sql-engine,integration-tests/kyuubi-flink-it" ./build/mvn -pl ${TEST_MODULES} -Pflink-${{ matrix.flink }} ${{ matrix.flink-archive }} -am clean install -DskipTests - name: Test Flink if: matrix.flink-archive == '' run: | TEST_MODULES="externals/kyuubi-flink-sql-engine,integration-tests/kyuubi-flink-it" ./build/mvn -pl ${TEST_MODULES} -Pflink-${{ matrix.flink }} ${{ matrix.flink-archive }} test - name: Cross-version test Flink if: matrix.flink-archive != '' run: | IT_FLINK=`echo "${{ matrix.flink-archive }}" | grep -E 'flink\-([0-9]+\.[0-9]+.[0-9]+)\-bin' -o | grep -E '[0-9]+\.[0-9]+' -o` IT_MODULE="integration-tests/kyuubi-flink-it" ./build/mvn -pl ${IT_MODULE} -Pflink-${IT_FLINK} ${{ matrix.flink-archive }} test - name: Upload test logs if: failure() uses: actions/upload-artifact@v4 with: name: unit-tests-log-java-${{ matrix.java }}-flink-${{ matrix.flink }}-${{ matrix.comment }} path: | **/target/unit-tests.log **/kyuubi-flink-sql-engine.log* hive-it: name: Hive Test runs-on: ubuntu-24.04 strategy: fail-fast: false max-parallel: 1 matrix: java: - 8 hive-archive: [ "" ] comment: [ "normal" ] include: - java: 8 hive-archive: '-Dhive.archive.mirror=https://archive.apache.org/dist/hive/hive-2.3.10 -Dhive.archive.name=apache-hive-2.3.10-bin.tar.gz' comment: 'verify-on-hive-2.3-binary' - java: 8 hive-archive: '-Dhive.archive.mirror=https://github.com/pan3793/cdh-hive/releases/download/cdh6.3.2-release -Dhive.archive.name=apache-hive-2.1.1-cdh6.3.2-bin.tar.gz' comment: 'verify-on-hive-2.1-cdh6-binary' steps: - uses: actions/checkout@v4 - name: Free up disk space run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} uses: actions/setup-java@v4 with: distribution: temurin java-version: ${{ matrix.java }} cache: 'maven' - name: Setup Maven uses: ./.github/actions/setup-maven - name: Cache Engine Archives uses: ./.github/actions/cache-engine-archives - name: Build and test Hive with maven w/o linters run: | TEST_MODULES="externals/kyuubi-hive-sql-engine,integration-tests/kyuubi-hive-it" ./build/mvn ${{ matrix.hive-archive }} -pl ${TEST_MODULES} -am clean install -DskipTests # unpack avatica-1.8.0.jar and delete Jackson classes to workaround CALCITE-1224 if [[ "${{ matrix.hive-archive }}" == *apache-hive-2.3.10-bin.tar.gz* ]]; then HIVE_2310_LIB="$PWD/externals/kyuubi-download/target/apache-hive-2.3.10-bin/lib" mkdir avatica-1.8.0-unpack cd avatica-1.8.0-unpack jar xf $HIVE_2310_LIB/avatica-1.8.0.jar rm -rf com/fasterxml/jackson org/slf4j jar cf avatica-1.8.0-patched.jar . rm $HIVE_2310_LIB/avatica-1.8.0.jar mv avatica-1.8.0-patched.jar $HIVE_2310_LIB/ cd .. rm -rf avatica-1.8.0-unpack fi ./build/mvn ${{ matrix.hive-archive }} -pl ${TEST_MODULES} test - name: Upload test logs if: failure() uses: actions/upload-artifact@v4 with: name: unit-tests-log-java-${{ matrix.java }}-hive-${{ matrix.comment }} path: | **/target/unit-tests.log **/kyuubi-hive-sql-engine.log* jdbc-trino-tpc-it: name: JDBC Trino TPC Tests runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: java: [ 8 ] comment: [ "normal" ] steps: - uses: actions/checkout@v4 - name: Free up disk space run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} uses: actions/setup-java@v4 with: distribution: temurin java-version: ${{ matrix.java }} cache: 'maven' - name: Setup Maven uses: ./.github/actions/setup-maven - name: Cache Engine Archives uses: ./.github/actions/cache-engine-archives - name: Build and test JDBC and Trino engines, TPC-H/TPC-DS with maven w/o linters run: | JDBC_TEST_MODULES="externals/kyuubi-jdbc-engine,integration-tests/kyuubi-jdbc-it" TRINO_TEST_MODULES="externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it" TPC_TEST_MODULES="kyuubi-server,extensions/spark/kyuubi-spark-connector-tpcds,extensions/spark/kyuubi-spark-connector-tpch" ./build/mvn -pl ${JDBC_TEST_MODULES},${TRINO_TEST_MODULES},${TPC_TEST_MODULES} -am clean install -DskipTests ./build/mvn -pl ${JDBC_TEST_MODULES} test ./build/mvn -pl ${TRINO_TEST_MODULES} test ./build/mvn -pl ${TPC_TEST_MODULES} test \ -Dmaven.plugin.scalatest.exclude.tags='' \ -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds,org.apache.kyuubi.spark.connector.tpcds.TPCDSQuerySuite,org.apache.kyuubi.spark.connector.tpch.TPCHQuerySuite - name: Upload test logs if: failure() uses: actions/upload-artifact@v4 with: name: unit-tests-log-java-${{ matrix.java }}-hive-${{ matrix.comment }} path: | **/target/unit-tests.log **/kyuubi-jdbc-engine.log* **/kyuubi-trino-engine.log* kyuubi-on-k8s-it: name: Kyuubi Server On Kubernetes Integration Test runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Free up disk space run: ./.github/scripts/free_disk_space.sh # https://github.com/docker/build-push-action - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: cache-binary: false - name: Build Kyuubi Docker Image uses: docker/build-push-action@v6 with: # passthrough CI into build container build-args: | CI=${CI} MVN_ARG=--flink-provided --hive-provided context: . file: build/Dockerfile.CI load: true tags: apache/kyuubi:latest - name: Cache Engine Archives uses: ./.github/actions/cache-engine-archives - name: Setup Minikube run: | # https://minikube.sigs.k8s.io/docs/start/ curl -LO https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64 sudo install minikube-linux-amd64 /usr/local/bin/minikube minikube start --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --force # https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime minikube image load apache/kyuubi:latest # pre-install spark into minikube docker pull apache/spark:3.5.5 minikube image load apache/spark:3.5.5 - name: kubectl pre-check run: | kubectl get nodes kubectl get serviceaccount kubectl create serviceaccount kyuubi kubectl create clusterrolebinding kyuubi-role --clusterrole=edit --serviceaccount=default:kyuubi kubectl get serviceaccount - name: start kyuubi run: kubectl apply -f integration-tests/kyuubi-kubernetes-it/src/test/resources/kyuubi-server.yaml - name: kyuubi pod check run: | kubectl get pods kubectl describe pods kyuubi-test - name: integration tests run: >- ./build/mvn clean install -pl integration-tests/kyuubi-kubernetes-it -am -Pkubernetes-it -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.deployment,org.apache.kyuubi.kubernetes.test.KubernetesUtilsTest - name: Cat kyuubi server log if: failure() run: kubectl logs kyuubi-test - name: Copy spark engine log from kyuubi pod if: failure() run: kubectl cp kyuubi-test:/opt/kyuubi/work ./target/work - name: Cat spark driver log if: failure() run: | kubectl get pods kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I {} kubectl logs {} - name: Upload test logs if: failure() uses: actions/upload-artifact@v4 with: name: unit-tests-log-kyuubi-on-k8s-it path: | **/target/unit-tests.log **/target/work/** spark-on-k8s-it: name: Spark Engine On Kubernetes Integration Test runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Free up disk space run: ./.github/scripts/free_disk_space.sh - name: Cache Engine Archives uses: ./.github/actions/cache-engine-archives - name: Setup JDK 11 uses: actions/setup-java@v4 with: distribution: temurin java-version: 11 cache: 'maven' - name: Setup Minikube run: | # https://minikube.sigs.k8s.io/docs/start/ curl -LO https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64 sudo install minikube-linux-amd64 /usr/local/bin/minikube minikube start --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --force # in case: https://spark.apache.org/docs/latest/running-on-kubernetes.html#rbac - name: Create Service Account run: | kubectl create serviceaccount spark kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=default:spark --namespace=default kubectl get serviceaccount # TODO Support more Spark version - name: integration tests run: >- ./build/mvn clean install -Pflink-provided,hive-provided -Pkubernetes-it -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.spark - name: Print Driver Pod logs if: failure() run: | kubectl get pods kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I {} kubectl logs {} - name: Upload test logs if: failure() uses: actions/upload-artifact@v4 with: name: unit-tests-log-spark-on-k8s-it path: | **/target/unit-tests.log **/kyuubi-spark-sql-engine.log* **/kyuubi-spark-batch-submit.log* zookeeper-it: name: Zookeeper Integration Test runs-on: ubuntu-24.04 strategy: fail-fast: false max-parallel: 1 matrix: java: [ 8 ] zookeeper: ["3.4", "3.5", "3.6", "3.7" ] comment: [ "normal" ] steps: - uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} uses: actions/setup-java@v4 with: distribution: temurin java-version: ${{ matrix.java }} cache: 'maven' - name: Setup Maven uses: ./.github/actions/setup-maven - name: Cache Engine Archives uses: ./.github/actions/cache-engine-archives - name: zookeeper integration tests run: | export KYUUBI_IT_ZOOKEEPER_VERSION=${{ matrix.zookeeper }} TEST_MODULES="integration-tests/kyuubi-zookeeper-it" ./build/mvn -pl ${TEST_MODULES} -am clean install -DskipTests ./build/mvn -pl ${TEST_MODULES} test - name: Upload test logs if: failure() uses: actions/upload-artifact@v4 with: name: unit-tests-log-java-${{ matrix.java }}-zookeeper-${{ matrix.comment }} path: | **/target/unit-tests.log