kyuubi/.github/workflows/master.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: CI

on:
  push:
    branches:
      - master
      - branch-*
  pull_request:
    branches:
      - master
      - branch-*

concurrency:
  group: test-${{ github.ref }}
  cancel-in-progress: true

jobs:

  build:
    name: Test
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        java:
          - 8
          - 11
        spark:
          - '3.0'
          - '3.1'
          - '3.2'
        spark-archive: [""]
        exclude-tags: [""]
        include:
          - java: 8
            spark: '3.2'
            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz'
            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest'
          - java: 8
            spark: '3.2'
            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3 -Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz'
            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest'
    env:
      SPARK_LOCAL_IP: localhost
    steps:
      - uses: actions/checkout@v2
      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm
      - name: Setup JDK ${{ matrix.java }}
        uses: actions/setup-java@v2
        with:
          distribution: zulu
          java-version: ${{ matrix.java }}
          cache: 'maven'
          check-latest: false
      - name: Build and test with maven w/o linters
        run: >-
          ./build/mvn clean install
          -Dmaven.javadoc.skip=true
          -Drat.skip=true
          -Dscalastyle.skip=true
          -Dspotless.check.skip
          -Dorg.slf4j.simpleLogger.defaultLogLevel=warn
          -Pspark-${{ matrix.spark }}
          ${{ matrix.spark-archive }}
          ${{ matrix.exclude-tags }}          
      - name: Code coverage
        if: |
          matrix.java == 8 &&
          matrix.spark == '3.2' &&
          matrix.spark-archive == ''          
        uses: codecov/codecov-action@v2
        with:
          verbose: true
      - name: Upload test logs
        if: failure()
        uses: actions/upload-artifact@v2
        with:
          name: unit-tests-log
          path: |
            **/target/unit-tests.log
            **/kyuubi-flink-sql-engine.log*
            **/kyuubi-spark-sql-engine.log*
            **/kyuubi-trino-sql-engine.log*            

  tpcds:
    name: TPC-DS Tests
    runs-on: ubuntu-20.04
    env:
      SPARK_LOCAL_IP: localhost
    steps:
      - uses: actions/checkout@v2
      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm
      - name: Setup JDK 8
        uses: actions/setup-java@v2
        with:
          distribution: zulu
          java-version: 8
          cache: 'maven'
          check-latest: false
      - name: Run TPC-DS Tests
        run: >-
          ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn
          -pl kyuubi-server -am
          -Dmaven.plugin.scalatest.exclude.tags=''
          -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds          

  kyuubi-on-k8s-it:
    name: Kyuubi Server On Kubernetes Integration Test
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      # https://github.com/docker/build-push-action
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      - name: Build Kyuubi Docker Image
        uses: docker/build-push-action@v2
        with:
          # passthrough CI into build container
          build-args: CI=${CI}
          context: .
          file: build/Dockerfile
          load: true
          tags: apache/kyuubi:latest
      # from https://github.com/marketplace/actions/setup-minikube-kubernetes-cluster
      - name: Setup Minikube
        uses: manusa/actions-setup-minikube@v2.4.3
        with:
          minikube version: 'v1.25.2'
          kubernetes version: 'v1.23.3'
      - name: kubectl pre-check
        run: |
          kubectl get serviceaccount
          kubectl create serviceaccount kyuubi
          kubectl get serviceaccount          
      - name: start kyuubi
        run: kubectl apply -f integration-tests/kyuubi-kubernetes-it/src/test/resources/kyuubi-server.yaml
      - name: kyuubi pod check
        run: kubectl get pods
      - name: integration tests
        run: >-
          ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -V
          -pl integration-tests/kyuubi-kubernetes-it -am
          -Pkubernetes-it
          -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.deployment          
      - name: Upload test logs
        if: failure()
        uses: actions/upload-artifact@v2
        with:
          name: unit-tests-log
          path: |
            **/target/unit-tests.log            

  spark-on-k8s-it:
    name: Spark Engine On Kubernetes Integration Test
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      # from https://github.com/marketplace/actions/setup-minikube-kubernetes-cluster
      - name: Setup Minikube
        uses: manusa/actions-setup-minikube@v2.4.3
        with:
          minikube version: 'v1.25.2'
          kubernetes version: 'v1.23.3'
          driver: docker
          start args: '--extra-config=kubeadm.ignore-preflight-errors=NumCPU --force --cpus 2 --memory 4096'
      # in case: https://spark.apache.org/docs/latest/running-on-kubernetes.html#rbac
      - name: Create Service Account
        run: |
          kubectl create serviceaccount spark
          kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=default:spark --namespace=default
          kubectl get serviceaccount          
      - name: integration tests
        run: >-
          ./build/mvn clean install
          -Dmaven.javadoc.skip=true
          -Drat.skip=true
          -Dscalastyle.skip=true
          -Dspotless.check.skip
          -Dspark.version=3.2.1
          -Pflink-provided,hive-provided
          -Pkubernetes-it
          -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.spark          
      - name: Print Driver Pod logs
        if: failure()
        run: |
          kubectl get pods
          kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I {} kubectl logs {}          
      - name: Upload test logs
        if: failure()
        uses: actions/upload-artifact@v2
        with:
          name: unit-tests-log
          path: |
            **/target/unit-tests.log
            **/kyuubi-spark-sql-engine.log*