kyuubi/.github/workflows/master.yml

name: Kyuubi

on:
  push:
    branches:
      - master
      - branch-*
  pull_request:
    branches:
      - master
      - branch-*

jobs:

  build:
    name: Build
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        include:
          - java: 8
            spark: '3.0'
            spark-hadoop: '2.7'
          - java: 8
            spark: '3.1'
            spark-hadoop: '2.7'
            profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest'
          - java: 8
            spark: '3.1'
            spark-hadoop: '3.2'
            profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.0 -Dspark.archive.name=spark-3.2.0-bin-hadoop3.2.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest'
          - java: 8
            spark: '3.1'
            spark-hadoop: '3.2'
            codecov: 'true'
          - java: 8
            spark: '3.2'
            spark-hadoop: '3.2'
          - java: 11
            spark: '3.1'
            spark-hadoop: '3.2'
            profiles: '-DskipTests -Pflink-provided,spark-provided'

    env:
      SPARK_LOCAL_IP: localhost
    steps:
      - uses: actions/checkout@v2
      - name: Setup JDK ${{ matrix.java }}
        uses: actions/setup-java@v2
        with:
          distribution: zulu
          java-version: ${{ matrix.java }}
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository/com
          key: ${{ runner.os }}-maven-com-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-com-            
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository/org
          key: ${{ runner.os }}-maven-org-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-org-            
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository/net
          key: ${{ runner.os }}-maven-net-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-net-            
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository/io
          key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-io-            
      - name: Build with Maven
        run: >-
          ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -V
          -Pspark-${{ matrix.spark }} -Pspark-hadoop-${{ matrix.spark-hadoop }} ${{ matrix.profiles }}          
      - name: Code coverage
        if: ${{ matrix.codecov == 'true' }}
        uses: codecov/codecov-action@v2
        with:
          verbose: true
      - name: Detected Dependency List Change
        if: ${{ matrix.java == 8 }}
        run: build/dependency.sh
      - name: Upload test logs
        if: failure()
        uses: actions/upload-artifact@v2
        with:
          name: unit-tests-log
          path: |
            **/target/unit-tests.log
            **/kyuubi-flink-sql-engine.log*
            **/kyuubi-spark-sql-engine.log*
            **/target/scalastyle-output.xml            

  tpcds:
    name: TPC-DS Tests
    runs-on: ubuntu-20.04
    env:
      SPARK_LOCAL_IP: localhost
    steps:
      - uses: actions/checkout@v2
      - name: Setup JDK 8
        uses: actions/setup-java@v2
        with:
          distribution: zulu
          java-version: 8
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository/com
          key: ${{ runner.os }}-maven-com-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-com-            
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository/org
          key: ${{ runner.os }}-maven-org-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-org-            
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository/net
          key: ${{ runner.os }}-maven-net-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-net-            
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository/io
          key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-io-            
      - name: Run TPC-DS Tests
        run: >-
          ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -V
          -pl kyuubi-server -am
          -Pspark-3.1
          -Dmaven.plugin.scalatest.exclude.tags=''
          -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds          

  minikube-it:
    name: Minikube Integration Test
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      # https://github.com/docker/build-push-action
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      - name: Build Kyuubi Docker Image
        uses: docker/build-push-action@v2
        with:
          # passthrough CI into build container
          build-args: CI=${CI}
          context: .
          file: build/Dockerfile
          load: true
          tags: apache/kyuubi:latest
      # from https://github.com/marketplace/actions/setup-minikube-kubernetes-cluster
      - name: Setup Minikube
        uses: manusa/actions-setup-minikube@v2.4.2
        with:
          minikube version: 'v1.16.0'
          kubernetes version: 'v1.19.2'
      - name: kubectl pre-check
        run: |
          kubectl get serviceaccount
          kubectl create serviceaccount kyuubi
          kubectl get serviceaccount          
      - name: start kyuubi
        run: kubectl apply -f kubernetes/integration-tests/test-k8s.yaml
      - name: kyuubi pod check
        run: kubectl get pods
      - name: integration tests
        run: >-
          ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -V
          -pl kubernetes/integration-tests -am
          -Pkubernetes
          -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test          
      - name: Upload test logs
        if: failure()
        uses: actions/upload-artifact@v2
        with:
          name: unit-tests-log
          path: |
            **/target/unit-tests.log