38814691b2
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> 1. remove the unnecessary scala installation 2. maven compiler log to warn level to filter a lot of annoying message ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1763 from yaooqinn/ci2. Closes #1763 c0f11145 [Kent Yao] fix dfd75dd7 [Kent Yao] fix 01fcc3f1 [Kent Yao] fix 94c62e46 [Kent Yao] Merge branch 'master' of github.com:apache/incubator-kyuubi into ci2 aab62e62 [Kent Yao] fix b38b52cc [Kent Yao] ci 51cc7db8 [Kent Yao] nit Authored-by: Kent Yao <yao@apache.org> Signed-off-by: Kent Yao <yao@apache.org>
191 lines
6.7 KiB
YAML
191 lines
6.7 KiB
YAML
name: Kyuubi
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- master
|
|
- branch-*
|
|
pull_request:
|
|
branches:
|
|
- master
|
|
- branch-*
|
|
|
|
jobs:
|
|
|
|
build:
|
|
name: Build
|
|
runs-on: ubuntu-20.04
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- java: 8
|
|
spark: '3.0'
|
|
spark-hadoop: '2.7'
|
|
- java: 8
|
|
spark: '3.1'
|
|
spark-hadoop: '2.7'
|
|
profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest'
|
|
- java: 8
|
|
spark: '3.1'
|
|
spark-hadoop: '3.2'
|
|
profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.0 -Dspark.archive.name=spark-3.2.0-bin-hadoop3.2.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest'
|
|
- java: 8
|
|
spark: '3.1'
|
|
spark-hadoop: '3.2'
|
|
codecov: 'true'
|
|
- java: 8
|
|
spark: '3.2'
|
|
spark-hadoop: '3.2'
|
|
- java: 11
|
|
spark: '3.1'
|
|
spark-hadoop: '3.2'
|
|
profiles: '-DskipTests -Pflink-provided,spark-provided'
|
|
|
|
env:
|
|
SPARK_LOCAL_IP: localhost
|
|
steps:
|
|
- uses: actions/checkout@v2.3.5
|
|
- name: Tune Runner VM
|
|
uses: ./.github/actions/tune-runner-vm
|
|
- name: Setup JDK ${{ matrix.java }}
|
|
uses: actions/setup-java@v2
|
|
with:
|
|
distribution: zulu
|
|
java-version: ${{ matrix.java }}
|
|
- uses: actions/cache@v2
|
|
with:
|
|
path: ~/.m2/repository/com
|
|
key: ${{ runner.os }}-maven-com-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-maven-com-
|
|
- uses: actions/cache@v2
|
|
with:
|
|
path: ~/.m2/repository/org
|
|
key: ${{ runner.os }}-maven-org-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-maven-org-
|
|
- uses: actions/cache@v2
|
|
with:
|
|
path: ~/.m2/repository/net
|
|
key: ${{ runner.os }}-maven-net-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-maven-net-
|
|
- uses: actions/cache@v2
|
|
with:
|
|
path: ~/.m2/repository/io
|
|
key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-maven-io-
|
|
- name: Build with Maven
|
|
run: >-
|
|
./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -V
|
|
-Pspark-${{ matrix.spark }} -Pspark-hadoop-${{ matrix.spark-hadoop }} ${{ matrix.profiles }}
|
|
- name: Code coverage
|
|
if: ${{ matrix.codecov == 'true' }}
|
|
uses: codecov/codecov-action@v2
|
|
with:
|
|
verbose: true
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@v2
|
|
with:
|
|
name: unit-tests-log
|
|
path: |
|
|
**/target/unit-tests.log
|
|
**/kyuubi-flink-sql-engine.log*
|
|
**/kyuubi-spark-sql-engine.log*
|
|
**/target/scalastyle-output.xml
|
|
|
|
tpcds:
|
|
name: TPC-DS Tests
|
|
runs-on: ubuntu-20.04
|
|
env:
|
|
SPARK_LOCAL_IP: localhost
|
|
steps:
|
|
- uses: actions/checkout@v2.3.5
|
|
- name: Tune Runner VM
|
|
uses: ./.github/actions/tune-runner-vm
|
|
- name: Setup JDK 8
|
|
uses: actions/setup-java@v2
|
|
with:
|
|
distribution: zulu
|
|
java-version: 8
|
|
- uses: actions/cache@v2
|
|
with:
|
|
path: ~/.m2/repository/com
|
|
key: ${{ runner.os }}-maven-com-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-maven-com-
|
|
- uses: actions/cache@v2
|
|
with:
|
|
path: ~/.m2/repository/org
|
|
key: ${{ runner.os }}-maven-org-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-maven-org-
|
|
- uses: actions/cache@v2
|
|
with:
|
|
path: ~/.m2/repository/net
|
|
key: ${{ runner.os }}-maven-net-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-maven-net-
|
|
- uses: actions/cache@v2
|
|
with:
|
|
path: ~/.m2/repository/io
|
|
key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-maven-io-
|
|
- name: Run TPC-DS Tests
|
|
run: >-
|
|
./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -V
|
|
-pl kyuubi-server -am
|
|
-Pspark-3.1
|
|
-Dmaven.plugin.scalatest.exclude.tags=''
|
|
-Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds
|
|
|
|
minikube-it:
|
|
name: Minikube Integration Test
|
|
runs-on: ubuntu-20.04
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v2.3.5
|
|
# https://github.com/docker/build-push-action
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v1
|
|
- name: Build Kyuubi Docker Image
|
|
uses: docker/build-push-action@v2
|
|
with:
|
|
# passthrough CI into build container
|
|
build-args: CI=${CI}
|
|
context: .
|
|
file: build/Dockerfile
|
|
load: true
|
|
tags: apache/kyuubi:latest
|
|
# from https://github.com/marketplace/actions/setup-minikube-kubernetes-cluster
|
|
- name: Setup Minikube
|
|
uses: manusa/actions-setup-minikube@v2.4.2
|
|
with:
|
|
minikube version: 'v1.16.0'
|
|
kubernetes version: 'v1.19.2'
|
|
- name: kubectl pre-check
|
|
run: |
|
|
kubectl get serviceaccount
|
|
kubectl create serviceaccount kyuubi
|
|
kubectl get serviceaccount
|
|
- name: start kyuubi
|
|
run: kubectl apply -f integration-tests/kyuubi-kubernetes-deployment-it/test-k8s.yaml
|
|
- name: kyuubi pod check
|
|
run: kubectl get pods
|
|
- name: integration tests
|
|
run: >-
|
|
./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -V
|
|
-pl integration-tests/kyuubi-kubernetes-deployment-it -am
|
|
-Pkubernetes-deployment-it
|
|
-Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@v2
|
|
with:
|
|
name: unit-tests-log
|
|
path: |
|
|
**/target/unit-tests.log
|