43f40dcb8a
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/NetEase/kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> The added sql module structure looks like: ``` kyuubi | - dev | - kyuubi-extension-spark_3.1 ``` This PR mainly add 3 feature: * merging small files automatically (include dynamic partition insertion case) * insert shuffle node before Join to make AQE `OptimizeSkewedJoin` work * stage level config isolation in AQE Note that, the sql rule depend on the Apache Spark interface so we need make the sql module verion independence. Currently, this PR only supports the Spark 3.1.1. Due to the version issue, we need to check and deploy this extension manually currently. ### _How was this patch tested?_ Add new test. Closes #631 from ulysses-you/add-sql-module. Closes #631 2cf12f1 [ulysses-you] version cfbf72c [ulysses-you] address comment 7740ca6 [ulysses-you] module name 0f723eb [ulysses-you] workflow 45c23d8 [ulysses-you] line 80378f5 [ulysses-you] assembly 95528aa [ulysses-you] move module 5fe5d87 [ulysses-you] license 6578440 [ulysses-you] init work Authored-by: ulysses-you <ulyssesyou18@gmail.com> Signed-off-by: Kent Yao <yao@apache.org>
94 lines
3.4 KiB
YAML
94 lines
3.4 KiB
YAML
name: Kyuubi
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- master
|
|
- branch-*
|
|
pull_request:
|
|
branches:
|
|
- master
|
|
- branch-*
|
|
|
|
jobs:
|
|
rat:
|
|
name: Check License
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- uses: actions/setup-java@v1
|
|
with:
|
|
java-version: '1.8'
|
|
- run: build/mvn org.apache.rat:apache-rat-plugin:check
|
|
- name: Upload rat report
|
|
if: failure()
|
|
uses: actions/upload-artifact@v2
|
|
with:
|
|
name: rat
|
|
path: "**/target/rat*.txt"
|
|
|
|
build:
|
|
name: Build
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
profiles:
|
|
- ''
|
|
- '-Pspark-3.0 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.1 -Dspark.archive.name=spark-3.1.1-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest'
|
|
- '-Pspark-3.1 -Pkyuubi-sql-spark_3.1'
|
|
- '-Pspark-3.1 -Pkyuubi-sql-spark_3.1 -Pspark-hadoop-3.2'
|
|
- '-Pspark-3.2-snapshot -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest'
|
|
- '-DwildcardSuites=org.apache.kyuubi.operation.tpcds.TPCDSOutputSchemaSuite,org.apache.kyuubi.operation.tpcds.TPCDSDDLSuite -Dmaven.plugin.scalatest.exclude.tags=""'
|
|
env:
|
|
SPARK_LOCAL_IP: localhost
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Setup JDK 1.8
|
|
uses: actions/setup-java@v1
|
|
with:
|
|
java-version: '1.8'
|
|
- uses: actions/cache@v1
|
|
with:
|
|
path: ~/.m2/repository/com
|
|
key: ${{ runner.os }}-maven-com-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-maven-com-
|
|
- uses: actions/cache@v1
|
|
with:
|
|
path: ~/.m2/repository/org
|
|
key: ${{ runner.os }}-maven-org-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-maven-org-
|
|
- uses: actions/cache@v1
|
|
with:
|
|
path: ~/.m2/repository/net
|
|
key: ${{ runner.os }}-maven-net-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-maven-net-
|
|
- uses: actions/cache@v1
|
|
with:
|
|
path: ~/.m2/repository/io
|
|
key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-maven-io-
|
|
- name: Build with Maven
|
|
if: ${{ ! contains(matrix.profiles, 'wildcardSuites') }}
|
|
run: mvn clean install ${{ matrix.profiles }} -Dmaven.javadoc.skip=true -V
|
|
- name: Wild Card Suites Tests
|
|
if: ${{ contains(matrix.profiles, 'wildcardSuites') }}
|
|
run: mvn clean install -DskipTests -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper && mvn -Dtest=none ${{ matrix.profiles }} test
|
|
- name: Code coverage
|
|
if: ${{ matrix.profiles == '' }}
|
|
run: bash <(curl -s https://codecov.io/bash)
|
|
- name: Detected Dependency List Change
|
|
if: ${{ ! contains(matrix.profiles, 'spark-3.2-snapshot') && ! contains(matrix.profiles, 'wildcardSuites') }}
|
|
run: build/dependency.sh
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@v2
|
|
with:
|
|
name: unit-tests-log
|
|
path: |
|
|
**/target/unit-tests.log
|
|
**/kyuubi-spark-sql-engine.log*
|