[KYUUBI #755] [BUILD] Default build against spark 3.1
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/NetEase/kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request Closes #755 from pan3793/spark-31. Closes #755 bb25a395 [Cheng Pan] typo 6b6fd4f7 [Cheng Pan] fix ut a79af708 [Cheng Pan] change cross version test to build against Spark 3.1 and test on Spark 3.0 binary 2fd0afb4 [Cheng Pan] enable CODECOV with spark 3.1 in travis f0c8b55c [Cheng Pan] [BUILD] Default build against spark 3.1 Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: fwang12 <fwang12@ebay.com>
This commit is contained in:
Cheng Pan
fwang12
parent
64bcfe761e
commit
819484fd57
7
.github/workflows/master.yml
vendored
7
.github/workflows/master.yml
vendored
@ -37,13 +37,14 @@ jobs:
|
||||
spark: '3.0'
|
||||
spark-hadoop: '2.7'
|
||||
- java: 8
|
||||
spark: '3.0'
|
||||
spark: '3.1'
|
||||
spark-hadoop: '2.7'
|
||||
profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.2 -Dspark.archive.name=spark-3.1.2-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
|
||||
profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
|
||||
- java: 8
|
||||
spark: '3.1'
|
||||
spark-hadoop: '3.2'
|
||||
profiles: '-Pkyuubi-extension-spark_3.1'
|
||||
codecov: 'true'
|
||||
- java: 11
|
||||
spark: '3.1'
|
||||
spark-hadoop: '3.2'
|
||||
@ -86,7 +87,7 @@ jobs:
|
||||
./build/mvn clean install -Dmaven.javadoc.skip=true -V
|
||||
-Pspark-${{ matrix.spark }} -Pspark-hadoop-${{ matrix.spark-hadoop }} ${{ matrix.profiles }}
|
||||
- name: Code coverage
|
||||
if: ${{ matrix.profiles == '' }}
|
||||
if: ${{ matrix.codecov == 'true' }}
|
||||
run: bash <(curl -s https://codecov.io/bash)
|
||||
- name: Detected Dependency List Change
|
||||
run: build/dependency.sh
|
||||
|
||||
12
.travis.yml
12
.travis.yml
@ -18,7 +18,7 @@ sudo: required
|
||||
dist: bionic
|
||||
language: scala
|
||||
scala:
|
||||
- 2.12.10
|
||||
- 2.12.14
|
||||
jdk:
|
||||
- openjdk8
|
||||
- openjdk11
|
||||
@ -29,19 +29,19 @@ cache:
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- name: Test Kyuubi w/ Default Profile
|
||||
- name: Test Kyuubi w/ -Pspark-3.0 -Pspark-hadoop-2.7
|
||||
env:
|
||||
- PROFILE=""
|
||||
- PROFILE="-Pspark-3.0 -Pspark-hadoop-2.7"
|
||||
- JOB="build"
|
||||
- CODECOV="true"
|
||||
- name: Test Kyuubi w/ -Pspark-3.0 and Spark 3.1 binary
|
||||
- name: Test Kyuubi w/ -Pspark-3.1 and Spark 3.0 binary
|
||||
env:
|
||||
- PROFILE="-Pspark-3.0 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.2 -Dspark.archive.name=spark-3.1.2-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest"
|
||||
- PROFILE="-Pspark-3.1 -Pspark-hadoop-2.7 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest"
|
||||
- JOB="build"
|
||||
- name: Test Kyuubi w/ -Pspark-3.1 -Pkyuubi-extension-spark_3.1
|
||||
env:
|
||||
- PROFILE="-Pspark-3.1 -Pkyuubi-extension-spark_3.1"
|
||||
- JOB="build"
|
||||
- CODECOV="true"
|
||||
- name: Test Kyuubi w/ -Pspark-master
|
||||
env:
|
||||
- PROFILE="-Pspark-master"
|
||||
|
||||
10
pom.xml
10
pom.xml
@ -42,7 +42,7 @@
|
||||
<licenses>
|
||||
<license>
|
||||
<name>The Apache Software License, Version 2.0</name>
|
||||
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
|
||||
<url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
|
||||
<distribution>manual</distribution>
|
||||
</license>
|
||||
</licenses>
|
||||
@ -68,10 +68,10 @@
|
||||
<commons-collections.version>3.2.2</commons-collections.version>
|
||||
<commons-lang3.version>3.10</commons-lang3.version>
|
||||
<curator.version>2.12.0</curator.version>
|
||||
<delta.version>0.8.0</delta.version>
|
||||
<delta.version>1.0.0</delta.version>
|
||||
<guava.version>30.1-jre</guava.version>
|
||||
<hadoop.version>3.2.2</hadoop.version>
|
||||
<hadoop.binary.version>2.7</hadoop.binary.version>
|
||||
<hadoop.binary.version>3.2</hadoop.binary.version>
|
||||
<hive.version>2.3.7</hive.version>
|
||||
<jackson.version>2.11.4</jackson.version>
|
||||
<jakarta.servlet-api.version>4.0.4</jakarta.servlet-api.version>
|
||||
@ -80,7 +80,7 @@
|
||||
<jetty.version>9.4.41.v20210516</jetty.version>
|
||||
<ldapsdk.version>5.1.4</ldapsdk.version>
|
||||
<prometheus.version>0.10.0</prometheus.version>
|
||||
<spark.version>3.0.3</spark.version>
|
||||
<spark.version>3.1.2</spark.version>
|
||||
<spark.archive.name>spark-${spark.version}-bin-hadoop${hadoop.binary.version}.tgz</spark.archive.name>
|
||||
<spark.archive.mirror>https://archive.apache.org/dist/spark/spark-${spark.version}</spark.archive.mirror>
|
||||
<spark.archive.download.skip>false</spark.archive.download.skip>
|
||||
@ -103,7 +103,7 @@
|
||||
<maven.plugin.scala.version>4.3.0</maven.plugin.scala.version>
|
||||
<maven.plugin.surefire.version>2.22.0</maven.plugin.surefire.version>
|
||||
<maven.plugin.scalatest.version>2.0.2</maven.plugin.scalatest.version>
|
||||
<maven.plugin.scalatest.exclude.tags>org.apache.kyuubi.tags.ExtendedSQLTest</maven.plugin.scalatest.exclude.tags>
|
||||
<maven.plugin.scalatest.exclude.tags>org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.IcebergTest</maven.plugin.scalatest.exclude.tags>
|
||||
<maven.plugin.scalatest.include.tags></maven.plugin.scalatest.include.tags>
|
||||
<maven.plugin.jacoco.version>0.8.6</maven.plugin.jacoco.version>
|
||||
<maven.plugin.shade.version>3.2.4</maven.plugin.shade.version>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user