[KYUUBI #606] Introduce ExtendedSQLTest tag to split build jobs
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/NetEase/kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> Involve ExtendedSQLTest tag to split build jobs ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request Closes #606 from turboFei/slow_tags. Closes #606 5f8464d [fwang12] do not use build/dist 9e813dc [fwang12] exclude extended sql test for default profile 4c5ea67 [fwang12] Involve ExtendedSQLTest to split build jobs Authored-by: fwang12 <fwang12@ebay.com> Signed-off-by: Kent Yao <yao@apache.org>
This commit is contained in:
fwang12
Kent Yao
parent
2af8b53dbf
commit
0e2263f759
11
.github/workflows/master.yml
vendored
11
.github/workflows/master.yml
vendored
@ -34,10 +34,11 @@ jobs:
|
||||
matrix:
|
||||
profiles:
|
||||
- ''
|
||||
- '-Pspark-3.0 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.1 -Dspark.archive.name=spark-3.1.1-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.DataLakeTest'
|
||||
- '-Pspark-3.0 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.1 -Dspark.archive.name=spark-3.1.1-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest'
|
||||
- '-Pspark-3.1'
|
||||
- '-Pspark-3.1 -Pspark-hadoop-3.2'
|
||||
- '-Pspark-3.2-snapshot -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.DataLakeTest'
|
||||
- '-Pspark-3.2-snapshot -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest'
|
||||
- '-DwildcardSuites=org.apache.kyuubi.operation.tpcds.TPCDSOutputSchemaSuite,org.apache.kyuubi.operation.tpcds.TPCDSDDLSuite -Dmaven.plugin.scalatest.exclude.tags=""'
|
||||
env:
|
||||
SPARK_LOCAL_IP: localhost
|
||||
steps:
|
||||
@ -71,12 +72,16 @@ jobs:
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-io-
|
||||
- name: Build with Maven
|
||||
if: ${{ ! contains(matrix.profiles, 'wildcardSuites') }}
|
||||
run: mvn clean install ${{ matrix.profiles }} -Dmaven.javadoc.skip=true -V
|
||||
- name: Wild Card Suites Tests
|
||||
if: ${{ contains(matrix.profiles, 'wildcardSuites') }}
|
||||
run: mvn clean install -DskipTests -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper && mvn -Dtest=none ${{ matrix.profiles }} test
|
||||
- name: Code coverage
|
||||
if: ${{ matrix.profiles == '' }}
|
||||
run: bash <(curl -s https://codecov.io/bash)
|
||||
- name: Detected Dependency List Change
|
||||
if: ${{ ! contains(matrix.profiles, 'spark-3.2-snapshot') }}
|
||||
if: ${{ ! contains(matrix.profiles, 'spark-3.2-snapshot') && ! contains(matrix.profiles, 'wildcardSuites') }}
|
||||
run: build/dependency.sh
|
||||
- name: Upload test logs
|
||||
if: failure()
|
||||
|
||||
16
.travis.yml
16
.travis.yml
@ -31,29 +31,33 @@ matrix:
|
||||
- name: Test Kyuubi w/ Default Profile
|
||||
env:
|
||||
- PROFILE=""
|
||||
- EXCLUDE_TAGS=""
|
||||
- EXCLUDE_TAGS="org.apache.kyuubi.tags.ExtendedSQLTest"
|
||||
- name: Test Kyuubi w/ -Pspark-3.0 and Spark 3.1 binary
|
||||
env:
|
||||
- PROFILE="-Pspark-3.0 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.1 -Dspark.archive.name=spark-3.1.1-bin-hadoop2.7.tgz"
|
||||
- EXCLUDE_TAGS="org.apache.kyuubi.tags.DataLakeTest"
|
||||
- EXCLUDE_TAGS="org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest"
|
||||
- name: Test Kyuubi w/ -Pspark-3.1
|
||||
env:
|
||||
- PROFILE="-Pspark-3.1"
|
||||
- EXCLUDE_TAGS="org.apache.kyuubi.tags.DataLakeTest"
|
||||
- EXCLUDE_TAGS="org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest"
|
||||
- name: Test Kyuubi w/ -Pspark-3.1 -Pspark-hadoop-3.2
|
||||
env:
|
||||
- PROFILE="-Pspark-3.1 -Dspark-hadoop-3.2"
|
||||
- EXCLUDE_TAGS="org.apache.kyuubi.tags.DataLakeTest"
|
||||
- EXCLUDE_TAGS="org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest"
|
||||
- name: Test Kyuubi w/ -Pspark-3.2-snapshot w/ Spark 3.2 nightly build
|
||||
env:
|
||||
- PROFILE="-Pspark-3.2-snapshot -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper"
|
||||
- EXCLUDE_TAGS="org.apache.kyuubi.tags.DataLakeTest"
|
||||
- EXCLUDE_TAGS="org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest"
|
||||
- name: Test Kyuubi w/ -Pspark-3.0 TPCDS Tests
|
||||
env:
|
||||
- TPCDS_TESTS=true
|
||||
|
||||
install:
|
||||
- mvn --version
|
||||
|
||||
script:
|
||||
- mvn clean install $PROFILE -Dmaven.plugin.scalatest.exclude.tags=$EXCLUDE_TAGS -Dmaven.javadoc.skip=true -V
|
||||
- if [[ "$TPCDS_TESTS" == "true" ]]; then mvn clean install -DskipTests -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper && mvn -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds.TPCDSOutputSchemaSuite,org.apache.kyuubi.operation.tpcds.TPCDSDDLSuite -Dmaven.plugin.scalatest.exclude.tags="" test; fi
|
||||
- if [[ "$TPCDS_TESTS" != "true" ]]; then mvn clean install $PROFILE -Dmaven.plugin.scalatest.exclude.tags=$EXCLUDE_TAGS -Dmaven.javadoc.skip=true -V ;fi
|
||||
|
||||
after_success:
|
||||
- if [[ -z $EXCLUDE_TAGS ]]; then bash <(curl -s https://codecov.io/bash); fi
|
||||
|
||||
@ -0,0 +1,30 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kyuubi.tags;
|
||||
|
||||
import org.scalatest.TagAnnotation;
|
||||
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
@TagAnnotation
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target({ElementType.METHOD, ElementType.TYPE})
|
||||
public @interface ExtendedSQLTest { }
|
||||
@ -19,7 +19,9 @@ package org.apache.kyuubi.operation.tpcds
|
||||
|
||||
import org.apache.kyuubi.config.KyuubiConf
|
||||
import org.apache.kyuubi.operation.{JDBCTestUtils, WithKyuubiServer}
|
||||
import org.apache.kyuubi.tags.ExtendedSQLTest
|
||||
|
||||
@ExtendedSQLTest
|
||||
class TPCDSDDLSuite extends WithKyuubiServer with JDBCTestUtils with TPCDSHelper {
|
||||
override protected val conf: KyuubiConf = KyuubiConf()
|
||||
|
||||
|
||||
@ -22,6 +22,7 @@ import java.nio.file.{Files, Path, Paths}
|
||||
|
||||
import org.apache.kyuubi.config.KyuubiConf
|
||||
import org.apache.kyuubi.operation.{JDBCTestUtils, WithKyuubiServer}
|
||||
import org.apache.kyuubi.tags.ExtendedSQLTest
|
||||
|
||||
// scalastyle:off line.size.limit
|
||||
/**
|
||||
@ -36,6 +37,7 @@ import org.apache.kyuubi.operation.{JDBCTestUtils, WithKyuubiServer}
|
||||
* }}}
|
||||
*/
|
||||
// scalastyle:on line.size.limit
|
||||
@ExtendedSQLTest
|
||||
class TPCDSOutputSchemaSuite extends WithKyuubiServer with JDBCTestUtils with TPCDSHelper {
|
||||
override protected val conf: KyuubiConf = KyuubiConf()
|
||||
override protected def jdbcUrl: String = getJdbcUrl
|
||||
@ -103,7 +105,7 @@ class TPCDSOutputSchemaSuite extends WithKyuubiServer with JDBCTestUtils with TP
|
||||
}.toSeq.sortBy(q => getQueryIndex(q.getFileName.toString))
|
||||
|
||||
validQueries.foreach { q =>
|
||||
test(name + "-" + q.getFileName.toString) {
|
||||
test(name + "/" + q.getFileName.toString) {
|
||||
q.toFile.listFiles().filter(_.getName.endsWith(".sql")).foreach { qf =>
|
||||
val schemaFile = Paths.get(
|
||||
baseResourcePath.toFile.getAbsolutePath,
|
||||
|
||||
6
pom.xml
6
pom.xml
@ -95,7 +95,8 @@
|
||||
<maven.plugin.scala.version>4.3.0</maven.plugin.scala.version>
|
||||
<maven.plugin.surefire.version>2.22.0</maven.plugin.surefire.version>
|
||||
<maven.plugin.scalatest.version>2.0.0</maven.plugin.scalatest.version>
|
||||
<maven.plugin.scalatest.exclude.tags></maven.plugin.scalatest.exclude.tags>
|
||||
<maven.plugin.scalatest.exclude.tags>org.apache.kyuubi.tags.ExtendedSQLTest</maven.plugin.scalatest.exclude.tags>
|
||||
<maven.plugin.scalatest.include.tags></maven.plugin.scalatest.include.tags>
|
||||
<maven.plugin.jacoco.version>0.8.6</maven.plugin.jacoco.version>
|
||||
<maven.plugin.shade.version>3.2.4</maven.plugin.shade.version>
|
||||
<maven.plugin.jar.version>3.2.0</maven.plugin.jar.version>
|
||||
@ -1217,6 +1218,7 @@
|
||||
<kyuubi.frontend.bind.host>localhost</kyuubi.frontend.bind.host>
|
||||
</systemProperties>
|
||||
<tagsToExclude>${maven.plugin.scalatest.exclude.tags}</tagsToExclude>
|
||||
<tagsToInclude>${maven.plugin.scalatest.include.tags}</tagsToInclude>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
@ -1498,7 +1500,7 @@
|
||||
<id>spark-3.1</id>
|
||||
<properties>
|
||||
<spark.version>3.1.1</spark.version>
|
||||
<maven.plugin.scalatest.exclude.tags>org.apache.kyuubi.tags.DataLakeTest</maven.plugin.scalatest.exclude.tags>
|
||||
<maven.plugin.scalatest.exclude.tags>org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest</maven.plugin.scalatest.exclude.tags>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user