[KYUUBI #6178] Improve Gluten integration test
# 🔍 Description ## Issue References 🔗 `kyuubi-gluten-it` is always enabled and uses a different Spark version, which mess the code jumping in IDEA. ## Describe Your Solution 🔧 - conditional enable `kyuubi-gluten-it` module via profile `gluten-it` - refactor POM and GHA workflow to reduce duplicated definition ## Types of changes 🔖 - [x] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan 🧪 Review, and wait for daily GHA results. --- # Checklist 📝 - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes #6178 from pan3793/gluten-it. Closes #6178 77b7bb809 [Cheng Pan] fix c62ce40f8 [Cheng Pan] Improve Gluten intergartion test Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
Cheng Pan
parent
3df21205b6
commit
c137270edc
5
.github/workflows/gluten.yml
vendored
5
.github/workflows/gluten.yml
vendored
@ -115,9 +115,8 @@ jobs:
|
||||
- name: Run Gluten Integration TPC-H/DS Test
|
||||
run: |
|
||||
TEST_MODULES="integration-tests/kyuubi-gluten-it"
|
||||
./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests -Pgluten-spark-${{ matrix.spark }}
|
||||
./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test -Pgluten-spark-${{ matrix.spark }} \
|
||||
-Dmaven.plugin.scalatest.exclude.tags='' -Dtest=none -Dmaven.plugin.scalatest.include.tags='org.apache.kyuubi.tags.GlutenTest'
|
||||
./build/mvn ${MVN_OPT} -Pgluten-it -Pspark-${{ matrix.spark }} -pl ${TEST_MODULES} -am clean install -DskipTests
|
||||
./build/mvn ${MVN_OPT} -Pgluten-it -Pspark-${{ matrix.spark }} -pl ${TEST_MODULES} test
|
||||
- name: Upload test logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
|
||||
2
.github/workflows/master.yml
vendored
2
.github/workflows/master.yml
vendored
@ -160,7 +160,7 @@ jobs:
|
||||
uses: ./.github/actions/cache-engine-archives
|
||||
- name: Build on Scala ${{ matrix.scala }}
|
||||
run: |
|
||||
TEST_MODULES="!externals/kyuubi-flink-sql-engine,!integration-tests/kyuubi-flink-it,!integration-tests/kyuubi-gluten-it"
|
||||
TEST_MODULES="!externals/kyuubi-flink-sql-engine,!integration-tests/kyuubi-flink-it"
|
||||
./build/mvn clean install ${MVN_OPT} -pl ${TEST_MODULES} -am \
|
||||
-Pscala-${{ matrix.scala }} -Pjava-${{ matrix.java }} -Pspark-${{ matrix.spark }}
|
||||
- name: Upload test logs
|
||||
|
||||
@ -31,11 +31,18 @@
|
||||
|
||||
<properties>
|
||||
<gluten.version>1.2.0-SNAPSHOT</gluten.version>
|
||||
<spark.version>3.4.2</spark.version>
|
||||
<spark.binary.version>3.4</spark.binary.version>
|
||||
<gluten.artifact.name>gluten-velox-bundle-spark${spark.binary.version}_${scala.binary.version}-ubuntu_22.04</gluten.artifact.name>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>io.glutenproject</groupId>
|
||||
<artifactId>${gluten.artifact.name}</artifactId>
|
||||
<version>${gluten.version}</version>
|
||||
<scope>system</scope>
|
||||
<systemPath>${project.basedir}/../../gluten/package/target/${gluten.artifact.name}_x86_64-${gluten.version}.jar</systemPath>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>kyuubi-common_${scala.binary.version}</artifactId>
|
||||
@ -90,38 +97,18 @@
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>gluten-spark-3.4</id>
|
||||
<id>gluten-it</id>
|
||||
<properties>
|
||||
<maven.plugin.scalatest.exclude.tags></maven.plugin.scalatest.exclude.tags>
|
||||
<maven.plugin.scalatest.include.tags>org.apache.kyuubi.tags.GlutenTest</maven.plugin.scalatest.include.tags>
|
||||
<spark.version>3.4.2</spark.version>
|
||||
<spark.binary.version>3.4</spark.binary.version>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>io.glutenproject</groupId>
|
||||
<artifactId>gluten-velox-bundle-spark3.4_2.12-ubuntu_22.04</artifactId>
|
||||
<version>${gluten.version}</version>
|
||||
<scope>system</scope>
|
||||
<systemPath>${project.basedir}/../../gluten/package/target/gluten-velox-bundle-spark3.4_2.12-ubuntu_22.04_x86_64-${gluten.version}.jar</systemPath>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
<profile>
|
||||
<id>gluten-spark-3.3</id>
|
||||
<id>spark-3.3</id>
|
||||
<properties>
|
||||
<maven.plugin.scalatest.include.tags>org.apache.kyuubi.tags.GlutenTest</maven.plugin.scalatest.include.tags>
|
||||
<!-- Gluten does not support the latest Spark 3.3 now -->
|
||||
<spark.version>3.3.1</spark.version>
|
||||
<spark.binary.version>3.3</spark.binary.version>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>io.glutenproject</groupId>
|
||||
<artifactId>gluten-velox-bundle-spark3.3_2.12-ubuntu_22.04</artifactId>
|
||||
<version>${gluten.version}</version>
|
||||
<scope>system</scope>
|
||||
<systemPath>${project.basedir}/../../gluten/package/target/gluten-velox-bundle-spark3.3_2.12-ubuntu_22.04_x86_64-${gluten.version}.jar</systemPath>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
||||
@ -31,7 +31,6 @@
|
||||
|
||||
<modules>
|
||||
<module>kyuubi-flink-it</module>
|
||||
<module>kyuubi-gluten-it</module>
|
||||
<module>kyuubi-hive-it</module>
|
||||
<module>kyuubi-trino-it</module>
|
||||
<module>kyuubi-jdbc-it</module>
|
||||
@ -45,5 +44,11 @@
|
||||
<module>kyuubi-kubernetes-it</module>
|
||||
</modules>
|
||||
</profile>
|
||||
<profile>
|
||||
<id>gluten-it</id>
|
||||
<modules>
|
||||
<module>kyuubi-gluten-it</module>
|
||||
</modules>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user