[KYUUBI #807] Make kyuubi-extension-spark_3.1 test pass with Spark 3.2.0-snapshot

<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->
Make this extesion more robust.

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request

Closes #807 from ulysses-you/sql-extension.

Closes #807

27155c1e [ulysses-you] fix

Authored-by: ulysses-you <ulyssesyou18@gmail.com>
Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
ulysses-you 2021-07-15 16:35:21 +08:00 committed by Cheng Pan
parent 4790cee274
commit ca94d7b46c
No known key found for this signature in database
GPG Key ID: F07E6C29ED4E2E5B

View File

@ -39,6 +39,7 @@ class KyuubiExtensionSuite extends QueryTest with SQLTestUtils with AdaptiveSpar
"org.apache.kyuubi.sql.KyuubiSparkSQLExtension")
.config(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")
.config("spark.hadoop.hive.exec.dynamic.partition.mode", "nonstrict")
.config("spark.hadoop.hive.metastore.client.capability.check", "false")
.config("spark.ui.enabled", "false")
.enableHiveSupport()
.getOrCreate()
@ -328,6 +329,7 @@ class KyuubiExtensionSuite extends QueryTest with SQLTestUtils with AdaptiveSpar
SQLConf.SHUFFLE_PARTITIONS.key -> "3",
KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true",
"spark.sql.adaptive.advisoryPartitionSizeInBytes" -> "1",
"spark.sql.adaptive.coalescePartitions.minPartitionSize" -> "1",
"spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes" -> "10000000") {
// use loop to double check final stage config doesn't affect the sql query each other
@ -372,10 +374,10 @@ class KyuubiExtensionSuite extends QueryTest with SQLTestUtils with AdaptiveSpar
// test ReusedExchange
checkPartitionNum(
"""
|SELECT t0.c2 FROM (
|SELECT t1.c1, count(*) as c2 FROM t1 GROUP BY t1.c1
|SELECT /*+ REPARTITION */ t0.c2 FROM (
|SELECT t1.c1, (count(*) + c1) as c2 FROM t1 GROUP BY t1.c1
|) t0 JOIN (
|SELECT t1.c1, count(*) as c2 FROM t1 GROUP BY t1.c1
|SELECT t1.c1, (count(*) + c1) as c2 FROM t1 GROUP BY t1.c1
|) t1 ON t0.c2 = t1.c2
|""".stripMargin,
3,