[KYUUBI #1214] Refine sql extension and avoid setup data for unused test

<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->
- Refine rule order
- Speed up test

### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #1214 from ulysses-you/tests.

Closes #1214

261271f0 [ulysses-you] test

Authored-by: ulysses-you <ulyssesyou18@gmail.com>
Signed-off-by: ulysses-you <ulyssesyou@apache.org>
This commit is contained in:
ulysses-you 2021-10-12 10:50:03 +08:00 committed by ulysses-you
parent 5230d09e1c
commit 6160b9de30
No known key found for this signature in database
GPG Key ID: 4C500BC62D576766
5 changed files with 18 additions and 4 deletions

View File

@ -38,13 +38,14 @@ class KyuubiSparkSQLExtension extends (SparkSessionExtensions => Unit) {
extensions.injectParser{ case (_, parser) => new ZorderSparkSqlExtensionsParser(parser) }
extensions.injectResolutionRule(ResolveZorder)
// a help rule for ForcedMaxOutputRowsRule
extensions.injectResolutionRule(MarkAggregateOrderRule)
// Note that:
// InsertZorderBeforeWritingDatasource and InsertZorderBeforeWritingHive
// should be applied before
// RepartitionBeforeWrite and RepartitionBeforeWriteHive
// because we can only apply one of them (i.e. Global Sort or Repartition)
extensions.injectResolutionRule(MarkAggregateOrderRule)
extensions.injectPostHocResolutionRule(InsertZorderBeforeWritingDatasource)
extensions.injectPostHocResolutionRule(InsertZorderBeforeWritingHive)
extensions.injectPostHocResolutionRule(KyuubiSqlClassification)

View File

@ -23,6 +23,11 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.kyuubi.sql.{FinalStageConfigIsolation, KyuubiSQLConf}
class FinalStageConfigIsolationSuite extends KyuubiSparkSQLExtensionTest {
protected override def beforeAll(): Unit = {
super.beforeAll()
setupData()
}
test("final stage config set reset check") {
withSQLConf(KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true",
"spark.sql.finalStage.adaptive.coalescePartitions.minPartitionNum" -> "1",

View File

@ -23,6 +23,11 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.kyuubi.sql.KyuubiSQLConf
class InsertShuffleNodeBeforeJoinSuite extends KyuubiSparkSQLExtensionTest {
protected override def beforeAll(): Unit = {
super.beforeAll()
setupData()
}
test("force shuffle before join") {
def checkShuffleNodeNum(sqlString: String, num: Int): Unit = {
var expectedResult: Seq[Row] = Seq.empty

View File

@ -42,7 +42,6 @@ trait KyuubiSparkSQLExtensionTest extends QueryTest
.enableHiveSupport()
.getOrCreate())
}
setupData()
super.beforeAll()
}
@ -52,7 +51,7 @@ trait KyuubiSparkSQLExtensionTest extends QueryTest
_spark.foreach(_.stop)
}
private def setupData(): Unit = {
protected def setupData(): Unit = {
val self = spark
import self.implicits._
spark.sparkContext.parallelize(

View File

@ -23,6 +23,10 @@ import org.apache.kyuubi.sql.KyuubiSQLConf
import org.apache.kyuubi.sql.watchdog.MaxHivePartitionExceedException
class WatchDogSuite extends KyuubiSparkSQLExtensionTest {
protected override def beforeAll(): Unit = {
super.beforeAll()
setupData()
}
case class LimitAndExpected(limit: Int, expected: Int)
val limitAndExpecteds = List(LimitAndExpected(1, 1), LimitAndExpected(11, 10))