[KYUUBI #1214] Refine sql extension and avoid setup data for unused test
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> - Refine rule order - Speed up test ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1214 from ulysses-you/tests. Closes #1214 261271f0 [ulysses-you] test Authored-by: ulysses-you <ulyssesyou18@gmail.com> Signed-off-by: ulysses-you <ulyssesyou@apache.org>
This commit is contained in:
parent
5230d09e1c
commit
6160b9de30
@ -38,13 +38,14 @@ class KyuubiSparkSQLExtension extends (SparkSessionExtensions => Unit) {
|
||||
extensions.injectParser{ case (_, parser) => new ZorderSparkSqlExtensionsParser(parser) }
|
||||
extensions.injectResolutionRule(ResolveZorder)
|
||||
|
||||
// a help rule for ForcedMaxOutputRowsRule
|
||||
extensions.injectResolutionRule(MarkAggregateOrderRule)
|
||||
|
||||
// Note that:
|
||||
// InsertZorderBeforeWritingDatasource and InsertZorderBeforeWritingHive
|
||||
// should be applied before
|
||||
// RepartitionBeforeWrite and RepartitionBeforeWriteHive
|
||||
// because we can only apply one of them (i.e. Global Sort or Repartition)
|
||||
extensions.injectResolutionRule(MarkAggregateOrderRule)
|
||||
|
||||
extensions.injectPostHocResolutionRule(InsertZorderBeforeWritingDatasource)
|
||||
extensions.injectPostHocResolutionRule(InsertZorderBeforeWritingHive)
|
||||
extensions.injectPostHocResolutionRule(KyuubiSqlClassification)
|
||||
|
||||
@ -23,6 +23,11 @@ import org.apache.spark.sql.internal.SQLConf
|
||||
import org.apache.kyuubi.sql.{FinalStageConfigIsolation, KyuubiSQLConf}
|
||||
|
||||
class FinalStageConfigIsolationSuite extends KyuubiSparkSQLExtensionTest {
|
||||
protected override def beforeAll(): Unit = {
|
||||
super.beforeAll()
|
||||
setupData()
|
||||
}
|
||||
|
||||
test("final stage config set reset check") {
|
||||
withSQLConf(KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true",
|
||||
"spark.sql.finalStage.adaptive.coalescePartitions.minPartitionNum" -> "1",
|
||||
|
||||
@ -23,6 +23,11 @@ import org.apache.spark.sql.internal.SQLConf
|
||||
import org.apache.kyuubi.sql.KyuubiSQLConf
|
||||
|
||||
class InsertShuffleNodeBeforeJoinSuite extends KyuubiSparkSQLExtensionTest {
|
||||
protected override def beforeAll(): Unit = {
|
||||
super.beforeAll()
|
||||
setupData()
|
||||
}
|
||||
|
||||
test("force shuffle before join") {
|
||||
def checkShuffleNodeNum(sqlString: String, num: Int): Unit = {
|
||||
var expectedResult: Seq[Row] = Seq.empty
|
||||
|
||||
@ -42,7 +42,6 @@ trait KyuubiSparkSQLExtensionTest extends QueryTest
|
||||
.enableHiveSupport()
|
||||
.getOrCreate())
|
||||
}
|
||||
setupData()
|
||||
super.beforeAll()
|
||||
}
|
||||
|
||||
@ -52,7 +51,7 @@ trait KyuubiSparkSQLExtensionTest extends QueryTest
|
||||
_spark.foreach(_.stop)
|
||||
}
|
||||
|
||||
private def setupData(): Unit = {
|
||||
protected def setupData(): Unit = {
|
||||
val self = spark
|
||||
import self.implicits._
|
||||
spark.sparkContext.parallelize(
|
||||
|
||||
@ -23,6 +23,10 @@ import org.apache.kyuubi.sql.KyuubiSQLConf
|
||||
import org.apache.kyuubi.sql.watchdog.MaxHivePartitionExceedException
|
||||
|
||||
class WatchDogSuite extends KyuubiSparkSQLExtensionTest {
|
||||
protected override def beforeAll(): Unit = {
|
||||
super.beforeAll()
|
||||
setupData()
|
||||
}
|
||||
|
||||
case class LimitAndExpected(limit: Int, expected: Int)
|
||||
val limitAndExpecteds = List(LimitAndExpected(1, 1), LimitAndExpected(11, 10))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user