From b3ecaef35e92058ffb3febf8ffbdbff3f9b4d76d Mon Sep 17 00:00:00 2001 From: senmiaoliu Date: Mon, 5 Sep 2022 15:02:58 +0800 Subject: [PATCH] [KYUUBI #3311] Operation language should set UNKONWN when language value is incorrect ### _Why are the changes needed?_ close https://github.com/apache/incubator-kyuubi/issues/3311 ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #3324 from lsm1/fix/operation_language_incorrect. Closes #3311 9192f61f [senmiaoliu] revert delete PlanOnlyOperationSuite.scala 9d0079b1 [senmiaoliu] move ut in SparkQueryTests e8b160da [senmiaoliu] add unknown for operation language Authored-by: senmiaoliu Signed-off-by: Kent Yao --- .../operation/SparkSQLOperationManager.scala | 11 +++++++---- .../org/apache/kyuubi/config/KyuubiConf.scala | 13 +++++++++++-- .../apache/kyuubi/operation/SparkQueryTests.scala | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkSQLOperationManager.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkSQLOperationManager.scala index dc3cb65af..13b8bbc51 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkSQLOperationManager.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkSQLOperationManager.scala @@ -17,7 +17,6 @@ package org.apache.kyuubi.engine.spark.operation -import java.util.Locale import java.util.concurrent.ConcurrentHashMap import scala.collection.JavaConverters._ @@ -62,11 +61,11 @@ class SparkSQLOperationManager private (name: String) extends OperationManager(n return catalogDatabaseOperation } } - val lang = confOverlay.getOrElse( + val lang = OperationLanguages(confOverlay.getOrElse( OPERATION_LANGUAGE.key, - spark.conf.get(OPERATION_LANGUAGE.key, operationLanguageDefault)) + spark.conf.get(OPERATION_LANGUAGE.key, operationLanguageDefault))) val operation = - OperationLanguages.withName(lang.toUpperCase(Locale.ROOT)) match { + lang match { case OperationLanguages.SQL => val mode = OperationModes(spark.conf.get(OPERATION_PLAN_ONLY_MODE.key, operationModeDefault)) @@ -82,6 +81,10 @@ class SparkSQLOperationManager private (name: String) extends OperationManager(n case OperationLanguages.SCALA => val repl = sessionToRepl.getOrElseUpdate(session.handle, KyuubiSparkILoop(spark)) new ExecuteScala(session, repl, statement) + case OperationLanguages.UNKNOWN => + spark.conf.unset(OPERATION_LANGUAGE.key) + throw KyuubiSQLException(s"The operation language $lang" + + " doesn't support in Spark SQL engine.") } addOperation(operation) } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index c249b8be8..5628dfef2 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -1721,9 +1721,18 @@ object KyuubiConf { "UseStatement", "SetCatalogAndNamespace")) - object OperationLanguages extends Enumeration { + object OperationLanguages extends Enumeration with Logging { type OperationLanguage = Value - val SQL, SCALA = Value + val SQL, SCALA, UNKNOWN = Value + def apply(language: String): OperationLanguage = { + language.toUpperCase(Locale.ROOT) match { + case "SQL" => SQL + case "SCALA" => SCALA + case other => + warn(s"Unsupported operation language: $language, using UNKNOWN instead") + UNKNOWN + } + } } val OPERATION_LANGUAGE: ConfigEntry[String] = diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala index 8b8d26854..8c9f120c7 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala @@ -27,6 +27,7 @@ import org.apache.commons.lang3.StringUtils import org.apache.hive.service.rpc.thrift.{TExecuteStatementReq, TFetchResultsReq, TOpenSessionReq, TStatusCode} import org.apache.kyuubi.{KYUUBI_VERSION, Utils} +import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.engine.SemanticVersion trait SparkQueryTests extends HiveJDBCTestHelper { @@ -675,6 +676,20 @@ trait SparkQueryTests extends HiveJDBCTestHelper { } } + test("kyuubi #3311: Operation language with an incorrect value") { + withSessionConf()(Map(KyuubiConf.OPERATION_LANGUAGE.key -> "SQL"))(Map.empty) { + withJdbcStatement() { statement => + statement.executeQuery(s"set ${KyuubiConf.OPERATION_LANGUAGE.key}=AAA") + val e = intercept[SQLException](statement.executeQuery("select 1")) + assert(e.getMessage.contains("The operation language UNKNOWN doesn't support")) + statement.executeQuery(s"set ${KyuubiConf.OPERATION_LANGUAGE.key}=SQL") + val result = statement.executeQuery("select 1") + assert(result.next()) + assert(result.getInt(1) === 1) + } + } + } + def sparkEngineMajorMinorVersion: (Int, Int) = { var sparkRuntimeVer = "" withJdbcStatement() { stmt =>