apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Ignore static and other unmodifiable settting in session

Browse Source
This commit is contained in:
Kent Yao 2020-11-11 11:15:09 +08:00
parent 574c119712
commit fb4bace6a5
4 changed files with 13 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/deployment/settings.md
View File

@ -191,7 +191,7 @@ Setting them in the JDBC Connection URL supplies session-specific for each SQL e
- **Runtime SQL Configuration**
- For [Runtime SQL Configurations](http://spark.apache.org/docs/latest/configuration.html#runtime-sql-configuration), they will take affect every time
- **Static SQL and Spark Core Configuration**
- For [Static SQL Configurations](http://spark.apache.org/docs/latest/configuration.html#static-sql-configuration) and other spark core configs, e.g. `spark.executor.memory`, they will take affect if there is no existing SQL engine application. Otherwise, they will just be ignored or sometimes fail the connection.
- For [Static SQL Configurations](http://spark.apache.org/docs/latest/configuration.html#static-sql-configuration) and other spark core configs, e.g. `spark.executor.memory`, they will take affect if there is no existing SQL engine application. Otherwise, they will just be ignored
### Via SET Syntax
Please refer to the Spark official online documentation for [SET Command](http://spark.apache.org/docs/latest/sql-ref-syntax-aux-conf-mgmt-set.html)

4
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala vendored
View File

@ -54,7 +54,9 @@ class SparkSQLSessionManager private (name: String, spark: SparkSession)
case (HIVE_VAR_PREFIX(key), value) => sparkSession.conf.set(key, value)
case (HIVE_CONF_PREFIX(key), value) => sparkSession.conf.set(key, value)
case ("use:database", database) => sparkSession.catalog.setCurrentDatabase(database)
case (key, value) => sparkSession.conf.set(key, value)
case (key, value) if sparkSession.conf.isModifiable(key) =>
sparkSession.conf.set(key, value)
case (key, _) => warn(s"Spark config $key is static and will be ignored")
}
sessionImpl.open()
operationManager.setSparkSession(handle, sparkSession)

12
externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala vendored
View File

@ -511,17 +511,21 @@ class SparkOperationSuite extends WithSparkSQLEngine {
}
}
test("set session conf - static") {
test("set session conf - static and core") {
withThriftClient { client =>
val req = new TOpenSessionReq()
req.setUsername("kentyao")
req.setPassword("anonymous")
val conf = Map("use:database" -> "default", "spark.sql.globalTempDatabase" -> "temp")
val queue = "spark.yarn.queue"
val conf = Map("use:database" -> "default",
"spark.sql.globalTempDatabase" -> "temp",
queue -> "new")
req.setConfiguration(conf.asJava)
val tOpenSessionResp = client.OpenSession(req)
val status = tOpenSessionResp.getStatus
assert(status.getStatusCode === TStatusCode.ERROR_STATUS)
assert(status.getErrorMessage.contains("spark.sql.globalTempDatabase"))
assert(status.getStatusCode === TStatusCode.SUCCESS_STATUS)
assert(spark.conf.get("spark.sql.globalTempDatabase") === "global_temp")
assert(spark.conf.getOption(queue).isEmpty)
}
}

3
kyuubi-main/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala
View File

@ -166,8 +166,7 @@ class AllKyuubiConfiguration extends KyuubiFunSuite {
writeWithNewLine(" - For [Static SQL Configurations](" +
"http://spark.apache.org/docs/latest/configuration.html#static-sql-configuration) and" +
" other spark core configs, e.g. `spark.executor.memory`, they will take affect if there" +
" is no existing SQL engine application. Otherwise, they will just be ignored or sometimes" +
" fail the connection.")
" is no existing SQL engine application. Otherwise, they will just be ignored")
writeWith2Line("### Via SET Syntax")
writeWithNewLine("Please refer to the Spark official online documentation for" +
" [SET Command](http://spark.apache.org/docs/latest/sql-ref-syntax-aux-conf-mgmt-set.html)")