From 3e199ccdce6ae5dae7f23e59efdab9eda4904600 Mon Sep 17 00:00:00 2001 From: "guoqing.yang" Date: Mon, 18 Oct 2021 10:21:12 +0800 Subject: [PATCH] [KYUUBI #1209] kyuubi.engine.share.level.subdomain can be more tolerant feature: kyuubi.engine.share.level.subdomain can be more tolerant now end-user can configure kyuubi.engine.share.level.subdomain by zookeeper valid path I refer to the zookeeper doc http://zookeeper.apache.org/doc/r3.7.0/zookeeperProgrammers.html#:~:text=ZooKeeper%20Basic%20Operations.-,The%20ZooKeeper%20Data%20Model,-ZooKeeper%20has%20a and source code: https://github.com/apache/zookeeper/blob/master/zookeeper-server/src/main/java/org/apache/zookeeper/common/PathUtils.java#:~:text=public%20static%20void-,validatePath,-(String%20path)%20throws Closes #1232 from TyrealBM/feature/valid-subdomain-path. Closes #1209 8d00c260 [guoqing.yang] [KYUUBI #1209] kyuubi.engine.share.level.subdomain can be more tolerant Authored-by: guoqing.yang Signed-off-by: Kent Yao --- docs/deployment/settings.md | 2 +- .../org/apache/kyuubi/config/KyuubiConf.scala | 12 +++++++--- .../kyuubi/config/KyuubiConfSuite.scala | 23 +++++++++++++++++++ .../engine/spark/SparkSqlEngineSuite.scala | 2 +- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md index d9c5fa703..aecf08a78 100644 --- a/docs/deployment/settings.md +++ b/docs/deployment/settings.md @@ -186,7 +186,7 @@ kyuubi\.engine\.pool
\.size\.threshold|
SHOW DATABASES
|
SemiColon-separated list of SQL statements to be initialized in the newly created engine session before queries. This configuration can not be used in JDBC url due to the limitation of Beeline/JDBC driver.
|
seq
|
1.3.0
kyuubi\.engine\.share
\.level|
USER
|
Engines will be shared in different levels, available configs are:
  • CONNECTION: engine will not be shared but only used by the current client connection
  • USER: engine will be shared by all sessions created by a unique username, see also kyuubi.engine.share.level.subdomain
  • GROUP: engine will be shared by all sessions created by all users belong to the same primary group name. The engine will be launched by the group name as the effective username, so here the group name is kind of special user who is able to visit the compute resources/data of a team. It follows the [Hadoop GroupsMapping](https://reurl.cc/xE61Y5) to map user to a primary group. If the primary group is not found, it fallback to the USER level.
  • SERVER: the App will be shared by Kyuubi servers
|
string
|
1.2.0
kyuubi\.engine\.share
\.level\.sub\.domain|
<undefined>
|
(deprecated) - Using kyuubi.engine.share.level.subdomain instead
|
string
|
1.2.0
-kyuubi\.engine\.share
\.level\.subdomain|
<undefined>
|
Allow end-users to create a subdomain for the share level of an engine. A subdomain is a case-insensitive string values in `^[a-zA-Z_-]{1,14}$` form. For example, for `USER` share level, an end-user can share a certain engine within a subdomain, not for all of its clients. End-users are free to create multiple engines in the `USER` share level
|
string
|
1.4.0
+kyuubi\.engine\.share
\.level\.subdomain|
<undefined>
|
Allow end-users to create a subdomain for the share level of an engine. A subdomain is a case-insensitive string values that must be a valid zookeeper sub path. For example, for `USER` share level, an end-user can share a certain engine within a subdomain, not for all of its clients. End-users are free to create multiple engines in the `USER` share level
|
string
|
1.4.0
kyuubi\.engine\.single
\.spark\.session|
false
|
When set to true, this engine is running in a single session mode. All the JDBC/ODBC connections share the temporary views, function registries, SQL configuration and the current database.
|
boolean
|
1.3.0
kyuubi\.engine\.ui
\.retainedSessions|
200
|
The number of SQL client sessions kept in the Kyuubi Query Engine web UI.
|
int
|
1.4.0
kyuubi\.engine\.ui\.stop
\.enabled|
true
|
When true, allows Kyuubi engine to be killed from the Spark Web UI.
|
boolean
|
1.3.0
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 3447a18ed..ec4d5522f 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -714,6 +714,11 @@ object KyuubiConf { private val validEngineSubDomain: Pattern = "^[a-zA-Z_-]{1,14}$".r.pattern + // [ZooKeeper Data Model] + // (http://zookeeper.apache.org/doc/r3.7.0/zookeeperProgrammers.html#ch_zkDataModel) + private val validEngineSubdomain: Pattern = ("(?!^[\\u002e]{1,2}$)" + + "(^[\\u0020-\\u002e\\u0030-\\u007e\\u00a0-\\ud7ff\\uf900-\\uffef]{1,}$)").r.pattern + @deprecated(s"using kyuubi.engine.share.level.subdomain instead", "1.4.0") val ENGINE_SHARE_LEVEL_SUB_DOMAIN: OptionalConfigEntry[String] = buildConf("engine.share.level.sub.domain") @@ -721,14 +726,15 @@ object KyuubiConf { .version("1.2.0") .stringConf .transform(_.toLowerCase(Locale.ROOT)) - .checkValue(validEngineSubDomain.matcher(_).matches(), - "must be [1, 14] length alphabet string, e.g. 'abc', 'apache'") + .checkValue(validEngineSubdomain.matcher(_).matches(), + "must be valid zookeeper sub path." + ) .createOptional val ENGINE_SHARE_LEVEL_SUBDOMAIN: ConfigEntry[Option[String]] = buildConf("engine.share.level.subdomain") .doc("Allow end-users to create a subdomain for the share level of an engine. A" + - " subdomain is a case-insensitive string values in `^[a-zA-Z_-]{1,14}$` form." + + " subdomain is a case-insensitive string values that must be a valid zookeeper sub path." + " For example, for `USER` share level, an end-user can share a certain engine within" + " a subdomain, not for all of its clients. End-users are free to create multiple" + " engines in the `USER` share level") diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala index 944316db2..fb1bd16c1 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala @@ -150,4 +150,27 @@ class KyuubiConfSuite extends KyuubiFunSuite { val e1 = intercept[IllegalArgumentException](kyuubiConf.get(OPERATION_QUERY_TIMEOUT)) assert(e1.getMessage.contains("must >= 1s if set")) } + + test("kyuubi conf engine.share.level.subdomain valid path test") { + val kyuubiConf = KyuubiConf() + kyuubiConf.set(ENGINE_SHARE_LEVEL_SUBDOMAIN.key, "") + assertThrows[IllegalArgumentException](kyuubiConf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN)) + kyuubiConf.set(ENGINE_SHARE_LEVEL_SUBDOMAIN.key, ".") + assertThrows[IllegalArgumentException](kyuubiConf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN)) + kyuubiConf.set(ENGINE_SHARE_LEVEL_SUBDOMAIN.key, "..") + assertThrows[IllegalArgumentException](kyuubiConf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN)) + kyuubiConf.set(ENGINE_SHARE_LEVEL_SUBDOMAIN.key, "/") + assertThrows[IllegalArgumentException](kyuubiConf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN)) + kyuubiConf.set(ENGINE_SHARE_LEVEL_SUBDOMAIN.key, "/tmp/") + assertThrows[IllegalArgumentException](kyuubiConf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN)) + kyuubiConf.set(ENGINE_SHARE_LEVEL_SUBDOMAIN.key, "tmp/") + assertThrows[IllegalArgumentException](kyuubiConf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN)) + kyuubiConf.set(ENGINE_SHARE_LEVEL_SUBDOMAIN.key, "/tmp") + assertThrows[IllegalArgumentException](kyuubiConf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN)) + kyuubiConf.set(ENGINE_SHARE_LEVEL_SUBDOMAIN.key, "abc/efg") + assertThrows[IllegalArgumentException](kyuubiConf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN)) + val path = "kyuubi!@#$%^&*()_+-=[]{};:,.<>?" + kyuubiConf.set(ENGINE_SHARE_LEVEL_SUBDOMAIN.key, path) + assert(kyuubiConf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN).get == path) + } } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkSqlEngineSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkSqlEngineSuite.scala index 7cd1004ca..a695f63fe 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkSqlEngineSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkSqlEngineSuite.scala @@ -72,7 +72,7 @@ class SparkSqlEngineSuite extends WithKyuubiServer with JDBCTestUtils { test("Fail connections on invalid sub domains") { - Seq("1", ",", "", "a" * 15, "abc.xyz").foreach { invalid => + Seq("/", "/tmp", "", "abc/efg", ".", "..").foreach { invalid => val sparkHiveConfigs = Map( ENGINE_SHARE_LEVEL.key -> "USER", ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> invalid)