diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index 1cf1d36a4..cd84ee8cb 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -457,7 +457,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co | kyuubi.session.conf.advisor | <undefined> | A config advisor plugin for Kyuubi Server. This plugin can provide a list of custom configs for different users or session configs and overwrite the session configs before opening a new session. This config value should be a subclass of `org.apache.kyuubi.plugin.SessionConfAdvisor` which has a zero-arg constructor. | seq | 1.5.0 | | kyuubi.session.conf.file.reload.interval | PT10M | When `FileSessionConfAdvisor` is used, this configuration defines the expired time of `$KYUUBI_CONF_DIR/kyuubi-session-.conf` in the cache. After exceeding this value, the file will be reloaded. | duration | 1.7.0 | | kyuubi.session.conf.ignore.list || A comma-separated list of ignored keys. If the client connection contains any of them, the key and the corresponding value will be removed silently during engine bootstrap and connection setup. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering but will not forbid users to set dynamic configurations via SET syntax. | set | 1.2.0 | -| kyuubi.session.conf.profile | <undefined> | Specify a profile to load session-level configurations from `$KYUUBI_CONF_DIR/kyuubi-session-.conf`. This configuration will be ignored if the file does not exist. This configuration only takes effect when `kyuubi.session.conf.advisor` is set as `org.apache.kyuubi.session.FileSessionConfAdvisor`. | string | 1.7.0 | +| kyuubi.session.conf.profile | <undefined> | Specify a profile to load session-level configurations from multiple `$KYUUBI_CONF_DIR/kyuubi-session-.conf` files. This configuration will be ignored if the file does not exist. This configuration only takes effect when `kyuubi.session.conf.advisor` is set as `org.apache.kyuubi.session.FileSessionConfAdvisor`. | seq | 1.7.0 | | kyuubi.session.conf.restrict.list || A comma-separated list of restricted keys. If the client connection contains any of them, the connection will be rejected explicitly during engine bootstrap and connection setup. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering but will not forbid users to set dynamic configurations via SET syntax. | set | 1.2.0 | | kyuubi.session.engine.alive.max.failures | 3 | The maximum number of failures allowed for the engine. | int | 1.8.1 | | kyuubi.session.engine.alive.probe.enabled | false | Whether to enable the engine alive probe, it true, we will create a companion thrift client that keeps sending simple requests to check whether the engine is alive. | boolean | 1.6.0 | diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 6cb75db9c..be4ed0c42 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -1359,15 +1359,16 @@ object KyuubiConf { .stringConf .createOptional - val SESSION_CONF_PROFILE: OptionalConfigEntry[String] = + val SESSION_CONF_PROFILE: OptionalConfigEntry[Seq[String]] = buildConf("kyuubi.session.conf.profile") .doc("Specify a profile to load session-level configurations from " + - "`$KYUUBI_CONF_DIR/kyuubi-session-.conf`. " + + "multiple `$KYUUBI_CONF_DIR/kyuubi-session-.conf` files. " + "This configuration will be ignored if the file does not exist. " + "This configuration only takes effect when `kyuubi.session.conf.advisor` " + "is set as `org.apache.kyuubi.session.FileSessionConfAdvisor`.") .version("1.7.0") .stringConf + .toSequence() .createOptional val SESSION_CONF_FILE_RELOAD_INTERVAL: ConfigEntry[Long] = diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/session/FileSessionConfAdvisor.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/session/FileSessionConfAdvisor.scala index d480520c4..0e9ce7ca6 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/session/FileSessionConfAdvisor.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/session/FileSessionConfAdvisor.scala @@ -33,11 +33,15 @@ class FileSessionConfAdvisor extends SessionConfAdvisor { override def getConfOverlay( user: String, sessionConf: JMap[String, String]): JMap[String, String] = { - val profile: String = sessionConf.get(KyuubiConf.SESSION_CONF_PROFILE.key) - profile match { + val profiles: String = sessionConf.get(KyuubiConf.SESSION_CONF_PROFILE.key) + profiles match { case null => Collections.emptyMap() case _ => - sessionConfCache.get(profile) + val confMap = scala.collection.mutable.Map[String, String]() + profiles.split(",").map(_.trim).filter(_.nonEmpty).foreach { profile => + confMap ++= sessionConfCache.get(profile).asScala + } + confMap.asJava } } } diff --git a/kyuubi-server/src/test/resources/kyuubi-session-cluster-b.conf b/kyuubi-server/src/test/resources/kyuubi-session-cluster-b.conf new file mode 100644 index 000000000..1f3327e42 --- /dev/null +++ b/kyuubi-server/src/test/resources/kyuubi-session-cluster-b.conf @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kyuubi.ha.namespace kyuubi-ns-b +kyuubi.engineEnv.HIVE_DIR /opt/hive_conf_dir diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/plugin/PluginLoaderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/plugin/PluginLoaderSuite.scala index bd7f78e24..fa4505cc2 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/plugin/PluginLoaderSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/plugin/PluginLoaderSuite.scala @@ -50,12 +50,12 @@ class PluginLoaderSuite extends KyuubiFunSuite { advisor.map(_.getConfOverlay("chris", conf.getAll.asJava).asScala).reduce(_ ++ _).asJava assert(emptyConfig.isEmpty) - conf.set(KyuubiConf.SESSION_CONF_PROFILE, "non.exists") + conf.set(KyuubiConf.SESSION_CONF_PROFILE, Seq("non.exists")) val nonExistsConfig = advisor.map(_.getConfOverlay("chris", conf.getAll.asJava).asScala).reduce(_ ++ _).asJava assert(nonExistsConfig.isEmpty) - conf.set(KyuubiConf.SESSION_CONF_PROFILE, "cluster-a") + conf.set(KyuubiConf.SESSION_CONF_PROFILE, Seq("cluster-a")) val clusterAConf = advisor.map(_.getConfOverlay("chris", conf.getAll.asJava).asScala).reduce(_ ++ _).asJava assert(clusterAConf.get("kyuubi.ha.namespace") == "kyuubi-ns-a") @@ -67,6 +67,21 @@ class PluginLoaderSuite extends KyuubiFunSuite { assert(clusterAConfFromCache.get("kyuubi.ha.namespace") == "kyuubi-ns-a") assert(clusterAConfFromCache.get("kyuubi.zk.ha.namespace") == null) assert(clusterAConfFromCache.size() == 5) + + conf.set(KyuubiConf.SESSION_CONF_PROFILE, Seq("cluster-a", "cluster-b")) + val clusterABConf = + advisor.map(_.getConfOverlay("chris", conf.getAll.asJava).asScala).reduce(_ ++ _).asJava + assert(clusterABConf.get("kyuubi.ha.namespace") == "kyuubi-ns-b") + assert(clusterABConf.get("kyuubi.zk.ha.namespace") == null) + assert(clusterABConf.get("kyuubi.engineEnv.HIVE_DIR") == "/opt/hive_conf_dir") + assert(clusterABConf.size() == 6) + + val clusterABConfFromCache = + advisor.map(_.getConfOverlay("chris", conf.getAll.asJava).asScala).reduce(_ ++ _).asJava + assert(clusterABConfFromCache.get("kyuubi.ha.namespace") == "kyuubi-ns-b") + assert(clusterABConfFromCache.get("kyuubi.zk.ha.namespace") == null) + assert(clusterABConf.get("kyuubi.engineEnv.HIVE_DIR") == "/opt/hive_conf_dir") + assert(clusterABConfFromCache.size() == 6) } test("SessionConfAdvisor - multi class") { @@ -75,7 +90,7 @@ class PluginLoaderSuite extends KyuubiFunSuite { KyuubiConf.SESSION_CONF_ADVISOR, Seq(classOf[FileSessionConfAdvisor].getName, classOf[TestSessionConfAdvisor].getName)) val advisor = PluginLoader.loadSessionConfAdvisor(conf) - conf.set(KyuubiConf.SESSION_CONF_PROFILE, "cluster-a") + conf.set(KyuubiConf.SESSION_CONF_PROFILE, Seq("cluster-a")) val clusterAConf = advisor.map(_.getConfOverlay("chris", conf.getAll.asJava).asScala).reduce(_ ++ _).asJava assert(clusterAConf.get("kyuubi.ha.namespace") == "kyuubi-ns-a")