[KYUUBI #6585] Support Multiple Configuration Files in FileSessionConfAdvisor
# 🔍 Description ## Issue References 🔗 This pull request fixes # ## Describe Your Solution 🔧 The current FileSessionConfAdvisor only supports configuring a single file through kyuubi.session.conf.profiler. However, users may need to use multiple configuration files, and the existing setup does not support this. To address this need, we have updated the FileSessionConfAdvisor to support multiple configuration files. ## Types of changes 🔖 - [ ] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan 🧪 #### Behavior Without This Pull Request ⚰️ #### Behavior With This Pull Request 🎉 #### Related Unit Tests --- # Checklist 📝 - [ ] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes #6585 from lsm1/branch-file-advidor-support-chain. Closes #6585 15b6524e0 [senmiaoliu] fix style fcedc4fe3 [senmiaoliu] FileSessionConfAdvisor support multi files Authored-by: senmiaoliu <senmiaoliu@trip.com> Signed-off-by: senmiaoliu <senmiaoliu@trip.com>
This commit is contained in:
senmiaoliu
parent
bdd91f4539
commit
72396e7f2d
@ -457,7 +457,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
|
||||
| kyuubi.session.conf.advisor | <undefined> | A config advisor plugin for Kyuubi Server. This plugin can provide a list of custom configs for different users or session configs and overwrite the session configs before opening a new session. This config value should be a subclass of `org.apache.kyuubi.plugin.SessionConfAdvisor` which has a zero-arg constructor. | seq | 1.5.0 |
|
||||
| kyuubi.session.conf.file.reload.interval | PT10M | When `FileSessionConfAdvisor` is used, this configuration defines the expired time of `$KYUUBI_CONF_DIR/kyuubi-session-<profile>.conf` in the cache. After exceeding this value, the file will be reloaded. | duration | 1.7.0 |
|
||||
| kyuubi.session.conf.ignore.list || A comma-separated list of ignored keys. If the client connection contains any of them, the key and the corresponding value will be removed silently during engine bootstrap and connection setup. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering but will not forbid users to set dynamic configurations via SET syntax. | set | 1.2.0 |
|
||||
| kyuubi.session.conf.profile | <undefined> | Specify a profile to load session-level configurations from `$KYUUBI_CONF_DIR/kyuubi-session-<profile>.conf`. This configuration will be ignored if the file does not exist. This configuration only takes effect when `kyuubi.session.conf.advisor` is set as `org.apache.kyuubi.session.FileSessionConfAdvisor`. | string | 1.7.0 |
|
||||
| kyuubi.session.conf.profile | <undefined> | Specify a profile to load session-level configurations from multiple `$KYUUBI_CONF_DIR/kyuubi-session-<profile>.conf` files. This configuration will be ignored if the file does not exist. This configuration only takes effect when `kyuubi.session.conf.advisor` is set as `org.apache.kyuubi.session.FileSessionConfAdvisor`. | seq | 1.7.0 |
|
||||
| kyuubi.session.conf.restrict.list || A comma-separated list of restricted keys. If the client connection contains any of them, the connection will be rejected explicitly during engine bootstrap and connection setup. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering but will not forbid users to set dynamic configurations via SET syntax. | set | 1.2.0 |
|
||||
| kyuubi.session.engine.alive.max.failures | 3 | The maximum number of failures allowed for the engine. | int | 1.8.1 |
|
||||
| kyuubi.session.engine.alive.probe.enabled | false | Whether to enable the engine alive probe, it true, we will create a companion thrift client that keeps sending simple requests to check whether the engine is alive. | boolean | 1.6.0 |
|
||||
|
||||
@ -1359,15 +1359,16 @@ object KyuubiConf {
|
||||
.stringConf
|
||||
.createOptional
|
||||
|
||||
val SESSION_CONF_PROFILE: OptionalConfigEntry[String] =
|
||||
val SESSION_CONF_PROFILE: OptionalConfigEntry[Seq[String]] =
|
||||
buildConf("kyuubi.session.conf.profile")
|
||||
.doc("Specify a profile to load session-level configurations from " +
|
||||
"`$KYUUBI_CONF_DIR/kyuubi-session-<profile>.conf`. " +
|
||||
"multiple `$KYUUBI_CONF_DIR/kyuubi-session-<profile>.conf` files. " +
|
||||
"This configuration will be ignored if the file does not exist. " +
|
||||
"This configuration only takes effect when `kyuubi.session.conf.advisor` " +
|
||||
"is set as `org.apache.kyuubi.session.FileSessionConfAdvisor`.")
|
||||
.version("1.7.0")
|
||||
.stringConf
|
||||
.toSequence()
|
||||
.createOptional
|
||||
|
||||
val SESSION_CONF_FILE_RELOAD_INTERVAL: ConfigEntry[Long] =
|
||||
|
||||
@ -33,11 +33,15 @@ class FileSessionConfAdvisor extends SessionConfAdvisor {
|
||||
override def getConfOverlay(
|
||||
user: String,
|
||||
sessionConf: JMap[String, String]): JMap[String, String] = {
|
||||
val profile: String = sessionConf.get(KyuubiConf.SESSION_CONF_PROFILE.key)
|
||||
profile match {
|
||||
val profiles: String = sessionConf.get(KyuubiConf.SESSION_CONF_PROFILE.key)
|
||||
profiles match {
|
||||
case null => Collections.emptyMap()
|
||||
case _ =>
|
||||
sessionConfCache.get(profile)
|
||||
val confMap = scala.collection.mutable.Map[String, String]()
|
||||
profiles.split(",").map(_.trim).filter(_.nonEmpty).foreach { profile =>
|
||||
confMap ++= sessionConfCache.get(profile).asScala
|
||||
}
|
||||
confMap.asJava
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,19 @@
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
kyuubi.ha.namespace kyuubi-ns-b
|
||||
kyuubi.engineEnv.HIVE_DIR /opt/hive_conf_dir
|
||||
@ -50,12 +50,12 @@ class PluginLoaderSuite extends KyuubiFunSuite {
|
||||
advisor.map(_.getConfOverlay("chris", conf.getAll.asJava).asScala).reduce(_ ++ _).asJava
|
||||
assert(emptyConfig.isEmpty)
|
||||
|
||||
conf.set(KyuubiConf.SESSION_CONF_PROFILE, "non.exists")
|
||||
conf.set(KyuubiConf.SESSION_CONF_PROFILE, Seq("non.exists"))
|
||||
val nonExistsConfig =
|
||||
advisor.map(_.getConfOverlay("chris", conf.getAll.asJava).asScala).reduce(_ ++ _).asJava
|
||||
assert(nonExistsConfig.isEmpty)
|
||||
|
||||
conf.set(KyuubiConf.SESSION_CONF_PROFILE, "cluster-a")
|
||||
conf.set(KyuubiConf.SESSION_CONF_PROFILE, Seq("cluster-a"))
|
||||
val clusterAConf =
|
||||
advisor.map(_.getConfOverlay("chris", conf.getAll.asJava).asScala).reduce(_ ++ _).asJava
|
||||
assert(clusterAConf.get("kyuubi.ha.namespace") == "kyuubi-ns-a")
|
||||
@ -67,6 +67,21 @@ class PluginLoaderSuite extends KyuubiFunSuite {
|
||||
assert(clusterAConfFromCache.get("kyuubi.ha.namespace") == "kyuubi-ns-a")
|
||||
assert(clusterAConfFromCache.get("kyuubi.zk.ha.namespace") == null)
|
||||
assert(clusterAConfFromCache.size() == 5)
|
||||
|
||||
conf.set(KyuubiConf.SESSION_CONF_PROFILE, Seq("cluster-a", "cluster-b"))
|
||||
val clusterABConf =
|
||||
advisor.map(_.getConfOverlay("chris", conf.getAll.asJava).asScala).reduce(_ ++ _).asJava
|
||||
assert(clusterABConf.get("kyuubi.ha.namespace") == "kyuubi-ns-b")
|
||||
assert(clusterABConf.get("kyuubi.zk.ha.namespace") == null)
|
||||
assert(clusterABConf.get("kyuubi.engineEnv.HIVE_DIR") == "/opt/hive_conf_dir")
|
||||
assert(clusterABConf.size() == 6)
|
||||
|
||||
val clusterABConfFromCache =
|
||||
advisor.map(_.getConfOverlay("chris", conf.getAll.asJava).asScala).reduce(_ ++ _).asJava
|
||||
assert(clusterABConfFromCache.get("kyuubi.ha.namespace") == "kyuubi-ns-b")
|
||||
assert(clusterABConfFromCache.get("kyuubi.zk.ha.namespace") == null)
|
||||
assert(clusterABConf.get("kyuubi.engineEnv.HIVE_DIR") == "/opt/hive_conf_dir")
|
||||
assert(clusterABConfFromCache.size() == 6)
|
||||
}
|
||||
|
||||
test("SessionConfAdvisor - multi class") {
|
||||
@ -75,7 +90,7 @@ class PluginLoaderSuite extends KyuubiFunSuite {
|
||||
KyuubiConf.SESSION_CONF_ADVISOR,
|
||||
Seq(classOf[FileSessionConfAdvisor].getName, classOf[TestSessionConfAdvisor].getName))
|
||||
val advisor = PluginLoader.loadSessionConfAdvisor(conf)
|
||||
conf.set(KyuubiConf.SESSION_CONF_PROFILE, "cluster-a")
|
||||
conf.set(KyuubiConf.SESSION_CONF_PROFILE, Seq("cluster-a"))
|
||||
val clusterAConf =
|
||||
advisor.map(_.getConfOverlay("chris", conf.getAll.asJava).asScala).reduce(_ ++ _).asJava
|
||||
assert(clusterAConf.get("kyuubi.ha.namespace") == "kyuubi-ns-a")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user