apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #7055] Support to filter out server only configs with prefixes

Browse Source 
### Why are the changes needed?

To filter out server only configs with prefixes.

For some kyuubi configs, there is no related defined ConfigEntry, and we can not filter out them and have to populate them to engien end.

For example:
```
kyuubi.kubernetes.28.master.address=k8s://master
kyuubi.backend.server.event.kafka.broker=localhost:9092
kyuubi.metadata.store.jdbc.driver=com.mysql.cj.jdbc.Driver
kyuubi.metadata.store.jdbc.datasource.maximumPoolSize=600
kyuubi.metadata.store.jdbc.datasource.minimumIdle=100
kyuubi.metadata.store.jdbc.datasource.idleTimeout=60000
```

This PR supports to exclude them by setting:
```
kyuubi.config.server.only.prefixes=kyuubi.backend.server.event.kafka.,kyuubi.metadata.store.jdbc.datasource.,kyuubi.kubernetes.28.
```

### How was this patch tested?

UT
### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #7055 from turboFei/server_only_configs.

Closes #7055

6c804ff91 [Cheng Pan] Update kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
bd391a664 [Wang, Fei] exclude

Lead-authored-by: Fei Wang <fwang12@ebay.com>
Co-authored-by: Cheng Pan <pan3793@gmail.com>
Co-authored-by: Wang, Fei <fwang12@ebay.com>
Signed-off-by: Wang, Fei <fwang12@ebay.com>
This commit is contained in:
Fei Wang 2025-05-11 22:34:46 -07:00 committed by Wang, Fei
parent 61487acfa0
commit 2ec8b46f02
2 changed files with 33 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
View File

@ -193,6 +193,16 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {
cloned
}
private lazy val serverOnlyPrefixes = get(KyuubiConf.SERVER_ONLY_PREFIXES)
private lazy val serverOnlyPrefixConfigKeys = settings.keys().asScala
// for ConfigEntry, respect the serverOnly flag and exclude it here
.filter(key => getConfigEntry(key) == null)
.filter { key =>
serverOnlyPrefixes.exists { prefix =>
key.startsWith(prefix)
}
}
def getUserDefaults(user: String): KyuubiConf = {
val cloned = KyuubiConf(false)
@ -205,6 +215,7 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {
cloned.set(k, v)
}
serverOnlyConfEntries.foreach(cloned.unset)
serverOnlyPrefixConfigKeys.foreach(cloned.unset)
cloned
}
@ -2905,6 +2916,22 @@ object KyuubiConf {
.stringConf
.createWithDefault("ENGINE")
val SERVER_ONLY_PREFIXES: ConfigEntry[Set[String]] =
buildConf("kyuubi.config.server.only.prefixes")
.internal
.serverOnly
.doc("A comma-separated list of prefixes for server-only configs. It's used to filter out " +
"the server-only configs to prevent passing them to the engine end. Note that, " +
"it only take affects for the configs that is not defined as a Kyuubi ConfigEntry. " +
"For example, you can exclude `kyuubi.kubernetes.28.master.address=k8s://master` by " +
"setting it to `kyuubi.kubernetes.28.`.")
.version("1.11.0")
.stringConf
.toSet()
.createWithDefault(Set(
"kyuubi.backend.server.event.kafka.",
"kyuubi.metadata.store.jdbc.datasource."))
val ENGINE_SPARK_SHOW_PROGRESS: ConfigEntry[Boolean] =
buildConf("kyuubi.session.engine.spark.showProgress")
.doc("When true, show the progress bar in the Spark's engine log.")

6
kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala
View File

@ -230,4 +230,10 @@ class KyuubiConfSuite extends KyuubiFunSuite {
assert(kubernetesConf2.get(KyuubiConf.KUBERNETES_AUTHENTICATE_OAUTH_TOKEN_FILE) ==
Some("/var/run/secrets/kubernetes.io/token.ns2"))
}
test("KYUUBI #7053 - Support to exclude server only configs with prefixes") {
val kyuubiConf = KyuubiConf(false)
kyuubiConf.set("kyuubi.backend.server.event.kafka.broker", "localhost:9092")
assert(kyuubiConf.getUserDefaults("kyuubi").getAll.size == 0)
}
}