From efe9f5552c2b9cbc7c089816d8166bd30d6d86fa Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 9 Jul 2025 10:14:15 +0800 Subject: [PATCH] [KYUUBI #6876] Fix hadoopConf for autoCreateFileUploadPath ### Why are the changes needed? This change fixes two issues: 1. `KyuubiHadoopUtils.newHadoopConf` should `loadDefaults`, otherwise `core-site.xml`, `hdfs-site.xml` won't take effect. 2. To make it aware of Hadoop conf hot reload, we should use `KyuubiServer.getHadoopConf()`. ### How was this patch tested? Manual test that `core-site.xml` takes affect, previously not. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #7102 from pan3793/6876-followup. Closes #6876 24989d688 [Cheng Pan] [KYUUBI #6876] Fix hadoopConf for autoCreateFileUploadPath Authored-by: Cheng Pan Signed-off-by: Cheng Pan --- .../org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala index 7c862f521..600d96a22 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala @@ -41,7 +41,8 @@ import org.apache.kyuubi.ha.HighAvailabilityConf import org.apache.kyuubi.ha.HighAvailabilityConf.HA_ZK_ENGINE_AUTH_TYPE import org.apache.kyuubi.ha.client.AuthTypes import org.apache.kyuubi.operation.log.OperationLog -import org.apache.kyuubi.util.{JavaUtils, KubernetesUtils, KyuubiHadoopUtils, Validator} +import org.apache.kyuubi.server.KyuubiServer +import org.apache.kyuubi.util.{JavaUtils, KubernetesUtils, Validator} import org.apache.kyuubi.util.command.CommandLineUtils._ class SparkProcessBuilder( @@ -287,7 +288,7 @@ class SparkProcessBuilder( // Create the `uploadPath` using permission 777, otherwise, spark just creates the // `$uploadPath/spark-upload-$uuid` using default permission 511, which might prevent // other users from creating the staging dir under `uploadPath` later. - val hadoopConf = KyuubiHadoopUtils.newHadoopConf(conf, loadDefaults = false) + val hadoopConf = KyuubiServer.getHadoopConf() val path = new Path(uploadPath) var fs: FileSystem = null try {