From 97f0bae87d2c84f9587cc9873cf570bf4a401e3e Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 22 Jul 2025 16:41:32 +0800 Subject: [PATCH] [KYUUBI #7148] Fix spark.kubernetes.file.upload.path permission ### Why are the changes needed? The default behavior of HDFS is to set the permission of a file created with `FileSystem.create` or `FileSystem.mkdirs` to `(P & ^umask)`, where `P` is the permission in the API call and umask is a system value set by `fs.permissions.umask-mode` and defaults to `0022`. This means, with default settings, any mkdirs call can have at most `755` permissions. The same issue also got reported in [SPARK-30860](https://issues.apache.org/jira/browse/SPARK-30860) ### How was this patch tested? Manual test. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #7148 from pan3793/fs-mkdirs. Closes #7148 7527060ac [Cheng Pan] fix f64913277 [Cheng Pan] Fix spark.kubernetes.file.upload.path permission Authored-by: Cheng Pan Signed-off-by: Cheng Pan --- .../test/deployment/KyuubiOnKubernetesTestsSuite.scala | 4 ++-- .../apache/kyuubi/engine/spark/SparkProcessBuilder.scala | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala index 0b5852fab..2a17d182e 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala @@ -117,11 +117,11 @@ class KyuubiOnKubernetesWithClusterSparkTestsSuite override def beforeAll(): Unit = { super.beforeAll() val fs = FileSystem.get(getHadoopConf) - fs.mkdirs( + FileSystem.mkdirs( + fs, new Path("/spark"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) fs.setPermission(new Path("/"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) - fs.setPermission(new Path("/spark"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) fs.copyFromLocalFile(new Path(driverTemplate.getPath), new Path("/spark/driver.yml")) } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala index 3b84f6946..05a09e029 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala @@ -296,7 +296,8 @@ class SparkProcessBuilder( fs = path.getFileSystem(hadoopConf) if (!fs.exists(path)) { info(s"Try creating $KUBERNETES_FILE_UPLOAD_PATH: $uploadPath") - fs.mkdirs(path, KUBERNETES_UPLOAD_PATH_PERMISSION) + // SPARK-30860: use the class method to avoid the umask causing permission issues + FileSystem.mkdirs(fs, path, KUBERNETES_UPLOAD_PATH_PERMISSION) } } catch { case ioe: IOException => @@ -410,7 +411,8 @@ object SparkProcessBuilder { final val INTERNAL_RESOURCE = "spark-internal" final val KUBERNETES_FILE_UPLOAD_PATH = "spark.kubernetes.file.upload.path" - final val KUBERNETES_UPLOAD_PATH_PERMISSION = new FsPermission(Integer.parseInt("777", 8).toShort) + final val KUBERNETES_UPLOAD_PATH_PERMISSION = + FsPermission.createImmutable(Integer.parseInt("777", 8).toShort) final val YEAR_FMT = DateTimeFormatter.ofPattern("yyyy") final val MONTH_FMT = DateTimeFormatter.ofPattern("MM")