diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala index 0b5852fab..2a17d182e 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala @@ -117,11 +117,11 @@ class KyuubiOnKubernetesWithClusterSparkTestsSuite override def beforeAll(): Unit = { super.beforeAll() val fs = FileSystem.get(getHadoopConf) - fs.mkdirs( + FileSystem.mkdirs( + fs, new Path("/spark"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) fs.setPermission(new Path("/"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) - fs.setPermission(new Path("/spark"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) fs.copyFromLocalFile(new Path(driverTemplate.getPath), new Path("/spark/driver.yml")) } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala index 3b84f6946..05a09e029 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala @@ -296,7 +296,8 @@ class SparkProcessBuilder( fs = path.getFileSystem(hadoopConf) if (!fs.exists(path)) { info(s"Try creating $KUBERNETES_FILE_UPLOAD_PATH: $uploadPath") - fs.mkdirs(path, KUBERNETES_UPLOAD_PATH_PERMISSION) + // SPARK-30860: use the class method to avoid the umask causing permission issues + FileSystem.mkdirs(fs, path, KUBERNETES_UPLOAD_PATH_PERMISSION) } } catch { case ioe: IOException => @@ -410,7 +411,8 @@ object SparkProcessBuilder { final val INTERNAL_RESOURCE = "spark-internal" final val KUBERNETES_FILE_UPLOAD_PATH = "spark.kubernetes.file.upload.path" - final val KUBERNETES_UPLOAD_PATH_PERMISSION = new FsPermission(Integer.parseInt("777", 8).toShort) + final val KUBERNETES_UPLOAD_PATH_PERMISSION = + FsPermission.createImmutable(Integer.parseInt("777", 8).toShort) final val YEAR_FMT = DateTimeFormatter.ofPattern("yyyy") final val MONTH_FMT = DateTimeFormatter.ofPattern("MM")