From cebbd230b080db32a8ae995e0cc8826775024ee4 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Thu, 20 Oct 2022 13:29:21 +0800 Subject: [PATCH] [KYUUBI #3670] [SPARK][K8S] Fix Spark master match rule on tagging Spark application ### _Why are the changes needed?_ Fix kyuubi application manager tag spark kubernetes application with yarn tag. The reason for this is the need to determine whether resourceManager start with K8S instead of being equal to K8S. Remove duplicate code from spark batch process builder. ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #3670 from zwangsheng/bugfix/kyuubi_application_manager_tag_application. Closes #3670 d4e8aa30 [zwangsheng] fix 7b5419eb [zwangsheng] back a976f994 [zwangsheng] fix Authored-by: zwangsheng <2213335496@qq.com> Signed-off-by: Cheng Pan --- .../kyuubi/engine/KyuubiApplicationManager.scala | 2 +- .../engine/KyuubiApplicationManagerSuite.scala | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala index 7f2501ec2..b772afc04 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala @@ -160,7 +160,7 @@ object KyuubiApplicationManager { conf: KyuubiConf): Unit = { (applicationType.toUpperCase, resourceManager.map(_.toUpperCase())) match { case ("SPARK", Some("YARN")) => setupSparkYarnTag(applicationTag, conf) - case ("SPARK", Some("K8S")) => setupSparkK8sTag(applicationTag, conf) + case ("SPARK", Some(rm)) if rm.startsWith("K8S") => setupSparkK8sTag(applicationTag, conf) case ("SPARK", _) => // if the master is not identified ahead, add all tags setupSparkYarnTag(applicationTag, conf) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala index 93ef02cdc..b01f82a24 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala @@ -19,6 +19,7 @@ package org.apache.kyuubi.engine import org.apache.kyuubi.{KyuubiException, KyuubiFunSuite} import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.engine.KubernetesApplicationOperation.LABEL_KYUUBI_UNIQUE_KEY class KyuubiApplicationManagerSuite extends KyuubiFunSuite { test("application access path") { @@ -70,4 +71,19 @@ class KyuubiApplicationManagerSuite extends KyuubiFunSuite { localDirLimitConf) } } + + test("Test kyuubi application Manager tag spark on kubernetes application") { + val conf: KyuubiConf = KyuubiConf() + val tag = "kyuubi-test-tag" + KyuubiApplicationManager.tagApplication( + tag, + "SPARK", + Some("k8s://https://kyuubi-test:8443"), + conf) + + val kubernetesTag = conf.getOption("spark.kubernetes.driver.label." + LABEL_KYUUBI_UNIQUE_KEY) + val yarnTag = conf.getOption("spark.yarn.tags") + assert(kubernetesTag.nonEmpty && tag.equals(kubernetesTag.get)) + assert(yarnTag.isEmpty) + } }