From 2fee652efc02ef7c2dee645bb6ba7f16abb2ffcd Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Sun, 27 Aug 2023 23:16:23 +0800 Subject: [PATCH] [KYUUBI #5201] Allow disabling batch impl v2 on globally ### _Why are the changes needed?_ It makes no sense to enable batch impl v2 when `kyuubi.batch.submitter.enabled` is disabled, to avoid misuse, globally disable batch impl v2 in such case. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request ### _Was this patch authored or co-authored using generative AI tooling?_ No. Closes #5201 from pan3793/batch-v2-switch. Closes #5201 2cc24032f [Cheng Pan] nit f5c8bbf44 [Cheng Pan] nit d280a209b [Cheng Pan] Allow disabling batch impl v2 on globally Authored-by: Cheng Pan Signed-off-by: Cheng Pan --- .../org/apache/kyuubi/config/KyuubiConf.scala | 29 ++++++++++--------- .../server/api/v1/BatchesResource.scala | 2 ++ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 1e8ce8c8a..c1b2052d9 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -1670,24 +1670,12 @@ object KyuubiConf { .booleanConf .createWithDefault(true) - val BATCH_IMPL_VERSION: ConfigEntry[String] = - buildConf("kyuubi.batch.impl.version") - .internal - .serverOnly - .doc("Batch API version, candidates: 1, 2. " + - "Note: Batch API v2 is experimental and under rapid development, this configuration " + - "is added to allow explorers conveniently testing the developing Batch v2 API, not " + - "intended exposing to end users, it may be removed in anytime.") - .version("1.8.0") - .stringConf - .createWithDefault("1") - val BATCH_SUBMITTER_ENABLED: ConfigEntry[Boolean] = buildConf("kyuubi.batch.submitter.enabled") .internal .serverOnly - .doc("When Batch API v2 is enabled, Kyuubi server requires to pick the INITIALIZED " + - "batch job from metastore and submits it to Resource Manager. " + + .doc("Batch API v2 requires batch submitter to pick the INITIALIZED batch job " + + "from metastore and submits it to Resource Manager. " + "Note: Batch API v2 is experimental and under rapid development, this configuration " + "is added to allow explorers conveniently testing the developing Batch v2 API, not " + "intended exposing to end users, it may be removed in anytime.") @@ -1705,6 +1693,19 @@ object KyuubiConf { .intConf .createWithDefault(100) + val BATCH_IMPL_VERSION: ConfigEntry[String] = + buildConf("kyuubi.batch.impl.version") + .internal + .serverOnly + .doc("Batch API version, candidates: 1, 2. Only take effect when " + + s"${BATCH_SUBMITTER_ENABLED.key} is true, otherwise always use v1 implementation. " + + "Note: Batch API v2 is experimental and under rapid development, this configuration " + + "is added to allow explorers conveniently testing the developing Batch v2 API, not " + + "intended exposing to end users, it may be removed in anytime.") + .version("1.8.0") + .stringConf + .createWithDefault("1") + val SERVER_EXEC_POOL_SIZE: ConfigEntry[Int] = buildConf("kyuubi.backend.server.exec.pool.size") .doc("Number of threads in the operation execution thread pool of Kyuubi server") diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala index caa768a66..d7e8b615a 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala @@ -42,6 +42,7 @@ import org.apache.kyuubi.config.KyuubiConf._ import org.apache.kyuubi.config.KyuubiReservedKeys._ import org.apache.kyuubi.engine.{ApplicationInfo, ApplicationManagerInfo, KillResponse, KyuubiApplicationManager} import org.apache.kyuubi.operation.{BatchJobSubmission, FetchOrientation, OperationState} +import org.apache.kyuubi.server.KyuubiServer import org.apache.kyuubi.server.api.ApiRequestContext import org.apache.kyuubi.server.api.v1.BatchesResource._ import org.apache.kyuubi.server.metadata.MetadataManager @@ -59,6 +60,7 @@ private[v1] class BatchesResource extends ApiRequestContext with Logging { fe.getConf.get(BATCH_INTERNAL_REST_CLIENT_CONNECT_TIMEOUT).toInt private def batchV2Enabled(reqConf: Map[String, String]): Boolean = { + KyuubiServer.kyuubiServer.getConf.get(BATCH_SUBMITTER_ENABLED) && reqConf.getOrElse(BATCH_IMPL_VERSION.key, fe.getConf.get(BATCH_IMPL_VERSION)) == "2" }