apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #5201] Allow disabling batch impl v2 on globally

Browse Source 
### _Why are the changes needed?_

It makes no sense to enable batch impl v2 when `kyuubi.batch.submitter.enabled` is disabled, to avoid misuse, globally disable batch impl v2 in such case.

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request

### _Was this patch authored or co-authored using generative AI tooling?_

No.

Closes #5201 from pan3793/batch-v2-switch.

Closes #5201

2cc24032f [Cheng Pan] nit
f5c8bbf44 [Cheng Pan] nit
d280a209b [Cheng Pan] Allow disabling batch impl v2 on globally

Authored-by: Cheng Pan <chengpan@apache.org>
Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
Cheng Pan 2023-08-27 23:16:23 +08:00
parent 56cbd582c4
commit 2fee652efc
No known key found for this signature in database
GPG Key ID: 8001952629BCC75D
2 changed files with 17 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
View File

@ -1670,24 +1670,12 @@ object KyuubiConf {
.booleanConf
.createWithDefault(true)
val BATCH_IMPL_VERSION: ConfigEntry[String] =
buildConf("kyuubi.batch.impl.version")
.internal
.serverOnly
.doc("Batch API version, candidates: 1, 2. " +
"Note: Batch API v2 is experimental and under rapid development, this configuration " +
"is added to allow explorers conveniently testing the developing Batch v2 API, not " +
"intended exposing to end users, it may be removed in anytime.")
.version("1.8.0")
.stringConf
.createWithDefault("1")
val BATCH_SUBMITTER_ENABLED: ConfigEntry[Boolean] =
buildConf("kyuubi.batch.submitter.enabled")
.internal
.serverOnly
.doc("When Batch API v2 is enabled, Kyuubi server requires to pick the INITIALIZED " +
"batch job from metastore and submits it to Resource Manager. " +
.doc("Batch API v2 requires batch submitter to pick the INITIALIZED batch job " +
"from metastore and submits it to Resource Manager. " +
"Note: Batch API v2 is experimental and under rapid development, this configuration " +
"is added to allow explorers conveniently testing the developing Batch v2 API, not " +
"intended exposing to end users, it may be removed in anytime.")
@ -1705,6 +1693,19 @@ object KyuubiConf {
.intConf
.createWithDefault(100)
val BATCH_IMPL_VERSION: ConfigEntry[String] =
buildConf("kyuubi.batch.impl.version")
.internal
.serverOnly
.doc("Batch API version, candidates: 1, 2. Only take effect when " +
s"${BATCH_SUBMITTER_ENABLED.key} is true, otherwise always use v1 implementation. " +
"Note: Batch API v2 is experimental and under rapid development, this configuration " +
"is added to allow explorers conveniently testing the developing Batch v2 API, not " +
"intended exposing to end users, it may be removed in anytime.")
.version("1.8.0")
.stringConf
.createWithDefault("1")
val SERVER_EXEC_POOL_SIZE: ConfigEntry[Int] =
buildConf("kyuubi.backend.server.exec.pool.size")
.doc("Number of threads in the operation execution thread pool of Kyuubi server")

2
kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala
View File

@ -42,6 +42,7 @@ import org.apache.kyuubi.config.KyuubiConf._
import org.apache.kyuubi.config.KyuubiReservedKeys._
import org.apache.kyuubi.engine.{ApplicationInfo, ApplicationManagerInfo, KillResponse, KyuubiApplicationManager}
import org.apache.kyuubi.operation.{BatchJobSubmission, FetchOrientation, OperationState}
import org.apache.kyuubi.server.KyuubiServer
import org.apache.kyuubi.server.api.ApiRequestContext
import org.apache.kyuubi.server.api.v1.BatchesResource._
import org.apache.kyuubi.server.metadata.MetadataManager
@ -59,6 +60,7 @@ private[v1] class BatchesResource extends ApiRequestContext with Logging {
fe.getConf.get(BATCH_INTERNAL_REST_CLIENT_CONNECT_TIMEOUT).toInt
private def batchV2Enabled(reqConf: Map[String, String]): Boolean = {
KyuubiServer.kyuubiServer.getConf.get(BATCH_SUBMITTER_ENABLED) &&
reqConf.getOrElse(BATCH_IMPL_VERSION.key, fe.getConf.get(BATCH_IMPL_VERSION)) == "2"
}