diff --git a/bin/kyuubi-admin b/bin/kyuubi-admin index 74b5f39a1..a1f176ec1 100755 --- a/bin/kyuubi-admin +++ b/bin/kyuubi-admin @@ -17,7 +17,7 @@ # ## Kyuubi Admin Control Client Entrance -CLASS="org.apache.kyuubi.ctl.AdminControlCli" +CLASS="org.apache.kyuubi.ctl.cli.AdminControlCli" export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)" diff --git a/bin/kyuubi-ctl b/bin/kyuubi-ctl index d989eeb2d..16809c075 100755 --- a/bin/kyuubi-ctl +++ b/bin/kyuubi-ctl @@ -17,7 +17,7 @@ # ## Kyuubi Control Client Entrance -CLASS="org.apache.kyuubi.ctl.ControlCli" +CLASS="org.apache.kyuubi.ctl.cli.ControlCli" export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)" diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/RestClientFactory.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/RestClientFactory.scala index 5821598fa..bbaa5f668 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/RestClientFactory.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/RestClientFactory.scala @@ -24,6 +24,7 @@ import org.apache.kyuubi.KyuubiException import org.apache.kyuubi.client.KyuubiRestClient import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.ctl.CtlConf._ +import org.apache.kyuubi.ctl.opt.CliConfig object RestClientFactory { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/AdminControlCli.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCli.scala similarity index 95% rename from kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/AdminControlCli.scala rename to kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCli.scala index d857febb1..728f2b7fe 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/AdminControlCli.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCli.scala @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.kyuubi.ctl +package org.apache.kyuubi.ctl.cli import org.apache.kyuubi.Logging +import org.apache.kyuubi.ctl.{ControlCliException, KyuubiOEffectSetup} import org.apache.kyuubi.ctl.util.CommandLineUtils class AdminControlCli extends ControlCli { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/AdminControlCliArguments.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala similarity index 62% rename from kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/AdminControlCliArguments.scala rename to kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala index 9bc33808e..4bc1e1317 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/AdminControlCliArguments.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala @@ -15,23 +15,34 @@ * limitations under the License. */ -package org.apache.kyuubi.ctl +package org.apache.kyuubi.ctl.cli import scopt.OParser import org.apache.kyuubi.KyuubiException import org.apache.kyuubi.ctl.cmd.Command +import org.apache.kyuubi.ctl.cmd.delete.AdminDeleteEngineCommand +import org.apache.kyuubi.ctl.cmd.list.AdminListEngineCommand import org.apache.kyuubi.ctl.cmd.refresh.RefreshConfigCommand +import org.apache.kyuubi.ctl.opt.{AdminCommandLine, CliConfig, ControlAction, ControlObject} class AdminControlCliArguments(args: Seq[String], env: Map[String, String] = sys.env) extends ControlCliArguments(args, env) { override def parser(): OParser[Unit, CliConfig] = { val builder = OParser.builder[CliConfig] - CommandLine.getAdminCtlOptionParser(builder) + AdminCommandLine.getAdminCtlOptionParser(builder) } override protected def getCommand(cliConfig: CliConfig): Command[_] = { cliConfig.action match { + case ControlAction.LIST => cliConfig.resource match { + case ControlObject.ENGINE => new AdminListEngineCommand(cliConfig) + case _ => throw new KyuubiException(s"Invalid resource: ${cliConfig.resource}") + } + case ControlAction.DELETE => cliConfig.resource match { + case ControlObject.ENGINE => new AdminDeleteEngineCommand(cliConfig) + case _ => throw new KyuubiException(s"Invalid resource: ${cliConfig.resource}") + } case ControlAction.REFRESH => cliConfig.resource match { case ControlObject.CONFIG => new RefreshConfigCommand(cliConfig) case _ => throw new KyuubiException(s"Invalid resource: ${cliConfig.resource}") @@ -42,6 +53,14 @@ class AdminControlCliArguments(args: Seq[String], env: Map[String, String] = sys override def toString: String = { cliConfig.resource match { + case ControlObject.ENGINE => + s"""Parsed arguments: + | action ${cliConfig.action} + | resource ${cliConfig.resource} + | type ${cliConfig.engineOpts.engineType} + | sharelevel ${cliConfig.engineOpts.engineShareLevel} + | sharesubdomain ${cliConfig.engineOpts.engineSubdomain} + """.stripMargin case ControlObject.CONFIG => s"""Parsed arguments: | action ${cliConfig.action} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ControlCli.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCli.scala similarity index 96% rename from kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ControlCli.scala rename to kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCli.scala index 9bc537b13..8214263c6 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ControlCli.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCli.scala @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.kyuubi.ctl +package org.apache.kyuubi.ctl.cli import org.apache.kyuubi.Logging +import org.apache.kyuubi.ctl.{ControlCliException, KyuubiOEffectSetup} import org.apache.kyuubi.ctl.util.CommandLineUtils /** diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ControlCliArguments.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCliArguments.scala similarity index 96% rename from kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ControlCliArguments.scala rename to kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCliArguments.scala index 0d64ef3eb..5c55689f1 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ControlCliArguments.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCliArguments.scala @@ -15,11 +15,12 @@ * limitations under the License. */ -package org.apache.kyuubi.ctl +package org.apache.kyuubi.ctl.cli import scopt.OParser import org.apache.kyuubi.{KyuubiException, Logging} +import org.apache.kyuubi.ctl.{opt, KyuubiOEffectSetup} import org.apache.kyuubi.ctl.cmd._ import org.apache.kyuubi.ctl.cmd.create.{CreateBatchCommand, CreateServerCommand} import org.apache.kyuubi.ctl.cmd.delete.{DeleteBatchCommand, DeleteEngineCommand, DeleteServerCommand} @@ -27,6 +28,7 @@ import org.apache.kyuubi.ctl.cmd.get.{GetBatchCommand, GetEngineCommand, GetServ import org.apache.kyuubi.ctl.cmd.list.{ListBatchCommand, ListEngineCommand, ListServerCommand} import org.apache.kyuubi.ctl.cmd.log.LogBatchCommand import org.apache.kyuubi.ctl.cmd.submit.SubmitBatchCommand +import org.apache.kyuubi.ctl.opt.{CliConfig, CommandLine, ControlAction, ControlObject} class ControlCliArguments(args: Seq[String], env: Map[String, String] = sys.env) extends ControlCliArgumentsParser with Logging { @@ -48,7 +50,7 @@ class ControlCliArguments(args: Seq[String], env: Map[String, String] = sys.env) private[kyuubi] lazy val effectSetup = new KyuubiOEffectSetup override def parse(args: Seq[String]): Unit = { - OParser.runParser(cliParser, args, CliConfig()) match { + OParser.runParser(cliParser, args, opt.CliConfig()) match { case (result, effects) => OParser.runEffects(effects, effectSetup) result match { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ControlCliArgumentsParser.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCliArgumentsParser.scala similarity index 93% rename from kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ControlCliArgumentsParser.scala rename to kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCliArgumentsParser.scala index 4a6707f39..a49e17aea 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ControlCliArgumentsParser.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCliArgumentsParser.scala @@ -15,10 +15,12 @@ * limitations under the License. */ -package org.apache.kyuubi.ctl +package org.apache.kyuubi.ctl.cli import scopt.OParser +import org.apache.kyuubi.ctl.opt.CliConfig + abstract private[kyuubi] class ControlCliArgumentsParser { /** diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/AdminCtlCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/AdminCtlCommand.scala index 7e5eb5cd3..f527b7de1 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/AdminCtlCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/AdminCtlCommand.scala @@ -17,8 +17,8 @@ package org.apache.kyuubi.ctl.cmd -import org.apache.kyuubi.ctl.AdminControlCli -import org.apache.kyuubi.ctl.CliConfig +import org.apache.kyuubi.ctl.cli.AdminControlCli +import org.apache.kyuubi.ctl.opt.CliConfig abstract class AdminCtlCommand[T](cliConfig: CliConfig) extends Command[T](cliConfig) { override def info(msg: => Any): Unit = AdminControlCli.printMessage(msg) diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/Command.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/Command.scala index 6dd88e5e7..1cf44e8cd 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/Command.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/Command.scala @@ -18,8 +18,8 @@ package org.apache.kyuubi.ctl.cmd import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiException, Logging} import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.ctl.CliConfig -import org.apache.kyuubi.ctl.ControlCli +import org.apache.kyuubi.ctl.cli.ControlCli +import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ha.HighAvailabilityConf._ abstract class Command[T](cliConfig: CliConfig) extends Logging { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateBatchCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateBatchCommand.scala index 3c5c6c55c..18618e37e 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateBatchCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateBatchCommand.scala @@ -22,9 +22,10 @@ import scala.collection.JavaConverters._ import org.apache.kyuubi.client.BatchRestApi import org.apache.kyuubi.client.api.v1.dto.{Batch, BatchRequest} -import org.apache.kyuubi.ctl.{CliConfig, ControlCliException} +import org.apache.kyuubi.ctl.ControlCliException import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient import org.apache.kyuubi.ctl.cmd.Command +import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator} class CreateBatchCommand(cliConfig: CliConfig) extends Command[Batch](cliConfig) { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateServerCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateServerCommand.scala index 3a942e0ab..66f75fc5f 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateServerCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateServerCommand.scala @@ -18,8 +18,8 @@ package org.apache.kyuubi.ctl.cmd.create import scala.collection.mutable.ListBuffer -import org.apache.kyuubi.ctl.{CliConfig, ControlObject} import org.apache.kyuubi.ctl.cmd.Command +import org.apache.kyuubi.ctl.opt.{CliConfig, ControlObject} import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator} import org.apache.kyuubi.ha.HighAvailabilityConf._ import org.apache.kyuubi.ha.client.{DiscoveryClient, DiscoveryPaths, ServiceNodeInfo} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/AdminDeleteEngineCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/AdminDeleteEngineCommand.scala new file mode 100644 index 000000000..989b38edc --- /dev/null +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/AdminDeleteEngineCommand.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.ctl.cmd.delete + +import org.apache.kyuubi.client.AdminRestApi +import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient +import org.apache.kyuubi.ctl.cmd.AdminCtlCommand +import org.apache.kyuubi.ctl.opt.CliConfig +import org.apache.kyuubi.ctl.util.Tabulator + +class AdminDeleteEngineCommand(cliConfig: CliConfig) extends AdminCtlCommand[String](cliConfig) { + + override def validate(): Unit = {} + + def doRun(): String = { + withKyuubiRestClient(normalizedCliConfig, null, conf) { kyuubiRestClient => + val adminRestApi = new AdminRestApi(kyuubiRestClient) + adminRestApi.deleteEngine( + normalizedCliConfig.engineOpts.engineType, + normalizedCliConfig.engineOpts.engineShareLevel, + normalizedCliConfig.engineOpts.engineSubdomain, + normalizedCliConfig.commonOpts.hs2ProxyUser) + } + } + + def render(resp: String): Unit = { + info(Tabulator.format("", Array("Response"), Array(Array(resp)))) + } + +} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteBatchCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteBatchCommand.scala index 548faa008..3988620ad 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteBatchCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteBatchCommand.scala @@ -19,9 +19,10 @@ package org.apache.kyuubi.ctl.cmd.delete import org.apache.kyuubi.client.BatchRestApi import org.apache.kyuubi.client.api.v1.dto.Batch import org.apache.kyuubi.client.util.{BatchUtils, JsonUtils} -import org.apache.kyuubi.ctl.{CliConfig, ControlCliException} +import org.apache.kyuubi.ctl.ControlCliException import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient import org.apache.kyuubi.ctl.cmd.Command +import org.apache.kyuubi.ctl.opt.CliConfig class DeleteBatchCommand(cliConfig: CliConfig) extends Command[Batch](cliConfig) { def validate(): Unit = { @@ -35,7 +36,7 @@ class DeleteBatchCommand(cliConfig: CliConfig) extends Command[Batch](cliConfig) val batchRestApi: BatchRestApi = new BatchRestApi(kyuubiRestClient) val batchId = normalizedCliConfig.batchOpts.batchId - val result = batchRestApi.deleteBatch(batchId, normalizedCliConfig.batchOpts.hs2ProxyUser) + val result = batchRestApi.deleteBatch(batchId, normalizedCliConfig.commonOpts.hs2ProxyUser) info(JsonUtils.toJson(result)) diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteCommand.scala index d42e23507..69479259a 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteCommand.scala @@ -18,8 +18,8 @@ package org.apache.kyuubi.ctl.cmd.delete import scala.collection.mutable.ListBuffer -import org.apache.kyuubi.ctl.CliConfig import org.apache.kyuubi.ctl.cmd.Command +import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator} import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient import org.apache.kyuubi.ha.client.ServiceNodeInfo diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteEngineCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteEngineCommand.scala index d0b0067a1..7be607467 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteEngineCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteEngineCommand.scala @@ -16,7 +16,7 @@ */ package org.apache.kyuubi.ctl.cmd.delete -import org.apache.kyuubi.ctl.CliConfig +import org.apache.kyuubi.ctl.opt.CliConfig class DeleteEngineCommand(cliConfig: CliConfig) extends DeleteCommand(cliConfig) { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteServerCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteServerCommand.scala index fea1014a2..6debba4d5 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteServerCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteServerCommand.scala @@ -16,6 +16,6 @@ */ package org.apache.kyuubi.ctl.cmd.delete -import org.apache.kyuubi.ctl.CliConfig +import org.apache.kyuubi.ctl.opt.CliConfig class DeleteServerCommand(cliConfig: CliConfig) extends DeleteCommand(cliConfig) {} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetBatchCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetBatchCommand.scala index 4822cb3f0..48921b99f 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetBatchCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetBatchCommand.scala @@ -18,9 +18,9 @@ package org.apache.kyuubi.ctl.cmd.get import org.apache.kyuubi.client.BatchRestApi import org.apache.kyuubi.client.api.v1.dto.Batch -import org.apache.kyuubi.ctl.CliConfig import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient import org.apache.kyuubi.ctl.cmd.Command +import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ctl.util.Render class GetBatchCommand(cliConfig: CliConfig) extends Command[Batch](cliConfig) { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetCommand.scala index 8f668b332..d78f0b995 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetCommand.scala @@ -16,8 +16,8 @@ */ package org.apache.kyuubi.ctl.cmd.get -import org.apache.kyuubi.ctl.CliConfig import org.apache.kyuubi.ctl.cmd.Command +import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator} import org.apache.kyuubi.ha.client.ServiceNodeInfo diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetEngineCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetEngineCommand.scala index 17557ceb6..4d9101625 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetEngineCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetEngineCommand.scala @@ -16,7 +16,7 @@ */ package org.apache.kyuubi.ctl.cmd.get -import org.apache.kyuubi.ctl.CliConfig +import org.apache.kyuubi.ctl.opt.CliConfig class GetEngineCommand(cliConfig: CliConfig) extends GetCommand(cliConfig) { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetServerCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetServerCommand.scala index fd0f52bd9..71b868453 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetServerCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetServerCommand.scala @@ -16,6 +16,6 @@ */ package org.apache.kyuubi.ctl.cmd.get -import org.apache.kyuubi.ctl.CliConfig +import org.apache.kyuubi.ctl.opt.CliConfig class GetServerCommand(cliConfig: CliConfig) extends GetCommand(cliConfig) {} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListEngineCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListEngineCommand.scala new file mode 100644 index 000000000..bc0b16e67 --- /dev/null +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListEngineCommand.scala @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.ctl.cmd.list + +import scala.collection.JavaConverters._ + +import org.apache.kyuubi.client.AdminRestApi +import org.apache.kyuubi.client.api.v1.dto.Engine +import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient +import org.apache.kyuubi.ctl.cmd.AdminCtlCommand +import org.apache.kyuubi.ctl.opt.CliConfig +import org.apache.kyuubi.ctl.util.Render + +class AdminListEngineCommand(cliConfig: CliConfig) extends AdminCtlCommand[Seq[Engine]](cliConfig) { + + override def validate(): Unit = {} + + def doRun(): Seq[Engine] = { + withKyuubiRestClient(normalizedCliConfig, null, conf) { kyuubiRestClient => + val adminRestApi = new AdminRestApi(kyuubiRestClient) + adminRestApi.listEngines( + normalizedCliConfig.engineOpts.engineType, + normalizedCliConfig.engineOpts.engineShareLevel, + normalizedCliConfig.engineOpts.engineSubdomain, + normalizedCliConfig.commonOpts.hs2ProxyUser).asScala + } + } + + def render(resp: Seq[Engine]): Unit = { + info(Render.renderEngineNodesInfo(resp)) + } +} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListBatchCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListBatchCommand.scala index f6877cf92..4ce1b49b2 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListBatchCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListBatchCommand.scala @@ -18,9 +18,9 @@ package org.apache.kyuubi.ctl.cmd.list import org.apache.kyuubi.client.BatchRestApi import org.apache.kyuubi.client.api.v1.dto.GetBatchesResponse -import org.apache.kyuubi.ctl.CliConfig import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient import org.apache.kyuubi.ctl.cmd.Command +import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ctl.util.Render class ListBatchCommand(cliConfig: CliConfig) extends Command[GetBatchesResponse](cliConfig) { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListCommand.scala index 4ad62c92b..0cfeb8e4e 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListCommand.scala @@ -16,8 +16,8 @@ */ package org.apache.kyuubi.ctl.cmd.list -import org.apache.kyuubi.ctl.CliConfig import org.apache.kyuubi.ctl.cmd.Command +import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator} import org.apache.kyuubi.ha.client.ServiceNodeInfo diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListEngineCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListEngineCommand.scala index 1cad9ac23..6a78a9e97 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListEngineCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListEngineCommand.scala @@ -16,7 +16,7 @@ */ package org.apache.kyuubi.ctl.cmd.list -import org.apache.kyuubi.ctl.CliConfig +import org.apache.kyuubi.ctl.opt.CliConfig class ListEngineCommand(cliConfig: CliConfig) extends ListCommand(cliConfig) { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListServerCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListServerCommand.scala index cea56e70c..8c3219ece 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListServerCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListServerCommand.scala @@ -16,6 +16,6 @@ */ package org.apache.kyuubi.ctl.cmd.list -import org.apache.kyuubi.ctl.CliConfig +import org.apache.kyuubi.ctl.opt.CliConfig class ListServerCommand(cliConfig: CliConfig) extends ListCommand(cliConfig) {} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/log/LogBatchCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/log/LogBatchCommand.scala index 42d492896..24197d8d6 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/log/LogBatchCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/log/LogBatchCommand.scala @@ -23,10 +23,10 @@ import scala.collection.JavaConverters._ import org.apache.kyuubi.client.BatchRestApi import org.apache.kyuubi.client.api.v1.dto.{Batch, OperationLog} import org.apache.kyuubi.client.util.BatchUtils -import org.apache.kyuubi.ctl.CliConfig import org.apache.kyuubi.ctl.CtlConf._ import org.apache.kyuubi.ctl.RestClientFactory.{withKyuubiInstanceRestClient, withKyuubiRestClient} import org.apache.kyuubi.ctl.cmd.Command +import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ctl.util.Render class LogBatchCommand( diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/refresh/RefreshConfigCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/refresh/RefreshConfigCommand.scala index 18830f7b8..80d673327 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/refresh/RefreshConfigCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/refresh/RefreshConfigCommand.scala @@ -19,9 +19,9 @@ package org.apache.kyuubi.ctl.cmd.refresh import org.apache.kyuubi.KyuubiException import org.apache.kyuubi.client.AdminRestApi -import org.apache.kyuubi.ctl.CliConfig import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient import org.apache.kyuubi.ctl.cmd.AdminCtlCommand +import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ctl.util.{Tabulator, Validator} class RefreshConfigCommand(cliConfig: CliConfig) extends AdminCtlCommand[String](cliConfig) { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/submit/SubmitBatchCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/submit/SubmitBatchCommand.scala index 8a846ce17..863da98d4 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/submit/SubmitBatchCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/submit/SubmitBatchCommand.scala @@ -18,10 +18,11 @@ package org.apache.kyuubi.ctl.cmd.submit import org.apache.kyuubi.client.api.v1.dto.Batch import org.apache.kyuubi.client.util.{BatchUtils, JsonUtils} -import org.apache.kyuubi.ctl.{BatchOpts, CliConfig, ControlCliException, LogOpts} +import org.apache.kyuubi.ctl.ControlCliException import org.apache.kyuubi.ctl.cmd.Command import org.apache.kyuubi.ctl.cmd.create.CreateBatchCommand import org.apache.kyuubi.ctl.cmd.log.LogBatchCommand +import org.apache.kyuubi.ctl.opt.{BatchOpts, CliConfig, LogOpts} import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator} class SubmitBatchCommand(cliConfig: CliConfig) extends Command[Batch](cliConfig) { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala new file mode 100644 index 000000000..524f2954e --- /dev/null +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.ctl.opt + +import scopt.{OParser, OParserBuilder} + +import org.apache.kyuubi.KYUUBI_VERSION + +object AdminCommandLine extends CommonCommandLine { + + def getAdminCtlOptionParser(builder: OParserBuilder[CliConfig]): OParser[Unit, CliConfig] = { + import builder._ + OParser.sequence( + programName("kyuubi-admin"), + head("kyuubi", KYUUBI_VERSION), + common(builder), + list(builder), + delete(builder), + refresh(builder), + checkConfig(f => { + if (f.action == null) { + failure("Must specify action command: [list|delete|refresh].") + } else { + success + } + }), + note(""), + help('h', "help").text("Show help message and exit.")) + } + + private def delete(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { + import builder._ + OParser.sequence( + note(""), + cmd("delete") + .text("\tDelete resources.") + .action((_, c) => c.copy(action = ControlAction.DELETE)) + .children( + engineCmd(builder).text("\tDelete the specified engine node for user."))) + + } + + private def list(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { + import builder._ + OParser.sequence( + note(""), + cmd("list") + .text("\tList information about resources.") + .action((_, c) => c.copy(action = ControlAction.LIST)) + .children( + engineCmd(builder).text("\tList all the engine nodes for a user"))) + + } + + private def refresh(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { + import builder._ + OParser.sequence( + note(""), + cmd("refresh") + .text("\tRefresh the resource.") + .action((_, c) => c.copy(action = ControlAction.REFRESH)) + .children( + refreshConfigCmd(builder).text("\tRefresh the config with specified type."))) + } + + private def engineCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { + import builder._ + cmd("engine").action((_, c) => c.copy(resource = ControlObject.ENGINE)) + .children( + opt[String]("engine-type").abbr("et") + .action((v, c) => c.copy(engineOpts = c.engineOpts.copy(engineType = v))) + .text("The engine type this engine belong to."), + opt[String]("engine-subdomain").abbr("es") + .action((v, c) => c.copy(engineOpts = c.engineOpts.copy(engineSubdomain = v))) + .text("The engine subdomain this engine belong to."), + opt[String]("engine-share-level").abbr("esl") + .action((v, c) => c.copy(engineOpts = c.engineOpts.copy(engineShareLevel = v))) + .text("The engine share level this engine belong to.")) + } + + private def refreshConfigCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { + import builder._ + cmd("config").action((_, c) => c.copy(resource = ControlObject.CONFIG)) + .children( + arg[String]("") + .optional() + .action((v, c) => c.copy(adminConfigOpts = c.adminConfigOpts.copy(configType = v))) + .text("The valid config type can be one of the following: hadoopConf.")) + } +} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CliConfig.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CliConfig.scala similarity index 91% rename from kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CliConfig.scala rename to kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CliConfig.scala index f2f4aab51..12bd51d6d 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CliConfig.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CliConfig.scala @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kyuubi.ctl +package org.apache.kyuubi.ctl.opt -import org.apache.kyuubi.ctl.ControlAction.ControlAction -import org.apache.kyuubi.ctl.ControlObject.ControlObject +import org.apache.kyuubi.ctl.opt.ControlAction.ControlAction +import org.apache.kyuubi.ctl.opt.ControlObject.ControlObject private[ctl] object ControlAction extends Enumeration { type ControlAction = Value @@ -47,7 +47,8 @@ case class CommonOpts( authSchema: String = null, username: String = null, password: String = null, - spnegoHost: String = null) + spnegoHost: String = null, + hs2ProxyUser: String = null) case class ZookeeperOpts( zkQuorum: String = null, @@ -69,7 +70,6 @@ case class BatchOpts( endTime: Long = 0, from: Int = -1, size: Int = 100, - hs2ProxyUser: String = null, waitCompletion: Boolean = true) case class EngineOpts( diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CommandLine.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CommandLine.scala similarity index 78% rename from kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CommandLine.scala rename to kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CommandLine.scala index 56c381848..aeab8bcdf 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CommandLine.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CommandLine.scala @@ -14,14 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kyuubi.ctl + +package org.apache.kyuubi.ctl.opt import scopt.{OParser, OParserBuilder} -import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiException} -import org.apache.kyuubi.ctl.util.DateTimeUtils._ +import org.apache.kyuubi.KYUUBI_VERSION +import org.apache.kyuubi.ctl.util.DateTimeUtils.dateStringToMillis -object CommandLine { +object CommandLine extends CommonCommandLine { def getCtlOptionParser(builder: OParserBuilder[CliConfig]): OParser[Unit, CliConfig] = { import builder._ @@ -47,55 +48,6 @@ object CommandLine { help('h', "help").text("Show help message and exit.")) } - def getAdminCtlOptionParser(builder: OParserBuilder[CliConfig]): OParser[Unit, CliConfig] = { - import builder._ - OParser.sequence( - programName("kyuubi-admin"), - head("kyuubi", KYUUBI_VERSION), - common(builder), - refresh(builder), - checkConfig(f => { - if (f.action == null) { - failure("Must specify action command: [refresh].") - } else { - success - } - }), - note(""), - help('h', "help").text("Show help message and exit.")) - } - - private def common(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { - import builder._ - OParser.sequence( - opt[Unit]('b', "verbose") - .action((_, c) => c.copy(commonOpts = c.commonOpts.copy(verbose = true))) - .text("Print additional debug output."), - opt[String]("hostUrl") - .action((v, c) => c.copy(commonOpts = c.commonOpts.copy(hostUrl = v))) - .text("Host url for rest api."), - opt[String]("authSchema") - .action((v, c) => c.copy(commonOpts = c.commonOpts.copy(authSchema = v))) - .text("Auth schema for rest api, valid values are basic, spnego."), - opt[String]("username") - .action((v, c) => c.copy(commonOpts = c.commonOpts.copy(username = v))) - .text("Username for basic authentication."), - opt[String]("password") - .action((v, c) => c.copy(commonOpts = c.commonOpts.copy(password = v))) - .text("Password for basic authentication."), - opt[String]("spnegoHost") - .action((v, c) => c.copy(commonOpts = c.commonOpts.copy(spnegoHost = v))) - .text("Spnego host for spnego authentication."), - opt[String]("conf") - .action((v, c) => { - v.split("=", 2).toSeq match { - case Seq(k, v) => c.copy(conf = c.conf ++ Map(k -> v)) - case _ => throw new KyuubiException(s"Kyuubi config without '=': $v") - } - }) - .text("Kyuubi config property pair, formatted key=value.")) - } - private def zooKeeper(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { import builder._ OParser.sequence( @@ -203,17 +155,6 @@ object CommandLine { submitBatchCmd(builder).text("\topen batch session and wait for completion."))) } - private def refresh(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { - import builder._ - OParser.sequence( - note(""), - cmd("refresh") - .text("\tRefresh the resource.") - .action((_, c) => c.copy(action = ControlAction.REFRESH)) - .children( - refreshConfigCmd(builder).text("\tRefresh the config with specified type."))) - } - private def serverCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { import builder._ cmd("server").action((_, c) => c.copy(resource = ControlObject.SERVER)) @@ -259,10 +200,7 @@ object CommandLine { arg[String]("") .optional() .action((v, c) => c.copy(batchOpts = c.batchOpts.copy(batchId = v))) - .text("Batch id."), - opt[String]("hs2ProxyUser") - .action((v, c) => c.copy(createOpts = c.createOpts.copy(filename = v))) - .text("The value of hive.server2.proxy.user config.")) + .text("Batch id.")) } private def listBatchCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { @@ -353,13 +291,4 @@ object CommandLine { "when the batch is no longer in PENDING state.")) } - private def refreshConfigCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { - import builder._ - cmd("config").action((_, c) => c.copy(resource = ControlObject.CONFIG)) - .children( - arg[String]("") - .optional() - .action((v, c) => c.copy(adminConfigOpts = c.adminConfigOpts.copy(configType = v))) - .text("The valid config type can be one of the following: hadoopConf.")) - } } diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CommonCommandLine.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CommonCommandLine.scala new file mode 100644 index 000000000..da855bcd0 --- /dev/null +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CommonCommandLine.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.ctl.opt + +import scopt.{OParser, OParserBuilder} + +import org.apache.kyuubi.KyuubiException + +trait CommonCommandLine { + + def common(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { + import builder._ + OParser.sequence( + opt[Unit]('b', "verbose") + .action((_, c) => c.copy(commonOpts = c.commonOpts.copy(verbose = true))) + .text("Print additional debug output."), + opt[String]("hostUrl") + .action((v, c) => c.copy(commonOpts = c.commonOpts.copy(hostUrl = v))) + .text("Host url for rest api."), + opt[String]("authSchema") + .action((v, c) => c.copy(commonOpts = c.commonOpts.copy(authSchema = v))) + .text("Auth schema for rest api, valid values are basic, spnego."), + opt[String]("username") + .action((v, c) => c.copy(commonOpts = c.commonOpts.copy(username = v))) + .text("Username for basic authentication."), + opt[String]("password") + .action((v, c) => c.copy(commonOpts = c.commonOpts.copy(password = v))) + .text("Password for basic authentication."), + opt[String]("spnegoHost") + .action((v, c) => c.copy(commonOpts = c.commonOpts.copy(spnegoHost = v))) + .text("Spnego host for spnego authentication."), + opt[String]("hs2ProxyUser") + .action((v, c) => c.copy(commonOpts = c.commonOpts.copy(hs2ProxyUser = v))) + .text("The value of hive.server2.proxy.user config."), + opt[String]("conf") + .action((v, c) => { + v.split("=", 2).toSeq match { + case Seq(k, v) => c.copy(conf = c.conf ++ Map(k -> v)) + case _ => throw new KyuubiException(s"Kyuubi config without '=': $v") + } + }) + .text("Kyuubi config property pair, formatted key=value.")) + } +} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/CtlUtils.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/CtlUtils.scala index 305447993..194f691e5 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/CtlUtils.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/CtlUtils.scala @@ -25,7 +25,7 @@ import org.yaml.snakeyaml.Yaml import org.apache.kyuubi.KyuubiException import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.{ENGINE_SHARE_LEVEL, ENGINE_SHARE_LEVEL_SUBDOMAIN, ENGINE_TYPE} -import org.apache.kyuubi.ctl.{CliConfig, ControlObject} +import org.apache.kyuubi.ctl.opt.{CliConfig, ControlObject} import org.apache.kyuubi.ha.client.{DiscoveryClient, DiscoveryPaths, ServiceNodeInfo} import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala index 190c2b239..b3977b21f 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala @@ -19,7 +19,7 @@ package org.apache.kyuubi.ctl.util import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer -import org.apache.kyuubi.client.api.v1.dto.{Batch, GetBatchesResponse} +import org.apache.kyuubi.client.api.v1.dto.{Batch, Engine, GetBatchesResponse} import org.apache.kyuubi.ctl.util.DateTimeUtils._ import org.apache.kyuubi.ha.client.ServiceNodeInfo @@ -33,6 +33,15 @@ private[ctl] object Render { Tabulator.format(title, header, rows) } + def renderEngineNodesInfo(engineNodesInfo: Seq[Engine]): String = { + val title = s"Engine Node List (total ${engineNodesInfo.size})" + val header = Array("Namespace", "Instance", "Version") + val rows = engineNodesInfo.map { engine => + Array(engine.getNamespace, engine.getInstance, engine.getVersion) + }.toArray + Tabulator.format(title, header, rows) + } + def renderBatchListInfo(batchListInfo: GetBatchesResponse): String = { val title = s"Batch List (from ${batchListInfo.getFrom} total ${batchListInfo.getTotal})" val rows = batchListInfo.getBatches.asScala.sortBy(_.getCreateTime).map(buildBatchRow).toArray diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Validator.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Validator.scala index 05df08c1e..896f1f1c2 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Validator.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Validator.scala @@ -22,7 +22,7 @@ import java.nio.file.{Files, Paths} import org.apache.commons.lang3.StringUtils import org.apache.kyuubi.KyuubiException -import org.apache.kyuubi.ctl.CliConfig +import org.apache.kyuubi.ctl.opt.CliConfig private[ctl] object Validator { diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala index 48f2d6854..03b606d34 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala @@ -18,6 +18,8 @@ package org.apache.kyuubi.ctl import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite} +import org.apache.kyuubi.ctl.cli.AdminControlCliArguments +import org.apache.kyuubi.ctl.opt.{ControlAction, ControlObject} class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit { @@ -71,11 +73,34 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi testPrematureExitForAdminControlCli(args, "Invalid config type:otherConf") } + test("test list engine") { + Seq("list", "delete").foreach { op => + val args = Array( + op, + "engine", + "-et", + "spark-sql", + "-esl", + "user", + "--engine-subdomain", + "default", + "--hs2ProxyUser", + "b_kyuubi") + val opArgs = new AdminControlCliArguments(args) + assert(opArgs.cliConfig.action.toString === op.toUpperCase) + assert(opArgs.cliConfig.resource.toString === "ENGINE") + assert(opArgs.cliConfig.engineOpts.engineType === "spark-sql") + assert(opArgs.cliConfig.engineOpts.engineShareLevel === "user") + assert(opArgs.cliConfig.engineOpts.engineSubdomain === "default") + assert(opArgs.cliConfig.commonOpts.hs2ProxyUser === "b_kyuubi") + } + } + test("test --help") { // scalastyle:off val helpString = s"""kyuubi $KYUUBI_VERSION - |Usage: kyuubi-admin [refresh] [options] + |Usage: kyuubi-admin [list|delete|refresh] [options] | | -b, --verbose Print additional debug output. | --hostUrl Host url for rest api. @@ -83,8 +108,31 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi | --username Username for basic authentication. | --password Password for basic authentication. | --spnegoHost Spnego host for spnego authentication. + | --hs2ProxyUser The value of hive.server2.proxy.user config. | --conf Kyuubi config property pair, formatted key=value. | + |Command: list [engine] + | List information about resources. + |Command: list engine [options] + | List all the engine nodes for a user + | -et, --engine-type + | The engine type this engine belong to. + | -es, --engine-subdomain + | The engine subdomain this engine belong to. + | -esl, --engine-share-level + | The engine share level this engine belong to. + | + |Command: delete [engine] + | Delete resources. + |Command: delete engine [options] + | Delete the specified engine node for user. + | -et, --engine-type + | The engine type this engine belong to. + | -es, --engine-subdomain + | The engine subdomain this engine belong to. + | -esl, --engine-share-level + | The engine share level this engine belong to. + | |Command: refresh [config] ... | Refresh the resource. |Command: refresh config [] diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/BatchCliArgumentsSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/BatchCliArgumentsSuite.scala index 47abee364..7563d985a 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/BatchCliArgumentsSuite.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/BatchCliArgumentsSuite.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.ctl import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.ctl.cli.ControlCliArguments import org.apache.kyuubi.ctl.util.DateTimeUtils._ class BatchCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit { @@ -118,6 +119,18 @@ class BatchCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit { } } + test("delete batch with hs2ProxyUser") { + val args = Array( + "delete", + "batch", + "f7fd702c-e54e-11ec-8fea-0242ac120002", + "--hs2ProxyUser", + "b_user") + val opArgs = new ControlCliArguments(args) + assert(opArgs.cliConfig.batchOpts.batchId == "f7fd702c-e54e-11ec-8fea-0242ac120002") + assert(opArgs.cliConfig.commonOpts.hs2ProxyUser == "b_user") + } + test("test list batch option") { val args = Array( "list", diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliArgumentsSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliArgumentsSuite.scala index 037f26aa1..215de65f5 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliArgumentsSuite.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliArgumentsSuite.scala @@ -19,6 +19,8 @@ package org.apache.kyuubi.ctl import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite} import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient +import org.apache.kyuubi.ctl.cli.ControlCliArguments +import org.apache.kyuubi.ctl.opt.ControlAction import org.apache.kyuubi.ha.HighAvailabilityConf.HA_NAMESPACE class ControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit { @@ -363,6 +365,7 @@ class ControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit { | --username Username for basic authentication. | --password Password for basic authentication. | --spnegoHost Spnego host for spnego authentication. + | --hs2ProxyUser The value of hive.server2.proxy.user config. | --conf Kyuubi config property pair, formatted key=value. | -zk, --zk-quorum | $zkHelpString @@ -398,10 +401,9 @@ class ControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit { | |Command: delete [batch|server|engine] ... |${"\t"}Delete resources. - |Command: delete batch [options] [] + |Command: delete batch [] |${"\t"}Close batch session. | Batch id. - | --hs2ProxyUser The value of hive.server2.proxy.user config. |Command: delete server |${"\t"}Delete the specified service node for a domain |Command: delete engine [options] diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliSuite.scala index 32187daf3..d27f3ec2a 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliSuite.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliSuite.scala @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicInteger import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite} import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.ctl.cli.{ControlCli, ControlCliArguments} import org.apache.kyuubi.ctl.util.{CtlUtils, Render} import org.apache.kyuubi.ha.HighAvailabilityConf.{HA_ADDRESSES, HA_NAMESPACE} import org.apache.kyuubi.ha.client.{DiscoveryClientProvider, ServiceNodeInfo} diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/TestPrematureExit.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/TestPrematureExit.scala index 43e1454f3..0e4cc1302 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/TestPrematureExit.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/TestPrematureExit.scala @@ -22,6 +22,7 @@ import java.io.{OutputStream, PrintStream} import scala.collection.mutable.ArrayBuffer import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.ctl.cli.{AdminControlCli, AdminControlCliArguments, ControlCli, ControlCliArguments} import org.apache.kyuubi.ctl.util.CommandLineUtils trait TestPrematureExit { diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java index 46853bb51..7d7c341ab 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java @@ -17,6 +17,12 @@ package org.apache.kyuubi.client; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.kyuubi.client.api.v1.dto.Engine; + public class AdminRestApi { private KyuubiRestClient client; @@ -33,6 +39,29 @@ public class AdminRestApi { return this.getClient().post(path, null, client.getAuthHeader()); } + public String deleteEngine( + String engineType, String shareLevel, String subdomain, String hs2ProxyUser) { + Map params = new HashMap<>(); + params.put("type", engineType); + params.put("sharelevel", shareLevel); + params.put("subdomain", subdomain); + params.put("hive.server2.proxy.user", hs2ProxyUser); + return this.getClient().delete(API_BASE_PATH + "/engine", params, client.getAuthHeader()); + } + + public List listEngines( + String engineType, String shareLevel, String subdomain, String hs2ProxyUser) { + Map params = new HashMap<>(); + params.put("type", engineType); + params.put("sharelevel", shareLevel); + params.put("subdomain", subdomain); + params.put("hive.server2.proxy.user", hs2ProxyUser); + Engine[] result = + this.getClient() + .get(API_BASE_PATH + "/engine", params, Engine[].class, client.getAuthHeader()); + return Arrays.asList(result); + } + private IRestClient getClient() { return this.client.getHttpClient(); } diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Engine.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Engine.java index 0f4d70adc..5566414fa 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Engine.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Engine.java @@ -29,6 +29,7 @@ public class Engine { private String sharelevel; private String subdomain; private String instance; + private String namespace; public Engine() {} @@ -38,13 +39,15 @@ public class Engine { String engineType, String sharelevel, String subdomain, - String instance) { + String instance, + String namespace) { this.version = version; this.user = user; this.engineType = engineType; this.sharelevel = sharelevel; this.subdomain = subdomain; this.instance = instance; + this.namespace = namespace; } public String getVersion() { @@ -95,6 +98,14 @@ public class Engine { this.instance = instance; } + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala index 674b475a2..ad7ea2114 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala @@ -135,7 +135,8 @@ private[v1] class AdminResource extends ApiRequestContext with Logging { engine.getEngineType, engine.getSharelevel, node.namespace.split("/").last, - node.instance)) + node.instance, + node.namespace)) } private def getEngine( @@ -161,6 +162,7 @@ private[v1] class AdminResource extends ApiRequestContext with Logging { normalizedEngineType, engineShareLevel, engineSubdomain, + null, null) } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala index b52c6f67e..10c636924 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala @@ -17,15 +17,21 @@ package org.apache.kyuubi.server.rest.client -import org.apache.kyuubi.RestClientTestHelper +import java.util.UUID + +import org.apache.kyuubi.{KYUUBI_VERSION, RestClientTestHelper} +import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.ctl.{CtlConf, TestPrematureExit} +import org.apache.kyuubi.engine.EngineRef +import org.apache.kyuubi.ha.HighAvailabilityConf +import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient +import org.apache.kyuubi.ha.client.DiscoveryPaths class AdminCtlSuite extends RestClientTestHelper with TestPrematureExit { override def beforeAll(): Unit = { super.beforeAll() System.setProperty(CtlConf.CTL_REST_CLIENT_BASE_URL.key, baseUri.toString) System.setProperty(CtlConf.CTL_REST_CLIENT_SPNEGO_HOST.key, "localhost") - System.setProperty(CtlConf.CTL_REST_CLIENT_AUTH_SCHEMA.key, "spnego") } override def afterAll(): Unit = { @@ -36,9 +42,61 @@ class AdminCtlSuite extends RestClientTestHelper with TestPrematureExit { } test("refresh config - hadoop conf") { - val args = Array("refresh", "config", "hadoopConf") + val args = Array("refresh", "config", "hadoopConf", "--authSchema", "spnego") testPrematureExitForAdminControlCli( args, s"Refresh the hadoop conf for ${fe.connectionUrl} successfully.") } + + test("engine list/delete operation") { + val id = UUID.randomUUID().toString + conf.set(HighAvailabilityConf.HA_NAMESPACE, "kyuubi_test") + conf.set(KyuubiConf.ENGINE_IDLE_TIMEOUT, 180000L) + conf.set(KyuubiConf.AUTHENTICATION_METHOD, Seq("LDAP", "CUSTOM")) + val user = ldapUser + val engine = new EngineRef(conf.clone, user, id, null) + + val engineSpace = DiscoveryPaths.makePath( + s"kyuubi_test_${KYUUBI_VERSION}_USER_SPARK_SQL", + user, + Array("default")) + + withDiscoveryClient(conf) { client => + engine.getOrCreate(client) + + } + + var args = Array( + "list", + "engine", + "--username", + ldapUser, + "--password", + ldapUserPasswd) + testPrematureExitForAdminControlCli( + args, + "Engine Node List (total 1)") + + args = Array( + "delete", + "engine", + "--username", + ldapUser, + "--password", + ldapUserPasswd) + testPrematureExitForAdminControlCli( + args, + s"Engine ${engineSpace} is deleted successfully.") + + args = Array( + "list", + "engine", + "--username", + ldapUser, + "--password", + ldapUserPasswd) + testPrematureExitForAdminControlCli( + args, + "Engine Node List (total 0)") + } } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala index 9ba0b89a4..b6ec539d3 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala @@ -17,8 +17,17 @@ package org.apache.kyuubi.server.rest.client -import org.apache.kyuubi.RestClientTestHelper +import java.util.UUID + +import scala.collection.JavaConverters.asScalaBufferConverter + +import org.apache.kyuubi.{KYUUBI_VERSION, RestClientTestHelper} import org.apache.kyuubi.client.{AdminRestApi, KyuubiRestClient} +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.engine.EngineRef +import org.apache.kyuubi.ha.HighAvailabilityConf +import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient +import org.apache.kyuubi.ha.client.DiscoveryPaths class AdminRestApiSuite extends RestClientTestHelper { test("refresh kyuubi server hadoop conf") { @@ -31,4 +40,47 @@ class AdminRestApiSuite extends RestClientTestHelper { val result = adminRestApi.refreshHadoopConf() assert(result === s"Refresh the hadoop conf for ${fe.connectionUrl} successfully.") } + + test("engine list/delete operation") { + val id = UUID.randomUUID().toString + conf.set(HighAvailabilityConf.HA_NAMESPACE, "kyuubi_test") + conf.set(KyuubiConf.ENGINE_IDLE_TIMEOUT, 180000L) + conf.set(KyuubiConf.AUTHENTICATION_METHOD, Seq("LDAP", "CUSTOM")) + val user = ldapUser + val engine = new EngineRef(conf.clone, user, id, null) + + val engineSpace = DiscoveryPaths.makePath( + s"kyuubi_test_${KYUUBI_VERSION}_USER_SPARK_SQL", + user, + Array("default")) + + withDiscoveryClient(conf) { client => + engine.getOrCreate(client) + + } + + val basicKyuubiRestClient: KyuubiRestClient = + KyuubiRestClient.builder(baseUri.toString) + .authHeaderMethod(KyuubiRestClient.AuthHeaderMethod.BASIC) + .username(ldapUser) + .password(ldapUserPasswd) + .socketTimeout(30000) + .build() + + val adminRestApi = new AdminRestApi(basicKyuubiRestClient) + var engines = adminRestApi.listEngines("spark_sql", "user", "default", "").asScala + assert(engines.size == 1) + assert(engines(0).getUser == user) + assert(engines(0).getVersion == KYUUBI_VERSION) + assert(engines(0).getEngineType == "SPARK_SQL") + assert(engines(0).getSharelevel == "USER") + assert(engines(0).getSubdomain == "default") + assert(engines(0).getNamespace == engineSpace) + + val result = adminRestApi.deleteEngine("spark_sql", "user", "default", "") + assert(result == s"Engine ${engineSpace} is deleted successfully.") + + engines = adminRestApi.listEngines("spark_sql", "user", "default", "").asScala + assert(engines.size == 0) + } }