apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #3679] Admin command line supports delete/list engine operation

Browse Source 
### _Why are the changes needed?_

Close #3679

### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #3680 from lightning-L/admin-ctl.

Closes #3679

9c748f073 [Tianlin Liao] [KYUUBI #3679] Admin command line supports delete/list engine operation

Authored-by: Tianlin Liao <tiliao@ebay.com>
Signed-off-by: Fei Wang <fwang12@ebay.com>
This commit is contained in:
Tianlin Liao 2022-10-24 11:30:26 +08:00 committed by Fei Wang
parent caf2f2f3a6
commit 2d937f7589
46 changed files with 565 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/kyuubi-admin
View File

@ -17,7 +17,7 @@
#
## Kyuubi Admin Control Client Entrance
CLASS="org.apache.kyuubi.ctl.AdminControlCli"
CLASS="org.apache.kyuubi.ctl.cli.AdminControlCli"
export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)"

2
bin/kyuubi-ctl
View File

@ -17,7 +17,7 @@
#
## Kyuubi Control Client Entrance
CLASS="org.apache.kyuubi.ctl.ControlCli"
CLASS="org.apache.kyuubi.ctl.cli.ControlCli"
export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)"

1
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/RestClientFactory.scala
View File

@ -24,6 +24,7 @@ import org.apache.kyuubi.KyuubiException
import org.apache.kyuubi.client.KyuubiRestClient
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ctl.CtlConf._
import org.apache.kyuubi.ctl.opt.CliConfig
object RestClientFactory {

3
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/AdminControlCli.scala → kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCli.scala
View File

@ -15,9 +15,10 @@
* limitations under the License.
*/
package org.apache.kyuubi.ctl
package org.apache.kyuubi.ctl.cli
import org.apache.kyuubi.Logging
import org.apache.kyuubi.ctl.{ControlCliException, KyuubiOEffectSetup}
import org.apache.kyuubi.ctl.util.CommandLineUtils
class AdminControlCli extends ControlCli {

23
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/AdminControlCliArguments.scala → kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala
View File

@ -15,23 +15,34 @@
* limitations under the License.
*/
package org.apache.kyuubi.ctl
package org.apache.kyuubi.ctl.cli
import scopt.OParser
import org.apache.kyuubi.KyuubiException
import org.apache.kyuubi.ctl.cmd.Command
import org.apache.kyuubi.ctl.cmd.delete.AdminDeleteEngineCommand
import org.apache.kyuubi.ctl.cmd.list.AdminListEngineCommand
import org.apache.kyuubi.ctl.cmd.refresh.RefreshConfigCommand
import org.apache.kyuubi.ctl.opt.{AdminCommandLine, CliConfig, ControlAction, ControlObject}
class AdminControlCliArguments(args: Seq[String], env: Map[String, String] = sys.env)
extends ControlCliArguments(args, env) {
override def parser(): OParser[Unit, CliConfig] = {
val builder = OParser.builder[CliConfig]
CommandLine.getAdminCtlOptionParser(builder)
AdminCommandLine.getAdminCtlOptionParser(builder)
}
override protected def getCommand(cliConfig: CliConfig): Command[_] = {
cliConfig.action match {
case ControlAction.LIST => cliConfig.resource match {
case ControlObject.ENGINE => new AdminListEngineCommand(cliConfig)
case _ => throw new KyuubiException(s"Invalid resource: ${cliConfig.resource}")
}
case ControlAction.DELETE => cliConfig.resource match {
case ControlObject.ENGINE => new AdminDeleteEngineCommand(cliConfig)
case _ => throw new KyuubiException(s"Invalid resource: ${cliConfig.resource}")
}
case ControlAction.REFRESH => cliConfig.resource match {
case ControlObject.CONFIG => new RefreshConfigCommand(cliConfig)
case _ => throw new KyuubiException(s"Invalid resource: ${cliConfig.resource}")
@ -42,6 +53,14 @@ class AdminControlCliArguments(args: Seq[String], env: Map[String, String] = sys
override def toString: String = {
cliConfig.resource match {
case ControlObject.ENGINE =>
s"""Parsed arguments:
| action ${cliConfig.action}
| resource ${cliConfig.resource}
| type ${cliConfig.engineOpts.engineType}
| sharelevel ${cliConfig.engineOpts.engineShareLevel}
| sharesubdomain ${cliConfig.engineOpts.engineSubdomain}
""".stripMargin
case ControlObject.CONFIG =>
s"""Parsed arguments:
| action ${cliConfig.action}

3
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ControlCli.scala → kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCli.scala
View File

@ -15,9 +15,10 @@
* limitations under the License.
*/
package org.apache.kyuubi.ctl
package org.apache.kyuubi.ctl.cli
import org.apache.kyuubi.Logging
import org.apache.kyuubi.ctl.{ControlCliException, KyuubiOEffectSetup}
import org.apache.kyuubi.ctl.util.CommandLineUtils
/**

6
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ControlCliArguments.scala → kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCliArguments.scala
View File

@ -15,11 +15,12 @@
* limitations under the License.
*/
package org.apache.kyuubi.ctl
package org.apache.kyuubi.ctl.cli
import scopt.OParser
import org.apache.kyuubi.{KyuubiException, Logging}
import org.apache.kyuubi.ctl.{opt, KyuubiOEffectSetup}
import org.apache.kyuubi.ctl.cmd._
import org.apache.kyuubi.ctl.cmd.create.{CreateBatchCommand, CreateServerCommand}
import org.apache.kyuubi.ctl.cmd.delete.{DeleteBatchCommand, DeleteEngineCommand, DeleteServerCommand}
@ -27,6 +28,7 @@ import org.apache.kyuubi.ctl.cmd.get.{GetBatchCommand, GetEngineCommand, GetServ
import org.apache.kyuubi.ctl.cmd.list.{ListBatchCommand, ListEngineCommand, ListServerCommand}
import org.apache.kyuubi.ctl.cmd.log.LogBatchCommand
import org.apache.kyuubi.ctl.cmd.submit.SubmitBatchCommand
import org.apache.kyuubi.ctl.opt.{CliConfig, CommandLine, ControlAction, ControlObject}
class ControlCliArguments(args: Seq[String], env: Map[String, String] = sys.env)
extends ControlCliArgumentsParser with Logging {
@ -48,7 +50,7 @@ class ControlCliArguments(args: Seq[String], env: Map[String, String] = sys.env)
private[kyuubi] lazy val effectSetup = new KyuubiOEffectSetup
override def parse(args: Seq[String]): Unit = {
OParser.runParser(cliParser, args, CliConfig()) match {
OParser.runParser(cliParser, args, opt.CliConfig()) match {
case (result, effects) =>
OParser.runEffects(effects, effectSetup)
result match {

4
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ControlCliArgumentsParser.scala → kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCliArgumentsParser.scala
View File

@ -15,10 +15,12 @@
* limitations under the License.
*/
package org.apache.kyuubi.ctl
package org.apache.kyuubi.ctl.cli
import scopt.OParser
import org.apache.kyuubi.ctl.opt.CliConfig
abstract private[kyuubi] class ControlCliArgumentsParser {
/**

4
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/AdminCtlCommand.scala
View File

@ -17,8 +17,8 @@
package org.apache.kyuubi.ctl.cmd
import org.apache.kyuubi.ctl.AdminControlCli
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.cli.AdminControlCli
import org.apache.kyuubi.ctl.opt.CliConfig
abstract class AdminCtlCommand[T](cliConfig: CliConfig) extends Command[T](cliConfig) {
override def info(msg: => Any): Unit = AdminControlCli.printMessage(msg)

4
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/Command.scala
View File

@ -18,8 +18,8 @@ package org.apache.kyuubi.ctl.cmd
import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiException, Logging}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.ControlCli
import org.apache.kyuubi.ctl.cli.ControlCli
import org.apache.kyuubi.ctl.opt.CliConfig
import org.apache.kyuubi.ha.HighAvailabilityConf._
abstract class Command[T](cliConfig: CliConfig) extends Logging {

3
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateBatchCommand.scala
View File

@ -22,9 +22,10 @@ import scala.collection.JavaConverters._
import org.apache.kyuubi.client.BatchRestApi
import org.apache.kyuubi.client.api.v1.dto.{Batch, BatchRequest}
import org.apache.kyuubi.ctl.{CliConfig, ControlCliException}
import org.apache.kyuubi.ctl.ControlCliException
import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient
import org.apache.kyuubi.ctl.cmd.Command
import org.apache.kyuubi.ctl.opt.CliConfig
import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator}
class CreateBatchCommand(cliConfig: CliConfig) extends Command[Batch](cliConfig) {

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateServerCommand.scala
View File

@ -18,8 +18,8 @@ package org.apache.kyuubi.ctl.cmd.create
import scala.collection.mutable.ListBuffer
import org.apache.kyuubi.ctl.{CliConfig, ControlObject}
import org.apache.kyuubi.ctl.cmd.Command
import org.apache.kyuubi.ctl.opt.{CliConfig, ControlObject}
import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator}
import org.apache.kyuubi.ha.HighAvailabilityConf._
import org.apache.kyuubi.ha.client.{DiscoveryClient, DiscoveryPaths, ServiceNodeInfo}

45
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/AdminDeleteEngineCommand.scala Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.ctl.cmd.delete
import org.apache.kyuubi.client.AdminRestApi
import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient
import org.apache.kyuubi.ctl.cmd.AdminCtlCommand
import org.apache.kyuubi.ctl.opt.CliConfig
import org.apache.kyuubi.ctl.util.Tabulator
class AdminDeleteEngineCommand(cliConfig: CliConfig) extends AdminCtlCommand[String](cliConfig) {
override def validate(): Unit = {}
def doRun(): String = {
withKyuubiRestClient(normalizedCliConfig, null, conf) { kyuubiRestClient =>
val adminRestApi = new AdminRestApi(kyuubiRestClient)
adminRestApi.deleteEngine(
normalizedCliConfig.engineOpts.engineType,
normalizedCliConfig.engineOpts.engineShareLevel,
normalizedCliConfig.engineOpts.engineSubdomain,
normalizedCliConfig.commonOpts.hs2ProxyUser)
}
}
def render(resp: String): Unit = {
info(Tabulator.format("", Array("Response"), Array(Array(resp))))
}
}

5
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteBatchCommand.scala
View File

@ -19,9 +19,10 @@ package org.apache.kyuubi.ctl.cmd.delete
import org.apache.kyuubi.client.BatchRestApi
import org.apache.kyuubi.client.api.v1.dto.Batch
import org.apache.kyuubi.client.util.{BatchUtils, JsonUtils}
import org.apache.kyuubi.ctl.{CliConfig, ControlCliException}
import org.apache.kyuubi.ctl.ControlCliException
import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient
import org.apache.kyuubi.ctl.cmd.Command
import org.apache.kyuubi.ctl.opt.CliConfig
class DeleteBatchCommand(cliConfig: CliConfig) extends Command[Batch](cliConfig) {
def validate(): Unit = {
@ -35,7 +36,7 @@ class DeleteBatchCommand(cliConfig: CliConfig) extends Command[Batch](cliConfig)
val batchRestApi: BatchRestApi = new BatchRestApi(kyuubiRestClient)
val batchId = normalizedCliConfig.batchOpts.batchId
val result = batchRestApi.deleteBatch(batchId, normalizedCliConfig.batchOpts.hs2ProxyUser)
val result = batchRestApi.deleteBatch(batchId, normalizedCliConfig.commonOpts.hs2ProxyUser)
info(JsonUtils.toJson(result))

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteCommand.scala
View File

@ -18,8 +18,8 @@ package org.apache.kyuubi.ctl.cmd.delete
import scala.collection.mutable.ListBuffer
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.cmd.Command
import org.apache.kyuubi.ctl.opt.CliConfig
import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator}
import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient
import org.apache.kyuubi.ha.client.ServiceNodeInfo

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteEngineCommand.scala
View File

@ -16,7 +16,7 @@
*/
package org.apache.kyuubi.ctl.cmd.delete
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.opt.CliConfig
class DeleteEngineCommand(cliConfig: CliConfig) extends DeleteCommand(cliConfig) {

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteServerCommand.scala
View File

@ -16,6 +16,6 @@
*/
package org.apache.kyuubi.ctl.cmd.delete
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.opt.CliConfig
class DeleteServerCommand(cliConfig: CliConfig) extends DeleteCommand(cliConfig) {}

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetBatchCommand.scala
View File

@ -18,9 +18,9 @@ package org.apache.kyuubi.ctl.cmd.get
import org.apache.kyuubi.client.BatchRestApi
import org.apache.kyuubi.client.api.v1.dto.Batch
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient
import org.apache.kyuubi.ctl.cmd.Command
import org.apache.kyuubi.ctl.opt.CliConfig
import org.apache.kyuubi.ctl.util.Render
class GetBatchCommand(cliConfig: CliConfig) extends Command[Batch](cliConfig) {

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetCommand.scala
View File

@ -16,8 +16,8 @@
*/
package org.apache.kyuubi.ctl.cmd.get
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.cmd.Command
import org.apache.kyuubi.ctl.opt.CliConfig
import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator}
import org.apache.kyuubi.ha.client.ServiceNodeInfo

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetEngineCommand.scala
View File

@ -16,7 +16,7 @@
*/
package org.apache.kyuubi.ctl.cmd.get
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.opt.CliConfig
class GetEngineCommand(cliConfig: CliConfig) extends GetCommand(cliConfig) {

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetServerCommand.scala
View File

@ -16,6 +16,6 @@
*/
package org.apache.kyuubi.ctl.cmd.get
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.opt.CliConfig
class GetServerCommand(cliConfig: CliConfig) extends GetCommand(cliConfig) {}

47
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListEngineCommand.scala Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.ctl.cmd.list
import scala.collection.JavaConverters._
import org.apache.kyuubi.client.AdminRestApi
import org.apache.kyuubi.client.api.v1.dto.Engine
import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient
import org.apache.kyuubi.ctl.cmd.AdminCtlCommand
import org.apache.kyuubi.ctl.opt.CliConfig
import org.apache.kyuubi.ctl.util.Render
class AdminListEngineCommand(cliConfig: CliConfig) extends AdminCtlCommand[Seq[Engine]](cliConfig) {
override def validate(): Unit = {}
def doRun(): Seq[Engine] = {
withKyuubiRestClient(normalizedCliConfig, null, conf) { kyuubiRestClient =>
val adminRestApi = new AdminRestApi(kyuubiRestClient)
adminRestApi.listEngines(
normalizedCliConfig.engineOpts.engineType,
normalizedCliConfig.engineOpts.engineShareLevel,
normalizedCliConfig.engineOpts.engineSubdomain,
normalizedCliConfig.commonOpts.hs2ProxyUser).asScala
}
}
def render(resp: Seq[Engine]): Unit = {
info(Render.renderEngineNodesInfo(resp))
}
}

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListBatchCommand.scala
View File

@ -18,9 +18,9 @@ package org.apache.kyuubi.ctl.cmd.list
import org.apache.kyuubi.client.BatchRestApi
import org.apache.kyuubi.client.api.v1.dto.GetBatchesResponse
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient
import org.apache.kyuubi.ctl.cmd.Command
import org.apache.kyuubi.ctl.opt.CliConfig
import org.apache.kyuubi.ctl.util.Render
class ListBatchCommand(cliConfig: CliConfig) extends Command[GetBatchesResponse](cliConfig) {

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListCommand.scala
View File

@ -16,8 +16,8 @@
*/
package org.apache.kyuubi.ctl.cmd.list
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.cmd.Command
import org.apache.kyuubi.ctl.opt.CliConfig
import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator}
import org.apache.kyuubi.ha.client.ServiceNodeInfo

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListEngineCommand.scala
View File

@ -16,7 +16,7 @@
*/
package org.apache.kyuubi.ctl.cmd.list
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.opt.CliConfig
class ListEngineCommand(cliConfig: CliConfig) extends ListCommand(cliConfig) {

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListServerCommand.scala
View File

@ -16,6 +16,6 @@
*/
package org.apache.kyuubi.ctl.cmd.list
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.opt.CliConfig
class ListServerCommand(cliConfig: CliConfig) extends ListCommand(cliConfig) {}

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/log/LogBatchCommand.scala
View File

@ -23,10 +23,10 @@ import scala.collection.JavaConverters._
import org.apache.kyuubi.client.BatchRestApi
import org.apache.kyuubi.client.api.v1.dto.{Batch, OperationLog}
import org.apache.kyuubi.client.util.BatchUtils
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.CtlConf._
import org.apache.kyuubi.ctl.RestClientFactory.{withKyuubiInstanceRestClient, withKyuubiRestClient}
import org.apache.kyuubi.ctl.cmd.Command
import org.apache.kyuubi.ctl.opt.CliConfig
import org.apache.kyuubi.ctl.util.Render
class LogBatchCommand(

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/refresh/RefreshConfigCommand.scala
View File

@ -19,9 +19,9 @@ package org.apache.kyuubi.ctl.cmd.refresh
import org.apache.kyuubi.KyuubiException
import org.apache.kyuubi.client.AdminRestApi
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient
import org.apache.kyuubi.ctl.cmd.AdminCtlCommand
import org.apache.kyuubi.ctl.opt.CliConfig
import org.apache.kyuubi.ctl.util.{Tabulator, Validator}
class RefreshConfigCommand(cliConfig: CliConfig) extends AdminCtlCommand[String](cliConfig) {

3
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/submit/SubmitBatchCommand.scala
View File

@ -18,10 +18,11 @@ package org.apache.kyuubi.ctl.cmd.submit
import org.apache.kyuubi.client.api.v1.dto.Batch
import org.apache.kyuubi.client.util.{BatchUtils, JsonUtils}
import org.apache.kyuubi.ctl.{BatchOpts, CliConfig, ControlCliException, LogOpts}
import org.apache.kyuubi.ctl.ControlCliException
import org.apache.kyuubi.ctl.cmd.Command
import org.apache.kyuubi.ctl.cmd.create.CreateBatchCommand
import org.apache.kyuubi.ctl.cmd.log.LogBatchCommand
import org.apache.kyuubi.ctl.opt.{BatchOpts, CliConfig, LogOpts}
import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator}
class SubmitBatchCommand(cliConfig: CliConfig) extends Command[Batch](cliConfig) {

105
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala Normal file
View File

@ -0,0 +1,105 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.ctl.opt
import scopt.{OParser, OParserBuilder}
import org.apache.kyuubi.KYUUBI_VERSION
object AdminCommandLine extends CommonCommandLine {
def getAdminCtlOptionParser(builder: OParserBuilder[CliConfig]): OParser[Unit, CliConfig] = {
import builder._
OParser.sequence(
programName("kyuubi-admin"),
head("kyuubi", KYUUBI_VERSION),
common(builder),
list(builder),
delete(builder),
refresh(builder),
checkConfig(f => {
if (f.action == null) {
failure("Must specify action command: [list|delete|refresh].")
} else {
success
}
}),
note(""),
help('h', "help").text("Show help message and exit."))
}
private def delete(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
import builder._
OParser.sequence(
note(""),
cmd("delete")
.text("\tDelete resources.")
.action((_, c) => c.copy(action = ControlAction.DELETE))
.children(
engineCmd(builder).text("\tDelete the specified engine node for user.")))
}
private def list(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
import builder._
OParser.sequence(
note(""),
cmd("list")
.text("\tList information about resources.")
.action((_, c) => c.copy(action = ControlAction.LIST))
.children(
engineCmd(builder).text("\tList all the engine nodes for a user")))
}
private def refresh(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
import builder._
OParser.sequence(
note(""),
cmd("refresh")
.text("\tRefresh the resource.")
.action((_, c) => c.copy(action = ControlAction.REFRESH))
.children(
refreshConfigCmd(builder).text("\tRefresh the config with specified type.")))
}
private def engineCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
import builder._
cmd("engine").action((_, c) => c.copy(resource = ControlObject.ENGINE))
.children(
opt[String]("engine-type").abbr("et")
.action((v, c) => c.copy(engineOpts = c.engineOpts.copy(engineType = v)))
.text("The engine type this engine belong to."),
opt[String]("engine-subdomain").abbr("es")
.action((v, c) => c.copy(engineOpts = c.engineOpts.copy(engineSubdomain = v)))
.text("The engine subdomain this engine belong to."),
opt[String]("engine-share-level").abbr("esl")
.action((v, c) => c.copy(engineOpts = c.engineOpts.copy(engineShareLevel = v)))
.text("The engine share level this engine belong to."))
}
private def refreshConfigCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
import builder._
cmd("config").action((_, c) => c.copy(resource = ControlObject.CONFIG))
.children(
arg[String]("<configType>")
.optional()
.action((v, c) => c.copy(adminConfigOpts = c.adminConfigOpts.copy(configType = v)))
.text("The valid config type can be one of the following: hadoopConf."))
}
}

10
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CliConfig.scala → kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CliConfig.scala
View File

@ -14,10 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.ctl
package org.apache.kyuubi.ctl.opt
import org.apache.kyuubi.ctl.ControlAction.ControlAction
import org.apache.kyuubi.ctl.ControlObject.ControlObject
import org.apache.kyuubi.ctl.opt.ControlAction.ControlAction
import org.apache.kyuubi.ctl.opt.ControlObject.ControlObject
private[ctl] object ControlAction extends Enumeration {
type ControlAction = Value
@ -47,7 +47,8 @@ case class CommonOpts(
authSchema: String = null,
username: String = null,
password: String = null,
spnegoHost: String = null)
spnegoHost: String = null,
hs2ProxyUser: String = null)
case class ZookeeperOpts(
zkQuorum: String = null,
@ -69,7 +70,6 @@ case class BatchOpts(
endTime: Long = 0,
from: Int = -1,
size: Int = 100,
hs2ProxyUser: String = null,
waitCompletion: Boolean = true)
case class EngineOpts(

83
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CommandLine.scala → kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CommandLine.scala
View File

@ -14,14 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.ctl
package org.apache.kyuubi.ctl.opt
import scopt.{OParser, OParserBuilder}
import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiException}
import org.apache.kyuubi.ctl.util.DateTimeUtils._
import org.apache.kyuubi.KYUUBI_VERSION
import org.apache.kyuubi.ctl.util.DateTimeUtils.dateStringToMillis
object CommandLine {
object CommandLine extends CommonCommandLine {
def getCtlOptionParser(builder: OParserBuilder[CliConfig]): OParser[Unit, CliConfig] = {
import builder._
@ -47,55 +48,6 @@ object CommandLine {
help('h', "help").text("Show help message and exit."))
}
def getAdminCtlOptionParser(builder: OParserBuilder[CliConfig]): OParser[Unit, CliConfig] = {
import builder._
OParser.sequence(
programName("kyuubi-admin"),
head("kyuubi", KYUUBI_VERSION),
common(builder),
refresh(builder),
checkConfig(f => {
if (f.action == null) {
failure("Must specify action command: [refresh].")
} else {
success
}
}),
note(""),
help('h', "help").text("Show help message and exit."))
}
private def common(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
import builder._
OParser.sequence(
opt[Unit]('b', "verbose")
.action((_, c) => c.copy(commonOpts = c.commonOpts.copy(verbose = true)))
.text("Print additional debug output."),
opt[String]("hostUrl")
.action((v, c) => c.copy(commonOpts = c.commonOpts.copy(hostUrl = v)))
.text("Host url for rest api."),
opt[String]("authSchema")
.action((v, c) => c.copy(commonOpts = c.commonOpts.copy(authSchema = v)))
.text("Auth schema for rest api, valid values are basic, spnego."),
opt[String]("username")
.action((v, c) => c.copy(commonOpts = c.commonOpts.copy(username = v)))
.text("Username for basic authentication."),
opt[String]("password")
.action((v, c) => c.copy(commonOpts = c.commonOpts.copy(password = v)))
.text("Password for basic authentication."),
opt[String]("spnegoHost")
.action((v, c) => c.copy(commonOpts = c.commonOpts.copy(spnegoHost = v)))
.text("Spnego host for spnego authentication."),
opt[String]("conf")
.action((v, c) => {
v.split("=", 2).toSeq match {
case Seq(k, v) => c.copy(conf = c.conf ++ Map(k -> v))
case _ => throw new KyuubiException(s"Kyuubi config without '=': $v")
}
})
.text("Kyuubi config property pair, formatted key=value."))
}
private def zooKeeper(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
import builder._
OParser.sequence(
@ -203,17 +155,6 @@ object CommandLine {
submitBatchCmd(builder).text("\topen batch session and wait for completion.")))
}
private def refresh(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
import builder._
OParser.sequence(
note(""),
cmd("refresh")
.text("\tRefresh the resource.")
.action((_, c) => c.copy(action = ControlAction.REFRESH))
.children(
refreshConfigCmd(builder).text("\tRefresh the config with specified type.")))
}
private def serverCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
import builder._
cmd("server").action((_, c) => c.copy(resource = ControlObject.SERVER))
@ -259,10 +200,7 @@ object CommandLine {
arg[String]("<batchId>")
.optional()
.action((v, c) => c.copy(batchOpts = c.batchOpts.copy(batchId = v)))
.text("Batch id."),
opt[String]("hs2ProxyUser")
.action((v, c) => c.copy(createOpts = c.createOpts.copy(filename = v)))
.text("The value of hive.server2.proxy.user config."))
.text("Batch id."))
}
private def listBatchCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
@ -353,13 +291,4 @@ object CommandLine {
"when the batch is no longer in PENDING state."))
}
private def refreshConfigCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
import builder._
cmd("config").action((_, c) => c.copy(resource = ControlObject.CONFIG))
.children(
arg[String]("<configType>")
.optional()
.action((v, c) => c.copy(adminConfigOpts = c.adminConfigOpts.copy(configType = v)))
.text("The valid config type can be one of the following: hadoopConf."))
}
}

59
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CommonCommandLine.scala Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.ctl.opt
import scopt.{OParser, OParserBuilder}
import org.apache.kyuubi.KyuubiException
trait CommonCommandLine {
def common(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
import builder._
OParser.sequence(
opt[Unit]('b', "verbose")
.action((_, c) => c.copy(commonOpts = c.commonOpts.copy(verbose = true)))
.text("Print additional debug output."),
opt[String]("hostUrl")
.action((v, c) => c.copy(commonOpts = c.commonOpts.copy(hostUrl = v)))
.text("Host url for rest api."),
opt[String]("authSchema")
.action((v, c) => c.copy(commonOpts = c.commonOpts.copy(authSchema = v)))
.text("Auth schema for rest api, valid values are basic, spnego."),
opt[String]("username")
.action((v, c) => c.copy(commonOpts = c.commonOpts.copy(username = v)))
.text("Username for basic authentication."),
opt[String]("password")
.action((v, c) => c.copy(commonOpts = c.commonOpts.copy(password = v)))
.text("Password for basic authentication."),
opt[String]("spnegoHost")
.action((v, c) => c.copy(commonOpts = c.commonOpts.copy(spnegoHost = v)))
.text("Spnego host for spnego authentication."),
opt[String]("hs2ProxyUser")
.action((v, c) => c.copy(commonOpts = c.commonOpts.copy(hs2ProxyUser = v)))
.text("The value of hive.server2.proxy.user config."),
opt[String]("conf")
.action((v, c) => {
v.split("=", 2).toSeq match {
case Seq(k, v) => c.copy(conf = c.conf ++ Map(k -> v))
case _ => throw new KyuubiException(s"Kyuubi config without '=': $v")
}
})
.text("Kyuubi config property pair, formatted key=value."))
}
}

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/CtlUtils.scala
View File

@ -25,7 +25,7 @@ import org.yaml.snakeyaml.Yaml
import org.apache.kyuubi.KyuubiException
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.{ENGINE_SHARE_LEVEL, ENGINE_SHARE_LEVEL_SUBDOMAIN, ENGINE_TYPE}
import org.apache.kyuubi.ctl.{CliConfig, ControlObject}
import org.apache.kyuubi.ctl.opt.{CliConfig, ControlObject}
import org.apache.kyuubi.ha.client.{DiscoveryClient, DiscoveryPaths, ServiceNodeInfo}
import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient

11
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala
View File

@ -19,7 +19,7 @@ package org.apache.kyuubi.ctl.util
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
import org.apache.kyuubi.client.api.v1.dto.{Batch, GetBatchesResponse}
import org.apache.kyuubi.client.api.v1.dto.{Batch, Engine, GetBatchesResponse}
import org.apache.kyuubi.ctl.util.DateTimeUtils._
import org.apache.kyuubi.ha.client.ServiceNodeInfo
@ -33,6 +33,15 @@ private[ctl] object Render {
Tabulator.format(title, header, rows)
}
def renderEngineNodesInfo(engineNodesInfo: Seq[Engine]): String = {
val title = s"Engine Node List (total ${engineNodesInfo.size})"
val header = Array("Namespace", "Instance", "Version")
val rows = engineNodesInfo.map { engine =>
Array(engine.getNamespace, engine.getInstance, engine.getVersion)
}.toArray
Tabulator.format(title, header, rows)
}
def renderBatchListInfo(batchListInfo: GetBatchesResponse): String = {
val title = s"Batch List (from ${batchListInfo.getFrom} total ${batchListInfo.getTotal})"
val rows = batchListInfo.getBatches.asScala.sortBy(_.getCreateTime).map(buildBatchRow).toArray

2
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Validator.scala
View File

@ -22,7 +22,7 @@ import java.nio.file.{Files, Paths}
import org.apache.commons.lang3.StringUtils
import org.apache.kyuubi.KyuubiException
import org.apache.kyuubi.ctl.CliConfig
import org.apache.kyuubi.ctl.opt.CliConfig
private[ctl] object Validator {

50
kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala
View File

@ -18,6 +18,8 @@
package org.apache.kyuubi.ctl
import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite}
import org.apache.kyuubi.ctl.cli.AdminControlCliArguments
import org.apache.kyuubi.ctl.opt.{ControlAction, ControlObject}
class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit {
@ -71,11 +73,34 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi
testPrematureExitForAdminControlCli(args, "Invalid config type:otherConf")
}
test("test list engine") {
Seq("list", "delete").foreach { op =>
val args = Array(
op,
"engine",
"-et",
"spark-sql",
"-esl",
"user",
"--engine-subdomain",
"default",
"--hs2ProxyUser",
"b_kyuubi")
val opArgs = new AdminControlCliArguments(args)
assert(opArgs.cliConfig.action.toString === op.toUpperCase)
assert(opArgs.cliConfig.resource.toString === "ENGINE")
assert(opArgs.cliConfig.engineOpts.engineType === "spark-sql")
assert(opArgs.cliConfig.engineOpts.engineShareLevel === "user")
assert(opArgs.cliConfig.engineOpts.engineSubdomain === "default")
assert(opArgs.cliConfig.commonOpts.hs2ProxyUser === "b_kyuubi")
}
}
test("test --help") {
// scalastyle:off
val helpString =
s"""kyuubi $KYUUBI_VERSION
|Usage: kyuubi-admin [refresh] [options]
|Usage: kyuubi-admin [list|delete|refresh] [options]
|
| -b, --verbose Print additional debug output.
| --hostUrl <value> Host url for rest api.
@ -83,8 +108,31 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi
| --username <value> Username for basic authentication.
| --password <value> Password for basic authentication.
| --spnegoHost <value> Spnego host for spnego authentication.
| --hs2ProxyUser <value> The value of hive.server2.proxy.user config.
| --conf <value> Kyuubi config property pair, formatted key=value.
|
|Command: list [engine]
| List information about resources.
|Command: list engine [options]
| List all the engine nodes for a user
| -et, --engine-type <value>
| The engine type this engine belong to.
| -es, --engine-subdomain <value>
| The engine subdomain this engine belong to.
| -esl, --engine-share-level <value>
| The engine share level this engine belong to.
|
|Command: delete [engine]
| Delete resources.
|Command: delete engine [options]
| Delete the specified engine node for user.
| -et, --engine-type <value>
| The engine type this engine belong to.
| -es, --engine-subdomain <value>
| The engine subdomain this engine belong to.
| -esl, --engine-share-level <value>
| The engine share level this engine belong to.
|
|Command: refresh [config] <args>...
| Refresh the resource.
|Command: refresh config [<configType>]

13
kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/BatchCliArgumentsSuite.scala
View File

@ -17,6 +17,7 @@
package org.apache.kyuubi.ctl
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.ctl.cli.ControlCliArguments
import org.apache.kyuubi.ctl.util.DateTimeUtils._
class BatchCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit {
@ -118,6 +119,18 @@ class BatchCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit {
}
}
test("delete batch with hs2ProxyUser") {
val args = Array(
"delete",
"batch",
"f7fd702c-e54e-11ec-8fea-0242ac120002",
"--hs2ProxyUser",
"b_user")
val opArgs = new ControlCliArguments(args)
assert(opArgs.cliConfig.batchOpts.batchId == "f7fd702c-e54e-11ec-8fea-0242ac120002")
assert(opArgs.cliConfig.commonOpts.hs2ProxyUser == "b_user")
}
test("test list batch option") {
val args = Array(
"list",

6
kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliArgumentsSuite.scala
View File

@ -19,6 +19,8 @@ package org.apache.kyuubi.ctl
import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite}
import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient
import org.apache.kyuubi.ctl.cli.ControlCliArguments
import org.apache.kyuubi.ctl.opt.ControlAction
import org.apache.kyuubi.ha.HighAvailabilityConf.HA_NAMESPACE
class ControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit {
@ -363,6 +365,7 @@ class ControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit {
| --username <value> Username for basic authentication.
| --password <value> Password for basic authentication.
| --spnegoHost <value> Spnego host for spnego authentication.
| --hs2ProxyUser <value> The value of hive.server2.proxy.user config.
| --conf <value> Kyuubi config property pair, formatted key=value.
| -zk, --zk-quorum <value>
| $zkHelpString
@ -398,10 +401,9 @@ class ControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit {
|
|Command: delete [batch|server|engine] <args>...
|${"\t"}Delete resources.
|Command: delete batch [options] [<batchId>]
|Command: delete batch [<batchId>]
|${"\t"}Close batch session.
| <batchId> Batch id.
| --hs2ProxyUser <value> The value of hive.server2.proxy.user config.
|Command: delete server
|${"\t"}Delete the specified service node for a domain
|Command: delete engine [options]

1
kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliSuite.scala
View File

@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicInteger
import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ctl.cli.{ControlCli, ControlCliArguments}
import org.apache.kyuubi.ctl.util.{CtlUtils, Render}
import org.apache.kyuubi.ha.HighAvailabilityConf.{HA_ADDRESSES, HA_NAMESPACE}
import org.apache.kyuubi.ha.client.{DiscoveryClientProvider, ServiceNodeInfo}

1
kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/TestPrematureExit.scala
View File

@ -22,6 +22,7 @@ import java.io.{OutputStream, PrintStream}
import scala.collection.mutable.ArrayBuffer
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.ctl.cli.{AdminControlCli, AdminControlCliArguments, ControlCli, ControlCliArguments}
import org.apache.kyuubi.ctl.util.CommandLineUtils
trait TestPrematureExit {

29
kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java
View File

@ -17,6 +17,12 @@
package org.apache.kyuubi.client;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.kyuubi.client.api.v1.dto.Engine;
public class AdminRestApi {
private KyuubiRestClient client;
@ -33,6 +39,29 @@ public class AdminRestApi {
return this.getClient().post(path, null, client.getAuthHeader());
}
public String deleteEngine(
String engineType, String shareLevel, String subdomain, String hs2ProxyUser) {
Map<String, Object> params = new HashMap<>();
params.put("type", engineType);
params.put("sharelevel", shareLevel);
params.put("subdomain", subdomain);
params.put("hive.server2.proxy.user", hs2ProxyUser);
return this.getClient().delete(API_BASE_PATH + "/engine", params, client.getAuthHeader());
}
public List<Engine> listEngines(
String engineType, String shareLevel, String subdomain, String hs2ProxyUser) {
Map<String, Object> params = new HashMap<>();
params.put("type", engineType);
params.put("sharelevel", shareLevel);
params.put("subdomain", subdomain);
params.put("hive.server2.proxy.user", hs2ProxyUser);
Engine[] result =
this.getClient()
.get(API_BASE_PATH + "/engine", params, Engine[].class, client.getAuthHeader());
return Arrays.asList(result);
}
private IRestClient getClient() {
return this.client.getHttpClient();
}

13
kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Engine.java
View File

@ -29,6 +29,7 @@ public class Engine {
private String sharelevel;
private String subdomain;
private String instance;
private String namespace;
public Engine() {}
@ -38,13 +39,15 @@ public class Engine {
String engineType,
String sharelevel,
String subdomain,
String instance) {
String instance,
String namespace) {
this.version = version;
this.user = user;
this.engineType = engineType;
this.sharelevel = sharelevel;
this.subdomain = subdomain;
this.instance = instance;
this.namespace = namespace;
}
public String getVersion() {
@ -95,6 +98,14 @@ public class Engine {
this.instance = instance;
}
public String getNamespace() {
return namespace;
}
public void setNamespace(String namespace) {
this.namespace = namespace;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;

4
kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala
View File

@ -135,7 +135,8 @@ private[v1] class AdminResource extends ApiRequestContext with Logging {
engine.getEngineType,
engine.getSharelevel,
node.namespace.split("/").last,
node.instance))
node.instance,
node.namespace))
}
private def getEngine(
@ -161,6 +162,7 @@ private[v1] class AdminResource extends ApiRequestContext with Logging {
normalizedEngineType,
engineShareLevel,
engineSubdomain,
null,
null)
}

64
kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala
View File

@ -17,15 +17,21 @@
package org.apache.kyuubi.server.rest.client
import org.apache.kyuubi.RestClientTestHelper
import java.util.UUID
import org.apache.kyuubi.{KYUUBI_VERSION, RestClientTestHelper}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ctl.{CtlConf, TestPrematureExit}
import org.apache.kyuubi.engine.EngineRef
import org.apache.kyuubi.ha.HighAvailabilityConf
import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient
import org.apache.kyuubi.ha.client.DiscoveryPaths
class AdminCtlSuite extends RestClientTestHelper with TestPrematureExit {
override def beforeAll(): Unit = {
super.beforeAll()
System.setProperty(CtlConf.CTL_REST_CLIENT_BASE_URL.key, baseUri.toString)
System.setProperty(CtlConf.CTL_REST_CLIENT_SPNEGO_HOST.key, "localhost")
System.setProperty(CtlConf.CTL_REST_CLIENT_AUTH_SCHEMA.key, "spnego")
}
override def afterAll(): Unit = {
@ -36,9 +42,61 @@ class AdminCtlSuite extends RestClientTestHelper with TestPrematureExit {
}
test("refresh config - hadoop conf") {
val args = Array("refresh", "config", "hadoopConf")
val args = Array("refresh", "config", "hadoopConf", "--authSchema", "spnego")
testPrematureExitForAdminControlCli(
args,
s"Refresh the hadoop conf for ${fe.connectionUrl} successfully.")
}
test("engine list/delete operation") {
val id = UUID.randomUUID().toString
conf.set(HighAvailabilityConf.HA_NAMESPACE, "kyuubi_test")
conf.set(KyuubiConf.ENGINE_IDLE_TIMEOUT, 180000L)
conf.set(KyuubiConf.AUTHENTICATION_METHOD, Seq("LDAP", "CUSTOM"))
val user = ldapUser
val engine = new EngineRef(conf.clone, user, id, null)
val engineSpace = DiscoveryPaths.makePath(
s"kyuubi_test_${KYUUBI_VERSION}_USER_SPARK_SQL",
user,
Array("default"))
withDiscoveryClient(conf) { client =>
engine.getOrCreate(client)
}
var args = Array(
"list",
"engine",
"--username",
ldapUser,
"--password",
ldapUserPasswd)
testPrematureExitForAdminControlCli(
args,
"Engine Node List (total 1)")
args = Array(
"delete",
"engine",
"--username",
ldapUser,
"--password",
ldapUserPasswd)
testPrematureExitForAdminControlCli(
args,
s"Engine ${engineSpace} is deleted successfully.")
args = Array(
"list",
"engine",
"--username",
ldapUser,
"--password",
ldapUserPasswd)
testPrematureExitForAdminControlCli(
args,
"Engine Node List (total 0)")
}
}

54
kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala
View File

@ -17,8 +17,17 @@
package org.apache.kyuubi.server.rest.client
import org.apache.kyuubi.RestClientTestHelper
import java.util.UUID
import scala.collection.JavaConverters.asScalaBufferConverter
import org.apache.kyuubi.{KYUUBI_VERSION, RestClientTestHelper}
import org.apache.kyuubi.client.{AdminRestApi, KyuubiRestClient}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.engine.EngineRef
import org.apache.kyuubi.ha.HighAvailabilityConf
import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient
import org.apache.kyuubi.ha.client.DiscoveryPaths
class AdminRestApiSuite extends RestClientTestHelper {
test("refresh kyuubi server hadoop conf") {
@ -31,4 +40,47 @@ class AdminRestApiSuite extends RestClientTestHelper {
val result = adminRestApi.refreshHadoopConf()
assert(result === s"Refresh the hadoop conf for ${fe.connectionUrl} successfully.")
}
test("engine list/delete operation") {
val id = UUID.randomUUID().toString
conf.set(HighAvailabilityConf.HA_NAMESPACE, "kyuubi_test")
conf.set(KyuubiConf.ENGINE_IDLE_TIMEOUT, 180000L)
conf.set(KyuubiConf.AUTHENTICATION_METHOD, Seq("LDAP", "CUSTOM"))
val user = ldapUser
val engine = new EngineRef(conf.clone, user, id, null)
val engineSpace = DiscoveryPaths.makePath(
s"kyuubi_test_${KYUUBI_VERSION}_USER_SPARK_SQL",
user,
Array("default"))
withDiscoveryClient(conf) { client =>
engine.getOrCreate(client)
}
val basicKyuubiRestClient: KyuubiRestClient =
KyuubiRestClient.builder(baseUri.toString)
.authHeaderMethod(KyuubiRestClient.AuthHeaderMethod.BASIC)
.username(ldapUser)
.password(ldapUserPasswd)
.socketTimeout(30000)
.build()
val adminRestApi = new AdminRestApi(basicKyuubiRestClient)
var engines = adminRestApi.listEngines("spark_sql", "user", "default", "").asScala
assert(engines.size == 1)
assert(engines(0).getUser == user)
assert(engines(0).getVersion == KYUUBI_VERSION)
assert(engines(0).getEngineType == "SPARK_SQL")
assert(engines(0).getSharelevel == "USER")
assert(engines(0).getSubdomain == "default")
assert(engines(0).getNamespace == engineSpace)
val result = adminRestApi.deleteEngine("spark_sql", "user", "default", "")
assert(result == s"Engine ${engineSpace} is deleted successfully.")
engines = adminRestApi.listEngines("spark_sql", "user", "default", "").asScala
assert(engines.size == 0)
}
}