From 52d3bf25ed5690f30ddfffcd01e2fc43e73223ce Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sun, 28 May 2023 09:32:23 +0800 Subject: [PATCH] [KYUUBI #4889] Admin command line supports list server command ### _Why are the changes needed?_ Support to list server with kyuubi-admin/kyuubi rest client. ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - UT for AdminRestAPI - UT for AminCtlArgument - UT for AdminCtl - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4889 from turboFei/list_server. Closes #4889 bfd13fbde [fwang12] nit 0a0131552 [fwang12] list server Authored-by: fwang12 Signed-off-by: fwang12 --- docs/tools/kyuubi-admin.rst | 9 ++++ .../ctl/cli/AdminControlCliArguments.scala | 8 +++- .../ctl/cmd/list/AdminListServerCommand.scala | 44 +++++++++++++++++++ .../kyuubi/ctl/opt/AdminCommandLine.scala | 8 +++- .../org/apache/kyuubi/ctl/util/Render.scala | 15 ++++++- .../ctl/AdminControlCliArgumentsSuite.scala | 11 ++++- .../apache/kyuubi/client/AdminRestApi.java | 8 ++++ .../kyuubi/client/api/v1/dto/ServerData.java | 2 + .../server/rest/client/AdminCtlSuite.scala | 18 +++++++- .../rest/client/AdminRestApiSuite.scala | 25 ++++++++++- 10 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListServerCommand.scala diff --git a/docs/tools/kyuubi-admin.rst b/docs/tools/kyuubi-admin.rst index 606396593..7fd07e973 100644 --- a/docs/tools/kyuubi-admin.rst +++ b/docs/tools/kyuubi-admin.rst @@ -98,6 +98,15 @@ Usage: ``bin/kyuubi-admin list engine [options]`` * - --hs2ProxyUser - The proxy user to impersonate. When specified, it will list engines for the hs2ProxyUser. +.. _list_server: + +List Servers +------------------------------------- + +Prints a table of the key information about the servers. + +Usage: ``bin/kyuubi-admin list server`` + .. _delete_engine: Delete an Engine diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala index 4bc1e1317..5a45630c6 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala @@ -22,7 +22,7 @@ import scopt.OParser import org.apache.kyuubi.KyuubiException import org.apache.kyuubi.ctl.cmd.Command import org.apache.kyuubi.ctl.cmd.delete.AdminDeleteEngineCommand -import org.apache.kyuubi.ctl.cmd.list.AdminListEngineCommand +import org.apache.kyuubi.ctl.cmd.list.{AdminListEngineCommand, AdminListServerCommand} import org.apache.kyuubi.ctl.cmd.refresh.RefreshConfigCommand import org.apache.kyuubi.ctl.opt.{AdminCommandLine, CliConfig, ControlAction, ControlObject} @@ -37,6 +37,7 @@ class AdminControlCliArguments(args: Seq[String], env: Map[String, String] = sys cliConfig.action match { case ControlAction.LIST => cliConfig.resource match { case ControlObject.ENGINE => new AdminListEngineCommand(cliConfig) + case ControlObject.SERVER => new AdminListServerCommand(cliConfig) case _ => throw new KyuubiException(s"Invalid resource: ${cliConfig.resource}") } case ControlAction.DELETE => cliConfig.resource match { @@ -61,6 +62,11 @@ class AdminControlCliArguments(args: Seq[String], env: Map[String, String] = sys | sharelevel ${cliConfig.engineOpts.engineShareLevel} | sharesubdomain ${cliConfig.engineOpts.engineSubdomain} """.stripMargin + case ControlObject.SERVER => + s"""Parsed arguments: + | action ${cliConfig.action} + | resource ${cliConfig.resource} + """.stripMargin case ControlObject.CONFIG => s"""Parsed arguments: | action ${cliConfig.action} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListServerCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListServerCommand.scala new file mode 100644 index 000000000..a8e7e5d06 --- /dev/null +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListServerCommand.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.ctl.cmd.list + +import scala.collection.JavaConverters._ + +import org.apache.kyuubi.client.AdminRestApi +import org.apache.kyuubi.client.api.v1.dto.ServerData +import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient +import org.apache.kyuubi.ctl.cmd.AdminCtlCommand +import org.apache.kyuubi.ctl.opt.CliConfig +import org.apache.kyuubi.ctl.util.Render + +class AdminListServerCommand(cliConfig: CliConfig) + extends AdminCtlCommand[Seq[ServerData]](cliConfig) { + + override def validate(): Unit = {} + + override def doRun(): Seq[ServerData] = { + withKyuubiRestClient(normalizedCliConfig, null, conf) { kyuubiRestClient => + val adminRestApi = new AdminRestApi(kyuubiRestClient) + adminRestApi.listServers().asScala + } + } + + override def render(resp: Seq[ServerData]): Unit = { + info(Render.renderServerNodesInfo(resp)) + } +} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala index b1a70935b..71e4068e5 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala @@ -64,7 +64,8 @@ object AdminCommandLine extends CommonCommandLine { .text("\tList information about resources.") .action((_, c) => c.copy(action = ControlAction.LIST)) .children( - engineCmd(builder).text("\tList all the engine nodes for a user"))) + engineCmd(builder).text("\tList all the engine nodes for a user"), + serverCmd(builder).text("\tList all the server nodes"))) } @@ -94,6 +95,11 @@ object AdminCommandLine extends CommonCommandLine { .text("The engine share level this engine belong to.")) } + private def serverCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { + import builder._ + cmd("server").action((_, c) => c.copy(resource = ControlObject.SERVER)) + } + private def refreshConfigCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { import builder._ cmd("config").action((_, c) => c.copy(resource = ControlObject.CONFIG)) diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala index 2d4879e42..bb6e3529d 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala @@ -19,7 +19,7 @@ package org.apache.kyuubi.ctl.util import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer -import org.apache.kyuubi.client.api.v1.dto.{Batch, Engine, GetBatchesResponse, SessionData} +import org.apache.kyuubi.client.api.v1.dto.{Batch, Engine, GetBatchesResponse, ServerData, SessionData} import org.apache.kyuubi.ctl.util.DateTimeUtils._ import org.apache.kyuubi.ha.client.ServiceNodeInfo @@ -45,6 +45,19 @@ private[ctl] object Render { Tabulator.format(title, header, rows) } + def renderServerNodesInfo(serverNodesInfo: Seq[ServerData]): String = { + val title = s"Server Node List (total ${serverNodesInfo.size})" + val header = Array("Namespace", "Instance", "Attributes", "Status") + val rows = serverNodesInfo.map { server => + Array( + server.getNamespace, + server.getInstance, + server.getAttributes.asScala.map { case (k, v) => s"$k=$v" }.mkString("\n"), + server.getStatus) + }.toArray + Tabulator.format(title, header, rows) + } + def renderSessionDataListInfo(sessions: Seq[SessionData]): String = { val title = s"Live Session List (total ${sessions.size})" val header = Array( diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala index dab796127..fdadd011b 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala @@ -115,6 +115,13 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi } } + test("test list server") { + val args = Array("list", "server") + val opArgs = new AdminControlCliArguments(args) + assert(opArgs.cliConfig.action.toString === "LIST") + assert(opArgs.cliConfig.resource.toString === "SERVER") + } + test("test --help") { // scalastyle:off val helpString = @@ -130,7 +137,7 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi | --hs2ProxyUser The value of hive.server2.proxy.user config. | --conf Kyuubi config property pair, formatted key=value. | - |Command: list [engine] + |Command: list [engine|server] | List information about resources. |Command: list engine [options] | List all the engine nodes for a user @@ -140,6 +147,8 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi | The engine subdomain this engine belong to. | -esl, --engine-share-level | The engine share level this engine belong to. + |Command: list server + | List all the server nodes | |Command: delete [engine] | Delete resources. diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java index c81af593a..a983827c8 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import org.apache.kyuubi.client.api.v1.dto.Engine; import org.apache.kyuubi.client.api.v1.dto.OperationData; +import org.apache.kyuubi.client.api.v1.dto.ServerData; import org.apache.kyuubi.client.api.v1.dto.SessionData; public class AdminRestApi { @@ -99,6 +100,13 @@ public class AdminRestApi { return this.getClient().delete(url, null, client.getAuthHeader()); } + public List listServers() { + ServerData[] result = + this.getClient() + .get(API_BASE_PATH + "/server", null, ServerData[].class, client.getAuthHeader()); + return Arrays.asList(result); + } + private IRestClient getClient() { return this.client.getHttpClient(); } diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ServerData.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ServerData.java index d64d43a72..6fe036162 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ServerData.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ServerData.java @@ -29,6 +29,8 @@ public class ServerData { private Map attributes; private String status; + public ServerData() {} + public ServerData( String nodeName, String namespace, diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala index 389b67e47..986b171c1 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala @@ -19,13 +19,16 @@ package org.apache.kyuubi.server.rest.client import java.util.UUID +import org.mockito.Mockito.lenient +import org.scalatestplus.mockito.MockitoSugar.mock + import org.apache.kyuubi.{KYUUBI_VERSION, RestClientTestHelper} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.ctl.{CtlConf, TestPrematureExit} import org.apache.kyuubi.engine.EngineRef import org.apache.kyuubi.ha.HighAvailabilityConf +import org.apache.kyuubi.ha.client.{DiscoveryPaths, ServiceDiscovery} import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient -import org.apache.kyuubi.ha.client.DiscoveryPaths import org.apache.kyuubi.plugin.PluginLoader class AdminCtlSuite extends RestClientTestHelper with TestPrematureExit { @@ -102,4 +105,17 @@ class AdminCtlSuite extends RestClientTestHelper with TestPrematureExit { args, "Engine Node List (total 0)") } + + test("list server") { + // Mock Kyuubi Server + val serverDiscovery = mock[ServiceDiscovery] + lenient.when(serverDiscovery.fe).thenReturn(fe) + val namespace = conf.get(HighAvailabilityConf.HA_NAMESPACE) + withDiscoveryClient(conf) { client => + client.registerService(conf, namespace, serverDiscovery) + + val args = Array("list", "server", "--authSchema", "spnego") + testPrematureExitForAdminControlCli(args, "Server Node List (total 1)") + } + } } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala index b79e62a12..91cd33e58 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala @@ -22,14 +22,16 @@ import java.util.UUID import scala.collection.JavaConverters.asScalaBufferConverter import org.apache.hive.service.rpc.thrift.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V2 +import org.mockito.Mockito.lenient +import org.scalatestplus.mockito.MockitoSugar.mock import org.apache.kyuubi.{KYUUBI_VERSION, RestClientTestHelper} import org.apache.kyuubi.client.{AdminRestApi, KyuubiRestClient} import org.apache.kyuubi.config.{KyuubiConf, KyuubiReservedKeys} import org.apache.kyuubi.engine.EngineRef import org.apache.kyuubi.ha.HighAvailabilityConf +import org.apache.kyuubi.ha.client.{DiscoveryPaths, ServiceDiscovery} import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient -import org.apache.kyuubi.ha.client.DiscoveryPaths import org.apache.kyuubi.plugin.PluginLoader class AdminRestApiSuite extends RestClientTestHelper { @@ -148,4 +150,25 @@ class AdminRestApiSuite extends RestClientTestHelper { assert(!operations.map(op => op.getIdentifier).contains(operation.identifier.toString)) } + + test("list server") { + val spnegoKyuubiRestClient: KyuubiRestClient = + KyuubiRestClient.builder(baseUri.toString) + .authHeaderMethod(KyuubiRestClient.AuthHeaderMethod.SPNEGO) + .spnegoHost("localhost") + .build() + val adminRestApi = new AdminRestApi(spnegoKyuubiRestClient) + + // Mock Kyuubi Server + val serverDiscovery = mock[ServiceDiscovery] + lenient.when(serverDiscovery.fe).thenReturn(fe) + val namespace = conf.get(HighAvailabilityConf.HA_NAMESPACE) + withDiscoveryClient(conf) { client => + client.registerService(conf, namespace, serverDiscovery) + + val servers = adminRestApi.listServers().asScala + assert(servers.nonEmpty) + assert(servers.map(s => s.getInstance()).contains(server.frontendServices.last.connectionUrl)) + } + } }