From 0aafc55bbacbbf5e16daee16a07461301b33e5ab Mon Sep 17 00:00:00 2001 From: fwang12 Date: Mon, 29 Mar 2021 20:43:13 +0800 Subject: [PATCH] [KYUUBI #304] Add Kyuubi Ctl arguments parser ![turboFei](https://badgen.net/badge/Hello/turboFei/green) [![Closes #465](https://badgen.net/badge/Preview/Closes%20%23465/blue)](https://github.com/yaooqinn/kyuubi/pull/465) ![919](https://badgen.net/badge/%2B/919/red) ![0](https://badgen.net/badge/-/0/green) ![22](https://badgen.net/badge/commits/22/yellow) ![Test Plan](https://badgen.net/badge/Missing/Test%20Plan/ff0000) [Powered by Pull Request Badge](https://pullrequestbadge.com/?utm_medium=github&utm_source=yaooqinn&utm_campaign=badge_info) ### _Why are the changes needed?_ ``` bin/kyuubi-service --zkAddress ... --namespace ... --user ... --host ... --port ... --version Operations: - create - expose a service to a namespace, this case is rare but sometimes we may want one server to be reached in 2 or more namespaces by different user groups - get - get the service node info - delete - delete the specified serviceNode - list - list all the service nodes for a particular domain Role: - server default - engine Args: --zkAddress - one of the zk ensemble address, using kyuubi-defaults/conf if absent --namespace - the namespace, using kyuubi-defaults/conf if absent --user - --host --port --version ``` ### _How was this patch tested?_ UT Closes #465 from turboFei/KYUUBI_304_CMD. Closes #304 4bab34b [fwang12] retest pleaes 8083a12 [fwang12] to increase code converage c7e51a2 [fwang12] complete 7249cd6 [fwang12] add ut c809c27 [fwang12] enable set verbose at first bb3cbb6 [fwang12] fix ut 604820d [fwang12] validate for each action a01ac1f [fwang12] fix scala style issue 76c9b4c [fwang12] save 7139dd5 [fwang12] increase test converage 318ebce [fwang12] add more ut 72978a6 [fwang12] save 2931f93 [fwang12] address comments 10b855d [fwang12] save 420912a [fwang12] treat help as an action b27d0a6 [fwang12] treat help as action 896e20d [fwang12] add ctl module into codecov ea43d69 [fwang12] rename kyuubi-service to kyuubi-ctl 65a0e30 [fwang12] save db718b0 [fwang12] refactor 41b503e [fwang12] Add kyuubi-ctl arguments parser cb3f6a8 [fwang12] with log appender Authored-by: fwang12 Signed-off-by: Kent Yao --- dev/kyuubi-codecov/pom.xml | 6 + kyuubi-assembly/pom.xml | 6 + .../org/apache/kyuubi/KyuubiFunSuite.scala | 43 +++ kyuubi-ctl/pom.xml | 106 +++++++ .../kyuubi/ctl/KyuubiCtlOptionParser.java | 150 +++++++++ .../org/apache/kyuubi/ctl/KyuubiCtl.scala | 28 ++ .../kyuubi/ctl/KyuubiCtlArguments.scala | 292 ++++++++++++++++++ .../kyuubi/ctl/KyuubiCtlArgumentsParser.scala | 20 ++ .../kyuubi/ctl/KyuubiCtlException.scala | 23 ++ .../src/test/resources/log4j.properties | 40 +++ .../kyuubi/ctl/KyuubiCtlArgumentsSuite.scala | 204 ++++++++++++ pom.xml | 1 + 12 files changed, 919 insertions(+) create mode 100644 kyuubi-ctl/pom.xml create mode 100644 kyuubi-ctl/src/main/java/org/apache/kyuubi/ctl/KyuubiCtlOptionParser.java create mode 100644 kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtl.scala create mode 100644 kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtlArguments.scala create mode 100644 kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtlArgumentsParser.scala create mode 100644 kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtlException.scala create mode 100644 kyuubi-ctl/src/test/resources/log4j.properties create mode 100644 kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/KyuubiCtlArgumentsSuite.scala diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml index 57876eecd..9b26cb730 100644 --- a/dev/kyuubi-codecov/pom.xml +++ b/dev/kyuubi-codecov/pom.xml @@ -38,6 +38,12 @@ ${project.version} + + org.apache.kyuubi + kyuubi-ctl + ${project.version} + + org.apache.kyuubi kyuubi-ha diff --git a/kyuubi-assembly/pom.xml b/kyuubi-assembly/pom.xml index 74b32bcab..43200390a 100644 --- a/kyuubi-assembly/pom.xml +++ b/kyuubi-assembly/pom.xml @@ -57,6 +57,12 @@ ${project.version} + + org.apache.kyuubi + kyuubi-ctl + ${project.version} + + org.apache.hadoop hadoop-client-api diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala index 081f5768a..58d28c88c 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala @@ -17,7 +17,11 @@ package org.apache.kyuubi +import scala.collection.mutable.ArrayBuffer + // scalastyle:off +import org.apache.log4j.{Appender, AppenderSkeleton, Level, Logger} +import org.apache.log4j.spi.LoggingEvent import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, FunSuite, Outcome} import org.scalatest.concurrent.Eventually @@ -52,4 +56,43 @@ trait KyuubiFunSuite extends FunSuite info(s"\n\n===== FINISHED $shortSuiteName: '$testName' =====\n") } } + + /** + * Adds a log appender and optionally sets a log level to the root logger or the logger with + * the specified name, then executes the specified function, and in the end removes the log + * appender and restores the log level if necessary. + */ + protected def withLogAppender( + appender: Appender, + loggerName: Option[String] = None, + level: Option[Level] = None)( + f: => Unit): Unit = { + val logger = loggerName.map(Logger.getLogger).getOrElse(Logger.getRootLogger) + val restoreLevel = logger.getLevel + logger.addAppender(appender) + if (level.isDefined) { + logger.setLevel(level.get) + } + try f finally { + logger.removeAppender(appender) + if (level.isDefined) { + logger.setLevel(restoreLevel) + } + } + } + + class LogAppender(msg: String = "", maxEvents: Int = 1000) extends AppenderSkeleton { + val loggingEvents = new ArrayBuffer[LoggingEvent]() + + override def append(loggingEvent: LoggingEvent): Unit = { + if (loggingEvents.size >= maxEvents) { + val loggingInfo = if (msg == "") "." else s" while logging $msg." + throw new IllegalStateException( + s"Number of events reached the limit of $maxEvents$loggingInfo") + } + loggingEvents.append(loggingEvent) + } + override def close(): Unit = {} + override def requiresLayout(): Boolean = false + } } diff --git a/kyuubi-ctl/pom.xml b/kyuubi-ctl/pom.xml new file mode 100644 index 000000000..b54ffd37e --- /dev/null +++ b/kyuubi-ctl/pom.xml @@ -0,0 +1,106 @@ + + + + + + org.apache.kyuubi + kyuubi + 1.2.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + kyuubi-ctl + jar + Kyuubi Project Control + + + + org.apache.kyuubi + kyuubi-common + ${project.version} + + + org.apache.kyuubi + kyuubi-ha + ${project.version} + + + + org.apache.hadoop + hadoop-client-api + provided + + + + org.apache.hadoop + hadoop-client-runtime + provided + + + + org.apache.curator + curator-framework + + + + org.apache.curator + curator-recipes + + + + org.apache.zookeeper + zookeeper + + + + + org.apache.curator + curator-test + provided + + + + + org.apache.kyuubi + kyuubi-common + ${project.version} + test-jar + test + + + + org.apache.hadoop + hadoop-minikdc + test + + + + org.apache.directory.server + apacheds-service + test + + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + diff --git a/kyuubi-ctl/src/main/java/org/apache/kyuubi/ctl/KyuubiCtlOptionParser.java b/kyuubi-ctl/src/main/java/org/apache/kyuubi/ctl/KyuubiCtlOptionParser.java new file mode 100644 index 000000000..214b76794 --- /dev/null +++ b/kyuubi-ctl/src/main/java/org/apache/kyuubi/ctl/KyuubiCtlOptionParser.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.ctl; + +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +abstract class KyuubiCtlOptionParser { + protected final String CREATE = "create"; + protected final String GET = "get"; + protected final String DELETE = "delete"; + protected final String LIST = "list"; + protected final String SERVER = "server"; + protected final String ENGINE = "engine"; + + protected final String ZK_ADDRESS = "--zkAddress"; + protected final String NAMESPACE = "--namespace"; + protected final String USER = "--user"; + protected final String HOST = "--host"; + protected final String PORT = "--port"; + protected final String VERSION = "--version"; + + // Options that do not take arguments. + protected final String HELP = "--help"; + protected final String VERBOSE = "--verbose"; + + final String[][] opts = { + { ZK_ADDRESS, "-zk" }, + { NAMESPACE, "-ns" }, + { USER, "-u" }, + { HOST, "-h" }, + { PORT, "-p" }, + { VERSION, "-V" }, + }; + + /** + * List of switches (command line options that do not take parameters) recognized by + * kyuubi-ctl. + */ + final String[][] switches = { + { HELP, "-I" }, + { VERBOSE, "-v" }, + }; + + /** + * Parse action type and service type. + * + * @return offset of remaining arguments. + */ + protected abstract int parseActionAndService(List args); + + /** + * Parse a list of kyuubi-ctl command line options. + *

+ * See KyuubiCtlArguments.scala for a more formal description of available options. + * + * @throws IllegalArgumentException If an error is found during parsing. + */ + protected final void parse(List args) { + Pattern eqSeparatedOpt = Pattern.compile("(--[^=]+)=(.+)"); + + int idx = parseActionAndService(args); + for (; idx < args.size(); idx++) { + String arg = args.get(idx); + String value = null; + + Matcher m = eqSeparatedOpt.matcher(arg); + if (m.matches()) { + arg = m.group(1); + value = m.group(2); + } + + // Look for options with a value. + String name = findCliOption(arg, opts); + if (name != null) { + if (value == null) { + if (idx == args.size() - 1) { + throw new IllegalArgumentException( + String.format("Missing argument for option '%s'.", arg)); + } + idx++; + value = args.get(idx); + } + if (!handle(name, value)) { + break; + } + continue; + } + + // Look for a switch. + name = findCliOption(arg, switches); + if (name != null) { + if (!handle(name, null)) { + break; + } + continue; + } + + handleUnknown(arg); + } + } + + /** + * Callback for when an option with an argument is parsed. + * + * @param opt The long name of the cli option (might differ from actual command line). + * @param value The value. This will be null if the option does not take a value. + * @return Whether to continue parsing the argument list. + */ + protected abstract boolean handle(String opt, String value); + + /** + * Callback for when an unrecognized option is parsed. + * + * @param opt Unrecognized option from the command line. + * @return Whether to continue parsing the argument list. + */ + protected abstract boolean handleUnknown(String opt); + + private String findCliOption(String name, String[][] available) { + for (String[] candidates : available) { + for (String candidate : candidates) { + if (candidate.equals(name)) { + return candidates[0]; + } + } + } + return null; + } + + protected String findSwitches(String name) { + return findCliOption(name, switches); + } +} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtl.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtl.scala new file mode 100644 index 000000000..25dd50b19 --- /dev/null +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtl.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.ctl + +private[ctl] object KyuubiCtlAction extends Enumeration { + type KyuubiCtlAction = Value + val CREATE, GET, DELETE, LIST, HELP = Value +} + +private[ctl] object KyuubiCtlActionService extends Enumeration { + type KyuubiCtlActionService = Value + val SERVER, ENGINE = Value +} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtlArguments.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtlArguments.scala new file mode 100644 index 000000000..06aacadd5 --- /dev/null +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtlArguments.scala @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.ctl + +import java.util.{List => JList} + +import scala.collection.JavaConverters._ + +import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiException, Logging, Utils} +import org.apache.kyuubi.ctl.KyuubiCtlAction._ +import org.apache.kyuubi.ctl.KyuubiCtlActionService._ +import org.apache.kyuubi.ha.HighAvailabilityConf._ + +class KyuubiCtlArguments(args: Seq[String], env: Map[String, String] = sys.env) + extends KyuubiCtlArgumentsParser with Logging { + var action: KyuubiCtlAction = null + var service: KyuubiCtlActionService = null + var zkAddress: String = null + var nameSpace: String = null + var user: String = null + var host: String = null + var port: String = null + var version: String = null + var verbose: Boolean = false + + /** Default properties present in the currently defined default file. */ + lazy val defaultKyuubiProperties: Map[String, String] = { + val maybeConfigFile = Utils.getDefaultPropertiesFile(env) + if (verbose) { + info(s"Using properties file: $maybeConfigFile") + } + val defaultProperties = Utils.getPropertiesFromFile(maybeConfigFile) + if (verbose) { + defaultProperties.foreach { case (k, v) => + info(s"Adding default property: $k=$v") + } + } + defaultProperties + } + + // Set parameters from command line arguments + parse(args.asJava) + + // Use default property value if not set + useDefaultPropertyValueIfMissing() + + validateArguments() + + if (verbose) { + info(toString) + } + + private def useDefaultPropertyValueIfMissing(): Unit = { + if (zkAddress == null) { + zkAddress = defaultKyuubiProperties.getOrElse(HA_ZK_QUORUM.key, null) + } + if (nameSpace == null) { + nameSpace = defaultKyuubiProperties.getOrElse(HA_ZK_NAMESPACE.key, null) + } + if (version == null) { + version = KYUUBI_VERSION + } + } + + /** Ensure that required fields exists. Call this only once all defaults are loaded. */ + private def validateArguments(): Unit = { + service = Option(service).getOrElse(KyuubiCtlActionService.SERVER) + action match { + case KyuubiCtlAction.CREATE => validateCreateGetDeleteArguments() + case KyuubiCtlAction.GET => validateCreateGetDeleteArguments() + case KyuubiCtlAction.DELETE => validateCreateGetDeleteArguments() + case KyuubiCtlAction.LIST => validateListArguments() + case KyuubiCtlAction.HELP => + case _ => printUsageAndExit(-1) + } + } + + private def validateCreateGetDeleteArguments(): Unit = { + if (zkAddress == null) { + fail("Zookeeper address is not specified and no default value to load") + } + if (nameSpace == null) { + fail("Zookeeper namespace is not specified and no default value to load") + } + if (version == null) { + fail("Kyuubi version is not specified and could not found KYUUBI_VERSION in " + + "kyuubi-build-info") + } + if (host == null) { + fail("Must specify host for service") + } + if (port == null) { + fail("Must specify port for service") + } + if (service == KyuubiCtlActionService.ENGINE && user == null) { + fail("Must specify user name for engine") + } + } + + private def validateListArguments(): Unit = { + if (zkAddress == null) { + fail("Zookeeper address is not specified and no default value to load") + } + if (nameSpace == null) { + fail("Zookeeper namespace is not specified and no default value to load") + } + } + + private def printUsageAndExit(exitCode: Int, unknownParam: Any = null): Unit = { + if (unknownParam != null) { + info("Unknown/unsupported param " + unknownParam) + } + val command = sys.env.getOrElse("_KYUUBI_CMD_USAGE", + s""" + |Kyuubi Ver $KYUUBI_VERSION. + |Usage: kyuubi-ctl --zkAddress ... + |--namespace ... --user ... --host ... --port ... --version""".stripMargin) + info(command) + + info( + s""" + |Command: + | - create expose a service to a namespace on the zookeeper cluster of + | zkAddress manually + | - get get the service node info + | - delete delete the specified serviceNode + | - list list all the service nodes for a particular domain + | + |Service: + | - server default + | - engine + | + |Arguments: + | --zkAddress, -zk one of the zk ensemble address, using kyuubi-defaults/conf + | if absent + | --namespace, -ns the namespace, using kyuubi-defaults/conf if absent + | --host, -h hostname or IP address of a service + | --port, -p listening port of a service + | --version, -V using the compiled KYUUBI_VERSION default, change it if the + | active service is running in another + | --user, -u for engine service only, the user name this engine belong to + | --help, -I Show this help message and exit. + | --verbose, -v Print additional debug output. + """.stripMargin + ) + + throw new KyuubiCtlException(exitCode) + } + + override protected def parseActionAndService(args: JList[String]): Int = { + if (args.isEmpty) { + printUsageAndExit(-1) + } + + var actionParsed = false + var serviceParsed = false + var offset = 0 + + while(offset < args.size() && needContinueHandle() && !(actionParsed && serviceParsed)) { + val arg = args.get(offset) + if (!actionParsed) { + arg match { + case CREATE => + action = KyuubiCtlAction.CREATE + actionParsed = true + case GET => + action = KyuubiCtlAction.GET + actionParsed = true + case DELETE => + action = KyuubiCtlAction.DELETE + actionParsed = true + case LIST => + action = KyuubiCtlAction.LIST + actionParsed = true + case _ => findSwitches(arg) match { + case HELP => + action = KyuubiCtlAction.HELP + actionParsed = true + case VERBOSE => + verbose = true + case _ => + printUsageAndExit(-1, arg) + } + } + offset += 1 + } else if (needContinueHandle() && !serviceParsed) { + arg match { + case SERVER => + service = KyuubiCtlActionService.SERVER + serviceParsed = true + offset += 1 + case ENGINE => + service = KyuubiCtlActionService.ENGINE + serviceParsed = true + offset += 1 + case _ => findSwitches(arg) match { + case HELP => + action = KyuubiCtlAction.HELP + offset += 1 + case VERBOSE => + verbose = true + offset += 1 + case _ => + service = KyuubiCtlActionService.SERVER + serviceParsed = true + } + } + } + } + offset + } + + override def toString: String = { + s"""Parsed arguments: + | action $action + | service $service + | zkAddress $zkAddress + | namespace $nameSpace + | user $user + | host $host + | port $port + | version $version + | verbose $verbose + """.stripMargin + } + + private def needContinueHandle(): Boolean = { + action != KyuubiCtlAction.HELP + } + + /** Fill in values by parsing user options. */ + override protected def handle(opt: String, value: String): Boolean = { + if (!needContinueHandle()) { + return false + } + opt match { + case ZK_ADDRESS => + zkAddress = value + + case NAMESPACE => + nameSpace = value + + case USER => + user = value + + case HOST => + host = value + + case PORT => + port = value + + case VERSION => + version = value + + case HELP => + action = KyuubiCtlAction.HELP + + case VERBOSE => + verbose = true + + case _ => + fail(s"Unexpected argument '$opt'.") + } + needContinueHandle() + } + + override protected def handleUnknown(opt: String): Boolean = { + if (!needContinueHandle()) { + false + } else { + printUsageAndExit(-1, opt) + false + } + } + + private def fail(msg: String): Unit = throw new KyuubiException(msg) +} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtlArgumentsParser.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtlArgumentsParser.scala new file mode 100644 index 000000000..e7ace8606 --- /dev/null +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtlArgumentsParser.scala @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.ctl + +private[kyuubi] abstract class KyuubiCtlArgumentsParser extends KyuubiCtlOptionParser diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtlException.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtlException.scala new file mode 100644 index 000000000..c6e77ae26 --- /dev/null +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/KyuubiCtlException.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.ctl + +import org.apache.kyuubi.KyuubiException + +class KyuubiCtlException(exitCode: Int) + extends KyuubiException(s"Kyuubi Ctl exited with $exitCode") diff --git a/kyuubi-ctl/src/test/resources/log4j.properties b/kyuubi-ctl/src/test/resources/log4j.properties new file mode 100644 index 000000000..fee75dc67 --- /dev/null +++ b/kyuubi-ctl/src/test/resources/log4j.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +log4j.rootLogger=DEBUG, CA, FA + +#Console Appender +log4j.appender.CA=org.apache.log4j.ConsoleAppender +log4j.appender.CA.layout=org.apache.log4j.PatternLayout +log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n +log4j.appender.CA.Threshold = FATAL + +#File Appender +log4j.appender.FA=org.apache.log4j.FileAppender +log4j.appender.FA.append=false +log4j.appender.FA.file=target/unit-tests.log +log4j.appender.FA.layout=org.apache.log4j.PatternLayout +log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n + +# Set the logger level of File Appender to WARN +log4j.appender.FA.Threshold = DEBUG + +# SPARK-34128：Suppress undesirable TTransportException warnings involved in THRIFT-4805 +log4j.appender.console.filter.1=org.apache.log4j.varia.StringMatchFilter +log4j.appender.console.filter.1.StringToMatch=Thrift error occurred during processing of message +log4j.appender.console.filter.1.AcceptOnMatch=false diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/KyuubiCtlArgumentsSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/KyuubiCtlArgumentsSuite.scala new file mode 100644 index 000000000..5f4f81d28 --- /dev/null +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/KyuubiCtlArgumentsSuite.scala @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.ctl + +import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite} + +class KyuubiCtlArgumentsSuite extends KyuubiFunSuite { + val zkAddress = "localhost:2181" + val namespace = "kyuubi" + val user = "kyuubi" + val host = "localhost" + val port = "10000" + + /** Check whether the script exits and the given search string is printed. */ + private def testPrematureExit(args: Array[String], searchString: String): Unit = { + val logAppender = new LogAppender("test premature exit") + withLogAppender(logAppender) { + val thread = new Thread { + override def run(): Unit = try { + new KyuubiCtlArguments(args) + } catch { + case e: Exception => + error(e) + } + } + thread.start() + thread.join() + assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(searchString))) + } + } + + test("test basic kyuubi service arguments parser") { + Seq("create", "get", "list", "delete").foreach { command => + Seq("server", "engine").foreach { service => + val args = Seq( + command, service, + "--zkAddress", zkAddress, + "--namespace", namespace, + "--user", user, + "--host", host, + "--port", port, + "--version", KYUUBI_VERSION + ) + val opArgs = new KyuubiCtlArguments(args) + assert(opArgs.action.toString.equalsIgnoreCase(command)) + assert(opArgs.service.toString.equalsIgnoreCase(service)) + assert(opArgs.zkAddress == zkAddress) + assert(opArgs.nameSpace == namespace) + assert(opArgs.user == user) + assert(opArgs.host == host) + assert(opArgs.port == port) + assert(opArgs.version == KYUUBI_VERSION) + } + } + } + + test("treat --help as action") { + val args = Seq("-I") + val opArgs = new KyuubiCtlArguments(args) + assert(opArgs.action == KyuubiCtlAction.HELP) + assert(opArgs.version == KYUUBI_VERSION) + + val args2 = Seq( + "create", "server", + s"--user=$user", + "-h", host, + "--verbose", + "--help", + "--port", port + ) + val opArgs2 = new KyuubiCtlArguments(args2) + assert(opArgs2.action == KyuubiCtlAction.HELP) + assert(opArgs2.user == user) + assert(opArgs2.host == host) + assert(opArgs2.verbose) + } + + test("prints usage on empty input") { + testPrematureExit(Array.empty[String], "Usage: kyuubi-ctl") + testPrematureExit(Array("--verbose"), "Usage: kyuubi-ctl") + testPrematureExit(Array("-v"), "Usage: kyuubi-ctl") + } + + test("prints error with unrecognized options") { + testPrematureExit(Array("create", "--unkonwn"), "Unknown/unsupported param --unkonwn") + testPrematureExit(Array("--unkonwn"), "Unknown/unsupported param --unkonwn") + } + + test("test invalid arguments") { + testPrematureExit(Array("create", "--user"), "Missing argument for option '--user'") + } + + test("test extra unused arguments") { + val args = Array( + "list", + "extraArg1", "extraArg2" + ) + testPrematureExit(args, "Unknown/unsupported param extraArg1") + } + + test("test list action arguments") { + val args = Array( + "list" + ) + testPrematureExit(args, "Zookeeper address is not specified") + + val args2 = Array( + "list", + "--zkAddress", zkAddress + ) + testPrematureExit(args2, "Zookeeper namespace is not specified") + + val args3 = Array( + "list", + "--zkAddress", zkAddress, + "--namespace", namespace + ) + val opArgs = new KyuubiCtlArguments(args3) + assert(opArgs.action == KyuubiCtlAction.LIST) + } + + test("test create/get/delete action arguments") { + Seq("create", "get", "delete").foreach { op => + val args = Array( + op + ) + testPrematureExit(args, "Zookeeper address is not specified") + + val args2 = Array( + op, + "--zkAddress", zkAddress + ) + testPrematureExit(args2, "Zookeeper namespace is not specified") + + val args3 = Array( + op, + "--zkAddress", zkAddress, + "--namespace", namespace + ) + testPrematureExit(args3, "Must specify host") + + val args4 = Array( + op, + "--zkAddress", zkAddress, + "--namespace", namespace, + "-h", host + ) + testPrematureExit(args4, "Must specify port") + + val args5 = Array( + op, "engine", + "--zkAddress", zkAddress, + "--namespace", namespace, + "-h", host, + "-p", port + ) + testPrematureExit(args5, "Must specify user name for engine") + + val args6 = Array( + op, "server", + "--zkAddress", zkAddress, + "--namespace", namespace, + "-h", host, + "-p", port + ) + val opArgs6 = new KyuubiCtlArguments(args6) + assert(opArgs6.action.toString.equalsIgnoreCase(op)) + } + } + + test("test with switches at head") { + val args = Seq("--verbose", "list", "engine", "-zk", zkAddress, "-ns", namespace) + val opArgs = new KyuubiCtlArguments(args) + assert(opArgs.verbose) + assert(opArgs.action == KyuubiCtlAction.LIST) + assert(opArgs.service == KyuubiCtlActionService.ENGINE) + + val args2 = Seq("list", "-v", "engine", "-zk", zkAddress, "-ns", namespace) + val opArgs2 = new KyuubiCtlArguments(args2) + assert(opArgs2.verbose) + assert(opArgs2.action == KyuubiCtlAction.LIST) + assert(opArgs2.service == KyuubiCtlActionService.ENGINE) + + val args3 = Seq("list", "--verbose", "--help", "engine", "-zk", zkAddress, "-ns", namespace) + val opArgs3 = new KyuubiCtlArguments(args3) + assert(opArgs3.verbose) + assert(opArgs3.action == KyuubiCtlAction.HELP) + } +} diff --git a/pom.xml b/pom.xml index b1948c383..f04e17343 100644 --- a/pom.xml +++ b/pom.xml @@ -30,6 +30,7 @@ externals/kyuubi-spark-sql-engine kyuubi-assembly kyuubi-common + kyuubi-ctl kyuubi-ha kyuubi-main kyuubi-metrics