apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #684]Improve kyuubi ctl verbose output (#685)

Browse Source 
* [KYUUBI #684]Improve kyuubi ctl verbose output

* fix
This commit is contained in:
hongdd 2021-06-28 14:09:29 +08:00 committed by GitHub
parent e8abfadd2d
commit 164bd03799
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 138 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/kyuubi-ctl
View File

@ -21,9 +21,7 @@ CLASS="org.apache.kyuubi.ctl.ServiceControlCli"
export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)"
echo "Starting Kyuubi Service Control Client from ${KYUUBI_HOME}"
. "${KYUUBI_HOME}/bin/load-kyuubi-env.sh"
. "${KYUUBI_HOME}/bin/load-kyuubi-env.sh" -s
if [[ -z ${JAVA_HOME} ]]; then
echo "Error: JAVA_HOME IS NOT SET! CANNOT PROCEED."

38
bin/load-kyuubi-env.sh
View File

@ -21,10 +21,26 @@ export KYUUBI_HOME="${KYUUBI_HOME:-"$(cd "$(dirname "$0")"/.. || exit; pwd)"}"
export KYUUBI_CONF_DIR="${KYUUBI_CONF_DIR:-"${KYUUBI_HOME}"/conf}"
silent=0
while getopts "s" arg
do
case $arg in
s)
silent=1
;;
?)
echo "unknown argument"
exit 1
;;
esac
done
KYUUBI_ENV_SH="${KYUUBI_CONF_DIR}"/kyuubi-env.sh
if [[ -f ${KYUUBI_ENV_SH} ]]; then
set -a
echo "Using kyuubi environment file ${KYUUBI_ENV_SH} to initialize..."
if [ $silent -eq 0 ]; then
echo "Using kyuubi environment file ${KYUUBI_ENV_SH} to initialize..."
fi
. "${KYUUBI_ENV_SH}"
set +a
else
@ -76,15 +92,17 @@ fi
export SPARK_HOME="${SPARK_HOME:-"${SPARK_BUILTIN}"}"
# Print essential environment variables to console
echo "JAVA_HOME: ${JAVA_HOME}"
if [ $silent -eq 0 ]; then
echo "JAVA_HOME: ${JAVA_HOME}"
echo "KYUUBI_HOME: ${KYUUBI_HOME}"
echo "KYUUBI_CONF_DIR: ${KYUUBI_CONF_DIR}"
echo "KYUUBI_LOG_DIR: ${KYUUBI_LOG_DIR}"
echo "KYUUBI_PID_DIR: ${KYUUBI_PID_DIR}"
echo "KYUUBI_WORK_DIR_ROOT: ${KYUUBI_WORK_DIR_ROOT}"
echo "KYUUBI_HOME: ${KYUUBI_HOME}"
echo "KYUUBI_CONF_DIR: ${KYUUBI_CONF_DIR}"
echo "KYUUBI_LOG_DIR: ${KYUUBI_LOG_DIR}"
echo "KYUUBI_PID_DIR: ${KYUUBI_PID_DIR}"
echo "KYUUBI_WORK_DIR_ROOT: ${KYUUBI_WORK_DIR_ROOT}"
echo "SPARK_HOME: ${SPARK_HOME}"
echo "SPARK_CONF_DIR: ${SPARK_CONF_DIR}"
echo "SPARK_HOME: ${SPARK_HOME}"
echo "SPARK_CONF_DIR: ${SPARK_CONF_DIR}"
echo "HADOOP_CONF_DIR: ${HADOOP_CONF_DIR}"
echo "HADOOP_CONF_DIR: ${HADOOP_CONF_DIR}"
fi

4
conf/log4j.properties.template
View File

@ -21,3 +21,7 @@ log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c{2}: %m%n
# Set the default kyuubi-ctl log level to WARN. When running the kyuubi-ctl, the
# log level for this class is used to overwrite the root logger's log level.
log4j.logger.org.apache.kyuubi.ctl.ServiceControlCli=ERROR

4
docs/deployment/settings.md
View File

@ -304,6 +304,10 @@ log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c{2}: %m%n
# Set the default kyuubi-ctl log level to WARN. When running the kyuubi-ctl, the
# log level for this class is used to overwrite the root logger's log level.
log4j.logger.org.apache.kyuubi.ctl.ServiceControlCli=ERROR
```
## Other Configurations

31
kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala
View File

@ -30,20 +30,14 @@ trait Logging {
// Method to get the logger name for this object
protected def loggerName: String = {
// Ignore trailing $'s in the class names for Scala objects
this.getClass.getName.stripSuffix("$")
// Ignore anon$'s of super class and trailing $'s in the class names for Scala objects
this.getClass.getName.split('$')(0)
}
// Method to get or create the logger for this object
protected def logger: Logger = {
if (log_ == null) {
if (!Logging.initialized) {
Logging.initLock.synchronized {
if (!Logging.initialized) {
initializeLogging()
}
}
}
initializeLoggerIfNecessary(false)
log_ = LoggerFactory.getLogger(loggerName)
}
log_
@ -85,7 +79,17 @@ trait Logging {
}
}
private def initializeLogging(): Unit = {
protected def initializeLoggerIfNecessary(isInterpreter: Boolean): Unit = {
if (!Logging.initialized) {
Logging.initLock.synchronized {
if (!Logging.initialized) {
initializeLogging(isInterpreter)
}
}
}
}
private def initializeLogging(isInterpreter: Boolean): Unit = {
if (Logging.isLog4j12) {
val log4j12Initialized = LogManager.getRootLogger.getAllAppenders.hasMoreElements
// scalastyle:off println
@ -104,6 +108,13 @@ trait Logging {
if (Logging.defaultRootLevel == null) {
Logging.defaultRootLevel = rootLogger.getLevel
}
if (isInterpreter) {
// set kyuubi ctl log level, default ERROR
val ctlLogger = LogManager.getLogger(loggerName)
val ctlLevel = Option(ctlLogger.getLevel()).getOrElse(Level.ERROR)
rootLogger.setLevel(ctlLevel)
}
// scalastyle:on println
}
Logging.initialized = true

32
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CommandLineUtils.scala
View File

@ -74,8 +74,9 @@ private[kyuubi] object Tabulator {
if (str == null) 0 else str.length + fullWidthRegex.findAllIn(str).size
}
def format(title: String, header: Seq[String], rows: Seq[Seq[String]]): String = {
val data = Seq(header).union(rows)
def format(title: String, header: Seq[String], rows: Seq[Seq[String]],
verbose: Boolean): String = {
val data = if (verbose) Seq(header).union(rows) else rows
val sb = new StringBuilder
val numCols = header.size
// We set a minimum column width at '10'
@ -94,20 +95,25 @@ private[kyuubi] object Tabulator {
}
}
// Create SeparateLine
val sep: String = colWidths.map("-" * _).addString(sb, "+", "+", "+\n").toString()
if (verbose) {
// Create SeparateLine
val sep: String = colWidths.map("-" * _).addString(sb, "+", "+", "+\n").toString()
val titleNewLine = "\n " + StringUtils.center(title, colWidths.sum) + "\n"
val titleNewLine = "\n " + StringUtils.center(title, colWidths.sum) + "\n"
// column names
paddedRows.head.addString(sb, "|", "|", "|\n")
sb.append(sep)
// column names
paddedRows.head.addString(sb, "|", "|", "|\n")
sb.append(sep)
// data
paddedRows.tail.foreach(_.addString(sb, "|", "|", "|\n"))
sb.append(sep)
// data
paddedRows.tail.foreach(_.addString(sb, "|", "|", "|\n"))
sb.append(sep)
sb.append(s"${rows.size} row(s)\n")
titleNewLine + sb.toString()
sb.append(s"${rows.size} row(s)\n")
titleNewLine + sb.toString()
} else {
paddedRows.foreach(_.addString(sb, "", "", "\n"))
sb.toString()
}
}
}

22
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ServiceControlCli.scala
View File

@ -45,10 +45,17 @@ private[kyuubi] class ServiceControlCli extends Logging {
import ServiceControlCli._
import ServiceDiscovery._
private var verbose: Boolean = false
def doAction(args: Array[String]): Unit = {
// Initialize logging if it hasn't been done yet.
// Set log level ERROR
initializeLoggerIfNecessary(true)
val ctlArgs = parseArguments(args)
if (ctlArgs.verbose) {
info(ctlArgs.toString)
verbose = ctlArgs.verbose
if (verbose) {
super.info(ctlArgs.toString)
}
ctlArgs.action match {
case ServiceControlAction.CREATE => create(ctlArgs)
@ -99,7 +106,7 @@ private[kyuubi] class ServiceControlCli extends Logging {
}
val title = "Created zookeeper service nodes"
info(renderServiceNodesInfo(title, exposedServiceNodes))
info(renderServiceNodesInfo(title, exposedServiceNodes, verbose))
}
}
@ -113,7 +120,7 @@ private[kyuubi] class ServiceControlCli extends Logging {
val nodes = getServiceNodes(zkClient, znodeRoot, hostPortOpt)
val title = "Zookeeper service nodes"
info(renderServiceNodesInfo(title, nodes))
info(renderServiceNodesInfo(title, nodes, verbose))
}
}
@ -153,7 +160,7 @@ private[kyuubi] class ServiceControlCli extends Logging {
}
val title = "Deleted zookeeper service nodes"
info(renderServiceNodesInfo(title, deletedNodes))
info(renderServiceNodesInfo(title, deletedNodes, verbose))
}
}
@ -207,11 +214,12 @@ object ServiceControlCli extends CommandLineUtils with Logging {
}
private[ctl] def renderServiceNodesInfo(
title: String, serviceNodeInfo: Seq[ServiceNodeInfo]): String = {
title: String, serviceNodeInfo: Seq[ServiceNodeInfo],
verbose: Boolean): String = {
val header = Seq("Namespace", "Host", "Port", "Version")
val rows = serviceNodeInfo.sortBy(_.nodeName).map { sn =>
Seq(sn.namespace, sn.host, sn.port.toString, sn.version.getOrElse(""))
}
Tabulator.format(title, header, rows)
Tabulator.format(title, header, rows, verbose)
}
}

7
kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/ServiceControlCliArguments.scala
View File

@ -54,7 +54,7 @@ class ServiceControlCliArguments(args: Seq[String], env: Map[String, String] = s
if (zkQuorum == null) {
conf.getOption(HA_ZK_QUORUM.key).foreach { v =>
if (verbose) {
info(s"Zookeeper quorum is not specified, use value from default conf:$v")
super.info(s"Zookeeper quorum is not specified, use value from default conf:$v")
}
zkQuorum = v
}
@ -65,14 +65,13 @@ class ServiceControlCliArguments(args: Seq[String], env: Map[String, String] = s
if (action != ServiceControlAction.CREATE && namespace == null) {
namespace = conf.get(HA_ZK_NAMESPACE)
if (verbose) {
info(s"Zookeeper namespace is not specified," +
s" use value from default conf:$namespace")
super.info(s"Zookeeper namespace is not specified, use value from default conf:$namespace")
}
}
if (version == null) {
if (verbose) {
info(s"version is not specified, use built-in KYUUBI_VERSION:$KYUUBI_VERSION")
super.info(s"version is not specified, use built-in KYUUBI_VERSION:$KYUUBI_VERSION")
}
version = KYUUBI_VERSION
}

52
kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ServiceControlCliSuite.scala
View File

@ -120,9 +120,14 @@ class ServiceControlCliSuite extends KyuubiFunSuite with TestPrematureExit {
}
/** Get the rendered service node info without title */
private def getRenderedNodesInfoWithoutTitle(nodesInfo: Seq[ServiceNodeInfo]): String = {
val renderedInfo = renderServiceNodesInfo("", nodesInfo)
renderedInfo.substring(renderedInfo.indexOf("|"))
private def getRenderedNodesInfoWithoutTitle(nodesInfo: Seq[ServiceNodeInfo],
verbose: Boolean): String = {
val renderedInfo = renderServiceNodesInfo("", nodesInfo, verbose)
if (verbose) {
renderedInfo.substring(renderedInfo.indexOf("|"))
} else {
renderedInfo
}
}
test("test help") {
@ -159,7 +164,7 @@ class ServiceControlCliSuite extends KyuubiFunSuite with TestPrematureExit {
test("test render zookeeper service node info") {
val title = "test render"
val nodes = Seq(ServiceNodeInfo("/kyuubi", "serviceNode", "localhost", 10000, Some("version")))
val renderedInfo = renderServiceNodesInfo(title, nodes)
val renderedInfo = renderServiceNodesInfo(title, nodes, true)
val expected = {
s"\n $title " +
"""
@ -172,7 +177,7 @@ class ServiceControlCliSuite extends KyuubiFunSuite with TestPrematureExit {
|""".stripMargin
}
assert(renderedInfo == expected)
assert(renderedInfo.contains(getRenderedNodesInfoWithoutTitle(nodes)))
assert(renderedInfo.contains(getRenderedNodesInfoWithoutTitle(nodes, true)))
}
test("test expose zk service node to another namespace") {
@ -201,7 +206,7 @@ class ServiceControlCliSuite extends KyuubiFunSuite with TestPrematureExit {
ServiceNodeInfo(s"/$newNamespace", "", "localhost", 10001, Some(KYUUBI_VERSION))
)
testPrematureExit(args, getRenderedNodesInfoWithoutTitle(expectedCreatedNodes))
testPrematureExit(args, getRenderedNodesInfoWithoutTitle(expectedCreatedNodes, false))
val znodeRoot = s"/$newNamespace"
val children = framework.getChildren.forPath(znodeRoot).asScala.sorted
assert(children.size == 2)
@ -257,7 +262,7 @@ class ServiceControlCliSuite extends KyuubiFunSuite with TestPrematureExit {
ServiceNodeInfo(s"/$uniqueNamespace", "", "localhost", 10001, Some(KYUUBI_VERSION))
)
testPrematureExit(args, getRenderedNodesInfoWithoutTitle(expectedNodes))
testPrematureExit(args, getRenderedNodesInfoWithoutTitle(expectedNodes, false))
}
}
@ -286,7 +291,7 @@ class ServiceControlCliSuite extends KyuubiFunSuite with TestPrematureExit {
ServiceNodeInfo(s"/$uniqueNamespace", "", "localhost", 10000, Some(KYUUBI_VERSION))
)
testPrematureExit(args, getRenderedNodesInfoWithoutTitle(expectedNodes))
testPrematureExit(args, getRenderedNodesInfoWithoutTitle(expectedNodes, false))
}
}
@ -317,7 +322,36 @@ class ServiceControlCliSuite extends KyuubiFunSuite with TestPrematureExit {
ServiceNodeInfo(s"/$uniqueNamespace", "", "localhost", 10000, Some(KYUUBI_VERSION))
)
testPrematureExit(args, getRenderedNodesInfoWithoutTitle(expectedDeletedNodes))
testPrematureExit(args, getRenderedNodesInfoWithoutTitle(expectedDeletedNodes, false))
}
}
test("test verbose output") {
val uniqueNamespace = getUniqueNamespace()
conf
.unset(KyuubiConf.SERVER_KEYTAB)
.unset(KyuubiConf.SERVER_PRINCIPAL)
.set(HA_ZK_QUORUM, zkServer.getConnectString)
.set(HA_ZK_NAMESPACE, uniqueNamespace)
.set(KyuubiConf.FRONTEND_BIND_PORT, 0)
withZkClient(conf) { framework =>
createZkServiceNode(conf, framework, uniqueNamespace, "localhost:10000")
createZkServiceNode(conf, framework, uniqueNamespace, "localhost:10001")
val args = Array(
"list", "server",
"--zk-quorum", zkServer.getConnectString,
"--namespace", uniqueNamespace,
"--verbose"
)
val expectedNodes = Seq(
ServiceNodeInfo(s"/$uniqueNamespace", "", "localhost", 10000, Some(KYUUBI_VERSION)),
ServiceNodeInfo(s"/$uniqueNamespace", "", "localhost", 10001, Some(KYUUBI_VERSION))
)
testPrematureExit(args, getRenderedNodesInfoWithoutTitle(expectedNodes, true))
}
}
}