[KYUUBI #630] [METRICS] Support expose Prometheus metrics
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/NetEase/kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> Support expose Prometheus metrics ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request Closes #630 from pan3793/prometheus. Closes #630 fff68b0 [Cheng Pan] nit 47c19eb [Cheng Pan] simplify 203e776 [Cheng Pan] address comments 10a877c [Cheng Pan] synchronized 26eb26e [Cheng Pan] simplify random port 827427c [Cheng Pan] fix import 476ee3a [Cheng Pan] address comments 9787c77 [Cheng Pan] update ut c8e1b0e [Cheng Pan] update license 2387a97 [Cheng Pan] [METRICS] Support expose Prometheus metrics Authored-by: Cheng Pan <379377944@qq.com> Signed-off-by: Kent Yao <yao@apache.org>
This commit is contained in:
Cheng Pan
Kent Yao
parent
d2be0f34e8
commit
0b4f4f242c
@ -220,6 +220,13 @@ com.fasterxml.jackson.core:jackson-annotations
|
||||
com.fasterxml.jackson.core:jackson-core
|
||||
com.fasterxml.jackson.core:jackson-databind
|
||||
org.javassist:javassist
|
||||
org.eclipse.jetty:jetty-http
|
||||
org.eclipse.jetty:jetty-io
|
||||
org.eclipse.jetty:jetty-security
|
||||
org.eclipse.jetty:jetty-server
|
||||
org.eclipse.jetty:jetty-servlet
|
||||
org.eclipse.jetty:jetty-util-ajax
|
||||
org.eclipse.jetty:jetty-util
|
||||
org.apache.thrift:libfb303
|
||||
org.apache.thrift:libthrift
|
||||
log4j:log4j
|
||||
@ -227,6 +234,10 @@ io.dropwizard.metrics:metrics-core
|
||||
io.dropwizard.metrics:metrics-jmx
|
||||
io.dropwizard.metrics:metrics-json
|
||||
io.dropwizard.metrics:metrics-jvm
|
||||
io.prometheus:simpleclient
|
||||
io.prometheus:simpleclient_common
|
||||
io.prometheus:simpleclient_dropwizard
|
||||
io.prometheus:simpleclient_servlet
|
||||
org.apache.zookeeper:zookeeper
|
||||
spark-*-bin-*.tgz
|
||||
|
||||
@ -234,6 +245,10 @@ BSD 3-Clause
|
||||
------------
|
||||
org.scala-lang:scala-library
|
||||
|
||||
Eclipse Public License (EPL) 2.0
|
||||
--------------------------------
|
||||
jakarta.servlet:jakarta.servlet-api https://projects.eclipse.org/projects/ee4j.servlet
|
||||
|
||||
MIT License
|
||||
-----------
|
||||
org.slf4j:slf4j-api
|
||||
|
||||
@ -30,8 +30,16 @@ htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
|
||||
jackson-annotations/2.11.4//jackson-annotations-2.11.4.jar
|
||||
jackson-core/2.11.4//jackson-core-2.11.4.jar
|
||||
jackson-databind/2.11.4//jackson-databind-2.11.4.jar
|
||||
jakarta.servlet-api/4.0.4//jakarta.servlet-api-4.0.4.jar
|
||||
javassist/3.18.1-GA//javassist-3.18.1-GA.jar
|
||||
jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
|
||||
jetty-http/9.4.40.v20210413//jetty-http-9.4.40.v20210413.jar
|
||||
jetty-io/9.4.40.v20210413//jetty-io-9.4.40.v20210413.jar
|
||||
jetty-security/9.4.40.v20210413//jetty-security-9.4.40.v20210413.jar
|
||||
jetty-server/9.4.40.v20210413//jetty-server-9.4.40.v20210413.jar
|
||||
jetty-servlet/9.4.40.v20210413//jetty-servlet-9.4.40.v20210413.jar
|
||||
jetty-util-ajax/9.4.40.v20210413//jetty-util-ajax-9.4.40.v20210413.jar
|
||||
jetty-util/9.4.40.v20210413//jetty-util-9.4.40.v20210413.jar
|
||||
libfb303/0.9.3//libfb303-0.9.3.jar
|
||||
libthrift/0.9.3//libthrift-0.9.3.jar
|
||||
log4j/1.2.17//log4j-1.2.17.jar
|
||||
@ -40,6 +48,10 @@ metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar
|
||||
metrics-json/4.1.1//metrics-json-4.1.1.jar
|
||||
metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar
|
||||
scala-library/2.12.10//scala-library-2.12.10.jar
|
||||
simpleclient/0.10.0//simpleclient-0.10.0.jar
|
||||
simpleclient_common/0.10.0//simpleclient_common-0.10.0.jar
|
||||
simpleclient_dropwizard/0.10.0//simpleclient_dropwizard-0.10.0.jar
|
||||
simpleclient_servlet/0.10.0//simpleclient_servlet-0.10.0.jar
|
||||
slf4j-api/1.7.30//slf4j-api-1.7.30.jar
|
||||
slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
|
||||
zookeeper/3.4.14//zookeeper-3.4.14.jar
|
||||
|
||||
@ -194,7 +194,9 @@ kyuubi\.metrics<br>\.console\.interval|<div style='width: 65pt;word-wrap: break-
|
||||
kyuubi\.metrics<br>\.enabled|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>true</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Set to true to enable kyuubi metrics system</div>|<div style='width: 30pt'>boolean</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics\.json<br>\.interval|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT5S</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>How often should report metrics to json file</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics\.json<br>\.location|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>metrics</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Where the json metrics file located</div>|<div style='width: 30pt'>string</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics<br>\.reporters|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>JSON</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>A comma separated list for all metrics reporters<ul> <li>CONSOLE - ConsoleReporter which outputs measurements to CONSOLE periodically.</li> <li>JMX - JmxReporter which listens for new metrics and exposes them as MBeans.</li> <li>JSON - JsonReporter which outputs measurements to json file periodically.</li> <li>SLF4J - Slf4jReporter which outputs measurements to system log periodically.</li></ul></div>|<div style='width: 30pt'>seq</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics<br>\.prometheus\.path|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>/metrics</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>URI context path of prometheus metrics HTTP server</div>|<div style='width: 30pt'>string</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics<br>\.prometheus\.port|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>10019</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Prometheus metrics HTTP server port</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics<br>\.reporters|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>JSON</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>A comma separated list for all metrics reporters<ul> <li>CONSOLE - ConsoleReporter which outputs measurements to CONSOLE periodically.</li> <li>JMX - JmxReporter which listens for new metrics and exposes them as MBeans.</li> <li>JSON - JsonReporter which outputs measurements to json file periodically.</li> <li>PROMETHEUS - PrometheusReporter which exposes metrics in prometheus format.</li> <li>SLF4J - Slf4jReporter which outputs measurements to system log periodically.</li></ul></div>|<div style='width: 30pt'>seq</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics\.slf4j<br>\.interval|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT5S</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>How often should report metrics to SLF4J logger</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
|
||||
|
||||
|
||||
@ -8,13 +8,16 @@ The metrics provide instrumentation for specific activities and Kyuubi server.
|
||||
|
||||
The metrics system is configured via `$KYUUBI_HOME/conf/kyuubi-defaults.conf`.
|
||||
|
||||
Key | Default | Meaning | Since
|
||||
--- | --- | --- | ---
|
||||
kyuubi\.metrics<br>\.enabled|<div style='width: 80pt;word-wrap: break-word;white-space: normal'>true</div>|<div style='width: 200pt;word-wrap: break-word;white-space: normal'>Set to true to enable kyuubi metrics system</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics\.json<br>\.report\.location|<div style='width: 80pt;word-wrap: break-word;white-space: normal'>metrics</div>|<div style='width: 200pt;word-wrap: break-word;white-space: normal'>Where the json metrics file located</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics\.report<br>\.interval|<div style='width: 80pt;word-wrap: break-word;white-space: normal'>PT5S</div>|<div style='width: 200pt;word-wrap: break-word;white-space: normal'>How often should report metrics to json/console. no effect on JMX</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics<br>\.reporters|<div style='width: 80pt;word-wrap: break-word;white-space: normal'>JSON</div>|<div style='width: 200pt;word-wrap: break-word;white-space: normal'>A comma separated list for all metrics reporters<ul> <li>JSON - default reporter which outputs measurements to json file periodically</li> <li>CONSOLE - ConsoleReporter which outputs measurements to CONSOLE.</li> <li>SLF4J - Slf4jReporter which outputs measurements to system log.</li> <li>JMX - JmxReporter which listens for new metrics and exposes them as namespaced MBeans.</li> </ul></div>|<div style='width: 20pt'>1.2.0</div>
|
||||
|
||||
Key | Default | Meaning | Type | Since
|
||||
--- | --- | --- | --- | ---
|
||||
kyuubi\.metrics<br>\.enabled|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>true</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Set to true to enable kyuubi metrics system</div>|<div style='width: 30pt'>boolean</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics<br>\.reporters|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>JSON</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>A comma separated list for all metrics reporters<ul> <li>CONSOLE - ConsoleReporter which outputs measurements to CONSOLE periodically.</li> <li>JMX - JmxReporter which listens for new metrics and exposes them as MBeans.</li> <li>JSON - JsonReporter which outputs measurements to json file periodically.</li> <li>PROMETHEUS - PrometheusReporter which exposes metrics in prometheus format.</li> <li>SLF4J - Slf4jReporter which outputs measurements to system log periodically.</li></ul></div>|<div style='width: 30pt'>seq</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics<br>\.console\.interval|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT5S</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>How often should report metrics to console</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics\.json<br>\.interval|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT5S</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>How often should report metrics to json file</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics\.json<br>\.location|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>metrics</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Where the json metrics file located</div>|<div style='width: 30pt'>string</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics<br>\.prometheus\.path|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>/metrics</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>URI context path of prometheus metrics HTTP server</div>|<div style='width: 30pt'>string</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics<br>\.prometheus\.port|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>10019</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Prometheus metrics HTTP server port</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
kyuubi\.metrics\.slf4j<br>\.interval|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT5S</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>How often should report metrics to SLF4J logger</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.2.0</div>
|
||||
|
||||
## Metrics
|
||||
|
||||
|
||||
@ -58,6 +58,31 @@
|
||||
<artifactId>metrics-jmx</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.prometheus</groupId>
|
||||
<artifactId>simpleclient_dropwizard</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.prometheus</groupId>
|
||||
<artifactId>simpleclient_servlet</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>jakarta.servlet</groupId>
|
||||
<artifactId>jakarta.servlet-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-server</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-servlet</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
@ -71,6 +96,12 @@
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-client</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
|
||||
@ -39,6 +39,7 @@ object MetricsConf {
|
||||
" <li>CONSOLE - ConsoleReporter which outputs measurements to CONSOLE periodically.</li>" +
|
||||
" <li>JMX - JmxReporter which listens for new metrics and exposes them as MBeans.</li> " +
|
||||
" <li>JSON - JsonReporter which outputs measurements to json file periodically.</li>" +
|
||||
" <li>PROMETHEUS - PrometheusReporter which exposes metrics in prometheus format.</li>" +
|
||||
" <li>SLF4J - Slf4jReporter which outputs measurements to system log periodically.</li>" +
|
||||
"</ul>")
|
||||
.version("1.2.0")
|
||||
@ -65,6 +66,20 @@ object MetricsConf {
|
||||
.timeConf
|
||||
.createWithDefault(Duration.ofSeconds(5).toMillis)
|
||||
|
||||
val METRICS_PROMETHEUS_PORT: ConfigEntry[Int] = buildConf("metrics.prometheus.port")
|
||||
.doc("Prometheus metrics HTTP server port")
|
||||
.version("1.2.0")
|
||||
.intConf
|
||||
.checkValue(p => p == 0 || (p > 1024 && p < 65535), "Invalid Port number")
|
||||
.createWithDefault(10019)
|
||||
|
||||
val METRICS_PROMETHEUS_PATH: ConfigEntry[String] = buildConf("metrics.prometheus.path")
|
||||
.doc("URI context path of prometheus metrics HTTP server")
|
||||
.version("1.2.0")
|
||||
.stringConf
|
||||
.checkValue(path => path.startsWith("/"), "Context path must start with '/'")
|
||||
.createWithDefault("/metrics")
|
||||
|
||||
val METRICS_SLF4J_INTERVAL: ConfigEntry[Long] = buildConf("metrics.slf4j.interval")
|
||||
.doc("How often should report metrics to SLF4J logger")
|
||||
.version("1.2.0")
|
||||
|
||||
@ -81,6 +81,7 @@ class MetricsSystem extends CompositeService("MetricsSystem") {
|
||||
case SLF4J => addService(new Slf4jReporterService(registry))
|
||||
case CONSOLE => addService(new ConsoleReporterService(registry))
|
||||
case JMX => addService(new JMXReporterService(registry))
|
||||
case PROMETHEUS => addService(new PrometheusReporterService(registry))
|
||||
}
|
||||
super.initialize(conf)
|
||||
}
|
||||
|
||||
@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kyuubi.metrics
|
||||
|
||||
import com.codahale.metrics.MetricRegistry
|
||||
import com.google.common.annotations.VisibleForTesting
|
||||
import io.prometheus.client.CollectorRegistry
|
||||
import io.prometheus.client.dropwizard.DropwizardExports
|
||||
import io.prometheus.client.exporter.MetricsServlet
|
||||
import org.eclipse.jetty.server.Server
|
||||
import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
|
||||
|
||||
import org.apache.kyuubi.KyuubiException
|
||||
import org.apache.kyuubi.config.KyuubiConf
|
||||
import org.apache.kyuubi.service.AbstractService
|
||||
|
||||
class PrometheusReporterService(registry: MetricRegistry)
|
||||
extends AbstractService("PrometheusReporterService") {
|
||||
|
||||
private val bridgeRegistry = new CollectorRegistry
|
||||
|
||||
@VisibleForTesting
|
||||
private[metrics] var httpServer: Server = _
|
||||
@volatile protected var isStarted = false
|
||||
|
||||
override def initialize(conf: KyuubiConf): Unit = {
|
||||
val port = conf.get(MetricsConf.METRICS_PROMETHEUS_PORT)
|
||||
val contextPath = conf.get(MetricsConf.METRICS_PROMETHEUS_PATH)
|
||||
httpServer = new Server(port)
|
||||
val context = new ServletContextHandler
|
||||
context.setContextPath("/")
|
||||
httpServer.setHandler(context)
|
||||
|
||||
new DropwizardExports(registry).register(bridgeRegistry)
|
||||
val metricsServlet = new MetricsServlet(bridgeRegistry)
|
||||
context.addServlet(new ServletHolder(metricsServlet), contextPath)
|
||||
|
||||
super.initialize(conf)
|
||||
}
|
||||
|
||||
override def start(): Unit = synchronized {
|
||||
if (!isStarted) {
|
||||
try {
|
||||
httpServer.start()
|
||||
info(s"Prometheus metrics HTTP server has started at ${httpServer.getURI}.")
|
||||
} catch {
|
||||
case rethrow: Exception =>
|
||||
stopHttpServer()
|
||||
throw new KyuubiException("Cannot start prometheus metrics HTTP server", rethrow)
|
||||
}
|
||||
isStarted = true
|
||||
}
|
||||
super.start()
|
||||
}
|
||||
|
||||
override def stop(): Unit = synchronized {
|
||||
if (isStarted) {
|
||||
stopHttpServer()
|
||||
isStarted = false
|
||||
}
|
||||
super.stop()
|
||||
}
|
||||
|
||||
private def stopHttpServer(): Unit = {
|
||||
if (httpServer != null) {
|
||||
try {
|
||||
httpServer.stop()
|
||||
info("Prometheus metrics HTTP server has stopped.")
|
||||
} catch {
|
||||
case err: Exception => error("Cannot safely stop prometheus metrics HTTP server", err)
|
||||
} finally {
|
||||
httpServer = null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -21,5 +21,5 @@ object ReporterType extends Enumeration {
|
||||
|
||||
type ReporterType = Value
|
||||
|
||||
val CONSOLE, JMX, JSON, SLF4J = Value
|
||||
val CONSOLE, JMX, JSON, PROMETHEUS, SLF4J = Value
|
||||
}
|
||||
|
||||
@ -23,12 +23,15 @@ import java.time.Duration
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.concurrent.duration._
|
||||
|
||||
import org.eclipse.jetty.client.HttpClient
|
||||
import org.eclipse.jetty.client.api.ContentResponse
|
||||
|
||||
import org.apache.kyuubi.{KyuubiFunSuite, Utils}
|
||||
import org.apache.kyuubi.config.KyuubiConf
|
||||
|
||||
class MetricsSystemSuite extends KyuubiFunSuite {
|
||||
|
||||
def checkMetrics(path: Path, searchKey: String): Unit = {
|
||||
def checkJsonFileMetrics(path: Path, searchKey: String): Unit = {
|
||||
eventually(timeout(10.seconds), interval(500.milliseconds)) {
|
||||
val reader = Files.newBufferedReader(path)
|
||||
val logs = new java.util.ArrayList[String]
|
||||
@ -41,26 +44,53 @@ class MetricsSystemSuite extends KyuubiFunSuite {
|
||||
}
|
||||
}
|
||||
|
||||
test("MetricsSystem") {
|
||||
test("metrics - PrometheusReporter") {
|
||||
val testContextPath = "/prometheus-metrics"
|
||||
|
||||
val conf = KyuubiConf()
|
||||
.set(MetricsConf.METRICS_ENABLED, true)
|
||||
.set(MetricsConf.METRICS_REPORTERS, Seq(ReporterType.PROMETHEUS.toString))
|
||||
.set(MetricsConf.METRICS_PROMETHEUS_PORT, 0) // random port
|
||||
.set(MetricsConf.METRICS_PROMETHEUS_PATH, testContextPath)
|
||||
val metricsSystem = new MetricsSystem()
|
||||
metricsSystem.initialize(conf)
|
||||
metricsSystem.start()
|
||||
|
||||
metricsSystem.registerGauge(MetricsConstants.CONN_OPEN, 2021, 0)
|
||||
|
||||
val prometheusHttpServer = metricsSystem.getServices.head
|
||||
.asInstanceOf[PrometheusReporterService].httpServer
|
||||
|
||||
val client: HttpClient = new HttpClient
|
||||
client.start()
|
||||
val res: ContentResponse = client.GET(prometheusHttpServer.getURI.resolve(testContextPath))
|
||||
assert(res.getContentAsString.contains("PS_MarkSweep_count"))
|
||||
assert(res.getContentAsString.contains("kyuubi_connection_opened 2021.0"))
|
||||
client.stop()
|
||||
|
||||
metricsSystem.stop()
|
||||
}
|
||||
|
||||
test("metrics - other reporters") {
|
||||
val reportPath = Utils.createTempDir()
|
||||
val conf = KyuubiConf()
|
||||
.set(MetricsConf.METRICS_ENABLED, true)
|
||||
.set(MetricsConf.METRICS_REPORTERS, ReporterType.values.map(_.toString).toSeq)
|
||||
.set(MetricsConf.METRICS_REPORTERS,
|
||||
ReporterType.values.filterNot(_ == ReporterType.PROMETHEUS).map(_.toString).toSeq)
|
||||
.set(MetricsConf.METRICS_JSON_INTERVAL, Duration.ofSeconds(1).toMillis)
|
||||
.set(MetricsConf.METRICS_JSON_LOCATION, reportPath.toString)
|
||||
val metricsSystem = new MetricsSystem()
|
||||
metricsSystem.initialize(conf)
|
||||
metricsSystem.start()
|
||||
val reportFile = Paths.get(reportPath.toString, "report.json")
|
||||
checkMetrics(reportFile, "PS-MarkSweep.count")
|
||||
checkJsonFileMetrics(reportFile, "PS-MarkSweep.count")
|
||||
metricsSystem.incAndGetCount(MetricsConstants.STATEMENT_TOTAL)
|
||||
|
||||
checkMetrics(reportFile, MetricsConstants.STATEMENT_TOTAL)
|
||||
checkJsonFileMetrics(reportFile, MetricsConstants.STATEMENT_TOTAL)
|
||||
metricsSystem.decAndGetCount(MetricsConstants.STATEMENT_TOTAL)
|
||||
metricsSystem.registerGauge(MetricsConstants.CONN_OPEN, 20181117, 0)
|
||||
checkMetrics(reportFile, MetricsConstants.CONN_OPEN)
|
||||
checkMetrics(reportFile, "20181117")
|
||||
checkJsonFileMetrics(reportFile, MetricsConstants.CONN_OPEN)
|
||||
checkJsonFileMetrics(reportFile, "20181117")
|
||||
metricsSystem.stop()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
277
licenses-binary/LICENSE-jakarta-servlet-api.txt
Normal file
277
licenses-binary/LICENSE-jakarta-servlet-api.txt
Normal file
@ -0,0 +1,277 @@
|
||||
Eclipse Public License - v 2.0
|
||||
|
||||
THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE
|
||||
PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION
|
||||
OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT.
|
||||
|
||||
1. DEFINITIONS
|
||||
|
||||
"Contribution" means:
|
||||
|
||||
a) in the case of the initial Contributor, the initial content
|
||||
Distributed under this Agreement, and
|
||||
|
||||
b) in the case of each subsequent Contributor:
|
||||
i) changes to the Program, and
|
||||
ii) additions to the Program;
|
||||
where such changes and/or additions to the Program originate from
|
||||
and are Distributed by that particular Contributor. A Contribution
|
||||
"originates" from a Contributor if it was added to the Program by
|
||||
such Contributor itself or anyone acting on such Contributor's behalf.
|
||||
Contributions do not include changes or additions to the Program that
|
||||
are not Modified Works.
|
||||
|
||||
"Contributor" means any person or entity that Distributes the Program.
|
||||
|
||||
"Licensed Patents" mean patent claims licensable by a Contributor which
|
||||
are necessarily infringed by the use or sale of its Contribution alone
|
||||
or when combined with the Program.
|
||||
|
||||
"Program" means the Contributions Distributed in accordance with this
|
||||
Agreement.
|
||||
|
||||
"Recipient" means anyone who receives the Program under this Agreement
|
||||
or any Secondary License (as applicable), including Contributors.
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source Code or other
|
||||
form, that is based on (or derived from) the Program and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship.
|
||||
|
||||
"Modified Works" shall mean any work in Source Code or other form that
|
||||
results from an addition to, deletion from, or modification of the
|
||||
contents of the Program, including, for purposes of clarity any new file
|
||||
in Source Code form that contains any contents of the Program. Modified
|
||||
Works shall not include works that contain only declarations,
|
||||
interfaces, types, classes, structures, or files of the Program solely
|
||||
in each case in order to link to, bind by name, or subclass the Program
|
||||
or Modified Works thereof.
|
||||
|
||||
"Distribute" means the acts of a) distributing or b) making available
|
||||
in any manner that enables the transfer of a copy.
|
||||
|
||||
"Source Code" means the form of a Program preferred for making
|
||||
modifications, including but not limited to software source code,
|
||||
documentation source, and configuration files.
|
||||
|
||||
"Secondary License" means either the GNU General Public License,
|
||||
Version 2.0, or any later versions of that license, including any
|
||||
exceptions or additional permissions as identified by the initial
|
||||
Contributor.
|
||||
|
||||
2. GRANT OF RIGHTS
|
||||
|
||||
a) Subject to the terms of this Agreement, each Contributor hereby
|
||||
grants Recipient a non-exclusive, worldwide, royalty-free copyright
|
||||
license to reproduce, prepare Derivative Works of, publicly display,
|
||||
publicly perform, Distribute and sublicense the Contribution of such
|
||||
Contributor, if any, and such Derivative Works.
|
||||
|
||||
b) Subject to the terms of this Agreement, each Contributor hereby
|
||||
grants Recipient a non-exclusive, worldwide, royalty-free patent
|
||||
license under Licensed Patents to make, use, sell, offer to sell,
|
||||
import and otherwise transfer the Contribution of such Contributor,
|
||||
if any, in Source Code or other form. This patent license shall
|
||||
apply to the combination of the Contribution and the Program if, at
|
||||
the time the Contribution is added by the Contributor, such addition
|
||||
of the Contribution causes such combination to be covered by the
|
||||
Licensed Patents. The patent license shall not apply to any other
|
||||
combinations which include the Contribution. No hardware per se is
|
||||
licensed hereunder.
|
||||
|
||||
c) Recipient understands that although each Contributor grants the
|
||||
licenses to its Contributions set forth herein, no assurances are
|
||||
provided by any Contributor that the Program does not infringe the
|
||||
patent or other intellectual property rights of any other entity.
|
||||
Each Contributor disclaims any liability to Recipient for claims
|
||||
brought by any other entity based on infringement of intellectual
|
||||
property rights or otherwise. As a condition to exercising the
|
||||
rights and licenses granted hereunder, each Recipient hereby
|
||||
assumes sole responsibility to secure any other intellectual
|
||||
property rights needed, if any. For example, if a third party
|
||||
patent license is required to allow Recipient to Distribute the
|
||||
Program, it is Recipient's responsibility to acquire that license
|
||||
before distributing the Program.
|
||||
|
||||
d) Each Contributor represents that to its knowledge it has
|
||||
sufficient copyright rights in its Contribution, if any, to grant
|
||||
the copyright license set forth in this Agreement.
|
||||
|
||||
e) Notwithstanding the terms of any Secondary License, no
|
||||
Contributor makes additional grants to any Recipient (other than
|
||||
those set forth in this Agreement) as a result of such Recipient's
|
||||
receipt of the Program under the terms of a Secondary License
|
||||
(if permitted under the terms of Section 3).
|
||||
|
||||
3. REQUIREMENTS
|
||||
|
||||
3.1 If a Contributor Distributes the Program in any form, then:
|
||||
|
||||
a) the Program must also be made available as Source Code, in
|
||||
accordance with section 3.2, and the Contributor must accompany
|
||||
the Program with a statement that the Source Code for the Program
|
||||
is available under this Agreement, and informs Recipients how to
|
||||
obtain it in a reasonable manner on or through a medium customarily
|
||||
used for software exchange; and
|
||||
|
||||
b) the Contributor may Distribute the Program under a license
|
||||
different than this Agreement, provided that such license:
|
||||
i) effectively disclaims on behalf of all other Contributors all
|
||||
warranties and conditions, express and implied, including
|
||||
warranties or conditions of title and non-infringement, and
|
||||
implied warranties or conditions of merchantability and fitness
|
||||
for a particular purpose;
|
||||
|
||||
ii) effectively excludes on behalf of all other Contributors all
|
||||
liability for damages, including direct, indirect, special,
|
||||
incidental and consequential damages, such as lost profits;
|
||||
|
||||
iii) does not attempt to limit or alter the recipients' rights
|
||||
in the Source Code under section 3.2; and
|
||||
|
||||
iv) requires any subsequent distribution of the Program by any
|
||||
party to be under a license that satisfies the requirements
|
||||
of this section 3.
|
||||
|
||||
3.2 When the Program is Distributed as Source Code:
|
||||
|
||||
a) it must be made available under this Agreement, or if the
|
||||
Program (i) is combined with other material in a separate file or
|
||||
files made available under a Secondary License, and (ii) the initial
|
||||
Contributor attached to the Source Code the notice described in
|
||||
Exhibit A of this Agreement, then the Program may be made available
|
||||
under the terms of such Secondary Licenses, and
|
||||
|
||||
b) a copy of this Agreement must be included with each copy of
|
||||
the Program.
|
||||
|
||||
3.3 Contributors may not remove or alter any copyright, patent,
|
||||
trademark, attribution notices, disclaimers of warranty, or limitations
|
||||
of liability ("notices") contained within the Program from any copy of
|
||||
the Program which they Distribute, provided that Contributors may add
|
||||
their own appropriate notices.
|
||||
|
||||
4. COMMERCIAL DISTRIBUTION
|
||||
|
||||
Commercial distributors of software may accept certain responsibilities
|
||||
with respect to end users, business partners and the like. While this
|
||||
license is intended to facilitate the commercial use of the Program,
|
||||
the Contributor who includes the Program in a commercial product
|
||||
offering should do so in a manner which does not create potential
|
||||
liability for other Contributors. Therefore, if a Contributor includes
|
||||
the Program in a commercial product offering, such Contributor
|
||||
("Commercial Contributor") hereby agrees to defend and indemnify every
|
||||
other Contributor ("Indemnified Contributor") against any losses,
|
||||
damages and costs (collectively "Losses") arising from claims, lawsuits
|
||||
and other legal actions brought by a third party against the Indemnified
|
||||
Contributor to the extent caused by the acts or omissions of such
|
||||
Commercial Contributor in connection with its distribution of the Program
|
||||
in a commercial product offering. The obligations in this section do not
|
||||
apply to any claims or Losses relating to any actual or alleged
|
||||
intellectual property infringement. In order to qualify, an Indemnified
|
||||
Contributor must: a) promptly notify the Commercial Contributor in
|
||||
writing of such claim, and b) allow the Commercial Contributor to control,
|
||||
and cooperate with the Commercial Contributor in, the defense and any
|
||||
related settlement negotiations. The Indemnified Contributor may
|
||||
participate in any such claim at its own expense.
|
||||
|
||||
For example, a Contributor might include the Program in a commercial
|
||||
product offering, Product X. That Contributor is then a Commercial
|
||||
Contributor. If that Commercial Contributor then makes performance
|
||||
claims, or offers warranties related to Product X, those performance
|
||||
claims and warranties are such Commercial Contributor's responsibility
|
||||
alone. Under this section, the Commercial Contributor would have to
|
||||
defend claims against the other Contributors related to those performance
|
||||
claims and warranties, and if a court requires any other Contributor to
|
||||
pay any damages as a result, the Commercial Contributor must pay
|
||||
those damages.
|
||||
|
||||
5. NO WARRANTY
|
||||
|
||||
EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT
|
||||
PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS"
|
||||
BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR
|
||||
IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF
|
||||
TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR
|
||||
PURPOSE. Each Recipient is solely responsible for determining the
|
||||
appropriateness of using and distributing the Program and assumes all
|
||||
risks associated with its exercise of rights under this Agreement,
|
||||
including but not limited to the risks and costs of program errors,
|
||||
compliance with applicable laws, damage to or loss of data, programs
|
||||
or equipment, and unavailability or interruption of operations.
|
||||
|
||||
6. DISCLAIMER OF LIABILITY
|
||||
|
||||
EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT
|
||||
PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS
|
||||
SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST
|
||||
PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE
|
||||
EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGES.
|
||||
|
||||
7. GENERAL
|
||||
|
||||
If any provision of this Agreement is invalid or unenforceable under
|
||||
applicable law, it shall not affect the validity or enforceability of
|
||||
the remainder of the terms of this Agreement, and without further
|
||||
action by the parties hereto, such provision shall be reformed to the
|
||||
minimum extent necessary to make such provision valid and enforceable.
|
||||
|
||||
If Recipient institutes patent litigation against any entity
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that the
|
||||
Program itself (excluding combinations of the Program with other software
|
||||
or hardware) infringes such Recipient's patent(s), then such Recipient's
|
||||
rights granted under Section 2(b) shall terminate as of the date such
|
||||
litigation is filed.
|
||||
|
||||
All Recipient's rights under this Agreement shall terminate if it
|
||||
fails to comply with any of the material terms or conditions of this
|
||||
Agreement and does not cure such failure in a reasonable period of
|
||||
time after becoming aware of such noncompliance. If all Recipient's
|
||||
rights under this Agreement terminate, Recipient agrees to cease use
|
||||
and distribution of the Program as soon as reasonably practicable.
|
||||
However, Recipient's obligations under this Agreement and any licenses
|
||||
granted by Recipient relating to the Program shall continue and survive.
|
||||
|
||||
Everyone is permitted to copy and distribute copies of this Agreement,
|
||||
but in order to avoid inconsistency the Agreement is copyrighted and
|
||||
may only be modified in the following manner. The Agreement Steward
|
||||
reserves the right to publish new versions (including revisions) of
|
||||
this Agreement from time to time. No one other than the Agreement
|
||||
Steward has the right to modify this Agreement. The Eclipse Foundation
|
||||
is the initial Agreement Steward. The Eclipse Foundation may assign the
|
||||
responsibility to serve as the Agreement Steward to a suitable separate
|
||||
entity. Each new version of the Agreement will be given a distinguishing
|
||||
version number. The Program (including Contributions) may always be
|
||||
Distributed subject to the version of the Agreement under which it was
|
||||
received. In addition, after a new version of the Agreement is published,
|
||||
Contributor may elect to Distribute the Program (including its
|
||||
Contributions) under the new version.
|
||||
|
||||
Except as expressly stated in Sections 2(a) and 2(b) above, Recipient
|
||||
receives no rights or licenses to the intellectual property of any
|
||||
Contributor under this Agreement, whether expressly, by implication,
|
||||
estoppel or otherwise. All rights in the Program not expressly granted
|
||||
under this Agreement are reserved. Nothing in this Agreement is intended
|
||||
to be enforceable by any entity that is not a Contributor or Recipient.
|
||||
No third-party beneficiary rights are created under this Agreement.
|
||||
|
||||
Exhibit A - Form of Secondary Licenses Notice
|
||||
|
||||
"This Source Code may also be made available under the following
|
||||
Secondary Licenses when the conditions for such availability set forth
|
||||
in the Eclipse Public License, v. 2.0 are satisfied: {name license(s),
|
||||
version(s), and exceptions or additional permissions here}."
|
||||
|
||||
Simply including a copy of this Agreement, including this Exhibit A
|
||||
is not sufficient to license the Source Code under Secondary Licenses.
|
||||
|
||||
If it is not possible or desirable to put the notice in a particular
|
||||
file, then You may include the notice in a location (such as a LICENSE
|
||||
file in a relevant directory) where a recipient would be likely to
|
||||
look for such a notice.
|
||||
|
||||
You may add additional accurate notices of copyright ownership.
|
||||
56
pom.xml
56
pom.xml
@ -73,7 +73,10 @@
|
||||
<hadoop.binary.version>2.7</hadoop.binary.version>
|
||||
<hive.version>2.3.7</hive.version>
|
||||
<jackson.version>2.11.4</jackson.version>
|
||||
<jakarta.servlet-api.version>4.0.4</jakarta.servlet-api.version>
|
||||
<jetty.version>9.4.40.v20210413</jetty.version>
|
||||
<ldapsdk.version>5.1.4</ldapsdk.version>
|
||||
<prometheus.version>0.10.0</prometheus.version>
|
||||
<spark.version>3.0.2</spark.version>
|
||||
<spark.archive.name>spark-${spark.version}-bin-hadoop${hadoop.binary.version}.tgz</spark.archive.name>
|
||||
<spark.archive.mirror>https://archive.apache.org/dist/spark/spark-${spark.version}</spark.archive.mirror>
|
||||
@ -999,6 +1002,58 @@
|
||||
<version>${jackson.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!--
|
||||
compatible with dropwizard 3.1.x, 3.2.x, 4.0.x, 4.1.x, 4.2.x
|
||||
see https://github.com/prometheus/client_java/issues/662
|
||||
-->
|
||||
<dependency>
|
||||
<groupId>io.prometheus</groupId>
|
||||
<artifactId>simpleclient_dropwizard</artifactId>
|
||||
<version>${prometheus.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>io.dropwizard.metrics</groupId>
|
||||
<artifactId>metrics-core</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.prometheus</groupId>
|
||||
<artifactId>simpleclient_servlet</artifactId>
|
||||
<version>${prometheus.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>jakarta.servlet</groupId>
|
||||
<artifactId>jakarta.servlet-api</artifactId>
|
||||
<version>${jakarta.servlet-api.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-client</artifactId>
|
||||
<version>${jetty.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-server</artifactId>
|
||||
<version>${jetty.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>javax.servlet</groupId>
|
||||
<artifactId>javax.servlet-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-servlet</artifactId>
|
||||
<version>${jetty.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.scalatest</groupId>
|
||||
<artifactId>scalatest_${scala.binary.version}</artifactId>
|
||||
@ -1379,6 +1434,7 @@
|
||||
<exclude>**/dependency-reduced-pom.xml</exclude>
|
||||
<exclude>**/scalastyle-output.xml</exclude>
|
||||
<exclude>docs/**</exclude>
|
||||
<exclude>build/apache-maven-*/**</exclude>
|
||||
<exclude>build/scala-*/**</exclude>
|
||||
<exclude>**/**/operation_logs/**/**</exclude>
|
||||
<exclude>**/*.output.schema</exclude>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user