From 4bb67bdf3bcabd135102bd85c8992e9b52ba9356 Mon Sep 17 00:00:00 2001 From: hezhao2 Date: Mon, 16 Oct 2023 11:46:56 +0800 Subject: [PATCH] [KYUUBI #5381] Change the default metrics reporter to Prometheus ### _Why are the changes needed?_ Close #5381 change default metrics reporter to prometheus since Kyuubi 1.8 ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request ### _Was this patch authored or co-authored using generative AI tooling?_ Closes #5344 from zhaohehuhu/Improvement-0928. Closes #5381 84f4c8208 [hezhao2] reset METRICS_REPORTERS for test case b9ee5f711 [Cheng Pan] Update kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala 86165a6fe [Cheng Pan] Update kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala a3605b626 [hezhao2] set METRICS_PROMETHEUS_PORT to 0 for test cases f1a4d2861 [hezhao2] restore version number for kyuubi.metrics.reporters in doc dae40e1a2 [hezhao2] change default metrics reporter to prometheus Lead-authored-by: hezhao2 Co-authored-by: Cheng Pan Signed-off-by: Cheng Pan --- docs/configuration/settings.md | 20 +++++++++---------- docs/deployment/migration-guide.md | 3 +++ .../apache/kyuubi/metrics/MetricsConf.scala | 2 +- .../ServerJsonLoggingEventHandlerSuite.scala | 2 ++ 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index 832099764..2869a59cd 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -352,16 +352,16 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co ### Metrics -| Key | Default | Meaning | Type | Since | -|---------------------------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| -| kyuubi.metrics.console.interval | PT5S | How often should report metrics to console | duration | 1.2.0 | -| kyuubi.metrics.enabled | true | Set to true to enable kyuubi metrics system | boolean | 1.2.0 | -| kyuubi.metrics.json.interval | PT5S | How often should report metrics to JSON file | duration | 1.2.0 | -| kyuubi.metrics.json.location | metrics | Where the JSON metrics file located | string | 1.2.0 | -| kyuubi.metrics.prometheus.path | /metrics | URI context path of prometheus metrics HTTP server | string | 1.2.0 | -| kyuubi.metrics.prometheus.port | 10019 | Prometheus metrics HTTP server port | int | 1.2.0 | -| kyuubi.metrics.reporters | JSON | A comma-separated list for all metrics reporters
  • CONSOLE - ConsoleReporter which outputs measurements to CONSOLE periodically.
  • JMX - JmxReporter which listens for new metrics and exposes them as MBeans.
  • JSON - JsonReporter which outputs measurements to json file periodically.
  • PROMETHEUS - PrometheusReporter which exposes metrics in Prometheus format.
  • SLF4J - Slf4jReporter which outputs measurements to system log periodically.
| set | 1.2.0 | -| kyuubi.metrics.slf4j.interval | PT5S | How often should report metrics to SLF4J logger | duration | 1.2.0 | +| Key | Default | Meaning | Type | Since | +|---------------------------------|------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| +| kyuubi.metrics.console.interval | PT5S | How often should report metrics to console | duration | 1.2.0 | +| kyuubi.metrics.enabled | true | Set to true to enable kyuubi metrics system | boolean | 1.2.0 | +| kyuubi.metrics.json.interval | PT5S | How often should report metrics to JSON file | duration | 1.2.0 | +| kyuubi.metrics.json.location | metrics | Where the JSON metrics file located | string | 1.2.0 | +| kyuubi.metrics.prometheus.path | /metrics | URI context path of prometheus metrics HTTP server | string | 1.2.0 | +| kyuubi.metrics.prometheus.port | 10019 | Prometheus metrics HTTP server port | int | 1.2.0 | +| kyuubi.metrics.reporters | PROMETHEUS | A comma-separated list for all metrics reporters
  • CONSOLE - ConsoleReporter which outputs measurements to CONSOLE periodically.
  • JMX - JmxReporter which listens for new metrics and exposes them as MBeans.
  • JSON - JsonReporter which outputs measurements to json file periodically.
  • PROMETHEUS - PrometheusReporter which exposes metrics in Prometheus format.
  • SLF4J - Slf4jReporter which outputs measurements to system log periodically.
| set | 1.2.0 | +| kyuubi.metrics.slf4j.interval | PT5S | How often should report metrics to SLF4J logger | duration | 1.2.0 | ### Operation diff --git a/docs/deployment/migration-guide.md b/docs/deployment/migration-guide.md index 27dad2aba..58df0fcc6 100644 --- a/docs/deployment/migration-guide.md +++ b/docs/deployment/migration-guide.md @@ -24,6 +24,9 @@ To restore previous behavior, set `kyuubi.metadata.store.jdbc.database.type=DERBY` and `kyuubi.metadata.store.jdbc.url=jdbc:derby:memory:kyuubi_state_store_db;create=true`. +* Since Kyuubi 1.8, PROMETHEUS is changed as the default metrics reporter. To restore previous behavior, + set `kyuubi.metrics.reporters=JSON`. + ## Upgrading from Kyuubi 1.7.1 to 1.7.2 * Since Kyuubi 1.7.2, for Kyuubi BeeLine, please use `--python-mode` option to run python code or script. diff --git a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala index fe11f6eb1..9bc2e6324 100644 --- a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala +++ b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala @@ -46,7 +46,7 @@ object MetricsConf { .transformToUpperCase .toSet() .checkValues(ReporterType) - .createWithDefault(Set(JSON.toString)) + .createWithDefault(Set(PROMETHEUS.toString)) val METRICS_CONSOLE_INTERVAL: ConfigEntry[Long] = buildConf("kyuubi.metrics.console.interval") .doc("How often should report metrics to console") diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala index 2f794ed48..1dc24aeec 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala @@ -34,6 +34,7 @@ import org.apache.kyuubi._ import org.apache.kyuubi.client.util.BatchUtils._ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.events.ServerEventHandlerRegister +import org.apache.kyuubi.metrics.MetricsConf import org.apache.kyuubi.operation.HiveJDBCTestHelper import org.apache.kyuubi.operation.OperationState._ import org.apache.kyuubi.server.KyuubiServer @@ -56,6 +57,7 @@ class ServerJsonLoggingEventHandlerSuite extends WithKyuubiServer with HiveJDBCT .set(KyuubiConf.SERVER_EVENT_JSON_LOG_PATH, serverLogRoot) .set(KyuubiConf.ENGINE_SPARK_EVENT_LOGGERS, Seq("JSON")) .set(KyuubiConf.ENGINE_EVENT_JSON_LOG_PATH, engineLogRoot) + .set(MetricsConf.METRICS_REPORTERS, Set.empty[String]) } override protected def jdbcUrl: String = getJdbcUrl