[CELEBORN-1389] Bump Dropwizard version from 3.2.6 to 4.2.25

### What changes were proposed in this pull request? Bump Dropwizard version from 3.2.6 to 4.2.25. Meanwhile, introduce `metrics_jvm_thread_peak_count_Value` and `metrics_jvm_thread_total_started_count_Value` in `celeborn-jvm-dashboard.json`. ### Why are the changes needed? Dropwizard metrics has released v4.2.25 including some bugfixes and improvements including: * [JVM] Fix maximum/total memory calculation: https://github.com/dropwizard/metrics/pull/3125 * [Thread] Add peak and total started thread count to `ThreadStatesGaugeSet`: https://github.com/dropwizard/metrics/pull/1601 Meanwhile, Ratis version has upgraded to 3.0.1 which has no compatibility problem with Dropwizard 4.2.25. Backport: - https://github.com/apache/spark/pull/26332 - https://github.com/apache/spark/pull/29426 - https://github.com/apache/spark/pull/37372 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manual test. Closes #2540 from SteNicholas/CELEBORN-1389. Authored-by: SteNicholas <programgeek@163.com> Signed-off-by: mingji <fengmingxiao.fmx@alibaba-inc.com>
2024-06-04 19:26:20 +08:00 · 2024-06-04 19:26:20 +08:00 · 4fc42d7fef
commit 4fc42d7fef
parent aa8ff3c17c
19 changed files with 95 additions and 58 deletions
--- a/METRICS.md
+++ b/METRICS.md
@ -180,9 +180,11 @@ Here is an example of Grafana dashboard importing.
 |        jvm_thread_blocked_count        |        JVM        |                                  The current number of threads having blocked state.                                   |
 |       jvm_thread_deadlock_count        |        JVM        |                                  The current number of threads having deadlock state.                                  |
 |          jvm_thread_new_count          |        JVM        |                                    The current number of threads having new state.                                     |
+|         jvm_thread_peak_count          |        JVM        |                                        The current number of peak live threads.                                        |
 |       jvm_thread_runnable_count        |        JVM        |                                  The current number of threads having runnable state.                                  |
 |      jvm_thread_terminated_count       |        JVM        |                                 The current number of threads having terminated state.                                 |
 |     jvm_thread_timed_waiting_count     |        JVM        |                               The current number of threads having timed_waiting state.                                |
+|     jvm_thread_total_started_count     |        JVM        |                                         The current number of started threads.                                         |
 |        jvm_thread_waiting_count        |        JVM        |                                  The current number of threads having waiting state.                                   |
 |         jvm_classloader_loaded         |        JVM        |                             The total number of classes loaded since the start of the JVM.                             |
 |        jvm_classloader_unloaded        |        JVM        |                            The total number of classes unloaded since the start of the JVM.                            |
--- a/assets/grafana/celeborn-jvm-dashboard.json
+++ b/assets/grafana/celeborn-jvm-dashboard.json
@ -1245,6 +1245,42 @@
          "legendFormat": "daemon_${baseLegend}",
          "range": true,
          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "metrics_jvm_thread_peak_count_Value{instance=~\"${instance}\"}",
+          "hide": false,
+          "legendFormat": "peak_${baseLegend}",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "metrics_jvm_thread_total_started_count_Value{instance=~\"${instance}\"}",
+          "hide": false,
+          "legendFormat": "total_started_${baseLegend}",
+          "range": true,
+          "refId": "D"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "metrics_jvm_thread_deadlock_count_Value{instance=~\"${instance}\"}",
+          "hide": false,
+          "legendFormat": "deadlock_${baseLegend}",
+          "range": true,
+          "refId": "F"
        }
      ],
      "title": "Thread Counts",
@ -1400,18 +1436,6 @@
          "legendFormat": "terminated_${baseLegend}",
          "range": true,
          "refId": "F"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "editorMode": "code",
-          "expr": "metrics_jvm_thread_deadlock_count_Value{instance=~\"${instance}\"}",
-          "hide": false,
-          "legendFormat": "deadlock_${baseLegend}",
-          "range": true,
-          "refId": "G"
        }
      ],
      "title": "Thread States",
--- a/dev/deps/dependencies-client-flink-1.14
+++ b/dev/deps/dependencies-client-flink-1.14
@ -34,9 +34,9 @@ jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
 netty-codec-dns/4.1.109.Final//netty-codec-dns-4.1.109.Final.jar
--- a/dev/deps/dependencies-client-flink-1.15
+++ b/dev/deps/dependencies-client-flink-1.15
@ -34,9 +34,9 @@ jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
 netty-codec-dns/4.1.109.Final//netty-codec-dns-4.1.109.Final.jar
--- a/dev/deps/dependencies-client-flink-1.17
+++ b/dev/deps/dependencies-client-flink-1.17
@ -34,9 +34,9 @@ jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
 netty-codec-dns/4.1.109.Final//netty-codec-dns-4.1.109.Final.jar
--- a/dev/deps/dependencies-client-flink-1.18
+++ b/dev/deps/dependencies-client-flink-1.18
@ -34,9 +34,9 @@ jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
 netty-codec-dns/4.1.109.Final//netty-codec-dns-4.1.109.Final.jar
--- a/dev/deps/dependencies-client-flink-1.19
+++ b/dev/deps/dependencies-client-flink-1.19
@ -34,9 +34,9 @@ jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
 netty-codec-dns/4.1.109.Final//netty-codec-dns-4.1.109.Final.jar
--- a/dev/deps/dependencies-client-mr
+++ b/dev/deps/dependencies-client-mr
@ -136,9 +136,9 @@ kotlin-stdlib/1.4.10//kotlin-stdlib-1.4.10.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 mssql-jdbc/6.2.1.jre7//mssql-jdbc-6.2.1.jre7.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
--- a/dev/deps/dependencies-client-spark-2.4
+++ b/dev/deps/dependencies-client-spark-2.4
@ -34,9 +34,9 @@ jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.4.0//lz4-java-1.4.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
 netty-codec-dns/4.1.109.Final//netty-codec-dns-4.1.109.Final.jar
--- a/dev/deps/dependencies-client-spark-3.0
+++ b/dev/deps/dependencies-client-spark-3.0
@ -34,9 +34,9 @@ jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.7.1//lz4-java-1.7.1.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
 netty-codec-dns/4.1.109.Final//netty-codec-dns-4.1.109.Final.jar
--- a/dev/deps/dependencies-client-spark-3.1
+++ b/dev/deps/dependencies-client-spark-3.1
@ -34,9 +34,9 @@ jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.7.1//lz4-java-1.7.1.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
 netty-codec-dns/4.1.109.Final//netty-codec-dns-4.1.109.Final.jar
--- a/dev/deps/dependencies-client-spark-3.2
+++ b/dev/deps/dependencies-client-spark-3.2
@ -34,9 +34,9 @@ jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.7.1//lz4-java-1.7.1.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
 netty-codec-dns/4.1.109.Final//netty-codec-dns-4.1.109.Final.jar
--- a/dev/deps/dependencies-client-spark-3.3
+++ b/dev/deps/dependencies-client-spark-3.3
@ -34,9 +34,9 @@ jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
 netty-codec-dns/4.1.109.Final//netty-codec-dns-4.1.109.Final.jar
--- a/dev/deps/dependencies-client-spark-3.4
+++ b/dev/deps/dependencies-client-spark-3.4
@ -34,9 +34,9 @@ jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
 netty-codec-dns/4.1.109.Final//netty-codec-dns-4.1.109.Final.jar
--- a/dev/deps/dependencies-client-spark-3.5
+++ b/dev/deps/dependencies-client-spark-3.5
@ -34,9 +34,9 @@ jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
 netty-buffer/4.1.109.Final//netty-buffer-4.1.109.Final.jar
 netty-codec-dns/4.1.109.Final//netty-codec-dns-4.1.109.Final.jar
--- a/dev/deps/dependencies-server
+++ b/dev/deps/dependencies-server
@ -75,9 +75,9 @@ log4j-core/2.17.2//log4j-core-2.17.2.jar
 log4j-slf4j-impl/2.17.2//log4j-slf4j-impl-2.17.2.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
-metrics-core/3.2.6//metrics-core-3.2.6.jar
-metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
-metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 mimepull/1.9.15//mimepull-1.9.15.jar
 mybatis/3.5.15//mybatis-3.5.15.jar
 netty-all/4.1.109.Final//netty-all-4.1.109.Final.jar
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@ -24,7 +24,7 @@ There are two ways to monitor Celeborn cluster: Prometheus metrics and REST API.
 ## Metrics

 Celeborn has a configurable metrics system based on the
-[Dropwizard Metrics Library](http://metrics.dropwizard.io/4.2.0).
+[Dropwizard Metrics Library](https://metrics.dropwizard.io/4.2.0).
 This allows users to report Celeborn metrics to a variety of sinks including HTTP, JMX, CSV
 files and prometheus servlet. The metrics are generated by sources embedded in the Celeborn code base.
 They provide instrumentation for specific activities and Celeborn components.
--- a/pom.xml
+++ b/pom.xml
@ -70,7 +70,11 @@
    <!-- use hadoop-3 as default  -->
    <hadoop.version>3.3.6</hadoop.version>

-    <codahale.metrics.version>3.2.6</codahale.metrics.version>
+    <!--
+    If you change codahale.metrics.version, you also need to change
+    the link to metrics.dropwizard.io in docs/monitoring.md.
+    -->
+    <codahale.metrics.version>4.2.25</codahale.metrics.version>
    <commons-lang3.version>3.12.0</commons-lang3.version>
    <commons-io.version>2.13.0</commons-io.version>
    <commons-crypto.version>1.0.0</commons-crypto.version>
@ -236,6 +240,12 @@
        <groupId>io.dropwizard.metrics</groupId>
        <artifactId>metrics-graphite</artifactId>
        <version>${codahale.metrics.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>com.rabbitmq</groupId>
+            <artifactId>amqp-client</artifactId>
+          </exclusion>
+        </exclusions>
      </dependency>
      <dependency>
        <groupId>io.dropwizard.metrics</groupId>
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@ -53,7 +53,7 @@ object Dependencies {
  val leveldbJniVersion = "1.8"
  val log4j2Version = "2.17.2"
  val jdkToolsVersion = "0.1"
-  val metricsVersion = "3.2.6"
+  val metricsVersion = "4.2.25"
  val mockitoVersion = "4.11.0"
  val nettyVersion = "4.1.109.Final"
  val ratisVersion = "3.0.1"
@ -107,7 +107,8 @@ object Dependencies {
    ExclusionRule("log4j", "log4j"),
    ExclusionRule("org.slf4j", "slf4j-log4j12"))
  val ioDropwizardMetricsCore = "io.dropwizard.metrics" % "metrics-core" % metricsVersion
-  val ioDropwizardMetricsGraphite = "io.dropwizard.metrics" % "metrics-graphite" % metricsVersion
+  val ioDropwizardMetricsGraphite = "io.dropwizard.metrics" % "metrics-graphite" % metricsVersion excludeAll (
+    ExclusionRule("com.rabbitmq", "amqp-client"))
  val ioDropwizardMetricsJvm = "io.dropwizard.metrics" % "metrics-jvm" % metricsVersion
  val ioNetty = "io.netty" % "netty-all" % nettyVersion excludeAll(
    ExclusionRule("io.netty", "netty-handler-ssl-ocsp"))