[CELEBORN-1994] Introduce disruptor dependency to support asynchronous logging of log4j2

### What changes were proposed in this pull request?

Introduce disruptor dependency to support asynchronous logging of log4j2.

### Why are the changes needed?

We add `-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector` in `CELEBORN_MASTER_JAVA_OPTS` and `CELEBORN_WOKRER_JAVA_OPTS` for production environment. `AsyncLoggerContextSelector` depends on disruptor dependency. Therefore, it's recommend to introduce disruptor dependency to support log4j2 asynchronous loggers.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Cluster test.

Closes #3246 from SteNicholas/CELEBORN-1994.

Authored-by: SteNicholas <programgeek@163.com>
Signed-off-by: mingji <fengmingxiao.fmx@alibaba-inc.com>

LICENSE-binary

@@ -216,6 +216,7 @@ com.fasterxml.jackson.module:jackson-module-scala
 com.google.code.findbugs:jsr305
 com.google.guava:failureaccess
 com.google.guava:guava
+com.lmax:disruptor
 com.thoughtworks.paranamer:paranamer
 commons-cli:commons-cli
 commons-io:commons-io

dev/deps/dependencies-server

@@ -25,6 +25,7 @@ commons-crypto/1.0.0//commons-crypto-1.0.0.jar
 commons-io/2.17.0//commons-io-2.17.0.jar
 commons-lang3/3.17.0//commons-lang3-3.17.0.jar
 commons-logging/1.1.3//commons-logging-1.1.3.jar
+disruptor/3.4.4//disruptor-3.4.4.jar
 failureaccess/1.0.2//failureaccess-1.0.2.jar
 guava/33.1.0-jre//guava-33.1.0-jre.jar
 hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar

master/pom.xml

@@ -86,6 +86,10 @@
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-1.2-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.lmax</groupId>
+      <artifactId>disruptor</artifactId>
+    </dependency>
 
     <dependency>
       <groupId>org.apache.celeborn</groupId>

pom.xml

@@ -90,6 +90,7 @@
     <junit.version>4.13.2</junit.version>
     <leveldb.version>1.8</leveldb.version>
     <log4j2.version>2.24.3</log4j2.version>
+    <disruptor.version>3.4.4</disruptor.version>
     <lz4-java.version>1.8.0</lz4-java.version>
     <mockito.version>4.11.0</mockito.version>
     <mockito-scalatest.version>1.17.14</mockito-scalatest.version>
@@ -245,6 +246,12 @@
         <artifactId>log4j-1.2-api</artifactId>
         <version>${log4j2.version}</version>
       </dependency>
+      <dependency>
+        <!-- log4j2 has an optional dependency on disruptor like AsyncLoggerContextSelector. -->
+        <groupId>com.lmax</groupId>
+        <artifactId>disruptor</artifactId>
+        <version>${disruptor.version}</version>
+      </dependency>
       <dependency>
         <groupId>io.dropwizard.metrics</groupId>
         <artifactId>metrics-core</artifactId>

project/CelebornBuild.scala

@@ -58,6 +58,7 @@ object Dependencies {
   val junitVersion = "4.13.2"
   val leveldbJniVersion = "1.8"
   val log4j2Version = "2.24.3"
+  val disruptorVersion = "3.4.4"
   val jdkToolsVersion = "0.1"
   val metricsVersion = "4.2.25"
   val mockitoVersion = "4.11.0"
@@ -148,6 +149,7 @@ object Dependencies {
   val log4jCore = "org.apache.logging.log4j" % "log4j-core" % log4j2Version
   val log4j12Api = "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version
   val log4jSlf4jImpl = "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2Version
+  val disruptor = "com.lmax" % "disruptor" % disruptorVersion
   val lz4Java = "org.lz4" % "lz4-java" % lz4JavaVersion
   val protobufJava = "com.google.protobuf" % "protobuf-java" % protoVersion
   val ratisClient = "org.apache.ratis" % "ratis-client" % ratisVersion
@@ -748,6 +750,7 @@ object CelebornMaster {
         Dependencies.hadoopClientApi,
         Dependencies.log4j12Api,
         Dependencies.log4jSlf4jImpl,
+        Dependencies.disruptor,
         Dependencies.ratisClient,
         Dependencies.ratisCommon,
         Dependencies.ratisGrpc,
@@ -783,6 +786,7 @@ object CelebornWorker {
         Dependencies.ioNetty,
         Dependencies.log4j12Api,
         Dependencies.log4jSlf4jImpl,
+        Dependencies.disruptor,
         Dependencies.leveldbJniAll,
         Dependencies.roaringBitmap,
         Dependencies.rocksdbJni,

worker/pom.xml

@@ -66,6 +66,10 @@
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-1.2-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.lmax</groupId>
+      <artifactId>disruptor</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.rocksdb</groupId>
       <artifactId>rocksdbjni</artifactId>