diff --git a/LICENSE-binary b/LICENSE-binary index 20d746b1b..8228d461e 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -216,6 +216,7 @@ com.fasterxml.jackson.module:jackson-module-scala com.google.code.findbugs:jsr305 com.google.guava:failureaccess com.google.guava:guava +com.lmax:disruptor com.thoughtworks.paranamer:paranamer commons-cli:commons-cli commons-io:commons-io diff --git a/dev/deps/dependencies-server b/dev/deps/dependencies-server index 8f24085e3..eb08ed80f 100644 --- a/dev/deps/dependencies-server +++ b/dev/deps/dependencies-server @@ -25,6 +25,7 @@ commons-crypto/1.0.0//commons-crypto-1.0.0.jar commons-io/2.17.0//commons-io-2.17.0.jar commons-lang3/3.17.0//commons-lang3-3.17.0.jar commons-logging/1.1.3//commons-logging-1.1.3.jar +disruptor/3.4.4//disruptor-3.4.4.jar failureaccess/1.0.2//failureaccess-1.0.2.jar guava/33.1.0-jre//guava-33.1.0-jre.jar hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar diff --git a/master/pom.xml b/master/pom.xml index 94858ba63..b2b2b06cd 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -86,6 +86,10 @@ org.apache.logging.log4j log4j-1.2-api + + com.lmax + disruptor + org.apache.celeborn diff --git a/pom.xml b/pom.xml index 0b7d37af0..31a38cda1 100644 --- a/pom.xml +++ b/pom.xml @@ -90,6 +90,7 @@ 4.13.2 1.8 2.24.3 + 3.4.4 1.8.0 4.11.0 1.17.14 @@ -245,6 +246,12 @@ log4j-1.2-api ${log4j2.version} + + + com.lmax + disruptor + ${disruptor.version} + io.dropwizard.metrics metrics-core diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index e8818977f..ecd4fb4be 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -58,6 +58,7 @@ object Dependencies { val junitVersion = "4.13.2" val leveldbJniVersion = "1.8" val log4j2Version = "2.24.3" + val disruptorVersion = "3.4.4" val jdkToolsVersion = "0.1" val metricsVersion = "4.2.25" val mockitoVersion = "4.11.0" @@ -148,6 +149,7 @@ object Dependencies { val log4jCore = "org.apache.logging.log4j" % "log4j-core" % log4j2Version val log4j12Api = "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version val log4jSlf4jImpl = "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2Version + val disruptor = "com.lmax" % "disruptor" % disruptorVersion val lz4Java = "org.lz4" % "lz4-java" % lz4JavaVersion val protobufJava = "com.google.protobuf" % "protobuf-java" % protoVersion val ratisClient = "org.apache.ratis" % "ratis-client" % ratisVersion @@ -748,6 +750,7 @@ object CelebornMaster { Dependencies.hadoopClientApi, Dependencies.log4j12Api, Dependencies.log4jSlf4jImpl, + Dependencies.disruptor, Dependencies.ratisClient, Dependencies.ratisCommon, Dependencies.ratisGrpc, @@ -783,6 +786,7 @@ object CelebornWorker { Dependencies.ioNetty, Dependencies.log4j12Api, Dependencies.log4jSlf4jImpl, + Dependencies.disruptor, Dependencies.leveldbJniAll, Dependencies.roaringBitmap, Dependencies.rocksdbJni, diff --git a/worker/pom.xml b/worker/pom.xml index 6d9c858a1..ad6b1156b 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -66,6 +66,10 @@ org.apache.logging.log4j log4j-1.2-api + + com.lmax + disruptor + org.rocksdb rocksdbjni