diff --git a/LICENSE-binary b/LICENSE-binary index 5a35fd16a..034141317 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -302,7 +302,7 @@ BSD 3-clause See licenses/LICENSE-protobuf.txt for details. com.google.protobuf:protobuf-java See licenses/LICENSE-leveldbjni.txt for details. -org.openlabtesting.leveldbjni:leveldbjni-all +org.fusesource.leveldbjni:leveldbjni-all Common Development and Distribution License (CDDL) 1.0 diff --git a/README.md b/README.md index 525b97894..e375122d5 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ To compile for Spark 3.5 with Java21, please use the following command ./build/make-distribution.sh --sbt-enabled -Pspark-3.5 -Pjdk-21 ``` +> **_NOTE:_** Celeborn supports automatic builds on linux aarch64 platform via `aarch64` profile. `aarch64` profile requires glibc version 3.4.21. There is potential problematic frame `C [libc.so.6+0x8412a]` for other glibc version like 2.x etc. ### Package Details Build procedure will create a compressed package. diff --git a/common/pom.xml b/common/pom.xml index ab9dba6b1..3273303dd 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -87,7 +87,7 @@ netty-all - org.openlabtesting.leveldbjni + ${leveldbjni.group} leveldbjni-all diff --git a/docs/deploy_on_k8s.md b/docs/deploy_on_k8s.md index 16597d6ce..3c7edcfb4 100644 --- a/docs/deploy_on_k8s.md +++ b/docs/deploy_on_k8s.md @@ -38,6 +38,8 @@ You can find released version of Celeborn on [Downloading Page](https://celeborn Of course, you can build binary package from master branch or your own branch by using `./build/make-distribution.sh` in source code. +> Notice: Celeborn supports automatic builds on linux aarch64 platform via `aarch64` profile. `aarch64` profile requires glibc version 3.4.21. There is potential problematic frame `C [libc.so.6+0x8412a]` for other glibc version like 2.x etc. + Anyway, you should unzip and into binary package. ### 2. Modify Celeborn Configurations diff --git a/pom.xml b/pom.xml index f9a824a2f..307b95227 100644 --- a/pom.xml +++ b/pom.xml @@ -113,6 +113,9 @@ 9.4.52.v20230823 4.0.4 + + org.fusesource.leveldbjni + org.apache.celeborn.shaded 3.0.0 @@ -394,7 +397,7 @@ - org.openlabtesting.leveldbjni + ${leveldbjni.group} leveldbjni-all ${leveldb.version} @@ -1697,5 +1700,20 @@ tests/kubernetes-it + + + + aarch64 + + + linux + aarch64 + + + + + org.openlabtesting.leveldbjni + + diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index d75c6ba46..c8a61cbaa 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -111,7 +111,15 @@ object Dependencies { val ioDropwizardMetricsJvm = "io.dropwizard.metrics" % "metrics-jvm" % metricsVersion val ioNetty = "io.netty" % "netty-all" % nettyVersion excludeAll( ExclusionRule("io.netty", "netty-handler-ssl-ocsp")) - val leveldbJniAll = "org.openlabtesting.leveldbjni" % "leveldbjni-all" % leveldbJniVersion + val leveldbJniGroup = if (System.getProperty("os.name").startsWith("Linux") + && System.getProperty("os.arch").equals("aarch64")) { + // use org.openlabtesting.leveldbjni on aarch64 platform except MacOS + // org.openlabtesting.leveldbjni requires glibc version 3.4.21 + "org.openlabtesting.leveldbjni" + } else { + "org.fusesource.leveldbjni" + } + val leveldbJniAll = leveldbJniGroup % "leveldbjni-all" % leveldbJniVersion val log4j12Api = "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version val log4jSlf4jImpl = "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2Version val lz4Java = "org.lz4" % "lz4-java" % lz4JavaVersion diff --git a/worker/pom.xml b/worker/pom.xml index ef32e97aa..e0be5e904 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -49,7 +49,7 @@ ${project.version} - org.openlabtesting.leveldbjni + ${leveldbjni.group} leveldbjni-all