### What changes were proposed in this pull request? Dependency leveldbjni uses `org.openlabtesting.leveldbjni` to support linux aarch64 platform for leveldb via `aarch64` profile. Follow up #2476. ### Why are the changes needed? Celeborn worker could not start on arm arch devices if db backend is `LevelDB`, which should support leveldbjni on the aarch64 platform. aarch64 uses `org.openlabtesting.leveldbjni:leveldbjni-all.1.8`, and other platforms use `org.fusesource.leveldbjni:leveldbjni-all.1.8`. Meanwhile, because some hadoop dependencies packages are also depend on `org.fusesource.leveldbjni:leveldbjni-all`, but hadoop merge the similar change on trunk, details see [HADOOP-16614](https://issues.apache.org/jira/browse/HADOOP-16614), therefore it should exclude the dependency of `org.fusesource.leveldbjni` for these hadoop packages related. In addtion, `org.openlabtesting.leveldbjni` requires glibc version 3.4.21. Otherwise, there will be the following potential runtime risks: ``` # # A fatal error has been detected by the Java Runtime Environment: # # SIGBUS (0x7) at pc=0x00007fad3630b12a, pid=62, tid=0x00007f93394ef700 # # JRE version: Java(TM) SE Runtime Environment (8.0_162-b12) (build 1.8.0_162-b12) # Java VM: Java HotSpot(TM) 64-Bit Server VM (25.162-b12 mixed mode linux-amd64 ) # Problematic frame: # C [libc.so.6+0x8412a] # # Core dump written. Default location: /data/service/celeborn/core or core.62 # # If you would like to submit a bug report, please visit: # http://bugreport.java.com/bugreport/crash.jsp # The crash happened outside the Java Virtual Machine in native code. # See problematic frame for where to report the bug. # --------------- T H R E A D --------------- Current thread (0x00007f9308001000): JavaThread "leveldb" [_thread_in_native, id=878, stack(0x00007f9338cf0000,0x00007f93394f0000)] siginfo: si_signo: 7 (SIGBUS), si_code: 2 (BUS_ADRERR), si_addr: 0x00007f97380d2220 ``` Backport: - https://github.com/apache/spark/pull/26636 - https://github.com/apache/spark/pull/31036 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? No. Closes #2530 from SteNicholas/CELEBORN-1380. Authored-by: SteNicholas <programgeek@163.com> Signed-off-by: mingji <fengmingxiao.fmx@alibaba-inc.com>
217 lines
6.7 KiB
XML
217 lines
6.7 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
|
|
<parent>
|
|
<groupId>org.apache.celeborn</groupId>
|
|
<artifactId>celeborn-parent_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<relativePath>../pom.xml</relativePath>
|
|
</parent>
|
|
|
|
<artifactId>celeborn-common_${scala.binary.version}</artifactId>
|
|
<packaging>jar</packaging>
|
|
<name>Celeborn Common</name>
|
|
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.ratis</groupId>
|
|
<artifactId>ratis-common</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.ratis</groupId>
|
|
<artifactId>ratis-client</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.dropwizard.metrics</groupId>
|
|
<artifactId>metrics-core</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.dropwizard.metrics</groupId>
|
|
<artifactId>metrics-graphite</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.dropwizard.metrics</groupId>
|
|
<artifactId>metrics-jvm</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.yaml</groupId>
|
|
<artifactId>snakeyaml</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.xerial.snappy</groupId>
|
|
<artifactId>snappy-java</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>jul-to-slf4j</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>jcl-over-slf4j</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-io</groupId>
|
|
<artifactId>commons-io</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-crypto</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-lang3</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty-all</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${leveldbjni.group}</groupId>
|
|
<artifactId>leveldbjni-all</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.google.code.findbugs</groupId>
|
|
<artifactId>jsr305</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.google.protobuf</groupId>
|
|
<artifactId>protobuf-java</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-library</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-reflect</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.roaringbitmap</groupId>
|
|
<artifactId>RoaringBitmap</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.module</groupId>
|
|
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-databind</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-annotations</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-core</artifactId>
|
|
</dependency>
|
|
<!-- Test dependencies -->
|
|
<dependency>
|
|
<groupId>org.mockito</groupId>
|
|
<artifactId>mockito-core</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-slf4j-impl</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-1.2-api</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<!-- for SSL support -->
|
|
<dependency>
|
|
<groupId>org.bouncycastle</groupId>
|
|
<artifactId>bcprov-jdk18on</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.bouncycastle</groupId>
|
|
<artifactId>bcpkix-jdk18on</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
</dependencies>
|
|
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.xolstice.maven.plugins</groupId>
|
|
<artifactId>protobuf-maven-plugin</artifactId>
|
|
</plugin>
|
|
</plugins>
|
|
<extensions>
|
|
<extension>
|
|
<groupId>kr.motd.maven</groupId>
|
|
<artifactId>os-maven-plugin</artifactId>
|
|
<version>${maven.plugin.os.version}</version>
|
|
</extension>
|
|
</extensions>
|
|
</build>
|
|
|
|
<profiles>
|
|
<profile>
|
|
<id>hadoop-3</id>
|
|
<activation>
|
|
<property>
|
|
<name>hadoop-3-deps</name>
|
|
</property>
|
|
</activation>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client-api</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client-runtime</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
</dependency>
|
|
</dependencies>
|
|
</profile>
|
|
<profile>
|
|
<id>hadoop-2</id>
|
|
<activation>
|
|
<property>
|
|
<name>hadoop-2-deps</name>
|
|
</property>
|
|
</activation>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
</dependency>
|
|
</dependencies>
|
|
</profile>
|
|
</profiles>
|
|
</project>
|