e0c00ecd38
### What changes were proposed in this pull request? 1. Map side merge and push. 2. Support hadoop2 & 3. 3. Reduce in-memory merge. 4. Integrate LifecycleManager to RmApplicationMaster. ### Why are the changes needed? Ditto. ### Does this PR introduce _any_ user-facing change? NO. ### How was this patch tested? Cluster. I tested this PR on a cluster with a 4x 16 CPU 64G Mem 4ESSD cluster. Hadoop 2.8.5 1TB Terasort, 8400 mappers, 1000 reducers Celeborn 81min vs MR shuffle 89min   1GB wordcount, 8 mappers, 8 reducers Celeborn 35s VS MR shuffle 38s   Closes #1830 from FMX/CELEBORN-839. Lead-authored-by: mingji <fengmingxiao.fmx@alibaba-inc.com> Co-authored-by: Ethan Feng <fengmingxiao.fmx@alibaba-inc.com> Signed-off-by: zky.zhoukeyong <zky.zhoukeyong@alibaba-inc.com>
123 lines
4.1 KiB
XML
123 lines
4.1 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<parent>
|
|
<groupId>org.apache.celeborn</groupId>
|
|
<artifactId>celeborn-parent_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</parent>
|
|
|
|
<artifactId>celeborn-master_${scala.binary.version}</artifactId>
|
|
<packaging>jar</packaging>
|
|
<name>Celeborn Master</name>
|
|
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.celeborn</groupId>
|
|
<artifactId>celeborn-common_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.celeborn</groupId>
|
|
<artifactId>celeborn-service_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty-all</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.google.protobuf</groupId>
|
|
<artifactId>protobuf-java</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.ratis</groupId>
|
|
<artifactId>ratis-common</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.ratis</groupId>
|
|
<artifactId>ratis-client</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.ratis</groupId>
|
|
<artifactId>ratis-server</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.ratis</groupId>
|
|
<artifactId>ratis-netty</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.ratis</groupId>
|
|
<artifactId>ratis-grpc</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.ratis</groupId>
|
|
<artifactId>ratis-shell</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-slf4j-impl</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-1.2-api</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client-api</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client-runtime</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.mockito</groupId>
|
|
<artifactId>mockito-core</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
</dependencies>
|
|
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-dependency-plugin</artifactId>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.xolstice.maven.plugins</groupId>
|
|
<artifactId>protobuf-maven-plugin</artifactId>
|
|
</plugin>
|
|
</plugins>
|
|
<extensions>
|
|
<extension>
|
|
<groupId>kr.motd.maven</groupId>
|
|
<artifactId>os-maven-plugin</artifactId>
|
|
<version>${maven.plugin.os.version}</version>
|
|
</extension>
|
|
</extensions>
|
|
</build>
|
|
</project>
|