apache/celeborn
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Feature] Shade netty native libraries (#908)

Browse Source 
* shade netty native libraries.

* To ensure netty use the correct native library.

* To ensure netty use the correct native library.

* update.
This commit is contained in:
Ethan Feng 2022-11-02 19:02:03 +08:00 committed by GitHub
parent ea4ed10e5c
commit b06ab31cee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 81 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
client-spark/spark-2-shaded/pom.xml
View File

@ -45,19 +45,19 @@
<relocations>
<relocation>
<pattern>com.google.protobuf</pattern>
<shadedPattern>org.apache.celeborn.com.google.protobuf</shadedPattern>
<shadedPattern>${shading.prefix}.com.google.protobuf</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>org.apache.celeborn.com.google.common</shadedPattern>
<shadedPattern>${shading.prefix}.com.google.common</shadedPattern>
</relocation>
<relocation>
<pattern>io.netty</pattern>
<shadedPattern>org.apache.celeborn.io.netty</shadedPattern>
<shadedPattern>${shading.prefix}.io.netty</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons</pattern>
<shadedPattern>org.apache.celeborn.org.apache.commons</shadedPattern>
<shadedPattern>${shading.prefix}.org.apache.commons</shadedPattern>
</relocation>
</relocations>
<artifactSet>
@ -82,6 +82,42 @@
</filters>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>${maven.plugin.antrun.version}</version>
<executions>
<execution>
<id>rename-native-library</id>
<goals>
<goal>run</goal>
</goals>
<phase>package</phase>
<configuration>
<target>
<echo message="unpacking netty jar"></echo>
<unzip dest="${project.build.directory}/unpacked/" src="${project.build.directory}/${artifactId}-${version}.jar"></unzip>
<echo message="renaming native epoll library"></echo>
<move includeemptydirs="false" todir="${project.build.directory}/unpacked/META-INF/native">
<fileset dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper from="libnetty_transport_native_epoll_x86_64.so" to="liborg_apache_celeborn_shaded_netty_transport_native_epoll_x86_64.so" type="glob"></mapper>
</move>
<move includeemptydirs="false" todir="${project.build.directory}/unpacked/META-INF/native">
<fileset dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper from="libnetty_transport_native_epoll_aarch_64.so" to="liborg_apache_celeborn_shaded_netty_transport_native_epoll_aarch_64.so.so" type="glob"></mapper>
</move>
<echo message="deleting native kqueue library"></echo>
<delete file="${project.build.directory}/unpacked/META-INF/native/libnetty_transport_native_kqueue_x86_64.jnilib"></delete>
<delete file="${project.build.directory}/unpacked/META-INF/native/libnetty_transport_native_kqueue_aarch_64.jnilib"></delete>
<delete file="${project.build.directory}/unpacked/META-INF/native/libnetty_resolver_dns_native_macos_aarch_64.jnilib"></delete>
<delete file="${project.build.directory}/unpacked/META-INF/native/libnetty_resolver_dns_native_macos_x86_64.jnilib"></delete>
<echo message="repackaging netty jar"></echo>
<jar basedir="${project.build.directory}/unpacked" destfile="${project.build.directory}/${artifactId}-${version}.jar"></jar>
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

44
client-spark/spark-3-shaded/pom.xml
View File

@ -45,19 +45,19 @@
<relocations>
<relocation>
<pattern>com.google.protobuf</pattern>
<shadedPattern>org.apache.celeborn.com.google.protobuf</shadedPattern>
<shadedPattern>${shading.prefix}.com.google.protobuf</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>org.apache.celeborn.com.google.common</shadedPattern>
<shadedPattern>${shading.prefix}.com.google.common</shadedPattern>
</relocation>
<relocation>
<pattern>io.netty</pattern>
<shadedPattern>org.apache.celeborn.io.netty</shadedPattern>
<shadedPattern>${shading.prefix}.io.netty</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons</pattern>
<shadedPattern>org.apache.celeborn.org.apache.commons</shadedPattern>
<shadedPattern>${shading.prefix}.org.apache.commons</shadedPattern>
</relocation>
</relocations>
<artifactSet>
@ -82,6 +82,42 @@
</filters>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>${maven.plugin.antrun.version}</version>
<executions>
<execution>
<id>rename-native-library</id>
<goals>
<goal>run</goal>
</goals>
<phase>package</phase>
<configuration>
<target>
<echo message="unpacking netty jar"></echo>
<unzip dest="${project.build.directory}/unpacked/" src="${project.build.directory}/${artifactId}-${version}.jar"></unzip>
<echo message="renaming native epoll library"></echo>
<move includeemptydirs="false" todir="${project.build.directory}/unpacked/META-INF/native">
<fileset dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper from="libnetty_transport_native_epoll_x86_64.so" to="liborg_apache_celeborn_shaded_netty_transport_native_epoll_x86_64.so" type="glob"></mapper>
</move>
<move includeemptydirs="false" todir="${project.build.directory}/unpacked/META-INF/native">
<fileset dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper from="libnetty_transport_native_epoll_aarch_64.so" to="liborg_apache_celeborn_shaded_netty_transport_native_epoll_aarch_64.so.so" type="glob"></mapper>
</move>
<echo message="deleting native kqueue library"></echo>
<delete file="${project.build.directory}/unpacked/META-INF/native/libnetty_transport_native_kqueue_x86_64.jnilib"></delete>
<delete file="${project.build.directory}/unpacked/META-INF/native/libnetty_transport_native_kqueue_aarch_64.jnilib"></delete>
<delete file="${project.build.directory}/unpacked/META-INF/native/libnetty_resolver_dns_native_macos_aarch_64.jnilib"></delete>
<delete file="${project.build.directory}/unpacked/META-INF/native/libnetty_resolver_dns_native_macos_x86_64.jnilib"></delete>
<echo message="repackaging netty jar"></echo>
<jar basedir="${project.build.directory}/unpacked" destfile="${project.build.directory}/${artifactId}-${version}.jar"></jar>
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

1
pom.xml
View File

@ -86,6 +86,7 @@
<snakeyaml.version>1.30</snakeyaml.version>
<!-- default hadoop version -->
<hadoop.version>3.3.1</hadoop.version>
<shading.prefix>org.apache.celeborn.shaded</shading.prefix>
<maven.plugin.antrun.version>3.0.0</maven.plugin.antrun.version>
<maven.plugin.clean.version>3.2.0</maven.plugin.clean.version>