apache/celeborn
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CELEBORN-1331] Remove third-party dependencies in shaded clients' pom

Browse Source 
### What changes were proposed in this pull request?
celeborn-client-spark-3-shaded_2.12 should not expose the shipped transitive deps

### Why are the changes needed?
maven dependency

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
CI

Closes #2391 from kerwin-zk/issue-1331.

Authored-by: xiyu.zk <xiyu.zk@alibaba-inc.com>
Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
xiyu.zk 2024-04-08 13:51:30 +08:00 committed by Cheng Pan
parent c788c38025
commit 6ce8b8894d
No known key found for this signature in database
GPG Key ID: 8001952629BCC75D
1 changed files with 31 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
project/CelebornBuild.scala
View File

@ -20,6 +20,8 @@ import java.nio.charset.StandardCharsets.UTF_8
import java.util.Locale
import scala.util.Properties
import scala.xml._
import scala.xml.transform._
import sbtassembly.AssemblyPlugin.autoImport._
import sbtprotoc.ProtocPlugin.autoImport._
@ -389,6 +391,21 @@ object Utils {
require(ALL_SCALA_VERSIONS.contains(v), s"found not allow scala version: $v")
v
}
/**
* The deps for shaded clients are already packaged in the jar,
* so we should not expose the shipped transitive deps.
*/
def removeDependenciesTransformer: xml.Node => xml.Node = { node =>
new RuleTransformer(new RewriteRule {
override def transform(n: xml.Node): Seq[xml.Node] = n match {
case e: Elem if e.label == "dependencies" =>
Nil
case _ =>
n
}
}).transform(node).head
}
}
object CelebornCommon {
@ -773,9 +790,14 @@ trait SparkClientProjects {
}
def sparkClientShade: Project = {
val p = Project(sparkClientShadedProjectName, file(sparkClientShadedProjectPath))
var p = Project(sparkClientShadedProjectName, file(sparkClientShadedProjectPath))
.dependsOn(sparkClient)
.disablePlugins(AddMetaInfLicenseFiles)
if (includeColumnarShuffle) {
p = p.dependsOn(sparkColumnarShuffle)
}
p = p.disablePlugins(AddMetaInfLicenseFiles)
.settings (
commonSettings,
releaseSettings,
@ -836,13 +858,10 @@ trait SparkClientProjects {
case _ => MergeStrategy.first
},
Compile / packageBin := assembly.value
Compile / packageBin := assembly.value,
pomPostProcess := removeDependenciesTransformer
)
if (includeColumnarShuffle) {
p.dependsOn(sparkColumnarShuffle)
} else {
p
}
p
}
}
@ -1052,7 +1071,8 @@ trait FlinkClientProjects {
case _ => MergeStrategy.first
},
Compile / packageBin := assembly.value
Compile / packageBin := assembly.value,
pomPostProcess := removeDependenciesTransformer
)
}
}
@ -1164,7 +1184,8 @@ object MRClientProjects {
case _ => MergeStrategy.first
},
Compile / packageBin := assembly.value
Compile / packageBin := assembly.value,
pomPostProcess := removeDependenciesTransformer
)
}