[CELEBORN-1600] Enable check server dependencies

### What changes were proposed in this pull request? ### Why are the changes needed? Server module missing checks. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? GA Closes #2742 from cxzl25/check_server_deps. Authored-by: sychen <sychen@ctrip.com> Signed-off-by: mingji <fengmingxiao.fmx@alibaba-inc.com>
2024-09-20 15:14:56 +08:00 · 2024-09-20 15:14:56 +08:00 · 589100ea91
commit 589100ea91
parent 9b83587dd5
7 changed files with 62 additions and 67 deletions
--- a/dev/dependencies.sh
+++ b/dev/dependencies.sh
@ -63,6 +63,22 @@ function sbt_build_client_classpath() {
      awk 'NR==2' | \
      tr ":" "\n"
  )
+  sbt_process_classpath "$deps"
+ }
+
+function sbt_build_server_classpath() {
+  deps=$(
+    $SBT "error; clean; export managedClasspath" | \
+     awk '/managedClasspath/ { found=1 } found { print }' | \
+     awk 'NR % 2 == 0' | \
+     # This will skip the last line
+     sed '$d' | \
+     tr ":" "\n"
+  )
+  sbt_process_classpath "$deps"
+}
+
+function sbt_process_classpath() {
  deps1=$(echo "$deps" | grep -v ".sbt")
  deps2=$(echo "$deps" | grep ".sbt" || true)
  result1=$(
@ -95,24 +111,6 @@ function sbt_build_client_classpath() {
  echo "${result[@]}" | tr ' ' '\n' | sort -u >> "${DEP_PR}"
 }

-function sbt_build_server_classpath() {
-  $SBT "error; clean; export managedClasspath" | \
-    awk '/managedClasspath/ { found=1 } found { print }' | \
-    awk 'NR % 2 == 0' | \
-    # This will skip the last line 
-    sed '$d' | \
-    tr ":" "\n" | \
-    awk -F '/' '{
-      artifact_id=$(NF-2);
-      version=$(NF-1);
-      jar_name=$NF;
-      classifier_start_index=length(artifact_id"-"version"-") + 1;
-      classifier_end_index=index(jar_name, ".jar") - 1;
-      classifier=substr(jar_name, classifier_start_index, classifier_end_index - classifier_start_index + 1);
-      print artifact_id"/"version"/"classifier"/"jar_name
-    }' | sort -u >> "${DEP_PR}"
-}
-
 function check_diff() {
    set +e
    the_diff=$(diff "${DEP}" "${DEP_PR}")
@ -208,7 +206,7 @@ case "$MODULE" in
    ;;
  *)
    MODULE="server"
-    MVN_MODULES="worker,master"
+    MVN_MODULES="worker,master,cli"
    ;;
 esac

@ -258,7 +256,6 @@ if [ "$REPLACE" == "true" ]; then
  exit 0
 fi

-# skip check for sbt server now until sbt_build_server_classpath is refactored
-if [ "$CHECK" == "true" -a "$MODULE" != "server" ]; then
+if [ "$CHECK" == "true" ]; then
  check_diff
 fi
--- a/dev/deps/dependencies-client-mr
+++ b/dev/deps/dependencies-client-mr
@ -72,13 +72,13 @@ jackson-annotations/2.15.3//jackson-annotations-2.15.3.jar
 jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
 jackson-core/2.15.3//jackson-core-2.15.3.jar
 jackson-databind/2.15.3//jackson-databind-2.15.3.jar
-jackson-jaxrs-base/2.12.7//jackson-jaxrs-base-2.12.7.jar
-jackson-jaxrs-json-provider/2.12.7//jackson-jaxrs-json-provider-2.12.7.jar
+jackson-jaxrs-base/2.15.3//jackson-jaxrs-base-2.15.3.jar
+jackson-jaxrs-json-provider/2.15.3//jackson-jaxrs-json-provider-2.15.3.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations/2.12.7//jackson-module-jaxb-annotations-2.12.7.jar
+jackson-module-jaxb-annotations/2.15.3//jackson-module-jaxb-annotations-2.15.3.jar
 jackson-module-scala_2.12/2.15.3//jackson-module-scala_2.12-2.15.3.jar
 jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
-jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
+jakarta.xml.bind-api/2.3.3//jakarta.xml.bind-api-2.3.3.jar
 javax-websocket-client-impl/9.4.51.v20230217//javax-websocket-client-impl-9.4.51.v20230217.jar
 javax-websocket-server-impl/9.4.51.v20230217//javax-websocket-server-impl-9.4.51.v20230217.jar
 javax.annotation-api/1.3.2//javax.annotation-api-1.3.2.jar
--- a/dev/deps/dependencies-server
+++ b/dev/deps/dependencies-server
@ -19,7 +19,6 @@ HikariCP/4.0.3//HikariCP-4.0.3.jar
 RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar
 aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
 ap-loader-all/3.0-8//ap-loader-all-3.0-8.jar
-aws-java-sdk-bundle/1.12.367//aws-java-sdk-bundle-1.12.367.jar
 classgraph/4.8.138//classgraph-4.8.138.jar
 commons-cli/1.5.0//commons-cli-1.5.0.jar
 commons-crypto/1.0.0//commons-crypto-1.0.0.jar
@ -28,7 +27,6 @@ commons-lang3/3.13.0//commons-lang3-3.13.0.jar
 commons-logging/1.1.3//commons-logging-1.1.3.jar
 failureaccess/1.0.2//failureaccess-1.0.2.jar
 guava/33.1.0-jre//guava-33.1.0-jre.jar
-hadoop-aws/3.3.6//hadoop-aws-3.3.6.jar
 hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
 hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
 hk2-api/2.6.1//hk2-api-2.6.1.jar
@ -122,18 +120,19 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
+picocli/4.7.6//picocli-4.7.6.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
 ratis-client/3.1.0//ratis-client-3.1.0.jar
 ratis-common/3.1.0//ratis-common-3.1.0.jar
 ratis-grpc/3.1.0//ratis-grpc-3.1.0.jar
-ratis-metrics-default/3.1.0/ratis-metrics-default-3.1.0.jar
+ratis-metrics-api/3.1.0//ratis-metrics-api-3.1.0.jar
+ratis-metrics-default/3.1.0//ratis-metrics-default-3.1.0.jar
 ratis-netty/3.1.0//ratis-netty-3.1.0.jar
 ratis-proto/3.1.0//ratis-proto-3.1.0.jar
 ratis-server-api/3.1.0//ratis-server-api-3.1.0.jar
 ratis-server/3.1.0//ratis-server-3.1.0.jar
 ratis-shell/3.1.0//ratis-shell-3.1.0.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
-reflections/0.10.2//reflections-0.10.2.jar
+ratis-thirdparty-misc/1.0.6//ratis-thirdparty-misc-1.0.6.jar
 rocksdbjni/8.11.3//rocksdbjni-8.11.3.jar
 scala-library/2.12.18//scala-library-2.12.18.jar
 scala-reflect/2.12.18//scala-reflect-2.12.18.jar
--- a/openapi/openapi-client/pom.xml
+++ b/openapi/openapi-client/pom.xml
@ -29,22 +29,6 @@
  <packaging>jar</packaging>
  <name>Celeborn OpenAPI Client</name>

-  <dependencyManagement>
-    <dependencies>
-      <dependency>
-        <groupId>com.fasterxml.jackson.jaxrs</groupId>
-        <artifactId>jackson-jaxrs-json-provider</artifactId>
-        <version>${jackson.version}</version>
-        <exclusions>
-          <exclusion>
-            <groupId>jakarta.activation</groupId>
-            <artifactId>jakarta.activation-api</artifactId>
-          </exclusion>
-        </exclusions>
-      </dependency>
-    </dependencies>
-  </dependencyManagement>
-
  <dependencies>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
--- a/pom.xml
+++ b/pom.xml
@ -692,6 +692,28 @@
        <artifactId>jackson-datatype-jsr310</artifactId>
        <version>${jackson.version}</version>
      </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.jaxrs</groupId>
+        <artifactId>jackson-jaxrs-json-provider</artifactId>
+        <version>${jackson.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>jakarta.activation</groupId>
+            <artifactId>jakarta.activation-api</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.module</groupId>
+        <artifactId>jackson-module-jaxb-annotations</artifactId>
+        <version>${jackson.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>jakarta.activation</groupId>
+            <artifactId>jakarta.activation-api</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
      <dependency>
        <groupId>org.xerial.snappy</groupId>
        <artifactId>snappy-java</artifactId>
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@ -63,6 +63,7 @@ object Dependencies {
  val roaringBitmapVersion = "1.0.6"
  val rocksdbJniVersion = "8.11.3"
  val jacksonVersion = "2.15.3"
+  val jakartaActivationApiVersion = "1.2.1"
  val scalatestMockitoVersion = "1.17.14"
  val scalatestVersion = "3.2.16"
  val slf4jVersion = "1.7.36"
@ -79,7 +80,7 @@ object Dependencies {
  val openApiToolsJacksonBindNullableVersion = "0.2.6"
  val httpClient5Version = "5.3.1"
  val httpCore5Version = "5.2.4"
-  val javaxAnnotationApiVersion = "1.3.2"
+  val jakartaAnnotationApiVersion = "1.3.5"
  val picocliVersion = "4.7.6"

  // For SSL support
@ -111,6 +112,7 @@ object Dependencies {
    ExclusionRule("com.google.guava", "guava"),
    ExclusionRule("com.fasterxml.jackson.core", "jackson-annotations"),
    ExclusionRule("com.fasterxml.jackson.core", "jackson-databind"),
+    ExclusionRule("jakarta.activation", "jakarta.activation-api"),
    ExclusionRule("jline", "jline"),
    ExclusionRule("log4j", "log4j"),
    ExclusionRule("org.slf4j", "slf4j-log4j12"))
@ -152,8 +154,11 @@ object Dependencies {
  val jacksonDataTypeJsr310 = "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonVersion
  val jacksonDataFormatYam = "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % jacksonVersion
  val jacksonJaxrsBase = "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-base" % jacksonVersion
-  val jacksonJaxrsJsonProvider = "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-json-provider" % jacksonVersion
-  val jacksonModuleJaxbAnnotations = "com.fasterxml.jackson.module" % "jackson-module-jaxb-annotations" % jacksonVersion
+  val jacksonJaxrsJsonProvider = "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-json-provider" % jacksonVersion excludeAll (
+    ExclusionRule("jakarta.activation", "jakarta.activation-api"))
+  val jacksonModuleJaxbAnnotations = "com.fasterxml.jackson.module" % "jackson-module-jaxb-annotations" % jacksonVersion excludeAll (
+    ExclusionRule("jakarta.activation", "jakarta.activation-api"))
+  val jakartaActivationApi = "jakarta.activation" % "jakarta.activation-api" % jakartaActivationApiVersion
  val scalaReflect = "org.scala-lang" % "scala-reflect" % projectScalaVersion
  val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion
  val slf4jJulToSlf4j = "org.slf4j" % "jul-to-slf4j" % slf4jVersion
@ -185,7 +190,7 @@ object Dependencies {
  val httpClient5 = "org.apache.httpcomponents.client5" % "httpclient5" % httpClient5Version
  val httpCore5 = "org.apache.httpcomponents.core5" % "httpcore5" % httpCore5Version
  val httpCore5H2 = "org.apache.httpcomponents.core5" % "httpcore5-h2" % httpCore5Version
-  val javaxAnnotationApi = "javax.annotation" % "javax.annotation-api" % javaxAnnotationApiVersion
+  val jakartaAnnotationApi = "jakarta.annotation" % "jakarta.annotation-api" % jakartaAnnotationApiVersion

  // Test dependencies
  // https://www.scala-sbt.org/1.x/docs/Testing.html
@ -585,6 +590,7 @@ object CelebornService {
        Dependencies.slf4jApi,
        Dependencies.mybatis,
        Dependencies.hikaricp,
+        Dependencies.jacksonDataFormatYam,
        Dependencies.swaggerJaxrs2,
        Dependencies.swaggerUi,
        Dependencies.jakartaServletApi,
@ -1200,7 +1206,9 @@ object MRClientProjects {
        libraryDependencies ++= Seq(
          Dependencies.hadoopClientApi,
          Dependencies.hadoopClientRuntime,
-          Dependencies.hadoopMapreduceClientApp
+          Dependencies.hadoopMapreduceClientApp,
+          Dependencies.jacksonJaxrsJsonProvider,
+          Dependencies.jakartaActivationApi
        ) ++ commonUnitTestDependencies,
        dependencyOverrides += Dependencies.commonsCompress
      )
@ -1392,7 +1400,7 @@ object CelebornOpenApi {
        Dependencies.jacksonDataTypeJsr310,
        Dependencies.jacksonJaxrsJsonProvider,
        Dependencies.findbugsJsr305,
-        Dependencies.javaxAnnotationApi,
+        Dependencies.jakartaAnnotationApi,
        Dependencies.httpClient5,
        Dependencies.httpCore5,
        Dependencies.httpCore5H2,
--- a/service/pom.xml
+++ b/service/pom.xml
@ -56,21 +56,6 @@
        <artifactId>jetty-proxy</artifactId>
        <version>${jetty.version}</version>
      </dependency>
-      <dependency>
-        <groupId>com.fasterxml.jackson.jaxrs</groupId>
-        <artifactId>jackson-jaxrs-base</artifactId>
-        <version>${jackson.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>com.fasterxml.jackson.jaxrs</groupId>
-        <artifactId>jackson-jaxrs-json-provider</artifactId>
-        <version>${jackson.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>com.fasterxml.jackson.module</groupId>
-        <artifactId>jackson-module-jaxb-annotations</artifactId>
-        <version>${jackson.version}</version>
-      </dependency>
    </dependencies>
  </dependencyManagement>