[KYUUBI #6458] Remove commons-logging from binary release

# 🔍 Description [`jcl-over-slf4j`](https://www.slf4j.org/legacy.html#jcl-over-slf4j) is a drop-in replacement of `commons-logging`, the latter one should not be present in the final classpath, otherwise, there are potential class conflict issues. The current dep check is problematic, this PR also changes it to always perform "install" to fix the false negative report. ## Types of changes 🔖 - [x] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan 🧪 Simply delete `commons-logging-1.1.3.jar` from `apache-kyuubi-1.9.1-bin.tgz` and everything goes well. --- # Checklist 📝 - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes #6458 from pan3793/commons-logging. Closes #6458 114ec766a [Cheng Pan] fix 79d4121a1 [Cheng Pan] fix 6633e83ee [Cheng Pan] fix 21127ed0b [Cheng Pan] always perform install on dep check 98b13dfcf [Cheng Pan] Remove commons-logging from binary release Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org> (cherry picked from commit 017d8ccd7e) Signed-off-by: Cheng Pan <chengpan@apache.org>
2024-06-11 11:32:30 +08:00 · 2024-06-11 11:32:30 +08:00 · 460e553002
commit 460e553002
parent 20d639d313
12 changed files with 61 additions and 38 deletions
--- a/.github/workflows/dep.yml
+++ b/.github/workflows/dep.yml
@ -48,27 +48,15 @@ jobs:
          check-latest: false
      - name: Setup Maven
        uses: ./.github/actions/setup-maven
-      - name: Check kyuubi modules available
-        id: modules-check
-        run: >-
-          build/mvn dependency:resolve validate
-          -DincludeGroupIds="org.apache.kyuubi" -DincludeScope="compile"
-          -Pfast -Denforcer.skip=false
-          -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
-        continue-on-error: true
-      - name: build
+      - name: Install modules
        env:
-          MAVEN_OPTS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error
-        if: steps.modules-check.conclusion == 'success' && steps.modules-check.outcome == 'failure'
-        run: >-
-          build/mvn clean install
-          -Pflink-provided,spark-provided,hive-provided
-          -Dmaven.javadoc.skip=true
-          -Drat.skip=true
-          -Dscalastyle.skip=true
-          -Dspotless.check.skip
-          -DskipTests
-          -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
+          MAVEN_ARGS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error
+        run: |
+          build/mvn clean install \
+            -Pflink-provided,spark-provided,hive-provided \
+            -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip \
+            -DskipTests \
+            -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
      - name: Check dependency list
        run: build/dependency.sh
      - name: Dependency Review
--- a/2
+++ b/2
@ -209,8 +209,6 @@ Apache License Version 2.0
 --------------------------
 com.zaxxer:HikariCP
 com.google.android:annotations
-commons-lang:commons-lang
-commons-logging:commons-logging
 org.apache.commons:commons-lang3
 com.google.errorprone:error_prone_annotations
 dev.failsafe:failsafe
--- a/11
+++ b/11
@ -115,9 +115,6 @@ Copyright 2002-2020 The Apache Software Foundation
 Apache Commons Lang
 Copyright 2001-2020 The Apache Software Foundation

-Apache Commons Logging
-Copyright 2003-2013 The Apache Software Foundation
-
 Apache Hadoop
 Copyright 2006 and onwards The Apache Software Foundation.

@ -986,14 +983,6 @@ benchmarking framework, which can be obtained at:
  * HOMEPAGE:
    * https://github.com/google/caliper

-This product optionally depends on 'Apache Commons Logging', a logging
-framework, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.commons-logging.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * https://commons.apache.org/logging/
-
 This product optionally depends on 'Apache Log4J', a logging framework, which
 can be obtained at:

--- a/dev/dependencyList
+++ b/dev/dependencyList
@ -30,7 +30,6 @@ checker-qual/3.42.0//checker-qual-3.42.0.jar
 classgraph/4.8.138//classgraph-4.8.138.jar
 commons-codec/1.15//commons-codec-1.15.jar
 commons-lang3/3.13.0//commons-lang3-3.13.0.jar
-commons-logging/1.1.3//commons-logging-1.1.3.jar
 error_prone_annotations/2.20.0//error_prone_annotations-2.20.0.jar
 failsafe/3.3.2//failsafe-3.3.2.jar
 failureaccess/1.0.1//failureaccess-1.0.1.jar
--- a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala
+++ b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala
@ -105,7 +105,14 @@ trait WithFlinkSQLEngineOnYarn extends KyuubiFunSuite with WithFlinkTestResource
    zkServer.start()
    conf.set(HA_ADDRESSES, zkServer.getConnectString)

-    hdfsCluster = new MiniDFSCluster.Builder(new Configuration)
+    val hdfsConf = new Configuration()
+    // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+    // commons-logging, we should disable it explicitly, otherwise, it throws
+    // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+    hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+    hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
+
+    hdfsCluster = new MiniDFSCluster.Builder(hdfsConf)
      .numDataNodes(1)
      .checkDataNodeAddrConfig(true)
      .checkDataNodeHostConfig(true)
--- a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala
+++ b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala
@ -20,6 +20,7 @@ package org.apache.kyuubi.it.flink
 import java.io.{File, FileWriter}
 import java.nio.file.Paths

+import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.yarn.conf.YarnConfiguration

 import org.apache.kyuubi.{KyuubiFunSuite, Utils, WithKyuubiServer}
@ -84,7 +85,13 @@ trait WithKyuubiServerAndYarnMiniCluster extends KyuubiFunSuite with WithKyuubiS
  }

  override def beforeAll(): Unit = {
-    miniHdfsService = new MiniDFSService()
+    val hdfsConf = new Configuration()
+    // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+    // commons-logging, we should disable it explicitly, otherwise, it throws
+    // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+    hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+    hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
+    miniHdfsService = new MiniDFSService(hdfsConf)
    miniHdfsService.initialize(conf)
    miniHdfsService.start()

--- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
@ -130,6 +130,11 @@ class KyuubiOnKubernetesWithClusterSparkTestsSuite
    hdfsConf.set("dfs.namenode.servicerpc-bind-host", "0.0.0.0")
    hdfsConf.set("dfs.datanode.hostname", localhostAddress)
    hdfsConf.set("dfs.datanode.address", s"0.0.0.0:${NetUtils.getFreeSocketPort}")
+    // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+    // commons-logging, we should disable it explicitly, otherwise, it throws
+    // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+    hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+    hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
    // spark use 185 as userid in docker
    hdfsConf.set("hadoop.proxyuser.185.groups", "*")
    hdfsConf.set("hadoop.proxyuser.185.hosts", "*")
--- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
@ -106,6 +106,11 @@ class SparkClusterModeOnKubernetesSuiteBase
    hdfsConf.set("dfs.namenode.servicerpc-bind-host", "0.0.0.0")
    hdfsConf.set("dfs.datanode.hostname", localhostAddress)
    hdfsConf.set("dfs.datanode.address", s"0.0.0.0:${NetUtils.getFreeSocketPort}")
+    // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+    // commons-logging, we should disable it explicitly, otherwise, it throws
+    // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+    hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+    hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
    // spark use 185 as userid in docker
    hdfsConf.set("hadoop.proxyuser.185.groups", "*")
    hdfsConf.set("hadoop.proxyuser.185.hosts", "*")
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala
@ -18,6 +18,7 @@ package org.apache.kyuubi

 import java.io.File

+import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.yarn.conf.YarnConfiguration

 import org.apache.kyuubi.config.KyuubiConf
@ -38,7 +39,13 @@ trait WithKyuubiServerAndHadoopMiniCluster extends KyuubiFunSuite with WithKyuub
  protected var miniYarnService: MiniYarnService = _

  override def beforeAll(): Unit = {
-    miniHdfsService = new MiniDFSService()
+    val hdfsConf = new Configuration()
+    // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+    // commons-logging, we should disable it explicitly, otherwise, it throws
+    // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+    hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+    hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
+    miniHdfsService = new MiniDFSService(hdfsConf)
    miniHdfsService.initialize(conf)
    miniHdfsService.start()

--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala
@ -37,6 +37,12 @@ trait WithSecuredDFSService extends KerberizedTestHelper {
    hdfsConf.set("dfs.namenode.kerberos.internal.spnego.principal", testPrincipal)
    hdfsConf.set("dfs.web.authentication.kerberos.principal", testPrincipal)

+    // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+    // commons-logging, we should disable it explicitly, otherwise, it throws
+    // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+    hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+    hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
+
    hdfsConf.set("dfs.datanode.address", "0.0.0.0:1025")
    hdfsConf.set("dfs.datanode.kerberos.principal", testPrincipal)
    hdfsConf.set("dfs.datanode.keytab.file", testKeytab)
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala
@ -26,7 +26,15 @@ trait WithSimpleDFSService extends KyuubiFunSuite {

  private var miniDFSService: MiniDFSService = _

-  def hadoopConf: Configuration = new Configuration()
+  def hadoopConf: Configuration = {
+    val hdfsConf = new Configuration()
+    // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+    // commons-logging, we should disable it explicitly, otherwise, it throws
+    // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+    hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+    hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
+    hdfsConf
+  }

  override def beforeAll(): Unit = {
    miniDFSService = new MiniDFSService(hadoopConf)
--- a/pom.xml
+++ b/pom.xml
@ -355,6 +355,10 @@
                        <groupId>org.xerial.snappy</groupId>
                        <artifactId>snappy-java</artifactId>
                    </exclusion>
+                    <exclusion>
+                        <groupId>commons-logging</groupId>
+                        <artifactId>commons-logging</artifactId>
+                    </exclusion>
                </exclusions>
            </dependency>