From 460e553002eb773c596a286bf378b0ddbea1e9d8 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 11 Jun 2024 11:32:30 +0800 Subject: [PATCH] [KYUUBI #6458] Remove commons-logging from binary release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # :mag: Description [`jcl-over-slf4j`](https://www.slf4j.org/legacy.html#jcl-over-slf4j) is a drop-in replacement of `commons-logging`, the latter one should not be present in the final classpath, otherwise, there are potential class conflict issues. The current dep check is problematic, this PR also changes it to always perform "install" to fix the false negative report. ## Types of changes :bookmark: - [x] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan ๐Ÿงช Simply delete `commons-logging-1.1.3.jar` from `apache-kyuubi-1.9.1-bin.tgz` and everything goes well. --- # Checklist ๐Ÿ“ - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes #6458 from pan3793/commons-logging. Closes #6458 114ec766a [Cheng Pan] fix 79d4121a1 [Cheng Pan] fix 6633e83ee [Cheng Pan] fix 21127ed0b [Cheng Pan] always perform install on dep check 98b13dfcf [Cheng Pan] Remove commons-logging from binary release Authored-by: Cheng Pan Signed-off-by: Cheng Pan (cherry picked from commit 017d8ccd7ecb2f819363aa75bc7f047834c7fec1) Signed-off-by: Cheng Pan --- .github/workflows/dep.yml | 28 ++++++------------- LICENSE-binary | 2 -- NOTICE-binary | 11 -------- dev/dependencyList | 1 - .../flink/WithFlinkSQLEngineOnYarn.scala | 9 +++++- .../WithKyuubiServerAndYarnMiniCluster.scala | 9 +++++- .../KyuubiOnKubernetesTestsSuite.scala | 5 ++++ .../spark/SparkOnKubernetesTestsSuite.scala | 5 ++++ ...WithKyuubiServerAndHadoopMiniCluster.scala | 9 +++++- .../apache/kyuubi/WithSecuredDFSService.scala | 6 ++++ .../apache/kyuubi/WithSimpleDFSService.scala | 10 ++++++- pom.xml | 4 +++ 12 files changed, 61 insertions(+), 38 deletions(-) diff --git a/.github/workflows/dep.yml b/.github/workflows/dep.yml index 96f49d8d9..c9cfdbf06 100644 --- a/.github/workflows/dep.yml +++ b/.github/workflows/dep.yml @@ -48,27 +48,15 @@ jobs: check-latest: false - name: Setup Maven uses: ./.github/actions/setup-maven - - name: Check kyuubi modules available - id: modules-check - run: >- - build/mvn dependency:resolve validate - -DincludeGroupIds="org.apache.kyuubi" -DincludeScope="compile" - -Pfast -Denforcer.skip=false - -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am - continue-on-error: true - - name: build + - name: Install modules env: - MAVEN_OPTS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error - if: steps.modules-check.conclusion == 'success' && steps.modules-check.outcome == 'failure' - run: >- - build/mvn clean install - -Pflink-provided,spark-provided,hive-provided - -Dmaven.javadoc.skip=true - -Drat.skip=true - -Dscalastyle.skip=true - -Dspotless.check.skip - -DskipTests - -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am + MAVEN_ARGS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error + run: | + build/mvn clean install \ + -Pflink-provided,spark-provided,hive-provided \ + -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip \ + -DskipTests \ + -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am - name: Check dependency list run: build/dependency.sh - name: Dependency Review diff --git a/LICENSE-binary b/LICENSE-binary index c4d8b185a..e78047f54 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -209,8 +209,6 @@ Apache License Version 2.0 -------------------------- com.zaxxer:HikariCP com.google.android:annotations -commons-lang:commons-lang -commons-logging:commons-logging org.apache.commons:commons-lang3 com.google.errorprone:error_prone_annotations dev.failsafe:failsafe diff --git a/NOTICE-binary b/NOTICE-binary index 237fdda56..b9bcdb214 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -115,9 +115,6 @@ Copyright 2002-2020 The Apache Software Foundation Apache Commons Lang Copyright 2001-2020 The Apache Software Foundation -Apache Commons Logging -Copyright 2003-2013 The Apache Software Foundation - Apache Hadoop Copyright 2006 and onwards The Apache Software Foundation. @@ -986,14 +983,6 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/logging/ - This product optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: diff --git a/dev/dependencyList b/dev/dependencyList index 348bf08b6..69bfd2de5 100644 --- a/dev/dependencyList +++ b/dev/dependencyList @@ -30,7 +30,6 @@ checker-qual/3.42.0//checker-qual-3.42.0.jar classgraph/4.8.138//classgraph-4.8.138.jar commons-codec/1.15//commons-codec-1.15.jar commons-lang3/3.13.0//commons-lang3-3.13.0.jar -commons-logging/1.1.3//commons-logging-1.1.3.jar error_prone_annotations/2.20.0//error_prone_annotations-2.20.0.jar failsafe/3.3.2//failsafe-3.3.2.jar failureaccess/1.0.1//failureaccess-1.0.1.jar diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala index 001768946..d85d93334 100644 --- a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala +++ b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala @@ -105,7 +105,14 @@ trait WithFlinkSQLEngineOnYarn extends KyuubiFunSuite with WithFlinkTestResource zkServer.start() conf.set(HA_ADDRESSES, zkServer.getConnectString) - hdfsCluster = new MiniDFSCluster.Builder(new Configuration) + val hdfsConf = new Configuration() + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") + + hdfsCluster = new MiniDFSCluster.Builder(hdfsConf) .numDataNodes(1) .checkDataNodeAddrConfig(true) .checkDataNodeHostConfig(true) diff --git a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala index b672bfb38..fc58a7833 100644 --- a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala +++ b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala @@ -20,6 +20,7 @@ package org.apache.kyuubi.it.flink import java.io.{File, FileWriter} import java.nio.file.Paths +import org.apache.hadoop.conf.Configuration import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.kyuubi.{KyuubiFunSuite, Utils, WithKyuubiServer} @@ -84,7 +85,13 @@ trait WithKyuubiServerAndYarnMiniCluster extends KyuubiFunSuite with WithKyuubiS } override def beforeAll(): Unit = { - miniHdfsService = new MiniDFSService() + val hdfsConf = new Configuration() + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") + miniHdfsService = new MiniDFSService(hdfsConf) miniHdfsService.initialize(conf) miniHdfsService.start() diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala index 1256687ae..6ffd157d6 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala @@ -130,6 +130,11 @@ class KyuubiOnKubernetesWithClusterSparkTestsSuite hdfsConf.set("dfs.namenode.servicerpc-bind-host", "0.0.0.0") hdfsConf.set("dfs.datanode.hostname", localhostAddress) hdfsConf.set("dfs.datanode.address", s"0.0.0.0:${NetUtils.getFreeSocketPort}") + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") // spark use 185 as userid in docker hdfsConf.set("hadoop.proxyuser.185.groups", "*") hdfsConf.set("hadoop.proxyuser.185.hosts", "*") diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala index cf4b3ff3b..be07dd557 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala @@ -106,6 +106,11 @@ class SparkClusterModeOnKubernetesSuiteBase hdfsConf.set("dfs.namenode.servicerpc-bind-host", "0.0.0.0") hdfsConf.set("dfs.datanode.hostname", localhostAddress) hdfsConf.set("dfs.datanode.address", s"0.0.0.0:${NetUtils.getFreeSocketPort}") + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") // spark use 185 as userid in docker hdfsConf.set("hadoop.proxyuser.185.groups", "*") hdfsConf.set("hadoop.proxyuser.185.hosts", "*") diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala index 90046ca37..64a521413 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala @@ -18,6 +18,7 @@ package org.apache.kyuubi import java.io.File +import org.apache.hadoop.conf.Configuration import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.kyuubi.config.KyuubiConf @@ -38,7 +39,13 @@ trait WithKyuubiServerAndHadoopMiniCluster extends KyuubiFunSuite with WithKyuub protected var miniYarnService: MiniYarnService = _ override def beforeAll(): Unit = { - miniHdfsService = new MiniDFSService() + val hdfsConf = new Configuration() + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") + miniHdfsService = new MiniDFSService(hdfsConf) miniHdfsService.initialize(conf) miniHdfsService.start() diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala index 43bea7c6a..f5f3eedc7 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala @@ -37,6 +37,12 @@ trait WithSecuredDFSService extends KerberizedTestHelper { hdfsConf.set("dfs.namenode.kerberos.internal.spnego.principal", testPrincipal) hdfsConf.set("dfs.web.authentication.kerberos.principal", testPrincipal) + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.address", "0.0.0.0:1025") hdfsConf.set("dfs.datanode.kerberos.principal", testPrincipal) hdfsConf.set("dfs.datanode.keytab.file", testKeytab) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala index 0e29f4411..d3307c269 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala @@ -26,7 +26,15 @@ trait WithSimpleDFSService extends KyuubiFunSuite { private var miniDFSService: MiniDFSService = _ - def hadoopConf: Configuration = new Configuration() + def hadoopConf: Configuration = { + val hdfsConf = new Configuration() + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") + hdfsConf + } override def beforeAll(): Unit = { miniDFSService = new MiniDFSService(hadoopConf) diff --git a/pom.xml b/pom.xml index 0642f39b9..c900617c0 100644 --- a/pom.xml +++ b/pom.xml @@ -355,6 +355,10 @@ org.xerial.snappy snappy-java + + commons-logging + commons-logging +