diff --git a/.github/workflows/dep.yml b/.github/workflows/dep.yml index 96f49d8d9..c9cfdbf06 100644 --- a/.github/workflows/dep.yml +++ b/.github/workflows/dep.yml @@ -48,27 +48,15 @@ jobs: check-latest: false - name: Setup Maven uses: ./.github/actions/setup-maven - - name: Check kyuubi modules available - id: modules-check - run: >- - build/mvn dependency:resolve validate - -DincludeGroupIds="org.apache.kyuubi" -DincludeScope="compile" - -Pfast -Denforcer.skip=false - -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am - continue-on-error: true - - name: build + - name: Install modules env: - MAVEN_OPTS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error - if: steps.modules-check.conclusion == 'success' && steps.modules-check.outcome == 'failure' - run: >- - build/mvn clean install - -Pflink-provided,spark-provided,hive-provided - -Dmaven.javadoc.skip=true - -Drat.skip=true - -Dscalastyle.skip=true - -Dspotless.check.skip - -DskipTests - -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am + MAVEN_ARGS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error + run: | + build/mvn clean install \ + -Pflink-provided,spark-provided,hive-provided \ + -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip \ + -DskipTests \ + -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am - name: Check dependency list run: build/dependency.sh - name: Dependency Review diff --git a/LICENSE-binary b/LICENSE-binary index c4d8b185a..e78047f54 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -209,8 +209,6 @@ Apache License Version 2.0 -------------------------- com.zaxxer:HikariCP com.google.android:annotations -commons-lang:commons-lang -commons-logging:commons-logging org.apache.commons:commons-lang3 com.google.errorprone:error_prone_annotations dev.failsafe:failsafe diff --git a/NOTICE-binary b/NOTICE-binary index 237fdda56..b9bcdb214 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -115,9 +115,6 @@ Copyright 2002-2020 The Apache Software Foundation Apache Commons Lang Copyright 2001-2020 The Apache Software Foundation -Apache Commons Logging -Copyright 2003-2013 The Apache Software Foundation - Apache Hadoop Copyright 2006 and onwards The Apache Software Foundation. @@ -986,14 +983,6 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/logging/ - This product optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: diff --git a/dev/dependencyList b/dev/dependencyList index 348bf08b6..69bfd2de5 100644 --- a/dev/dependencyList +++ b/dev/dependencyList @@ -30,7 +30,6 @@ checker-qual/3.42.0//checker-qual-3.42.0.jar classgraph/4.8.138//classgraph-4.8.138.jar commons-codec/1.15//commons-codec-1.15.jar commons-lang3/3.13.0//commons-lang3-3.13.0.jar -commons-logging/1.1.3//commons-logging-1.1.3.jar error_prone_annotations/2.20.0//error_prone_annotations-2.20.0.jar failsafe/3.3.2//failsafe-3.3.2.jar failureaccess/1.0.1//failureaccess-1.0.1.jar diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala index 001768946..d85d93334 100644 --- a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala +++ b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala @@ -105,7 +105,14 @@ trait WithFlinkSQLEngineOnYarn extends KyuubiFunSuite with WithFlinkTestResource zkServer.start() conf.set(HA_ADDRESSES, zkServer.getConnectString) - hdfsCluster = new MiniDFSCluster.Builder(new Configuration) + val hdfsConf = new Configuration() + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") + + hdfsCluster = new MiniDFSCluster.Builder(hdfsConf) .numDataNodes(1) .checkDataNodeAddrConfig(true) .checkDataNodeHostConfig(true) diff --git a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala index b672bfb38..fc58a7833 100644 --- a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala +++ b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala @@ -20,6 +20,7 @@ package org.apache.kyuubi.it.flink import java.io.{File, FileWriter} import java.nio.file.Paths +import org.apache.hadoop.conf.Configuration import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.kyuubi.{KyuubiFunSuite, Utils, WithKyuubiServer} @@ -84,7 +85,13 @@ trait WithKyuubiServerAndYarnMiniCluster extends KyuubiFunSuite with WithKyuubiS } override def beforeAll(): Unit = { - miniHdfsService = new MiniDFSService() + val hdfsConf = new Configuration() + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") + miniHdfsService = new MiniDFSService(hdfsConf) miniHdfsService.initialize(conf) miniHdfsService.start() diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala index 1256687ae..6ffd157d6 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala @@ -130,6 +130,11 @@ class KyuubiOnKubernetesWithClusterSparkTestsSuite hdfsConf.set("dfs.namenode.servicerpc-bind-host", "0.0.0.0") hdfsConf.set("dfs.datanode.hostname", localhostAddress) hdfsConf.set("dfs.datanode.address", s"0.0.0.0:${NetUtils.getFreeSocketPort}") + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") // spark use 185 as userid in docker hdfsConf.set("hadoop.proxyuser.185.groups", "*") hdfsConf.set("hadoop.proxyuser.185.hosts", "*") diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala index cf4b3ff3b..be07dd557 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala @@ -106,6 +106,11 @@ class SparkClusterModeOnKubernetesSuiteBase hdfsConf.set("dfs.namenode.servicerpc-bind-host", "0.0.0.0") hdfsConf.set("dfs.datanode.hostname", localhostAddress) hdfsConf.set("dfs.datanode.address", s"0.0.0.0:${NetUtils.getFreeSocketPort}") + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") // spark use 185 as userid in docker hdfsConf.set("hadoop.proxyuser.185.groups", "*") hdfsConf.set("hadoop.proxyuser.185.hosts", "*") diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala index 90046ca37..64a521413 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala @@ -18,6 +18,7 @@ package org.apache.kyuubi import java.io.File +import org.apache.hadoop.conf.Configuration import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.kyuubi.config.KyuubiConf @@ -38,7 +39,13 @@ trait WithKyuubiServerAndHadoopMiniCluster extends KyuubiFunSuite with WithKyuub protected var miniYarnService: MiniYarnService = _ override def beforeAll(): Unit = { - miniHdfsService = new MiniDFSService() + val hdfsConf = new Configuration() + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") + miniHdfsService = new MiniDFSService(hdfsConf) miniHdfsService.initialize(conf) miniHdfsService.start() diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala index 43bea7c6a..f5f3eedc7 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala @@ -37,6 +37,12 @@ trait WithSecuredDFSService extends KerberizedTestHelper { hdfsConf.set("dfs.namenode.kerberos.internal.spnego.principal", testPrincipal) hdfsConf.set("dfs.web.authentication.kerberos.principal", testPrincipal) + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.address", "0.0.0.0:1025") hdfsConf.set("dfs.datanode.kerberos.principal", testPrincipal) hdfsConf.set("dfs.datanode.keytab.file", testKeytab) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala index 0e29f4411..d3307c269 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala @@ -26,7 +26,15 @@ trait WithSimpleDFSService extends KyuubiFunSuite { private var miniDFSService: MiniDFSService = _ - def hadoopConf: Configuration = new Configuration() + def hadoopConf: Configuration = { + val hdfsConf = new Configuration() + // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on + // commons-logging, we should disable it explicitly, otherwise, it throws + // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger + hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0") + hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0") + hdfsConf + } override def beforeAll(): Unit = { miniDFSService = new MiniDFSService(hadoopConf) diff --git a/pom.xml b/pom.xml index 0642f39b9..c900617c0 100644 --- a/pom.xml +++ b/pom.xml @@ -355,6 +355,10 @@ org.xerial.snappy snappy-java + + commons-logging + commons-logging +