From 53d59a02bf04bc5e1b81119ee56cdc6e819c4d8a Mon Sep 17 00:00:00 2001 From: Fei Wang Date: Wed, 26 Jan 2022 10:53:16 +0800 Subject: [PATCH] [KYUUBI #1769] [BUILD] Migrate from log4j1 to log4j2 ### _Why are the changes needed?_ This patch proposes to migrate from log4j1 to log4j2 in Kyuubi. Refer the spark patch: https://github.com/apache/spark/pull/34895 ### Does this PR introduce any user-facing change? Yes. Users may need to rewrite their log4j properties file for log4j2. As of version 2.4, log4j now supports configuration via properties files. Note that the property syntax is not the same as the syntax used in log4j 1, but during the migration I found the syntax is pretty close so the migration should be straightforward. ### _How was this patch tested?_ Passed all existing tests. Closes #1769 from turboFei/log4j2. Closes #1769 8613779c [Fei Wang] remove log4j dependencies from spark-sql-engine b2fe6dba [Fei Wang] Use String to present default log level 762e2d03 [Fei Wang] only remove org.apache.logging.log4j:log4j-slf4j-impl 8a912086 [Fei Wang] remove dependencies from spark-sql-engine 7e3a4980 [Fei Wang] address commments 051f49f5 [Fei Wang] address comments 85316a0b [Fei Wang] Keep compatible with log4j12 01d1a84e [Fei Wang] for log4j1 b9e17e1b [Fei Wang] refactor e24391ed [Fei Wang] revert log count 38803002 [Fei Wang] add log4j2.properties.template 4f0b22fc [Fei Wang] save 7ce84119 [Fei Wang] modify log level 1ea5ca53 [Fei Wang] add log4j to engine c4a86d4d [Fei Wang] use AbstractFilter 27b08b6a [Fei Wang] remove more 8cc15ae7 [Fei Wang] reformat c13ec29e [Fei Wang] save temporarily 33a38e2e [Fei Wang] exclude log4j12 from spark-sql 9129a64a [Fei Wang] refactor 5362b43d [Fei Wang] make it run at first 7f27f519 [Fei Wang] more 56f4f1ff [Fei Wang] fix logging a74b6d37 [Fei Wang] start appender dea964aa [Fei Wang] fix build erorr at first e20b7500 [Fei Wang] address comments 2ec02b4d [Fei Wang] fix LogDivertAppender dded1290 [Fei Wang] more c63e0008 [Fei Wang] add log4j2.properties Authored-by: Fei Wang Signed-off-by: Fei Wang --- conf/log4j2.properties.template | 51 +++++++++ dev/dependencyList | 4 + .../src/test/resources/log4j2.properties | 57 ++++++++++ .../src/test/resources/log4j2.properties | 57 ++++++++++ .../src/test/resources/log4j2.properties | 57 ++++++++++ docs/deployment/settings.md | 38 +++++-- .../src/test/resources/log4j2.properties | 57 ++++++++++ .../src/test/resources/log4j2.properties | 57 ++++++++++ .../src/test/resources/log4j2.properties | 57 ++++++++++ .../src/test/resources/log4j2.properties | 57 ++++++++++ kyuubi-assembly/pom.xml | 19 +++- kyuubi-common/pom.xml | 19 +++- .../main/resources/log4j2-defaults.properties | 38 +++++++ .../scala/org/apache/kyuubi/Logging.scala | 60 ++++++++-- .../operation/log/Log4j12DivertAppender.scala | 63 +++++++++++ .../operation/log/Log4j2DivertAppender.scala | 103 ++++++++++++++++++ .../operation/log/LogDivertAppender.scala | 49 ++------- .../src/test/resources/log4j.properties | 2 +- .../src/test/resources/log4j2.properties | 57 ++++++++++ .../org/apache/kyuubi/KyuubiFunSuite.scala | 72 ++++++++---- .../src/test/resources/log4j.properties | 2 +- .../src/test/resources/log4j2.properties | 57 ++++++++++ .../ctl/ServiceControlCliArgumentsSuite.scala | 6 +- .../src/test/resources/log4j2.properties | 57 ++++++++++ .../ha/client/ServiceDiscoverySuite.scala | 3 +- kyuubi-hive-beeline/pom.xml | 19 +++- .../src/test/resources/log4j2.properties | 57 ++++++++++ .../src/test/resources/log4j2.properties | 57 ++++++++++ .../config/AllKyuubiConfiguration.scala | 4 +- .../src/test/resources/log4j2.properties | 57 ++++++++++ pom.xml | 33 +++++- 31 files changed, 1230 insertions(+), 96 deletions(-) create mode 100644 conf/log4j2.properties.template create mode 100644 dev/kyuubi-extension-spark-3-1/src/test/resources/log4j2.properties create mode 100644 dev/kyuubi-extension-spark-3-2/src/test/resources/log4j2.properties create mode 100644 dev/kyuubi-extension-spark-common/src/test/resources/log4j2.properties create mode 100644 externals/kyuubi-flink-sql-engine/src/test/resources/log4j2.properties create mode 100644 externals/kyuubi-spark-sql-engine/src/test/resources/log4j2.properties create mode 100644 externals/kyuubi-trino-engine/src/test/resources/log4j2.properties create mode 100644 integration-tests/kyuubi-kubernetes-deployment-it/src/test/resources/log4j2.properties create mode 100644 kyuubi-common/src/main/resources/log4j2-defaults.properties create mode 100644 kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j12DivertAppender.scala create mode 100644 kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala create mode 100644 kyuubi-common/src/test/resources/log4j2.properties create mode 100644 kyuubi-ctl/src/test/resources/log4j2.properties create mode 100644 kyuubi-ha/src/test/resources/log4j2.properties create mode 100644 kyuubi-metrics/src/test/resources/log4j2.properties create mode 100644 kyuubi-server/src/test/resources/log4j2.properties create mode 100644 kyuubi-zookeeper/src/test/resources/log4j2.properties diff --git a/conf/log4j2.properties.template b/conf/log4j2.properties.template new file mode 100644 index 000000000..04d7bdedc --- /dev/null +++ b/conf/log4j2.properties.template @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = info + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# Set the default kyuubi-ctl log level to WARN. When running the kyuubi-ctl, the +# log level for this class is used to overwrite the root logger's log level. +logger.ctl.name = org.apache.kyuubi.ctl.ServiceControlCli +logger.ctl.level = error + +# Analysis MySQLFrontend protocol traffic +# logger.mysql.name = org.apache.kyuubi.server.mysql.codec +# logger.mysql.level = trace + +# Kyuubi BeeLine +logger.beeline.name = org.apache.hive.beeline.KyuubiBeeLine +logger.beeline.level = error diff --git a/dev/dependencyList b/dev/dependencyList index 7f3b9426b..8ef7fb7bf 100644 --- a/dev/dependencyList +++ b/dev/dependencyList @@ -72,6 +72,10 @@ jetty-util/9.4.41.v20210516//jetty-util-9.4.41.v20210516.jar jline/0.9.94//jline-0.9.94.jar libfb303/0.9.3//libfb303-0.9.3.jar libthrift/0.9.3//libthrift-0.9.3.jar +log4j-1.2-api/2.17.1//log4j-1.2-api-2.17.1.jar +log4j-api/2.17.1//log4j-api-2.17.1.jar +log4j-core/2.17.1//log4j-core-2.17.1.jar +log4j-slf4j-impl/2.17.1//log4j-slf4j-impl-2.17.1.jar log4j/1.2.17//log4j-1.2.17.jar metrics-core/4.1.1//metrics-core-4.1.1.jar metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar diff --git a/dev/kyuubi-extension-spark-3-1/src/test/resources/log4j2.properties b/dev/kyuubi-extension-spark-3-1/src/test/resources/log4j2.properties new file mode 100644 index 000000000..6f4ea28ab --- /dev/null +++ b/dev/kyuubi-extension-spark-3-1/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/dev/kyuubi-extension-spark-3-2/src/test/resources/log4j2.properties b/dev/kyuubi-extension-spark-3-2/src/test/resources/log4j2.properties new file mode 100644 index 000000000..6f4ea28ab --- /dev/null +++ b/dev/kyuubi-extension-spark-3-2/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/dev/kyuubi-extension-spark-common/src/test/resources/log4j2.properties b/dev/kyuubi-extension-spark-common/src/test/resources/log4j2.properties new file mode 100644 index 000000000..6f4ea28ab --- /dev/null +++ b/dev/kyuubi-extension-spark-common/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md index 77ab1c677..edf7e4d84 100644 --- a/docs/deployment/settings.md +++ b/docs/deployment/settings.md @@ -370,7 +370,7 @@ Please refer to the Spark official online documentation for [SET Command](http:/ ## Logging -Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging. You can configure it using `$KYUUBI_HOME/conf/log4j.properties`. +Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging. You can configure it using `$KYUUBI_HOME/conf/log4j2.properties`. ```bash # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -389,22 +389,40 @@ Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging. You can # limitations under the License. # -# Set everything to be logged to the console -log4j.rootCategory=INFO, console -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c{2}: %m%n +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = info + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral # Set the default kyuubi-ctl log level to WARN. When running the kyuubi-ctl, the # log level for this class is used to overwrite the root logger's log level. -log4j.logger.org.apache.kyuubi.ctl.ServiceControlCli=ERROR +logger.ctl.name = org.apache.kyuubi.ctl.ServiceControlCli +logger.ctl.level = error # Analysis MySQLFrontend protocol traffic -# log4j.logger.org.apache.kyuubi.server.mysql.codec=TRACE +# logger.mysql.name = org.apache.kyuubi.server.mysql.codec +# logger.mysql.level = trace # Kyuubi BeeLine -log4j.logger.org.apache.hive.beeline.KyuubiBeeLine=ERROR +logger.beeline.name = org.apache.hive.beeline.KyuubiBeeLine +logger.beeline.level = error ``` ## Other Configurations diff --git a/externals/kyuubi-flink-sql-engine/src/test/resources/log4j2.properties b/externals/kyuubi-flink-sql-engine/src/test/resources/log4j2.properties new file mode 100644 index 000000000..fafe9b2a1 --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/externals/kyuubi-spark-sql-engine/src/test/resources/log4j2.properties b/externals/kyuubi-spark-sql-engine/src/test/resources/log4j2.properties new file mode 100644 index 000000000..fafe9b2a1 --- /dev/null +++ b/externals/kyuubi-spark-sql-engine/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/externals/kyuubi-trino-engine/src/test/resources/log4j2.properties b/externals/kyuubi-trino-engine/src/test/resources/log4j2.properties new file mode 100644 index 000000000..fafe9b2a1 --- /dev/null +++ b/externals/kyuubi-trino-engine/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/integration-tests/kyuubi-kubernetes-deployment-it/src/test/resources/log4j2.properties b/integration-tests/kyuubi-kubernetes-deployment-it/src/test/resources/log4j2.properties new file mode 100644 index 000000000..45d900984 --- /dev/null +++ b/integration-tests/kyuubi-kubernetes-deployment-it/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to INFO +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = info diff --git a/kyuubi-assembly/pom.xml b/kyuubi-assembly/pom.xml index baba1db1a..f7150cf83 100644 --- a/kyuubi-assembly/pom.xml +++ b/kyuubi-assembly/pom.xml @@ -107,8 +107,23 @@ - org.slf4j - slf4j-log4j12 + org.apache.logging.log4j + log4j-slf4j-impl + + + + org.apache.logging.log4j + log4j-api + + + + org.apache.logging.log4j + log4j-core + + + + org.apache.logging.log4j + log4j-1.2-api diff --git a/kyuubi-common/pom.xml b/kyuubi-common/pom.xml index a850cad77..a4fbadb3c 100644 --- a/kyuubi-common/pom.xml +++ b/kyuubi-common/pom.xml @@ -48,8 +48,23 @@ - org.slf4j - slf4j-log4j12 + org.apache.logging.log4j + log4j-slf4j-impl + + + + org.apache.logging.log4j + log4j-api + + + + org.apache.logging.log4j + log4j-core + + + + org.apache.logging.log4j + log4j-1.2-api diff --git a/kyuubi-common/src/main/resources/log4j2-defaults.properties b/kyuubi-common/src/main/resources/log4j2-defaults.properties new file mode 100644 index 000000000..ab54692b4 --- /dev/null +++ b/kyuubi-common/src/main/resources/log4j2-defaults.properties @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the console +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala index 4ec783dfa..9bf904578 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala @@ -17,7 +17,8 @@ package org.apache.kyuubi -import org.apache.log4j.{Level, LogManager, PropertyConfigurator} +import org.apache.logging.log4j.{Level, LogManager} +import org.apache.logging.log4j.core.LoggerContext import org.slf4j.{Logger, LoggerFactory} import org.slf4j.impl.StaticLoggerBinder @@ -90,32 +91,64 @@ trait Logging { } private def initializeLogging(isInterpreter: Boolean): Unit = { - if (Logging.isLog4j12) { - val log4j12Initialized = LogManager.getRootLogger.getAllAppenders.hasMoreElements + if (Logging.isLog4j2) { + val log4j2Initialized = !LogManager.getRootLogger + .asInstanceOf[org.apache.logging.log4j.core.Logger].getAppenders.isEmpty // scalastyle:off println - if (!log4j12Initialized) { + if (!log4j2Initialized) { Logging.useDefault = true - val defaultLogProps = "log4j-defaults.properties" + val defaultLogProps = "log4j2-defaults.properties" Option(Thread.currentThread().getContextClassLoader.getResource(defaultLogProps)) match { case Some(url) => - PropertyConfigurator.configure(url) + val context = LogManager.getContext(false).asInstanceOf[LoggerContext] + context.setConfigLocation(url.toURI) case None => System.err.println(s"Missing $defaultLogProps") } } val rootLogger = LogManager.getRootLogger + .asInstanceOf[org.apache.logging.log4j.core.Logger] if (Logging.defaultRootLevel == null) { - Logging.defaultRootLevel = rootLogger.getLevel + Logging.defaultRootLevel = rootLogger.getLevel.toString } if (isInterpreter) { // set kyuubi ctl log level, default ERROR val ctlLogger = LogManager.getLogger(loggerName) + .asInstanceOf[org.apache.logging.log4j.core.Logger] val ctlLevel = Option(ctlLogger.getLevel()).getOrElse(Level.ERROR) rootLogger.setLevel(ctlLevel) } // scalastyle:on println + } else if (Logging.isLog4j12) { + val log4j12Initialized = + org.apache.log4j.LogManager.getRootLogger.getAllAppenders.hasMoreElements + // scalastyle:off println + if (!log4j12Initialized) { + Logging.useDefault = true + val defaultLogProps = "log4j-defaults.properties" + Option(Thread.currentThread().getContextClassLoader.getResource(defaultLogProps)) match { + case Some(url) => + org.apache.log4j.PropertyConfigurator.configure(url) + + case None => + System.err.println(s"Missing $defaultLogProps") + } + + val rootLogger = org.apache.log4j.LogManager.getRootLogger + if (Logging.defaultRootLevel == null) { + Logging.defaultRootLevel = rootLogger.getLevel.toString + } + + if (isInterpreter) { + // set kyuubi ctl log level, default ERROR + val ctlLogger = org.apache.log4j.LogManager.getLogger(loggerName) + val ctlLevel = Option(ctlLogger.getLevel()).getOrElse(org.apache.log4j.Level.ERROR) + rootLogger.setLevel(ctlLevel) + } + // scalastyle:on println + } } Logging.initialized = true @@ -127,14 +160,23 @@ trait Logging { object Logging { @volatile private var useDefault = false - @volatile private var defaultRootLevel: Level = _ + @volatile private var defaultRootLevel: String = _ @volatile private var initialized = false val initLock = new Object() - private def isLog4j12: Boolean = { + + private[kyuubi] def isLog4j12: Boolean = { // This distinguishes the log4j 1.2 binding, currently // org.slf4j.impl.Log4jLoggerFactory, from the log4j 2.0 binding, currently // org.apache.logging.slf4j.Log4jLoggerFactory val binderClass = StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr "org.slf4j.impl.Log4jLoggerFactory".equals(binderClass) } + + private[kyuubi] def isLog4j2: Boolean = { + // This distinguishes the log4j 1.2 binding, currently + // org.slf4j.impl.Log4jLoggerFactory, from the log4j 2.0 binding, currently + // org.apache.logging.slf4j.Log4jLoggerFactory + val binderClass = StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr + "org.apache.logging.slf4j.Log4jLoggerFactory".equals(binderClass) + } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j12DivertAppender.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j12DivertAppender.scala new file mode 100644 index 000000000..1191e94ae --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j12DivertAppender.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.operation.log + +import java.io.CharArrayWriter + +import scala.collection.JavaConverters._ + +import org.apache.log4j._ +import org.apache.log4j.spi.{Filter, LoggingEvent} + +class Log4j12DivertAppender extends WriterAppender { + + final private val writer = new CharArrayWriter + + final private val lo = Logger.getRootLogger + .getAllAppenders.asScala + .find(_.isInstanceOf[ConsoleAppender]) + .map(_.asInstanceOf[Appender].getLayout) + .getOrElse(new PatternLayout("%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n")) + + setName("KyuubiSparkSQLEngineLogDivertAppender") + setWriter(writer) + setLayout(lo) + + addFilter { _: LoggingEvent => + if (OperationLog.getCurrentOperationLog == null) Filter.DENY else Filter.NEUTRAL + } + + /** + * Overrides WriterAppender.subAppend(), which does the real logging. No need + * to worry about concurrency since log4j calls this synchronously. + */ + override protected def subAppend(event: LoggingEvent): Unit = { + super.subAppend(event) + // That should've gone into our writer. Notify the LogContext. + val logOutput = writer.toString + writer.reset() + val log = OperationLog.getCurrentOperationLog + if (log != null) log.write(logOutput) + } +} + +object Log4j12DivertAppender { + def initialize(): Unit = { + org.apache.log4j.Logger.getRootLogger.addAppender(new Log4j12DivertAppender()) + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala new file mode 100644 index 000000000..82ce6feb9 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.operation.log + +import java.io.CharArrayWriter + +import scala.collection.JavaConverters._ + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.core.{Filter, LogEvent, StringLayout} +import org.apache.logging.log4j.core.appender.{AbstractWriterAppender, ConsoleAppender, WriterManager} +import org.apache.logging.log4j.core.config.Property +import org.apache.logging.log4j.core.filter.AbstractFilter +import org.apache.logging.log4j.core.layout.PatternLayout + +class Log4j2DivertAppender( + name: String, + layout: StringLayout, + filter: Filter, + ignoreExceptions: Boolean, + immediateFlush: Boolean, + writer: CharArrayWriter) + extends AbstractWriterAppender[WriterManager]( + name, + layout, + filter, + ignoreExceptions, + immediateFlush, + Property.EMPTY_ARRAY, + new WriterManager(writer, name, layout, true)) { + def this() = this( + "KyuubiEngineLogDivertAppender", + Log4j2DivertAppender.initLayout(), + null, + false, + true, + new CharArrayWriter()) + + addFilter(new AbstractFilter() { + override def filter(event: LogEvent): Filter.Result = { + if (OperationLog.getCurrentOperationLog == null) { + Filter.Result.DENY + } else { + Filter.Result.NEUTRAL + } + } + }) + + def initLayout(): StringLayout = { + LogManager.getRootLogger.asInstanceOf[org.apache.logging.log4j.core.Logger] + .getAppenders.values().asScala + .find(ap => ap.isInstanceOf[ConsoleAppender] && ap.getLayout.isInstanceOf[StringLayout]) + .map(_.getLayout.asInstanceOf[StringLayout]) + .getOrElse(PatternLayout.newBuilder().withPattern( + "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n").build()) + } + + /** + * Overrides AbstractWriterAppender.append(), which does the real logging. No need + * to worry about concurrency since log4j calls this synchronously. + */ + override def append(event: LogEvent): Unit = { + super.append(event) + // That should've gone into our writer. Notify the LogContext. + val logOutput = writer.toString + writer.reset() + val log = OperationLog.getCurrentOperationLog + if (log != null) log.write(logOutput) + } +} + +object Log4j2DivertAppender { + def initLayout(): StringLayout = { + LogManager.getRootLogger.asInstanceOf[org.apache.logging.log4j.core.Logger] + .getAppenders.values().asScala + .find(ap => ap.isInstanceOf[ConsoleAppender] && ap.getLayout.isInstanceOf[StringLayout]) + .map(_.getLayout.asInstanceOf[StringLayout]) + .getOrElse(PatternLayout.newBuilder().withPattern( + "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n").build()) + } + + def initialize(): Unit = { + val ap = new Log4j2DivertAppender() + org.apache.logging.log4j.LogManager.getRootLogger() + .asInstanceOf[org.apache.logging.log4j.core.Logger].addAppender(ap) + ap.start() + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/LogDivertAppender.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/LogDivertAppender.scala index 57585df88..6766962fe 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/LogDivertAppender.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/LogDivertAppender.scala @@ -17,47 +17,18 @@ package org.apache.kyuubi.operation.log -import java.io.CharArrayWriter +import org.slf4j.impl.StaticLoggerBinder -import scala.collection.JavaConverters._ +import org.apache.kyuubi.Logging -import org.apache.log4j._ -import org.apache.log4j.spi.{Filter, LoggingEvent} - -class LogDivertAppender extends WriterAppender { - - final private val writer = new CharArrayWriter - - final private val lo = Logger.getRootLogger - .getAllAppenders.asScala - .find(_.isInstanceOf[ConsoleAppender]) - .map(_.asInstanceOf[Appender].getLayout) - .getOrElse(new PatternLayout("%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n")) - - setName("KyuubiSparkSQLEngineLogDivertAppender") - setWriter(writer) - setLayout(lo) - - addFilter { _: LoggingEvent => - if (OperationLog.getCurrentOperationLog == null) Filter.DENY else Filter.NEUTRAL - } - - /** - * Overrides WriterAppender.subAppend(), which does the real logging. No need - * to worry about concurrency since log4j calls this synchronously. - */ - override protected def subAppend(event: LoggingEvent): Unit = { - super.subAppend(event) - // That should've gone into our writer. Notify the LogContext. - val logOutput = writer.toString - writer.reset() - val log = OperationLog.getCurrentOperationLog - if (log != null) log.write(logOutput) - } -} - -object LogDivertAppender { +object LogDivertAppender extends Logging { def initialize(): Unit = { - org.apache.log4j.Logger.getRootLogger.addAppender(new LogDivertAppender()) + if (Logging.isLog4j2) { + Log4j2DivertAppender.initialize() + } else if (Logging.isLog4j12) { + Log4j12DivertAppender.initialize() + } else { + warn(s"Unsupported SLF4J binding ${StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr}") + } } } diff --git a/kyuubi-common/src/test/resources/log4j.properties b/kyuubi-common/src/test/resources/log4j.properties index dc7155f25..fd7d4e8a8 100644 --- a/kyuubi-common/src/test/resources/log4j.properties +++ b/kyuubi-common/src/test/resources/log4j.properties @@ -31,7 +31,7 @@ log4j.appender.FA.file=target/unit-tests.log log4j.appender.FA.layout=org.apache.log4j.PatternLayout log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n -# Set the logger level of File Appender to WARN +# Set the logger level of File Appender to DEBUG log4j.appender.FA.Threshold = DEBUG # SPARK-34128: Suppress undesirable TTransportException warnings involved in THRIFT-4805 diff --git a/kyuubi-common/src/test/resources/log4j2.properties b/kyuubi-common/src/test/resources/log4j2.properties new file mode 100644 index 000000000..fafe9b2a1 --- /dev/null +++ b/kyuubi-common/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala index 029278305..d4b5fdf59 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala @@ -20,8 +20,10 @@ package org.apache.kyuubi import scala.collection.mutable.ArrayBuffer // scalastyle:off -import org.apache.log4j.{Appender, AppenderSkeleton, Level, Logger} -import org.apache.log4j.spi.LoggingEvent +import org.apache.logging.log4j._ +import org.apache.logging.log4j.core.{LogEvent, Logger, LoggerContext} +import org.apache.logging.log4j.core.appender.AbstractAppender +import org.apache.logging.log4j.core.config.Property import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Outcome} import org.scalatest.concurrent.Eventually import org.scalatest.funsuite.AnyFunSuite @@ -70,38 +72,66 @@ trait KyuubiFunSuite extends AnyFunSuite * appender and restores the log level if necessary. */ final def withLogAppender( - appender: Appender, - loggerName: Option[String] = None, + appender: AbstractAppender, + loggerNames: Seq[String] = Seq.empty, level: Option[Level] = None)( f: => Unit): Unit = { - val logger = loggerName.map(Logger.getLogger).getOrElse(Logger.getRootLogger) - val restoreLevel = logger.getLevel - logger.addAppender(appender) - if (level.isDefined) { - logger.setLevel(level.get) + val loggers = + if (loggerNames.nonEmpty) { + loggerNames.map(LogManager.getLogger) + } else { + Seq(LogManager.getRootLogger) + } + val restoreLevels = loggers.map(_.getLevel) + loggers.foreach { l => + val logger = l.asInstanceOf[Logger] + logger.addAppender(appender) + appender.start() + if (level.isDefined) { + logger.setLevel(level.get) + logger.get().setLevel(level.get) + LogManager.getContext(false).asInstanceOf[LoggerContext].updateLoggers() + } } try f finally { - logger.removeAppender(appender) + loggers.foreach(_.asInstanceOf[Logger].removeAppender(appender)) + appender.stop() if (level.isDefined) { - logger.setLevel(restoreLevel) + loggers.zipWithIndex.foreach { case (logger, i) => + logger.asInstanceOf[Logger].setLevel(restoreLevels(i)) + logger.asInstanceOf[Logger].get().setLevel(restoreLevels(i)) + } } } } - class LogAppender(msg: String = "", maxEvents: Int = 1000) extends AppenderSkeleton { - val loggingEvents = new ArrayBuffer[LoggingEvent]() + class LogAppender(msg: String = "", maxEvents: Int = 1000) + extends AbstractAppender("logAppender", null, null, true, Property.EMPTY_ARRAY) { + private val _loggingEvents = new ArrayBuffer[LogEvent]() + private var _threshold: Level = Level.INFO - override def append(loggingEvent: LoggingEvent): Unit = { - if (loggingEvents.size >= maxEvents) { - val loggingInfo = if (msg == "") "." else s" while logging $msg." - throw new IllegalStateException( - s"Number of events reached the limit of $maxEvents$loggingInfo") + override def append(loggingEvent: LogEvent): Unit = loggingEvent.synchronized { + val copyEvent = loggingEvent.toImmutable + if (copyEvent.getLevel.isMoreSpecificThan(_threshold)) { + _loggingEvents.synchronized { + if (_loggingEvents.size >= maxEvents) { + val loggingInfo = if (msg == "") "." else s" while logging $msg." + throw new IllegalStateException( + s"Number of events reached the limit of $maxEvents$loggingInfo") + } + _loggingEvents.append(copyEvent) + } } - loggingEvents.append(loggingEvent) } - override def close(): Unit = {} - override def requiresLayout(): Boolean = false + + def setThreshold(threshold: Level): Unit = { + _threshold = threshold + } + + def loggingEvents: ArrayBuffer[LogEvent] = _loggingEvents.synchronized { + _loggingEvents.filterNot(_ == null) + } } final def withSystemProperty(key: String, value: String)(f: => Unit): Unit = { diff --git a/kyuubi-ctl/src/test/resources/log4j.properties b/kyuubi-ctl/src/test/resources/log4j.properties index dc7155f25..fd7d4e8a8 100644 --- a/kyuubi-ctl/src/test/resources/log4j.properties +++ b/kyuubi-ctl/src/test/resources/log4j.properties @@ -31,7 +31,7 @@ log4j.appender.FA.file=target/unit-tests.log log4j.appender.FA.layout=org.apache.log4j.PatternLayout log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n -# Set the logger level of File Appender to WARN +# Set the logger level of File Appender to DEBUG log4j.appender.FA.Threshold = DEBUG # SPARK-34128: Suppress undesirable TTransportException warnings involved in THRIFT-4805 diff --git a/kyuubi-ctl/src/test/resources/log4j2.properties b/kyuubi-ctl/src/test/resources/log4j2.properties new file mode 100644 index 000000000..fafe9b2a1 --- /dev/null +++ b/kyuubi-ctl/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ServiceControlCliArgumentsSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ServiceControlCliArgumentsSuite.scala index ec5d36a1d..fb3dfb416 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ServiceControlCliArgumentsSuite.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ServiceControlCliArgumentsSuite.scala @@ -42,7 +42,8 @@ class ServiceControlCliArgumentsSuite extends KyuubiFunSuite { } thread.start() thread.join() - assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(searchString))) + assert(logAppender.loggingEvents.exists( + _.getMessage.getFormattedMessage.contains(searchString))) } } @@ -66,7 +67,8 @@ class ServiceControlCliArgumentsSuite extends KyuubiFunSuite { } thread.start() thread.join() - assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(searchString))) + assert(logAppender.loggingEvents.exists( + _.getMessage.getFormattedMessage.contains(searchString))) } } diff --git a/kyuubi-ha/src/test/resources/log4j2.properties b/kyuubi-ha/src/test/resources/log4j2.properties new file mode 100644 index 000000000..fafe9b2a1 --- /dev/null +++ b/kyuubi-ha/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ServiceDiscoverySuite.scala b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ServiceDiscoverySuite.scala index 4be1de538..182bf56e4 100644 --- a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ServiceDiscoverySuite.scala +++ b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ServiceDiscoverySuite.scala @@ -201,7 +201,8 @@ class ServiceDiscoverySuite extends KerberizedTestHelper { assert(server.getServiceState === ServiceState.STOPPED) val msg = s"This Kyuubi instance ${server.frontendServices.head.connectionUrl}" + s" is now de-registered" - assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(msg))) + assert(logAppender.loggingEvents.exists( + _.getMessage.getFormattedMessage.contains(msg))) } } finally { server.stop() diff --git a/kyuubi-hive-beeline/pom.xml b/kyuubi-hive-beeline/pom.xml index 95e394f9b..1af51688e 100644 --- a/kyuubi-hive-beeline/pom.xml +++ b/kyuubi-hive-beeline/pom.xml @@ -102,8 +102,23 @@ - org.slf4j - slf4j-log4j12 + org.apache.logging.log4j + log4j-slf4j-impl + + + + org.apache.logging.log4j + log4j-api + + + + org.apache.logging.log4j + log4j-core + + + + org.apache.logging.log4j + log4j-1.2-api diff --git a/kyuubi-metrics/src/test/resources/log4j2.properties b/kyuubi-metrics/src/test/resources/log4j2.properties new file mode 100644 index 000000000..fafe9b2a1 --- /dev/null +++ b/kyuubi-metrics/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/kyuubi-server/src/test/resources/log4j2.properties b/kyuubi-server/src/test/resources/log4j2.properties new file mode 100644 index 000000000..45d900984 --- /dev/null +++ b/kyuubi-server/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to INFO +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = info diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala index 1f8359e44..3972554c2 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala @@ -222,9 +222,9 @@ class AllKyuubiConfiguration extends KyuubiFunSuite { newOutput += ("## Logging") newOutput += "" newOutput += ("Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging." + - " You can configure it using `$KYUUBI_HOME/conf/log4j.properties`.") + " You can configure it using `$KYUUBI_HOME/conf/log4j2.properties`.") - rewriteToConf(Paths.get(kyuubiHome, "conf", "log4j.properties.template"), newOutput) + rewriteToConf(Paths.get(kyuubiHome, "conf", "log4j2.properties.template"), newOutput) newOutput += "" newOutput += ("## Other Configurations") diff --git a/kyuubi-zookeeper/src/test/resources/log4j2.properties b/kyuubi-zookeeper/src/test/resources/log4j2.properties new file mode 100644 index 000000000..fafe9b2a1 --- /dev/null +++ b/kyuubi-zookeeper/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/pom.xml b/pom.xml index 4a2360baa..e7b11d3b8 100644 --- a/pom.xml +++ b/pom.xml @@ -131,6 +131,7 @@ 3.2.9 4.0.1 1.7.30 + 2.17.1 3.1.2 spark-${spark.version}-bin-hadoop${hadoop.binary.version}.tgz https://archive.apache.org/dist/spark/spark-${spark.version} @@ -365,6 +366,14 @@ org.apache.hadoop hadoop-client + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + @@ -567,14 +576,30 @@ org.slf4j - slf4j-log4j12 + jul-to-slf4j ${slf4j.version} - org.slf4j - jul-to-slf4j - ${slf4j.version} + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version}