From 08f647ea3e8fa2bb71d173ca4c3ebd5e32f8b560 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Sat, 13 Aug 2022 11:10:48 +0800 Subject: [PATCH] Remove unexpected log4j configuration file from package jar and minor improvement (#331) --- .gitignore | 2 + .../emr/rss/common/internal/Logging.scala | 163 +++--------------- .../src/main/resources/log4j.properties | 25 --- 3 files changed, 28 insertions(+), 162 deletions(-) delete mode 100644 server-worker/src/main/resources/log4j.properties diff --git a/.gitignore b/.gitignore index ea64b8036..c2563b3b8 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,8 @@ unit-tests.log work/ assets/.jekyll-metadata *.crc +metastore_db/ +spark-warehouse/ logs pids diff --git a/common/src/main/scala/com/aliyun/emr/rss/common/internal/Logging.scala b/common/src/main/scala/com/aliyun/emr/rss/common/internal/Logging.scala index 6777350e9..d97f823e8 100644 --- a/common/src/main/scala/com/aliyun/emr/rss/common/internal/Logging.scala +++ b/common/src/main/scala/com/aliyun/emr/rss/common/internal/Logging.scala @@ -17,9 +17,7 @@ package com.aliyun.emr.rss.common.internal -import org.apache.log4j.{Level, LogManager, PropertyConfigurator} import org.slf4j.{Logger, LoggerFactory} -import org.slf4j.impl.StaticLoggerBinder import com.aliyun.emr.rss.common.util.Utils @@ -32,142 +30,58 @@ trait Logging { // Make the log field transient so that objects with Logging can // be serialized and used on another machine - @transient private var log_ : Logger = null + @transient protected lazy val log: Logger = LoggerFactory.getLogger(logName) // Method to get the logger name for this object - protected def logName = { + protected def logName: String = { // Ignore trailing $'s in the class names for Scala objects this.getClass.getName.stripSuffix("$") } - // Method to get or create the logger for this object - protected def log: Logger = { - if (log_ == null) { - initializeLogIfNecessary(false) - log_ = LoggerFactory.getLogger(logName) - } - log_ - } - // Log methods that take only a String - protected def logInfo(msg: => String) { - if (log.isInfoEnabled) log.info(msg) - } - - protected def logDebug(msg: => String) { - if (log.isDebugEnabled) log.debug(msg) - } - - protected def logTrace(msg: => String) { + protected def logTrace(msg: => String): Unit = { if (log.isTraceEnabled) log.trace(msg) } - protected def logWarning(msg: => String) { + protected def logDebug(msg: => String): Unit = { + if (log.isDebugEnabled) log.debug(msg) + } + + protected def logInfo(msg: => String): Unit = { + if (log.isInfoEnabled) log.info(msg) + } + + protected def logWarning(msg: => String): Unit = { if (log.isWarnEnabled) log.warn(msg) } - protected def logError(msg: => String) { + protected def logError(msg: => String): Unit = { if (log.isErrorEnabled) log.error(msg) } - // Log methods that take Throwables (Exceptions/Errors) too - protected def logInfo(msg: => String, throwable: Throwable) { - if (log.isInfoEnabled) log.info(msg, throwable) - } - - protected def logDebug(msg: => String, throwable: Throwable) { - if (log.isDebugEnabled) log.debug(msg, throwable) - } - - protected def logTrace(msg: => String, throwable: Throwable) { + // Log methods that take Throwable(Exception/Error) too + protected def logTrace(msg: => String, throwable: Throwable): Unit = { if (log.isTraceEnabled) log.trace(msg, throwable) } - protected def logWarning(msg: => String, throwable: Throwable) { + protected def logDebug(msg: => String, throwable: Throwable): Unit = { + if (log.isDebugEnabled) log.debug(msg, throwable) + } + + protected def logInfo(msg: => String, throwable: Throwable): Unit = { + if (log.isInfoEnabled) log.info(msg, throwable) + } + + protected def logWarning(msg: => String, throwable: Throwable): Unit = { if (log.isWarnEnabled) log.warn(msg, throwable) } - protected def logError(msg: => String, throwable: Throwable) { + protected def logError(msg: => String, throwable: Throwable): Unit = { if (log.isErrorEnabled) log.error(msg, throwable) } - - protected def isTraceEnabled(): Boolean = { - log.isTraceEnabled - } - - protected def initializeLogIfNecessary(isInterpreter: Boolean): Unit = { - initializeLogIfNecessary(isInterpreter, silent = false) - } - - protected def initializeLogIfNecessary( - isInterpreter: Boolean, - silent: Boolean = false): Boolean = { - if (!Logging.initialized) { - Logging.initLock.synchronized { - if (!Logging.initialized) { - initializeLogging(isInterpreter, silent) - return true - } - } - } - false - } - - private def initializeLogging(isInterpreter: Boolean, silent: Boolean): Unit = { - // Don't use a logger in here, as this is itself occurring during initialization of a logger - // If Log4j 1.2 is being used, but is not initialized, load a default properties file - if (Logging.isLog4j12()) { - val log4j12Initialized = LogManager.getRootLogger.getAllAppenders.hasMoreElements - // scalastyle:off println - if (!log4j12Initialized) { - Logging.defaultSparkLog4jConfig = true - val defaultLogProps = "log4j-defaults.properties" - Option(Utils.getClassLoader.getResource(defaultLogProps)) match { - case Some(url) => - PropertyConfigurator.configure(url) - if (!silent) { - System.err.println(s"Using Spark's default log4j profile: $defaultLogProps") - } - case None => - System.err.println(s"Spark was unable to load $defaultLogProps") - } - } - - val rootLogger = LogManager.getRootLogger() - if (Logging.defaultRootLevel == null) { - Logging.defaultRootLevel = rootLogger.getLevel() - } - - if (isInterpreter) { - // Use the repl's main class to define the default log level when running the shell, - // overriding the root logger's config if they're different. - val replLogger = LogManager.getLogger(logName) - val replLevel = Option(replLogger.getLevel()).getOrElse(Level.WARN) - if (replLevel != rootLogger.getEffectiveLevel()) { - if (!silent) { - System.err.printf("Setting default log level to \"%s\".\n", replLevel) - System.err.println("To adjust logging level use sc.setLogLevel(newLevel). " + - "For SparkR, use setLogLevel(newLevel).") - } - rootLogger.setLevel(replLevel) - } - } - // scalastyle:on println - } - Logging.initialized = true - - // Force a call into slf4j to initialize it. Avoids this happening from multiple threads - // and triggering this: http://mailman.qos.ch/pipermail/slf4j-dev/2010-April/002956.html - log - } } private[rss] object Logging { - @volatile private var initialized = false - @volatile private var defaultRootLevel: Level = null - @volatile private var defaultSparkLog4jConfig = false - - val initLock = new Object() try { // We use reflection here to handle the case where users remove the // slf4j-to-jul bridge order to route their logs to JUL. @@ -178,31 +92,6 @@ private[rss] object Logging { bridgeClass.getMethod("install").invoke(null) } } catch { - case e: ClassNotFoundException => // can't log anything yet so just fail silently - } - - /** - * Marks the logging system as not initialized. This does a best effort at resetting the - * logging system to its initial state so that the next class to use logging triggers - * initialization again. - */ - def uninitialize(): Unit = initLock.synchronized { - if (isLog4j12()) { - if (defaultSparkLog4jConfig) { - defaultSparkLog4jConfig = false - LogManager.resetConfiguration() - } else { - LogManager.getRootLogger().setLevel(defaultRootLevel) - } - } - this.initialized = false - } - - private def isLog4j12(): Boolean = { - // This distinguishes the log4j 1.2 binding, currently - // org.slf4j.impl.Log4jLoggerFactory, from the log4j 2.0 binding, currently - // org.apache.logging.slf4j.Log4jLoggerFactory - val binderClass = StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr - "org.slf4j.impl.Log4jLoggerFactory".equals(binderClass) + case _: ClassNotFoundException => // can't log anything yet so just fail silently } } diff --git a/server-worker/src/main/resources/log4j.properties b/server-worker/src/main/resources/log4j.properties deleted file mode 100644 index f0ecd3b1d..000000000 --- a/server-worker/src/main/resources/log4j.properties +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Set everything to be logged to the file target/unit-tests.log -log4j.rootCategory=ERROR, console -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}: %m%n -# Ignore messages below warning level from Jetty, because it's a bit verbose -log4j.logger.org.sparkproject.jetty=WARN -