From 7ee92ce5c1831fa20e62c1a703fd0564416f75cd Mon Sep 17 00:00:00 2001 From: Binjie Yang <2213335496@qq.com> Date: Mon, 19 Jul 2021 18:27:35 +0800 Subject: [PATCH] [KYUUBI #828] Manage Spark-Block-Cleaner log files ### _Why are the changes needed?_ Spark-Block-Cleaner log files are not governed by itself. Add log4j-defaults.properties to help input both console and file. And use rollingFileApeender to split big logfile to small logFiles. Create a log file under /logs/spark-block-cleaner-log/ so that you can manage the log yourself with the ability to clear the folder leading to `spark-`. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [X] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request Closes #831 from zwangsheng/feature/block-cleaner. Closes #828 83dc639e [ulysses] nit 5458f84b [Binjie Yang] make init log4j static f0d65dd7 [Binjie Yang] import format 48514496 [Binjie Yang] optimize properties 52713f1e [Binjie Yang] assign log4j.properties a8723f1a [Binjie Yang] fix 4d44a033 [Binjie Yang] use rollingfileappender Lead-authored-by: Binjie Yang <2213335496@qq.com> Co-authored-by: ulysses Signed-off-by: ulysses-you --- build/dist | 4 +-- .../kubernetes/docker/entrypoint.sh | 3 +- tools/spark-block-cleaner/pom.xml | 5 +++ .../resources/log4j-block-cleaner.properties | 33 +++++++++++++++++++ .../tools/KubernetesSparkBlockCleaner.scala | 5 +++ 5 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 tools/spark-block-cleaner/src/main/resources/log4j-block-cleaner.properties diff --git a/build/dist b/build/dist index 89a8732ef..645d792ac 100755 --- a/build/dist +++ b/build/dist @@ -195,9 +195,9 @@ cp "$KYUUBI_HOME/externals/kyuubi-spark-sql-engine/target/kyuubi-spark-sql-engin # Copy kyuubi tools if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner-$VERSION.jar" ]]; then - mkdir -p "$DISTDIR/tools/spark-block-cleaner/kubernetes/docker" + mkdir -p "$DISTDIR/tools/spark-block-cleaner/kubernetes" mkdir -p "$DISTDIR/tools/spark-block-cleaner/jars" - cp -r "$KYUUBI_HOME/tools/spark-block-cleaner/kubernetes/" "$DISTDIR/tools/spark-block-cleaner/kubernetes/" + cp -r "$KYUUBI_HOME/tools/spark-block-cleaner/kubernetes/" "$DISTDIR/tools/spark-block-cleaner/" cp "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner-$VERSION.jar" "$DISTDIR/tools/spark-block-cleaner/jars/" fi diff --git a/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh b/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh index 63a6e7c9f..953a80334 100644 --- a/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh +++ b/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh @@ -20,5 +20,4 @@ # shellcheck disable=SC2046 exec /usr/bin/tini -s -- java -cp "${CLASS_PATH}:${CLEANER_CLASSPATH}" \ - org.apache.kyuubi.tools.KubernetesSparkBlockCleaner \ - | tee /log/cleanerLog/cleaner$(date "+%Y%m%d%H%M%S").out + org.apache.kyuubi.tools.KubernetesSparkBlockCleaner diff --git a/tools/spark-block-cleaner/pom.xml b/tools/spark-block-cleaner/pom.xml index 7db8bda9c..b03d8517e 100644 --- a/tools/spark-block-cleaner/pom.xml +++ b/tools/spark-block-cleaner/pom.xml @@ -55,5 +55,10 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes + + + ${project.basedir}/src/main/resources + + diff --git a/tools/spark-block-cleaner/src/main/resources/log4j-block-cleaner.properties b/tools/spark-block-cleaner/src/main/resources/log4j-block-cleaner.properties new file mode 100644 index 000000000..2649bc49b --- /dev/null +++ b/tools/spark-block-cleaner/src/main/resources/log4j-block-cleaner.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the console +log4j.rootCategory=INFO, console, logFile + +### console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.out +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %d{yyyy} %p %c{2}: %m%n + +### logFile +log4j.appender.logFile=org.apache.log4j.RollingFileAppender +log4j.appender.logFile.File=/logs/spark-block-cleaner-log/cleaner-log.out +log4j.appender.logFile.MaxFileSize=20MB +log4j.appender.logFile.MaxBackupIndex=5 +log4j.appender.logFile.layout=org.apache.log4j.PatternLayout +log4j.appender.logFile.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c{2}: %m%n diff --git a/tools/spark-block-cleaner/src/main/scala/org/apache/kyuubi/tools/KubernetesSparkBlockCleaner.scala b/tools/spark-block-cleaner/src/main/scala/org/apache/kyuubi/tools/KubernetesSparkBlockCleaner.scala index 04b878737..7e67dd47d 100644 --- a/tools/spark-block-cleaner/src/main/scala/org/apache/kyuubi/tools/KubernetesSparkBlockCleaner.scala +++ b/tools/spark-block-cleaner/src/main/scala/org/apache/kyuubi/tools/KubernetesSparkBlockCleaner.scala @@ -21,6 +21,8 @@ import java.io.File import java.nio.file.{Files, Paths} import java.util.concurrent.{CountDownLatch, Executors} +import org.apache.log4j.PropertyConfigurator + import org.apache.kyuubi.Logging /* @@ -45,6 +47,9 @@ object KubernetesSparkBlockCleaner extends Logging { private val envMap = System.getenv() + PropertyConfigurator.configure( + Thread.currentThread().getContextClassLoader.getResource("log4j-block-cleaner.properties")) + private val freeSpaceThreshold = envMap.getOrDefault(FREE_SPACE_THRESHOLD_KEY, "60").toInt private val fileExpiredTime = envMap.getOrDefault(FILE_EXPIRED_TIME_KEY,