diff --git a/build/dist b/build/dist index 89a8732ef..645d792ac 100755 --- a/build/dist +++ b/build/dist @@ -195,9 +195,9 @@ cp "$KYUUBI_HOME/externals/kyuubi-spark-sql-engine/target/kyuubi-spark-sql-engin # Copy kyuubi tools if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner-$VERSION.jar" ]]; then - mkdir -p "$DISTDIR/tools/spark-block-cleaner/kubernetes/docker" + mkdir -p "$DISTDIR/tools/spark-block-cleaner/kubernetes" mkdir -p "$DISTDIR/tools/spark-block-cleaner/jars" - cp -r "$KYUUBI_HOME/tools/spark-block-cleaner/kubernetes/" "$DISTDIR/tools/spark-block-cleaner/kubernetes/" + cp -r "$KYUUBI_HOME/tools/spark-block-cleaner/kubernetes/" "$DISTDIR/tools/spark-block-cleaner/" cp "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner-$VERSION.jar" "$DISTDIR/tools/spark-block-cleaner/jars/" fi diff --git a/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh b/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh index 63a6e7c9f..953a80334 100644 --- a/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh +++ b/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh @@ -20,5 +20,4 @@ # shellcheck disable=SC2046 exec /usr/bin/tini -s -- java -cp "${CLASS_PATH}:${CLEANER_CLASSPATH}" \ - org.apache.kyuubi.tools.KubernetesSparkBlockCleaner \ - | tee /log/cleanerLog/cleaner$(date "+%Y%m%d%H%M%S").out + org.apache.kyuubi.tools.KubernetesSparkBlockCleaner diff --git a/tools/spark-block-cleaner/pom.xml b/tools/spark-block-cleaner/pom.xml index 7db8bda9c..b03d8517e 100644 --- a/tools/spark-block-cleaner/pom.xml +++ b/tools/spark-block-cleaner/pom.xml @@ -55,5 +55,10 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes + + + ${project.basedir}/src/main/resources + + diff --git a/tools/spark-block-cleaner/src/main/resources/log4j-block-cleaner.properties b/tools/spark-block-cleaner/src/main/resources/log4j-block-cleaner.properties new file mode 100644 index 000000000..2649bc49b --- /dev/null +++ b/tools/spark-block-cleaner/src/main/resources/log4j-block-cleaner.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the console +log4j.rootCategory=INFO, console, logFile + +### console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.out +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %d{yyyy} %p %c{2}: %m%n + +### logFile +log4j.appender.logFile=org.apache.log4j.RollingFileAppender +log4j.appender.logFile.File=/logs/spark-block-cleaner-log/cleaner-log.out +log4j.appender.logFile.MaxFileSize=20MB +log4j.appender.logFile.MaxBackupIndex=5 +log4j.appender.logFile.layout=org.apache.log4j.PatternLayout +log4j.appender.logFile.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c{2}: %m%n diff --git a/tools/spark-block-cleaner/src/main/scala/org/apache/kyuubi/tools/KubernetesSparkBlockCleaner.scala b/tools/spark-block-cleaner/src/main/scala/org/apache/kyuubi/tools/KubernetesSparkBlockCleaner.scala index 04b878737..7e67dd47d 100644 --- a/tools/spark-block-cleaner/src/main/scala/org/apache/kyuubi/tools/KubernetesSparkBlockCleaner.scala +++ b/tools/spark-block-cleaner/src/main/scala/org/apache/kyuubi/tools/KubernetesSparkBlockCleaner.scala @@ -21,6 +21,8 @@ import java.io.File import java.nio.file.{Files, Paths} import java.util.concurrent.{CountDownLatch, Executors} +import org.apache.log4j.PropertyConfigurator + import org.apache.kyuubi.Logging /* @@ -45,6 +47,9 @@ object KubernetesSparkBlockCleaner extends Logging { private val envMap = System.getenv() + PropertyConfigurator.configure( + Thread.currentThread().getContextClassLoader.getResource("log4j-block-cleaner.properties")) + private val freeSpaceThreshold = envMap.getOrDefault(FREE_SPACE_THRESHOLD_KEY, "60").toInt private val fileExpiredTime = envMap.getOrDefault(FILE_EXPIRED_TIME_KEY,