### Why are the changes needed? It's a common use case that the user may want to send the service logs in a structured format to Kafka and then collect them into centralized log services for further analysis, fortunately, the Kyuubi used logging frameworks Log4j2 has built-in [KafkaAppender](https://logging.apache.org/log4j/2.x/manual/appenders/message-queue.html#KafkaAppender) and [JSON Template Layout](https://logging.apache.org/log4j/2.x/manual/json-template-layout.html), thus the goal could be achieved by just a few configurations. To simplify the user setup steps, this PR adds `log4j-layout-template-json-<version>.jar` into Kyuubi binary tarball. PS: I also plan to support sending engine bootstrap process(e.g. `spark-submit`) logs into Kafka with specific labels in the follow-up PRs. ### How was this patch tested? Manually test. Configuration in `$KYUUBI_HOME/conf/log4j2.xml` ```xml <Configuration status="INFO"> <Appenders> <Kafka name="kafka" topic="ecs-json-logs" syncSend="false"> <JsonTemplateLayout> <EventTemplateAdditionalField key="app" value="kyuubi"/> <EventTemplateAdditionalField key="cluster" value="hadoop-testing"/> <EventTemplateAdditionalField key="host" value="${hostName}"/> </JsonTemplateLayout> <Property name="bootstrap.servers" value="kafka-1:9092,kafka-2:9092,kafka-3:9092"/> <Property name="compression.type" value="gzip"/> </Kafka> </Appenders> <Loggers> <Root level="INFO"> <AppenderRef ref="kafka"/> </Root> </Loggers> </Configuration> ``` Check that Kafka receives the expected structured logging message in the Elastic Common Schema(ECS) layout.  ### Was this patch authored or co-authored using generative AI tooling? No Closes #6861 from pan3793/structured-logging. Closes #6861 9556da2a7 [Cheng Pan] Structured Logs 7dc6dda86 [Cheng Pan] Add log4j-layout-template-json Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
98 lines
4.8 KiB
XML
98 lines
4.8 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
|
|
<!-- Provide log4j2.xml.template to fix `ERROR Filters contains invalid attributes "onMatch", "onMismatch"`, see KYUUBI #2247 -->
|
|
<!-- Extra logging related to initialization of Log4j.
|
|
Set to debug or trace if log4j initialization is failing. -->
|
|
<Configuration status="INFO">
|
|
<Properties>
|
|
<Property name="logDir">${env:KYUUBI_LOG_DIR}</Property>
|
|
<Property name="restAuditLogPath">rest-audit.log</Property>
|
|
<Property name="restAuditLogFilePattern">rest-audit-%d{yyyy-MM-dd}-%i.log</Property>
|
|
<Property name="k8sAuditLogPath">k8s-audit.log</Property>
|
|
<Property name="k8sAuditLogFilePattern">k8s-audit-%d{yyyy-MM-dd}-%i.log</Property>
|
|
<Property name="opAuditLogPath">operation-audit.log</Property>
|
|
<Property name="opAuditLogFilePattern">operation-audit-%d{yyyy-MM-dd}-%i.log</Property>
|
|
</Properties>
|
|
<Appenders>
|
|
<Console name="stdout" target="SYSTEM_OUT">
|
|
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %p %tn %c: %m%n%ex"/>
|
|
</Console>
|
|
<RollingFile name="restAudit" fileName="${sys:logDir}/${sys:restAuditLogPath}"
|
|
filePattern="${sys:logDir}/${sys:restAuditLogFilePattern}">
|
|
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c{1}: %m%n%ex"/>
|
|
<Policies>
|
|
<SizeBasedTriggeringPolicy size="51200KB" />
|
|
</Policies>
|
|
<DefaultRolloverStrategy max="10"/>
|
|
</RollingFile>
|
|
<RollingFile name="k8sAudit" fileName="${sys:logDir}/${sys:k8sAuditLogPath}"
|
|
filePattern="${sys:logDir}/${sys:k8sAuditLogFilePattern}">
|
|
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c{1}: %m%n%ex"/>
|
|
<Policies>
|
|
<SizeBasedTriggeringPolicy size="51200KB" />
|
|
</Policies>
|
|
<DefaultRolloverStrategy max="10"/>
|
|
</RollingFile>
|
|
<RollingFile name="opAudit" fileName="${sys:logDir}/${sys:opAuditLogPath}"
|
|
filePattern="${sys:logDir}/${sys:opAuditLogFilePattern}">
|
|
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c{1}: %m%n%ex"/>
|
|
<Policies>
|
|
<SizeBasedTriggeringPolicy size="51200KB" />
|
|
</Policies>
|
|
<DefaultRolloverStrategy max="10"/>
|
|
</RollingFile>
|
|
<!-- Kafka appender with Elastic Common Schema(ECS) JSON template layout
|
|
<Kafka name="kafka" topic="ecs-json-logs" syncSend="false">
|
|
<JsonTemplateLayout>
|
|
<EventTemplateAdditionalField key="app" value="kyuubi"/>
|
|
<EventTemplateAdditionalField key="cluster" value="kyuubi-cluster"/>
|
|
<EventTemplateAdditionalField key="host" value="${hostName}"/>
|
|
</JsonTemplateLayout>
|
|
<Property name="bootstrap.servers" value="kafka-1:9092,kafka-2:9092,kafka-3:9092"/>
|
|
<Property name="compression.type" value="gzip"/>
|
|
</Kafka>
|
|
-->
|
|
</Appenders>
|
|
<Loggers>
|
|
<Root level="INFO">
|
|
<AppenderRef ref="stdout"/>
|
|
</Root>
|
|
<Logger name="org.apache.kyuubi.ctl" level="error" additivity="false">
|
|
<AppenderRef ref="stdout"/>
|
|
</Logger>
|
|
<!--
|
|
<Logger name="org.apache.kyuubi.server.mysql.codec" level="trace" additivity="false">
|
|
<AppenderRef ref="stdout"/>
|
|
</Logger>
|
|
-->
|
|
<Logger name="org.apache.hive.beeline.KyuubiBeeLine" level="error" additivity="false">
|
|
<AppenderRef ref="stdout"/>
|
|
</Logger>
|
|
<Logger name="org.apache.kyuubi.server.http.authentication.AuthenticationAuditLogger" additivity="false">
|
|
<AppenderRef ref="restAudit" />
|
|
</Logger>
|
|
<Logger name="org.apache.kyuubi.engine.KubernetesApplicationAuditLogger" additivity="false">
|
|
<AppenderRef ref="k8sAudit" />
|
|
</Logger>
|
|
<Logger name="org.apache.kyuubi.operation.OperationAuditLogger" additivity="false">
|
|
<AppenderRef ref="opAudit" />
|
|
</Logger>
|
|
</Loggers>
|
|
</Configuration>
|