14e12e9aa4
### Why are the changes needed? It's a common use case that the user may want to send the service logs in a structured format to Kafka and then collect them into centralized log services for further analysis, fortunately, the Kyuubi used logging frameworks Log4j2 has built-in [KafkaAppender](https://logging.apache.org/log4j/2.x/manual/appenders/message-queue.html#KafkaAppender) and [JSON Template Layout](https://logging.apache.org/log4j/2.x/manual/json-template-layout.html), thus the goal could be achieved by just a few configurations. To simplify the user setup steps, this PR adds `log4j-layout-template-json-<version>.jar` into Kyuubi binary tarball. PS: I also plan to support sending engine bootstrap process(e.g. `spark-submit`) logs into Kafka with specific labels in the follow-up PRs. ### How was this patch tested? Manually test. Configuration in `$KYUUBI_HOME/conf/log4j2.xml` ```xml <Configuration status="INFO"> <Appenders> <Kafka name="kafka" topic="ecs-json-logs" syncSend="false"> <JsonTemplateLayout> <EventTemplateAdditionalField key="app" value="kyuubi"/> <EventTemplateAdditionalField key="cluster" value="hadoop-testing"/> <EventTemplateAdditionalField key="host" value="${hostName}"/> </JsonTemplateLayout> <Property name="bootstrap.servers" value="kafka-1:9092,kafka-2:9092,kafka-3:9092"/> <Property name="compression.type" value="gzip"/> </Kafka> </Appenders> <Loggers> <Root level="INFO"> <AppenderRef ref="kafka"/> </Root> </Loggers> </Configuration> ``` Check that Kafka receives the expected structured logging message in the Elastic Common Schema(ECS) layout.  ### Was this patch authored or co-authored using generative AI tooling? No Closes #6861 from pan3793/structured-logging. Closes #6861 9556da2a7 [Cheng Pan] Structured Logs 7dc6dda86 [Cheng Pan] Add log4j-layout-template-json Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
129 lines
4.5 KiB
XML
129 lines
4.5 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
|
|
<parent>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-parent</artifactId>
|
|
<version>1.11.0-SNAPSHOT</version>
|
|
<relativePath>../pom.xml</relativePath>
|
|
</parent>
|
|
|
|
<artifactId>kyuubi-assembly_${scala.binary.version}</artifactId>
|
|
<packaging>pom</packaging>
|
|
<name>Kyuubi Project Assembly</name>
|
|
<url>https://kyuubi.apache.org/</url>
|
|
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-common_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-ha_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-metrics_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-rest-client</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-server_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-ctl_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>${kyuubi-relocated-zookeeper.artifacts}</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client-api</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client-runtime</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.google.code.findbugs</groupId>
|
|
<artifactId>jsr305</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>failureaccess</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>jline</groupId>
|
|
<artifactId>jline</artifactId>
|
|
</dependency>
|
|
|
|
<!-- default package with log4j -->
|
|
<dependency>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-slf4j-impl</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-api</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-core</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-1.2-api</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-layout-template-json</artifactId>
|
|
</dependency>
|
|
</dependencies>
|
|
</project>
|