### _Why are the changes needed?_ Implements AtlasLineageDispatcher to send lineage to Apache Atlas. close #4814 Atlas Spark Model Definition: https://github.com/apache/atlas/blob/master/addons/models/1000-Hadoop/1100-spark_model.json spark process:  table lineage:  column lineage:  ### _How was this patch tested?_ - [X] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [X] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4815 from wForget/KYUUBI-4814. Closes #4814 3df8a7ec9 [wforget] comments c58eae7c5 [wforget] comments 926bcf211 [wforget] comment e0b4067c3 [wforget] comment e4cc3e3f8 [wforget] comments adc72b96f [Bowen Liang] Update extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasEntityHelper.scala e3bdd1c65 [Bowen Liang] Update extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasEntityHelper.scala baf1711ac [Bowen Liang] Update extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasLineageDispatcherSuite.scala 61e79f3d5 [Bowen Liang] Update extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasLineageDispatcherSuite.scala 541df3780 [Bowen Liang] Update extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasLineageDispatcherSuite.scala 5dd310657 [wforget] fix cea1e137d [wforget] fix f028d4b09 [wforget] fix 0c9b4516b [wforget] fix 6f8113032 [wforget] add close atlas client shutdown hook 3f4d2a7db [wforget] add remote user a0db58afc [wforget] comments 6dd3c66df [wforget] comments f2b2a30dc [wforget] style 83eb1e481 [wforget] add atlas.column.lineage.enable configuration 0719a2b65 [wforget] doc 05f936005 [wforget] fix d169b661d [wforget] fix 6da80d742 [wforget] fix 820ae5c5f [wforget] column lineages dabe8173e [wforget] license f22e044d2 [wforget] test b948bce90 [wforget] fix and add test 0aef1be6b [wforget] fix 368b5ab3f [wforget] [KYUUBI-4814] Implements AtlasLineageDispatcher to send lineage to Apache Atlas Lead-authored-by: wforget <643348094@qq.com> Co-authored-by: Bowen Liang <bowenliang@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
183 lines
6.6 KiB
XML
183 lines
6.6 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<parent>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-parent</artifactId>
|
|
<version>1.8.0-SNAPSHOT</version>
|
|
<relativePath>../../../pom.xml</relativePath>
|
|
</parent>
|
|
|
|
<artifactId>kyuubi-spark-lineage_2.12</artifactId>
|
|
<packaging>jar</packaging>
|
|
<name>Kyuubi Dev Spark Lineage Extension</name>
|
|
<url>https://kyuubi.apache.org/</url>
|
|
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-util-scala_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-common_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-events_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>commons-collections</groupId>
|
|
<artifactId>commons-collections</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-annotations</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-core</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-databind</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.httpcomponents</groupId>
|
|
<artifactId>httpclient</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>commons-lang</groupId>
|
|
<artifactId>commons-lang</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-lang3</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.atlas</groupId>
|
|
<artifactId>atlas-client-v2</artifactId>
|
|
<version>${atlas.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>jul-to-slf4j</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.springframework</groupId>
|
|
<artifactId>spring-context</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-text</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-common_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-hive_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
</dependencies>
|
|
|
|
<build>
|
|
<testResources>
|
|
<testResource>
|
|
<directory>${project.basedir}/src/test/resources</directory>
|
|
</testResource>
|
|
</testResources>
|
|
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
|
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
|
</build>
|
|
|
|
</project>
|