From d68507d23988dff02fb16b47c1d6448df32be99d Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Thu, 19 Oct 2023 13:04:25 +0800 Subject: [PATCH] [KYUUBI #5462] [AUTHZ] Support create table command for Paimon ### _Why are the changes needed?_ Apache Paimon is an incubating Apache project of data lake platform for high-speed data ingestion, changelog tracking and efficient real-time analytics. - Initial support for Paimon tables in Authz plugin - Create Table Command: https://paimon.apache.org/docs/master/engines/spark3/#create-table - Paimon `0.5.0-incubating` supports Spark 3.1/3.2/3.3/3.4. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request ### _Was this patch authored or co-authored using generative AI tooling?_ Closes #5462 from bowenliang123/authz-paimon-createtable. Closes #5462 8393f6ecc [liangbowen] remove spark-authz-paimon-test profile, use 3.4 as spark binary version for Paimon a4e01a2a8 [liangbowen] update workflow inclusions f372882aa [liangbowen] update workflow exclusions 7785dec52 [liangbowen] update workflow include 64ee20106 [liangbowen] update 73edea38f [liangbowen] add spark-authz-paimon-test profile and excluded for Spark 3.5 de58605bb [liangbowen] nit 81a4a4370 [liangbowen] nit eb2de8359 [Bowen Liang] exclude paimon tests for spark-3.5 4f58d7727 [Bowen Liang] fix typo cf360e6da [Bowen Liang] drop database af8ffda14 [Bowen Liang] drop database dc3ee9462 [Bowen Liang] support Paimon create table Lead-authored-by: Bowen Liang Co-authored-by: liangbowen Signed-off-by: Bowen Liang --- .github/workflows/master.yml | 2 +- extensions/spark/kyuubi-spark-authz/pom.xml | 6 ++ ...imonCatalogRangerSparkExtensionSuite.scala | 85 +++++++++++++++++++ .../org/apache/kyuubi/tags/PaimonTest.java | 29 +++++++ pom.xml | 12 ++- 5 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/PaimonCatalogRangerSparkExtensionSuite.scala create mode 100644 kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/PaimonTest.java diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index f590ea267..77b8922ca 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -75,7 +75,7 @@ jobs: - java: 8 spark: '3.4' spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.5.0 -Dspark.archive.name=spark-3.5.0-bin-hadoop3.tgz -Pzookeeper-3.6' - exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.SparkLocalClusterTest' + exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest' comment: 'verify-on-spark-3.5-binary' exclude: # SPARK-33772: Spark supports JDK 17 since 3.3.0 diff --git a/extensions/spark/kyuubi-spark-authz/pom.xml b/extensions/spark/kyuubi-spark-authz/pom.xml index 97145e514..1b8655612 100644 --- a/extensions/spark/kyuubi-spark-authz/pom.xml +++ b/extensions/spark/kyuubi-spark-authz/pom.xml @@ -323,6 +323,12 @@ scala-collection-compat_${scala.binary.version} test + + + org.apache.paimon + paimon-spark-${paimon.spark.binary.version} + test + diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/PaimonCatalogRangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/PaimonCatalogRangerSparkExtensionSuite.scala new file mode 100644 index 000000000..62cd9d627 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/PaimonCatalogRangerSparkExtensionSuite.scala @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.plugin.spark.authz.ranger + +import org.scalatest.Outcome + +import org.apache.kyuubi.Utils +import org.apache.kyuubi.plugin.spark.authz.AccessControlException +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.tags.PaimonTest +import org.apache.kyuubi.util.AssertionUtils._ + +/** + * Tests for RangerSparkExtensionSuite on Paimon + */ +@PaimonTest +class PaimonCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { + override protected val catalogImpl: String = "hive" + private def isSupportedVersion = !isSparkV35OrGreater + + val catalogV2 = "paimon_catalog" + val namespace1 = "paimon_ns" + val table1 = "table1" + + override def withFixture(test: NoArgTest): Outcome = { + assume(isSupportedVersion) + test() + } + + override def beforeAll(): Unit = { + if (isSupportedVersion) { + spark.conf.set(s"spark.sql.catalog.$catalogV2", "org.apache.paimon.spark.SparkCatalog") + spark.conf.set( + s"spark.sql.catalog.$catalogV2.warehouse", + Utils.createTempDir(catalogV2).toString) + super.beforeAll() + } + + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $catalogV2.$namespace1")) + } + + override def afterAll(): Unit = { + if (isSupportedVersion) { + doAs(admin, sql(s"DROP DATABASE IF EXISTS $catalogV2.$namespace1")) + + super.afterAll() + spark.sessionState.catalog.reset() + spark.sessionState.conf.clear() + } + } + + test("CreateTable") { + withCleanTmpResources(Seq((s"$catalogV2.$namespace1.$table1", "table"))) { + val createTable = + s""" + |CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$table1 + |(id int, name string, city string) + |USING paimon + |OPTIONS ( + | primaryKey = 'id' + |) + |""".stripMargin + + interceptContains[AccessControlException] { + doAs(someone, sql(createTable)) + }(s"does not have [create] privilege on [$namespace1/$table1]") + doAs(admin, createTable) + } + } +} diff --git a/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/PaimonTest.java b/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/PaimonTest.java new file mode 100644 index 000000000..e3e807584 --- /dev/null +++ b/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/PaimonTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.scalatest.TagAnnotation; + +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface PaimonTest {} diff --git a/pom.xml b/pom.xml index 00ee354a1..1f3fca53b 100644 --- a/pom.xml +++ b/pom.xml @@ -184,6 +184,8 @@ --> 4.1.93.Final 0.12.0 + 0.5.0-incubating + ${spark.binary.version} 1.10.1 6.0.0 0.16.0 @@ -1484,6 +1486,12 @@ hudi-spark${hudi.spark.binary.version}-bundle_${scala.binary.version} ${hudi.version} + + + org.apache.paimon + paimon-spark-${paimon.spark.binary.version} + ${paimon.version} + @@ -2266,9 +2274,11 @@ 3.0.0 3.4 + + 3.4 3.5.0 3.5 - org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.PySparkTest + org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.PySparkTest,org.apache.kyuubi.tags.PaimonTest