[KYUUBI #5462] [AUTHZ] Support create table command for Paimon
### _Why are the changes needed?_ Apache Paimon is an incubating Apache project of data lake platform for high-speed data ingestion, changelog tracking and efficient real-time analytics. - Initial support for Paimon tables in Authz plugin - Create Table Command: https://paimon.apache.org/docs/master/engines/spark3/#create-table - Paimon `0.5.0-incubating` supports Spark 3.1/3.2/3.3/3.4. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request ### _Was this patch authored or co-authored using generative AI tooling?_ Closes #5462 from bowenliang123/authz-paimon-createtable. Closes #5462 8393f6ecc [liangbowen] remove spark-authz-paimon-test profile, use 3.4 as spark binary version for Paimon a4e01a2a8 [liangbowen] update workflow inclusions f372882aa [liangbowen] update workflow exclusions 7785dec52 [liangbowen] update workflow include 64ee20106 [liangbowen] update 73edea38f [liangbowen] add spark-authz-paimon-test profile and excluded for Spark 3.5 de58605bb [liangbowen] nit 81a4a4370 [liangbowen] nit eb2de8359 [Bowen Liang] exclude paimon tests for spark-3.5 4f58d7727 [Bowen Liang] fix typo cf360e6da [Bowen Liang] drop database af8ffda14 [Bowen Liang] drop database dc3ee9462 [Bowen Liang] support Paimon create table Lead-authored-by: Bowen Liang <liangbowen@gf.com.cn> Co-authored-by: liangbowen <liangbowen@gf.com.cn> Signed-off-by: Bowen Liang <liangbowen@gf.com.cn>
This commit is contained in:
Bowen Liang
parent
411df60431
commit
d68507d239
2
.github/workflows/master.yml
vendored
2
.github/workflows/master.yml
vendored
@ -75,7 +75,7 @@ jobs:
|
||||
- java: 8
|
||||
spark: '3.4'
|
||||
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.5.0 -Dspark.archive.name=spark-3.5.0-bin-hadoop3.tgz -Pzookeeper-3.6'
|
||||
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
|
||||
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
|
||||
comment: 'verify-on-spark-3.5-binary'
|
||||
exclude:
|
||||
# SPARK-33772: Spark supports JDK 17 since 3.3.0
|
||||
|
||||
@ -323,6 +323,12 @@
|
||||
<artifactId>scala-collection-compat_${scala.binary.version}</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.paimon</groupId>
|
||||
<artifactId>paimon-spark-${paimon.spark.binary.version}</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.kyuubi.plugin.spark.authz.ranger
|
||||
|
||||
import org.scalatest.Outcome
|
||||
|
||||
import org.apache.kyuubi.Utils
|
||||
import org.apache.kyuubi.plugin.spark.authz.AccessControlException
|
||||
import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._
|
||||
import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._
|
||||
import org.apache.kyuubi.tags.PaimonTest
|
||||
import org.apache.kyuubi.util.AssertionUtils._
|
||||
|
||||
/**
|
||||
* Tests for RangerSparkExtensionSuite on Paimon
|
||||
*/
|
||||
@PaimonTest
|
||||
class PaimonCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite {
|
||||
override protected val catalogImpl: String = "hive"
|
||||
private def isSupportedVersion = !isSparkV35OrGreater
|
||||
|
||||
val catalogV2 = "paimon_catalog"
|
||||
val namespace1 = "paimon_ns"
|
||||
val table1 = "table1"
|
||||
|
||||
override def withFixture(test: NoArgTest): Outcome = {
|
||||
assume(isSupportedVersion)
|
||||
test()
|
||||
}
|
||||
|
||||
override def beforeAll(): Unit = {
|
||||
if (isSupportedVersion) {
|
||||
spark.conf.set(s"spark.sql.catalog.$catalogV2", "org.apache.paimon.spark.SparkCatalog")
|
||||
spark.conf.set(
|
||||
s"spark.sql.catalog.$catalogV2.warehouse",
|
||||
Utils.createTempDir(catalogV2).toString)
|
||||
super.beforeAll()
|
||||
}
|
||||
|
||||
doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $catalogV2.$namespace1"))
|
||||
}
|
||||
|
||||
override def afterAll(): Unit = {
|
||||
if (isSupportedVersion) {
|
||||
doAs(admin, sql(s"DROP DATABASE IF EXISTS $catalogV2.$namespace1"))
|
||||
|
||||
super.afterAll()
|
||||
spark.sessionState.catalog.reset()
|
||||
spark.sessionState.conf.clear()
|
||||
}
|
||||
}
|
||||
|
||||
test("CreateTable") {
|
||||
withCleanTmpResources(Seq((s"$catalogV2.$namespace1.$table1", "table"))) {
|
||||
val createTable =
|
||||
s"""
|
||||
|CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$table1
|
||||
|(id int, name string, city string)
|
||||
|USING paimon
|
||||
|OPTIONS (
|
||||
| primaryKey = 'id'
|
||||
|)
|
||||
|""".stripMargin
|
||||
|
||||
interceptContains[AccessControlException] {
|
||||
doAs(someone, sql(createTable))
|
||||
}(s"does not have [create] privilege on [$namespace1/$table1]")
|
||||
doAs(admin, createTable)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kyuubi.tags;
|
||||
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
import org.scalatest.TagAnnotation;
|
||||
|
||||
@TagAnnotation
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target({ElementType.METHOD, ElementType.TYPE})
|
||||
public @interface PaimonTest {}
|
||||
12
pom.xml
12
pom.xml
@ -184,6 +184,8 @@
|
||||
-->
|
||||
<netty.version>4.1.93.Final</netty.version>
|
||||
<openai.java.version>0.12.0</openai.java.version>
|
||||
<paimon.version>0.5.0-incubating</paimon.version>
|
||||
<paimon.spark.binary.version>${spark.binary.version}</paimon.spark.binary.version>
|
||||
<parquet.version>1.10.1</parquet.version>
|
||||
<phoenix.version>6.0.0</phoenix.version>
|
||||
<prometheus.version>0.16.0</prometheus.version>
|
||||
@ -1484,6 +1486,12 @@
|
||||
<artifactId>hudi-spark${hudi.spark.binary.version}-bundle_${scala.binary.version}</artifactId>
|
||||
<version>${hudi.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.paimon</groupId>
|
||||
<artifactId>paimon-spark-${paimon.spark.binary.version}</artifactId>
|
||||
<version>${paimon.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
@ -2266,9 +2274,11 @@
|
||||
<delta.version>3.0.0</delta.version>
|
||||
<!-- Remove this when Hudi supports Spark 3.5 -->
|
||||
<hudi.spark.binary.version>3.4</hudi.spark.binary.version>
|
||||
<!-- Remove this when Paimon supports Spark 3.5 -->
|
||||
<paimon.spark.binary.version>3.4</paimon.spark.binary.version>
|
||||
<spark.version>3.5.0</spark.version>
|
||||
<spark.binary.version>3.5</spark.binary.version>
|
||||
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.PySparkTest</maven.plugin.scalatest.exclude.tags>
|
||||
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.PySparkTest,org.apache.kyuubi.tags.PaimonTest</maven.plugin.scalatest.exclude.tags>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user