[KYUUBI #3050] Bump Apache Iceberg 0.14.0
### _Why are the changes needed?_ https://github.com/apache/iceberg/releases/tag/apache-iceberg-0.14.0 Notable changes: - Support Spark 3.3 - Unified package name ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #3050 from pan3793/iceberg. Closes #3050 3e700a11 [Cheng Pan] Fix ut caused by SPARK-37931 039c4e2d [Cheng Pan] Enable Iceberg test w/ Spark 3.3 53cac396 [Cheng Pan] Bump Apache Iceberg 0.14.0 Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
Cheng Pan
parent
720bc00c99
commit
6999622420
2
externals/kyuubi-spark-sql-engine/pom.xml
vendored
2
externals/kyuubi-spark-sql-engine/pom.xml
vendored
@ -137,7 +137,7 @@
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.iceberg</groupId>
|
||||
<artifactId>${iceberg.name}</artifactId>
|
||||
<artifactId>iceberg-spark-runtime-${spark.binary.version}_${scala.binary.version}</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
|
||||
@ -17,11 +17,22 @@
|
||||
|
||||
package org.apache.kyuubi.operation
|
||||
|
||||
import org.apache.kyuubi.IcebergSuiteMixin
|
||||
import org.apache.kyuubi.{IcebergSuiteMixin, SPARK_COMPILE_VERSION}
|
||||
import org.apache.kyuubi.Utils._
|
||||
import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
|
||||
|
||||
trait IcebergMetadataTests extends HiveJDBCTestHelper with IcebergSuiteMixin {
|
||||
|
||||
def isSparkVersionAtLeast(ver: String): Boolean = {
|
||||
val runtimeMajor = majorVersion(SPARK_COMPILE_VERSION)
|
||||
val targetMajor = majorVersion(ver)
|
||||
(runtimeMajor > targetMajor) || {
|
||||
val runtimeMinor = minorVersion(SPARK_COMPILE_VERSION)
|
||||
val targetMinor = minorVersion(ver)
|
||||
runtimeMajor == targetMajor && runtimeMinor >= targetMinor
|
||||
}
|
||||
}
|
||||
|
||||
test("get catalogs") {
|
||||
withJdbcStatement() { statement =>
|
||||
val metaData = statement.getConnection.getMetaData
|
||||
@ -151,9 +162,12 @@ trait IcebergMetadataTests extends HiveJDBCTestHelper with IcebergSuiteMixin {
|
||||
"map<int, bigint>",
|
||||
"date",
|
||||
"timestamp",
|
||||
"struct<`X`: bigint, `Y`: double>",
|
||||
// SPARK-37931
|
||||
if (isSparkVersionAtLeast("3.3")) "struct<X: bigint, Y: double>"
|
||||
else "struct<`X`: bigint, `Y`: double>",
|
||||
"binary",
|
||||
"struct<`X`: string>")
|
||||
// SPARK-37931
|
||||
if (isSparkVersionAtLeast("3.3")) "struct<X: string>" else "struct<`X`: string>")
|
||||
val cols = dataTypes.zipWithIndex.map { case (dt, idx) => s"c$idx" -> dt }
|
||||
val (colNames, _) = cols.unzip
|
||||
|
||||
|
||||
@ -384,7 +384,7 @@
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.iceberg</groupId>
|
||||
<artifactId>${iceberg.name}</artifactId>
|
||||
<artifactId>iceberg-spark-runtime-${spark.binary.version}_${scala.binary.version}</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
|
||||
15
pom.xml
15
pom.xml
@ -138,8 +138,7 @@
|
||||
<httpclient.version>4.5.13</httpclient.version>
|
||||
<httpcore.version>4.4.15</httpcore.version>
|
||||
<hudi.version>0.11.1</hudi.version>
|
||||
<iceberg.name>iceberg-spark-runtime-3.2_${scala.binary.version}</iceberg.name>
|
||||
<iceberg.version>0.13.2</iceberg.version>
|
||||
<iceberg.version>0.14.0</iceberg.version>
|
||||
<jackson.version>2.13.3</jackson.version>
|
||||
<jakarta.servlet-api.version>4.0.4</jakarta.servlet-api.version>
|
||||
<jakarta.xml-bind.version>2.3.2</jakarta.xml-bind.version>
|
||||
@ -165,9 +164,10 @@
|
||||
<snakeyaml.version>1.30</snakeyaml.version>
|
||||
<!--
|
||||
DO NOT forget to change the following properties when change the minor version of Spark:
|
||||
`delta.version`, `iceberg.name`, `maven.plugin.scalatest.exclude.tags`
|
||||
`delta.version`, `maven.plugin.scalatest.exclude.tags`
|
||||
-->
|
||||
<spark.version>3.2.2</spark.version>
|
||||
<spark.binary.version>3.2</spark.binary.version>
|
||||
<spark.archive.name>spark-${spark.version}-bin-hadoop3.2.tgz</spark.archive.name>
|
||||
<spark.archive.mirror>${apache.archive.dist}/spark/spark-${spark.version}</spark.archive.mirror>
|
||||
<spark.archive.download.skip>false</spark.archive.download.skip>
|
||||
@ -1075,7 +1075,7 @@
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.iceberg</groupId>
|
||||
<artifactId>${iceberg.name}</artifactId>
|
||||
<artifactId>iceberg-spark-runtime-${spark.binary.version}_${scala.binary.version}</artifactId>
|
||||
<version>${iceberg.version}</version>
|
||||
</dependency>
|
||||
|
||||
@ -2039,8 +2039,8 @@
|
||||
<id>spark-3.1</id>
|
||||
<properties>
|
||||
<spark.version>3.1.3</spark.version>
|
||||
<spark.binary.version>3.1</spark.binary.version>
|
||||
<delta.version>1.0.1</delta.version>
|
||||
<iceberg.name>iceberg-spark-runtime-3.1_${scala.binary.version}</iceberg.name>
|
||||
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow</maven.plugin.scalatest.exclude.tags>
|
||||
</properties>
|
||||
<modules>
|
||||
@ -2054,8 +2054,8 @@
|
||||
<id>spark-3.2</id>
|
||||
<properties>
|
||||
<spark.version>3.2.2</spark.version>
|
||||
<spark.binary.version>3.2</spark.binary.version>
|
||||
<delta.version>2.0.0rc1</delta.version>
|
||||
<iceberg.name>iceberg-spark-runtime-3.2_${scala.binary.version}</iceberg.name>
|
||||
<!--
|
||||
See https://github.com/apache/incubator-kyuubi/pull/2576
|
||||
Hudi 0.11 requires hudi-spark3.2-bundle to execute UT
|
||||
@ -2073,8 +2073,9 @@
|
||||
<id>spark-3.3</id>
|
||||
<properties>
|
||||
<spark.version>3.3.0</spark.version>
|
||||
<spark.binary.version>3.3</spark.binary.version>
|
||||
<spark.archive.name>spark-${spark.version}-bin-hadoop3.tgz</spark.archive.name>
|
||||
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.HudiTest</maven.plugin.scalatest.exclude.tags>
|
||||
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest</maven.plugin.scalatest.exclude.tags>
|
||||
</properties>
|
||||
<modules>
|
||||
<module>extensions/spark/kyuubi-extension-spark-common</module>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user