apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #4218] Using DB and table name when checking Delta table schema.

Browse Source 
### _Why are the changes needed?_

To close #4218 .
This change ensures BI tools can list columns on Delta Lake tables in all schemas.

<img width="312" alt="image" src="https://user-images.githubusercontent.com/89149767/215793967-722eb5f9-ffe4-4ffb-b7f9-1ded06c146d7.png">

<img width="725" alt="image" src="https://user-images.githubusercontent.com/89149767/215794036-871f005f-1494-487d-90aa-1f99891177c2.png">

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [x] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #4219 from nousot-cloud-guy/feature/delta-db-schema.

Closes #4218

569843213 [Alex Wiss-Wolferding] Reversing match order in getColumnsByCatalog.
a6d973a3e [Alex Wiss-Wolferding] Revert "[KYUUBI #1458] Delta lake table columns won't show up in DBeaver."
20337dc96 [Alex Wiss-Wolferding] Revert "Using DB and table name when checking Delta table schema."
f7e4675a7 [Alex Wiss-Wolferding] Using DB and table name when checking Delta table schema.

Authored-by: Alex Wiss-Wolferding <alex@nousot.com>
Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
Alex Wiss-Wolferding 2023-02-06 10:20:23 +08:00 committed by Cheng Pan
parent 8a99750a20
commit 2b958c69e1
No known key found for this signature in database
GPG Key ID: 8001952629BCC75D
2 changed files with 9 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala vendored
View File

@ -139,13 +139,7 @@ class CatalogShim_v2_4 extends SparkCatalogShim {
databases.flatMap { db =>
val identifiers = catalog.listTables(db, tablePattern, includeLocalTempViews = true)
catalog.getTablesByName(identifiers).flatMap { t =>
val tableSchema =
if (t.provider.getOrElse("").equalsIgnoreCase("delta")) {
spark.table(t.identifier.table).schema
} else {
t.schema
}
tableSchema.zipWithIndex.filter(f => columnPattern.matcher(f._1.name).matches())
t.schema.zipWithIndex.filter(f => columnPattern.matcher(f._1.name).matches())
.map { case (f, i) => toColumnResult(catalogName, t.database, t.identifier.table, f, i) }
}
}

16
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v3_0.scala vendored
View File

@ -188,14 +188,6 @@ class CatalogShim_v3_0 extends CatalogShim_v2_4 {
val catalog = getCatalog(spark, catalogName)
catalog match {
case builtin if builtin.name() == SESSION_CATALOG =>
super.getColumnsByCatalog(
spark,
SESSION_CATALOG,
schemaPattern,
tablePattern,
columnPattern)
case tc: TableCatalog =>
val namespaces = listNamespacesWithPattern(catalog, schemaPattern)
val tp = tablePattern.r.pattern
@ -210,6 +202,14 @@ class CatalogShim_v3_0 extends CatalogShim_v2_4 {
table.schema.zipWithIndex.filter(f => columnPattern.matcher(f._1.name).matches())
.map { case (f, i) => toColumnResult(tc.name(), namespace, tableName, f, i) }
}
case builtin if builtin.name() == SESSION_CATALOG =>
super.getColumnsByCatalog(
spark,
SESSION_CATALOG,
schemaPattern,
tablePattern,
columnPattern)
}
}
}