apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Hive JDBC Database MetaData API Auditing

Browse Source
This commit is contained in:
Kent Yao 2020-10-20 18:08:28 +08:00
parent 324c3c9c3f
commit 0849678453
9 changed files with 353 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala vendored
View File

@ -52,7 +52,7 @@ class GetTables(
.add(TABLE_TYPE, "string", nullable = true, "The table type, e.g. \"TABLE\", \"VIEW\"")
.add(REMARKS, "string", nullable = true, "Comments about the table.")
.add("TYPE_CAT", "string", nullable = true, "The types catalog." )
.add("TYPE_SCHEM", "string", nullable = true, "The types catalog")
.add("TYPE_SCHEM", "string", nullable = true, "the types schema (may be null)")
.add("TYPE_NAME", "string", nullable = true, "Type name.")
.add("SELF_REFERENCING_COL_NAME", "string", nullable = true,
"Name of the designated \"identifier\" column of a typed table.")

208
externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala vendored
View File

@ -17,10 +17,12 @@
package org.apache.kyuubi.engine.spark.operation
import java.sql.{Date, ResultSet, SQLException, Timestamp}
import java.sql.{DatabaseMetaData, Date, ResultSet, SQLException, SQLFeatureNotSupportedException, Timestamp}
import scala.collection.JavaConverters._
import scala.util.Random
import org.apache.hive.common.util.HiveVersionInfo
import org.apache.hive.service.cli.HiveSQLException
import org.apache.hive.service.rpc.thrift._
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
@ -473,16 +475,19 @@ class SparkOperationSuite extends WithSparkSQLEngine {
test("get functions") {
withJdbcStatement() { statement =>
val metaData = statement.getConnection.getMetaData
val apis = Seq(metaData.getFunctions _, metaData.getProcedures _)
Seq("to_timestamp", "date_part", "lpad", "date_format", "cos", "sin").foreach { func =>
val resultSet = metaData.getFunctions("", dftSchema, func)
while (resultSet.next()) {
val exprInfo = FunctionRegistry.expressions(func)._1
assert(resultSet.getString(FUNCTION_CAT).isEmpty)
assert(resultSet.getString(FUNCTION_SCHEM) === null)
assert(resultSet.getString(FUNCTION_NAME) === exprInfo.getName)
assert(resultSet.getString(REMARKS) ===
s"Usage: ${exprInfo.getUsage}\nExtended Usage:${exprInfo.getExtended}")
assert(resultSet.getString(SPECIFIC_NAME) === exprInfo.getClassName)
apis.foreach { apiFunc =>
val resultSet = apiFunc("", dftSchema, func)
while (resultSet.next()) {
val exprInfo = FunctionRegistry.expressions(func)._1
assert(resultSet.getString(FUNCTION_CAT).isEmpty)
assert(resultSet.getString(FUNCTION_SCHEM) === null)
assert(resultSet.getString(FUNCTION_NAME) === exprInfo.getName)
assert(resultSet.getString(REMARKS) ===
s"Usage: ${exprInfo.getUsage}\nExtended Usage:${exprInfo.getExtended}")
assert(resultSet.getString(SPECIFIC_NAME) === exprInfo.getClassName)
}
}
}
}
@ -878,4 +883,187 @@ class SparkOperationSuite extends WithSparkSQLEngine {
assert(tFetchResultsResp.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS)
}
}
test("Hive JDBC Database MetaData API Auditing") {
withJdbcStatement() { statement =>
val metaData = statement.getConnection.getMetaData
Seq(
() => metaData.allProceduresAreCallable(),
() => metaData.getURL,
() => metaData.getUserName,
() => metaData.isReadOnly,
() => metaData.nullsAreSortedHigh,
() => metaData.nullsAreSortedLow,
() => metaData.nullsAreSortedAtStart(),
() => metaData.nullsAreSortedAtEnd(),
() => metaData.usesLocalFiles(),
() => metaData.usesLocalFilePerTable(),
() => metaData.supportsMixedCaseIdentifiers(),
() => metaData.supportsMixedCaseQuotedIdentifiers(),
() => metaData.storesUpperCaseIdentifiers(),
() => metaData.storesUpperCaseQuotedIdentifiers(),
() => metaData.storesLowerCaseIdentifiers(),
() => metaData.storesLowerCaseQuotedIdentifiers(),
() => metaData.storesMixedCaseIdentifiers(),
() => metaData.storesMixedCaseQuotedIdentifiers(),
() => metaData.getSQLKeywords,
() => metaData.nullPlusNonNullIsNull,
() => metaData.supportsConvert,
() => metaData.supportsTableCorrelationNames,
() => metaData.supportsDifferentTableCorrelationNames,
() => metaData.supportsExpressionsInOrderBy(),
() => metaData.supportsOrderByUnrelated,
() => metaData.supportsGroupByUnrelated,
() => metaData.supportsGroupByBeyondSelect,
() => metaData.supportsLikeEscapeClause,
() => metaData.supportsMultipleTransactions,
() => metaData.supportsMinimumSQLGrammar,
() => metaData.supportsCoreSQLGrammar,
() => metaData.supportsExtendedSQLGrammar,
() => metaData.supportsANSI92EntryLevelSQL,
() => metaData.supportsANSI92IntermediateSQL,
() => metaData.supportsANSI92FullSQL,
() => metaData.supportsIntegrityEnhancementFacility,
() => metaData.isCatalogAtStart,
() => metaData.supportsSubqueriesInComparisons,
() => metaData.supportsSubqueriesInExists,
() => metaData.supportsSubqueriesInIns,
() => metaData.supportsSubqueriesInQuantifieds,
// Spark support this, see https://issues.apache.org/jira/browse/SPARK-18455
() => metaData.supportsCorrelatedSubqueries,
() => metaData.supportsOpenCursorsAcrossCommit,
() => metaData.supportsOpenCursorsAcrossRollback,
() => metaData.supportsOpenStatementsAcrossCommit,
() => metaData.supportsOpenStatementsAcrossRollback,
() => metaData.getMaxBinaryLiteralLength,
() => metaData.getMaxCharLiteralLength,
() => metaData.getMaxColumnsInGroupBy,
() => metaData.getMaxColumnsInIndex,
() => metaData.getMaxColumnsInOrderBy,
() => metaData.getMaxColumnsInSelect,
() => metaData.getMaxColumnsInTable,
() => metaData.getMaxConnections,
() => metaData.getMaxCursorNameLength,
() => metaData.getMaxIndexLength,
() => metaData.getMaxSchemaNameLength,
() => metaData.getMaxProcedureNameLength,
() => metaData.getMaxCatalogNameLength,
() => metaData.getMaxRowSize,
() => metaData.doesMaxRowSizeIncludeBlobs,
() => metaData.getMaxStatementLength,
() => metaData.getMaxStatements,
() => metaData.getMaxTableNameLength,
() => metaData.getMaxTablesInSelect,
() => metaData.getMaxUserNameLength,
() => metaData.supportsTransactionIsolationLevel(1),
() => metaData.supportsDataDefinitionAndDataManipulationTransactions,
() => metaData.supportsDataManipulationTransactionsOnly,
() => metaData.dataDefinitionCausesTransactionCommit,
() => metaData.dataDefinitionIgnoredInTransactions,
() => metaData.getColumnPrivileges("", "%", "%", "%"),
() => metaData.getTablePrivileges("", "%", "%"),
() => metaData.getBestRowIdentifier("", "%", "%", 0, true),
() => metaData.getVersionColumns("", "%", "%"),
() => metaData.getExportedKeys("", "default", ""),
() => metaData.supportsResultSetConcurrency(ResultSet.TYPE_FORWARD_ONLY, 2),
() => metaData.ownUpdatesAreVisible(ResultSet.TYPE_FORWARD_ONLY),
() => metaData.ownDeletesAreVisible(ResultSet.TYPE_FORWARD_ONLY),
() => metaData.ownInsertsAreVisible(ResultSet.TYPE_FORWARD_ONLY),
() => metaData.othersUpdatesAreVisible(ResultSet.TYPE_FORWARD_ONLY),
() => metaData.othersDeletesAreVisible(ResultSet.TYPE_FORWARD_ONLY),
() => metaData.othersInsertsAreVisible(ResultSet.TYPE_FORWARD_ONLY),
() => metaData.updatesAreDetected(ResultSet.TYPE_FORWARD_ONLY),
() => metaData.deletesAreDetected(ResultSet.TYPE_FORWARD_ONLY),
() => metaData.insertsAreDetected(ResultSet.TYPE_FORWARD_ONLY),
() => metaData.supportsNamedParameters(),
() => metaData.supportsMultipleOpenResults,
() => metaData.supportsGetGeneratedKeys,
() => metaData.getSuperTypes("", "%", "%"),
() => metaData.getSuperTables("", "%", "%"),
() => metaData.getAttributes("", "%", "%", "%"),
() => metaData.getResultSetHoldability,
() => metaData.locatorsUpdateCopy,
() => metaData.supportsStatementPooling,
() => metaData.getRowIdLifetime,
() => metaData.supportsStoredFunctionsUsingCallSyntax,
() => metaData.autoCommitFailureClosesAllResultSets,
() => metaData.getClientInfoProperties,
() => metaData.getFunctionColumns("", "%", "%", "%"),
() => metaData.getPseudoColumns("", "%", "%", "%"),
() => metaData.generatedKeyAlwaysReturned).foreach { func =>
val e = intercept[SQLFeatureNotSupportedException](func())
assert(e.getMessage === "Method not supported")
}
import org.apache.kyuubi.KYUUBI_VERSION
assert(metaData.allTablesAreSelectable)
assert(metaData.getDatabaseProductName === "Spark SQL")
assert(metaData.getDatabaseProductVersion === KYUUBI_VERSION)
assert(metaData.getDriverName === "Hive JDBC")
assert(metaData.getDriverVersion === HiveVersionInfo.getVersion)
assert(metaData.getDatabaseMajorVersion === Utils.majorVersion(KYUUBI_VERSION))
assert(metaData.getDatabaseMinorVersion === Utils.minorVersion(KYUUBI_VERSION))
assert(metaData.getIdentifierQuoteString === " ",
"This method returns a space \" \" if identifier quoting is not supported")
assert(metaData.getNumericFunctions === "")
assert(metaData.getStringFunctions === "")
assert(metaData.getSystemFunctions === "")
assert(metaData.getTimeDateFunctions === "")
assert(metaData.getSearchStringEscape === "\\")
assert(metaData.getExtraNameCharacters === "")
assert(metaData.supportsAlterTableWithAddColumn())
assert(!metaData.supportsAlterTableWithDropColumn())
assert(metaData.supportsColumnAliasing())
assert(metaData.supportsGroupBy)
assert(!metaData.supportsMultipleResultSets)
assert(!metaData.supportsNonNullableColumns)
assert(metaData.supportsOuterJoins)
assert(metaData.supportsFullOuterJoins)
assert(metaData.supportsLimitedOuterJoins)
assert(metaData.getSchemaTerm === "database")
assert(metaData.getProcedureTerm === "UDF")
assert(metaData.getCatalogTerm === "instance")
assert(metaData.getCatalogSeparator === ".")
assert(metaData.supportsSchemasInDataManipulation)
assert(!metaData.supportsSchemasInProcedureCalls)
assert(metaData.supportsSchemasInTableDefinitions)
assert(!metaData.supportsSchemasInIndexDefinitions)
assert(!metaData.supportsSchemasInPrivilegeDefinitions)
// This is actually supported, but hive jdbc package return false
assert(!metaData.supportsCatalogsInDataManipulation)
assert(!metaData.supportsCatalogsInProcedureCalls)
// This is actually supported, but hive jdbc package return false
assert(!metaData.supportsCatalogsInTableDefinitions)
assert(!metaData.supportsCatalogsInIndexDefinitions)
assert(!metaData.supportsCatalogsInPrivilegeDefinitions)
assert(!metaData.supportsPositionedDelete)
assert(!metaData.supportsPositionedUpdate)
assert(!metaData.supportsSelectForUpdate)
assert(!metaData.supportsStoredProcedures)
// This is actually supported, but hive jdbc package return false
assert(!metaData.supportsUnion)
assert(metaData.supportsUnionAll)
assert(metaData.getMaxColumnNameLength === 128)
assert(metaData.getDefaultTransactionIsolation === java.sql.Connection.TRANSACTION_NONE)
assert(!metaData.supportsTransactions)
assert(!metaData.getProcedureColumns("", "%", "%", "%").next())
intercept[HiveSQLException](metaData.getPrimaryKeys("", "default", ""))
assert(!metaData.getImportedKeys("", "default", "").next())
intercept[HiveSQLException] {
metaData.getCrossReference("", "default", "src", "", "default", "src2")
}
assert(!metaData.getIndexInfo("", "default", "src", true, true).next())
assert(metaData.supportsResultSetType(new Random().nextInt()))
assert(!metaData.supportsBatchUpdates)
assert(!metaData.getUDTs(",", "%", "%", null).next())
assert(!metaData.supportsSavepoints)
assert(!metaData.supportsResultSetHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT))
assert(metaData.getJDBCMajorVersion === 3)
assert(metaData.getJDBCMinorVersion === 0)
assert(metaData.getSQLStateType === DatabaseMetaData.sqlStateSQL)
assert(metaData.getMaxLogicalLobSize === 0)
assert(!metaData.supportsRefCursors)
}
}
}

43
kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
View File

@ -106,4 +106,47 @@ private[kyuubi] object Utils extends Logging {
}
def currentUser: String = UserGroupInformation.getCurrentUser.getShortUserName
private val majorMinorRegex = """^(\d+)\.(\d+)(\..*)?$""".r
private val shortVersionRegex = """^(\d+\.\d+\.\d+)(.*)?$""".r
/**
* Given a Kyuubi/Spark/Hive version string, return the major version number.
* E.g., for 2.0.1-SNAPSHOT, return 2.
*/
def majorVersion(version: String): Int = majorMinorVersion(version)._1
/**
* Given a Kyuubi/Spark/Hive version string, return the minor version number.
* E.g., for 2.0.1-SNAPSHOT, return 0.
*/
def minorVersion(version: String): Int = majorMinorVersion(version)._2
/**
* Given a Kyuubi/Spark/Hive version string, return the short version string.
* E.g., for 3.0.0-SNAPSHOT, return '3.0.0'.
*/
def shortVersion(version: String): String = {
shortVersionRegex.findFirstMatchIn(version) match {
case Some(m) => m.group(1)
case None =>
throw new IllegalArgumentException(s"Tried to parse '$version' as a project" +
s" version string, but it could not find the major/minor/maintenance version numbers.")
}
}
/**
* Given a Kyuubi/Spark/Hive version string,
* return the (major version number, minor version number).
* E.g., for 2.0.1-SNAPSHOT, return (2, 0).
*/
def majorMinorVersion(version: String): (Int, Int) = {
majorMinorRegex.findFirstMatchIn(version) match {
case Some(m) =>
(m.group(1).toInt, m.group(2).toInt)
case None =>
throw new IllegalArgumentException(s"Tried to parse '$version' as a project" +
s" version string, but it could not find the major and minor version numbers.")
}
}
}

2
kyuubi-common/src/main/scala/org/apache/kyuubi/config/ConfigProvider.scala
View File

@ -20,7 +20,7 @@ package org.apache.kyuubi.config
private[kyuubi] class ConfigProvider(conf: java.util.Map[String, String]) {
def get(key: String): Option[String] = {
if (key.startsWith(KYUUBI_PREFIX)) {
if (key.startsWith("kyuubi.")) {
Option(conf.get(key))
} else {
None

8
kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
View File

@ -33,7 +33,7 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {
}
private def loadFromMap(props: Map[String, String] = Utils.getSystemProperties): KyuubiConf = {
for ((key, value) <- props if key.startsWith(KYUUBI_PREFIX)) {
for ((key, value) <- props if key.startsWith("kyuubi.")) {
set(key, value)
}
this
@ -110,7 +110,9 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {
}
def toSparkPrefixedConf: Map[String, String] = {
settings.entrySet().asScala.map { e => SPARK_PREFIX + e.getKey -> e.getValue}.toMap
settings.entrySet().asScala.map { e =>
"spark." + e.getKey -> e.getValue
}.toMap
}
}
@ -132,7 +134,7 @@ object KyuubiConf {
}
def buildConf(key: String): ConfigBuilder = {
new ConfigBuilder(KYUUBI_PREFIX + key).onCreate(register)
new ConfigBuilder("kyuubi." + key).onCreate(register)
}
val EMBEDDED_ZK_PORT: ConfigEntry[Int] = buildConf("embedded.zookeeper.port")

23
kyuubi-common/src/main/scala/org/apache/kyuubi/config/package.scala
View File

@ -1,23 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi
package object config {
final val KYUUBI_PREFIX = "kyuubi."
final val SPARK_PREFIX = "spark."
}

92
kyuubi-common/src/main/scala/org/apache/kyuubi/operation/meta/ResultSetSchemaConstant.scala
View File

@ -24,6 +24,11 @@ object ResultSetSchemaConstant {
* Catalog name. NULL if not applicable
*/
final val TABLE_CAT = "TABLE_CAT"
/**
* String type.
* Catalog name. NULL if not applicable
*/
final val TABLE_CATALOG = "TABLE_CATALOG"
/**
@ -37,16 +42,52 @@ object ResultSetSchemaConstant {
* Table Name
*/
final val TABLE_NAME = "TABLE_NAME"
/**
* String
* table type. Typical types are "TABLE", "VIEW", ...
*/
final val TABLE_TYPE = "TABLE_TYPE"
/**
* String => explanatory comment on the table
*/
final val REMARKS = "REMARKS"
/**
* String => column name
*/
final val COLUMN_NAME = "COLUMN_NAME"
/**
* int => SQL type from [[java.sql.Types]]
*/
final val DATA_TYPE = "DATA_TYPE"
/**
* String => Data source dependent type name, for a UDT the type name is fully qualified
*/
final val TYPE_NAME = "TYPE_NAME"
/**
* int => column size
*/
final val COLUMN_SIZE = "COLUMN_SIZE"
/**
* is not used.
*/
final val BUFFER_LENGTH = "BUFFER_LENGTH"
/**
* int => the number of fractional digits.
* Null is returned for data types where DECIMAL_DIGITS is not applicable.
*/
final val DECIMAL_DIGITS = "DECIMAL_DIGITS"
/**
* int => Radix (typically either 10 or 2)
*/
final val NUM_PREC_RADIX = "NUM_PREC_RADIX"
/**
@ -54,11 +95,38 @@ object ResultSetSchemaConstant {
* Can you use NULL for this type?
*/
final val NULLABLE = "NULLABLE"
/**
* String => default value for the column, which should be interpreted as a string when the value
* is enclosed in single quotes (may be null)
*/
final val COLUMN_DEF = "COLUMN_DEF"
/**
* is not used.
*/
final val SQL_DATA_TYPE = "SQL_DATA_TYPE"
/**
* is not used.
*/
final val SQL_DATETIME_SUB = "SQL_DATETIME_SUB"
/**
* int => for char types the maximum number of bytes in the column
*/
final val CHAR_OCTET_LENGTH = "CHAR_OCTET_LENGTH"
/**
* int => index of column in table (starting at 1)
*/
final val ORDINAL_POSITION = "ORDINAL_POSITION"
/**
* String => ISO rules are used to determine the nullability for a column.
* - YES --- if the column can include NULLs
* - NO --- if the column cannot include NULLs
* - empty string --- if the nullability for the column is unknown
*/
final val IS_NULLABLE = "IS_NULLABLE"
final val SCOPE_CATALOG = "SCOPE_CATALOG"
final val SCOPE_SCHEMA = "SCOPE_SCHEMA"
@ -83,9 +151,33 @@ object ResultSetSchemaConstant {
*/
final val SEARCHABLE = "SEARCHABLE"
/**
* String => function catalog (may be null)
*/
final val FUNCTION_CAT = "FUNCTION_CAT"
/**
* String => function schema (may be null)
*/
final val FUNCTION_SCHEM = "FUNCTION_SCHEM"
/**
* String => function name. This is the name used to invoke the function
*/
final val FUNCTION_NAME = "FUNCTION_NAME"
/**
* short => kind of function:
* - functionResultUnknown - Cannot determine if a return value or table will be returned
* - functionNoTable- Does not return a table
* - functionReturnsTable - Returns a table
*/
final val FUNCTION_TYPE = "FUNCTION_TYPE"
/**
* String => the name which uniquely identifies this function within its schema.
* This is a user specified, or DBMS generated, name that may be different then the FUNCTION_NAME
* for example with overload functions
*/
final val SPECIFIC_NAME = "SPECIFIC_NAME"
}

13
kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala
View File

@ -108,4 +108,17 @@ class UtilsSuite extends KyuubiFunSuite {
}
)
}
test("version test") {
assert(Utils.majorVersion(KYUUBI_VERSION) ===
Utils.majorMinorVersion(KYUUBI_VERSION)._1)
assert(Utils.majorVersion(SPARK_COMPILE_VERSION) ===
Utils.majorMinorVersion(SPARK_COMPILE_VERSION)._1)
assert(Utils.majorVersion(HADOOP_COMPILE_VERSION) ===
Utils.majorMinorVersion(HADOOP_COMPILE_VERSION)._1)
assert(Utils.minorVersion(KYUUBI_VERSION) ===
Utils.majorMinorVersion(KYUUBI_VERSION)._2)
assert(Utils.shortVersion(KYUUBI_VERSION) ===
KYUUBI_VERSION.stripSuffix("-SNAPSHOT"))
}
}

2
kyuubi-common/src/test/scala/org/apache/kyuubi/config/ConfigProviderSuite.scala
View File

@ -29,7 +29,5 @@ class ConfigProviderSuite extends KyuubiFunSuite {
assert(provider.get("kyuubi.abc").get === "1")
assert(provider.get("kyuubi.xyz").get === "2")
assert(provider.get("spark.abc") === None)
}
}