From 4de2976fd41a14ea4dd1dfdcd96d7a92f86c2a8a Mon Sep 17 00:00:00 2001 From: Min Zhao Date: Sun, 22 Aug 2021 00:28:45 +0800 Subject: [PATCH] [KYUUBI #659] Add UDF engine_name ### _Why are the changes needed?_ Add UDF engine_name to show the runtime information about execution engine ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request Closes #747 from zhaomin1423/udf. Closes #659 8c3b3c16 [Min Zhao] Merge branch 'master' into udf f184a661 [Min Zhao] Merge branch 'master' into udf 9ea0c190 [Min Zhao] get appName from driver side 09f07523 [Min Zhao] get conf by SparkEnv.get.conf in the engine_name udf 6c000784 [Min Zhao] update the description of the engine_name udf 63ef2762 [Min Zhao] update class to org.apache.kyuubi.engine.spark.udf.KyuubiDefinedFunctionSuite in functions.md db02884c [Min Zhao] [KYUUBI #659] Add UDF engine_name Lead-authored-by: Min Zhao Co-authored-by: Min Zhao <49054376+zhaomin1423@users.noreply.github.com> Signed-off-by: Cheng Pan (cherry picked from commit 8bab5b8d6a05b33b8ad474811f89299860db60f4) Signed-off-by: Cheng Pan --- docs/sql/functions.md | 1 + .../apache/kyuubi/engine/spark/udf/KDFRegistry.scala | 10 ++++++++++ .../engine/spark/udf/KyuubiDefinedFunctionSuite.scala | 1 - .../scala/org/apache/kyuubi/operation/JDBCTests.scala | 9 +++++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/sql/functions.md b/docs/sql/functions.md index cc6e19e60..ba9df148c 100644 --- a/docs/sql/functions.md +++ b/docs/sql/functions.md @@ -30,5 +30,6 @@ Kyuubi provides several auxiliary SQL functions as supplement to Spark's [Built- Name | Description | Return Type | Since --- | --- | --- | --- kyuubi_version | Return the version of Kyuubi Server | string | 1.3.0 +engine_name | Return the spark application name for the associated query engine | string | 1.3.0 system_user | Return the system user name for the associated query engine | string | 1.3.0 diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/udf/KDFRegistry.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/udf/KDFRegistry.scala index 14c837af8..537a77117 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/udf/KDFRegistry.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/udf/KDFRegistry.scala @@ -29,6 +29,8 @@ object KDFRegistry { val registeredFunctions = new ArrayBuffer[KyuubiDefinedFunction]() + val appName = SparkSession.active.sparkContext.appName + val kyuubi_version: KyuubiDefinedFunction = create( "kyuubi_version", udf(() => KYUUBI_VERSION).asNonNullable(), @@ -36,6 +38,14 @@ object KDFRegistry { "string", "1.3.0") + val engine_name: KyuubiDefinedFunction = create( + "engine_name", + udf(() => appName).asNonNullable(), + "Return the spark application name for the associated query engine", + "string", + "1.3.0" + ) + val system_user: KyuubiDefinedFunction = create( "system_user", udf(() => System.getProperty("user.name")).asNonNullable(), diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala index 64b03d0bd..21e8e13e3 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala @@ -86,7 +86,6 @@ class KyuubiDefinedFunctionSuite extends KyuubiFunSuite { KDFRegistry.registeredFunctions.foreach { func => newOutput += s"${func.name} | ${func.description} | ${func.returnType} | ${func.since}" } - newOutput += "" TestUtils.verifyOutput(markdown, newOutput, getClass.getCanonicalName) } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala index 2bb533d42..13f8e0e2b 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala @@ -21,6 +21,7 @@ import java.sql.{Date, SQLException, SQLTimeoutException, Timestamp} import scala.collection.JavaConverters._ +import org.apache.commons.lang3.StringUtils import org.apache.hive.service.rpc.thrift.{TExecuteStatementReq, TFetchResultsReq, TOpenSessionReq, TStatusCode} import org.apache.kyuubi.KYUUBI_VERSION @@ -362,6 +363,14 @@ trait JDBCTests extends BasicJDBCTests { } } + test("kyuubi defined function - engine_name") { + withJdbcStatement() { statement => + val rs = statement.executeQuery("SELECT engine_name()") + assert(rs.next()) + assert(StringUtils.isNotBlank(rs.getString(1))) + } + } + // dockerfile use kyuubi as user which is not same with non-k8s env. ignore("kyuubi defined function - system_user") { withJdbcStatement() { statement =>