diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md index c12047443..1a3fee2fc 100644 --- a/docs/deployment/settings.md +++ b/docs/deployment/settings.md @@ -446,7 +446,7 @@ kyuubi.operation.plan.only.mode|none|Configures the statement performed mode, Th kyuubi.operation.plan.only.output.style|plain|Configures the planOnly output style, The value can be 'plain' and 'json', default value is 'plain', this configuration supports only the output styles of the Spark engine|string|1.7.0 kyuubi.operation.progress.enabled|false|Whether to enable the operation progress. When true, the operation progress will be returned in `GetOperationStatus`.|boolean|1.6.0 kyuubi.operation.query.timeout|<undefined>|Timeout for query executions at server-side, take affect with client-side timeout(`java.sql.Statement.setQueryTimeout`) together, a running query will be cancelled automatically if timeout. It's off by default, which means only client-side take fully control whether the query should timeout or not. If set, client-side timeout capped at this point. To cancel the queries right away without waiting task to finish, consider enabling kyuubi.operation.interrupt.on.cancel together.|duration|1.2.0 -kyuubi.operation.result.codec|simple|Specify the result codec, available configs are: |string|1.7.0 +kyuubi.operation.result.format|thrift|Specify the result format, available configs are: |string|1.7.0 kyuubi.operation.result.max.rows|0|Max rows of Spark query results. Rows that exceeds the limit would be ignored. By setting this value to 0 to disable the max rows limit.|int|1.6.0 kyuubi.operation.scheduler.pool|<undefined>|The scheduler pool of job. Note that, this config should be used after change Spark config spark.scheduler.mode=FAIR.|string|1.1.1 kyuubi.operation.spark.listener.enabled|true|When set to true, Spark engine registers a SQLOperationListener before executing the statement, logs a few summary statistics when each stage completes.|boolean|1.6.0 diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala index 27b9dde4d..2cdc2b500 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala @@ -176,5 +176,5 @@ class ExecuteStatement( } override def getResultSetMetadataHints(): Seq[String] = - Seq(s"__kyuubi_operation_result_codec__=$resultCodec") + Seq(s"__kyuubi_operation_result_format__=$resultFormat") } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala index 67ab29172..842ff944f 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala @@ -260,14 +260,14 @@ abstract class SparkOperation(session: Session) override def shouldRunAsync: Boolean = false protected def arrowEnabled(): Boolean = { - resultCodec().equalsIgnoreCase("arrow") && + resultFormat().equalsIgnoreCase("arrow") && // TODO: (fchen) make all operation support arrow getClass.getCanonicalName == classOf[ExecuteStatement].getCanonicalName } - protected def resultCodec(): String = { + protected def resultFormat(): String = { // TODO: respect the config of the operation ExecuteStatement, if it was set. - spark.conf.get("kyuubi.operation.result.codec", "simple") + spark.conf.get("kyuubi.operation.result.format", "thrift") } protected def setSessionUserSign(): Unit = { diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala index 9a4b3955e..e46456914 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala @@ -30,26 +30,26 @@ class SparkArrowbasedOperationSuite extends WithSparkSQLEngine with SparkDataTyp override def withKyuubiConf: Map[String, String] = Map.empty override def jdbcVars: Map[String, String] = { - Map(KyuubiConf.OPERATION_RESULT_CODEC.key -> resultCodec) + Map(KyuubiConf.OPERATION_RESULT_FORMAT.key -> resultFormat) } - override def resultCodec: String = "arrow" + override def resultFormat: String = "arrow" - test("detect resultSet codec") { + test("detect resultSet format") { withJdbcStatement() { statement => - checkResultSetCodec(statement, "arrow") - statement.executeQuery(s"set ${KyuubiConf.OPERATION_RESULT_CODEC.key}=simple") - checkResultSetCodec(statement, "simple") + checkResultSetFormat(statement, "arrow") + statement.executeQuery(s"set ${KyuubiConf.OPERATION_RESULT_FORMAT.key}=thrift") + checkResultSetFormat(statement, "thrift") } } - def checkResultSetCodec(statement: Statement, expectCodec: String): Unit = { + def checkResultSetFormat(statement: Statement, expectFormat: String): Unit = { val query = s""" - |SELECT '$${hivevar:${KyuubiConf.OPERATION_RESULT_CODEC.key}}' AS col + |SELECT '$${hivevar:${KyuubiConf.OPERATION_RESULT_FORMAT.key}}' AS col |""".stripMargin val resultSet = statement.executeQuery(query) assert(resultSet.next()) - assert(resultSet.getString("col") === expectCodec) + assert(resultSet.getString("col") === expectFormat) } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 0703412cc..349292d87 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -1534,18 +1534,18 @@ object KyuubiConf { .booleanConf .createWithDefault(false) - val OPERATION_RESULT_CODEC: ConfigEntry[String] = - buildConf("kyuubi.operation.result.codec") - .doc("Specify the result codec, available configs are: