diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala index dd6163ec9..2feadbced 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala @@ -299,7 +299,7 @@ object KyuubiArrowConverters extends SQLConfHelper with Logging { MessageSerializer.serialize(writeChannel, batch) // Always write the Ipc options at the end. - ArrowStreamWriter.writeEndOfStream(writeChannel, IpcOption.DEFAULT) + ArrowStreamWriter.writeEndOfStream(writeChannel, ARROW_IPC_OPTION_DEFAULT) batch.close() } { @@ -318,4 +318,7 @@ object KyuubiArrowConverters extends SQLConfHelper with Logging { context: TaskContext): Iterator[InternalRow] = { ArrowConverters.fromBatchIterator(arrowBatchIter, schema, timeZoneId, context) } + + // IpcOption.DEFAULT was introduced in ARROW-11081(ARROW-4.0.0), add this for adapt Spark-3.1/3.2 + final private val ARROW_IPC_OPTION_DEFAULT = new IpcOption() }