From cdbe05fa4c6a18cd52185f3152b8648f676b240d Mon Sep 17 00:00:00 2001 From: Fu Chen Date: Mon, 17 Apr 2023 16:52:54 +0800 Subject: [PATCH] [KYUUBI #4720] [ARROW] Fix java.lang.NoSuchFieldError: IpcOption.DEFAULT for Spark-3.1/3.2 ### _Why are the changes needed?_ `IpcOption.DEFAULT` was introduced in [ARROW-11081](https://github.com/apache/arrow/pull/9053)(ARROW-4.0.0), add `ARROW_IPC_OPTION_DEFAULT` for adapt Spark-3.1/3.2 ``` Caused by: java.lang.NoSuchFieldError: DEFAULT at org.apache.spark.sql.execution.arrow.KyuubiArrowConverters$ArrowBatchIterator.$anonfun$next$1(KyuubiArrowConverters.scala:304) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1491) at org.apache.spark.sql.execution.arrow.KyuubiArrowConverters$ArrowBatchIterator.next(KyuubiArrowConverters.scala:308) at org.apache.spark.sql.execution.arrow.KyuubiArrowConverters$ArrowBatchIterator.next(KyuubiArrowConverters.scala:231) at scala.collection.Iterator.foreach(Iterator.scala:943) at scala.collection.Iterator.foreach$(Iterator.scala:943) at org.apache.spark.sql.execution.arrow.KyuubiArrowConverters$ArrowBatchIterator.foreach(KyuubiArrowConverters.scala:231) ``` ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4720 from cfmcgrady/arrow-ipc-option. Closes #4720 2c80e670e [Fu Chen] fix style a8294f637 [Fu Chen] add ARROW_IPC_OPTION_DEFAULT Authored-by: Fu Chen Signed-off-by: Fu Chen --- .../spark/sql/execution/arrow/KyuubiArrowConverters.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala index dd6163ec9..2feadbced 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala @@ -299,7 +299,7 @@ object KyuubiArrowConverters extends SQLConfHelper with Logging { MessageSerializer.serialize(writeChannel, batch) // Always write the Ipc options at the end. - ArrowStreamWriter.writeEndOfStream(writeChannel, IpcOption.DEFAULT) + ArrowStreamWriter.writeEndOfStream(writeChannel, ARROW_IPC_OPTION_DEFAULT) batch.close() } { @@ -318,4 +318,7 @@ object KyuubiArrowConverters extends SQLConfHelper with Logging { context: TaskContext): Iterator[InternalRow] = { ArrowConverters.fromBatchIterator(arrowBatchIter, schema, timeZoneId, context) } + + // IpcOption.DEFAULT was introduced in ARROW-11081(ARROW-4.0.0), add this for adapt Spark-3.1/3.2 + final private val ARROW_IPC_OPTION_DEFAULT = new IpcOption() }