diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala index 67d1ecd69..cc945f7ff 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.engine.spark.operation +import java.io.IOException import java.time.ZoneId import org.apache.commons.lang3.StringUtils @@ -122,7 +123,12 @@ abstract class SparkOperation(spark: SparkSession, opType: OperationType, sessio override def close(): Unit = { cleanup(OperationState.CLOSED) - getOperationLog.foreach(_.close()) + try { + getOperationLog.foreach(_.close()) + } catch { + case e: IOException => + error(e.getMessage, e) + } } override def getResultSetSchema: TTableSchema = SchemaHelper.toTTableSchema(resultSchema) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/OperationLog.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/OperationLog.scala index 2976f3e0d..1e485532a 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/OperationLog.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/OperationLog.scala @@ -83,7 +83,7 @@ object OperationLog extends Logging { } } -class OperationLog(path: Path) extends Logging { +class OperationLog(path: Path) { private val writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8) private val reader = Files.newBufferedReader(path, StandardCharsets.UTF_8) @@ -134,7 +134,12 @@ class OperationLog(path: Path) extends Logging { Files.delete(path) } catch { case e: IOException => - error(s"Failed to remove corresponding log file of operation: ${path.toAbsolutePath}", e) + // Printing log here may cause a deadlock. The lock order of OperationLog.write + // is RootLogger -> LogDivertAppender -> OperationLog. If printing log here, the + // lock order is OperationLog -> RootLogger. So the exception is thrown and + // processing at the invocations + throw new IOException( + s"Failed to remove corresponding log file of operation: ${path.toAbsolutePath}", e) } } } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiOperation.scala index 5ba246441..256815837 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiOperation.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/operation/KyuubiOperation.scala @@ -17,6 +17,8 @@ package org.apache.kyuubi.operation +import java.io.IOException + import com.codahale.metrics.MetricRegistry import org.apache.commons.lang3.StringUtils import org.apache.hive.service.rpc.thrift._ @@ -97,6 +99,11 @@ abstract class KyuubiOperation( if (_remoteOpHandle != null && !isClosedOrCanceled) { try { getOperationLog.foreach(_.close()) + } catch { + case e: IOException => + error(e.getMessage, e) + } + try { client.closeOperation(_remoteOpHandle) setState(OperationState.CLOSED) } catch onError("closing")