From 6b2bf9f9ad6f6c2f620062fda78cded203f619c8 Mon Sep 17 00:00:00 2001 From: Guo Chenzhao Date: Wed, 29 May 2019 23:20:01 +0800 Subject: [PATCH] Fix files truncating according to maxRecordPerFile (#180) * Fix files truncating according to maxRecordPerFile * toDouble --- src/main/scala/com/databricks/spark/sql/perf/Tables.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/databricks/spark/sql/perf/Tables.scala b/src/main/scala/com/databricks/spark/sql/perf/Tables.scala index 4d3657b..177d38c 100644 --- a/src/main/scala/com/databricks/spark/sql/perf/Tables.scala +++ b/src/main/scala/com/databricks/spark/sql/perf/Tables.scala @@ -222,7 +222,7 @@ abstract class Tables(sqlContext: SQLContext, scaleFactor: String, log.info(s"Data has $numRows rows clustered $clusterByPartitionColumns for $maxRecordPerFile") if (maxRecordPerFile > 0 && numRows > maxRecordPerFile) { - val numFiles = ((numRows)/maxRecordPerFile).ceil.toInt + val numFiles = (numRows.toDouble/maxRecordPerFile).ceil.toInt println(s"Coalescing into $numFiles files") log.info(s"Coalescing into $numFiles files") data.coalesce(numFiles).write