github/spark-sql-perf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix files truncating according to maxRecordPerFile (#180)

Browse Source 
* Fix files truncating according to maxRecordPerFile

* toDouble
This commit is contained in:
Guo Chenzhao 2019-05-29 23:20:01 +08:00 committed by Wenchen Fan
parent 3f92a094cc
commit 6b2bf9f9ad
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/scala/com/databricks/spark/sql/perf/Tables.scala
View File

@ -222,7 +222,7 @@ abstract class Tables(sqlContext: SQLContext, scaleFactor: String,
log.info(s"Data has $numRows rows clustered $clusterByPartitionColumns for $maxRecordPerFile") log.info(s"Data has $numRows rows clustered $clusterByPartitionColumns for $maxRecordPerFile")
if (maxRecordPerFile > 0 && numRows > maxRecordPerFile) { if (maxRecordPerFile > 0 && numRows > maxRecordPerFile) {
val numFiles = ((numRows)/maxRecordPerFile).ceil.toInt val numFiles = (numRows.toDouble/maxRecordPerFile).ceil.toInt
println(s"Coalescing into $numFiles files") println(s"Coalescing into $numFiles files")
log.info(s"Coalescing into $numFiles files") log.info(s"Coalescing into $numFiles files")
data.coalesce(numFiles).write data.coalesce(numFiles).write