Write only one file per run

Author: Michael Armbrust <michael@databricks.com> Closes #35 from marmbrus/oneResultFile.
2015-12-08 15:46:20 -08:00 · 2015-12-08 15:46:20 -08:00 · 0aa2569a18
commit 0aa2569a18
parent 12b7537181
1 changed files with 3 additions and 1 deletions
--- a/src/main/scala/com/databricks/spark/sql/perf/Benchmark.scala
+++ b/src/main/scala/com/databricks/spark/sql/perf/Benchmark.scala
@ -212,7 +212,9 @@ abstract class Benchmark(
        try {
          val resultsTable = sqlContext.createDataFrame(results)
          currentMessages += s"Results written to table: 'sqlPerformance' at $resultsLocation/$timestamp"
-          results.toDF().write
+          results.toDF()
+            .coalesce(1)
+            .write
            .format("json")
            .save(s"$resultsLocation/$timestamp")