github/spark-sql-perf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix writing results

Browse Source
This commit is contained in:
Davies Liu 2016-03-30 11:56:55 -07:00
parent 2cd9b89388
commit 656f1bdb17
2 changed files with 14 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/scala/com/databricks/spark/sql/perf/Benchmark.scala
View File

@ -191,7 +191,7 @@ abstract class Benchmark(
}
// Run the benchmarks!
val results = (1 to iterations).flatMap { i =>
val results: Seq[ExperimentRun] = (1 to iterations).flatMap { i =>
combinations.map { setup =>
val currentOptions = variations.asInstanceOf[Seq[Variation[Any]]].zip(setup).map {
case (v, idx) =>
@ -246,13 +246,11 @@ abstract class Benchmark(
try {
val resultsTable = sqlContext.createDataFrame(results)
logMessage(s"Results written to table: 'sqlPerformance' at $resultPath")
results.toDF()
resultsTable
.coalesce(1)
.write
.format("json")
.save(resultPath)
results.toDF()
} catch {
case e: Throwable => logMessage(s"Failed to write data: $e")
}

35
src/main/scala/com/databricks/spark/sql/perf/tpcds/TPCDS_1_4_Queries.scala
View File

@ -3863,27 +3863,16 @@ trait Tpcds_1_4_Queries extends Benchmark {
}
val tpcds1_4QueriesMap = tpcds1_4Queries.map(q => q.name.split("-").get(0) -> q).toMap
// Queries that are plannable using the SQL dialect.
val sqlDialectPlannableQueries = Seq("q2", "q3", "q4", "q7", "q8", "q11", "q13", "q15", "q17",
"q19", "q21", "q25", "q26", "q28", "q29", "q31", "q34", "q37", "q38", "q39a", "q39b", "q40",
"q42", "q43", "q46", "q48", "q52", "q55", "q59", "q64", "q65", "q66", "q68", "q71", "q72",
"q73", "q74", "q75", "q76", "q78", "q79", "q82", "q84", "q85", "q87", "q88", "q90", "q91",
"q93", "q96", "q97", "qSsMax").map(tpcds1_4QueriesMap)
// Queries that are plannable using HiveQL
val hiveDialectPlannableQueries = Seq("q3", "q4", "q7", "q11", "q13", "q15", "q17", "q19", "q21",
"q25", "q26", "q28", "q29", "q31", "q34", "q37", "q39a", "q39b", "q40", "q42", "q43", "q46",
"q47", "q48", "q49", "q51", "q52", "q53", "q55", "q57", "q59", "q63", "q64", "q65", "q68",
"q71", "q72", "q73", "q74", "q75", "q76", "q78", "q79", "q82", "q84", "q85", "q88", "q89",
"q90", "q91", "q93", "q96", "q97", "qSsMax").map(tpcds1_4QueriesMap)
// check results q17, q25, q91
// OOM: q4, q11, q13, q48, q65, q78, q85
// Hang/OOM: q64
// q84 failed when run together.
// q93 sporadic OOM
val sqlDialectRunnable: Seq[Query] = Seq("q2", "q3", "q7", "q8", "q15", "q17", "q19", "q21",
"q25", "q26", "q28", "q29", "q31", "q34", "q37", "q38", "q39a", "q39b", "q40", "q42", "q43",
"q46", "q52", "q55", "q59", "q66", "q68", "q71", "q72", "q73", "q74", "q75", "q76", "q79",
"q82", "q84", "q87", "q88", "q90", "q91", "q93", "q96", "q97", "qSsMax").map(tpcds1_4QueriesMap)
// q72 slow on SC=1
val runnable: Seq[Query] = Seq(
"q1", "q2", "q3", "q4", "q5", "q7", "q8", "q9",
"q11", "q12", "q13", "q15", "q17", "q18", "q19",
"q20", "q21", "q22", "q25", "q26", "q27", "q28", "q29",
"q31", "q34", "q36", "q37", "q38", "q39a", "q39b",
"q40", "q42", "q43", "q44", "q46", "q47", "q48", "q49",
"q50", "q51", "q52", "q53", "q54", "q55", "q57", "q59",
"q61", "q62", "q63", "q64", "q65", "q66", "q67", "q68",
"q71", "q72", "q73", "q74", "q75", "q76", "q77", "q78", "q79",
"q80", "q82", "q84", "q85", "q86", "q87", "q88", "q89",
"q90", "q91", "q93", "q96", "q97", "q98", "q99", "qSsMax").map(tpcds1_4QueriesMap)
}