github/spark-sql-perf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Removed queryOutputLocation parameter

Browse Source
This commit is contained in:
Jean-Yves Stephan 2015-07-21 13:26:50 -07:00
parent 9640cd8c1e
commit 933f3f0bb5
2 changed files with 4 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/scala/com/databricks/spark/sql/perf/query.scala
View File

@ -50,7 +50,7 @@ case class QueryForTest(
(endTime - startTime).toDouble / 1000000
}
def benchmark(description: String = "", queryOutputLocation: Option[String]) = {
def benchmark(description: String = "") = {
try {
sparkContext.setJobDescription(s"Query: ${query.name}, $description")
val dataFrame = sqlContext.sql(query.sqlText)
@ -77,7 +77,7 @@ case class QueryForTest(
val executionTime = query.executionMode match {
case CollectResults => benchmarkMs { dataFrame.rdd.collect() }
case ForeachResults => benchmarkMs { dataFrame.rdd.foreach { row => Unit } }
case WriteParquet(location) => benchmarkMs { dataFrame.saveAsParquetFile(s"$location/${query.name}.parquet") }
case WriteParquet(location) => benchmarkMs { dataFrame.saveAsParquetFile(s"$location/$name.parquet") }
}
val joinTypes = dataFrame.queryExecution.executedPlan.collect {
@ -90,9 +90,7 @@ case class QueryForTest(
tableIdentifier.last
}
}
queryOutputLocation.foreach(dir => dataFrame.saveAsParquetFile(s"$dir/$name.parquet"))
BenchmarkResult(
name = query.name,
joinTypes = joinTypes,

3
src/main/scala/com/databricks/spark/sql/perf/runBenchmarks.scala
View File

@ -196,7 +196,6 @@ abstract class Dataset(
def runExperiment(
queries: Seq[Query],
resultsLocation: String,
queryOutputLocation: Option[String] = None,
includeBreakdown: Boolean = false,
iterations: Int = 3,
variations: Seq[Variation[_]] = Seq(Variation("StandardRun", Seq("")) { _ => {} }),
@ -239,7 +238,7 @@ abstract class Dataset(
currentMessages += s"Running query ${q.name} $setup"
currentQuery = q.name
val singleResult = try q.benchmark(setup, queryOutputLocation) :: Nil catch {
val singleResult = try q.benchmark(setup) :: Nil catch {
case e: Exception =>
currentMessages += s"Failed to run query ${q.name}: $e"
Nil