github/spark-sql-perf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add option to avoid cleaning after each run, to enable parallel runs

Browse Source
This commit is contained in:
Eric Liang 2017-03-14 19:45:27 -07:00
parent 53091a1935
commit 64728c7cff
3 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.sbt
View File

@ -5,7 +5,7 @@ name := "spark-sql-perf"
organization := "com.databricks"
scalaVersion := "2.10.6"
scalaVersion := "2.11.8"
crossScalaVersions := Seq("2.10.6", "2.11.8")

10
src/main/scala/com/databricks/spark/sql/perf/Benchmarkable.scala
View File

@ -58,10 +58,12 @@ trait Benchmarkable extends Logging {
private def afterBenchmark(sc: SparkContext): Unit = {
// Best-effort clean up of weakly referenced RDDs, shuffles, and broadcasts
System.gc()
// Remove any leftover blocks that still exist
sc.getExecutorStorageStatus
.flatMap { status => status.blocks.map { case (bid, _) => bid } }
.foreach { bid => SparkEnv.get.blockManager.master.removeBlock(bid) }
if (sparkContext.getConf.getBoolean("spark.databricks.benchmark.cleanBlocksAfter", true)) {
// Remove any leftover blocks that still exist
sc.getExecutorStorageStatus
.flatMap { status => status.blocks.map { case (bid, _) => bid } }
.foreach { bid => SparkEnv.get.blockManager.master.removeBlock(bid) }
}
}
private def runBenchmarkForked(

2
version.sbt
View File

@ -1 +1 @@
version in ThisBuild := "0.4.11-SNAPSHOT"
version in ThisBuild := "0.4.12-SNAPSHOT"