github/spark-sql-perf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Block completion of cpu collection

Browse Source
This commit is contained in:
Michael Armbrust 2015-08-24 16:13:26 -07:00
parent e5ac7f6b4a
commit 32215e05ee
1 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/scala/com/databricks/spark/sql/perf/Benchmark.scala
View File

@ -123,6 +123,10 @@ abstract class Benchmark(
@volatile var failures = 0
@volatile var startTime = 0L
/** An optional log collection task that will run after the experiment. */
@volatile var logCollection: () => Unit = () => {}
def cartesianProduct[T](xss: List[List[T]]): List[List[T]] = xss match {
case Nil => List(Nil)
case h :: t => for(xh <- h; xt <- cartesianProduct(t)) yield xh :: xt
@ -203,9 +207,12 @@ abstract class Benchmark(
} catch {
case e: Throwable => currentMessages += s"Failed to write data: $e"
}
logCollection()
}
def scheduleCpuCollection(fs: FS) = resultsFuture.onComplete { _ =>
def scheduleCpuCollection(fs: FS) = {
logCollection = () => {
currentMessages += s"Begining CPU log collection"
try {
val location = cpu.collectLogs(sqlContext, fs, timestamp)
@ -216,6 +223,7 @@ abstract class Benchmark(
throw e
}
}
}
def cpuProfile = new Profile(sqlContext, sqlContext.read.json(getCpuLocation(timestamp)))