github/spark-sql-perf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fail gracefully when invalid CPU logs are encountered

Browse Source 
Author: Michael Armbrust <michael@databricks.com>

Closes #18 from marmbrus/parseCpuFail.
This commit is contained in:
Michael Armbrust 2015-09-09 22:02:23 -07:00
parent e2dc749480
commit f03b3af719
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/scala/com/databricks/spark/sql/perf/CpuProfile.scala
View File

@ -35,6 +35,7 @@ import com.twitter.jvm.CpuProfile
*/
package object cpu {
// Placeholder for DBFS.
type FS = {
def cp(from: String, to: String, recurse: Boolean): Boolean
def rm(dir: String, recurse: Boolean): Boolean
@ -102,8 +103,9 @@ package object cpu {
case stackLine(cls, method, file, line) => new StackTraceElement(cls, method, file, line.stripPrefix(":").toInt)
}
val counts = cpuLogs.groupBy($"stack").agg(count($"*")).collect().map {
case Row(stackLines: Seq[String], count: Long) => stackLines.map(toStackElement) -> count
val counts = cpuLogs.groupBy($"stack").agg(count($"*")).collect().flatMap {
case Row(stackLines: Seq[String], count: Long) => stackLines.map(toStackElement) -> count :: Nil
case other => println(s"Failed to parse $other"); Nil
}.toMap
val profile = new com.twitter.jvm.CpuProfile(counts, com.twitter.util.Duration.fromSeconds(10), cpuLogs.count().toInt, 0)
@ -122,4 +124,4 @@ package object cpu {
s"""<a href="https://dogfood.staging.cloud.databricks.com/files/cpu.profiles/$timestamp.svg"/>CPU Usage Visualization</a>"""
}
}
}
}