From 9269f8f594cc3b81aa2d0c5e245447d7c0b9512b Mon Sep 17 00:00:00 2001 From: Michael Armbrust Date: Wed, 23 Dec 2015 11:03:06 -0800 Subject: [PATCH] Capture BuildInfo when available Author: Michael Armbrust Closes #45 from marmbrus/buildInfo. --- .../com/databricks/spark/sql/perf/Benchmark.scala | 15 ++++++++++++--- .../com/databricks/spark/sql/perf/results.scala | 3 ++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/main/scala/com/databricks/spark/sql/perf/Benchmark.scala b/src/main/scala/com/databricks/spark/sql/perf/Benchmark.scala index 67008a1..233706f 100644 --- a/src/main/scala/com/databricks/spark/sql/perf/Benchmark.scala +++ b/src/main/scala/com/databricks/spark/sql/perf/Benchmark.scala @@ -16,14 +16,14 @@ package com.databricks.spark.sql.perf -import org.apache.spark.rdd.RDD - import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.concurrent._ import scala.concurrent.duration._ import scala.concurrent.ExecutionContext.Implicits.global +import scala.util.Try +import org.apache.spark.rdd.RDD import org.apache.spark.sql.{Dataset, AnalysisException, DataFrame, SQLContext} import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation import org.apache.spark.sql.execution.SparkPlan @@ -54,10 +54,19 @@ abstract class Benchmark( protected implicit def toOption[A](a: A) = Option(a) + val buildInfo = Try(getClass.getClassLoader.loadClass("org.apache.spark.BuildInfo")).map { cls => + cls.getMethods + .filter(_.getReturnType == classOf[String]) + .filterNot(_.getName == "toString") + .map(m => m.getName -> m.invoke(cls).asInstanceOf[String]) + .toMap + }.getOrElse(Map.empty) + def currentConfiguration = BenchmarkConfiguration( sqlConf = sqlContext.getAllConfs, sparkConf = sparkContext.getConf.getAll.toMap, - defaultParallelism = sparkContext.defaultParallelism) + defaultParallelism = sparkContext.defaultParallelism, + buildInfo = buildInfo) /** * A Variation represents a setting (e.g. the number of shuffle partitions or if tables diff --git a/src/main/scala/com/databricks/spark/sql/perf/results.scala b/src/main/scala/com/databricks/spark/sql/perf/results.scala index 80e2e80..ebd5e41 100644 --- a/src/main/scala/com/databricks/spark/sql/perf/results.scala +++ b/src/main/scala/com/databricks/spark/sql/perf/results.scala @@ -43,7 +43,8 @@ case class BenchmarkConfiguration( sparkVersion: String = org.apache.spark.SPARK_VERSION, sqlConf: Map[String, String], sparkConf: Map[String, String], - defaultParallelism: Int) + defaultParallelism: Int, + buildInfo: Map[String, String]) /** * The result of a query.