From 9286007814f603f67ac61c627377108deacd1538 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Tue, 21 Jun 2016 10:24:32 -0700 Subject: [PATCH] fix checkResult, bump to 0.2.4 --- build.sbt | 4 ++-- .../scala/com/databricks/spark/sql/perf/Benchmark.scala | 7 ++----- version.sbt | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/build.sbt b/build.sbt index 4cfb7c5..332acf6 100644 --- a/build.sbt +++ b/build.sbt @@ -5,14 +5,14 @@ name := "spark-sql-perf" organization := "com.databricks" -scalaVersion := "2.10.4" +scalaVersion := "2.10.6" sparkPackageName := "databricks/spark-sql-perf" // All Spark Packages need a license licenses := Seq("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0")) -sparkVersion := "v1.6.0-preview" +sparkVersion := "1.6.1" resolvers += "Apache Preview Repo" at "https://repository.apache.org/content/repositories/orgapachespark-1156/" diff --git a/src/main/scala/com/databricks/spark/sql/perf/Benchmark.scala b/src/main/scala/com/databricks/spark/sql/perf/Benchmark.scala index 67008a1..ff39440 100644 --- a/src/main/scala/com/databricks/spark/sql/perf/Benchmark.scala +++ b/src/main/scala/com/databricks/spark/sql/perf/Benchmark.scala @@ -617,13 +617,10 @@ abstract class Benchmark( case ExecutionMode.WriteParquet(location) => dataFrame.saveAsParquetFile(s"$location/$name.parquet") case ExecutionMode.HashResults => - val columnStr = dataFrame.schema.map(_.name).mkString(",") - // SELECT SUM(HASH(col1, col2, ...)) FROM (benchmark query) + // SELECT SUM(CRC32(CONCAT_WS(",", *))) FROM (benchmark query) val row = dataFrame - .selectExpr(s"hash($columnStr) as hashValue") - .groupBy() - .sum("hashValue") + .selectExpr("""SUM(CRC32(CONCAT_WS(",", *)))""") .head() result = if (row.isNullAt(0)) None else Some(row.getLong(0)) } diff --git a/version.sbt b/version.sbt index 46265ff..0557689 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "0.2.3" \ No newline at end of file +version in ThisBuild := "0.2.4"