fix checking results and bump to 0.4.9

This commit is contained in:
Davies Liu 2016-06-17 12:53:12 -07:00
parent 0d1e9043f1
commit ea342c6165
2 changed files with 4 additions and 7 deletions

View File

@ -127,13 +127,10 @@ class Query(
case ExecutionMode.WriteParquet(location) =>
dataFrame.write.parquet(s"$location/$name.parquet")
case ExecutionMode.HashResults =>
val columnStr = dataFrame.schema.map(_.name).mkString(",")
// SELECT SUM(HASH(col1, col2, ...)) FROM (benchmark query)
// SELECT SUM(CRC32(CONCAT_WS(", ", *))) FROM (benchmark query)
val row =
dataFrame
.selectExpr(s"hash($columnStr) as hashValue")
.groupBy()
.sum("hashValue")
.selectExpr(s"sum(crc32(concat_ws(',', *)))")
.head()
result = if (row.isNullAt(0)) None else Some(row.getLong(0))
}
@ -169,4 +166,4 @@ class Query(
def checkResult: Query = {
new Query(name, buildDataFrame, description, sqlText, ExecutionMode.HashResults)
}
}
}

View File

@ -1 +1 @@
version in ThisBuild := "0.4.8-SNAPSHOT"
version in ThisBuild := "0.4.9-SNAPSHOT"