diff --git a/pom.xml b/pom.xml index ae4fcf556..fa05309fa 100644 --- a/pom.xml +++ b/pom.xml @@ -264,6 +264,12 @@ test-jar test + + ${hive.group} + hive-service + ${hive.version} + test + diff --git a/src/main/scala/yaooqinn/kyuubi/schema/Column.scala b/src/main/scala/yaooqinn/kyuubi/schema/Column.scala deleted file mode 100644 index 212f708e4..000000000 --- a/src/main/scala/yaooqinn/kyuubi/schema/Column.scala +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package yaooqinn.kyuubi.schema - -class Column { - -} diff --git a/src/test/scala/yaooqinn/kyuubi/schema/ColumnBasedSetSuite.scala b/src/test/scala/yaooqinn/kyuubi/schema/ColumnBasedSetSuite.scala new file mode 100644 index 000000000..8aeb1874a --- /dev/null +++ b/src/test/scala/yaooqinn/kyuubi/schema/ColumnBasedSetSuite.scala @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package yaooqinn.kyuubi.schema + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.StructType + +class ColumnBasedSetSuite extends SparkFunSuite { + val maxRows: Int = 5 + val schema = new StructType().add("a", "int").add("b", "string") + val rows = Seq( + Row(1, "11"), + Row(2, "22"), + Row(3, "33"), + Row(4, "44"), + Row(5, "55"), + Row(6, "66"), + Row(7, "77"), + Row(8, "88"), + Row(9, "99"), + Row(10, "000"), + Row(11, "111"), + Row(12, "222"), + Row(13, "333"), + Row(14, "444"), + Row(15, "555"), + Row(16, "666")) + + test("row set basic suites") { + // fetch next + val rowIterator = rows.iterator + var taken = rowIterator.take(maxRows).toSeq + var tRowSet = ColumnBasedSet(schema, taken).toTRowSet + assert(tRowSet.getColumns.size() === 2) + assert(tRowSet.getRowsSize === 0) + assert(tRowSet.getColumns.get(0).getI32Val.getValuesSize === 5) + assert(tRowSet.getColumns.get(0).getI32Val.getValues.get(0).intValue() === 1) + assert(tRowSet.getColumns.get(1).getStringVal.getValues.get(1) === "22") + assert(tRowSet.getColumns.get(1).getStringVal.getValues.get(2) === "33") + assert(tRowSet.getColumns.get(1).getStringVal.getValues.get(3) === "44") + assert(tRowSet.getColumns.get(1).getStringVal.getValues.get(4) === "55") + + taken = rowIterator.take(maxRows).toSeq + tRowSet = ColumnBasedSet(schema, taken).toTRowSet + assert(tRowSet.getColumns.get(0).getI32Val.getValues.get(0).intValue() === 6) + assert(tRowSet.getColumns.get(1).getStringVal.getValues.get(1) === "77") + assert(tRowSet.getColumns.get(1).getStringVal.getValues.get(2) === "88") + assert(tRowSet.getColumns.get(1).getStringVal.getValues.get(3) === "99") + assert(tRowSet.getColumns.get(1).getStringVal.getValues.get(4) === "000") + + taken = rowIterator.take(maxRows).toSeq + tRowSet = ColumnBasedSet(schema, taken).toTRowSet + assert(tRowSet.getColumns.get(0).getI32Val.getValues.get(0).intValue() === 11) + assert(tRowSet.getColumns.get(1).getStringVal.getValues.get(1) === "222") + assert(tRowSet.getColumns.get(1).getStringVal.getValues.get(2) === "333") + assert(tRowSet.getColumns.get(1).getStringVal.getValues.get(3) === "444") + assert(tRowSet.getColumns.get(1).getStringVal.getValues.get(4) === "555") + + taken = rowIterator.take(maxRows).toSeq + tRowSet = ColumnBasedSet(schema, taken).toTRowSet + assert(tRowSet.getColumns.get(0).getI32Val.getValues.get(0).intValue() === 16) + intercept[IndexOutOfBoundsException](tRowSet.getColumns.get(1).getStringVal.getValues.get(1)) + + assert(rowIterator.isEmpty) + } + + test("kyuubi set to TRowSet then to Hive Row Set") { + val rowIterator = rows.iterator + val taken = rowIterator.take(maxRows).toSeq + val tRowSet = ColumnBasedSet(schema, taken).toTRowSet + + val hiveRowSet = new org.apache.hive.service.cli.ColumnBasedSet(tRowSet) + assert(hiveRowSet.getColumns.get(0).get(0).asInstanceOf[Int] === 1) + assert(hiveRowSet.getColumns.get(1).get(0).equals("11")) + assert(hiveRowSet.getColumns.get(1).get(4).equals("55")) + } +} diff --git a/src/test/scala/yaooqinn/kyuubi/schema/RowBasedSetSuite.scala b/src/test/scala/yaooqinn/kyuubi/schema/RowBasedSetSuite.scala index 8fcad1db5..c1caf935f 100644 --- a/src/test/scala/yaooqinn/kyuubi/schema/RowBasedSetSuite.scala +++ b/src/test/scala/yaooqinn/kyuubi/schema/RowBasedSetSuite.scala @@ -17,35 +17,33 @@ package yaooqinn.kyuubi.schema +import org.apache.hive.service.cli import org.apache.spark.SparkFunSuite import org.apache.spark.sql.Row import org.apache.spark.sql.types.StructType class RowBasedSetSuite extends SparkFunSuite { + val maxRows: Int = 5 + val schema = new StructType().add("a", "int").add("b", "string") + val rows = Seq( + Row(1, "11"), + Row(2, "22"), + Row(3, "33"), + Row(4, "44"), + Row(5, "55"), + Row(6, "66"), + Row(7, "77"), + Row(8, "88"), + Row(9, "99"), + Row(10, "000"), + Row(11, "111"), + Row(12, "222"), + Row(13, "333"), + Row(14, "444"), + Row(15, "555"), + Row(16, "666")) test("row set basic suites") { - val maxRows: Int = 5 - - val schema = new StructType().add("a", "int").add("b", "string") - - val rows = Seq( - Row(1, "11"), - Row(2, "22"), - Row(3, "33"), - Row(4, "44"), - Row(5, "55"), - Row(6, "66"), - Row(7, "77"), - Row(8, "88"), - Row(9, "99"), - Row(10, "000"), - Row(11, "111"), - Row(12, "222"), - Row(13, "333"), - Row(14, "444"), - Row(15, "555"), - Row(16, "666")) - // fetch next val rowIterator = rows.iterator var taken = rowIterator.take(maxRows).toSeq @@ -116,4 +114,30 @@ class RowBasedSetSuite extends SparkFunSuite { assert(tRowSet.getRows.get(3).getColVals.get(1).getStringVal.getValue === "44") assert(tRowSet.getRows.get(4).getColVals.get(1).getStringVal.getValue === "55") } + + test("kyuubi row set to TRowSet then to hive row set") { + val rowIterator = rows.iterator + val taken = rowIterator.take(maxRows).toSeq + val tRowSet = RowBasedSet(schema, taken).toTRowSet + + val hiveSet = new cli.RowBasedSet(tRowSet) + assert(hiveSet.numRows() === 5) + assert(hiveSet.numColumns() === 2) + val hiveRowIter = hiveSet.iterator() + val row1 = hiveRowIter.next().iterator + assert(row1.next().asInstanceOf[Int] === 1) + assert(row1.next().equals("11")) + val row2 = hiveRowIter.next().iterator + assert(row2.next().asInstanceOf[Int] === 2) + assert(row2.next().equals("22")) + val row3 = hiveRowIter.next().iterator + assert(row3.next().asInstanceOf[Int] === 3) + assert(row3.next().equals("33")) + val row4 = hiveRowIter.next().iterator + assert(row4.next().asInstanceOf[Int] === 4) + assert(row4.next().equals("44")) + val row5 = hiveRowIter.next().iterator + assert(row5.next().asInstanceOf[Int] === 5) + assert(row5.next().equals("55")) + } } diff --git a/src/test/scala/yaooqinn/kyuubi/schema/RowSetBuilderSuite.scala b/src/test/scala/yaooqinn/kyuubi/schema/RowSetBuilderSuite.scala new file mode 100644 index 000000000..b9da08e02 --- /dev/null +++ b/src/test/scala/yaooqinn/kyuubi/schema/RowSetBuilderSuite.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package yaooqinn.kyuubi.schema + +import org.apache.hive.service.cli.thrift.TProtocolVersion +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.StructType + +class RowSetBuilderSuite extends SparkFunSuite { + + test("row set builder create right RowSet") { + val schema = new StructType().add("a", "int").add("b", "string") + val rows = Seq(Row(1, "11"), Row(2, "22")) + assert(RowSetBuilder.create( + schema, rows, TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V1).isInstanceOf[RowBasedSet]) + assert(RowSetBuilder.create( + schema, rows, TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V2).isInstanceOf[RowBasedSet]) + assert(RowSetBuilder.create( + schema, rows, TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V3).isInstanceOf[RowBasedSet]) + assert(RowSetBuilder.create( + schema, rows, TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V4).isInstanceOf[RowBasedSet]) + assert(RowSetBuilder.create( + schema, rows, TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V5).isInstanceOf[RowBasedSet]) + assert(RowSetBuilder.create( + schema, rows, TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6).isInstanceOf[ColumnBasedSet]) + assert(RowSetBuilder.create( + schema, rows, TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V7).isInstanceOf[ColumnBasedSet]) + assert(RowSetBuilder.create( + schema, rows, TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8).isInstanceOf[ColumnBasedSet]) + } + +}