From 92c2ebcbf36c8801a002a3d006b997e6179e2f00 Mon Sep 17 00:00:00 2001 From: hongdongdong Date: Mon, 30 Aug 2021 16:55:03 +0800 Subject: [PATCH] [KYUUBI #1000] Use underscore instead hyphen in path Use underscore instead hyphen in path, friendly to sql. - [X] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1000 from hddong/fix-underscore. Closes #1000 8b3459a5 [hongdongdong] Use underscore instead hyphen in path Authored-by: hongdongdong Signed-off-by: ulysses-you (cherry picked from commit 94d60ea505800bf61954f8d69ff772f569a83700) Signed-off-by: ulysses-you --- .../apache/kyuubi/events/KyuubiEvent.scala | 32 ++++++ .../events/EventLoggingServiceSuite.scala | 108 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 kyuubi-common/src/main/scala/org/apache/kyuubi/events/KyuubiEvent.scala create mode 100644 kyuubi-server/src/test/scala/org/apache/kyuubi/events/EventLoggingServiceSuite.scala diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/events/KyuubiEvent.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/events/KyuubiEvent.scala new file mode 100644 index 000000000..24aa75b20 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/events/KyuubiEvent.scala @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.events + +import java.util.Locale + +trait KyuubiEvent extends Product { + final lazy val eventType: String = { + this.getClass.getSimpleName.stripSuffix("Event") + .replaceAll("(.)([A-Z])", "$1_$2") + .toLowerCase(Locale.ROOT) + } + + def partitions: Seq[(String, String)] + + final def toJson: String = JsonProtocol.productToJson(this) +} diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/EventLoggingServiceSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/EventLoggingServiceSuite.scala new file mode 100644 index 000000000..c38e5e533 --- /dev/null +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/EventLoggingServiceSuite.scala @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.events + +import java.net.InetAddress +import java.nio.file.Paths + +import org.apache.kyuubi.{Utils, WithKyuubiServer} +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.operation.JDBCTestUtils +import org.apache.kyuubi.operation.OperationState._ + +class EventLoggingServiceSuite extends WithKyuubiServer with JDBCTestUtils { + + private val logRoot = Utils.createTempDir() + private val currentDate = Utils.getDateFromTimestamp(System.currentTimeMillis()) + + override protected val conf: KyuubiConf = { + KyuubiConf() + .set(KyuubiConf.SERVER_EVENT_LOGGERS, Seq("JSON")) + .set(KyuubiConf.SERVER_EVENT_JSON_LOG_PATH, logRoot.toString) + .set(KyuubiConf.ENGINE_EVENT_LOGGERS, Seq("JSON")) + .set(KyuubiConf.ENGINE_EVENT_JSON_LOG_PATH, logRoot.toString) + } + + override protected def jdbcUrl: String = getJdbcUrl + + test("statementEvent: generate, dump and query") { + val hostName = InetAddress.getLocalHost.getCanonicalHostName + val serverStatementEventPath = + Paths.get(logRoot.toString, "kyuubi_statement", s"day=$currentDate", s"server-$hostName.json") + val engineStatementEventPath = + Paths.get(logRoot.toString, "spark_statement", s"day=$currentDate", "*.json") + val sql = "select timestamp'2021-06-01'" + + withJdbcStatement() { statement => + statement.execute(sql) + + // check server statement events + val serverTable = serverStatementEventPath.getParent + val resultSet = statement.executeQuery(s"SELECT * FROM `json`.`${serverTable}`" + + "where statement = \"" + sql + "\"") + val states = Array(INITIALIZED, PENDING, RUNNING, FINISHED, CLOSED) + var stateIndex = 0 + while (resultSet.next()) { + assert(resultSet.getString("user") == Utils.currentUser) + assert(resultSet.getString("statement") == sql) + assert(resultSet.getString("state") == states(stateIndex).toString) + stateIndex += 1 + } + + // check engine statement events + val engineTable = engineStatementEventPath.getParent + val resultSet2 = statement.executeQuery(s"SELECT * FROM `json`.`${engineTable}`" + + "where statement = \"" + sql + "\"") + val engineStates = Array(INITIALIZED, PENDING, RUNNING, COMPILED, FINISHED) + stateIndex = 0 + while (resultSet2.next()) { + assert(resultSet2.getString("Event") == + "org.apache.kyuubi.engine.spark.events.SparkStatementEvent") + assert(resultSet2.getString("statement") == sql) + assert(resultSet2.getString("state") == engineStates(stateIndex).toString) + stateIndex += 1 + } + } + } + + test("test Kyuubi session event") { + withSessionConf()(Map.empty)(Map(KyuubiConf.SESSION_NAME.key -> "test1")) { + withJdbcStatement() { statement => + statement.execute("SELECT 1") + } + } + + val eventPath = + Paths.get(logRoot.toString, "kyuubi-session", s"day=$currentDate") + withSessionConf()(Map.empty)(Map("spark.sql.shuffle.partitions" -> "2")) { + withJdbcStatement() { statement => + val res = statement.executeQuery( + s"SELECT * FROM `json`.`$eventPath` where sessionName = 'test1' order by totalOperations") + assert(res.next()) + assert(res.getString("user") == Utils.currentUser) + assert(res.getString("sessionName") == "test1") + assert(res.getLong("startTime") > 0) + assert(res.getInt("totalOperations") == 0) + assert(res.next()) + assert(res.getInt("totalOperations") == 1) + assert(res.getLong("endTime") > 0) + assert(!res.next()) + } + } + } +}