diff --git a/.travis.yml b/.travis.yml index 74c740326..d65582b21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,22 +30,10 @@ deploy: jobs: include: - - stage: spark2.1 - language: scala - script: - - ./build/mvn clean install -Pspark-2.1 -Dmaven.javadoc.skip=true -B -V - - stage: spark2.2 - language: scala - script: - - ./build/mvn clean install -Pspark-2.2 -Dmaven.javadoc.skip=true -B -V - - stage: spark2.3 - language: scala - script: - - ./build/mvn clean install -Pspark-2.3 -Dmaven.javadoc.skip=true -B -V - stage: spark2.4 language: scala script: - - ./build/mvn clean install -Pspark-2.4 -Dmaven.javadoc.skip=true -B -V + - ./build/mvn clean install -Dmaven.javadoc.skip=true -B -V after_success: - bash <(curl -s https://codecov.io/bash) diff --git a/kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiSessionPage.scala b/kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiSessionPage.scala index a5790b232..b4ac5071f 100644 --- a/kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiSessionPage.scala +++ b/kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiSessionPage.scala @@ -49,7 +49,7 @@ class KyuubiSessionPage(parent: KyuubiSessionTab) extends WebUIPage("") { generateSessionStatsTable(request) ++ generateSQLStatsTable(request) } - KyuubiUIUtils.headerSparkPage(request, "Kyuubi Session - Application View", content, parent) + UIUtils.headerSparkPage(request, "Kyuubi Session - Application View", content, parent) } /** Generate basic stats of the kyuubi server program */ @@ -76,7 +76,7 @@ class KyuubiSessionPage(parent: KyuubiSessionTab) extends WebUIPage("") { def generateDataRow(info: ExecutionInfo): Seq[Node] = { val jobLink = info.jobId.map { id: String => + UIUtils.prependBaseUri(request, parent.basePath), id)}> [{id}] } @@ -147,7 +147,7 @@ class KyuubiSessionPage(parent: KyuubiSessionTab) extends WebUIPage("") { "Total Execute") def generateDataRow(session: SessionInfo): Seq[Node] = { val sessionLink = "%s/%s/session?id=%s".format( - KyuubiUIUtils.prependBaseUri(request, parent.basePath), parent.prefix, session.sessionId) + UIUtils.prependBaseUri(request, parent.basePath), parent.prefix, session.sessionId) {session.userName} {session.ip} diff --git a/kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiSessionSubPage.scala b/kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiSessionSubPage.scala index 50fbb9e46..e6418ccfe 100644 --- a/kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiSessionSubPage.scala +++ b/kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiSessionSubPage.scala @@ -54,7 +54,7 @@ class KyuubiSessionSubPage(parent: KyuubiSessionTab) extends WebUIPage("session" ++ generateSQLStatsTable(request, sessionStat.sessionId) } - KyuubiUIUtils.headerSparkPage(request, "Kyuubi Session", content, parent) + UIUtils.headerSparkPage(request, "Kyuubi Session", content, parent) } /** Generate basic stats of the kyuubi server program */ @@ -83,7 +83,7 @@ class KyuubiSessionSubPage(parent: KyuubiSessionTab) extends WebUIPage("session" def generateDataRow(info: ExecutionInfo): Seq[Node] = { val jobLink = info.jobId.map { id: String => + .format(UIUtils.prependBaseUri(request, parent.basePath), id)}> [{id}] } diff --git a/kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiUIUtils.scala b/kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiUIUtils.scala deleted file mode 100644 index e9e840744..000000000 --- a/kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiUIUtils.scala +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.ui - -import javax.servlet.http.HttpServletRequest - -import scala.xml.Node - -import org.apache.spark.KyuubiSparkUtil._ - -import yaooqinn.kyuubi.utils.ReflectUtils - -object KyuubiUIUtils { - - private val className = "org.apache.spark.ui.UIUtils" - - /** Returns a spark page with correctly formatted headers */ - def headerSparkPage( - request: HttpServletRequest, - title: String, - content: => Seq[Node], - activeTab: SparkUITab): Seq[Node] = { - val methodMirror = ReflectUtils.reflectStaticMethodScala(className, "headerSparkPage") - if (equalOrHigherThan("2.4")) { - methodMirror(request, title, content, activeTab, Some(5000), None, false, false) - .asInstanceOf[Seq[Node]] - } else { - methodMirror(title, content, activeTab, Some(5000), None, false, false) - .asInstanceOf[Seq[Node]] - } - } - - def prependBaseUri( - request: HttpServletRequest, - basePath: String = "", - resource: String = ""): String = { - val method = ReflectUtils.reflectStaticMethodScala(className, "prependBaseUri") - if (equalOrHigherThan("2.4")) { - method(request, basePath, resource).asInstanceOf[String] - } else { - method(basePath, resource).asInstanceOf[String] - } - } -} diff --git a/kyuubi-server/src/main/scala/yaooqinn/kyuubi/session/security/HiveTokenCollector.scala b/kyuubi-server/src/main/scala/yaooqinn/kyuubi/session/security/HiveTokenCollector.scala index 79fd30258..e1bcbb521 100644 --- a/kyuubi-server/src/main/scala/yaooqinn/kyuubi/session/security/HiveTokenCollector.scala +++ b/kyuubi-server/src/main/scala/yaooqinn/kyuubi/session/security/HiveTokenCollector.scala @@ -21,7 +21,6 @@ import scala.util.control.NonFatal import org.apache.commons.lang3.StringUtils import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier -import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.ql.metadata.Hive import org.apache.hadoop.io.Text import org.apache.hadoop.security.{Credentials, UserGroupInformation} diff --git a/kyuubi-server/src/main/scala/yaooqinn/kyuubi/ui/KyuubiServerMonitor.scala b/kyuubi-server/src/main/scala/yaooqinn/kyuubi/ui/KyuubiServerMonitor.scala index 06a189350..c8e56e506 100644 --- a/kyuubi-server/src/main/scala/yaooqinn/kyuubi/ui/KyuubiServerMonitor.scala +++ b/kyuubi-server/src/main/scala/yaooqinn/kyuubi/ui/KyuubiServerMonitor.scala @@ -19,7 +19,6 @@ package yaooqinn.kyuubi.ui import scala.collection.mutable.HashMap -import org.apache.spark.SparkException import org.apache.spark.ui.KyuubiSessionTab diff --git a/kyuubi-server/src/test/scala/org/apache/spark/ui/KyuubiUIUtilsSuite.scala b/kyuubi-server/src/test/scala/org/apache/spark/ui/KyuubiUIUtilsSuite.scala deleted file mode 100644 index cc8c9684a..000000000 --- a/kyuubi-server/src/test/scala/org/apache/spark/ui/KyuubiUIUtilsSuite.scala +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.ui - -import javax.servlet.http.HttpServletRequest - -import scala.util.Try - -import org.apache.spark.{KyuubiSparkUtil, SparkConf, SparkContext, SparkFunSuite} -import org.scalatest.mock.MockitoSugar - -class KyuubiUIUtilsSuite extends SparkFunSuite with MockitoSugar { - - var sc: SparkContext = _ - var user: String = _ - var tab: KyuubiSessionTab = _ - - override def beforeAll(): Unit = { - val conf = new SparkConf(loadDefaults = true).setMaster("local").setAppName("test") - KyuubiSparkUtil.setupCommonConfig(conf) - sc = new SparkContext(conf) - user = KyuubiSparkUtil.getCurrentUserName - tab = new KyuubiSessionTab(user, sc) - } - - override def afterAll(): Unit = { - sc.stop() - } - - test("spark page header") { - val request = mock[HttpServletRequest] - val title = "KyuubiServer test ui" * 10 - val content = - assert(Try { KyuubiUIUtils.headerSparkPage(request, title, content, tab) }.isSuccess) - } - - test("prepend base uri") { - val request = mock[HttpServletRequest] - val baseUri = KyuubiUIUtils.prependBaseUri(request) - - val basePath = KyuubiUIUtils.prependBaseUri(request, "1") - assert(basePath === baseUri + "1") - - val resourcePath = KyuubiUIUtils.prependBaseUri(request, "1", "2") - assert(resourcePath === baseUri + "12") - } - -} diff --git a/pom.xml b/pom.xml index 8edb654b3..7b3e9dd04 100644 --- a/pom.xml +++ b/pom.xml @@ -53,11 +53,11 @@ UTF-8 1.7 2.11.8 - 2.2.6 + 3.0.3 2.11 - 3.3.9 + 3.5.4 org.apache.spark - 2.1.3 + 2.4.5 provided 2.6.5 provided @@ -74,7 +74,6 @@ central - Maven Repository https://repo.maven.apache.org/maven2 @@ -504,40 +503,10 @@ - - spark-2.1 - - 2.1.3 - - - - - spark-2.2 - - 2.2.2 - - - - - spark-2.3 - - 2.3.2 - 3.0.3 - - - - - spark-2.4 - - 2.4.0 - 3.0.3 - - - hadoop-2.7 - 2.7.3 + 2.7.4 2.7.1