apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI-233] Make master branch target apache spark 2.4 #233 (#234)

Browse Source
This commit is contained in:
Kent Yao 2020-05-18 11:54:20 +08:00 committed by GitHub
parent d16d5b335c
commit 17c3c054b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 10 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.travis.yml
View File

@ -30,22 +30,10 @@ deploy:
jobs:
include:
- stage: spark2.1
language: scala
script:
- ./build/mvn clean install -Pspark-2.1 -Dmaven.javadoc.skip=true -B -V
- stage: spark2.2
language: scala
script:
- ./build/mvn clean install -Pspark-2.2 -Dmaven.javadoc.skip=true -B -V
- stage: spark2.3
language: scala
script:
- ./build/mvn clean install -Pspark-2.3 -Dmaven.javadoc.skip=true -B -V
- stage: spark2.4
language: scala
script:
- ./build/mvn clean install -Pspark-2.4 -Dmaven.javadoc.skip=true -B -V
- ./build/mvn clean install -Dmaven.javadoc.skip=true -B -V
after_success:
- bash <(curl -s https://codecov.io/bash)

6
kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiSessionPage.scala
View File

@ -49,7 +49,7 @@ class KyuubiSessionPage(parent: KyuubiSessionTab) extends WebUIPage("") {
generateSessionStatsTable(request) ++
generateSQLStatsTable(request)
}
KyuubiUIUtils.headerSparkPage(request, "Kyuubi Session - Application View", content, parent)
UIUtils.headerSparkPage(request, "Kyuubi Session - Application View", content, parent)
}
/** Generate basic stats of the kyuubi server program */
@ -76,7 +76,7 @@ class KyuubiSessionPage(parent: KyuubiSessionTab) extends WebUIPage("") {
def generateDataRow(info: ExecutionInfo): Seq[Node] = {
val jobLink = info.jobId.map { id: String =>
<a href={"%s/jobs/job?id=%s".format(
KyuubiUIUtils.prependBaseUri(request, parent.basePath), id)}>
UIUtils.prependBaseUri(request, parent.basePath), id)}>
[{id}]
</a>
}
@ -147,7 +147,7 @@ class KyuubiSessionPage(parent: KyuubiSessionTab) extends WebUIPage("") {
"Total Execute")
def generateDataRow(session: SessionInfo): Seq[Node] = {
val sessionLink = "%s/%s/session?id=%s".format(
KyuubiUIUtils.prependBaseUri(request, parent.basePath), parent.prefix, session.sessionId)
UIUtils.prependBaseUri(request, parent.basePath), parent.prefix, session.sessionId)
<tr>
<td> {session.userName} </td>
<td> {session.ip} </td>

4
kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiSessionSubPage.scala
View File

@ -54,7 +54,7 @@ class KyuubiSessionSubPage(parent: KyuubiSessionTab) extends WebUIPage("session"
</h4> ++
generateSQLStatsTable(request, sessionStat.sessionId)
}
KyuubiUIUtils.headerSparkPage(request, "Kyuubi Session", content, parent)
UIUtils.headerSparkPage(request, "Kyuubi Session", content, parent)
}
/** Generate basic stats of the kyuubi server program */
@ -83,7 +83,7 @@ class KyuubiSessionSubPage(parent: KyuubiSessionTab) extends WebUIPage("session"
def generateDataRow(info: ExecutionInfo): Seq[Node] = {
val jobLink = info.jobId.map { id: String =>
<a href={"%s/jobs/job?id=%s"
.format(KyuubiUIUtils.prependBaseUri(request, parent.basePath), id)}>
.format(UIUtils.prependBaseUri(request, parent.basePath), id)}>
[{id}]
</a>
}

59
kyuubi-server/src/main/scala/org/apache/spark/ui/KyuubiUIUtils.scala
View File

@ -1,59 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.ui
import javax.servlet.http.HttpServletRequest
import scala.xml.Node
import org.apache.spark.KyuubiSparkUtil._
import yaooqinn.kyuubi.utils.ReflectUtils
object KyuubiUIUtils {
private val className = "org.apache.spark.ui.UIUtils"
/** Returns a spark page with correctly formatted headers */
def headerSparkPage(
request: HttpServletRequest,
title: String,
content: => Seq[Node],
activeTab: SparkUITab): Seq[Node] = {
val methodMirror = ReflectUtils.reflectStaticMethodScala(className, "headerSparkPage")
if (equalOrHigherThan("2.4")) {
methodMirror(request, title, content, activeTab, Some(5000), None, false, false)
.asInstanceOf[Seq[Node]]
} else {
methodMirror(title, content, activeTab, Some(5000), None, false, false)
.asInstanceOf[Seq[Node]]
}
}
def prependBaseUri(
request: HttpServletRequest,
basePath: String = "",
resource: String = ""): String = {
val method = ReflectUtils.reflectStaticMethodScala(className, "prependBaseUri")
if (equalOrHigherThan("2.4")) {
method(request, basePath, resource).asInstanceOf[String]
} else {
method(basePath, resource).asInstanceOf[String]
}
}
}

1
kyuubi-server/src/main/scala/yaooqinn/kyuubi/session/security/HiveTokenCollector.scala
View File

@ -21,7 +21,6 @@ import scala.util.control.NonFatal
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.ql.metadata.Hive
import org.apache.hadoop.io.Text
import org.apache.hadoop.security.{Credentials, UserGroupInformation}

1
kyuubi-server/src/main/scala/yaooqinn/kyuubi/ui/KyuubiServerMonitor.scala
View File

@ -19,7 +19,6 @@ package yaooqinn.kyuubi.ui
import scala.collection.mutable.HashMap
import org.apache.spark.SparkException
import org.apache.spark.ui.KyuubiSessionTab

63
kyuubi-server/src/test/scala/org/apache/spark/ui/KyuubiUIUtilsSuite.scala
View File

@ -1,63 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.ui
import javax.servlet.http.HttpServletRequest
import scala.util.Try
import org.apache.spark.{KyuubiSparkUtil, SparkConf, SparkContext, SparkFunSuite}
import org.scalatest.mock.MockitoSugar
class KyuubiUIUtilsSuite extends SparkFunSuite with MockitoSugar {
var sc: SparkContext = _
var user: String = _
var tab: KyuubiSessionTab = _
override def beforeAll(): Unit = {
val conf = new SparkConf(loadDefaults = true).setMaster("local").setAppName("test")
KyuubiSparkUtil.setupCommonConfig(conf)
sc = new SparkContext(conf)
user = KyuubiSparkUtil.getCurrentUserName
tab = new KyuubiSessionTab(user, sc)
}
override def afterAll(): Unit = {
sc.stop()
}
test("spark page header") {
val request = mock[HttpServletRequest]
val title = "KyuubiServer test ui" * 10
val content = <td></td>
assert(Try { KyuubiUIUtils.headerSparkPage(request, title, content, tab) }.isSuccess)
}
test("prepend base uri") {
val request = mock[HttpServletRequest]
val baseUri = KyuubiUIUtils.prependBaseUri(request)
val basePath = KyuubiUIUtils.prependBaseUri(request, "1")
assert(basePath === baseUri + "1")
val resourcePath = KyuubiUIUtils.prependBaseUri(request, "1", "2")
assert(resourcePath === baseUri + "12")
}
}

39
pom.xml
View File

@ -53,11 +53,11 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>1.7</java.version>
<scala.version>2.11.8</scala.version>
<scalatest.version>2.2.6</scalatest.version>
<scalatest.version>3.0.3</scalatest.version>
<scala.binary.version>2.11</scala.binary.version>
<maven.version>3.3.9</maven.version>
<maven.version>3.5.4</maven.version>
<spark.group>org.apache.spark</spark.group>
<spark.version>2.1.3</spark.version>
<spark.version>2.4.5</spark.version>
<spark.scope>provided</spark.scope>
<hadoop.version>2.6.5</hadoop.version>
<hadoop.deps.scope>provided</hadoop.deps.scope>
@ -74,7 +74,6 @@
<repositories>
<repository>
<id>central</id>
<!-- This should be at top, it makes maven try the central repo first and then others and hence faster dep resolution -->
<name>Maven Repository</name>
<url>https://repo.maven.apache.org/maven2</url>
<releases>
@ -504,40 +503,10 @@
</build>
<profiles>
<profile>
<id>spark-2.1</id>
<properties>
<spark.version>2.1.3</spark.version>
</properties>
</profile>
<profile>
<id>spark-2.2</id>
<properties>
<spark.version>2.2.2</spark.version>
</properties>
</profile>
<profile>
<id>spark-2.3</id>
<properties>
<spark.version>2.3.2</spark.version>
<scalatest.version>3.0.3</scalatest.version>
</properties>
</profile>
<profile>
<id>spark-2.4</id>
<properties>
<spark.version>2.4.0</spark.version>
<scalatest.version>3.0.3</scalatest.version>
</properties>
</profile>
<profile>
<id>hadoop-2.7</id>
<properties>
<hadoop.version>2.7.3</hadoop.version>
<hadoop.version>2.7.4</hadoop.version>
<curator.version>2.7.1</curator.version>
</properties>
</profile>