[KYUUBI #6118] Migrate to shaded HMS client for getting delegation token on server
# 🔍 Description ## Issue References 🔗 ## Describe Your Solution 🔧 Kyuubi Shaded 0.3 introduces a light kyuubi-relocated-hive-metastore-client, for refreshing delegation token, this PR aims to migrate from the vanilla HMS client to this light shaded HMS client, then we can get rid of Hive dependencies, especially the vulnerable thrift 0.9, from the Kyuubi server. ## Types of changes 🔖 - [ ] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan 🧪 Pass GA. --- # Checklist 📝 - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes #6118 from zhouyifan279/relocated-hms-client. Closes #6118 4d7e50915 [Cheng Pan] comment 845e39f87 [Cheng Pan] notice b4a58a614 [Cheng Pan] Update NOTICE f4bfa9310 [zhouyifan279] Use kyuubi-relocated-hive-metastore-client af17be1fe [zhouyifan279] Use kyuubi-relocated-hive-metastore-client Lead-authored-by: zhouyifan279 <zhouyifan279@gmail.com> Co-authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
zhouyifan279
Cheng Pan
parent
02bf4f02cf
commit
6fc7552ef0
@ -230,14 +230,6 @@ com.google.guava:failureaccess
|
||||
com.google.guava:guava
|
||||
org.apache.hadoop:hadoop-client-api
|
||||
org.apache.hadoop:hadoop-client-runtime
|
||||
org.apache.hive:hive-common
|
||||
org.apache.hive:hive-metastore
|
||||
org.apache.hive:hive-standalone-metastore
|
||||
org.apache.hive:hive-llap-client
|
||||
org.apache.hive:hive-serde
|
||||
org.apache.hive:hive-service-rpc
|
||||
org.apache.hive:hive-shims-0.23
|
||||
org.apache.hive:hive-shims-common
|
||||
com.google.j2objc:j2objc-annotations
|
||||
com.fasterxml.jackson.core:jackson-annotations
|
||||
com.fasterxml.jackson.core:jackson-core
|
||||
@ -270,8 +262,6 @@ org.eclipse.jetty:jetty-servlet
|
||||
org.eclipse.jetty:jetty-util-ajax
|
||||
org.eclipse.jetty:jetty-util
|
||||
org.eclipse.jetty:jetty-proxy
|
||||
org.apache.thrift:libfb303
|
||||
org.apache.thrift:libthrift
|
||||
org.apache.logging.log4j:log4j-1.2-api
|
||||
org.apache.logging.log4j:log4j-api
|
||||
org.apache.logging.log4j:log4j-core
|
||||
|
||||
@ -198,33 +198,12 @@ Copyright 2022 The Apache Software Foundation
|
||||
Hive JDBC
|
||||
Copyright 2022 The Apache Software Foundation
|
||||
|
||||
Hive Llap Client
|
||||
Copyright 2022 The Apache Software Foundation
|
||||
|
||||
Hive Metastore
|
||||
Copyright 2022 The Apache Software Foundation
|
||||
|
||||
Hive Serde
|
||||
Copyright 2022 The Apache Software Foundation
|
||||
|
||||
Hive Service
|
||||
Copyright 2022 The Apache Software Foundation
|
||||
|
||||
Hive Service RPC
|
||||
Copyright 2022 The Apache Software Foundation
|
||||
|
||||
Hive Shims 0.23
|
||||
Copyright 2022 The Apache Software Foundation
|
||||
|
||||
Hive Shims Common
|
||||
Copyright 2022 The Apache Software Foundation
|
||||
|
||||
Hive Standalone Metastore
|
||||
Copyright 2022 The Apache Software Foundation
|
||||
|
||||
Hive Storage API
|
||||
Copyright 2020 The Apache Software Foundation
|
||||
|
||||
Apache HttpClient
|
||||
Copyright 1999-2020 The Apache Software Foundation
|
||||
|
||||
@ -798,9 +777,6 @@ Permission to use, copy, modify and distribute UnixCrypt
|
||||
for non-commercial or commercial purposes and without fee is
|
||||
granted provided that the copyright notice appears in all copies.
|
||||
|
||||
Apache Thrift
|
||||
Copyright 2006-2010 The Apache Software Foundation.
|
||||
|
||||
Apache Log4j 1.x Compatibility API
|
||||
Copyright 1999-2022 The Apache Software Foundation
|
||||
|
||||
|
||||
@ -51,14 +51,6 @@ gson/2.10.1//gson-2.10.1.jar
|
||||
guava/32.0.1-jre//guava-32.0.1-jre.jar
|
||||
hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
|
||||
hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
|
||||
hive-common/3.1.3//hive-common-3.1.3.jar
|
||||
hive-metastore/3.1.3//hive-metastore-3.1.3.jar
|
||||
hive-serde/3.1.3//hive-serde-3.1.3.jar
|
||||
hive-service-rpc/3.1.3//hive-service-rpc-3.1.3.jar
|
||||
hive-shims-0.23/3.1.3//hive-shims-0.23-3.1.3.jar
|
||||
hive-shims-common/3.1.3//hive-shims-common-3.1.3.jar
|
||||
hive-standalone-metastore/3.1.3//hive-standalone-metastore-3.1.3.jar
|
||||
hive-storage-api/2.7.0//hive-storage-api-2.7.0.jar
|
||||
hk2-api/2.6.1//hk2-api-2.6.1.jar
|
||||
hk2-locator/2.6.1//hk2-locator-2.6.1.jar
|
||||
hk2-utils/2.6.1//hk2-utils-2.6.1.jar
|
||||
@ -133,8 +125,6 @@ kubernetes-model-rbac/6.8.1//kubernetes-model-rbac-6.8.1.jar
|
||||
kubernetes-model-resource/6.8.1//kubernetes-model-resource-6.8.1.jar
|
||||
kubernetes-model-scheduling/6.8.1//kubernetes-model-scheduling-6.8.1.jar
|
||||
kubernetes-model-storageclass/6.8.1//kubernetes-model-storageclass-6.8.1.jar
|
||||
libfb303/0.9.3//libfb303-0.9.3.jar
|
||||
libthrift/0.9.3//libthrift-0.9.3.jar
|
||||
log4j-1.2-api/2.20.0//log4j-1.2-api-2.20.0.jar
|
||||
log4j-api/2.20.0//log4j-api-2.20.0.jar
|
||||
log4j-core/2.20.0//log4j-core-2.20.0.jar
|
||||
|
||||
@ -78,6 +78,12 @@
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>kyuubi-relocated-hive-metastore-client</artifactId>
|
||||
<version>${kyuubi-relocated.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.antlr</groupId>
|
||||
<artifactId>antlr4-runtime</artifactId>
|
||||
@ -107,6 +113,7 @@
|
||||
<groupId>org.apache.hive</groupId>
|
||||
<artifactId>hive-metastore</artifactId>
|
||||
<version>${hive.version}</version>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>*</groupId>
|
||||
@ -119,6 +126,7 @@
|
||||
<groupId>org.apache.hive</groupId>
|
||||
<artifactId>hive-standalone-metastore</artifactId>
|
||||
<version>${hive.version}</version>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>*</groupId>
|
||||
@ -131,6 +139,7 @@
|
||||
<groupId>org.apache.hive</groupId>
|
||||
<artifactId>hive-serde</artifactId>
|
||||
<version>${hive.version}</version>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>*</groupId>
|
||||
@ -143,6 +152,7 @@
|
||||
<groupId>org.apache.hive.shims</groupId>
|
||||
<artifactId>hive-shims-common</artifactId>
|
||||
<version>${hive.version}</version>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>*</groupId>
|
||||
@ -155,6 +165,7 @@
|
||||
<groupId>org.apache.hive.shims</groupId>
|
||||
<artifactId>hive-shims-0.23</artifactId>
|
||||
<version>${hive.version}</version>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>*</groupId>
|
||||
@ -167,6 +178,7 @@
|
||||
<groupId>org.apache.hive</groupId>
|
||||
<artifactId>hive-common</artifactId>
|
||||
<version>${hive.version}</version>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>*</groupId>
|
||||
@ -179,6 +191,7 @@
|
||||
<groupId>org.apache.hive</groupId>
|
||||
<artifactId>hive-storage-api</artifactId>
|
||||
<version>${hive.storage-api.version}</version>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>*</groupId>
|
||||
@ -190,16 +203,19 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.thrift</groupId>
|
||||
<artifactId>libfb303</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.thrift</groupId>
|
||||
<artifactId>libthrift</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hive</groupId>
|
||||
<artifactId>hive-service-rpc</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
@ -15,4 +15,4 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier
|
||||
org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier
|
||||
|
||||
@ -18,9 +18,6 @@
|
||||
package org.apache.kyuubi.credentials
|
||||
|
||||
import org.apache.hadoop.conf.Configuration
|
||||
import org.apache.hadoop.hive.conf.HiveConf
|
||||
import org.apache.hadoop.hive.metastore.{IMetaStoreClient, RetryingMetaStoreClient}
|
||||
import org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier
|
||||
import org.apache.hadoop.io.Text
|
||||
import org.apache.hadoop.security.{Credentials, SecurityUtil}
|
||||
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod
|
||||
@ -28,6 +25,9 @@ import org.apache.hadoop.security.token.Token
|
||||
|
||||
import org.apache.kyuubi.Logging
|
||||
import org.apache.kyuubi.config.KyuubiConf
|
||||
import org.apache.kyuubi.shaded.hive.metastore.{IMetaStoreClient, RetryingMetaStoreClient}
|
||||
import org.apache.kyuubi.shaded.hive.metastore.conf.MetastoreConf
|
||||
import org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier
|
||||
|
||||
class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with Logging {
|
||||
|
||||
@ -38,7 +38,7 @@ class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with Log
|
||||
override def serviceName: String = "hive"
|
||||
|
||||
override def initialize(hadoopConf: Configuration, kyuubiConf: KyuubiConf): Unit = {
|
||||
val conf = new HiveConf(hadoopConf, classOf[HiveConf])
|
||||
val conf = MetastoreConf.newMetastoreConf(hadoopConf)
|
||||
val metastoreUris = conf.getTrimmed("hive.metastore.uris", "")
|
||||
// SQL engine requires token alias to be `hive.metastore.uris`
|
||||
tokenAlias = new Text(metastoreUris)
|
||||
@ -51,7 +51,7 @@ class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with Log
|
||||
principal = conf.getTrimmed(principalKey, "")
|
||||
require(principal.nonEmpty, s"Hive principal $principalKey undefined")
|
||||
|
||||
client = Some(RetryingMetaStoreClient.getProxy(conf, false))
|
||||
client = Some(RetryingMetaStoreClient.getProxy(conf))
|
||||
info(s"Created HiveMetaStoreClient with metastore uris $metastoreUris")
|
||||
}
|
||||
}
|
||||
|
||||
@ -26,8 +26,6 @@ import javax.ws.rs.core.NewCookie
|
||||
|
||||
import scala.collection.mutable
|
||||
|
||||
import org.apache.hadoop.hive.shims.Utils
|
||||
|
||||
import org.apache.kyuubi.Logging
|
||||
import org.apache.kyuubi.config.KyuubiConf
|
||||
import org.apache.kyuubi.config.KyuubiConf.FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER
|
||||
@ -57,6 +55,8 @@ class ThriftHttpServlet(
|
||||
private var isHttpOnlyCookie = false
|
||||
private val X_FORWARDED_FOR_HEADER = "X-Forwarded-For"
|
||||
private val authenticationFilter = new AuthenticationFilter(conf)
|
||||
private val XSRF_HEADER_DEFAULT = "X-XSRF-HEADER"
|
||||
private val XSRF_METHODS_TO_IGNORE_DEFAULT = Set("GET", "OPTIONS", "HEAD", "TRACE")
|
||||
|
||||
override def init(): Unit = {
|
||||
isCookieAuthEnabled = conf.get(KyuubiConf.FRONTEND_THRIFT_HTTP_COOKIE_AUTH_ENABLED)
|
||||
@ -82,7 +82,7 @@ class ThriftHttpServlet(
|
||||
var requireNewCookie: Boolean = false
|
||||
try {
|
||||
if (conf.get(KyuubiConf.FRONTEND_THRIFT_HTTP_XSRF_FILTER_ENABLED)) {
|
||||
val continueProcessing = Utils.doXsrfFilter(request, response, null, null)
|
||||
val continueProcessing = doXsrfFilter(request, response)
|
||||
if (!continueProcessing) {
|
||||
warn("Request did not have valid XSRF header, rejecting.")
|
||||
return
|
||||
@ -303,4 +303,22 @@ class ThriftHttpServlet(
|
||||
|
||||
null
|
||||
}
|
||||
|
||||
private def doXsrfFilter(
|
||||
httpRequest: HttpServletRequest,
|
||||
response: HttpServletResponse): Boolean = {
|
||||
if (XSRF_METHODS_TO_IGNORE_DEFAULT.contains(httpRequest.getMethod)
|
||||
|| httpRequest.getHeader(XSRF_HEADER_DEFAULT) != null) {
|
||||
true
|
||||
} else {
|
||||
response.sendError(
|
||||
HttpServletResponse.SC_BAD_REQUEST,
|
||||
"Missing Required Header for Vulnerability Protection")
|
||||
// scalastyle:off println
|
||||
response.getWriter.println(
|
||||
"XSRF filter denial, requests must contain header : " + XSRF_HEADER_DEFAULT)
|
||||
// scalastyle:on println
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -31,7 +31,7 @@ import org.apache.hadoop.conf.Configuration
|
||||
import org.apache.hadoop.hive.conf.HiveConf
|
||||
import org.apache.hadoop.hive.conf.HiveConf.ConfVars._
|
||||
import org.apache.hadoop.hive.metastore.{HiveMetaException, HiveMetaStore}
|
||||
import org.apache.hadoop.hive.metastore.security.{DelegationTokenIdentifier, HadoopThriftAuthBridge, HadoopThriftAuthBridge23}
|
||||
import org.apache.hadoop.hive.metastore.security.{HadoopThriftAuthBridge, HadoopThriftAuthBridge23}
|
||||
import org.apache.hadoop.io.Text
|
||||
import org.apache.hadoop.security.{Credentials, UserGroupInformation}
|
||||
import org.apache.hadoop.security.authorize.ProxyUsers
|
||||
@ -44,6 +44,7 @@ import org.scalatest.time.SpanSugar.convertIntToGrainOfTime
|
||||
import org.apache.kyuubi.{KerberizedTestHelper, Logging, Utils}
|
||||
import org.apache.kyuubi.config.KyuubiConf
|
||||
import org.apache.kyuubi.credentials.LocalMetaServer.defaultHiveConf
|
||||
import org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier
|
||||
|
||||
class HiveDelegationTokenProviderSuite extends KerberizedTestHelper {
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user