apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #2983] Remove Hive llap-client dependencies from Kyuubi Hive JDBC

Browse Source 
### _Why are the changes needed?_

fix #2983

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #2986 from jiaoqingbo/kyuubi-2983.

Closes #2983

9d068edf [jiaoqingbo] [KYUUBI #2983] Remove Hive llap-client dependencies from Kyuubi Hive JDBC

Authored-by: jiaoqingbo <1178404354@qq.com>
Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
jiaoqingbo 2022-07-01 18:44:55 +08:00 committed by Cheng Pan
parent 3a80f33bf1
commit e41ef56611
No known key found for this signature in database
GPG Key ID: 8001952629BCC75D
11 changed files with 266 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
kyuubi-hive-jdbc/pom.xml
View File

@ -100,19 +100,6 @@
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-llap-client</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>

34
kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/HS2ActivePassiveHARegistry.java
View File

@ -19,7 +19,6 @@ package org.apache.kyuubi.jdbc.hive.server;
import com.google.common.base.Preconditions;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
@ -28,9 +27,6 @@ import org.apache.curator.framework.recipes.leader.LeaderLatch;
import org.apache.curator.utils.CloseableUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.llap.registry.ServiceRegistry;
import org.apache.hadoop.hive.registry.ServiceInstanceSet;
import org.apache.hadoop.hive.registry.ServiceInstanceStateChangeListener;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -40,13 +36,11 @@ public class HS2ActivePassiveHARegistry extends ZkRegistryBase<HiveServer2Instan
private static final Logger LOG = LoggerFactory.getLogger(HS2ActivePassiveHARegistry.class);
static final String ACTIVE_ENDPOINT = "activeEndpoint";
static final String PASSIVE_ENDPOINT = "passiveEndpoint";
private static final String SASL_LOGIN_CONTEXT_NAME = "HS2ActivePassiveHAZooKeeperClient";
private static final String INSTANCE_PREFIX = "instance-";
private static final String INSTANCE_GROUP = "instances";
private static final String LEADER_LATCH_PATH = "/_LEADER";
private LeaderLatch leaderLatch;
private String latchPath;
private ServiceRecord srv;
private final String uniqueId;
// There are 2 paths under which the instances get registered
@ -110,23 +104,6 @@ public class HS2ActivePassiveHARegistry extends ZkRegistryBase<HiveServer2Instan
LOG.info("Populating instances cache for client");
}
@Override
protected void unregisterInternal() {
super.unregisterInternal();
}
@Override
public String register() throws IOException {
updateEndpoint(srv, PASSIVE_ENDPOINT);
return registerServiceRecord(srv, uniqueId);
}
@Override
public void unregister() {
CloseableUtils.closeQuietly(leaderLatch);
unregisterInternal();
}
private void populateCache() throws IOException {
PathChildrenCache pcc = ensureInstancesCache(0);
populateCache(pcc, false);
@ -138,17 +115,6 @@ public class HS2ActivePassiveHARegistry extends ZkRegistryBase<HiveServer2Instan
throw new IOException("Not supported to get instances by component name");
}
private void updateEndpoint(final ServiceRecord srv, final String endpointName) {
final String instanceUri = srv.get(HiveServer2.INSTANCE_URI_CONFIG);
final String[] tokens = instanceUri.split(":");
final String hostname = tokens[0];
final int port = Integer.parseInt(tokens[1]);
Endpoint urlEndpoint =
RegistryTypeUtils.ipcEndpoint(endpointName, new InetSocketAddress(hostname, port));
srv.addInternalEndpoint(urlEndpoint);
LOG.info("Added {} endpoint to service record", urlEndpoint);
}
@Override
public void stop() {
CloseableUtils.closeQuietly(leaderLatch);

1
kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/HS2ActivePassiveHARegistryClient.java
View File

@ -24,7 +24,6 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.registry.impl.ZkRegistryBase;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

2
kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/HiveServer2HAInstanceSet.java
View File

@ -17,8 +17,6 @@
package org.apache.kyuubi.jdbc.hive.server;
import org.apache.hadoop.hive.registry.ServiceInstanceSet;
public interface HiveServer2HAInstanceSet extends ServiceInstanceSet<HiveServer2Instance> {
/**

56
kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/RegistryUtilities.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.jdbc.hive.server;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.UUID;
public class RegistryUtilities {
private static final String LOCALHOST = "localhost";
/**
* Will return hostname stored in InetAddress.
*
* @return hostname
*/
public static String getHostName() {
try {
return InetAddress.getLocalHost().getHostName();
} catch (UnknownHostException e) {
return LOCALHOST;
}
}
/**
* Will return FQDN of the host after doing reverse DNS lookip.
*
* @return FQDN of host
*/
public static String getCanonicalHostName() {
try {
return InetAddress.getLocalHost().getCanonicalHostName();
} catch (UnknownHostException e) {
return LOCALHOST;
}
}
public static String getUUID() {
return String.valueOf(UUID.randomUUID());
}
}

50
kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstance.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.jdbc.hive.server;
import java.util.Map;
public interface ServiceInstance {
/**
* Worker identity is a UUID (unique across restarts), to identify a node which died &amp; was
* brought back on the same host/port
*/
String getWorkerIdentity();
/**
* Hostname of the service instance
*
* @return service hostname
*/
String getHost();
/**
* RPC Endpoint for service instance
*
* @return rpc port
*/
int getRpcPort();
/**
* Config properties of the Service Instance (llap.daemon.*)
*
* @return properties
*/
Map<String, String> getProperties();
}

1
kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceBase.java
View File

@ -20,7 +20,6 @@ package org.apache.kyuubi.jdbc.hive.server;
import java.io.IOException;
import java.util.Map;
import java.util.Objects;
import org.apache.hadoop.hive.registry.ServiceInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

64
kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceSet.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.jdbc.hive.server;
import java.util.Collection;
import java.util.Set;
/**
* Note: For most of the implementations, there's no guarantee that the ServiceInstance returned by
* one invocation is the same as the instance returned by another invocation. e.g. the ZK registry
* returns a new ServiceInstance object each time a getInstance call is made.
*/
public interface ServiceInstanceSet<InstanceType extends ServiceInstance> {
/**
* Get an instance mapping which map worker identity to each instance.
*
* <p>The worker identity does not collide between restarts, so each restart will have a unique
* id, while having the same host/ip pair.
*
* @return instance list
*/
Collection<InstanceType> getAll();
/**
* Get an instance by worker identity.
*
* @param name worker id
* @return instance
*/
InstanceType getInstance(String name);
/**
* Get a list of service instances for a given host.
*
* <p>The list could include dead and alive instances.
*
* @param host hostname
* @return instance list
*/
Set<InstanceType> getByHost(String host);
/**
* Get number of instances in the currently available.
*
* @return - number of instances
*/
int size();
}

42
kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceStateChangeListener.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.jdbc.hive.server;
/** Callback listener for instance state change events */
public interface ServiceInstanceStateChangeListener<InstanceType extends ServiceInstance> {
/**
* Called when new {@link ServiceInstance} is created.
*
* @param serviceInstance - created service instance
*/
void onCreate(InstanceType serviceInstance, int ephSeqVersion);
/**
* Called when an existing {@link ServiceInstance} is updated.
*
* @param serviceInstance - updated service instance
*/
void onUpdate(InstanceType serviceInstance, int ephSeqVersion);
/**
* Called when an existing {@link ServiceInstance} is removed.
*
* @param serviceInstance - removed service instance
*/
void onRemove(InstanceType serviceInstance, int ephSeqVersion);
}

54
kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceRegistry.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.jdbc.hive.server;
import java.io.IOException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
/**
* ServiceRegistry interface for switching between fixed host and dynamic registry implementations.
*/
public interface ServiceRegistry<T extends ServiceInstance> {
/** Start the service registry */
void start() throws IOException;
/** Stop the service registry */
void stop() throws IOException;
/**
* Client API to get the list of instances registered via the current registry key.
*
* @param component
* @param clusterReadyTimeoutMs The time to wait for the cluster to be ready, if it's not started
* yet. 0 means do not wait.
*/
ServiceInstanceSet<T> getInstances(String component, long clusterReadyTimeoutMs)
throws IOException;
/**
* Adds state change listeners for service instances.
*
* @param listener - state change listener
*/
void registerStateChangeListener(ServiceInstanceStateChangeListener<T> listener)
throws IOException;
/** @return The application ID of the LLAP cluster. */
ApplicationId getApplicationId() throws IOException;
}

7
kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ZkRegistryBase.java
View File

@ -47,9 +47,6 @@ import org.apache.curator.utils.CloseableUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.registry.RegistryUtilities;
import org.apache.hadoop.hive.registry.ServiceInstance;
import org.apache.hadoop.hive.registry.ServiceInstanceStateChangeListener;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.kyuubi.jdbc.hive.server.RegistryUtils.ServiceRecordMarshal;
@ -665,10 +662,6 @@ public abstract class ZkRegistryBase<InstanceType extends ServiceInstance> {
}
}
protected void unregisterInternal() {
CloseableUtils.closeQuietly(znode);
}
public void stop() {
CloseableUtils.closeQuietly(znode);
CloseableUtils.closeQuietly(instancesCache);