diff --git a/kyuubi-hive-jdbc/pom.xml b/kyuubi-hive-jdbc/pom.xml index f60183903..398b3f509 100644 --- a/kyuubi-hive-jdbc/pom.xml +++ b/kyuubi-hive-jdbc/pom.xml @@ -100,19 +100,6 @@ - - org.apache.hive - hive-llap-client - ${hive.version} - - - * - * - - - - - commons-codec commons-codec diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/HS2ActivePassiveHARegistry.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/HS2ActivePassiveHARegistry.java index c569231bd..4d4649932 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/HS2ActivePassiveHARegistry.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/HS2ActivePassiveHARegistry.java @@ -19,7 +19,6 @@ package org.apache.kyuubi.jdbc.hive.server; import com.google.common.base.Preconditions; import java.io.IOException; -import java.net.InetSocketAddress; import java.util.Collection; import java.util.Set; import org.apache.commons.lang3.StringUtils; @@ -28,9 +27,6 @@ import org.apache.curator.framework.recipes.leader.LeaderLatch; import org.apache.curator.utils.CloseableUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.llap.registry.ServiceRegistry; -import org.apache.hadoop.hive.registry.ServiceInstanceSet; -import org.apache.hadoop.hive.registry.ServiceInstanceStateChangeListener; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,13 +36,11 @@ public class HS2ActivePassiveHARegistry extends ZkRegistryBase { /** diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/RegistryUtilities.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/RegistryUtilities.java new file mode 100644 index 000000000..f8a56e99e --- /dev/null +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/RegistryUtilities.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.jdbc.hive.server; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.UUID; + +public class RegistryUtilities { + private static final String LOCALHOST = "localhost"; + + /** + * Will return hostname stored in InetAddress. + * + * @return hostname + */ + public static String getHostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + return LOCALHOST; + } + } + + /** + * Will return FQDN of the host after doing reverse DNS lookip. + * + * @return FQDN of host + */ + public static String getCanonicalHostName() { + try { + return InetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + return LOCALHOST; + } + } + + public static String getUUID() { + return String.valueOf(UUID.randomUUID()); + } +} diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstance.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstance.java new file mode 100644 index 000000000..aa5a20509 --- /dev/null +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstance.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.jdbc.hive.server; + +import java.util.Map; + +public interface ServiceInstance { + + /** + * Worker identity is a UUID (unique across restarts), to identify a node which died & was + * brought back on the same host/port + */ + String getWorkerIdentity(); + + /** + * Hostname of the service instance + * + * @return service hostname + */ + String getHost(); + + /** + * RPC Endpoint for service instance + * + * @return rpc port + */ + int getRpcPort(); + + /** + * Config properties of the Service Instance (llap.daemon.*) + * + * @return properties + */ + Map getProperties(); +} diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceBase.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceBase.java index 5041c8cc2..264a73636 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceBase.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceBase.java @@ -20,7 +20,6 @@ package org.apache.kyuubi.jdbc.hive.server; import java.io.IOException; import java.util.Map; import java.util.Objects; -import org.apache.hadoop.hive.registry.ServiceInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceSet.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceSet.java new file mode 100644 index 000000000..0c231c77c --- /dev/null +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceSet.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.jdbc.hive.server; + +import java.util.Collection; +import java.util.Set; + +/** + * Note: For most of the implementations, there's no guarantee that the ServiceInstance returned by + * one invocation is the same as the instance returned by another invocation. e.g. the ZK registry + * returns a new ServiceInstance object each time a getInstance call is made. + */ +public interface ServiceInstanceSet { + + /** + * Get an instance mapping which map worker identity to each instance. + * + *

The worker identity does not collide between restarts, so each restart will have a unique + * id, while having the same host/ip pair. + * + * @return instance list + */ + Collection getAll(); + + /** + * Get an instance by worker identity. + * + * @param name worker id + * @return instance + */ + InstanceType getInstance(String name); + + /** + * Get a list of service instances for a given host. + * + *

The list could include dead and alive instances. + * + * @param host hostname + * @return instance list + */ + Set getByHost(String host); + + /** + * Get number of instances in the currently available. + * + * @return - number of instances + */ + int size(); +} diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceStateChangeListener.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceStateChangeListener.java new file mode 100644 index 000000000..25c1c51e9 --- /dev/null +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceInstanceStateChangeListener.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.jdbc.hive.server; + +/** Callback listener for instance state change events */ +public interface ServiceInstanceStateChangeListener { + /** + * Called when new {@link ServiceInstance} is created. + * + * @param serviceInstance - created service instance + */ + void onCreate(InstanceType serviceInstance, int ephSeqVersion); + + /** + * Called when an existing {@link ServiceInstance} is updated. + * + * @param serviceInstance - updated service instance + */ + void onUpdate(InstanceType serviceInstance, int ephSeqVersion); + + /** + * Called when an existing {@link ServiceInstance} is removed. + * + * @param serviceInstance - removed service instance + */ + void onRemove(InstanceType serviceInstance, int ephSeqVersion); +} diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceRegistry.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceRegistry.java new file mode 100644 index 000000000..fcb77e2f2 --- /dev/null +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ServiceRegistry.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.jdbc.hive.server; + +import java.io.IOException; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +/** + * ServiceRegistry interface for switching between fixed host and dynamic registry implementations. + */ +public interface ServiceRegistry { + + /** Start the service registry */ + void start() throws IOException; + + /** Stop the service registry */ + void stop() throws IOException; + + /** + * Client API to get the list of instances registered via the current registry key. + * + * @param component + * @param clusterReadyTimeoutMs The time to wait for the cluster to be ready, if it's not started + * yet. 0 means do not wait. + */ + ServiceInstanceSet getInstances(String component, long clusterReadyTimeoutMs) + throws IOException; + + /** + * Adds state change listeners for service instances. + * + * @param listener - state change listener + */ + void registerStateChangeListener(ServiceInstanceStateChangeListener listener) + throws IOException; + + /** @return The application ID of the LLAP cluster. */ + ApplicationId getApplicationId() throws IOException; +} diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ZkRegistryBase.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ZkRegistryBase.java index 144c35780..9bb3a3d89 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ZkRegistryBase.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/server/ZkRegistryBase.java @@ -47,9 +47,6 @@ import org.apache.curator.utils.CloseableUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.registry.RegistryUtilities; -import org.apache.hadoop.hive.registry.ServiceInstance; -import org.apache.hadoop.hive.registry.ServiceInstanceStateChangeListener; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.kyuubi.jdbc.hive.server.RegistryUtils.ServiceRecordMarshal; @@ -665,10 +662,6 @@ public abstract class ZkRegistryBase { } } - protected void unregisterInternal() { - CloseableUtils.closeQuietly(znode); - } - public void stop() { CloseableUtils.closeQuietly(znode); CloseableUtils.closeQuietly(instancesCache);