From 9fd392b7e94a47654471a88b6d33806215ea9802 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 11 Mar 2024 15:21:14 +0800 Subject: [PATCH] [KYUUBI #6153] Cut out Hive JDBC and Hive Service deps in beeline module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # :mag: Description ## Issue References ๐Ÿ”— This is the next step of https://github.com/apache/kyuubi/issues/6146, cutting out most Hive deps(excepting `hive-common`) and recovering the skipped tests via minor code tunning. ## Describe Your Solution ๐Ÿ”ง - Drop `hive-jdbc`, `hive-service`, `hive-service-rpc` deps in the beeline module. - Migrate from `commons-lang` to `commons-lang3` in the beeline module. - Recover the skipped test `TestClientCommandHookFactory#connectHook` ## Types of changes :bookmark: - [x] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan ๐Ÿงช Pass GA, and manually test to ensure the following error has gone. Before ``` roothadoop-master1:/opt/kyuubi# bin/beeline --version Warn: Not find kyuubi environment file /etc/kyuubi/conf/kyuubi-env.sh, using default ones... java.lang.NoClassDefFoundError: org/apache/curator/RetryPolicy ``` After ``` roothadoop-master1:/opt/kyuubi# bin/beeline --version Connecting to jdbc:hive2://hadoop-master1.orb.local:10000/default;password=hive;user=hive Connected to: Apache Hive (version 2.3.9) Driver: Kyuubi Project Hive JDBC Client (version 1.9.0-SNAPSHOT) Beeline version 1.9.0-SNAPSHOT by Apache Kyuubi 0: jdbc:hive2://hadoop-master1.orb.local:1000> ``` --- # Checklist ๐Ÿ“ - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes #6153 from pan3793/beeline-2. Closes #6153 8cd52e509 [Cheng Pan] notice d03c72992 [Cheng Pan] minor 5d16bf4ce [Cheng Pan] beeline test pass Authored-by: Cheng Pan Signed-off-by: Cheng Pan --- NOTICE-binary | 9 --- kyuubi-hive-beeline/pom.xml | 65 +++---------------- .../java/org/apache/hive/beeline/BeeLine.java | 12 ++-- .../beeline/ClientCommandHookFactory.java | 2 +- .../org/apache/hive/beeline/Commands.java | 39 ++++++----- .../hive/beeline/DatabaseConnection.java | 6 +- .../beeline/SeparatedValuesOutputFormat.java | 4 +- .../hs2connection/HS2ConnectionFileUtils.java | 2 +- .../apache/hive/beeline/ProxyAuthTest.java | 31 +++++++-- .../beeline/TestClientCommandHookFactory.java | 2 - pom.xml | 10 ++- 11 files changed, 71 insertions(+), 111 deletions(-) diff --git a/NOTICE-binary b/NOTICE-binary index eb8634f0f..a1b304dfa 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -192,15 +192,6 @@ JUnit (4.12) Hive Common Copyright 2022 The Apache Software Foundation -Hive JDBC -Copyright 2022 The Apache Software Foundation - -Hive Service -Copyright 2022 The Apache Software Foundation - -Hive Service RPC -Copyright 2022 The Apache Software Foundation - Apache HttpClient Copyright 1999-2020 The Apache Software Foundation diff --git a/kyuubi-hive-beeline/pom.xml b/kyuubi-hive-beeline/pom.xml index 33753ed94..0b3bbf48f 100644 --- a/kyuubi-hive-beeline/pom.xml +++ b/kyuubi-hive-beeline/pom.xml @@ -59,44 +59,13 @@ - org.apache.hive - hive-jdbc - ${hive.version} - - - * - * - - - - - - org.apache.hive - hive-service-rpc - ${hive.version} - - - * - * - - - - - - org.apache.hive - hive-service - ${hive.version} - - - * - * - - + org.apache.hadoop + hadoop-client-api org.apache.hadoop - hadoop-client-api + hadoop-client-runtime @@ -110,14 +79,8 @@ - org.mockito - mockito-core - test - - - - commons-lang - commons-lang + org.apache.commons + commons-lang3 @@ -128,13 +91,6 @@ net.sf.supercsv super-csv - ${hive.client.supercsv.version} - - - * - * - - @@ -163,14 +119,9 @@ - org.apache.hadoop - hadoop-client-runtime - - - * - * - - + org.mockito + mockito-core + test diff --git a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/BeeLine.java b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/BeeLine.java index 1706d1531..e3b29206c 100644 --- a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/BeeLine.java +++ b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/BeeLine.java @@ -86,10 +86,10 @@ import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.io.IOUtils; import org.apache.hive.beeline.cli.CliOptionsProcessor; import org.apache.hive.beeline.hs2connection.BeelineConfFileParseException; import org.apache.hive.beeline.hs2connection.BeelineSiteParseException; @@ -99,9 +99,9 @@ import org.apache.hive.beeline.hs2connection.HS2ConnectionFileUtils; import org.apache.hive.beeline.hs2connection.HiveSiteHS2ConnectionFileParser; import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; import org.apache.hive.common.util.ShutdownHookManager; -import org.apache.hive.jdbc.JdbcUriParseException; -import org.apache.hive.jdbc.Utils; -import org.apache.hive.jdbc.Utils.JdbcConnectionParams; +import org.apache.kyuubi.jdbc.hive.JdbcConnectionParams; +import org.apache.kyuubi.jdbc.hive.JdbcUriParseException; +import org.apache.kyuubi.jdbc.hive.Utils; import org.apache.kyuubi.shaded.thrift.transport.TTransportException; /** @@ -1178,7 +1178,7 @@ public class BeeLine implements Closeable { } private int embeddedConnect() { - if (!execCommandWithPrefix("!connect " + Utils.URL_PREFIX + " '' ''")) { + if (!execCommandWithPrefix("!connect jdbc:hive2:// '' ''")) { return ERRNO_OTHER; } else { return ERRNO_OK; @@ -1226,7 +1226,7 @@ public class BeeLine implements Closeable { handleException(t); return ERRNO_OTHER; } finally { - IOUtils.closeStream(fileStream); + IOUtils.closeQuietly(fileStream); } } diff --git a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/ClientCommandHookFactory.java b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/ClientCommandHookFactory.java index 935f1cfbf..cebcad1fe 100644 --- a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/ClientCommandHookFactory.java +++ b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/ClientCommandHookFactory.java @@ -18,7 +18,7 @@ package org.apache.hive.beeline; import java.sql.SQLException; -import org.apache.hive.jdbc.Utils; +import org.apache.kyuubi.jdbc.hive.Utils; /** We need to update some client side information after executing some Hive Commands */ public class ClientCommandHookFactory { diff --git a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/Commands.java b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/Commands.java index 6c59b0401..a1df5fb55 100644 --- a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/Commands.java +++ b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/Commands.java @@ -58,11 +58,12 @@ import org.apache.hadoop.hive.conf.HiveVariableSource; import org.apache.hadoop.hive.conf.SystemVariables; import org.apache.hadoop.hive.conf.VariableSubstitution; import org.apache.hadoop.io.IOUtils; +import org.apache.hive.beeline.logs.KyuubiBeelineInPlaceUpdateStream; import org.apache.hive.common.util.HiveStringUtils; -import org.apache.hive.jdbc.HiveStatement; -import org.apache.hive.jdbc.Utils; -import org.apache.hive.jdbc.Utils.JdbcConnectionParams; -import org.apache.hive.jdbc.logs.InPlaceUpdateStream; +import org.apache.kyuubi.jdbc.hive.JdbcConnectionParams; +import org.apache.kyuubi.jdbc.hive.KyuubiStatement; +import org.apache.kyuubi.jdbc.hive.Utils; +import org.apache.kyuubi.jdbc.hive.logs.InPlaceUpdateStream; public class Commands { @@ -986,10 +987,10 @@ public class Commands { logThread = new Thread(createLogRunnable(stmnt, eventNotifier)); logThread.setDaemon(true); logThread.start(); - if (stmnt instanceof HiveStatement) { - // HiveStatement hiveStatement = (HiveStatement) stmnt; - // hiveStatement.setInPlaceUpdateStream( - // new BeelineInPlaceUpdateStream(beeLine.getErrorStream(), eventNotifier)); + if (stmnt instanceof KyuubiStatement) { + KyuubiStatement hiveStatement = (KyuubiStatement) stmnt; + hiveStatement.setInPlaceUpdateStream( + new KyuubiBeelineInPlaceUpdateStream(beeLine.getErrorStream(), eventNotifier)); } hasResults = stmnt.execute(sql); logThread.interrupt(); @@ -1289,11 +1290,11 @@ public class Commands { private Runnable createLogRunnable( final Statement statement, InPlaceUpdateStream.EventNotifier eventNotifier) { - if (statement instanceof HiveStatement) { + if (statement instanceof KyuubiStatement) { return new LogRunnable( - this, (HiveStatement) statement, DEFAULT_QUERY_PROGRESS_INTERVAL, eventNotifier); + this, (KyuubiStatement) statement, DEFAULT_QUERY_PROGRESS_INTERVAL, eventNotifier); } else { - beeLine.debug("The statement instance is not HiveStatement type: " + statement.getClass()); + beeLine.debug("The statement instance is not KyuubiStatement type: " + statement.getClass()); return new Runnable() { @Override public void run() { @@ -1313,13 +1314,13 @@ public class Commands { static class LogRunnable implements Runnable { private final Commands commands; - private final HiveStatement hiveStatement; + private final KyuubiStatement hiveStatement; private final long queryProgressInterval; private final InPlaceUpdateStream.EventNotifier notifier; LogRunnable( Commands commands, - HiveStatement hiveStatement, + KyuubiStatement hiveStatement, long queryProgressInterval, InPlaceUpdateStream.EventNotifier eventNotifier) { this.hiveStatement = hiveStatement; @@ -1330,7 +1331,7 @@ public class Commands { private void updateQueryLog() { try { - List queryLogs = hiveStatement.getQueryLog(); + List queryLogs = hiveStatement.getExecLog(); for (String log : queryLogs) { if (!commands.beeLine.isTestMode()) { commands.beeLine.info(log); @@ -1371,12 +1372,12 @@ public class Commands { } private void showRemainingLogsIfAny(Statement statement) { - if (statement instanceof HiveStatement) { - HiveStatement hiveStatement = (HiveStatement) statement; + if (statement instanceof KyuubiStatement) { + KyuubiStatement hiveStatement = (KyuubiStatement) statement; List logs = null; do { try { - logs = hiveStatement.getQueryLog(); + logs = hiveStatement.getExecLog(); } catch (SQLException e) { beeLine.error(new SQLWarning(e)); return; @@ -1629,9 +1630,7 @@ public class Commands { props.setProperty(JdbcConnectionParams.AUTH_USER, username); if (password == null) { password = - beeLine - .getConsoleReader() - .readLine("Enter password for " + urlForPrompt + ": ", new Character('*')); + beeLine.getConsoleReader().readLine("Enter password for " + urlForPrompt + ": ", '*'); } props.setProperty(JdbcConnectionParams.AUTH_PASSWD, password); } diff --git a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/DatabaseConnection.java b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/DatabaseConnection.java index a57ae6331..624715597 100644 --- a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/DatabaseConnection.java +++ b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/DatabaseConnection.java @@ -38,7 +38,7 @@ import java.util.Set; import java.util.TreeSet; import jline.console.completer.ArgumentCompleter; import jline.console.completer.Completer; -import org.apache.hive.jdbc.HiveConnection; +import org.apache.kyuubi.jdbc.hive.KyuubiConnection; class DatabaseConnection { private static final String HIVE_VAR_PREFIX = "hivevar:"; @@ -272,8 +272,8 @@ class DatabaseConnection { } public String getConnectedUrl() { - if (connection instanceof HiveConnection) { - return ((HiveConnection) connection).getConnectedUrl(); + if (connection instanceof KyuubiConnection) { + return ((KyuubiConnection) connection).getConnectedUrl(); } return getUrl(); } diff --git a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java index d721f4822..4a93db9f8 100644 --- a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java +++ b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java @@ -23,8 +23,8 @@ package org.apache.hive.beeline; import org.apache.commons.io.output.StringBuilderWriter; -import org.apache.commons.lang.BooleanUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; import org.supercsv.encoder.CsvEncoder; import org.supercsv.encoder.DefaultCsvEncoder; import org.supercsv.encoder.SelectiveCsvEncoder; diff --git a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java index f44767955..c21f6120a 100644 --- a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java +++ b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java @@ -23,7 +23,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Properties; -import org.apache.hive.jdbc.Utils.JdbcConnectionParams; +import org.apache.kyuubi.jdbc.hive.JdbcConnectionParams; public class HS2ConnectionFileUtils { diff --git a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/ProxyAuthTest.java b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/ProxyAuthTest.java index 56595e83f..c4e9ae8e7 100644 --- a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/ProxyAuthTest.java +++ b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/ProxyAuthTest.java @@ -26,10 +26,11 @@ import java.sql.SQLException; import java.sql.Statement; import javax.security.auth.login.LoginException; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hive.jdbc.HiveConnection; -import org.apache.hive.service.auth.HiveAuthConstants; -import org.apache.hive.service.cli.session.SessionUtils; +import org.apache.hadoop.security.token.Token; +import org.apache.kyuubi.jdbc.hive.KyuubiConnection; /** * Simple client application to test various direct and proxy connection to HiveServer2 Note that @@ -40,7 +41,7 @@ import org.apache.hive.service.cli.session.SessionUtils; * */ public class ProxyAuthTest { - private static final String driverName = "org.apache.hive.jdbc.HiveDriver"; + private static final String driverName = "org.apache.kyuubi.jdbc.KyuubiHiveDriver"; private static final String BEELINE_EXIT = "beeline.system.exit"; private static Connection con = null; private static boolean noClose = false; @@ -86,7 +87,7 @@ public class ProxyAuthTest { con = DriverManager.getConnection(url); System.out.println("Connected successfully to " + url); // get delegation token for the given proxy user - String token = ((HiveConnection) con).getDelegationToken(proxyUser, serverPrincipal); + String token = ((KyuubiConnection) con).getDelegationToken(proxyUser, serverPrincipal); if ("true".equals(System.getProperty("proxyAuth.debug", "false"))) { System.out.println("Got token: " + token); } @@ -190,7 +191,7 @@ public class ProxyAuthTest { System.out.println("Connected successfully to " + url); runTest(); - ((HiveConnection) con).cancelDelegationToken(token); + ((KyuubiConnection) con).cancelDelegationToken(token); con.close(); } catch (SQLException e) { System.out.println("*** SQLException: " + e.getMessage() + " : " + e.getSQLState()); @@ -222,11 +223,27 @@ public class ProxyAuthTest { return UserGroupInformation.getCurrentUser(); } + private static final String HS2_CLIENT_TOKEN = "hiveserver2ClientToken"; + private static void storeTokenInJobConf(String tokenStr) throws Exception { - SessionUtils.setTokenStr(getUGI(), tokenStr, HiveAuthConstants.HS2_CLIENT_TOKEN); + setTokenStr(getUGI(), tokenStr, HS2_CLIENT_TOKEN); System.out.println("Stored token " + tokenStr); } + private static void setTokenStr(UserGroupInformation ugi, String tokenStr, String tokenService) + throws IOException { + Token delegationToken = createToken(tokenStr, tokenService); + ugi.addToken(delegationToken); + } + + private static Token createToken(String tokenStr, String tokenService) + throws IOException { + Token delegationToken = new Token<>(); + delegationToken.decodeFromUrlString(tokenStr); + delegationToken.setService(new Text(tokenService)); + return delegationToken; + } + // run sql operations private static void runTest() throws Exception { // craete table and check dir ownership diff --git a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/TestClientCommandHookFactory.java b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/TestClientCommandHookFactory.java index a8cd480f1..6aaef1222 100644 --- a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/TestClientCommandHookFactory.java +++ b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/TestClientCommandHookFactory.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.when; import java.sql.Connection; import java.sql.SQLException; import junit.framework.Assert; -import org.junit.Ignore; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -103,7 +102,6 @@ public class TestClientCommandHookFactory { } @Test - @Ignore("Fix later: NoClassDefFound org/apache/curator/RetryPolicy") public void testConnectHook() { BeeLine beeLine = setupMockData(true, true); ClientHook hook = diff --git a/pom.xml b/pom.xml index 8f3c0baaf..85b2e0232 100644 --- a/pom.xml +++ b/pom.xml @@ -207,6 +207,7 @@ ${apache.archive.dist}/spark/spark-${spark.version} false 3.42.0.0 + 2.2.0 2.2.1 0.41.0 @@ -222,9 +223,6 @@ v18.16.0 v8.6.1 - - 2.2.0 - kyuubi-hive-jdbc UTF-8 @@ -1259,6 +1257,12 @@ ${fliptables.verion} + + net.sf.supercsv + super-csv + ${supercsv.version} + + org.glassfish.jersey.core