From a71c511ba36ba496659cdb6983937ac2c2a681bf Mon Sep 17 00:00:00 2001 From: SteNicholas Date: Thu, 10 Mar 2022 15:03:11 +0800 Subject: [PATCH] [KYUUBI #2081] YARN_CONF_DIR shall be added to kyuubi server classpath as HADOOP_CONF_DIR ### _Why are the changes needed?_ YARN_CONF_DIR shall be added to kyuubi server classpath as HADOOP_CONF_DIR. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #2082 from SteNicholas/KYUUBI-2081. Closes #2081 c6f76c5c [SteNicholas] [KYUUBI #2081] YARN_CONF_DIR shall be added to kyuubi server classpath as HADOOP_CONF_DIR fee5c21f [SteNicholas] [KYUUBI #2081] YARN_CONF_DIR shall be added to kyuubi server classpath as HADOOP_CONF_DIR Authored-by: SteNicholas Signed-off-by: Kent Yao --- bin/beeline | 6 +++++- bin/kyuubi | 7 ++++++- bin/kyuubi-ctl | 6 +++++- bin/kyuubi-zk-cli | 6 +++++- bin/load-kyuubi-env.sh | 2 ++ conf/kyuubi-env.sh.template | 2 ++ docs/deployment/settings.md | 2 ++ docs/integrations/delta_lake_with_azure_blob.md | 1 + docs/monitor/logging.md | 1 + docs/quick_start/quick_start.md | 1 + 10 files changed, 30 insertions(+), 4 deletions(-) diff --git a/bin/beeline b/bin/beeline index 2dd72598a..91e6654bf 100755 --- a/bin/beeline +++ b/bin/beeline @@ -39,6 +39,10 @@ if [[ -z "$KYUUBI_BEELINE_JARS" ]]; then fi fi -KYUUBI_BEELINE_CLASSPATH="${KYUUBI_BEELINE_JARS}/*:${HADOOP_CONF_DIR}" +if [[ -z ${YARN_CONF_DIR} ]]; then + KYUUBI_BEELINE_CLASSPATH="${KYUUBI_BEELINE_JARS}/*:${HADOOP_CONF_DIR}" +else + KYUUBI_BEELINE_CLASSPATH="${KYUUBI_BEELINE_JARS}/*:${HADOOP_CONF_DIR}:${YARN_CONF_DIR}" +fi exec ${RUNNER} ${KYUUBI_BEELINE_OPTS} -cp ${KYUUBI_BEELINE_CLASSPATH} $CLASS "$@" diff --git a/bin/kyuubi b/bin/kyuubi index edd5465ed..0f1789e2a 100755 --- a/bin/kyuubi +++ b/bin/kyuubi @@ -85,7 +85,12 @@ if [[ -z "$KYUUBI_JAR_DIR" ]]; then fi fi -KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}" +if [[ -z ${YARN_CONF_DIR} ]]; then + KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}" +else + KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}:${YARN_CONF_DIR}" +fi + cmd="${RUNNER} ${KYUUBI_JAVA_OPTS} -cp ${KYUUBI_CLASSPATH} $CLASS" pid="${KYUUBI_PID_DIR}/kyuubi-$USER-$CLASS.pid" diff --git a/bin/kyuubi-ctl b/bin/kyuubi-ctl index f6c3be684..3ff03ddff 100755 --- a/bin/kyuubi-ctl +++ b/bin/kyuubi-ctl @@ -39,6 +39,10 @@ if [[ -z "$KYUUBI_JAR_DIR" ]]; then fi fi -KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}" +if [[ -z ${YARN_CONF_DIR} ]]; then + KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}" +else + KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}:${YARN_CONF_DIR}" +fi exec ${RUNNER} ${KYUUBI_CTL_JAVA_OPTS} -cp ${KYUUBI_CLASSPATH} $CLASS "$@" diff --git a/bin/kyuubi-zk-cli b/bin/kyuubi-zk-cli index 369a45d3f..089b7ad18 100755 --- a/bin/kyuubi-zk-cli +++ b/bin/kyuubi-zk-cli @@ -39,6 +39,10 @@ if [[ -z "$KYUUBI_JAR_DIR" ]]; then fi fi -KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}" +if [[ -z ${YARN_CONF_DIR} ]]; then + KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}" +else + KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}:${YARN_CONF_DIR}" +fi exec ${RUNNER} ${KYUUBI_JAVA_OPTS} -cp ${KYUUBI_CLASSPATH} $CLASS "$@" \ No newline at end of file diff --git a/bin/load-kyuubi-env.sh b/bin/load-kyuubi-env.sh index 7d65cca6f..087eb21d4 100755 --- a/bin/load-kyuubi-env.sh +++ b/bin/load-kyuubi-env.sh @@ -102,4 +102,6 @@ if [ $silent -eq 0 ]; then echo "TRINO_ENGINE_HOME: ${TRINO_ENGINE_HOME}" echo "HADOOP_CONF_DIR: ${HADOOP_CONF_DIR}" + + echo "YARN_CONF_DIR: ${YARN_CONF_DIR}" fi diff --git a/conf/kyuubi-env.sh.template b/conf/kyuubi-env.sh.template index 969d47f47..4afa70dbf 100755 --- a/conf/kyuubi-env.sh.template +++ b/conf/kyuubi-env.sh.template @@ -38,6 +38,7 @@ # - KYUUBI_WORK_DIR_ROOT Root directory for launching sql engine applications. # (Default: $KYUUBI_HOME/work) # - HADOOP_CONF_DIR Directory containing the Hadoop / YARN configuration to use. +# - YARN_CONF_DIR Directory containing the YARN configuration to use. # # - SPARK_HOME Spark distribution which you would like to use in Kyuubi. # - SPARK_CONF_DIR Optional directory where the Spark configuration lives. @@ -50,5 +51,6 @@ # export JAVA_HOME=/usr/jdk64/jdk1.8.0_152 # export SPARK_HOME=/opt/spark # export HADOOP_CONF_DIR=/usr/ndp/current/mapreduce_client/conf +# export YARN_CONF_DIR=/usr/ndp/current/yarn/conf # export KYUUBI_JAVA_OPTS="-Xmx10g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark -XX:MaxDirectMemorySize=1024m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./logs -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:./logs/kyuubi-server-gc-%t.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=5M -XX:NewRatio=3 -XX:MetaspaceSize=512m" # export KYUUBI_BEELINE_OPTS="-Xmx2g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark" diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md index fc07eb80f..38a176da9 100644 --- a/docs/deployment/settings.md +++ b/docs/deployment/settings.md @@ -73,6 +73,7 @@ You can configure the environment variables in `$KYUUBI_HOME/conf/kyuubi-env.sh` # - KYUUBI_WORK_DIR_ROOT Root directory for launching sql engine applications. # (Default: $KYUUBI_HOME/work) # - HADOOP_CONF_DIR Directory containing the Hadoop / YARN configuration to use. +# - YARN_CONF_DIR Directory containing the YARN configuration to use. # # - SPARK_HOME Spark distribution which you would like to use in Kyuubi. # - SPARK_CONF_DIR Optional directory where the Spark configuration lives. @@ -85,6 +86,7 @@ You can configure the environment variables in `$KYUUBI_HOME/conf/kyuubi-env.sh` # export JAVA_HOME=/usr/jdk64/jdk1.8.0_152 # export SPARK_HOME=/opt/spark # export HADOOP_CONF_DIR=/usr/ndp/current/mapreduce_client/conf +# export YARN_CONF_DIR=/usr/ndp/current/yarn/conf # export KYUUBI_JAVA_OPTS="-Xmx10g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark -XX:MaxDirectMemorySize=1024m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./logs -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:./logs/kyuubi-server-gc-%t.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=5M -XX:NewRatio=3 -XX:MetaspaceSize=512m" # export KYUUBI_BEELINE_OPTS="-Xmx2g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark" ``` diff --git a/docs/integrations/delta_lake_with_azure_blob.md b/docs/integrations/delta_lake_with_azure_blob.md index 03a981892..dfd60b835 100644 --- a/docs/integrations/delta_lake_with_azure_blob.md +++ b/docs/integrations/delta_lake_with_azure_blob.md @@ -195,6 +195,7 @@ KYUUBI_WORK_DIR_ROOT: /usr/apache/current/kyuubi/work SPARK_HOME: /usr/apache/current/spark SPARK_CONF_DIR: /usr/apache/current/spark/conf HADOOP_CONF_DIR: +YARN_CONF_DIR: Starting org.apache.kyuubi.server.KyuubiServer, logging to /usr/apache/current/kyuubi/logs/kyuubi-hadoop-org.apache.kyuubi.server.KyuubiServer-host.out Welcome to __ __ __ diff --git a/docs/monitor/logging.md b/docs/monitor/logging.md index f1655e8d2..9081446ef 100644 --- a/docs/monitor/logging.md +++ b/docs/monitor/logging.md @@ -75,6 +75,7 @@ KYUUBI_WORK_DIR_ROOT: /Users/kentyao/svn-kyuubi/v1.3.1-incubating-rc0/apache-kyu SPARK_HOME: /Users/kentyao/Downloads/spark/spark-3.2.0-bin-hadoop3.2 SPARK_CONF_DIR: /Users/kentyao/Downloads/spark/spark-3.2.0-bin-hadoop3.2/conf HADOOP_CONF_DIR: +YARN_CONF_DIR: Starting org.apache.kyuubi.server.KyuubiServer, logging to /Users/kentyao/svn-kyuubi/v1.3.1-incubating-rc0/apache-kyuubi-1.3.1-incubating-bin/logs/kyuubi-kentyao-org.apache.kyuubi.server.KyuubiServer-hulk.local.out Welcome to __ __ __ diff --git a/docs/quick_start/quick_start.md b/docs/quick_start/quick_start.md index f3d72dd6a..b03c03494 100644 --- a/docs/quick_start/quick_start.md +++ b/docs/quick_start/quick_start.md @@ -162,6 +162,7 @@ KYUUBI_WORK_DIR_ROOT: /Users/kentyao/svn-kyuubi/v1.3.1-incubating-rc0/apache-kyu SPARK_HOME: /Users/kentyao/Downloads/spark/spark-3.2.0-bin-hadoop3.2 SPARK_CONF_DIR: /Users/kentyao/Downloads/spark/spark-3.2.0-bin-hadoop3.2/conf HADOOP_CONF_DIR: +YARN_CONF_DIR: Starting org.apache.kyuubi.server.KyuubiServer, logging to /Users/kentyao/svn-kyuubi/v1.3.1-incubating-rc0/apache-kyuubi-1.3.1-incubating-bin/logs/kyuubi-kentyao-org.apache.kyuubi.server.KyuubiServer-hulk.local.out Welcome to __ __ __