[KYUUBI #1950] Remove ambiguous SPARK_HADOOP_VERSION
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> The original idea of SPARK_HADOOP_VERSION is used to concat spark release names only, now we need to remove it as - SPARK_HADOOP_VERSION is misunderstood by developers and misused somewhere like the one of kyuubi compiled - multi-engine support now - the release names of spark(or something else) are very easy to get through code with different environments, prod/test/dev - A `mvn` job is bundled with `bin/load-kyuubi-env.sh` which is truly worrisome - SPARK_HADOOP_VERSION on spark side hass broken already for spark 3.2 which actually bundled with hadoop 3.3, see https://github.com/apache/spark-website/pull/361#discussion_r730716668 ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1950 from yaooqinn/hadoop. Closes #1950 b47be7c6 [Kent Yao] Remove ambiguous SPARK_HADOOP_VERSION 3b33ee56 [Kent Yao] Remove ambiguous SPARK_HADOOP_VERSION Authored-by: Kent Yao <yao@apache.org> Signed-off-by: Kent Yao <yao@apache.org>
This commit is contained in:
Kent Yao
parent
f25e5c93b8
commit
74abe8510c
@ -71,32 +71,11 @@ fi
|
||||
export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.12"}"
|
||||
|
||||
if [[ -f ${KYUUBI_HOME}/RELEASE ]]; then
|
||||
KYUUBI_VERSION="$(grep "Kyuubi " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print $2}')"
|
||||
FLINK_VERSION_BUILD="$(grep "Flink " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print $2}')"
|
||||
SPARK_VERSION_BUILD="$(grep "Spark " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print $2}' | grep -v 'Hadoop')"
|
||||
HADOOP_VERSION_BUILD="$(grep "Spark Hadoop " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print $3}')"
|
||||
FLINK_BUILTIN="${KYUUBI_HOME}/externals/flink-$FLINK_VERSION_BUILD"
|
||||
SPARK_BUILTIN="${KYUUBI_HOME}/externals/spark-$SPARK_VERSION_BUILD-bin-hadoop${HADOOP_VERSION_BUILD:0:3}"
|
||||
FLINK_BUILTIN="$(find "$KYUUBI_HOME/externals" -name 'flink-*' -type d | head -n 1)"
|
||||
SPARK_BUILTIN="$(find "$KYUUBI_HOME/externals" -name 'spark-*' -type d | head -n 1)"
|
||||
else
|
||||
MVN="${MVN:-"${KYUUBI_HOME}/build/mvn"}"
|
||||
KYUUBI_VERSION=$("$MVN" help:evaluate -Dexpression=project.version 2>/dev/null\
|
||||
| grep -v "INFO"\
|
||||
| grep -v "WARNING"\
|
||||
| tail -n 1)
|
||||
FLINK_VERSION_BUILD=$("$MVN" help:evaluate -Dexpression=flink.version 2>/dev/null\
|
||||
| grep -v "INFO"\
|
||||
| grep -v "WARNING"\
|
||||
| tail -n 1)
|
||||
SPARK_VERSION_BUILD=$("$MVN" help:evaluate -Dexpression=spark.version 2>/dev/null\
|
||||
| grep -v "INFO"\
|
||||
| grep -v "WARNING"\
|
||||
| tail -n 1)
|
||||
HADOOP_VERSION_BUILD=$("$MVN" help:evaluate -Dexpression=hadoop.binary.version 2>/dev/null\
|
||||
| grep -v "INFO"\
|
||||
| grep -v "WARNING"\
|
||||
| tail -n 1)
|
||||
FLINK_BUILTIN="${KYUUBI_HOME}/externals/kyuubi-download/target/flink-$FLINK_VERSION_BUILD"
|
||||
SPARK_BUILTIN="${KYUUBI_HOME}/externals/kyuubi-download/target/spark-$SPARK_VERSION_BUILD-bin-hadoop${HADOOP_VERSION_BUILD}"
|
||||
FLINK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name 'flink-*' -type d | head -n 1)"
|
||||
SPARK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name 'spark-*' -type d | head -n 1)"
|
||||
fi
|
||||
|
||||
export FLINK_HOME="${FLINK_HOME:-"${FLINK_BUILTIN}"}"
|
||||
@ -109,7 +88,6 @@ if [ $silent -eq 0 ]; then
|
||||
echo "JAVA_HOME: ${JAVA_HOME}"
|
||||
|
||||
echo "KYUUBI_HOME: ${KYUUBI_HOME}"
|
||||
echo "KYUUBI_VERSION: ${KYUUBI_VERSION}"
|
||||
echo "KYUUBI_CONF_DIR: ${KYUUBI_CONF_DIR}"
|
||||
echo "KYUUBI_LOG_DIR: ${KYUUBI_LOG_DIR}"
|
||||
echo "KYUUBI_PID_DIR: ${KYUUBI_PID_DIR}"
|
||||
|
||||
16
build/dist
16
build/dist
@ -153,11 +153,6 @@ SPARK_VERSION=$("$MVN" help:evaluate -Dexpression=spark.version $@ 2>/dev/null\
|
||||
| grep -v "WARNING"\
|
||||
| tail -n 1)
|
||||
|
||||
SPARK_HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.binary.version $@ 2>/dev/null\
|
||||
| grep -v "INFO"\
|
||||
| grep -v "WARNING"\
|
||||
| tail -n 1)
|
||||
|
||||
HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.version $@ 2>/dev/null\
|
||||
| grep -v "INFO"\
|
||||
| grep -v "WARNING"\
|
||||
@ -175,7 +170,7 @@ if [[ "$NAME" == "none" ]]; then
|
||||
if [[ "$SPARK_PROVIDED" == "true" ]]; then
|
||||
SUFFIX=""
|
||||
else
|
||||
SUFFIX="-spark-${SPARK_VERSION:0:3}-hadoop${SPARK_HADOOP_VERSION}"
|
||||
SUFFIX="-spark-${SPARK_VERSION:0:3}"
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -219,7 +214,6 @@ echo "Java $JAVA_VERSION" >> "$DISTDIR/RELEASE"
|
||||
echo "Scala $SCALA_VERSION" >> "$DISTDIR/RELEASE"
|
||||
echo "Flink $FLINK_VERSION" >> "$DISTDIR/RELEASE"
|
||||
echo "Spark $SPARK_VERSION" >> "$DISTDIR/RELEASE"
|
||||
echo "Spark Hadoop $SPARK_HADOOP_VERSION" >> "$DISTDIR/RELEASE"
|
||||
echo "Kyuubi Hadoop $HADOOP_VERSION" >> "$DISTDIR/RELEASE"
|
||||
echo "Hive $HIVE_VERSION" >> "$DISTDIR/RELEASE"
|
||||
echo "Build flags: $@" >> "$DISTDIR/RELEASE"
|
||||
@ -275,14 +269,14 @@ done
|
||||
|
||||
if [[ "$FLINK_PROVIDED" != "true" ]]; then
|
||||
# Copy flink binary dist
|
||||
cp -r "$KYUUBI_HOME/externals/kyuubi-download/target/flink-$FLINK_VERSION/" \
|
||||
"$DISTDIR/externals/flink-$FLINK_VERSION/"
|
||||
FLINK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name 'flink-*' -type d)"
|
||||
cp -r "$FLINK_BUILTIN" "$DISTDIR/externals/"
|
||||
fi
|
||||
|
||||
if [[ "$SPARK_PROVIDED" != "true" ]]; then
|
||||
# Copy spark binary dist
|
||||
cp -r "$KYUUBI_HOME/externals/kyuubi-download/target/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}$HIVE_VERSION_SUFFIX/" \
|
||||
"$DISTDIR/externals/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}$HIVE_VERSION_SUFFIX/"
|
||||
SPARK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name 'spark-*' -type d)"
|
||||
cp -r "$SPARK_BUILTIN" "$DISTDIR/externals/"
|
||||
fi
|
||||
|
||||
# Copy license files
|
||||
|
||||
17
pom.xml
17
pom.xml
@ -110,7 +110,6 @@
|
||||
<guava.version>30.1-jre</guava.version>
|
||||
<guava.failureaccess.version>1.0.1</guava.failureaccess.version>
|
||||
<hadoop.version>3.3.1</hadoop.version>
|
||||
<hadoop.binary.version>3.2</hadoop.binary.version>
|
||||
<hive.version>2.3.9</hive.version>
|
||||
<hudi.version>0.10.0</hudi.version>
|
||||
<iceberg.name>iceberg-spark-runtime-3.2_${scala.binary.version}</iceberg.name>
|
||||
@ -138,7 +137,7 @@
|
||||
`delta.version`, `iceberg.name`
|
||||
-->
|
||||
<spark.version>3.2.1</spark.version>
|
||||
<spark.archive.name>spark-${spark.version}-bin-hadoop${hadoop.binary.version}.tgz</spark.archive.name>
|
||||
<spark.archive.name>spark-${spark.version}-bin-hadoop3.2.tgz</spark.archive.name>
|
||||
<spark.archive.mirror>https://archive.apache.org/dist/spark/spark-${spark.version}</spark.archive.mirror>
|
||||
<spark.archive.download.skip>false</spark.archive.download.skip>
|
||||
<swagger.version>2.1.11</swagger.version>
|
||||
@ -1901,20 +1900,6 @@
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>spark-hadoop-2.7</id>
|
||||
<properties>
|
||||
<hadoop.binary.version>2.7</hadoop.binary.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>spark-hadoop-3.2</id>
|
||||
<properties>
|
||||
<hadoop.binary.version>3.2</hadoop.binary.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>spark-provided</id>
|
||||
<properties>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user