From d862272645dab08878c8f99158977b622227e784 Mon Sep 17 00:00:00 2001 From: liangbowen Date: Fri, 10 Feb 2023 16:19:45 +0800 Subject: [PATCH] [KYUUBI #4274] [INFRA] Introduce `mvnd` to speed up CI jobs of Dependency, Licence and Style Check ### _Why are the changes needed?_ #### Motivation: - speed up maven building and testing with `mvnd` (https://github.com/apache/maven-mvnd). `mvnd` itself embeds exactly the same distribution of maven with more improvements and features on the client side and compilation server in daemon mode. - also inspired by custom github action for mvnd installing in Apache Camel (https://github.com/apache/camel/blob/main/.github/actions/install-mvnd/action.yml) #### Changes in this PR: - introducing `mvnd` by adding `build/mvnd` script - use `mvnd` version `0.9.0` which embeds `maven` 3.8.7. The maven version embedded in `mvnd` is also check and guaranteed the same as maven version required in pom by `build/mvnd` script. - use mvnd in CI jobs of Depedency Check and Style Check #### Comparision - CI jobs (both with maven dependencies cached in local repo) Job | with `build/mvn` | with `build/mvnd` --- | --- | --- Dependency Check | 9min1sec;[see log](https://github.com/apache/kyuubi/actions/runs/3966175262/jobs/6796688187) | 6min 46 sec ; [see log](https://github.com/apache/kyuubi/actions/runs/4124518481/jobs/7123867015) Style Check | 10min32sec ; [see log](https://github.com/apache/kyuubi/actions/runs/4122613274/jobs/7119603428) | 7min50s ; [see log](https://github.com/apache/kyuubi/actions/runs/4125013599/jobs/7125006946) Licence Check | 32s ; [see log](https://github.com/apache/kyuubi/actions/runs/4130799867/jobs/7137835062) | 21s ; [see log](https://github.com/apache/kyuubi/actions/runs/4130923126/jobs/7138088114) - building entire maven project on local machine (skipping test running and style checking with `fast` profile) `build/mvnd clean install -Pfast` Pasted Graphic `build/mvn clean install -Pfast` Pasted Graphic 1 ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4274 from bowenliang123/ci-mvnd. Closes #4274 380c1c9ea [liangbowen] update cache key a93bba7b7 [liangbowen] style bc8da133e [Bowen Liang] Merge branch 'master' into ci-mvnd 97becc035 [liangbowen] add `build/maven-mvnd-*/**` to rat-excludes 5b61f50b9 [liangbowen] apply mvnd to license check 7be181d0f [liangbowen] typo in build/mvnd c52861df8 [liangbowen] update f6d0eb28c [liangbowen] introduce mvnd Lead-authored-by: liangbowen Co-authored-by: Bowen Liang Signed-off-by: Cheng Pan --- .github/actions/setup-mvnd/action.yaml | 32 ++++++ .github/workflows/dep.yml | 5 +- .github/workflows/license.yml | 4 +- .github/workflows/style.yml | 16 +-- .gitignore | 1 + .rat-excludes | 1 + build/mvnd | 134 +++++++++++++++++++++++++ pom.xml | 1 + 8 files changed, 185 insertions(+), 9 deletions(-) create mode 100644 .github/actions/setup-mvnd/action.yaml create mode 100755 build/mvnd diff --git a/.github/actions/setup-mvnd/action.yaml b/.github/actions/setup-mvnd/action.yaml new file mode 100644 index 000000000..d7497e332 --- /dev/null +++ b/.github/actions/setup-mvnd/action.yaml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 'setup-mvnd' +description: 'Setup the maven daemon' +runs: + using: composite + steps: + - name: Cache Mvnd + uses: actions/cache@v3 + with: + path: | + build/maven-mvnd-* + build/apache-maven-* + key: setup-mvnd-${{ runner.os }}-mvnd + - name: Check Mvnd + run: build/mvnd -v + shell: bash diff --git a/.github/workflows/dep.yml b/.github/workflows/dep.yml index 16475ce0f..dc94e4ea7 100644 --- a/.github/workflows/dep.yml +++ b/.github/workflows/dep.yml @@ -26,6 +26,7 @@ on: # when pom or dependency workflow changes - '**/pom.xml' - '.github/workflows/dep.yml' + - .github/actions/setup-mvnd/*.yaml concurrency: group: dep-${{ github.head_ref || github.run_id }} @@ -44,11 +45,13 @@ jobs: java-version: 8 cache: 'maven' check-latest: false + - name: Setup Mvnd + uses: ./.github/actions/setup-mvnd - name: build env: MAVEN_OPTS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error run: >- - build/mvn clean install + build/mvnd clean install -Pflink-provided,spark-provided,hive-provided -Dmaven.javadoc.skip=true -Drat.skip=true diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml index 17591dacb..a490def91 100644 --- a/.github/workflows/license.yml +++ b/.github/workflows/license.yml @@ -42,8 +42,10 @@ jobs: java-version: 8 cache: 'maven' check-latest: false + - name: Setup Mvnd + uses: ./.github/actions/setup-mvnd - run: >- - build/mvn org.apache.rat:apache-rat-plugin:check + build/mvnd org.apache.rat:apache-rat-plugin:check -Ptpcds -Pspark-block-cleaner -Pkubernetes-it -Pspark-3.1 -Pspark-3.2 -Pspark-3.3 - name: Upload rat report diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index f9307e529..9ff484824 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -45,14 +45,16 @@ jobs: java-version: 8 cache: 'maven' check-latest: false + - name: Setup Mvnd + uses: ./.github/actions/setup-mvnd - name: Setup Python 3 uses: actions/setup-python@v4 with: python-version: '3.9' cache: 'pip' - - name: Check kyuubi modules avaliable + - name: Check kyuubi modules available id: modules-check - run: build/mvn dependency:resolve -DincludeGroupIds="org.apache.kyuubi" -DincludeScope="compile" -DexcludeTransitive=true ${{ matrix.profiles }} + run: build/mvnd dependency:resolve -DincludeGroupIds="org.apache.kyuubi" -DincludeScope="compile" -DexcludeTransitive=true ${{ matrix.profiles }} continue-on-error: true - name: Install @@ -61,13 +63,13 @@ jobs: if: steps.modules-check.conclusion == 'success' && steps.modules-check.outcome == 'failure' run: | MVN_OPT="-DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip" - build/mvn clean install ${MVN_OPT} -Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.2,tpcds - build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-1 -Pspark-3.1 - build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-kudu,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.3 + build/mvnd clean install ${MVN_OPT} -Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.2,tpcds + build/mvnd clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-1 -Pspark-3.1 + build/mvnd clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-kudu,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.3 - name: Scalastyle with maven id: scalastyle-check - run: build/mvn scalastyle:check ${{ matrix.profiles }} + run: build/mvnd scalastyle:check ${{ matrix.profiles }} - name: Print scalastyle error report if: failure() && steps.scalastyle-check.outcome != 'success' run: >- @@ -81,7 +83,7 @@ jobs: run: | SPOTLESS_BLACK_VERSION=$(build/mvn help:evaluate -Dexpression=spotless.python.black.version -q -DforceStdout) pip install black==$SPOTLESS_BLACK_VERSION - build/mvn spotless:check ${{ matrix.profiles }} -Pspotless-python + build/mvnd spotless:check ${{ matrix.profiles }} -Pspotless-python - name: setup npm uses: actions/setup-node@v3 with: diff --git a/.gitignore b/.gitignore index bbdd246c4..bb2df4fd0 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ .scala_dependencies .settings build/apache-maven* +build/maven-mvnd* build/release/tmp build/scala* build/test diff --git a/.rat-excludes b/.rat-excludes index 86c38ec99..2906e7cea 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -32,6 +32,7 @@ NOTICE* docs/** build/apache-maven-*/** +build/maven-mvnd-*/** build/scala-*/** **/**/operation_logs/**/** **/**/server_operation_logs/**/** diff --git a/build/mvnd b/build/mvnd new file mode 100755 index 000000000..493ee43ad --- /dev/null +++ b/build/mvnd @@ -0,0 +1,134 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Determine the current working directory +_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Preserve the calling directory +_CALLING_DIR="$(pwd)" +# Options used during compilation +_COMPILE_JVM_OPTS="-Xms2g -Xmx2g -XX:ReservedCodeCacheSize=1g -Xss128m" + +if [ "$CI" ]; then + export MAVEN_CLI_OPTS="--no-transfer-progress --errors --fail-fast" +fi + +# Installs any application tarball given a URL, the expected tarball name, +# and, optionally, a checkable binary path to determine if the binary has +# already been installed +## Arg1 - URL +## Arg2 - Tarball Name +## Arg3 - Checkable Binary +install_app() { + local remote_tarball="$1/$2" + local local_tarball="${_DIR}/$2" + local binary="${_DIR}/$3" + + # setup `curl` and `wget` silent options if we're running on Jenkins + local curl_opts="-L" + local wget_opts="" + curl_opts="--progress-bar ${curl_opts}" + wget_opts="--progress=bar:force ${wget_opts}" + + if [ -z "$3" ] || [ ! -f "$binary" ]; then + # check if we already have the tarball + # check if we have curl installed + # download application + rm -f "$local_tarball" + [ ! -f "${local_tarball}" ] && [ "$(command -v curl)" ] && \ + echo "exec: curl ${curl_opts} ${remote_tarball}" 1>&2 && \ + curl ${curl_opts} "${remote_tarball}" > "${local_tarball}" + # if the file still doesn't exist, lets try `wget` and cross our fingers + [ ! -f "${local_tarball}" ] && [ "$(command -v wget)" ] && \ + echo "exec: wget ${wget_opts} ${remote_tarball}" 1>&2 && \ + wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}" + # if both were unsuccessful, exit + [ ! -f "${local_tarball}" ] && \ + echo -n "ERROR: Cannot download $2 with cURL or wget; " && \ + echo "please install manually and try again." && \ + exit 2 + cd "${_DIR}" && tar -xzf "$2" + rm -rf "$local_tarball" + fi +} + +function get_os_type() { + local unameOsOut=$(uname -s) + local osType + case "${unameOsOut}" in + Linux*) osType=linux ;; + Darwin*) osType=darwin ;; + CYGWIN*) osType=windows ;; + MINGW*) osType=windows ;; + *) osType="UNKNOWN:${unameOsOut}" ;; + esac + echo "$osType" +} + +function get_os_arch() { + local unameArchOut="$(uname -m)" + local arch + case "${unameArchOut}" in + x86_64*) arch=amd64 ;; + arm64*) arch=aarch64 ;; + *) arch="UNKNOWN:${unameOsOut}" ;; + esac + echo "$arch" +} + +# Determine the Mvnd version from the root pom.xml file and +# install mvnd under the build/ folder if needed. +function install_mvnd() { + local MVND_VERSION=$(grep "" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}') + local MVN_VERSION=$(grep "" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}') + MVND_BIN="$(command -v mvnd)" + if [ "$MVND_BIN" ]; then + local MVND_DETECTED_VERSION="$(mvnd -v 2>&1 | grep '(mvnd)' | awk '{print $5}')" + local MVN_DETECTED_VERSION="$(mvnd -v 2>&1 | grep 'Apache Maven' | awk 'NR==2 {print $3}')" + fi + # See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers + function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; } + + if [ $(version $MVND_DETECTED_VERSION) -ne $(version $MVND_VERSION) ]; then + local APACHE_MIRROR=${APACHE_MIRROR:-'https://downloads.apache.org'} + local OS_TYPE=$(get_os_type) + local ARCH=$(get_os_arch) + + install_app \ + "${APACHE_MIRROR}/maven/mvnd/${MVND_VERSION}" \ + "maven-mvnd-${MVND_VERSION}-${OS_TYPE}-${ARCH}.tar.gz" \ + "maven-mvnd-${MVND_VERSION}-${OS_TYPE}-${ARCH}/bin/mvnd" + + MVND_BIN="${_DIR}/maven-mvnd-${MVND_VERSION}-${OS_TYPE}-${ARCH}/bin/mvnd" + else + if [ "$(version $MVN_DETECTED_VERSION)" -ne "$(version $MVN_VERSION)" ]; then + echo "Mvnd $MVND_DETECTED_VERSION embedded maven version $MVN_DETECTED_VERSION is not equivalent to $MVN_VERSION required in pom." + exit 1 + fi + fi +} + +install_mvnd + +cd "${_CALLING_DIR}" + +# Set any `mvn` options if not already present +export MAVEN_OPTS=${MAVEN_OPTS:-"$_COMPILE_JVM_OPTS"} + +echo "Using \`mvnd\` from path: $MVND_BIN" 1>&2 +${MVND_BIN} $MAVEN_CLI_OPTS "$@" diff --git a/pom.xml b/pom.xml index 6aae60a8f..b761ef158 100644 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,7 @@ 1.8 3.8.7 + 0.9.0 ${java.version} ${java.version} 2.12.17