[KYUUBI #954] Refactor Dockerfile

<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->

1. As the binary distributions contain all things that we want, we don't need a pre-build spark as a docker base image, use java as the base image instead.

2. Build kyuubi from source code.

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #955 from cfmcgrady/kyuubi-954.

Closes #954

c1c4ed31 [Fu Chen] only compile module kyuubi-compile/kyuubi-integration-tests
af96c7df [Fu Chen] update master.yaml
4c395c22 [Fu Chen] sleep for debug GA
770eef1f [Fu Chen] add local docker registry
749db9c4 [Fu Chen] build docker image
feb86b01 [Fu Chen] fix Dockerfile arg bug
0a599749 [Fu Chen] add .dockerignore
49dc6e31 [Fu Chen] review
dd1d73ce [Fu Chen] refactor Dockerfile

Authored-by: Fu Chen <cfmcgrady@gmail.com>
Signed-off-by: ulysses-you <ulyssesyou18@gmail.com>

.dockerignore

@@ -0,0 +1,6 @@
+externals/*/target
+*/target
+target
+.git
+.idea
+.github

.github/workflows/master.yml

@@ -145,6 +145,16 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+      # https://github.com/docker/build-push-action
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Build Kyuubi Docker Image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: docker/Dockerfile
+          load: true
+          tags: apache/kyuubi:latest
       # from https://github.com/marketplace/actions/setup-minikube-kubernetes-cluster
       - name: Setup Minikube
         uses: manusa/actions-setup-minikube@v2.4.2
@@ -161,7 +171,7 @@ jobs:
       - name: kyuubi pod check
         run: kubectl get pods
       - name: integration tests
-        run: ./build/mvn clean test -Pkubernetes -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test
+        run: ./build/mvn clean test -pl :kyuubi-common_2.12,:kubernetes-integration-tests_2.12 -Pkubernetes -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test
       - name: Upload test logs
         if: failure()
         uses: actions/upload-artifact@v2

docker/Dockerfile

@@ -16,18 +16,30 @@
 #
 
 # Usage:
-#   1. use ./build/dist to make binary distributions of Kyuubi or download a release
-#   2. Untar it and run the docker command below
-#      docker build -f docker/Dockerfile -t yaooqinn/kyuubi:tagname .
+#   Run the docker command below
+#      docker build --build-arg MVN_ARG="-Pspark-3.1,spark-hadoop-3.2" -f docker/Dockerfile -t apache/kyuubi:tagname .
 #   Options:
 #     -f this docker file
 #     -t the target repo and tag name
 #     more options can be found with -h
-# TODO: REPALCE it with offical spark image iff Apache Spark community deploy
-# Or make one after kyuubi to be setup under ASF
 
-ARG SPARK_IMAGE=yaooqinn/spark:3.0.3
-FROM ${SPARK_IMAGE}
+# declare the BASE_IMAGE argument in the first line. for more detail see: https://github.com/moby/moby/issues/38379
+ARG BASE_IMAGE=openjdk:8-jdk
+
+FROM maven:3.6-jdk-8 as builder
+
+ARG MVN_ARG
+
+ADD . /workspace/kyuubi
+WORKDIR /workspace/kyuubi
+
+RUN ./build/dist ${MVN_ARG} && \
+    mv /workspace/kyuubi/dist /opt/kyuubi && \
+    # Removing stuff saves time because docker creates a temporary layer
+    rm -rf ~/.m2 && \
+    rm -rf /workspace/kyuubi
+
+FROM ${BASE_IMAGE}
 
 ARG kyuubi_uid=10009
 
@@ -38,9 +50,11 @@ ENV KYUUBI_LOG_DIR ${KYUUBI_HOME}/logs
 ENV KYUUBI_PID_DIR ${KYUUBI_HOME}/pid
 ENV KYUUBI_WORK_DIR_ROOT ${KYUUBI_HOME}/work
 
+COPY --from=builder /opt/kyuubi ${KYUUBI_HOME}
+
 RUN set -ex && \
-    sed -i 's/http:\/\/deb.\(.*\)/https:\/\/deb.\1/g' /etc/apt/sources.list && \
     apt-get update && \
+    DEBIAN_FRONTEND=noninteractive \
     apt install -y bash tini libc6 libpam-modules krb5-user libnss3 procps && \
     useradd -u ${kyuubi_uid} -g root kyuubi && \
     mkdir -p ${KYUUBI_HOME} ${KYUUBI_LOG_DIR} ${KYUUBI_PID_DIR} ${KYUUBI_WORK_DIR_ROOT} && \
@@ -48,11 +62,6 @@ RUN set -ex && \
     chmod a+rwx -R ${KYUUBI_WORK_DIR_ROOT} && \
     rm -rf /var/cache/apt/*
 
-COPY bin ${KYUUBI_HOME}/bin
-COPY conf ${KYUUBI_HOME}/conf
-COPY jars ${KYUUBI_HOME}/jars
-COPY externals/engines/spark ${KYUUBI_HOME}/externals/engines/spark
-
 WORKDIR ${KYUUBI_HOME}
 
 CMD [ "./bin/kyuubi", "run" ]

kubernetes/integration-tests/test-k8s.yaml

@@ -25,8 +25,7 @@ spec:
   serviceAccountName: kyuubi
   containers:
     - name: kyuubi-server
-      # TODO: replace this with the official repo
-      image: yaooqinn/kyuubi:latest
+      image: apache/kyuubi:latest
       imagePullPolicy: IfNotPresent
       env:
         - name: KYUUBI_JAVA_OPTS