[KYUUBI #4147] [K8S][HELM] Add configuration support for multiple frontends to helm chart

### _Why are the changes needed?_ The changes allow to configure multiple frontends in the helm chart. Notes: - Removed unused code - `server.confDir` renamed to `kyuubiConfDir` - `server.conf` renamed to `kyuubiConf` - `livenessProbe` and `readinessProbe` changed to execute `bin/kyuubi status` which actually prints `Kyuubi is not running`. The issue needs to be reviewed. Also `livenessProbe` and `readinessProbe` should be revisited in the next PR. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4147 from dnskr/helm_frontend_configs. Closes #4147 ab10dd15b [dnskr] [K8S][HELM] Address PR comments 65e1f593e [dnskr] [K8S][HELM] Address some PR comments 36b5fc64f [dnskr] [K8S][HELM] Add configuration support for multiple frontends to helm chart Authored-by: dnskr <dnskrv88@gmail.com> Signed-off-by: Cheng Pan <chengpan@apache.org>
2023-02-06 10:11:00 +08:00 · 2023-02-06 10:11:00 +08:00 · 8a99750a20
commit 8a99750a20
parent 4dd00c1499
6 changed files with 128 additions and 79 deletions
--- a/charts/kyuubi/templates/NOTES.txt
+++ b/charts/kyuubi/templates/NOTES.txt
@ -15,7 +15,29 @@
 # limitations under the License.
 #

-Get kyuubi expose URL by running these commands:
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "kyuubi.fullname" . }}-nodeport)
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo $NODE_IP:$NODE_PORT
+The chart has been installed!
+
+In order to check the release status, use:
+  helm status {{ .Release.Name }} -n {{ .Release.Namespace }}
+    or for more detailed info
+  helm get all {{ .Release.Name }} -n {{ .Release.Namespace }}
+
+************************
+******* Services *******
+************************
+{{- range $name, $frontend := .Values.server }}
+{{- if $frontend.enabled }}
+{{ $name | snakecase | upper }}:
+- To access {{ $.Release.Name }}-{{ $name | kebabcase }} service within the cluster, use the following URL:
+    {{ $.Release.Name }}-{{ $name | kebabcase }}.{{ $.Release.Namespace }}.svc.cluster.local
+- To access {{ $.Release.Name }}-{{ $name | kebabcase }} service from outside the cluster for debugging, run the following command:
+    kubectl port-forward svc/{{ $.Release.Name }}-{{ $name | kebabcase }} {{ tpl $frontend.service.port $ }}:{{ tpl $frontend.service.port $ }} -n {{ $.Release.Namespace }}
+  and use 127.0.0.1:{{ tpl $frontend.service.port $ }}
+{{- if eq $frontend.service.type "NodePort" }}
+- To access {{ $.Release.Name }}-{{ $name | kebabcase }} service from outside the cluster through configured NodePort, run the following commands:
+    export NODE_PORT=$(kubectl get service {{ $.Release.Name }}-{{ $name | kebabcase }} -n {{ $.Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}")
+    export NODE_IP=$(kubectl get nodes -n {{ $.Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+    echo http://$NODE_IP:$NODE_PORT
+{{- end }}
+{{- end }}
+{{- end }}
--- a/charts/kyuubi/templates/_helpers.tpl
+++ b/charts/kyuubi/templates/_helpers.tpl
@ -16,33 +16,18 @@
 */}}

 {{/*
-Expand the name of the chart.
+A comma separated string of enabled frontend protocols, e.g. "REST,THRIFT_BINARY".
+For details, see 'kyuubi.frontend.protocols': https://kyuubi.readthedocs.io/en/master/deployment/settings.html#frontend
 */}}
-{{- define "kyuubi.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- define "kyuubi.frontend.protocols" -}}
+{{- $protocols := list }}
+{{- range $name, $frontend := .Values.server }}
+  {{- if $frontend.enabled }}
+    {{- $protocols = $name | snakecase | upper | append $protocols }}
+  {{- end }}
 {{- end }}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "kyuubi.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- if not $protocols }}
+  {{ fail "At least one frontend protocol must be enabled!" }}
 {{- end }}
+{{- $protocols |  join "," }}
 {{- end }}
-{{- end }}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "kyuubi.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
--- a/charts/kyuubi/templates/kyuubi-configmap.yaml
+++ b/charts/kyuubi/templates/kyuubi-configmap.yaml
@ -26,22 +26,26 @@ metadata:
    app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
 data:
-  {{- with .Values.server.conf.kyuubiEnv }}
+  {{- with .Values.kyuubiConf.kyuubiEnv }}
  kyuubi-env.sh: |
    #!/usr/bin/env bash
    {{- tpl . $ | nindent 4 }}
  {{- end }}
  kyuubi-defaults.conf: |
    ## Helm chart provided Kyuubi configurations
-    kyuubi.frontend.bind.host={{ .Values.server.bind.host }}
-    kyuubi.frontend.bind.port={{ .Values.server.bind.port }}
    kyuubi.kubernetes.namespace={{ .Release.Namespace }}
+    kyuubi.frontend.bind.host=localhost
+    kyuubi.frontend.thrift.binary.bind.port={{ .Values.server.thriftBinary.port }}
+    kyuubi.frontend.thrift.http.bind.port={{ .Values.server.thriftHttp.port }}
+    kyuubi.frontend.rest.bind.port={{ .Values.server.rest.port }}
+    kyuubi.frontend.mysql.bind.port={{ .Values.server.mysql.port }}
+    kyuubi.frontend.protocols={{ include "kyuubi.frontend.protocols" . }}

    ## User provided Kyuubi configurations
-    {{- with .Values.server.conf.kyuubiDefaults }}
+    {{- with .Values.kyuubiConf.kyuubiDefaults }}
    {{- tpl . $ | nindent 4 }}
    {{- end }}
-  {{- with .Values.server.conf.log4j2 }}
+  {{- with .Values.kyuubiConf.log4j2 }}
  log4j2.xml: |
    {{- tpl . $ | nindent 4 }}
  {{- end }}
--- a/charts/kyuubi/templates/kyuubi-deployment.yaml
+++ b/charts/kyuubi/templates/kyuubi-deployment.yaml
@ -57,13 +57,16 @@ spec:
          envFrom: {{- tpl (toYaml .) $ | nindent 12 }}
          {{- end }}
          ports:
-            - name: frontend-port
-              containerPort: {{ .Values.server.bind.port }}
-              protocol: TCP
+            {{- range $name, $frontend := .Values.server }}
+            {{- if $frontend.enabled }}
+            - name: {{ $name | kebabcase }}
+              containerPort: {{ $frontend.port }}
+            {{- end }}
+            {{- end }}
          {{- if .Values.probe.liveness.enabled }}
          livenessProbe:
-            tcpSocket:
-              port: {{ .Values.server.bind.port }}
+            exec:
+              command: ["/bin/bash", "-c", "bin/kyuubi status"]
            initialDelaySeconds: {{ .Values.probe.liveness.initialDelaySeconds }}
            periodSeconds: {{ .Values.probe.liveness.periodSeconds }}
            timeoutSeconds: {{ .Values.probe.liveness.timeoutSeconds }}
@ -72,8 +75,8 @@ spec:
          {{- end }}
          {{- if .Values.probe.readiness.enabled }}
          readinessProbe:
-            tcpSocket:
-              port: {{ .Values.server.bind.port }}
+            exec:
+              command: ["/bin/bash", "-c", "$KYUUBI_HOME/bin/kyuubi status"]
            initialDelaySeconds: {{ .Values.probe.readiness.initialDelaySeconds }}
            periodSeconds: {{ .Values.probe.readiness.periodSeconds }}
            timeoutSeconds: {{ .Values.probe.readiness.timeoutSeconds }}
@ -85,7 +88,7 @@ spec:
          {{- end }}
          volumeMounts:
            - name: conf
-              mountPath: {{ .Values.server.confDir }}
+              mountPath: {{ .Values.kyuubiConfDir }}
            {{- with .Values.volumeMounts }}
              {{- tpl (toYaml .) $ | nindent 12 }}
            {{- end }}
--- a/charts/kyuubi/templates/kyuubi-service.yaml
+++ b/charts/kyuubi/templates/kyuubi-service.yaml
@ -15,27 +15,34 @@
 # limitations under the License.
 #

+{{- range $name, $frontend := .Values.server }}
+{{- if $frontend.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ .Release.Name }}
+  name: {{ $.Release.Name }}-{{ $name | kebabcase }}
  labels:
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app.kubernetes.io/name: {{ .Chart.Name }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-  {{- with .Values.service.annotations }}
+    helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }}
+    app.kubernetes.io/name: {{ $.Chart.Name }}
+    app.kubernetes.io/instance: {{ $.Release.Name }}
+    app.kubernetes.io/version: {{ $.Values.image.tag | default $.Chart.AppVersion | quote }}
+    app.kubernetes.io/managed-by: {{ $.Release.Service }}
+  {{- with $frontend.service.annotations }}
  annotations:
    {{- toYaml . | nindent 4 }}
  {{- end }}
 spec:
+  type: {{ $frontend.service.type }}
  ports:
-    - name: http
-      nodePort: {{ .Values.service.port }}
-      port: {{ .Values.server.bind.port }}
-      protocol: TCP
-  type: {{ .Values.service.type }}
+    - name: {{ $name | kebabcase }}
+      port: {{ tpl $frontend.service.port $ }}
+      targetPort: {{ $frontend.port }}
+      {{- if and (eq $frontend.service.type "NodePort") ($frontend.service.nodePort) }}
+      nodePort: {{ $frontend.service.nodePort }}
+      {{- end }}
  selector:
-    app.kubernetes.io/name: {{ .Chart.Name }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/name: {{ $.Chart.Name }}
+    app.kubernetes.io/instance: {{ $.Release.Name }}
+---
+{{- end }}
+{{- end }}
--- a/charts/kyuubi/values.yaml
+++ b/charts/kyuubi/values.yaml
@ -58,22 +58,59 @@ probe:
    successThreshold: 1

 server:
-  bind:
-    host: 0.0.0.0
+  # Thrift Binary protocol (HiveServer2 compatible)
+  thriftBinary:
+    enabled: true
    port: 10009
-  confDir: /opt/kyuubi/conf
-  conf:
-    # The value (templated string) is used for kyuubi-env.sh file
-    # See https://kyuubi.apache.org/docs/latest/deployment/settings.html#environments for more details
-    kyuubiEnv: ~
+    service:
+      type: ClusterIP
+      port: "{{ .Values.server.thriftBinary.port }}"
+      nodePort: ~
+      annotations: {}

-    # The value (templated string) is used for kyuubi-defaults.conf file
-    # See https://kyuubi.apache.org/docs/latest/deployment/settings.html#kyuubi-configurations for more details
-    kyuubiDefaults: ~
+  # Thrift HTTP protocol (HiveServer2 compatible)
+  thriftHttp:
+    enabled: false
+    port: 10010
+    service:
+      type: ClusterIP
+      port: "{{ .Values.server.thriftHttp.port }}"
+      nodePort: ~
+      annotations: {}

-    # The value (templated string) is used for log4j2.xml file
-    # See https://kyuubi.apache.org/docs/latest/deployment/settings.html#logging for more details
-    log4j2: ~
+  # REST API protocol (experimental)
+  rest:
+    enabled: false
+    port: 10099
+    service:
+      type: ClusterIP
+      port: "{{ .Values.server.rest.port }}"
+      nodePort: ~
+      annotations: {}
+
+  # MySQL compatible text protocol (experimental)
+  mysql:
+    enabled: false
+    port: 3309
+    service:
+      type: ClusterIP
+      port: "{{ .Values.server.mysql.port }}"
+      nodePort: ~
+      annotations: {}
+
+kyuubiConfDir: /opt/kyuubi/conf
+kyuubiConf:
+  # The value (templated string) is used for kyuubi-env.sh file
+  # See https://kyuubi.apache.org/docs/latest/deployment/settings.html#environments for more details
+  kyuubiEnv: ~
+
+  # The value (templated string) is used for kyuubi-defaults.conf file
+  # See https://kyuubi.apache.org/docs/latest/deployment/settings.html#kyuubi-configurations for more details
+  kyuubiDefaults: ~
+
+  # The value (templated string) is used for log4j2.xml file
+  # See https://kyuubi.apache.org/docs/latest/deployment/settings.html#logging for more details
+  log4j2: ~

 # Environment variables (templated)
 env: []
@ -89,15 +126,6 @@ initContainers: []
 # Additional containers for Kyuubi pod (templated)
 containers: []

-service:
-  type: NodePort
-  # The default port limit of kubernetes is 30000-32767
-  # to change:
-  #   vim kube-apiserver.yaml (usually under path: /etc/kubernetes/manifests/)
-  #   add or change line 'service-node-port-range=1-32767' under kube-apiserver
-  port: 30009
-  annotations: {}
-
 resources: {}
  # Used to specify resource, default unlimited.
  # If you do want to specify resources: