From c83d498d99c27851b1c3aa84a4cb096697e12706 Mon Sep 17 00:00:00 2001 From: Yi Chen Date: Tue, 27 May 2025 16:32:46 -0700 Subject: [PATCH] [CELEBORN-1528][HELM] Use volume claim template to support various storage backend ### What changes were proposed in this pull request? - Add support for configuring volumeClaimTemplates by adding new values `master.volumeClaimTemplates` and `worker.volumeClaimTemplates`. ### Why are the changes needed? Use volume claim template to support various storage backend. ### Does this PR introduce _any_ user-facing change? Yes. New Helm values `master.volumeClaimTemplates` and `worker.volumeClaimTemplates` are added. ### How was this patch tested? ```bash helm unittest charts/celeborn --file "tests/**/*_test.yaml" --strict --debug ``` Closes #3277 from ChenYi015/helm/volume-claim-templates. Authored-by: Yi Chen Signed-off-by: Wang, Fei --- charts/celeborn/ci/values.yaml | 26 ------ .../templates/master/statefulset.yaml | 32 ++++--- .../templates/worker/statefulset.yaml | 31 ++++--- .../tests/master/statefulset_test.yaml | 52 +++++++++++ .../tests/worker/statefulset_test.yaml | 52 +++++++++++ charts/celeborn/values.yaml | 90 +++++++++++++++---- 6 files changed, 220 insertions(+), 63 deletions(-) diff --git a/charts/celeborn/ci/values.yaml b/charts/celeborn/ci/values.yaml index c6c77dbcc..1f15bf195 100644 --- a/charts/celeborn/ci/values.yaml +++ b/charts/celeborn/ci/values.yaml @@ -28,16 +28,6 @@ image: # -- Image pull policy pullPolicy: IfNotPresent -service: - # -- Specifies service type - type: ClusterIP - # -- Specifies service port - port: 9097 - -cluster: - # -- Specifies Kubernetes cluster name - name: cluster - celeborn: celeborn.metrics.enabled: false celeborn.worker.storage.dirs: /mnt/disk1:disktype=SSD:capacity=1Gi,/mnt/disk2:disktype=SSD:capacity=1Gi @@ -134,26 +124,10 @@ worker: podMonitor: # -- Specifies whether to enable creating pod monitors for Celeborn pods enable: false - # -- Specifies pod metrics endpoint - podMetricsEndpoint: - # Specifies scheme - scheme: http - # Specifies scrape interval - interval: 5s - # Specifies port name - portName: metrics serviceAccount: # -- Specifies whether to create a service account for Celeborn create: false - # -- Specifies the name of the service account - name: default rbac: create: false - roleName: default - roleBindingName: default - rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["create", "list", "delete"] diff --git a/charts/celeborn/templates/master/statefulset.yaml b/charts/celeborn/templates/master/statefulset.yaml index 335fd2671..2ccf49e72 100644 --- a/charts/celeborn/templates/master/statefulset.yaml +++ b/charts/celeborn/templates/master/statefulset.yaml @@ -23,10 +23,14 @@ metadata: {{- include "celeborn.master.labels" . | nindent 4 }} spec: replicas: {{ .Values.master.replicas }} - serviceName: {{ include "celeborn.master.service.name" . }} selector: matchLabels: {{- include "celeborn.master.selectorLabels" . | nindent 6 }} + {{- with .Values.master.volumeClaimTemplates }} + volumeClaimTemplates: + {{- toYaml . | nindent 2 }} + {{- end }} + serviceName: {{ include "celeborn.master.service.name" . }} template: metadata: labels: @@ -37,7 +41,18 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - serviceAccountName: {{ include "celeborn.serviceAccountName" . }} + {{- /* Add an init container to chown mount paths of Celeborn master volumes if necessary. */}} + {{- $paths := list }} + {{- range $volumeMount := .Values.master.volumeMounts }} + {{- range $volume := $.Values.master.volumes }} + {{- if eq $volume.name $volumeMount.name }} + {{- if or $volume.hostPath $volume.emptyDir }} + {{- $paths = append $paths $volumeMount.mountPath }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- if $paths }} initContainers: - name: chown-celeborn-master-volume image: {{ include "celeborn.image" . }} @@ -48,14 +63,8 @@ spec: - chown - -R - {{ .Values.master.podSecurityContext.runAsUser | default 10006 }}:{{ .Values.master.podSecurityContext.runAsGroup | default 10006 }} - {{- range $volumeMount := .Values.master.volumeMounts }} - {{- range $volume := $.Values.master.volumes }} - {{- if eq $volume.name $volumeMount.name }} - {{- if or $volume.hostPath $volume.emptyDir }} - - {{ $volumeMount.mountPath }} - {{- end }} - {{- end }} - {{- end }} + {{- range $path := $paths }} + - {{ $path }} {{- end }} {{- with .Values.master.volumeMounts }} volumeMounts: @@ -67,6 +76,7 @@ spec: {{- end }} securityContext: runAsUser: 0 + {{- end }} containers: - name: {{ .Chart.Name }} image: {{ include "celeborn.image" . }} @@ -161,8 +171,10 @@ spec: {{- with .Values.master.hostNetwork }} hostNetwork: {{ . }} {{- end }} + serviceAccountName: {{ include "celeborn.serviceAccountName" . }} {{- with .Values.master.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} {{- end }} terminationGracePeriodSeconds: 30 + diff --git a/charts/celeborn/templates/worker/statefulset.yaml b/charts/celeborn/templates/worker/statefulset.yaml index d3abc7009..b8441a3bd 100644 --- a/charts/celeborn/templates/worker/statefulset.yaml +++ b/charts/celeborn/templates/worker/statefulset.yaml @@ -23,10 +23,14 @@ metadata: {{- include "celeborn.worker.labels" . | nindent 4 }} spec: replicas: {{ .Values.worker.replicas }} - serviceName: {{ include "celeborn.worker.service.name" . }} selector: matchLabels: {{- include "celeborn.worker.selectorLabels" . | nindent 6 }} + {{- with .Values.worker.volumeClaimTemplates }} + volumeClaimTemplates: + {{- toYaml . | nindent 2 }} + {{- end }} + serviceName: {{ include "celeborn.worker.service.name" . }} template: metadata: labels: @@ -37,7 +41,18 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - serviceAccountName: {{ include "celeborn.serviceAccountName" . }} + {{- /* Add an init container to chown mount paths of Celeborn workers volume if necessary. */}} + {{- $paths := list }} + {{- range $volumeMount := .Values.worker.volumeMounts }} + {{- range $volume := $.Values.worker.volumes }} + {{- if eq $volume.name $volumeMount.name }} + {{- if or $volume.hostPath $volume.emptyDir }} + {{- $paths = append $paths $volumeMount.mountPath }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- if $paths }} initContainers: - name: chown-celeborn-worker-volume image: {{ include "celeborn.image" . }} @@ -48,14 +63,8 @@ spec: - chown - -R - {{ .Values.worker.podSecurityContext.runAsUser | default 10006 }}:{{ .Values.worker.podSecurityContext.runAsGroup | default 10006 }} - {{- range $volumeMount := .Values.worker.volumeMounts }} - {{- range $volume := $.Values.worker.volumes }} - {{- if eq $volume.name $volumeMount.name }} - {{- if or $volume.hostPath $volume.emptyDir }} - - {{ $volumeMount.mountPath }} - {{- end }} - {{- end }} - {{- end }} + {{- range $path := $paths }} + - {{ $path }} {{- end }} {{- with .Values.worker.volumeMounts }} volumeMounts: @@ -67,6 +76,7 @@ spec: {{- end }} securityContext: runAsUser: 0 + {{- end }} containers: - name: {{ .Chart.Name }} image: {{ include "celeborn.image" . }} @@ -160,6 +170,7 @@ spec: {{- with .Values.worker.hostNetwork }} hostNetwork: {{ . }} {{- end }} + serviceAccountName: {{ include "celeborn.serviceAccountName" . }} {{- with .Values.worker.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} diff --git a/charts/celeborn/tests/master/statefulset_test.yaml b/charts/celeborn/tests/master/statefulset_test.yaml index 2fa9bc581..981a80d6c 100644 --- a/charts/celeborn/tests/master/statefulset_test.yaml +++ b/charts/celeborn/tests/master/statefulset_test.yaml @@ -376,3 +376,55 @@ tests: - equal: path: spec.template.spec.securityContext.fsGroup value: 3000 + + - it: Should add volume claim templates if `master.volumeClaimTemplates` is set + set: + master: + volumeClaimTemplates: + - metadata: + name: test-volume-claim-template-1 + spec: + accessModes: + - ReadWriteMany + resources: + request: + storage: 100Gi + limits: + storage: 100Gi + - metadata: + name: test-volume-claim-template-2 + spec: + accessModes: + - ReadWriteOnce + resources: + request: + storage: 200Gi + limits: + storage: 200Gi + asserts: + - contains: + path: spec.volumeClaimTemplates + content: + metadata: + name: test-volume-claim-template-1 + spec: + accessModes: + - ReadWriteMany + resources: + request: + storage: 100Gi + limits: + storage: 100Gi + - contains: + path: spec.volumeClaimTemplates + content: + metadata: + name: test-volume-claim-template-2 + spec: + accessModes: + - ReadWriteOnce + resources: + request: + storage: 200Gi + limits: + storage: 200Gi diff --git a/charts/celeborn/tests/worker/statefulset_test.yaml b/charts/celeborn/tests/worker/statefulset_test.yaml index 8440310ed..6ec98a091 100644 --- a/charts/celeborn/tests/worker/statefulset_test.yaml +++ b/charts/celeborn/tests/worker/statefulset_test.yaml @@ -375,3 +375,55 @@ tests: - equal: path: spec.template.spec.securityContext.fsGroup value: 3000 + + - it: Should add volume claim templates if `worker.volumeClaimTemplates` is set + set: + worker: + volumeClaimTemplates: + - metadata: + name: test-volume-claim-template-1 + spec: + accessModes: + - ReadWriteMany + resources: + request: + storage: 100Gi + limits: + storage: 100Gi + - metadata: + name: test-volume-claim-template-2 + spec: + accessModes: + - ReadWriteOnce + resources: + request: + storage: 200Gi + limits: + storage: 200Gi + asserts: + - contains: + path: spec.volumeClaimTemplates + content: + metadata: + name: test-volume-claim-template-1 + spec: + accessModes: + - ReadWriteMany + resources: + request: + storage: 100Gi + limits: + storage: 100Gi + - contains: + path: spec.volumeClaimTemplates + content: + metadata: + name: test-volume-claim-template-2 + spec: + accessModes: + - ReadWriteOnce + resources: + request: + storage: 200Gi + limits: + storage: 200Gi diff --git a/charts/celeborn/values.yaml b/charts/celeborn/values.yaml index 5dcd85d5d..81c264479 100644 --- a/charts/celeborn/values.yaml +++ b/charts/celeborn/values.yaml @@ -195,13 +195,13 @@ master: # -- Tolerations for Celeborn master pods. tolerations: - # - key: key1 - # operator: Equal - # value: value1 - # effect: NoSchedule - # - key: key2 - # operator: Exists - # effect: NoSchedule + # - key: key1 + # operator: Equal + # value: value1 + # effect: NoSchedule + # - key: key2 + # operator: Exists + # effect: NoSchedule # Priority class for Celeborn master pods. priorityClass: @@ -227,6 +227,19 @@ master: # The group ID to use when modifying the ownership and permissions of the mounted volumes. fsGroup: 10006 + # -- Volume claim templates for Celeborn master statefulset. + volumeClaimTemplates: + # - metadata: + # name: celeborn-ratis + # spec: + # accessModes: + # - ReadWriteOnce + # resources: + # requests: + # storage: 100Gi + # limits: + # storage: 100Gi + worker: # -- Number of Celeborn worker replicas to deploy, should less than node number. replicas: 5 @@ -328,13 +341,13 @@ worker: # -- Tolerations for Celeborn worker pods. tolerations: - # - key: key1 - # operator: Equal - # value: value1 - # effect: NoSchedule - # - key: key2 - # operator: Exists - # effect: NoSchedule + # - key: key1 + # operator: Equal + # value: value1 + # effect: NoSchedule + # - key: key2 + # operator: Exists + # effect: NoSchedule # Priority class for Celeborn worker pods. priorityClass: @@ -360,6 +373,49 @@ worker: # The group ID to use when modifying the ownership and permissions of the mounted volumes. fsGroup: 10006 + # -- Volume claim templates for Celeborn worker pods. + volumeClaimTemplates: + # - metadata: + # name: disk1 + # spec: + # accessModes: + # - ReadWriteOnce + # resources: + # requests: + # storage: 100Gi + # limits: + # storage: 100Gi + # - metadata: + # name: disk2 + # spec: + # accessModes: + # - ReadWriteOnce + # resources: + # requests: + # storage: 100Gi + # limits: + # storage: 100Gi + # - metadata: + # name: disk3 + # spec: + # accessModes: + # - ReadWriteOnce + # resources: + # requests: + # storage: 100Gi + # limits: + # storage: 100Gi + # - metadata: + # name: disk4 + # spec: + # accessModes: + # - ReadWriteOnce + # resources: + # requests: + # storage: 100Gi + # limits: + # storage: 100Gi + podMonitor: # -- Specifies whether to enable creating pod monitors for Celeborn pods enable: true @@ -383,6 +439,6 @@ rbac: roleName: default roleBindingName: default rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["create", "list", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "list", "delete"]