Merge pull request #5187 from irbekrm/cleanup_kind_config
Clean up kind config
This commit is contained in:
jetstack-bot
GitHub
commit
bbf2b58a5e
@ -7,11 +7,13 @@ apiVersion: kind.x-k8s.io/v1alpha4
|
||||
kind: Cluster
|
||||
kubeadmConfigPatches:
|
||||
- |
|
||||
# config generated by kind
|
||||
apiVersion: kubeadm.k8s.io/v1beta2
|
||||
kind: ClusterConfiguration
|
||||
metadata:
|
||||
name: config
|
||||
etcd:
|
||||
local:
|
||||
extraArgs:
|
||||
unsafe-no-fsync: "true"
|
||||
networking:
|
||||
serviceSubnet: 10.0.0.0/16
|
||||
nodes:
|
||||
@ -62,17 +62,14 @@ fi
|
||||
|
||||
# Create the kind cluster
|
||||
$KIND_BIN create cluster \
|
||||
--config "${SCRIPT_ROOT}/config/v1beta2.yaml" \
|
||||
--config "${SCRIPT_ROOT}/config/kind.yaml" \
|
||||
--image "${KIND_IMAGE}" \
|
||||
--name "${KIND_CLUSTER_NAME}"
|
||||
|
||||
# kubectl cluster-info dump does not return output in format that could be
|
||||
# easily parsed with a json or yaml parser.
|
||||
service_ip_prefix=$(kubectl cluster-info dump | grep ip-range | head -n1 | cut -d= -f2 | cut -d. -f1,2,3)
|
||||
|
||||
# Get the current config
|
||||
original_coredns_config=$(kubectl get -ogo-template='{{.data.Corefile}}' -n=kube-system configmap/coredns)
|
||||
additional_coredns_config=$'example.com:53 {\n forward . '$service_ip_prefix$'.16\n}\n'
|
||||
additional_coredns_config=$'example.com:53 {\n forward . '$SERVICE_IP_PREFIX$'.16\n}\n'
|
||||
echo "Original CoreDNS config:"
|
||||
echo "${original_coredns_config}"
|
||||
# Patch it
|
||||
|
||||
@ -29,9 +29,7 @@ export K8S_VERSION=${K8S_VERSION:-1.24}
|
||||
export OPENSHIFT_VERSION=${OPENSHIFT_VERSION:-"3.11"}
|
||||
export IS_OPENSHIFT="${IS_OPENSHIFT:-"false"}"
|
||||
export OPENSHIFT_VERSION="${OPENSHIFT_VERSION:-"3.11"}"
|
||||
# kubectl cluster-info dump does not return output in format that could be
|
||||
# easily parsed with a json or yaml parser.
|
||||
export SERVICE_IP_PREFIX=$(kubectl cluster-info dump | grep ip-range | head -n1 | cut -d= -f2 | cut -d. -f1,2,3)
|
||||
export SERVICE_IP_PREFIX="10.0.0"
|
||||
export DNS_SERVER="${SERVICE_IP_PREFIX}.16"
|
||||
export INGRESS_IP="${SERVICE_IP_PREFIX}.15"
|
||||
export GATEWAY_IP="${SERVICE_IP_PREFIX}.14"
|
||||
|
||||
@ -138,24 +138,11 @@ setup_kind() {
|
||||
|
||||
# (1) Does the kind cluster already exist?
|
||||
if ! kind get clusters -q | grep -q "^$kind_cluster_name\$"; then
|
||||
# Kind with Kubernetes v1.18 and v1.19 need different config files
|
||||
config="config_etcd_no_fsync"
|
||||
if [[ "$k8s_version" == 1.18 || "$k8s_version" == 1.19 ]]; then
|
||||
config="config"
|
||||
fi
|
||||
trace kind create cluster --config "make/config/kind/${config}.yaml" \
|
||||
trace kind create cluster --config "make/config/kind/cluster.yaml" \
|
||||
--image "$image" \
|
||||
--name "$kind_cluster_name"
|
||||
fi
|
||||
|
||||
# Sleep to wait for cluster-info to be up to date
|
||||
# TODO (irbekrm): loop and repeatedly check for the info to become available instead of sleeping
|
||||
echo "Waiting for kubectl cluster-info to be up to date..."
|
||||
sleep 20
|
||||
# kubectl cluster-info dump does not return output in format that could be
|
||||
# easily parsed with a json or yaml parser.
|
||||
service_ip_prefix=$(kubectl cluster-info dump | grep ip-range | head -n 1 | cut -d= -f2 | cut -d. -f1,2,3)
|
||||
|
||||
# (2) Does the kube config contain the context for this existing kind cluster?
|
||||
if ! kubectl config get-contexts -oname 2>/dev/null | grep -q "^kind-${kind_cluster_name}$"; then
|
||||
printf "${red}${redcross}Error${end}: the kind cluster ${yel}$kind_cluster_name${end} already exists, but your current kube config does not contain the context ${yel}kind-$kind_cluster_name${end}. Run:\n" >&2
|
||||
@ -191,7 +178,7 @@ setup_kind() {
|
||||
|
||||
# (6) Has the Corefile been patched?
|
||||
corefile=$(kubectl get -ogo-template='{{.data.Corefile}}' -n=kube-system configmap/coredns)
|
||||
to_be_appended=$'example.com:53 {\n forward . '$service_ip_prefix$'.16\n}\n'
|
||||
to_be_appended=$'example.com:53 {\n forward . '$SERVICE_IP_PREFIX$'.16\n}\n'
|
||||
if ! grep -q --null-data -F "$(tr -d $'\n' <<<"$to_be_appended")" <(tr -d $'\n' <<<"$corefile"); then
|
||||
kubectl create configmap -oyaml coredns --dry-run=client --from-literal=Corefile="$(printf '%s\n%s' "$corefile" "$to_be_appended")" \
|
||||
| kubectl apply -n kube-system -f - >/dev/null
|
||||
|
||||
33
make/config/kind/cluster.yaml
Normal file
33
make/config/kind/cluster.yaml
Normal file
@ -0,0 +1,33 @@
|
||||
# This kind config is unversioned as we are using it to create kind clusters with a range of different versions of Kubernetes. Having the config unversioned allows kind to choose a suitable API version, see https://github.com/kubernetes-sigs/kind/issues/1839#issuecomment-1148968204
|
||||
# When making changes to this file ensure that the change works with all currently supported versions of Kubernetes.
|
||||
#
|
||||
# When making any changes to this file, make sure it works with all supported versions of Kubernetes.
|
||||
# The --unsafe-no-fsync decreases the load on the pod's filesystem [1], which in
|
||||
# turn decreases the end-to-end tests duration. It is OK for us to use this flag
|
||||
# because we are using a one-node etcd cluster. The fsync feature is used for
|
||||
# the raft consensus protocol and is thus only useful when using 3 or more etcd
|
||||
# nodes.
|
||||
#
|
||||
# [1]: https://github.com/etcd-io/etcd/pull/11946 [2]:
|
||||
# https://etcd.io/docs/v3.5/tuning/#disk [3]: https://etcd.io/docs/v3.5/faq/
|
||||
#
|
||||
# Custom service subnet allows us to have a fixed/predictable clusterIP for
|
||||
# various addon Services such as ingress-nginx, Gateway etc.
|
||||
# TODO: parameterize the service subnet range instead of hardcoding it so that it is defined in one place only
|
||||
# It could be interpolated with ytt like for addons i.e https://github.com/cert-manager/cert-manager/blob/134398e939bb2b1401697eaf589405ad469cd609/make/e2e-setup.mk#L379
|
||||
|
||||
apiVersion: kind.x-k8s.io/v1alpha4
|
||||
kind: Cluster
|
||||
kubeadmConfigPatches:
|
||||
- |
|
||||
kind: ClusterConfiguration
|
||||
metadata:
|
||||
name: config
|
||||
etcd:
|
||||
local:
|
||||
extraArgs:
|
||||
unsafe-no-fsync: "true"
|
||||
networking:
|
||||
serviceSubnet: 10.0.0.0/16
|
||||
nodes:
|
||||
- role: control-plane
|
||||
@ -1,13 +0,0 @@
|
||||
# Kind config for kind 1.18 and 1.19 node images. Use
|
||||
# make/config/kind/config_etcd_no_fsync.yaml for newer versions of kind as they
|
||||
# support --unsafe-no-fsync etcd flag, which improves test speed.
|
||||
apiVersion: kind.x-k8s.io/v1alpha4
|
||||
kind: Cluster
|
||||
kubeadmConfigPatches:
|
||||
- |
|
||||
apiVersion: kubeadm.k8s.io/v1beta2
|
||||
kind: ClusterConfiguration
|
||||
metadata:
|
||||
name: config
|
||||
nodes:
|
||||
- role: control-plane
|
||||
@ -1,23 +0,0 @@
|
||||
# The --unsafe-no-fsync decreases the load on the pod's filesystem [1],
|
||||
# which in turn decreases the end-to-end tests duration. It is OK for us to
|
||||
# use this flag because we are using a one-node etcd cluster. The fsync
|
||||
# feature is used for the raft consensus protocol and is thus only useful
|
||||
# when using 3 or more etcd nodes.
|
||||
#
|
||||
# [1]: https://github.com/etcd-io/etcd/pull/11946
|
||||
# [2]: https://etcd.io/docs/v3.5/tuning/#disk
|
||||
# [3]: https://etcd.io/docs/v3.5/faq/
|
||||
apiVersion: kind.x-k8s.io/v1alpha4
|
||||
kind: Cluster
|
||||
kubeadmConfigPatches:
|
||||
- |
|
||||
apiVersion: kubeadm.k8s.io/v1beta3
|
||||
kind: ClusterConfiguration
|
||||
metadata:
|
||||
name: config
|
||||
etcd:
|
||||
local:
|
||||
extraArgs:
|
||||
unsafe-no-fsync: "true"
|
||||
nodes:
|
||||
- role: control-plane
|
||||
@ -20,6 +20,11 @@ set -o nounset
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
export SERVICE_IP_PREFIX="10.0.0"
|
||||
export DNS_SERVER="${SERVICE_IP_PREFIX}.16"
|
||||
export INGRESS_IP="${SERVICE_IP_PREFIX}.15"
|
||||
export GATEWAY_IP="${SERVICE_IP_PREFIX}.14"
|
||||
|
||||
red=
|
||||
green=
|
||||
yel=
|
||||
|
||||
@ -38,6 +38,10 @@ IMAGE_kind_arm64 := $(IMAGE_kind_amd64)
|
||||
PEBBLE_COMMIT = ba5f81dd80fa870cbc19326f2d5a46f45f0b5ee3
|
||||
GATEWAY_API_VERSION = 0.4.1
|
||||
|
||||
# TODO: move the installation commands in this file to separate scripts like those in https://github.com/cert-manager/cert-manager/tree/master/devel/addon for readability
|
||||
# Once that is done, we can consume this variable from ./make/config/lib.sh
|
||||
SERVICE_IP_PREFIX = 10.0.0
|
||||
|
||||
.PHONY: e2e-setup-kind
|
||||
## Create a Kubernetes cluster using Kind, which is required for `make e2e`.
|
||||
## The Kind image is pre-pulled to avoid 'kind create' from blocking other make
|
||||
@ -60,7 +64,7 @@ e2e-setup-kind: kind-exists
|
||||
# used as a prerequisite. If we were to use .PHONY, then the file's
|
||||
# timestamp would not be used to check whether targets should be rebuilt,
|
||||
# and they would get constantly rebuilt.
|
||||
bin/scratch/kind-exists: make/config/kind/config.yaml make/config/kind/config_etcd_no_fsync.yaml preload-kind-image make/cluster.sh FORCE bin/tools/kind bin/tools/kubectl bin/tools/yq | bin/scratch
|
||||
bin/scratch/kind-exists: make/config/kind/cluster.yaml preload-kind-image make/cluster.sh FORCE bin/tools/kind bin/tools/kubectl bin/tools/yq | bin/scratch
|
||||
@$(eval KIND_CLUSTER_NAME ?= kind)
|
||||
@make/cluster.sh --name $(KIND_CLUSTER_NAME)
|
||||
@if [ "$(shell cat $@ 2>/dev/null)" != kind ]; then echo kind > $@; else touch $@; fi
|
||||
@ -169,9 +173,9 @@ feature_gates_controller := $(subst $(space),\$(comma),$(filter AllAlpha=% AllBe
|
||||
feature_gates_webhook := $(subst $(space),\$(comma),$(filter AllAlpha=% AllBeta=% AdditionalCertificateOutputFormats=% LiteralCertificateSubject=%, $(subst $(comma),$(space),$(FEATURE_GATES))))
|
||||
feature_gates_cainjector := $(subst $(space),\$(comma),$(filter AllAlpha=% AllBeta=%, $(subst $(comma),$(space),$(FEATURE_GATES))))
|
||||
|
||||
# TODO: move these commands to separate scripts for readability
|
||||
.PHONY: e2e-setup-certmanager
|
||||
e2e-setup-certmanager: bin/cert-manager.tgz $(foreach bin,controller acmesolver cainjector webhook ctl,bin/containers/cert-manager-$(bin)-linux-$(CRI_ARCH).tar) $(foreach bin,controller acmesolver cainjector webhook ctl,load-bin/containers/cert-manager-$(bin)-linux-$(CRI_ARCH).tar) e2e-setup-gatewayapi bin/scratch/kind-exists bin/tools/kubectl bin/tools/kind
|
||||
@$(eval SERVICE_IP_PREFIX = $(shell bin/tools/kubectl cluster-info dump | grep -m1 ip-range | cut -d= -f2 | cut -d. -f1,2,3))
|
||||
@$(eval TAG = $(shell tar xfO bin/containers/cert-manager-controller-linux-$(CRI_ARCH).tar manifest.json | jq '.[0].RepoTags[0]' -r | cut -d: -f2))
|
||||
bin/tools/helm upgrade \
|
||||
--install \
|
||||
@ -195,7 +199,6 @@ e2e-setup-certmanager: bin/cert-manager.tgz $(foreach bin,controller acmesolver
|
||||
|
||||
.PHONY: e2e-setup-bind
|
||||
e2e-setup-bind: $(call image-tar,bind) load-$(call image-tar,bind) $(wildcard make/config/bind/*.yaml) bin/scratch/kind-exists bin/tools/kubectl
|
||||
@$(eval SERVICE_IP_PREFIX = $(shell bin/tools/kubectl cluster-info dump | grep -m1 ip-range | cut -d= -f2 | cut -d. -f1,2,3))
|
||||
@$(eval IMAGE = $(shell tar xfO $< manifest.json | jq '.[0].RepoTags[0]' -r))
|
||||
bin/tools/kubectl get ns bind 2>/dev/null >&2 || bin/tools/kubectl create ns bind
|
||||
sed -e "s|{SERVICE_IP_PREFIX}|$(SERVICE_IP_PREFIX)|g" -e "s|{IMAGE}|$(IMAGE)|g" make/config/bind/*.yaml | bin/tools/kubectl apply -n bind -f - >/dev/null
|
||||
@ -214,7 +217,6 @@ e2e-setup-gatewayapi: bin/downloaded/gatewayapi-v$(GATEWAY_API_VERSION) bin/scra
|
||||
# https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml#L64-L67
|
||||
.PHONY: e2e-setup-ingressnginx
|
||||
e2e-setup-ingressnginx: $(call image-tar,ingressnginx) load-$(call image-tar,ingressnginx) bin/tools/helm
|
||||
@$(eval SERVICE_IP_PREFIX = $(shell bin/tools/kubectl cluster-info dump | grep -m1 ip-range | cut -d= -f2 | cut -d. -f1,2,3))
|
||||
@$(eval TAG=$(shell tar xfO $< manifest.json | jq '.[0].RepoTags[0]' -r | cut -d: -f2))
|
||||
bin/tools/helm repo add ingress-nginx --force-update https://kubernetes.github.io/ingress-nginx >/dev/null
|
||||
bin/tools/helm upgrade \
|
||||
@ -305,7 +307,6 @@ e2e-setup-samplewebhook: load-bin/downloaded/containers/$(CRI_ARCH)/samplewebhoo
|
||||
|
||||
.PHONY: e2e-setup-projectcontour
|
||||
e2e-setup-projectcontour: load-$(call image-tar,projectcontour) make/config/projectcontour/contour-gateway.yaml make/config/projectcontour/gateway.yaml bin/scratch/kind-exists bin/tools/kubectl bin/tools/ytt
|
||||
@$(eval SERVICE_IP_PREFIX = $(shell bin/tools/kubectl cluster-info dump | grep -m1 ip-range | cut -d= -f2 | cut -d. -f1,2,3))
|
||||
bin/tools/ytt --data-value service_ip_prefix="${SERVICE_IP_PREFIX}" \
|
||||
--file make/config/projectcontour/contour-gateway.yaml \
|
||||
--file make/config/projectcontour/gateway.yaml | bin/tools/kubectl apply -f-
|
||||
|
||||
@ -184,8 +184,6 @@ fi
|
||||
|
||||
mkdir -p "${ARTIFACTS}"
|
||||
|
||||
service_ip_prefix=$(kubectl cluster-info dump | grep ip-range | head -n1 | cut -d= -f2 | cut -d. -f1,2,3)
|
||||
|
||||
export CGO_ENABLED=0
|
||||
|
||||
trace ginkgo \
|
||||
@ -196,9 +194,9 @@ trace ginkgo \
|
||||
-- \
|
||||
--repo-root="$PWD" \
|
||||
--report-dir="${ARTIFACTS}" \
|
||||
--acme-dns-server="${service_ip_prefix}.16" \
|
||||
--acme-ingress-ip="${service_ip_prefix}.15" \
|
||||
--acme-gateway-ip="${service_ip_prefix}.14" \
|
||||
--acme-dns-server="${SERVICE_IP_PREFIX}.16" \
|
||||
--acme-ingress-ip="${SERVICE_IP_PREFIX}.15" \
|
||||
--acme-gateway-ip="${SERVICE_IP_PREFIX}.14" \
|
||||
--ingress-controller-domain=ingress-nginx.http01.example.com \
|
||||
--gateway-domain=gateway.http01.example.com \
|
||||
--feature-gates="$feature_gates" \
|
||||
|
||||
22
test/fixtures/kind/config-v1alpha2.yaml
vendored
22
test/fixtures/kind/config-v1alpha2.yaml
vendored
@ -1,22 +0,0 @@
|
||||
# this config file is similar to the default, except we set the cluster's
|
||||
# service cidr range to be 10.0.0.0/16.
|
||||
# we do this because we need a fixed/predictable clusterIP of 10.0.0.15 for the
|
||||
# nginx-ingress service, in order to perform HTTP01 validations during tests.
|
||||
|
||||
apiVersion: kind.sigs.k8s.io/v1alpha3
|
||||
kind: Cluster
|
||||
kubeadmConfigPatches:
|
||||
- |
|
||||
# config generated by kind
|
||||
apiVersion: kubeadm.k8s.io/v1alpha2
|
||||
kind: MasterConfiguration
|
||||
metadata:
|
||||
name: config
|
||||
networking:
|
||||
serviceSubnet: 10.0.0.0/16
|
||||
kubeletConfiguration:
|
||||
baseConfig:
|
||||
clusterDNS:
|
||||
- 10.0.0.10
|
||||
nodes:
|
||||
- role: control-plane
|
||||
18
test/fixtures/kind/config-v1alpha3.yaml
vendored
18
test/fixtures/kind/config-v1alpha3.yaml
vendored
@ -1,18 +0,0 @@
|
||||
# this config file is similar to the default, except we set the cluster's
|
||||
# service cidr range to be 10.0.0.0/16.
|
||||
# we do this because we need a fixed/predictable clusterIP of 10.0.0.15 for the
|
||||
# nginx-ingress service, in order to perform HTTP01 validations during tests.
|
||||
|
||||
apiVersion: kind.sigs.k8s.io/v1alpha3
|
||||
kind: Cluster
|
||||
kubeadmConfigPatches:
|
||||
- |
|
||||
# config generated by kind
|
||||
apiVersion: kubeadm.k8s.io/v1alpha3
|
||||
kind: ClusterConfiguration
|
||||
metadata:
|
||||
name: config
|
||||
networking:
|
||||
serviceSubnet: 10.0.0.0/16
|
||||
nodes:
|
||||
- role: control-plane
|
||||
18
test/fixtures/kind/config-v1beta1.yaml
vendored
18
test/fixtures/kind/config-v1beta1.yaml
vendored
@ -1,18 +0,0 @@
|
||||
# this config file is similar to the default, except we set the cluster's
|
||||
# service cidr range to be 10.0.0.0/16.
|
||||
# we do this because we need a fixed/predictable clusterIP of 10.0.0.15 for the
|
||||
# nginx-ingress service, in order to perform HTTP01 validations during tests.
|
||||
|
||||
apiVersion: kind.sigs.k8s.io/v1alpha3
|
||||
kind: Cluster
|
||||
kubeadmConfigPatches:
|
||||
- |
|
||||
# config generated by kind
|
||||
apiVersion: kubeadm.k8s.io/v1beta1
|
||||
kind: ClusterConfiguration
|
||||
metadata:
|
||||
name: config
|
||||
networking:
|
||||
serviceSubnet: 10.0.0.0/16
|
||||
nodes:
|
||||
- role: control-plane
|
||||
18
test/fixtures/kind/config-v1beta2.yaml
vendored
18
test/fixtures/kind/config-v1beta2.yaml
vendored
@ -1,18 +0,0 @@
|
||||
# this config file is similar to the default, except we set the cluster's
|
||||
# service cidr range to be 10.0.0.0/16.
|
||||
# we do this because we need a fixed/predictable clusterIP of 10.0.0.15 for the
|
||||
# nginx-ingress service, in order to perform HTTP01 validations during tests.
|
||||
|
||||
apiVersion: kind.sigs.k8s.io/v1alpha3
|
||||
kind: Cluster
|
||||
kubeadmConfigPatches:
|
||||
- |
|
||||
# config generated by kind
|
||||
apiVersion: kubeadm.k8s.io/v1beta2
|
||||
kind: ClusterConfiguration
|
||||
metadata:
|
||||
name: config
|
||||
networking:
|
||||
serviceSubnet: 10.0.0.0/16
|
||||
nodes:
|
||||
- role: control-plane
|
||||
Loading…
Reference in New Issue
Block a user