Merge pull request #1189 from munnerz/pebble-via-bazel

Use pebble built by bazel & pre-pull images during e2e tests

Makefile

@@ -113,7 +113,6 @@ e2e_test:
 			$$(bazel info bazel-genfiles)/test/e2e/e2e.test \
 			-- \
 			--helm-binary-path=$$(bazel info bazel-genfiles)/hack/bin/helm \
-			--tiller-image-tag=$$($$(bazel info bazel-genfiles)/hack/bin/helm version --client --template '{{.Client.SemVer}}') \
 			--repo-root="$$(pwd)" \
 			--report-dir="$${ARTIFACTS:-./_artifacts}" \
 			--ginkgo.skip="$(GINKGO_SKIP)"

WORKSPACE

@@ -74,7 +74,9 @@ container_pull(
     tag = "3.7-v20180822-0201cfb11",
 )
 
-## Fetch helm for use in template generation and testing
+## Fetch helm & tiller for use in template generation and testing
+## You can bump the version of Helm & Tiller used during e2e tests by tweaking
+## the version numbers in these rules.
 http_archive(
     name = "helm_darwin",
     sha256 = "7c4e6bfbc211d6b984ffb4fa490ce9ac112cc4b9b8d859ece27045b8514c1ed1",
@@ -107,6 +109,13 @@ filegroup(
 """,
 )
 
+container_pull(
+    name = "io_gcr_helm_tiller",
+    registry = "gcr.io",
+    repository = "kubernetes-helm/tiller",
+    tag = "v2.10.0",
+)
+
 ## Install 'kind', for creating kubernetes-in-docker clusters
 go_repository(
     name = "io_kubernetes_sigs_kind",
@@ -114,6 +123,45 @@ go_repository(
     importpath = "sigs.k8s.io/kind",
 )
 
+## Fetch pebble for use during e2e tests
+## You can change the version of Pebble used for tests by changing the 'commit'
+## field in this rule
+go_repository(
+    name = "org_letsencrypt_pebble",
+    commit = "2132a88193fdf0d7c0d93c33fce61db43d630fd4",
+    importpath = "github.com/letsencrypt/pebble",
+    build_external = "vendored",
+    # Expose the generated go_default_library as 'public' visibility
+    patch_cmds = ["sed -i -e 's/private/public/g' 'cmd/pebble/BUILD.bazel'"],
+)
+
+## Fetch nginx-ingress for use during e2e tests
+## You can change the version of nginx-ingress used for tests by changing the
+## 'tag' field in this rule
+container_pull(
+    name = "io_kubernetes_ingress-nginx",
+    registry = "quay.io",
+    repository = "kubernetes-ingress-controller/nginx-ingress-controller",
+    tag = "0.21.0",
+)
+
+container_pull(
+    name = "io_gcr_k8s_defaultbackend",
+    registry = "k8s.gcr.io",
+    repository = "defaultbackend",
+    tag = "1.4",
+)
+
+## Fetch vault for use during e2e tests
+## You can change the version of vault used for tests by changing the 'tag'
+## field in this rule
+container_pull(
+    name = "com_hashicorp_vault",
+    registry = "index.docker.io",
+    repository = "library/vault",
+    tag = "0.9.3",
+)
+
 ## Install buildozer, for mass-editing BUILD files
 http_file(
     name = "buildozer_darwin",

hack/ci/lib/build_images.sh

@@ -31,6 +31,8 @@ build_images() {
     DOCKER_REPO="${DOCKER_REPO}" \
     DOCKER_TAG="${DOCKER_TAG}" \
     bazel run //:images
+    # Build e2e test images
+    bazel run //test/e2e/charts:images
 
     local TMP_DIR=$(mktemp -d)
     local BUNDLE_FILE="${TMP_DIR}"/cmbundle.tar.gz
@@ -40,6 +42,11 @@ build_images() {
         "${DOCKER_REPO}"/cert-manager-controller:"${DOCKER_TAG}" \
         "${DOCKER_REPO}"/cert-manager-acmesolver:"${DOCKER_TAG}" \
         "${DOCKER_REPO}"/cert-manager-webhook:"${DOCKER_TAG}" \
+        "pebble:bazel" \
+        "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.21.0" \
+        "k8s.gcr.io/defaultbackend:bazel" \
+        "vault:bazel" \
+        "gcr.io/kubernetes-helm/tiller:bazel" \
         -o "${BUNDLE_FILE}"
 
     # Copy docker archive into the kind container
@@ -52,4 +59,4 @@ build_images() {
     rm -Rf "${TMP_DIR}"
 }
 
-build_images
+build_images

test/e2e/charts/BUILD.bazel

@@ -0,0 +1,13 @@
+load("@io_bazel_rules_docker//container:bundle.bzl", "container_bundle")
+
+container_bundle(
+    name = "images",
+    images = {
+        # A set of images to bundle up into a single tarball.
+        "pebble:bazel": "//test/e2e/charts/pebble:image",
+        "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.21.0": "@io_kubernetes_ingress-nginx//image",
+        "k8s.gcr.io/defaultbackend:bazel": "@io_gcr_k8s_defaultbackend//image",
+        "vault:bazel": "@com_hashicorp_vault//image",
+        "gcr.io/kubernetes-helm/tiller:bazel": "@io_gcr_helm_tiller//image",
+    }
+)

test/e2e/charts/pebble/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@io_bazel_rules_docker//go:image.bzl", "go_image")
+
+go_image(
+    name = "image",
+    base = "@alpine//image",
+    embed = ["@org_letsencrypt_pebble//cmd/pebble:go_default_library"],
+    goarch = "amd64",
+    goos = "linux",
+    pure = "on",
+    visibility = ["//visibility:public"],
+)

test/e2e/charts/pebble/templates/configmap.yaml

@@ -0,0 +1,68 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "fullname" . }}
+  labels:
+    app: {{ template "name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  config.json: |
+    {
+      "pebble": {
+        "listenAddress": "0.0.0.0:14000",
+        "certificate": "/config/cert.pem",
+        "privateKey": "/config/key.pem",
+        "httpPort": 80,
+        "tlsPort": 443
+      }
+    }
+  cert.pem: |
+    -----BEGIN CERTIFICATE-----
+    MIIDGzCCAgOgAwIBAgIIbEfayDFsBtwwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE
+    AxMVbWluaWNhIHJvb3QgY2EgMjRlMmRiMCAXDTE3MTIwNjE5NDIxMFoYDzIxMDcx
+    MjA2MTk0MjEwWjAUMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB
+    AQUAA4IBDwAwggEKAoIBAQCbFMW3DXXdErvQf2lCZ0qz0DGEWadDoF0O2neM5mVa
+    VQ7QGW0xc5Qwvn3Tl62C0JtwLpF0pG2BICIN+DHdVaIUwkf77iBS2doH1I3waE1I
+    8GkV9JrYmFY+j0dA1SwBmqUZNXhLNwZGq1a91nFSI59DZNy/JciqxoPX2K++ojU2
+    FPpuXe2t51NmXMsszpa+TDqF/IeskA9A/ws6UIh4Mzhghx7oay2/qqj2IIPjAmJj
+    i73kdUvtEry3wmlkBvtVH50+FscS9WmPC5h3lDTk5nbzSAXKuFusotuqy3XTgY5B
+    PiRAwkZbEY43JNfqenQPHo7mNTt29i+NVVrBsnAa5ovrAgMBAAGjYzBhMA4GA1Ud
+    DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0T
+    AQH/BAIwADAiBgNVHREEGzAZgglsb2NhbGhvc3SCBnBlYmJsZYcEfwAAATANBgkq
+    hkiG9w0BAQsFAAOCAQEAYIkXff8H28KS0KyLHtbbSOGU4sujHHVwiVXSATACsNAE
+    D0Qa8hdtTQ6AUqA6/n8/u1tk0O4rPE/cTpsM3IJFX9S3rZMRsguBP7BSr1Lq/XAB
+    7JP/CNHt+Z9aKCKcg11wIX9/B9F7pyKM3TdKgOpqXGV6TMuLjg5PlYWI/07lVGFW
+    /mSJDRs8bSCFmbRtEqc4lpwlrpz+kTTnX6G7JDLfLWYw/xXVqwFfdengcDTHCc8K
+    wtgGq/Gu6vcoBxIO3jaca+OIkMfxxXmGrcNdseuUCa3RMZ8Qy03DqGu6Y6XQyK4B
+    W8zIG6H9SVKkAznM2yfYhW8v2ktcaZ95/OBHY97ZIw==
+    -----END CERTIFICATE-----
+  key.pem: |
+    -----BEGIN RSA PRIVATE KEY-----
+    MIIEowIBAAKCAQEAmxTFtw113RK70H9pQmdKs9AxhFmnQ6BdDtp3jOZlWlUO0Blt
+    MXOUML5905etgtCbcC6RdKRtgSAiDfgx3VWiFMJH++4gUtnaB9SN8GhNSPBpFfSa
+    2JhWPo9HQNUsAZqlGTV4SzcGRqtWvdZxUiOfQ2TcvyXIqsaD19ivvqI1NhT6bl3t
+    redTZlzLLM6Wvkw6hfyHrJAPQP8LOlCIeDM4YIce6Gstv6qo9iCD4wJiY4u95HVL
+    7RK8t8JpZAb7VR+dPhbHEvVpjwuYd5Q05OZ280gFyrhbrKLbqst104GOQT4kQMJG
+    WxGONyTX6np0Dx6O5jU7dvYvjVVawbJwGuaL6wIDAQABAoIBAGW9W/S6lO+DIcoo
+    PHL+9sg+tq2gb5ZzN3nOI45BfI6lrMEjXTqLG9ZasovFP2TJ3J/dPTnrwZdr8Et/
+    357YViwORVFnKLeSCnMGpFPq6YEHj7mCrq+YSURjlRhYgbVPsi52oMOfhrOIJrEG
+    ZXPAwPRi0Ftqu1omQEqz8qA7JHOkjB2p0i2Xc/uOSJccCmUDMlksRYz8zFe8wHuD
+    XvUL2k23n2pBZ6wiez6Xjr0wUQ4ESI02x7PmYgA3aqF2Q6ECDwHhjVeQmAuypMF6
+    IaTjIJkWdZCW96pPaK1t+5nTNZ+Mg7tpJ/PRE4BkJvqcfHEOOl6wAE8gSk5uVApY
+    ZRKGmGkCgYEAzF9iRXYo7A/UphL11bR0gqxB6qnQl54iLhqS/E6CVNcmwJ2d9pF8
+    5HTfSo1/lOXT3hGV8gizN2S5RmWBrc9HBZ+dNrVo7FYeeBiHu+opbX1X/C1HC0m1
+    wJNsyoXeqD1OFc1WbDpHz5iv4IOXzYdOdKiYEcTv5JkqE7jomqBLQk8CgYEAwkG/
+    rnwr4ThUo/DG5oH+l0LVnHkrJY+BUSI33g3eQ3eM0MSbfJXGT7snh5puJW0oXP7Z
+    Gw88nK3Vnz2nTPesiwtO2OkUVgrIgWryIvKHaqrYnapZHuM+io30jbZOVaVTMR9c
+    X/7/d5/evwXuP7p2DIdZKQKKFgROm1XnhNqVgaUCgYBD/ogHbCR5RVsOVciMbRlG
+    UGEt3YmUp/vfMuAsKUKbT2mJM+dWHVlb+LZBa4pC06QFgfxNJi/aAhzSGvtmBEww
+    xsXbaceauZwxgJfIIUPfNZCMSdQVIVTi2Smcx6UofBz6i/Jw14MEwlvhamaa7qVf
+    kqflYYwelga1wRNCPopLaQKBgQCWsZqZKQqBNMm0Q9yIhN+TR+2d7QFjqeePoRPl
+    1qxNejhq25ojE607vNv1ff9kWUGuoqSZMUC76r6FQba/JoNbefI4otd7x/GzM9uS
+    8MHMJazU4okwROkHYwgLxxkNp6rZuJJYheB4VDTfyyH/ng5lubmY7rdgTQcNyZ5I
+    majRYQKBgAMKJ3RlII0qvAfNFZr4Y2bNIq+60Z+Qu2W5xokIHCFNly3W1XDDKGFe
+    CCPHSvQljinke3P9gPt2HVdXxcnku9VkTti+JygxuLkVg7E0/SWwrWfGsaMJs+84
+    fK+mTZay2d3v24r9WKEKwLykngYPyZw5+BdWU0E+xx5lGUd3U4gG
+    -----END RSA PRIVATE KEY-----

test/e2e/charts/pebble/templates/deployment.yaml

@@ -19,6 +19,12 @@ spec:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+          - -config=/config/config.json
+          volumeMounts:
+          - name: config
+            mountPath: /config
+            readOnly: true
           readinessProbe:
             tcpSocket:
               port: 14000
@@ -28,7 +34,11 @@ spec:
             successThreshold: 1
           resources:
 {{ toYaml .Values.resources | indent 12 }}
-    {{- if .Values.nodeSelector }}
+      volumes:
+      - name: config
+        configMap:
+          name: {{ template "fullname" . }}
+      {{- if .Values.nodeSelector }}
       nodeSelector:
 {{ toYaml .Values.nodeSelector | indent 8 }}
     {{- end }}

test/e2e/charts/pebble/values.yaml

@@ -1,8 +1,8 @@
 replicaCount: 1
 image:
-  repository: quay.io/munnerz/pebble
-  tag: "20180725"
-  pullPolicy: IfNotPresent
+  repository: pebble
+  tag: "bazel"
+  pullPolicy: Never
 service:
   type: ClusterIP
 resources:

test/e2e/charts/vault/templates/vault-deployment.yaml

@@ -18,6 +18,7 @@ spec:
       containers:
       - name: vault
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: Never
         command: ["vault", "server", "-dev", "-dev-listen-address=[::]:8202", "-config", "/vault/config/config.json"]
         # command: ["/bin/sh", "-c", "sleep 9999"]
         ports:

test/e2e/charts/vault/values.yaml

@@ -1,6 +1,6 @@
 image:
   repository: vault
-  tag: "0.9.3"
+  tag: "bazel"
 
 vault:
   publicKey:

test/e2e/framework/addon/nginxingress/nginx.go

@@ -94,6 +94,22 @@ func (n *Nginx) Setup(cfg *config.Config) error {
 		ChartName:    "stable/nginx-ingress",
 		ChartVersion: cfg.Addons.Nginx.ChartVersion,
 		Vars: []chart.StringTuple{
+			{
+				Key:   "controller.image.pullPolicy",
+				Value: "Never",
+			},
+			{
+				Key:   "controller.image.tag",
+				Value: "0.21.0",
+			},
+			{
+				Key:   "defaultBackend.image.pullPolicy",
+				Value: "Never",
+			},
+			{
+				Key:   "defaultBackend.image.tag",
+				Value: "bazel",
+			},
 			{
 				Key:   "controller.service.clusterIP",
 				Value: n.IPAddress,

test/e2e/framework/config/tiller.go

@@ -31,7 +31,7 @@ type Tiller struct {
 
 func (n *Tiller) AddFlags(fs *flag.FlagSet) {
 	fs.StringVar(&n.ImageRepo, "tiller-image-repo", "gcr.io/kubernetes-helm/tiller", "docker image repo for tiller-deploy")
-	fs.StringVar(&n.ImageTag, "tiller-image-tag", "v2.11.0", "docker image tag for tiller-deploy")
+	fs.StringVar(&n.ImageTag, "tiller-image-tag", "bazel", "docker image tag for tiller-deploy")
 }
 
 func (n *Tiller) Validate() []error {