diff --git a/contrib/charts/boulder/.helmignore b/contrib/charts/boulder/.helmignore
deleted file mode 100644
index f0c131944..000000000
--- a/contrib/charts/boulder/.helmignore
+++ /dev/null
@@ -1,21 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
diff --git a/contrib/charts/boulder/Chart.yaml b/contrib/charts/boulder/Chart.yaml
deleted file mode 100644
index 20eb0c115..000000000
--- a/contrib/charts/boulder/Chart.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-apiVersion: v1
-description: A Helm chart for Kubernetes
-name: boulder
-version: 0.1.0
diff --git a/contrib/charts/boulder/templates/_helpers.tpl b/contrib/charts/boulder/templates/_helpers.tpl
deleted file mode 100644
index f0d83d2ed..000000000
--- a/contrib/charts/boulder/templates/_helpers.tpl
+++ /dev/null
@@ -1,16 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-*/}}
-{{- define "fullname" -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
diff --git a/contrib/charts/boulder/templates/bhsm-deployment.yaml b/contrib/charts/boulder/templates/bhsm-deployment.yaml
deleted file mode 100644
index db14d34a3..000000000
--- a/contrib/charts/boulder/templates/bhsm-deployment.yaml
+++ /dev/null
@@ -1,31 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.7.0 ()
-  creationTimestamp: null
-  labels:
-    io.kompose.service: bhsm
-  name: bhsm
-spec:
-  replicas: 1
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        io.kompose.service: bhsm
-    spec:
-      containers:
-      - args:
-        - /usr/local/bin/pkcs11-daemon
-        - /usr/lib/softhsm/libsofthsm2.so
-        env:
-        - name: PKCS11_DAEMON_SOCKET
-          value: tcp://0.0.0.0:5657
-        image: letsencrypt/boulder-tools:2017-12-18
-        name: bhsm
-        resources: {}
-      restartPolicy: Always
-status: {}
diff --git a/contrib/charts/boulder/templates/bhsm-service.yaml b/contrib/charts/boulder/templates/bhsm-service.yaml
deleted file mode 100644
index 2cbc27858..000000000
--- a/contrib/charts/boulder/templates/bhsm-service.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.7.0 ()
-  creationTimestamp: null
-  labels:
-    io.kompose.service: bhsm
-  name: boulder-hsm
-spec:
-  ports:
-  - name: headless
-    port: 5657
-    targetPort: 5657
-  selector:
-    io.kompose.service: bhsm
-status:
-  loadBalancer: {}
diff --git a/contrib/charts/boulder/templates/bmysql-deployment.yaml b/contrib/charts/boulder/templates/bmysql-deployment.yaml
deleted file mode 100644
index 54458c72b..000000000
--- a/contrib/charts/boulder/templates/bmysql-deployment.yaml
+++ /dev/null
@@ -1,31 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.7.0 ()
-  creationTimestamp: null
-  labels:
-    io.kompose.service: bmysql
-  name: bmysql
-spec:
-  replicas: 1
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        io.kompose.service: bmysql
-    spec:
-      containers:
-      - args:
-        - mysqld
-        - --bind-address=0.0.0.0
-        env:
-        - name: MYSQL_ALLOW_EMPTY_PASSWORD
-          value: "yes"
-        image: mariadb:10.1
-        name: bmysql
-        resources: {}
-      restartPolicy: Always
-status: {}
diff --git a/contrib/charts/boulder/templates/bmysql-service.yaml b/contrib/charts/boulder/templates/bmysql-service.yaml
deleted file mode 100644
index e9625bb3e..000000000
--- a/contrib/charts/boulder/templates/bmysql-service.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.7.0 ()
-  creationTimestamp: null
-  labels:
-    io.kompose.service: bmysql
-  name: boulder-mysql
-spec:
-  ports:
-  - name: sql
-    port: 3306
-    targetPort: 3306
-  selector:
-    io.kompose.service: bmysql
-status:
-  loadBalancer: {}
diff --git a/contrib/charts/boulder/templates/boulder-deployment.yaml b/contrib/charts/boulder/templates/boulder-deployment.yaml
deleted file mode 100644
index aa3b7a709..000000000
--- a/contrib/charts/boulder/templates/boulder-deployment.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.7.0 ()
-  creationTimestamp: null
-  labels:
-    io.kompose.service: boulder
-  name: boulder
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        io.kompose.service: boulder
-    spec:
-      hostname: boulder
-      containers:
-      - env:
-        - name: BOULDER_CONFIG_DIR
-          value: test/config
-        - name: FAKE_DNS
-          value: 10.0.0.10
-        - name: PKCS11_PROXY_SOCKET
-          value: tcp://boulder-hsm:5657
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-        name: boulder
-        readinessProbe:
-          httpGet:
-            path: /
-            port: 4000
-        ports:
-        - containerPort: 4000
-        - containerPort: 4001
-        - containerPort: 4002
-        - containerPort: 4003
-        - containerPort: 4430
-        - containerPort: 4431
-        - containerPort: 4500
-        - containerPort: 6000
-        - containerPort: 8000
-        - containerPort: 8001
-        - containerPort: 8002
-        - containerPort: 8003
-        - containerPort: 8004
-        - containerPort: 8005
-        - containerPort: 8006
-        - containerPort: 8008
-        - containerPort: 8009
-        - containerPort: 8010
-        - containerPort: 8055
-        - containerPort: 9380
-        - containerPort: 9381
-        resources: {}
-        volumeMounts:
-        - mountPath: /tmp
-          name: boulder-claim0
-      restartPolicy: Always
-      volumes:
-      - name: boulder-claim0
-        emptyDir: {}
-status: {}
diff --git a/contrib/charts/boulder/templates/boulder-service.yaml b/contrib/charts/boulder/templates/boulder-service.yaml
deleted file mode 100644
index af8f27c28..000000000
--- a/contrib/charts/boulder/templates/boulder-service.yaml
+++ /dev/null
@@ -1,79 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.7.0 ()
-  creationTimestamp: null
-  labels:
-    io.kompose.service: boulder
-  name: boulder
-spec:
-  ports:
-  - name: "4000"
-    port: 4000
-    targetPort: 4000
-  - name: "4001"
-    port: 4001
-    targetPort: 4001
-  - name: "4002"
-    port: 4002
-    targetPort: 4002
-  - name: "4003"
-    port: 4003
-    targetPort: 4003
-  - name: "4430"
-    port: 4430
-    targetPort: 4430
-  - name: "4431"
-    port: 4431
-    targetPort: 4431
-  - name: "4500"
-    port: 4500
-    targetPort: 4500
-  - name: "6000"
-    port: 6000
-    targetPort: 6000
-  - name: "8000"
-    port: 8000
-    targetPort: 8000
-  - name: "8001"
-    port: 8001
-    targetPort: 8001
-  - name: "8002"
-    port: 8002
-    targetPort: 8002
-  - name: "8003"
-    port: 8003
-    targetPort: 8003
-  - name: "8004"
-    port: 8004
-    targetPort: 8004
-  - name: "8005"
-    port: 8005
-    targetPort: 8005
-  - name: "8006"
-    port: 8006
-    targetPort: 8006
-  - name: "8008"
-    port: 8008
-    targetPort: 8008
-  - name: "8009"
-    port: 8009
-    targetPort: 8009
-  - name: "8010"
-    port: 8010
-    targetPort: 8010
-  - name: "8055"
-    port: 8055
-    targetPort: 8055
-  - name: "9380"
-    port: 9380
-    targetPort: 9380
-  - name: "9381"
-    port: 9381
-    targetPort: 9381
-  selector:
-    io.kompose.service: boulder
-status:
-  loadBalancer: {}
diff --git a/contrib/charts/boulder/values.yaml b/contrib/charts/boulder/values.yaml
deleted file mode 100644
index 00d87e5c2..000000000
--- a/contrib/charts/boulder/values.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-image:
-  repository: quay.io/munnerz/boulder
-  tag: "20180115.1"
-  pullPolicy: IfNotPresent
diff --git a/hack/ci.sh b/hack/ci.sh
index 82653d8d5..aa9cd011a 100755
--- a/hack/ci.sh
+++ b/hack/ci.sh
@@ -10,11 +10,48 @@ set -o pipefail
 make build APP_VERSION=build
 
 # Wait for e2e service dependencies
-./hack/test/wait-minikube.sh
+echo "Waiting for minikube cluster to be ready..."
 
+while true; do if kubectl get nodes; then break; fi; echo "Waiting 5s for kubernetes to be ready..."; sleep 5; done
+
+echo "Installing helm with cluster-admin privileges..."
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: List
+items:
+
+### Tiller ###
+# Create a ServiceAccount for tiller to use
+- apiVersion: v1
+  kind: ServiceAccount
+  metadata:
+    name: tiller
+    namespace: kube-system
+# Bind tiller to the cluster-admin role
+- apiVersion: rbac.authorization.k8s.io/v1beta1
+  kind: ClusterRoleBinding
+  metadata:
+    name: "tiller"
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: "cluster-admin"
+  subjects:
+  - apiGroup: ""
+    kind: ServiceAccount
+    name: tiller
+    namespace: kube-system
+EOF
+helm init --service-account=tiller
+
+
+echo "Exposing nginx-ingress service with a stable IP (10.0.0.15)"
 # Setup service for nginx ingress controller. A DNS entry for *.certmanager.kubernetes.network has been setup to point to 10.0.0.15 for e2e tests
 while true; do if kubectl get rc nginx-ingress-controller -n kube-system; then break; fi; echo "Waiting 5s for nginx-ingress-controller rc to be installed..."; sleep 5; done
 kubectl expose -n kube-system --port 80 --target-port 80 --type ClusterIP rc nginx-ingress-controller --cluster-ip 10.0.0.15
+
+echo "Waiting for Tiller to be ready..."
 while true; do if timeout 5 helm version; then break; fi; echo "Waiting 5s for tiller to be ready..."; sleep 5; done
 
+echo "Running e2e tests"
 make e2e_test E2E_NGINX_CERTIFICATE_DOMAIN=certmanager.kubernetes.network
diff --git a/hack/test/setup-boulder.sh b/hack/test/setup-boulder.sh
deleted file mode 100755
index 4ca13de64..000000000
--- a/hack/test/setup-boulder.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-BOULDER_REPO="github.com/letsencrypt/boulder"
-BOULDER_URL="http://127.0.0.1:4000"
-
-echo "Fetching ${BOULDER_REPO}"
-go get -d github.com/letsencrypt/boulder || true
-echo "Retrieved boulder repository"
-cd "${GOPATH}/src/${BOULDER_REPO}"
-
-# Modify boulder configuration
-sed -i 's/FAKE_DNS: 127.0.0.1/FAKE_DNS: 10.0.0.10/' docker-compose.yml
-sed -i 's/127.0.0.1:8053/10.0.0.10:53/' test/config/va.json
-sed -i 's/5002/80/' test/config/va.json
-sed -i 's/good-caa-reserved.com/kubernetes.network/' test/rate-limit-policies.yml
-
-function start {
-    if ! docker-compose up; then
-        echo "Error running boulder"
-        exit 1
-    fi
-}
-
-start &
-
-echo "Started boulder process in background"
diff --git a/hack/test/setup-minikube.sh b/hack/test/setup-minikube.sh
deleted file mode 100755
index 5e2fc7501..000000000
--- a/hack/test/setup-minikube.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# Create a cluster. We do this as root as we are using the 'docker' driver.
-sudo -E CHANGE_MINIKUBE_NONE_USER=true minikube start --vm-driver=none --extra-config=apiserver.Authorization.Mode=RBAC --kubernetes-version="${KUBERNETES_VERSION}"
-sudo -E CHANGE_MINIKUBE_NONE_USER=true minikube addons enable ingress
diff --git a/hack/test/wait-boulder.sh b/hack/test/wait-boulder.sh
deleted file mode 100755
index 2149d7df4..000000000
--- a/hack/test/wait-boulder.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-BOULDER_URL="http://127.0.0.1:4000"
-
-while ! curl "${BOULDER_URL}" > /dev/null 2>&1 ; do
-  echo "Waiting for boulder API to be available..."
-  sleep 5
-done
-
-echo "Boulder API now available at ${BOULDER_URL}"
diff --git a/hack/test/wait-minikube.sh b/hack/test/wait-minikube.sh
deleted file mode 100755
index 3f4a1005b..000000000
--- a/hack/test/wait-minikube.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-while true; do if kubectl get nodes; then break; fi; echo "Waiting 5s for kubernetes to be ready..."; sleep 5; done
-
-echo "Installing helm..."
-cat <<EOF | kubectl apply -f -
-apiVersion: v1
-kind: List
-items:
-
-### Tiller ###
-# Create a ServiceAccount for tiller to use
-- apiVersion: v1
-  kind: ServiceAccount
-  metadata:
-    name: tiller
-    namespace: kube-system
-# Bind tiller to the cluster-admin role
-- apiVersion: rbac.authorization.k8s.io/v1beta1
-  kind: ClusterRoleBinding
-  metadata:
-    name: "tiller"
-  roleRef:
-    apiGroup: rbac.authorization.k8s.io
-    kind: ClusterRole
-    name: "cluster-admin"
-  subjects:
-  - apiGroup: ""
-    kind: ServiceAccount
-    name: tiller
-    namespace: kube-system
-EOF
-helm init --service-account=tiller
diff --git a/hack/ubuntu-nsenter/Dockerfile b/hack/ubuntu-nsenter/Dockerfile
deleted file mode 100644
index 8a728f88d..000000000
--- a/hack/ubuntu-nsenter/Dockerfile
+++ /dev/null
@@ -1,12 +0,0 @@
-FROM ubuntu:14.04
-
-RUN apt-get update && \
-    apt-get install -y git build-essential libncurses5-dev libslang2-dev gettext zlib1g-dev libselinux1-dev debhelper lsb-release pkg-config po-debconf autoconf automake autopoint libtool bison
-
-RUN git clone git://git.kernel.org/pub/scm/utils/util-linux/util-linux.git util-linux
-RUN cd util-linux/ && \
-    ./autogen.sh && \
-    ./configure --without-python --disable-all-programs --enable-nsenter && \
-    make
-
-RUN mv /util-linux/nsenter /nsenter
diff --git a/hack/ubuntu-nsenter/README.md b/hack/ubuntu-nsenter/README.md
deleted file mode 100644
index 5338b9060..000000000
--- a/hack/ubuntu-nsenter/README.md
+++ /dev/null
@@ -1,12 +0,0 @@
-# ubuntu-nsenter
-
-This is a simple Docker image that contains a dynamically linked version of
-nsenter for Ubuntu 14.04. It is used in the e2e CI environment in order to
-install socat on a Travis CI host.
-
-You can use it like so:
-
-```
-$ docker build -t ubuntu-nsenter .
-$ docker run -v /usr/local/bin:/hostbin ubuntu-nsenter cp /nsenter /hostbin/nsenter
-```
diff --git a/hack/verify-chart-version.sh b/hack/verify-chart-version.sh
index 40273e370..e6491006f 100755
--- a/hack/verify-chart-version.sh
+++ b/hack/verify-chart-version.sh
@@ -90,6 +90,10 @@ for directory in ${CHANGED_FOLDERS}; do
     if [ "${directory}" == "contrib/charts" ]; then
         continue
     fi
+    if [ ! -d "${directory}" ]; then
+        echo "Directory ${directory} has been deleted. Skipping version check..."
+        continue
+    fi
     semvercompare "${directory}"
 done