diff --git a/pkg/controller/certificates/metrics/controller.go b/pkg/controller/certificates/metrics/controller.go
index 6cf26e05a..94060e46b 100644
--- a/pkg/controller/certificates/metrics/controller.go
+++ b/pkg/controller/certificates/metrics/controller.go
@@ -93,7 +93,7 @@ func (c *controller) ProcessItem(ctx context.Context, key string) error {
 	crt, err := c.certificateLister.Certificates(namespace).Get(name)
 	if apierrors.IsNotFound(err) {
 		// If the Certificate no longer exists, remove it's metrics from being exposed.
-		c.metrics.RemoveCertificate(key, crt.Spec.IssuerRef.Name, crt.Spec.IssuerRef.Kind, crt.Spec.IssuerRef.Group)
+		c.metrics.RemoveCertificate(key)
 		return nil
 	}
 	if err != nil {
diff --git a/pkg/metrics/certificates.go b/pkg/metrics/certificates.go
index 7b756e78f..071e53897 100644
--- a/pkg/metrics/certificates.go
+++ b/pkg/metrics/certificates.go
@@ -109,16 +109,14 @@ func (m *Metrics) updateCertificateReadyStatus(crt *cmapi.Certificate, current c
 
 // RemoveCertificate will delete the Certificate metrics from continuing to be
 // exposed.
-func (m *Metrics) RemoveCertificate(key, issuerName, issuerKind, issuerGroup string) {
+func (m *Metrics) RemoveCertificate(key string) {
 	namespace, name, err := cache.SplitMetaNamespaceKey(key)
 	if err != nil {
 		m.log.Error(err, "failed to get namespace and name from key")
 		return
 	}
 
-	m.certificateExpiryTimeSeconds.DeleteLabelValues(name, namespace, issuerName, issuerKind, issuerGroup)
-	m.certificateRenewalTimeSeconds.DeleteLabelValues(name, namespace, issuerName, issuerKind, issuerGroup)
-	for _, condition := range readyConditionStatuses {
-		m.certificateReadyStatus.DeleteLabelValues(name, namespace, string(condition), issuerName, issuerKind, issuerGroup)
-	}
+	m.certificateExpiryTimeSeconds.DeletePartialMatch(prometheus.Labels{"name": name, "namespace": namespace})
+	m.certificateRenewalTimeSeconds.DeletePartialMatch(prometheus.Labels{"name": name, "namespace": namespace})
+	m.certificateReadyStatus.DeletePartialMatch(prometheus.Labels{"name": name, "namespace": namespace})
 }
diff --git a/pkg/metrics/certificates_test.go b/pkg/metrics/certificates_test.go
index 83397c4cb..e530a4395 100644
--- a/pkg/metrics/certificates_test.go
+++ b/pkg/metrics/certificates_test.go
@@ -323,7 +323,7 @@ func TestCertificateCache(t *testing.T) {
 	}
 
 	// Remove second certificate and check not exists
-	m.RemoveCertificate("default-unit-test-ns/crt2", "test-issuer", "test-issuer-kind", "test-issuer-group")
+	m.RemoveCertificate("default-unit-test-ns/crt2")
 	if err := testutil.CollectAndCompare(m.certificateReadyStatus,
 		strings.NewReader(readyMetadata+`
         certmanager_certificate_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="crt1",namespace="default-unit-test-ns"} 0
@@ -348,9 +348,9 @@ func TestCertificateCache(t *testing.T) {
 	}
 
 	// Remove all Certificates (even is already removed) and observe no Certificates
-	m.RemoveCertificate("default-unit-test-ns/crt1", "test-issuer", "test-issuer-kind", "test-issuer-group")
-	m.RemoveCertificate("default-unit-test-ns/crt2", "test-issuer", "test-issuer-kind", "test-issuer-group")
-	m.RemoveCertificate("default-unit-test-ns/crt3", "test-issuer", "test-issuer-kind", "test-issuer-group")
+	m.RemoveCertificate("default-unit-test-ns/crt1")
+	m.RemoveCertificate("default-unit-test-ns/crt2")
+	m.RemoveCertificate("default-unit-test-ns/crt3")
 	if err := testutil.CollectAndCompare(m.certificateReadyStatus,
 		strings.NewReader(readyMetadata),
 		"certmanager_certificate_ready_status",