diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go
index 9a6d1e159..4798b2472 100644
--- a/cmd/controller/app/options/options.go
+++ b/cmd/controller/app/options/options.go
@@ -243,6 +243,11 @@ func EnabledControllers(o *config.ControllerConfiguration) sets.Set[string] {
 		}
 	}
 
+	// Detect if "*" was implied (in case only disabled controllers were specified)
+	if len(disabled) > 0 && len(enabled) == 0 {
+		enabled = enabled.Insert(defaults.DefaultEnabledControllers...)
+	}
+
 	enabled = enabled.Delete(disabled...)
 
 	if utilfeature.DefaultFeatureGate.Enabled(feature.ExperimentalCertificateSigningRequestControllers) {
diff --git a/cmd/controller/app/options/options_test.go b/cmd/controller/app/options/options_test.go
index 8c86f308c..99256498e 100644
--- a/cmd/controller/app/options/options_test.go
+++ b/cmd/controller/app/options/options_test.go
@@ -50,6 +50,14 @@ func TestEnabledControllers(t *testing.T) {
 			controllers: []string{"*", "-clusterissuers", "-issuers"},
 			expEnabled:  sets.New(defaults.DefaultEnabledControllers...).Delete("clusterissuers", "issuers"),
 		},
+		"if only disabled controllers are specified, implicitly enable all default controllers": {
+			controllers: []string{"-clusterissuers", "-issuers"},
+			expEnabled:  sets.New(defaults.DefaultEnabledControllers...).Delete("clusterissuers", "issuers"),
+		},
+		"if both enabled and disabled controllers are specified, return specified controllers": {
+			controllers: []string{"foo", "-bar"},
+			expEnabled:  sets.New("foo"),
+		},
 	}
 
 	for name, test := range tests {