diff --git a/pkg/util/versionchecker/testdata/BUILD.bazel b/pkg/util/versionchecker/testdata/BUILD.bazel
index ef803540b..345ef58e9 100644
--- a/pkg/util/versionchecker/testdata/BUILD.bazel
+++ b/pkg/util/versionchecker/testdata/BUILD.bazel
@@ -1,8 +1,9 @@
-# Clone empty version of cert-manager repo and list all tags
+# List all tags of cert-manager repo
 genrule(
     name = "git_tags",
     outs = [":git_tags.txt"],
     cmd = "git ls-remote -t --refs https://github.com/jetstack/cert-manager.git | awk '{print $$2;}' | sed 's/refs\\/tags\\///' | sed -n '/v1.0.0/,$$p' > $@",
+    stamp = 1,  # make sure we re-run if 'bazel-out/stable-status.txt' changes (more frequent would also be ok, but is difficult with bazel)
 )
 
 genrule(