From 5003684092ddb9d9fe7dec4f2d569e907a99403c Mon Sep 17 00:00:00 2001
From: Kris Raney <kraney@google.com>
Date: Thu, 20 Aug 2026 21:33:51 +0000
Subject: [PATCH 1/2] Fix controller workqueue rate limits, resource limits,
 and deletion log noise

- Align controller workqueue RateLimiter with REST client QPS (100 QPS, 200 burst) in Options, resolving the default 10 QPS WorkQueue bottleneck.
- Increase controller manager deployment resource limits (CPU: 2, Memory: 1Gi) to prevent OOM kills and CPU throttling under high concurrency and scale.
- Check for apierrors.IsNotFound(err) before logging errors in Reconcile(), eliminating noisy ERROR stack traces during resource deletion/cleanup.
---
 operator/config/manager/manager.yaml       |  8 ++++----
 operator/controllers/lemming_controller.go | 17 +++++++++++++++--
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/operator/config/manager/manager.yaml b/operator/config/manager/manager.yaml
index 4e6debbf..69eee335 100644
--- a/operator/config/manager/manager.yaml
+++ b/operator/config/manager/manager.yaml
@@ -61,10 +61,10 @@ spec:
         # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
         resources:
           limits:
-            cpu: 500m
-            memory: 128Mi
+            cpu: "2"
+            memory: 1Gi
           requests:
-            cpu: 10m
-            memory: 64Mi
+            cpu: 100m
+            memory: 256Mi
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10
diff --git a/operator/controllers/lemming_controller.go b/operator/controllers/lemming_controller.go
index 0f1492df..0a1f22a0 100644
--- a/operator/controllers/lemming_controller.go
+++ b/operator/controllers/lemming_controller.go
@@ -20,10 +20,14 @@ import (
 	"path/filepath"
 	"sort"
 
+	"time"
+
+	"golang.org/x/time/rate"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/workqueue"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -60,8 +64,11 @@ func (r *LemmingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	log := log.FromContext(ctx)
 	lemming := &lemmingv1alpha1.Lemming{}
 	if err := r.Get(ctx, req.NamespacedName, lemming); err != nil {
+		if apierrors.IsNotFound(err) {
+			return ctrl.Result{}, nil
+		}
 		log.Error(err, "unable to get lemming")
-		return ctrl.Result{}, client.IgnoreNotFound(err)
+		return ctrl.Result{}, err
 	}
 
 	secret, err := r.reconcileSecrets(ctx, lemming)
@@ -345,6 +352,12 @@ func (r *LemmingReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&corev1.Pod{}).
 		Owns(&corev1.Service{}).
 		Owns(&corev1.Secret{}).
-		WithOptions(controller.Options{MaxConcurrentReconciles: 50}).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: 50,
+			RateLimiter: workqueue.NewMaxOfRateLimiter(
+				workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 1000*time.Second),
+				&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(100), 200)},
+			),
+		}).
 		Complete(r)
 }

From a1e9842443ae01ee83c95cc36c023a50e2b29bd8 Mon Sep 17 00:00:00 2001
From: Kris Raney <kraney@google.com>
Date: Thu, 20 Aug 2026 23:03:45 +0000
Subject: [PATCH 2/2] More dependencies

---
 MODULE.bazel               | 1 +
 operator/controllers/BUILD | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/MODULE.bazel b/MODULE.bazel
index d5ecb0ad..6eadbfc1 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -133,6 +133,7 @@ use_repo(
     "org_golang_google_protobuf",
     "org_golang_x_oauth2",
     "org_golang_x_sys",
+    "org_golang_x_time",
     "org_modernc_cc_v4",
     "org_uber_go_mock",
 )
diff --git a/operator/controllers/BUILD b/operator/controllers/BUILD
index 0d26bafa..84c0717e 100644
--- a/operator/controllers/BUILD
+++ b/operator/controllers/BUILD
@@ -16,11 +16,13 @@ go_library(
         "@io_k8s_apimachinery//pkg/types",
         "@io_k8s_apimachinery//pkg/util/intstr",
         "@io_k8s_client_go//util/cert",
+        "@io_k8s_client_go//util/workqueue",
         "@io_k8s_sigs_controller_runtime//:controller-runtime",
         "@io_k8s_sigs_controller_runtime//pkg/client",
         "@io_k8s_sigs_controller_runtime//pkg/controller",
         "@io_k8s_sigs_controller_runtime//pkg/log",
         "@io_k8s_utils//pointer",
+        "@org_golang_x_time//rate",
     ],
 )