Fix controller workqueue rate limits, resource limits, and deletion log noise

MODULE.bazel

@@ -133,6 +133,7 @@ use_repo(
     "org_golang_google_protobuf",
     "org_golang_x_oauth2",
     "org_golang_x_sys",
+    "org_golang_x_time",
     "org_modernc_cc_v4",
     "org_uber_go_mock",
 )

operator/config/manager/manager.yaml

@@ -61,10 +61,10 @@ spec:
         # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
         resources:
           limits:
-            cpu: 500m
-            memory: 128Mi
+            cpu: "2"
+            memory: 1Gi
           requests:
-            cpu: 10m
-            memory: 64Mi
+            cpu: 100m
+            memory: 256Mi
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10

operator/controllers/BUILD

@@ -16,11 +16,13 @@ go_library(
         "@io_k8s_apimachinery//pkg/types",
         "@io_k8s_apimachinery//pkg/util/intstr",
         "@io_k8s_client_go//util/cert",
+        "@io_k8s_client_go//util/workqueue",
         "@io_k8s_sigs_controller_runtime//:controller-runtime",
         "@io_k8s_sigs_controller_runtime//pkg/client",
         "@io_k8s_sigs_controller_runtime//pkg/controller",
         "@io_k8s_sigs_controller_runtime//pkg/log",
         "@io_k8s_utils//pointer",
+        "@org_golang_x_time//rate",
     ],
 )
 

operator/controllers/lemming_controller.go

@@ -20,10 +20,14 @@ import (
 	"path/filepath"
 	"sort"
 
+	"time"
+
+	"golang.org/x/time/rate"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/workqueue"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -60,8 +64,11 @@ func (r *LemmingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	log := log.FromContext(ctx)
 	lemming := &lemmingv1alpha1.Lemming{}
 	if err := r.Get(ctx, req.NamespacedName, lemming); err != nil {
+		if apierrors.IsNotFound(err) {
+			return ctrl.Result{}, nil
+		}
 		log.Error(err, "unable to get lemming")
-		return ctrl.Result{}, client.IgnoreNotFound(err)
+		return ctrl.Result{}, err
 	}
 
 	secret, err := r.reconcileSecrets(ctx, lemming)
@@ -346,6 +353,12 @@ func (r *LemmingReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&corev1.Pod{}).
 		Owns(&corev1.Service{}).
 		Owns(&corev1.Secret{}).
-		WithOptions(controller.Options{MaxConcurrentReconciles: 50}).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: 50,
+			RateLimiter: workqueue.NewMaxOfRateLimiter(
+				workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 1000*time.Second),
+				&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(100), 200)},
+			),
+		}).
 		Complete(r)
 }