From 58b64a820c68fd95d6b47a4610b2da4edab9a34a Mon Sep 17 00:00:00 2001 From: kkellerlbl Date: Fri, 28 Aug 2026 13:59:14 -0700 Subject: [PATCH 1/7] Create k8s_failed_cronjobs.py Initial commit of script checking Jobs spawned by CronJobs. --- lakehouse/k8s_failed_cronjobs.py | 93 ++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 lakehouse/k8s_failed_cronjobs.py diff --git a/lakehouse/k8s_failed_cronjobs.py b/lakehouse/k8s_failed_cronjobs.py new file mode 100644 index 0000000..48b2778 --- /dev/null +++ b/lakehouse/k8s_failed_cronjobs.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +""" +For every CronJob in the cluster, find its child Jobs +(via ownerReferences) and report any which failed. +Will automatically pick up new CronJob objects. + +Ensure that `ttlSecondsAfterFinished` is set to a high enough value +(e.g., 172800 for two days) in the CronJob. +Otherwise, Job objects will be purged too early, and the script +could report an inaccurate status. + +Nagios exit statuses (for use with mrpe): +Exits 2 if most recent Job in any CronJob failed +Exits 1 if any Job failed but a more recent Job of that CronJob succeeded +Exits 0 if all Jobs succeeded + +Requires: pip install kubernetes, credentials saved in default location +""" + +import kubernetes +import sys + +def load_kube_config(): + # Tries in-cluster config first (e.g. running as a Pod), falls back + # to your local kubeconfig (~/.kube/config or $KUBECONFIG). + # (for use in check_mk, will likely prefer $HOME/.kube/config + try: + kubernetes.config.load_incluster_config() + except kubernetes.config.ConfigException: + kubernetes.config.load_kube_config() + + +def main(): + load_kube_config() + batch = kubernetes.client.BatchV1Api() + + cronjobs = batch.list_cron_job_for_all_namespaces().items + jobs = batch.list_job_for_all_namespaces().items + + # Index jobs by their owning CronJob's uid + jobs_by_cronjob_uid = {} + for job in jobs: + for ref in job.metadata.owner_references or []: + if ref.kind == "CronJob": + jobs_by_cronjob_uid.setdefault(ref.uid, []).append(job) + + cronjob_names = [] + failed_children = [] + status=3 + + for cj in cronjobs: + ns = cj.metadata.namespace + name = cj.metadata.name + uid = cj.metadata.uid + + cronjob_names.append(cj.metadata.namespace+'/'+cj.metadata.name) + children = jobs_by_cronjob_uid.get(uid, []) + + cronjobstatus=3 + # setting the status like this depends on jobs being sorted by date + # which the script doesn't currently check for + for job in children: + #print (str(job)) + conditions = job.status.conditions or [] + if any(c.type == "Failed" and c.status == "True" for c in conditions): + failed_children.append(job.metadata.name) + cronjobstatus=2 + # explicitly look for success + elif any(c.type == "SuccessCriteriaMet" and c.status == "True" for c in conditions): + if (len(failed_children) > 0): + cronjobstatus=1 + else: + cronjobstatus=0 + # otherwise this Job object won't change cronjobstatus + + # set global status based on status of previous and current CronJob objects + if (status == 3): + status=cronjobstatus + if (cronjobstatus > status and cronjobstatus != 3): + status=cronjobstatus + + if failed_children: + print(f"failed CronJobs: {', '.join(failed_children)} ; CronJobs checked: {', '.join(cronjob_names)}") + else: +# print(str(status) + f" ok {ns}/{name} ({len(children)} job(s))") + print(f"all CronJobs OK, CronJobs checked: {', '.join(cronjob_names)}") + + # required in order to exit script with correct exit value + return status + +if __name__ == "__main__": + sys.exit(main()) From 6438d78b53fcffe330f8ec11cb9f13197b7ba1ff Mon Sep 17 00:00:00 2001 From: kkellerlbl Date: Tue, 1 Sep 2026 14:50:30 -0700 Subject: [PATCH 2/7] Update k8s_failed_cronjobs.py fix output if status is unknown --- lakehouse/k8s_failed_cronjobs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lakehouse/k8s_failed_cronjobs.py b/lakehouse/k8s_failed_cronjobs.py index 48b2778..30928e7 100644 --- a/lakehouse/k8s_failed_cronjobs.py +++ b/lakehouse/k8s_failed_cronjobs.py @@ -82,9 +82,11 @@ def main(): if failed_children: print(f"failed CronJobs: {', '.join(failed_children)} ; CronJobs checked: {', '.join(cronjob_names)}") - else: + elif status == 0: # print(str(status) + f" ok {ns}/{name} ({len(children)} job(s))") print(f"all CronJobs OK, CronJobs checked: {', '.join(cronjob_names)}") + else: + print(f"status of CronJobs unknown, CronJobs checked: {', '.join(cronjob_names)}") # required in order to exit script with correct exit value return status From 59f9bd63f9da75dd2d20297548f5b35c5d587b2a Mon Sep 17 00:00:00 2001 From: kkellerlbl Date: Tue, 1 Sep 2026 14:52:41 -0700 Subject: [PATCH 3/7] Copilot suggestion for ordering jobs by date Unverified Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- lakehouse/k8s_failed_cronjobs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lakehouse/k8s_failed_cronjobs.py b/lakehouse/k8s_failed_cronjobs.py index 30928e7..a719b12 100644 --- a/lakehouse/k8s_failed_cronjobs.py +++ b/lakehouse/k8s_failed_cronjobs.py @@ -55,7 +55,14 @@ def main(): uid = cj.metadata.uid cronjob_names.append(cj.metadata.namespace+'/'+cj.metadata.name) - children = jobs_by_cronjob_uid.get(uid, []) + children = sorted( + jobs_by_cronjob_uid.get(uid, []), + key=lambda j: ( + (j.status.start_time or j.metadata.creation_timestamp).timestamp() + if (j.status.start_time or j.metadata.creation_timestamp) + else 0.0 + ), + ) cronjobstatus=3 # setting the status like this depends on jobs being sorted by date From 6955aa733b51ca65fda22ab94312cf1506d22923 Mon Sep 17 00:00:00 2001 From: kkellerlbl Date: Wed, 2 Sep 2026 12:27:35 -0700 Subject: [PATCH 4/7] use Complete for success suggested by Copilot Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- lakehouse/k8s_failed_cronjobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lakehouse/k8s_failed_cronjobs.py b/lakehouse/k8s_failed_cronjobs.py index a719b12..2ace960 100644 --- a/lakehouse/k8s_failed_cronjobs.py +++ b/lakehouse/k8s_failed_cronjobs.py @@ -74,7 +74,7 @@ def main(): failed_children.append(job.metadata.name) cronjobstatus=2 # explicitly look for success - elif any(c.type == "SuccessCriteriaMet" and c.status == "True" for c in conditions): + elif any(c.type == "Complete" and c.status == "True" for c in conditions): if (len(failed_children) > 0): cronjobstatus=1 else: From fc204453f31bc4840a8040d866b1d0d9b025a7dc Mon Sep 17 00:00:00 2001 From: kkellerlbl Date: Wed, 2 Sep 2026 12:33:51 -0700 Subject: [PATCH 5/7] Update k8s_failed_cronjobs.py add try blocks to k8s API calls --- lakehouse/k8s_failed_cronjobs.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lakehouse/k8s_failed_cronjobs.py b/lakehouse/k8s_failed_cronjobs.py index 2ace960..2db7c73 100644 --- a/lakehouse/k8s_failed_cronjobs.py +++ b/lakehouse/k8s_failed_cronjobs.py @@ -33,17 +33,26 @@ def load_kube_config(): def main(): load_kube_config() - batch = kubernetes.client.BatchV1Api() - - cronjobs = batch.list_cron_job_for_all_namespaces().items - jobs = batch.list_job_for_all_namespaces().items - + try: + batch = kubernetes.client.BatchV1Api() + except: + print("can't connect to k8s") + + try: + cronjobs = batch.list_cron_job_for_all_namespaces().items + jobs = batch.list_job_for_all_namespaces().items + except: + print("can't get CronJobs from k8s") + # Index jobs by their owning CronJob's uid jobs_by_cronjob_uid = {} for job in jobs: for ref in job.metadata.owner_references or []: if ref.kind == "CronJob": - jobs_by_cronjob_uid.setdefault(ref.uid, []).append(job) + try: + jobs_by_cronjob_uid.setdefault(ref.uid, []).append(job) + except: + print("can't get Jobs from k8s") cronjob_names = [] failed_children = [] From 7c32cf0fe077de80983c71bcffd36541986bcfe9 Mon Sep 17 00:00:00 2001 From: kkellerlbl Date: Wed, 2 Sep 2026 12:37:23 -0700 Subject: [PATCH 6/7] add namespace to failed job output suggested by Copilot Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- lakehouse/k8s_failed_cronjobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lakehouse/k8s_failed_cronjobs.py b/lakehouse/k8s_failed_cronjobs.py index 2db7c73..bab73e1 100644 --- a/lakehouse/k8s_failed_cronjobs.py +++ b/lakehouse/k8s_failed_cronjobs.py @@ -80,7 +80,7 @@ def main(): #print (str(job)) conditions = job.status.conditions or [] if any(c.type == "Failed" and c.status == "True" for c in conditions): - failed_children.append(job.metadata.name) + failed_children.append(f"{job.metadata.namespace}/{job.metadata.name}") cronjobstatus=2 # explicitly look for success elif any(c.type == "Complete" and c.status == "True" for c in conditions): From d69706e4cf4226e9eb266a607effdbb65a09e299 Mon Sep 17 00:00:00 2001 From: kkellerlbl Date: Wed, 2 Sep 2026 12:50:38 -0700 Subject: [PATCH 7/7] try block for getting config suggested by copilot Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- lakehouse/k8s_failed_cronjobs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lakehouse/k8s_failed_cronjobs.py b/lakehouse/k8s_failed_cronjobs.py index bab73e1..04f9a40 100644 --- a/lakehouse/k8s_failed_cronjobs.py +++ b/lakehouse/k8s_failed_cronjobs.py @@ -32,7 +32,11 @@ def load_kube_config(): def main(): - load_kube_config() + try: + load_kube_config() + except kubernetes.config.ConfigException as exc: + print(f"can't load Kubernetes configuration: {exc}") + return 3 try: batch = kubernetes.client.BatchV1Api() except: