diff --git a/lakehouse/k8s_failed_cronjobs.py b/lakehouse/k8s_failed_cronjobs.py new file mode 100644 index 0000000..04f9a40 --- /dev/null +++ b/lakehouse/k8s_failed_cronjobs.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +""" +For every CronJob in the cluster, find its child Jobs +(via ownerReferences) and report any which failed. +Will automatically pick up new CronJob objects. + +Ensure that `ttlSecondsAfterFinished` is set to a high enough value +(e.g., 172800 for two days) in the CronJob. +Otherwise, Job objects will be purged too early, and the script +could report an inaccurate status. + +Nagios exit statuses (for use with mrpe): +Exits 2 if most recent Job in any CronJob failed +Exits 1 if any Job failed but a more recent Job of that CronJob succeeded +Exits 0 if all Jobs succeeded + +Requires: pip install kubernetes, credentials saved in default location +""" + +import kubernetes +import sys + +def load_kube_config(): + # Tries in-cluster config first (e.g. running as a Pod), falls back + # to your local kubeconfig (~/.kube/config or $KUBECONFIG). + # (for use in check_mk, will likely prefer $HOME/.kube/config + try: + kubernetes.config.load_incluster_config() + except kubernetes.config.ConfigException: + kubernetes.config.load_kube_config() + + +def main(): + try: + load_kube_config() + except kubernetes.config.ConfigException as exc: + print(f"can't load Kubernetes configuration: {exc}") + return 3 + try: + batch = kubernetes.client.BatchV1Api() + except: + print("can't connect to k8s") + + try: + cronjobs = batch.list_cron_job_for_all_namespaces().items + jobs = batch.list_job_for_all_namespaces().items + except: + print("can't get CronJobs from k8s") + + # Index jobs by their owning CronJob's uid + jobs_by_cronjob_uid = {} + for job in jobs: + for ref in job.metadata.owner_references or []: + if ref.kind == "CronJob": + try: + jobs_by_cronjob_uid.setdefault(ref.uid, []).append(job) + except: + print("can't get Jobs from k8s") + + cronjob_names = [] + failed_children = [] + status=3 + + for cj in cronjobs: + ns = cj.metadata.namespace + name = cj.metadata.name + uid = cj.metadata.uid + + cronjob_names.append(cj.metadata.namespace+'/'+cj.metadata.name) + children = sorted( + jobs_by_cronjob_uid.get(uid, []), + key=lambda j: ( + (j.status.start_time or j.metadata.creation_timestamp).timestamp() + if (j.status.start_time or j.metadata.creation_timestamp) + else 0.0 + ), + ) + + cronjobstatus=3 + # setting the status like this depends on jobs being sorted by date + # which the script doesn't currently check for + for job in children: + #print (str(job)) + conditions = job.status.conditions or [] + if any(c.type == "Failed" and c.status == "True" for c in conditions): + failed_children.append(f"{job.metadata.namespace}/{job.metadata.name}") + cronjobstatus=2 + # explicitly look for success + elif any(c.type == "Complete" and c.status == "True" for c in conditions): + if (len(failed_children) > 0): + cronjobstatus=1 + else: + cronjobstatus=0 + # otherwise this Job object won't change cronjobstatus + + # set global status based on status of previous and current CronJob objects + if (status == 3): + status=cronjobstatus + if (cronjobstatus > status and cronjobstatus != 3): + status=cronjobstatus + + if failed_children: + print(f"failed CronJobs: {', '.join(failed_children)} ; CronJobs checked: {', '.join(cronjob_names)}") + elif status == 0: +# print(str(status) + f" ok {ns}/{name} ({len(children)} job(s))") + print(f"all CronJobs OK, CronJobs checked: {', '.join(cronjob_names)}") + else: + print(f"status of CronJobs unknown, CronJobs checked: {', '.join(cronjob_names)}") + + # required in order to exit script with correct exit value + return status + +if __name__ == "__main__": + sys.exit(main())