From b2e0d29801f7f8cd2bb165f820c056d969b7f1d0 Mon Sep 17 00:00:00 2001
From: Daniel Langbein <daniel@systemli.org>
Date: Sun, 8 Oct 2023 20:03:00 +0200
Subject: [PATCH] monitor unhealthy containers

---
 hosts/yodaNas/host-specific.nix |  2 --
 modules/docker.nix              | 36 +++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/hosts/yodaNas/host-specific.nix b/hosts/yodaNas/host-specific.nix
index c7b6c65..f4824c6 100644
--- a/hosts/yodaNas/host-specific.nix
+++ b/hosts/yodaNas/host-specific.nix
@@ -144,6 +144,4 @@ in
     # Start after login.
     wantedBy = [ "multi-user.target" ];
   };
-
-  # TODO: Service which checks if all running docker containers are healthy. Also for yodaYoga.
 }
diff --git a/modules/docker.nix b/modules/docker.nix
index 510d4ec..cb1e25a 100644
--- a/modules/docker.nix
+++ b/modules/docker.nix
@@ -26,4 +26,40 @@
       ];
     };
   };
+
+  # Monitor unhealthy Docker containers.
+  systemd.timers."docker-health" = {
+    wantedBy = [ "timers.target" ];
+    partOf = [ "docker-health.service" ];
+    timerConfig = {
+      OnBootSec = "0m";
+      OnUnitInactiveSec = "3m";
+
+      AccuracySec = "15s";
+      RandomizedDelaySec = "15s";
+    };
+  };
+  systemd.services."docker-health" = {
+    serviceConfig = {
+      Type = "oneshot";
+      PrivateTmp = true;
+      # `docker` requires root access.
+      User = "root";
+      Nice = 19;
+      IOSchedulingClass = "idle";
+    };
+    path = with pkgs; [
+      docker
+    ];
+    # If there are no unhealthy Docker containers, the output of `docker ps -f health=unhealthy` is just one line:
+    #   CONTAINER ID   IMAGE     COMMAND   CREATED   STATUS    PORTS     NAMES
+    # We filter this line with `grep -v`.
+    # As a result, grep returns exit code 1 if there are no unhealthy containers (as not a single line is printed).
+    # Thus, we prefix the whole command with `!`.
+    # Lastly, we redirect stdout to stderr with `1>&2` so that unhealthy containers are written to stderr.
+    script = ''
+      set -eu -o pipefail
+      ! sudo docker ps -f health=unhealthy | grep -v 'CONTAINER ID   IMAGE     COMMAND   CREATED   STATUS    PORTS     NAMES' 1>&2
+    '';
+  };
 }