monitor unhealthy containers

hosts/yodaNas/host-specific.nix

@@ -144,6 +144,4 @@ in
     # Start after login.
     wantedBy = [ "multi-user.target" ];
   };
-
-  # TODO: Service which checks if all running docker containers are healthy. Also for yodaYoga.
 }

modules/docker.nix

@@ -26,4 +26,40 @@
       ];
     };
   };
+
+  # Monitor unhealthy Docker containers.
+  systemd.timers."docker-health" = {
+    wantedBy = [ "timers.target" ];
+    partOf = [ "docker-health.service" ];
+    timerConfig = {
+      OnBootSec = "0m";
+      OnUnitInactiveSec = "3m";
+
+      AccuracySec = "15s";
+      RandomizedDelaySec = "15s";
+    };
+  };
+  systemd.services."docker-health" = {
+    serviceConfig = {
+      Type = "oneshot";
+      PrivateTmp = true;
+      # `docker` requires root access.
+      User = "root";
+      Nice = 19;
+      IOSchedulingClass = "idle";
+    };
+    path = with pkgs; [
+      docker
+    ];
+    # If there are no unhealthy Docker containers, the output of `docker ps -f health=unhealthy` is just one line:
+    #   CONTAINER ID   IMAGE     COMMAND   CREATED   STATUS    PORTS     NAMES
+    # We filter this line with `grep -v`.
+    # As a result, grep returns exit code 1 if there are no unhealthy containers (as not a single line is printed).
+    # Thus, we prefix the whole command with `!`.
+    # Lastly, we redirect stdout to stderr with `1>&2` so that unhealthy containers are written to stderr.
+    script = ''
+      set -eu -o pipefail
+      ! sudo docker ps -f health=unhealthy | grep -v 'CONTAINER ID   IMAGE     COMMAND   CREATED   STATUS    PORTS     NAMES' 1>&2
+    '';
+  };
 }