From d7b6ef5a659ff86df611e0e8c2d94fb4c984850b Mon Sep 17 00:00:00 2001
From: Daniel Langbein <daniel@systemli.org>
Date: Wed, 15 Nov 2023 13:54:37 +0100
Subject: [PATCH] yodaHedgehog: WIP backup service

---
 examples/block-while-remote-file-exists.sh | 19 ++++++
 examples/systemd-units.sh                  | 28 +++++++++
 hosts/yodaHedgehog/host-specific.nix       | 73 +++++++++++++++++++---
 3 files changed, 110 insertions(+), 10 deletions(-)
 create mode 100644 examples/block-while-remote-file-exists.sh
 create mode 100644 examples/systemd-units.sh

diff --git a/examples/block-while-remote-file-exists.sh b/examples/block-while-remote-file-exists.sh
new file mode 100644
index 0000000..a39fab8
--- /dev/null
+++ b/examples/block-while-remote-file-exists.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# As long as there is the file `stay-alive` at host `yodaNas`,
+# this loop blocks/waits.
+while :; do
+  result="$(ssh yodaNas 'ls stay-alive 2>&1')"
+  case "${result}" in
+    *"No such file or directory")
+      break
+      ;;
+    "stay-alive")
+      printf '%s\n' 'Delaying suspend due to stay-alive file.'
+      ;;
+    *)
+      printf '%s\n' 'Delaying suspend due to SSH connectivity problems.'
+      ;;
+  esac
+  sleep 10s
+done
diff --git a/examples/systemd-units.sh b/examples/systemd-units.sh
new file mode 100644
index 0000000..7037652
--- /dev/null
+++ b/examples/systemd-units.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Some examples on how to list and iterate over systemd units (e.g. services or timers).
+# With an additional example on how to check if one of multiple services is running.
+
+function all_timers(){
+  # Newline separated list of systemd timers.
+  timers="$(systemctl list-units --type=timer --plain --quiet | awk '{ print $1 }')"
+  # For $timer in $timers.
+  while IFS= read -r timer; do
+    echo "Timer: ${timer}"
+  done <<< "${timers}"
+}
+
+function btrfs_scrub_timers(){
+  # Newline separated list of systemd timers which start with `btrfs-scrub`.
+  timers="$(systemctl list-units --type=timer --plain --quiet | awk '{ print $1 }' | grep '^btrfs-scrub')"
+  # For $timer in $timers.
+  while IFS= read -r timer; do
+    echo "Timer: ${timer}"
+  done <<< "${timers}"
+}
+
+function running_btrfs_scrub(){
+  systemctl list-units --type=service --plain --quiet | awk '{ print $1 }' | grep '^btrfs-scrub'
+}
+
+running_btrfs_scrub
diff --git a/hosts/yodaHedgehog/host-specific.nix b/hosts/yodaHedgehog/host-specific.nix
index ba05a55..2543223 100644
--- a/hosts/yodaHedgehog/host-specific.nix
+++ b/hosts/yodaHedgehog/host-specific.nix
@@ -1,5 +1,9 @@
 { config, pkgs, ... }:
-
+let
+  backup-source = "rootNas";
+  # The "stay-awake" file is located at `${backup-source}:${stay-awake-file}`.
+  stay-awake-file = "${config.networking.hostName}.stay-awake";
+in
 {
   # Suspend:
   #   sudo systemctl suspend
@@ -12,24 +16,73 @@
   #    2.6W suspended, with 1 RAM, 1 SSD, 2 HDDs
   #   18.9W idle, with 1 RAM, 1 SSD, 2 HDDs
 
-  # journalctl -u regular-wakeup
+  # journalctl -u daily-backup-and-suspend
 
-  systemd.timers."regular-wakeup" = {
+  assertions = [{
+    assertion = config.services.openssh.enable;
+    message = "systemd service daily-backup-and-suspend requires SSH.";
+  } {
+    assertion = config.services.journalwatch.enable;
+    message = "systemd service daily-backup-and-suspend requires journalwatch.";
+  }];
+
+  systemd.timers."daily-backup-and-suspend" = {
     wantedBy = [ "multi-user.target" ];
     timerConfig = {
       OnCalendar = [
-        # Testing
-        "Sun 16:15:00"
         # Daily
-        # *-*-* 00:00:05
+        "*-*-* 00:00:05"
       ];
       WakeSystem = true;
-      };
-   };
-  systemd.services."regular-wakeup" = {
+    };
+  };
+  systemd.services."daily-backup-and-suspend" = {
+    # Packages required for this script.
+    # For `ssh` and `journalwatch`, there are assertions above.
+    path = with pkgs; [
+      # Provides `echo`, `sleep`, `printf`.
+      coreutils
+    ];
+    # Script to execute as main process.
     script = ''
       set -eu -o pipefail
-      printf '%s%s\n' 'Wokeup at ' "$(date)"
+      printf '%s\n' 'Starting backup script.'
+
+      # TODO: Backup: Pull BTRFS snapshots from ${backup-source}.
+
+      # Don't suspend as long as `${backup-source}:${stay-awake-file}` exists.
+      while :; do
+        result="$(ssh ${backup-source} 'ls ${stay-awake-file} 2>&1')"
+        case "$${result}" in
+          *"No such file or directory")
+            break
+            ;;
+          "${stay-awake-file}")
+            printf '%s\n' 'Delaying suspend due to ${stay-awake-file} file.'
+            ;;
+          *)
+            printf '%s\n' 'Delaying suspend due to SSH connectivity problems.'
+            ;;
+        esac
+        sleep 10s
+      done
+
+      # Wait until no BTRFS scrub service is running.
+      while systemctl list-units --type=service --plain --quiet | awk '{ print $1 }' | grep '^btrfs-scrub'; do
+        printf '%s\n' 'Delaying suspend due to running BTRFS scrub service.'
+        sleep 60s
+      done
+
+      # Send filtered journal by email.
+      systemctl start journalwatch.service ||:
+      # Short delay to let sendmail send the email.
+      sleep 15s
+
+      printf '%s\n' 'Finished backup script.'
+
+      # Suspend to save power.
+      # TODO
+      #systemctl suspend
     '';
   };
 }