nix-git/hosts/yodaHedgehog/host-specific.nix

# Suspend:
#   sudo systemctl suspend
# Suspend for 60 seconds:
#   sudo rtcwake -m mem -s 60

# View service log:
#   journalctl -u daily-backup-and-suspend

# Print unit file:
#   cat "$(systemctl show -P FragmentPath daily-backup-and-suspend.service)"

{ config, pkgs, ... }:
let
  backup-source = "rootNas";
  # The "stay-awake" file is located at `${backup-source}:${stay-awake-file}`.
  # Example: ssh rootNas 'touch yodaHedgehog.stay-awake'
  stay-awake-file = "${config.networking.hostName}.stay-awake";

  # How often to try to establish an SSH connection with ${backup-source}.
  retries = "10";
  # How many seconds to wait between failed SSH connection attempts to ${backup-source}.
  wait-seconds = "15";
in
{
  assertions = [{
    assertion = config.services.openssh.enable;
    message = "systemd service daily-backup-and-suspend requires SSH.";
  } {
    assertion = config.services.journalwatch.enable;
    message = "systemd service daily-backup-and-suspend requires journalwatch.";
  }];

  systemd.timers."daily-backup-and-suspend" = {
    wantedBy = [ "multi-user.target" ];
    timerConfig = {
      OnCalendar = [
        # Daily
        "*-*-* 12:05:00"
      ];
      WakeSystem = true;
    };
  };
  systemd.services."daily-backup-and-suspend" = {
    after = [ "network-online.target" ];
    wants = [ "network-online.target" ];
    # Packages required for this script.
    # For `ssh` and `journalwatch`, there are assertions above.
    path = with pkgs; [
      # Provides `ssh`
      openssh
      # Provides `sync`, `readlink` (with support for parameter `-e`, required by `btrbk`)
      coreutils
      # Provides `awk`, `grep`, `sleep`, `printf`, `echo`, 'sendmail', `readlink` (without support for parameter `-e`)
      busybox
      # Provides `smtpctl`
      opensmtpd
      # Provides `btrbk`
      btrbk
      # Provides `sudo` required by `btrbk`.
      # Alternatively we could configure `btrbk` to use the "btrfs-progs" instead of the "btrfs-progs-sudo" backend. But the `btrbk` NixOS module has no option for this.
      sudo
    ];
    # Script to execute as main process.
    script = ''
      set -eu -o pipefail

      for i in $(seq 1 ${retries}); do
        # Check if ${backup-source} is reachable via SSH.
        #
        # This check is useful if ${backup-source} is disconnected for a short period.
        # Additionally, this is necessary because of the following issue:
        #   If the system resumes at 12:05, it is not directly connected to the Internet, even if "after" and "wants" are set to "network-online.target".
        #   TODO: How can we fix this?
        #   TODO: Once fixed, send notification already after first failed connection attempt (instead of fourth).
        #
        result="$(ssh ${backup-source} 'echo ${backup-source}')" && e=0 || e=$?
        if [ "''${e}" = 0 ] && [ "''${result}" = ${backup-source} ]; then
          # Continue if successful.
          break
        fi
        # Otherwise do some error handling and try again.

        printf '%s\n' 'Delaying backup due to SSH connectivity problems.'
        # After the fourth failed connection attempt, send a notification by email.
        if [ "''${i}" = "4" ]; then
          printf '%s\n\n%s' 'Subject: ${config.networking.hostName}' 'Error connecting to ${backup-source}. Will retry in some seconds.' | sendmail -f langbein@mail.de daniel@systemli.org
        fi
        # After ${retries} failed connection attempts, send a second notification by email and give up.
        if [ "''${i}" = "${retries}" ]; then
          printf '%s\n\n%s' 'Subject: ${config.networking.hostName}' 'Error connecting to ${backup-source} for ${retries} times. Giving up!' | sendmail -f langbein@mail.de daniel@systemli.org
          exit 1
        fi
        # Wait some seconds before repeating.
        sleep "${wait-seconds}"s
      done

      # Pull BTRFS snapshots from ${backup-source}.
      btrbk -c /etc/btrbk/remote-backup-ssd.conf run
      btrbk -c /etc/btrbk/remote-backup-hdd.conf run

      # Don't suspend as long as `${backup-source}:${stay-awake-file}` exists.
      while :; do
        result="$(ssh ${backup-source} 'ls ${stay-awake-file} 2>&1')" ||:
        case "''${result}" in
          *'No such file or directory')
            break
            ;;
          '${stay-awake-file}')
            printf '%s\n' 'Delaying suspend due to ${stay-awake-file} file.'
            ;;
          *)
            printf '%s\n' 'Delaying suspend due to SSH connectivity problems.'
            ;;
        esac
        sleep 60s
      done

      # Wait until no BTRFS scrub service is running.
      while :; do
        running_services="$(systemctl list-units --type=service --plain --quiet | awk '{ print $1 }')"
        if ! printf '%s' "''${running_services}" | grep '^btrfs-scrub'; then
          break;
        fi
        printf '%s\n' 'Delaying suspend due to running BTRFS scrub service.'
        sleep 60s
      done

      # Send filtered journal by email.
      systemctl start journalwatch.service ||:
      # Send notification by email.
      printf '%s\n\n%s' 'Subject: ${config.networking.hostName}' 'Finished backup.' | sendmail -f langbein@mail.de daniel@systemli.org

      # Let sendmail send emails.
      #while :; do
      #  # TODO: Plain usage of `smtpctl` gives the error:
      #  #   smtpctl: this program must be setgid smtpq
      #  queue="$(smtpctl show queue)"
      #  if [ "''${queue}" = "" ]; then
      #    break
      #  fi
      #  printf '%s\n' 'Delaying suspend due to non-empty smtpd email queue.'
      #  sleep 1s
      #done
      sleep 15s

      #printf '%s\n' 'Finished backup script.'
      # Sync changed files to disk to reduce risk of file corruption in case of power loss.
      sync
      # Suspend to save power.
      systemctl suspend
    '';
  };
}