nix-git/hosts/yodaHedgehog/host-specific.nix

# Suspend:
#   sudo systemctl suspend
# Suspend for 60 seconds:
#   sudo rtcwake -m mem -s 60

# View service log:
#   journalctl -u daily-backup-and-suspend

# Print unit file:
#   cat "$(systemctl show -P FragmentPath daily-backup-and-suspend.service)"

{ config, pkgs, ... }:
let
  backup-source = "rootNas";
  # The "stay-awake" file is located at `${backup-source}:${stay-awake-file}`.
  # Example: ssh rootNas 'touch yodaHedgehog.stay-awake'
  stay-awake-file = "${config.networking.hostName}.stay-awake";
in
{
  assertions = [{
    assertion = config.services.openssh.enable;
    message = "systemd service daily-backup-and-suspend requires SSH.";
  } {
    assertion = config.services.journalwatch.enable;
    message = "systemd service daily-backup-and-suspend requires journalwatch.";
  }];

  systemd.timers."daily-backup-and-suspend" = {
    wantedBy = [ "multi-user.target" ];
    timerConfig = {
      OnCalendar = [
        # Daily
        "*-*-* 00:05:00"
      ];
      WakeSystem = true;
    };
  };
  systemd.services."daily-backup-and-suspend" = {
    after = [ "network-online.target" ];
    # Packages required for this script.
    # For `ssh` and `journalwatch`, there are assertions above.
    path = with pkgs; [
      # Provides `ssh`
      openssh
      # Provides `sync`, `readlink` (with support for parameter `-e`, required by `btrbk`)
      coreutils
      # Provides `awk`, `grep`, `sleep`, `printf`, `echo`, 'sendmail', `readlink` (without support for parameter `-e`)
      busybox
      # Provides `smtpctl`
      opensmtpd
      # Provides `btrbk`
      btrbk
      # Provides `sudo` required by `btrbk`.
      # Alternatively we could configure `btrbk` to use the "btrfs-progs" instead of the "btrfs-progs-sudo" backend. But the `btrbk` NixOS module has no option for this.
      sudo
    ];
    # Script to execute as main process.
    script = ''
      set -eu -o pipefail
      #printf '%s\n' 'Starting backup script.'

      # Wait until ${backup-source} is reachable.
      #
      # This test is necessary because of the following:
      # If the system wakes up at 00:05, it is not directly connected to the Internet.
      # The config option `after = [ "network-online.target" ];` does not help in this regard.
      # Thus, `btrbk` might fail with the following error while connecting to ${backup-source}:
      #   ssh: Could not resolve hostname p1st.de: Name or service not known
      #
      while :; do
        result="$(ssh ${backup-source} 'echo ${backup-source}')" && e=0 || e=$?
        if [ "''${e}" = 0 ] && [ "''${result}" = '${backup-source}' ]; then
          break
        fi
        printf '%s\n' 'Delaying backup due to SSH connectivity problems.'
        sleep 10s
      done

      # Pull BTRFS snapshots from ${backup-source}.
      btrbk -c /etc/btrbk/remote-backup-ssd.conf run
      btrbk -c /etc/btrbk/remote-backup-hdd.conf run

      # Don't suspend as long as `${backup-source}:${stay-awake-file}` exists.
      while :; do
        result="$(ssh ${backup-source} 'ls ${stay-awake-file} 2>&1')" ||:
        case "''${result}" in
          *'No such file or directory')
            break
            ;;
          '${stay-awake-file}')
            printf '%s\n' 'Delaying suspend due to ${stay-awake-file} file.'
            ;;
          *)
            printf '%s\n' 'Delaying suspend due to SSH connectivity problems.'
            ;;
        esac
        sleep 60s
      done

      # Wait until no BTRFS scrub service is running.
      while :; do
        running_services="$(systemctl list-units --type=service --plain --quiet | awk '{ print $1 }')"
        if ! printf '%s' "''${running_services}" | grep '^btrfs-scrub'; then
          break;
        fi
        printf '%s\n' 'Delaying suspend due to running BTRFS scrub service.'
        sleep 60s
      done

      # Send filtered journal by email.
      systemctl start journalwatch.service ||:
      # Send notification by email.
      printf '%s\n\n%s' 'Subject: ${config.networking.hostName}' 'Finished backup.' | sendmail -f langbein@mail.de daniel@systemli.org

      # Let sendmail send emails.
      #while :; do
      #  # TODO: Plain usage of `smtpctl` gives the error:
      #  #   smtpctl: this program must be setgid smtpq
      #  queue="$(smtpctl show queue)"
      #  if [ "''${queue}" = "" ]; then
      #    break
      #  fi
      #  printf '%s\n' 'Delaying suspend due to non-empty smtpd email queue.'
      #  sleep 1s
      #done
      sleep 15s

      #printf '%s\n' 'Finished backup script.'
      # Sync changed files to disk to reduce risk of file corruption in case of power loss.
      sync
      # Suspend to save power.
      systemctl suspend
    '';
  };
}