Mirror/nix-git
Mirror/nix-git
1
0
Fork 0
mirror of https://codeberg.org/privacy1st/nix-git synced 2024-11-21 22:03:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

improved systemd service

Browse Source 
add wants network-online.target, better error handling
This commit is contained in:
Daniel Langbein 2024-08-28 21:03:14 +02:00
parent 4f3b0745ee
commit a52c830c3e
Signed by: langfingaz
GPG Key ID: 6C47C753F0823002
1 changed files with 30 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
hosts/yodaHedgehog/host-specific.nix
View File

@ -15,6 +15,11 @@ let
# The "stay-awake" file is located at `${backup-source}:${stay-awake-file}`.
# Example: ssh rootNas 'touch yodaHedgehog.stay-awake'
stay-awake-file = "${config.networking.hostName}.stay-awake";
# How often to try to establish an SSH connection with ${backup-source}.
retries = "10";
# How many seconds to wait between failed SSH connection attempts to ${backup-source}.
wait-seconds = "15";
in
{
assertions = [{
@ -37,6 +42,7 @@ in
};
systemd.services."daily-backup-and-suspend" = {
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
# Packages required for this script.
# For `ssh` and `journalwatch`, there are assertions above.
path = with pkgs; [
@ -57,23 +63,35 @@ in
# Script to execute as main process.
script = ''
set -eu -o pipefail
#printf '%s\n' 'Starting backup script.'
# Wait until ${backup-source} is reachable.
for i in $(seq 1 ${retries}); do
# Check if ${backup-source} is reachable via SSH.
#
# This test is necessary because of the following:
# If the system wakes up at 12:05, it is not directly connected to the Internet.
# The config option `after = [ "network-online.target" ];` does not help in this regard.
# Thus, `btrbk` might fail with the following error while connecting to ${backup-source}:
# ssh: Could not resolve hostname p1st.de: Name or service not known
# This check is useful if ${backup-source} is disconnected for a short period.
# Additionally, this is necessary because of the following issue:
# If the system resumes at 12:05, it is not directly connected to the Internet, even if "after" and "wants" are set to "network-online.target".
# TODO: How can we fix this?
# TODO: Once fixed, send notification already after first failed connection attempt (instead of fourth).
#
while :; do
result="$(ssh ${backup-source} 'echo ${backup-source}')" && e=0 || e=$?
if [ "''${e}" = 0 ] && [ "''${result}" = '${backup-source}' ]; then
if [ "''${e}" = 0 ] && [ "''${result}" = ${backup-source} ]; then
# Continue if successful.
break
fi
# Otherwise do some error handling and try again.
printf '%s\n' 'Delaying backup due to SSH connectivity problems.'
sleep 10s
# After the fourth failed connection attempt, send a notification by email.
if [ "''${i}" = "4" ]; then
printf '%s\n\n%s' 'Subject: ${config.networking.hostName}' 'Error connecting to ${backup-source}. Will retry in some seconds.' | sendmail -f langbein@mail.de daniel@systemli.org
fi
# After ${retries} failed connection attempts, send a second notification by email and give up.
if [ "''${i}" = "${retries}" ]; then
printf '%s\n\n%s' 'Subject: ${config.networking.hostName}' 'Error connecting to ${backup-source} for ${retries} times. Giving up!' | sendmail -f langbein@mail.de daniel@systemli.org
exit 1
fi
# Wait some seconds before repeating.
sleep "${wait-seconds}"s
done
# Pull BTRFS snapshots from ${backup-source}.