From 239354ab9f2002d3b5b1c7901e38d80aba7ace18 Mon Sep 17 00:00:00 2001 From: Daniel Langbein Date: Sat, 8 Feb 2025 15:14:33 +0100 Subject: [PATCH] btrbk: config update and zstd compression --- hosts/yodaHedgehog/btrbk-config.nix | 8 ++++---- hosts/yodaHedgehog/host-specific.nix | 2 ++ hosts/yodaNas/btrbk-config.nix | 2 +- modules/btrbk/README.md | 6 +++--- modules/btrbk/backup.nix | 9 ++++++--- modules/btrbk/default.nix | 16 ++++++++++++++++ 6 files changed, 32 insertions(+), 11 deletions(-) diff --git a/hosts/yodaHedgehog/btrbk-config.nix b/hosts/yodaHedgehog/btrbk-config.nix index 2ae35ec..c349b50 100644 --- a/hosts/yodaHedgehog/btrbk-config.nix +++ b/hosts/yodaHedgehog/btrbk-config.nix @@ -13,7 +13,6 @@ let "changedetection.p1st.de" = {}; "cloud.privacy1st.de" = {}; "cloud.fykml.eu" = {}; - "git.privacy1st.de" = {}; "mastodon-toot-follower.privacy1st.de" = {}; "money.p1st.de" = {}; "music.privacy1st.de" = {}; @@ -26,15 +25,16 @@ let "arch.p1st.de" = {}; "cloud.privacy1st.de" = {}; "cloud.fykml.eu" = {}; + "git.privacy1st.de" = {}; }; in { - # TODO: Enable lz4 when yodaHedgehog no longer has direct LAN connection. + # TODO: Disable zstd if yodaHedgehog has direct LAN connection to yodaHedgehog. yoda.btrbkBackups = [ { instance = "remote-backup-ssd"; enable = false; - lz4 = false; + zstd = true; ssh_identity = "/root/.ssh/rootNas_ed25519"; volume = "ssh://rootNas/jc-data"; snapshot_dir = "/snap"; @@ -44,7 +44,7 @@ in { instance = "remote-backup-hdd"; enable = false; - lz4 = false; + zstd = true; ssh_identity = "/root/.ssh/rootNas_ed25519"; volume = "ssh://rootNas/mnt/data/jc-data"; snapshot_dir = "/mnt/data/snap2"; diff --git a/hosts/yodaHedgehog/host-specific.nix b/hosts/yodaHedgehog/host-specific.nix index fccefd6..a86e085 100644 --- a/hosts/yodaHedgehog/host-specific.nix +++ b/hosts/yodaHedgehog/host-specific.nix @@ -56,6 +56,8 @@ in opensmtpd # Provides `btrbk` btrbk + # Provides `zstd` required by `btrbk`. + zstd # Provides `sudo` required by `btrbk`. # Alternatively we could configure `btrbk` to use the "btrfs-progs" instead of the "btrfs-progs-sudo" backend. But the `btrbk` NixOS module has no option for this. sudo diff --git a/hosts/yodaNas/btrbk-config.nix b/hosts/yodaNas/btrbk-config.nix index 76cc3c2..75d2026 100644 --- a/hosts/yodaNas/btrbk-config.nix +++ b/hosts/yodaNas/btrbk-config.nix @@ -5,7 +5,6 @@ let "changedetection.p1st.de" = {}; "cloud.privacy1st.de" = {}; "cloud.fykml.eu" = {}; - "git.privacy1st.de" = {}; "mastodon-toot-follower.privacy1st.de" = {}; "money.p1st.de" = {}; "music.privacy1st.de" = {}; @@ -18,6 +17,7 @@ let "arch.p1st.de" = {}; "cloud.privacy1st.de" = {}; "cloud.fykml.eu" = {}; + "git.privacy1st.de" = {}; }; in { diff --git a/modules/btrbk/README.md b/modules/btrbk/README.md index 4630de0..23cedb1 100644 --- a/modules/btrbk/README.md +++ b/modules/btrbk/README.md @@ -367,7 +367,7 @@ sudo btrbk -c ~/btrbk.cfg list # Configuration options for btrbk. Nested attrsets translate to subsections. settings = { timestamp_format = "long"; - stream_compress = "lz4"; + stream_compress = "zstd"; snapshot_preserve_min = "2d"; snapshot_preserve = "24h 7d 4w 6m"; @@ -396,7 +396,7 @@ Another remote backup server periodically pulls snapshots to create remote backu ```shell cat > ~/btrbk.cfg <<'EOF' timestamp_format long -stream_compress lz4 +stream_compress zstd ssh_identity /mnt/backup/rootNas_ed25519 # Create backups. @@ -417,7 +417,6 @@ volume ssh://rootNas/jc-data subvolume blogger.privacy1st.de subvolume changedetection.p1st.de subvolume cloud.privacy1st.de - subvolume git.privacy1st.de subvolume mastodon-toot-follower.privacy1st.de subvolume money.p1st.de subvolume music.privacy1st.de @@ -430,5 +429,6 @@ volume ssh://rootNas/mnt/data/jc-data target /mnt/backup/snap2 subvolume cloud.privacy1st.de subvolume cloud.fykml.eu + subvolume git.privacy1st.de EOF ``` diff --git a/modules/btrbk/backup.nix b/modules/btrbk/backup.nix index 4322428..6d2ff43 100644 --- a/modules/btrbk/backup.nix +++ b/modules/btrbk/backup.nix @@ -18,8 +18,8 @@ in # Can be used to disable timer. #enable = false; # Optional. - # If this is `true` and `volume` starts with `ssh://`, `lz4` transport compression is enabled. - lz4 = true; + # If this is `true` and `volume` starts with `ssh://`, `zstd` transport compression is enabled. + zstd = true; # Optional. #ssh_identity = /root/.ssh/rootNas_ed25519; volume = "/jc-data"; @@ -41,6 +41,8 @@ in # lib.attrsets.mergeAttrsList: # https://github.com/NixOS/nixpkgs/blob/54f00576aa6139a9d54062d0edc2fb31423f0ffb/lib/attrsets.nix#L786 config = { + # In case a btrbk instance uses stream_compress with zstd. + services.btrbk.extraPackages = [ pkgs.zstd ]; services.btrbk.instances = # Merge list of attr sets into one attr set. attrsets.mergeAttrsList ( @@ -53,7 +55,8 @@ in onCalendar = if (x.enable or true) then "12:05" else null; settings = { timestamp_format = "long"; - stream_compress = mkIf ((x.lz4 or false) && strings.hasPrefix "ssh://" x.volume) "lz4"; + stream_compress = mkIf ((x.zstd or false) && strings.hasPrefix "ssh://" x.volume) "zstd"; + stream_compress_level = mkIf ((x.zstd or false) && strings.hasPrefix "ssh://" x.volume) "9"; ssh_identity = mkIf (x?ssh_identity) x.ssh_identity; # Create backups. diff --git a/modules/btrbk/default.nix b/modules/btrbk/default.nix index 268f262..05bbf87 100644 --- a/modules/btrbk/default.nix +++ b/modules/btrbk/default.nix @@ -17,6 +17,22 @@ ./backup.nix ]; + # Compression + # + # `btrbk` can be configured to either + # - compress the BTRFS data stream itself + # - or to enable SSH compression + # + # The latter supports only ZLIB (LZ77) compression, see https://www.ietf.org/rfc/rfc4253.txt + # Thus, we choose the first option instead. + # + # lz4 is a good option, but zstd-9 seems to be better in the mean time. + # + # Install the compression package so that other hosts can pull compressed snapshots from us. + environment.systemPackages = [ + pkgs.zstd + ]; + services.btrbk = { # Lowest scheduling priority. niceness = 19;