btrbk: config update and zstd compression

hosts/yodaHedgehog/btrbk-config.nix

@@ -13,7 +13,6 @@ let
     "changedetection.p1st.de" = {};
     "cloud.privacy1st.de" = {};
     "cloud.fykml.eu" = {};
-    "git.privacy1st.de" = {};
     "mastodon-toot-follower.privacy1st.de" = {};
     "money.p1st.de" = {};
     "music.privacy1st.de" = {};
@@ -26,15 +25,16 @@ let
     "arch.p1st.de" = {};
     "cloud.privacy1st.de" = {};
     "cloud.fykml.eu" = {};
+    "git.privacy1st.de" = {};
   };
 in
 {
-  # TODO: Enable lz4 when yodaHedgehog no longer has direct LAN connection.
+  # TODO: Disable zstd if yodaHedgehog has direct LAN connection to yodaHedgehog.
   yoda.btrbkBackups = [
     {
       instance = "remote-backup-ssd";
       enable = false;
-      lz4 = false;
+      zstd = true;
       ssh_identity = "/root/.ssh/rootNas_ed25519";
       volume = "ssh://rootNas/jc-data";
       snapshot_dir = "/snap";
@@ -44,7 +44,7 @@ in
     {
       instance = "remote-backup-hdd";
       enable = false;
-      lz4 = false;
+      zstd = true;
       ssh_identity = "/root/.ssh/rootNas_ed25519";
       volume = "ssh://rootNas/mnt/data/jc-data";
       snapshot_dir = "/mnt/data/snap2";

hosts/yodaHedgehog/host-specific.nix

@@ -56,6 +56,8 @@ in
       opensmtpd
       # Provides `btrbk`
       btrbk
+      # Provides `zstd` required by `btrbk`.
+      zstd
       # Provides `sudo` required by `btrbk`.
       # Alternatively we could configure `btrbk` to use the "btrfs-progs" instead of the "btrfs-progs-sudo" backend. But the `btrbk` NixOS module has no option for this.
       sudo

hosts/yodaNas/btrbk-config.nix

@@ -5,7 +5,6 @@ let
     "changedetection.p1st.de" = {};
     "cloud.privacy1st.de" = {};
     "cloud.fykml.eu" = {};
-    "git.privacy1st.de" = {};
     "mastodon-toot-follower.privacy1st.de" = {};
     "money.p1st.de" = {};
     "music.privacy1st.de" = {};
@@ -18,6 +17,7 @@ let
     "arch.p1st.de" = {};
     "cloud.privacy1st.de" = {};
     "cloud.fykml.eu" = {};
+    "git.privacy1st.de" = {};
   };
 in
 {

modules/btrbk/README.md

@@ -367,7 +367,7 @@ sudo btrbk -c ~/btrbk.cfg list
         # Configuration options for btrbk. Nested attrsets translate to subsections.
         settings = {
           timestamp_format = "long";
-          stream_compress = "lz4";
+          stream_compress = "zstd";
 
           snapshot_preserve_min = "2d";
           snapshot_preserve = "24h 7d 4w 6m";
@@ -396,7 +396,7 @@ Another remote backup server periodically pulls snapshots to create remote backu
 ```shell
 cat > ~/btrbk.cfg <<'EOF'
 timestamp_format    long
-stream_compress     lz4
+stream_compress     zstd
 ssh_identity        /mnt/backup/rootNas_ed25519
 
 # Create backups.
@@ -417,7 +417,6 @@ volume ssh://rootNas/jc-data
   subvolume    blogger.privacy1st.de
   subvolume    changedetection.p1st.de
   subvolume    cloud.privacy1st.de
-  subvolume    git.privacy1st.de
   subvolume    mastodon-toot-follower.privacy1st.de
   subvolume    money.p1st.de
   subvolume    music.privacy1st.de
@@ -430,5 +429,6 @@ volume ssh://rootNas/mnt/data/jc-data
   target       /mnt/backup/snap2
   subvolume    cloud.privacy1st.de
   subvolume    cloud.fykml.eu
+  subvolume    git.privacy1st.de
 EOF
 ```

modules/btrbk/backup.nix

@@ -18,8 +18,8 @@ in
         # Can be used to disable timer.
         #enable = false;
         # Optional.
-        # If this is `true` and `volume` starts with `ssh://`, `lz4` transport compression is enabled.
+        # If this is `true` and `volume` starts with `ssh://`, `zstd` transport compression is enabled.
-        lz4 = true;
+        zstd = true;
         # Optional.
         #ssh_identity = /root/.ssh/rootNas_ed25519;
         volume = "/jc-data";
@@ -41,6 +41,8 @@ in
   # lib.attrsets.mergeAttrsList:
   #   https://github.com/NixOS/nixpkgs/blob/54f00576aa6139a9d54062d0edc2fb31423f0ffb/lib/attrsets.nix#L786
   config = {
+    # In case a btrbk instance uses stream_compress with zstd.
+    services.btrbk.extraPackages = [ pkgs.zstd ];
     services.btrbk.instances =
       # Merge list of attr sets into one attr set.
       attrsets.mergeAttrsList (
@@ -53,7 +55,8 @@ in
               onCalendar = if (x.enable or true) then "12:05" else null;
               settings = {
                 timestamp_format = "long";
-                stream_compress =  mkIf ((x.lz4 or false) && strings.hasPrefix "ssh://" x.volume) "lz4";
+                stream_compress =       mkIf ((x.zstd or false) && strings.hasPrefix "ssh://" x.volume) "zstd";
+                stream_compress_level = mkIf ((x.zstd or false) && strings.hasPrefix "ssh://" x.volume) "9";
                 ssh_identity = mkIf (x?ssh_identity) x.ssh_identity;
 
                 # Create backups.

modules/btrbk/default.nix

@@ -17,6 +17,22 @@
     ./backup.nix
   ];
 
+  # Compression
+  #
+  # `btrbk` can be configured to either
+  # - compress the BTRFS data stream itself
+  # - or to enable SSH compression
+  #
+  # The latter supports only ZLIB (LZ77) compression, see https://www.ietf.org/rfc/rfc4253.txt
+  # Thus, we choose the first option instead.
+  #
+  # lz4 is a good option, but zstd-9 seems to be better in the mean time.
+  #
+  # Install the compression package so that other hosts can pull compressed snapshots from us.
+  environment.systemPackages = [
+    pkgs.zstd
+  ];
+
   services.btrbk = {
     # Lowest scheduling priority.
     niceness = 19;