diff --git a/modules/btrbk/README.md b/modules/btrbk/README.md new file mode 100644 index 0000000..ff2cae0 --- /dev/null +++ b/modules/btrbk/README.md @@ -0,0 +1,271 @@ +# btrbk + +* https://digint.ch/btrbk/doc/readme.html +* https://digint.ch/btrbk/doc/btrbk.1.html +* https://digint.ch/btrbk/doc/btrbk.conf.5.html + +> btrbk - backup tool for btrfs subvolumes + +- Take snapshots (on same disk) +- Create multiple backups (on different disks) over SSH + +## Snapshots and Backups + +subvolume: + +snapshot: .[] + +timestamp: Can be configured using the `timestamp_format` option. + +N: If multiple snapshots are created at the same time, the increasing counter N differentiates them. + +## Actions + +`run [filter...]`: In multiple steps, perform snapshots and backups. +* Read Data: Perform sanity checks +* Create Snapshots +* Create Backups +* Delete Backups +* Delete Snapshots + +`clean [filter...]`: Delete incomplete backups (network error, kill signal while send/receive was ongoing). + +## Configuration + +### General + +`volume `: Usually the mount point of a btrfs filesystem mounted with `subvolid=5`. + +`subvolume `: Subvolume to be backed up, relative to ``. + +`snapshot_dir `: Directory in which the btrfs snapshots are created, relative to ``. Hast to be created **manually**! + +`target |`: Target directory where the backup subvolumes are to be created. Multiple target sections are allowed. + +`url`: Accepted formats are: +* `ssh://[:]/` +* `:` + +`hostname`: Either a host name, IPv4 or IPv6 address + +`timestamp_format short|long|long-iso`: This becomes relevant for setups with multiple btrbk instances, e.g. many snapshot-only instances (spread around the world), and a fetch-only instance on the backup server. Make sure to run btrbk with the same time zone on every host, e.g. by setting the TZ environment variable (see tzset(3)). + +### Retention + +`snapshot_preserve ` + +`snapshot_preserve_min all|latest|{h,d,w,m,y}`: Preserve all snapshots for a minimum amount of hours (h), days (d), weeks (w), months (m) or years (y). + +`target_preserve ` + +`target_preserve_min all|latest|no|{h,d,w,m,y}`: Preserve all snapshots for a minimum amount of hours (h), days (d), weeks (w), months (m) or years (y). If set to "no", only the backups following the `target_preserve` policy are created. + +``: `[h] [d] [w] [m] [y]`. Defines how many backups of each period should be preserved. + +### SSH and Data Stream Options + +`ssh_compression yes|no` + +`stream_compress `: Compress the btrfs send stream before transferring it from/to remote locations. + +``: gzip, pigz, bzip2, pbzip2, bzip3, xz, lzo, lz4, zstd. + +`stream_buffer |no`: This can give a speed improvement (measured up to 20%) on both local or remote operations, but also increases system load. Make sure that the `mbuffer` command is available. Leave this option disabled if your main concern is a stable backup process: while recent versions of mbuffer have proven reliable, it is often desirable to keep things simple rather than adding an additional, multi-threaded process to the command pipe. + +`rate_limit |no`: Limit the read rate of the btrfs send stream to `` bytes per second. Implies the `stream_buffer` option. + +### System Options + +`transaction_syslog |no`: Log all transactions to syslog. + +`lockfile |no` + +## Example: Local Snapshots and Backup + +``` +timestamp_format long + +snapshot_preserve_min 2d +snapshot_preserve 24h 7d 4w 6m + +target_preserve_min no +target_preserve 7d 4w 6m + +volume /mnt/12tb1-top-level + snapshot_dir @snap + target /mnt/12tb2-top-level/@snap + subvolume @home + subvolume @ +``` + +Mount top-level subvolume: + +```shell +mount -o subvolid=5 /dev/mapper/some-disk /mnt/12tb1-top-level +``` + +Manually create snapshot directory: + +```shell +btrfs subvolume create /mnt/12tb1-top-level/@snap +btrfs subvolume create /mnt/12tb2-top-level/@snap +``` + +Snapshots only: + +```shell +btrbk snapshot +``` + +Backup only: + +```shell +btrbk resume +``` + +Create an additional snapshot of @home: + +```shell +btrbk snapshot @home +``` + +## Testing on Local Host + +```shell +sudo mkdir -p /mnt/top-level +ls /mnt/top-level/@ || sudo mount -o subvolid=5 /dev/mapper/luks-ea7099e3-320d-4eb3-a4c3-9910a9af817b /mnt/top-level +ls /mnt/top-level/@snap || sudo btrfs subvolume create /mnt/top-level/@snap +ls /mnt/top-level/@foo || { + sudo btrfs subvolume create /mnt/top-level/@foo + sudo touch /mnt/top-level/@foo/bar +} +``` + +```shell +ls ~/btrbk-usb.img || { + fallocate -l 1G ~/btrbk-usb.img + sudo mkfs.btrfs ~/btrbk-usb.img + # Mount and create subvolume @ +} + +sudo mkdir -p /mnt/usb-top-level +ls /mnt/usb-top-level/@ || sudo mount -o subvolid=5 ~/btrbk-usb.img /mnt/usb-top-level +ls /mnt/usb-top-level/@snap || sudo btrfs subvolume create /mnt/usb-top-level/@snap +} +``` + +```shell +cat > ~/btrbk.cfg <<'EOF' +timestamp_format long + +snapshot_preserve_min 2d +snapshot_preserve 24h 7d 4w 6m + +target_preserve_min no +target_preserve 7d 4w 6m + +volume /mnt/top-level + snapshot_dir @snap + target /mnt/usb-top-level/@snap + subvolume @foo +EOF +``` + +```shell +# Check configuration. +sudo btrbk -c ~/btrbk.cfg config print +sudo btrbk -c ~/btrbk.cfg config list + + +# Dry run. +sudo btrbk -c ~/btrbk.cfg run -n +# Create first snapshot. +sudo btrbk -c ~/btrbk.cfg run +# Print schedule. +sudo btrbk -c ~/btrbk.cfg run -n -S + +sleep 5m +sudo btrbk -c ~/btrbk.cfg run + +# List snapshots +sudo btrbk -c ~/btrbk.cfg list +``` + +### NixOS Config + +```nix +{ config, pkgs, ... }: +{ + services.btrbk = { + extraPackages = [ pkgs.lz4 ]; + # Lowest scheduling priority. + niceness = 19; + # Set of btrbk instances. The instance named btrbk is the default one. + instances = { + # This is the configuration equivalent to + # README.md section "Testing on Local Host". + "testing-on-local-host" = { + # The timer is disabled. The service has to be invoked manually. + onCalendar = null; + #onCalendar = "hourly"; + #onCalendar = "*:0/15"; # Every 15min + # Configuration options for btrbk. Nested attrsets translate to subsections. + settings = { + timestamp_format = "long"; + stream_compress = "lz4"; + + snapshot_preserve_min = "2d"; + snapshot_preserve = "24h 7d 4w 6m"; + + target_preserve_min = "no"; + target_preserve = "7d 4w 6m"; + + volume."/mnt/top-level" = { + snapshot_dir = "@snap"; + target = "/mnt/usb-top-level/@snap"; + subvolume = "@foo"; + }; + }; + }; + }; + }; +} +``` + +## pi3bplus Host Config + +A local `btrbk` instance manages snapshots and local backups. + +Another remote backup server periodically pulls snapshots to create remote backups. Here is its config: + +```shell +cat > ~/btrbk.cfg <<'EOF' +timestamp_format long +ssh_identity /mnt/backup/rootnas_ed25519 + +target_preserve_min no +# Daily backups -> @daily cron entry +target_preserve 7d 4w 6m +# Hourly backups -> @hourly cron entry +#target_preserve 24h 7d 4w 6m + +# Don't create or delete snapshots on remote server "rootnas". +snapshot_preserve_min all +snapshot_create no + +volume ssh://rootnas/mnt/data/data + snapshot_dir /mnt/data/snap + target /mnt/backup/snap + subvolume arch.p1st.de + subvolume blogger.privacy1st.de + subvolume changedetection.p1st.de + subvolume cloud.media-kollektiv.eu + subvolume cloud.privacy1st.de + subvolume git.privacy1st.de + subvolume money.p1st.de + subvolume music.privacy1st.de + subvolume proxy + subvolume recipe.privacy1st.de + subvolume traggo.privacy1st.de +EOF +``` diff --git a/modules/btrbk/default.nix b/modules/btrbk/default.nix index 8afb8aa..839d271 100644 --- a/modules/btrbk/default.nix +++ b/modules/btrbk/default.nix @@ -45,32 +45,6 @@ # Set of btrbk instances. The instance named btrbk is the default one. instances = { -# # This is the configuration equivalent to -# # README.md section "Testing on Local Host". -# "testing-on-local-host" = { -# # The timer is disabled. The service has to be invoked manually. -# onCalendar = null; -# #onCalendar = "hourly"; -# #onCalendar = "*:0/15"; # Every 15min -# # Configuration options for btrbk. Nested attrsets translate to subsections. -# settings = { -# timestamp_format = "long"; -# stream_compress = "lz4"; -# -# snapshot_preserve_min = "2d"; -# snapshot_preserve = "24h 7d 4w 6m"; -# -# target_preserve_min = "no"; -# target_preserve = "7d 4w 6m"; -# -# volume."/mnt/top-level" = { -# snapshot_dir = "@snap"; -# target = "/mnt/usb-top-level/@snap"; -# subvolume = "@foo"; -# }; -# }; -# }; - "local-snapshots" = { onCalendar = "hourly"; #onCalendar = "*:0/15"; # Every 15min