{ config, pkgs, ... }: { # Systemd Journal Monitoring. # Alternative: # journal-biref # https://github.com/twaugh/journal-brief # https://opensource.com/article/20/7/systemd-journals-email # Write to Systemd Journal: # echo 'hello' | systemd-cat -p emerg # echo 'hello' | systemd-cat -t someapp -p emerg # View Systemd Journal. # Output similar to dmesg # journalctl -b -k # Filter by app: # journalctl -b -t someapp # Filter by priority: # journalctl -b -p 5 # Manually execute journalwatch timer: # sudo systemctl start journalwatch.service # Find a message and view its details # journalctl -b -p5 -o json-pretty # Then press "/" and enter a pattern, then press "Enter". assertions = [{ assertion = config.services.opensmtpd.enable; message = "journalwatch requires a configured sendmail MTA, see sendmail-mta.nix."; }]; services.journalwatch = { enable = true; # TODO: Same as configured by sendmail MTA. mailFrom = "langbein@mail.de"; mailTo = "daniel+journalwatch@systemli.org"; #interval = "hourly"; # Lowest priority of message to be considered. A value between 7 (“debug”), and 0 (“emerg”). Defaults to 6 (“info”). If you don't care about anything with “info” priority, you can reduce this to e.g. 5 (“notice”) to considerably reduce the amount of messages without needing many filterBlocks. priority = 5; # Default patterns: https://github.com/The-Compiler/journalwatch/blob/363725ac4b8aa841d87654fa8a63403a59ad1275/journalwatch.py#L71 # If the value of `match` starts and ends with a forward-slash, it is interpreted as a regular expression, if not, it's an exact match. # `filters` are always regular expressions. # All regular expressions have to match the full string! # # TODO: A "\s" in the double-quoted string `match` is inserted as `s` in the generated config file! We have to use "\\s" to insert `\s`. # TODO: A ''\s'' in the multiline string `filter` is inserted as `\s` in the generated config file. # filterBlocks = [ # # _TRANSPORT # { # yodaNas filters = '' booting system configuration /nix/store/\S+\.05pre-git ''; match = "_TRANSPORT = kernel"; } # # _EXE # { # yodaNas filters = '' # Ignore any invocation of sudo. .* ''; match = "_EXE = //nix/store/[a-z0-9]+-sudo-[0-9]+\.[0-9]+\.[0-9]+[a-z0-9]+/bin/sudo/"; } # # _SYSTEMD_CGROUP # { # yodaYoga filters = '' parent not found! continent_id [0-9]+ ''; match = "_SYSTEMD_CGROUP = /system.slice/docker.service"; } # # CONTAINER_NAME # { # yodaNas filters = '' .* ''; match = "CONTAINER_NAME = /(docker-compose-btp-proxy-1|nc_web_[^-_\\s]+|nginx_arch\.p1st\.de|recipe\.privacy1st\.de_nginx)/"; } { # yodaNas # TODO: Open issue on GitHub https://github.com/nginx-proxy/nginx-proxy/issues/1256. Maybe set env variable RESOLVERS=1.1.1.1? filters = '' \S+ \S+ \[warn\] [0-9]+#[0-9]+: no resolver defined to resolve r3\.o\.lencr\.org while requesting certificate status, responder: r3\.o\.lencr\.org, certificate: "/etc/nginx/certs/\S+\.crt" \S+ \S+ \[error\] [0-9]+#[0-9]+: OCSP responder sent invalid "Content-Type" header: "text/html" while requesting certificate status, responder: r3\.o\.lencr\.org, peer: \S+, certificate: "/etc/nginx/certs/\S+\.crt" \S+ \S+ \[error\] [0-9]+#[0-9]+: \*[0-9]+ connect\(\) failed \(111: Connection refused\) while connecting to upstream, client: \S+, server: \S+, request: "[^"]+", upstream: "[^"]+", host: "[^"]+" ''; match = "CONTAINER_NAME = nginx-proxy"; } # # IMAGE_NAME # { # yodaNas, yodaYoga filters = '' .* ''; match = "IMAGE_NAME = /(p1st/docker-gen:.+|nginxproxy/acme-companion|biketripplanner/digitransit-ui:.+|thetorproject/snowflake-proxy:.+|collabora/code|wordpress|danielszabo99/microbin|p1st/mastodon-toot-follower:.+|browserless/chrome)/"; } { # yodaNas filters = '' \S+ \S+ \[error\] [0-9]+#[0-9]+: \S+ open\(\) "/usr/share/nginx/html/robots\.txt" failed \(2: No such file or directory\), client: \S+, server: localhost, request: "GET /robots\.txt HTTP/[^"]+", host: "[^"]+" # \S+ \S+ \[notice\] [0-9]+#[0-9]+: signal 3 \(SIGQUIT\) received, shutting down \S+ \S+ \[notice\] [0-9]+#[0-9]+: gracefully shutting down \S+ \S+ \[notice\] [0-9]+#[0-9]+: exiting \S+ \S+ \[notice\] [0-9]+#[0-9]+: exit \S+ \S+ \[notice\] [0-9]+#[0-9]+: signal 17 \(SIGCHLD\) received from [0-9]+ \S+ \S+ \[notice\] [0-9]+#[0-9]+: worker process [0-9]+ exited with code 0 # \S+ \S+ \[notice\] [0-9]+#[0-9]+: signal 1 \(SIGHUP\) received, reconfiguring \S+ \S+ \[notice\] [0-9]+#[0-9]+: reconfiguring \S+ \S+ \[notice\] [0-9]+#[0-9]+: using the "epoll" event method \S+ \S+ \[notice\] [0-9]+#[0-9]+: nginx/\S+ \S+ \S+ \[notice\] [0-9]+#[0-9]+: built by gcc \S+ \S+ \(Alpine \S+\) \S+ \S+ \[notice\] [0-9]+#[0-9]+: OS: Linux \S+ \S+ \S+ \[notice\] [0-9]+#[0-9]+: getrlimit\(RLIMIT_NOFILE\): [0-9]+:[0-9]+ \S+ \S+ \[notice\] [0-9]+#[0-9]+: start worker processes \S+ \S+ \[notice\] [0-9]+#[0-9]+: start worker process [0-9]+ ''; match = "IMAGE_NAME = /nginx:\S+/"; } { # yodaNas # TODO: logged IP is not the public one, but always 172.24.0.6 filters = '' Could not yet connect with DB\. Retrying in 10s \.\.\. Nextcloud or one of the apps require upgrade - only a limited number of commands are available You may use your browser or the occ upgrade command to do the upgrade \[\S+ \S+\] NOTICE: ready to handle connections [0-9] \[[>-]+\] [0-9] \[[>-]+\] [0-9] \[[>-]+\] \[\S+ \S+\] NOTICE: fpm is running, pid [0-9]+ \[\S+ \S+\] NOTICE: Finishing \.\.\. \[\S+ \S+\] NOTICE: exiting, bye-bye! # Ignore: At least one idle child was left. \[\S+ \S+\] WARNING: \[pool www\] seems busy \(you may need to increase pm\.start_servers, or pm\.min/max_spare_servers\), spawning [0-9]+ children, there are [1-9][0-9]* idle, and [0-9]+ total children # # 1.1.1.1 - 28/Sep/2023:21:03:39 +0000 "GET /status.php" 200 # 1.1.1.1 - 28/Sep/2023:21:12:16 +0000 "GET /index.php" 200 # 1.1.1.1 - my-username 28/Sep/2023:21:20:16 +0000 "DELETE /index.php" 200 # 1.1.1.1 - my-username 28/Sep/2023:21:10:18 +0000 "PROPFIND /remote\.php" 207 # 1.1.1.1 - my-username 28/Sep/2023:21:11:48 +0000 "GET /ocs/v2.php" 304 # 1.1.1.1 - 28/Sep/2023:21:13:10 +0000 "GET /ocs/v2.php" 304 [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(DELETE|GET|HEAD|PATCH|POST|PROPFIND|PUT) /(index|status)\.php" (200|204|302|303|304|401|404|405|412|500) [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(DELETE|GET|HEAD|MKCOL|MOVE|OPTIONS|POST|PROPFIND|PUT|REPORT|SEARCH) /remote\.php" (200|201|204|207|401|404|405|423|500) [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(DELETE|GET|POST|PUT) /ocs/(v1|v2)\.php" (200|202|204|304|404) [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "GET /(ocm|ocs)-provider/index\.php" 200 [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "PROPFIND /public\.php" (201|207|401|404) ''; match = "IMAGE_NAME = p1st/nextcloud:stable-fpm-alpine"; } { # yodaNas filters = '' \S+ \S+ 0 \[Note\] Starting MariaDB \S+ source revision \S+ as process 1 \S+ \S+ 0 \[Note\] InnoDB: Compressed tables use zlib \S+ \S+ \S+ 0 \[Note\] InnoDB: Number of transaction pools: 1 \S+ \S+ 0 \[Note\] InnoDB: Using crc32 \+ pclmulqdq instructions \S+ \S+ 0 \[Note\] InnoDB: Using liburing \S+ \S+ 0 \[Note\] InnoDB: Initializing buffer pool, total size = 1\.000GiB, chunk size = 16\.000MiB \S+ \S+ 0 \[Note\] InnoDB: Completed initialization of buffer pool \S+ \S+ 0 \[Note\] InnoDB: Buffered log writes \(block size=512 bytes\) \S+ \S+ 0 \[Note\] InnoDB: End of log at LSN=\S+ \S+ \S+ 0 \[Note\] InnoDB: Opened 3 undo tablespaces \S+ \S+ 0 \[Note\] InnoDB: 128 rollback segments in 3 undo tablespaces are active\. \S+ \S+ 0 \[Note\] InnoDB: 128 rollback segments are active\. \S+ \S+ 0 \[Note\] InnoDB: Setting file '\./\S+' size to 12\.000MiB\. Physically writing the file full; Please wait \.\.\. \S+ \S+ 0 \[Note\] InnoDB: File '\./\S+' size is now 12\.000MiB\. \S+ \S+ 0 \[Note\] InnoDB: log sequence number \S+; transaction id \S+ \S+ \S+ 0 \[Note\] InnoDB: Loading buffer pool\(s\) from /var/lib/mysql/ib_buffer_pool \S+ \S+ 0 \[Note\] InnoDB: Read redo log up to LSN=\S+ \S+ \S+ 0 \[Note\] InnoDB: Buffer pool\(s\) load completed at \S+ \S+ \S+ \S+ 0 \[Note\] Plugin 'FEEDBACK' is disabled\. \S+ \S+ 0 \[Note\] Server socket created on IP: '0.0.0.0'\. \S+ \S+ 0 \[Note\] Server socket created on IP: '::'\. \S+ \S+ 0 \[Note\] mariadbd: ready for connections\. Version: '\S+' socket: '/run/mysqld/mysqld.sock' port: \S+ mariadb\.org binary distribution # \S+ \S+ 0 \[Note\] mariadbd \(initiated by: unknown\): Normal shutdown \S+ \S+ 0 \[Note\] InnoDB: FTS optimize thread exiting\. \S+ \S+ 0 \[Note\] InnoDB: Starting shutdown\.\.\. \S+ \S+ 0 \[Note\] InnoDB: Dumping buffer pool\(s\) to /var/lib/mysql/ib_buffer_pool \S+ \S+ 0 \[Note\] InnoDB: Buffer pool\(s\) dump completed at \S+ \S+ \S+ \S+ 0 \[Note\] InnoDB: Removed temporary tablespace data file: "\./ibtmp1" \S+ \S+ 0 \[Note\] InnoDB: Shutdown completed; log sequence number \S+; transaction id \S+ \S+ \S+ 0 \[Note\] mariadbd: Shutdown complete \s* ''; match = "IMAGE_NAME = /mariadb(:.+)?/"; } { # yodaNas filters = '' \S+ \S+ UTC \[[0-9]+\] LOG: listening on IPv4 address "0\.0\.0\.0", port 5432 \S+ \S+ UTC \[[0-9]+\] LOG: listening on IPv6 address "::", port 5432 \S+ \S+ UTC \[[0-9]+\] LOG: listening on Unix socket "/var/run/postgresql/\.s\.PGSQL\.5432" \S+ \S+ UTC \[[0-9]+\] LOG: database system was shut down at 2023-10-03 17:21:05 UTC \S+ \S+ UTC \[[0-9]+\] LOG: database system is ready to accept connections # \S+ \S+ UTC \[[0-9]+\] LOG: received fast shutdown request \S+ \S+ UTC \[[0-9]+\] LOG: aborting any active transactions \S+ \S+ UTC \[[0-9]+\] FATAL: terminating connection due to administrator command \S+ \S+ UTC \[[0-9]+\] LOG: shutting down \S+ \S+ UTC \[[0-9]+\] LOG: database system is shut down ''; match = "IMAGE_NAME = postgres:.+"; } { # yodaNas filters = '' \S+ [0-9]+ \[Warning\] \[MY-013360\] \[Server\] Plugin mysql_native_password reported: '''mysql_native_password' is deprecated and will be removed in a future release\. Please use caching_sha2_password instead' ''; match = "IMAGE_NAME = /mysql:[0-9]+/"; } { # yodaNas filters = '' crond: crond \(busybox \S+\) started, log level [0-9]+ crond: USER root pid\s+[0-9]+ cmd wget -qO- http://money\.p1st\.de:8080/api/v1/cron/\S+ > /proc/1/fd/1 2>/proc/1/fd/2 ''; match = "IMAGE_NAME = busybox"; } { # yodaNas filters = '' \[\S+ \S+ \S+ \S+ \S+\] \[mpm_prefork:notice\] \[pid 1\] AH00170: caught SIGWINCH, shutting down gracefully ''; match = "IMAGE_NAME = fireflyiii/core:latest"; } { # yodaNas filters = '' \[\S+ \S+ \S+ \S+ \S+\] \[mpm_prefork:notice\] \[pid 1\] AH00170: caught SIGWINCH, shutting down gracefully ''; match = "IMAGE_NAME = fireflyiii/data-importer:latest"; } { # yodaNas filters = '' .* ''; match = "IMAGE_NAME = /(deluan/navidrome|ghcr\.io/dgtlmoon/changedetection\.io)/"; } # # _SYSTEMD_UNIT # { # yodaTux filters = '' .* ''; match = "_SYSTEMD_UNIT = /(bluetooth\.service|cups\.service)/"; } { # yodaNas. TODO: This is maybe a bug. It occurs when deploying and switching to a NixOS config with colmena. filters = '' \S+: Process '/nix/store/\S+-bash-\S+/bin/sh -c 'echo 2 > /proc/sys/net/ipv6/conf/\S+/use_tempaddr''' failed with exit code 1. ''; match = "_SYSTEMD_UNIT = systemd-udevd.service"; } { # yodaTux, yodaYoga filters = '' The system will suspend now! The system will power off now! The system will reboot now! System is powering down\. System is rebooting\. ''; match = "_SYSTEMD_UNIT = systemd-logind.service"; } { # yodaTux filters = '' Reexecuting\. (finished )?switching to system configuration /nix/store/.+-nixos-system-.+-[0-9]+\.[0-9]+pre-git ''; match = "_SYSTEMD_UNIT = user@0.service"; } { # yodaTux filters = '' Reexecuting\. (finished )?switching to system configuration /nix/store/.+-nixos-system-.+-[0-9]+\.[0-9]+pre-git ''; match = "_SYSTEMD_UNIT = user@1000.service"; } { # yodaTux filters = '' Reloading rules Collecting garbage unconditionally\.\.\. Loading rules from directory /.+ Finished loading, compiling and executing [0-9]+ rules ''; match = "_SYSTEMD_UNIT = polkit.service"; } { # yodaTux filters = '' .+ error name="org\.bluez\.MediaEndpoint1\.Error\.NotImplemented" .+ # Open issue: https://github.com/NixOS/nixpkgs/issues/79220 Unknown (username|group) .+ in message bus configuration file ''; match = "_SYSTEMD_UNIT = dbus.service"; } { # yodaTux filters = '' Mounted /dev/\S+ at /\S+ on behalf of uid [0-9]+ Cleaning up mount point /\S+ \(device \S+ is not mounted\) Unmounted /dev/\S+ on behalf of uid [0-9]+ Successfully sent SCSI command SYNCHRONIZE CACHE to /dev/\S+ Successfully sent SCSI command START STOP UNIT to /dev/\S+ Powered off /dev/\S+ - successfully wrote to sysfs path /sys/devices/\S+ ''; match = "_SYSTEMD_UNIT = udisks2.service"; } # # SYSLOG_IDENTIFIER # { # yodaYoga, yodaNas filters = '' # Somebody evil ... error: kex_exchange_identification: banner line contains invalid characters # error: kex_exchange_identification: client sent invalid protocol identifier "MGLNDD_188.194.209.73_2222" # error: kex_exchange_identification: client sent invalid protocol identifier "GET / HTTP/1.1" error: kex_exchange_identification: client sent invalid protocol identifier "[^"]+" error: kex_exchange_identification: Connection closed by remote host error: kex_exchange_identification: read: Connection reset by peer error: PAM: Authentication failure for \S+ from \S+ error: Protocol major versions differ: 2 vs\. 1 fatal: Timeout before authentication for \S+ port [0-9]+ ''; match = "SYSLOG_IDENTIFIER = sshd"; } { # yodaTux. If the user `yoda` runs a command with `sudo`. filters = '' \s+yoda : TTY=pts/[0-9] ; PWD=/\S+ ; USER=root ; COMMAND=/.+ ''; match = "SYSLOG_IDENTIFIER = sudo"; } { # yodaYoga filters = '' (finished )?switching to system configuration /nix/store/.+-nixos-system-.+-[0-9]+\.[0-9]+pre-git ''; match = "SYSLOG_IDENTIFIER = nixos"; } { # yodaYoga filters = '' \S+\.(service|scope|slice|mount): Consumed .+ CPU time, read .+ from disk, written .+ to disk.+ \S+\.(service|scope|slice|mount): Consumed .+ CPU time, received .+ IP traffic, sent .+ IP traffic\. \S+\.(service|scope|slice|mount): Consumed .+ CPU time, no IP traffic\. # Shutting down\. ''; match = "SYSLOG_IDENTIFIER = systemd"; } { # yodaTux filters = '' .* ''; match = "SYSLOG_IDENTIFIER = //nix/store/.+/libexec/gdm-x-session/"; } { # yodaTux, yodaTab filters = '' # # yodaNas # # Can be ignored: https://unix.stackexchange.com/a/249660/315162 watchdog: watchdog0: watchdog did not stop! # Ignore. \s*#[0-9]* audit: type=2000 audit\([^(]\): state=initialized audit_enabled=0 res=1 # Ingore: Probably just copying of boot graphics (.bmp) failed. https://patchwork.kernel.org/project/linux-fbdev/patch/20180617153235.16219-3-hdegoede@redhat.com/ efifb: Ignoring BGRT: unexpected or invalid BMP data # Probably safe to ignore. resource sanity check: requesting \[[^\]]\], which spans more than Reserved \[[^\]]\] # # yodaTux # # Bug. ACPI: FW issue: working around C-state latencies out of order # Kernel WiFi driver bug. #iwlwifi 0000:01:00\.0: .* iwlwifi 0000:01:00\.0: Unhandled alg: 0x707 iwlwifi 0000:01:00\.0: Not associated and the session protection is over already... iwlwifi 0000:01:00\.0: api flags index 2 larger than supported by driver # Ignore. audit: type=2000 audit([0-9]+\.[0-9]+:[0-9]+): state=initialized audit_enabled=0 res=1 ENERGY_PERF_BIAS: Set to 'normal', was 'performance' Kernel command line: initrd=\\efi\\nixos\\\S+-initrd-linux-\S+-initrd\.efi init=/nix/store/\S+-nixos-system-\S+-[0-9]+\.[0-9]+pre-git/init ip=dhcp loglevel=[0-9] Linux version \S+ \(nixbld@localhost\) \(gcc \(GCC\) \S+, GNU ld \(GNU Binutils\) \S+\) #1-NixOS SMP PREEMPT_DYNAMIC \S+ \S+ \S+ \S+ UTC \S+ random: crng reseeded on system resumption random: crng init done sd [0-9]:0:0:0: \[sd[a-z]\] [0-9]+ 512-byte logical blocks: \([0-9]+ (GB|TB)/[0-9]+ (GiB|TiB)\) sd [0-9]:0:0:0: \[sd[a-z]\] [0-9]+-byte physical blocks sd [0-9]:0:0:0: \[sd[a-z]\] Write Protect is off sd [0-9]:0:0:0: \[sd[a-z]\] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA sd [0-9]:0:0:0: \[sd[a-z]\] Optimal transfer size [0-9]+ bytes not a multiple of preferred minimum block size ([0-9] bytes) sd [0-9]:0:0:0: \[sd[a-z]\] Attached SCSI disk sd [0-9]:0:0:0: \[sd[a-z]\] Synchronizing SCSI cache sd [0-9]:0:0:0: \[sd[a-z]\] supports TCG Opal \#3 # # yodaTab # # Ignore. mmc0: cannot verify signal voltage switch Initialise system trusted keyrings Key type asymmetric registered Asymmetric key parser 'x509' registered Loading compiled-in X\.509 certificates Key type \.fscrypt registered Key type fscrypt-provisioning registered Key type encrypted registered Bridge firewalling registered SCSI subsystem initialized scsi [0-9]:0:0:0: Direct-Access\s+ATA.+PQ: 0 ANSI: 5 scsi [0-9]:0:0:0: Direct-Access.+PQ: 0 ANSI: 6 thinkpad_acpi: Disabling thinkpad-acpi brightness events by default\.\.\. VFS: Disk quotas dquot_[0-9]+\.[0-9]+\.[0-9]+ ata1\.00: supports DRM functions and may not be fully accessible # done\. ''; match = "SYSLOG_IDENTIFIER = kernel"; } { # yodaTux filters = '' .* ''; match = "SYSLOG_IDENTIFIER = simple-scan"; } # # _SYSTEMD_USER_UNIT # { # yodaTux, yodaTab filters = '' .+ Setting AttentionNeeded to FALSE because EnsureCredentials\(\) succeded Connecting to org\.freedesktop\.Tracker3\.Miner\.Files ''; match = "_SYSTEMD_USER_UNIT = dbus.service"; } { # yodaTux filters = '' .* ''; match = "_SYSTEMD_USER_UNIT = /(org\.gnome\..+\.service|pipewire\.service|wireplumber\.service|app-gnome-org\.gnome\.Software-[0-9]+\.scope)/"; } ]; }; }