nix-git/modules/journalwatch.nix

{ config, pkgs, ... }:
{
  # Systemd Journal Monitoring.
  # Alternative:
  #   journal-biref
  #     https://github.com/twaugh/journal-brief
  #     https://opensource.com/article/20/7/systemd-journals-email

  # Write to Systemd Journal:
  #   echo 'hello' | systemd-cat -p emerg
  #   echo 'hello' | systemd-cat -t someapp -p emerg

  # View Systemd Journal.
  #   Output similar to dmesg
  #     journalctl -b -k
  #   Filter by app:
  #     journalctl -b -t someapp
  #   Filter by priority:
  #     journalctl -b -p 5

  # Manually execute journalwatch timer:
  #   sudo systemctl start journalwatch.service

  # View generated journalwatch config:
  #   cat "$(systemctl show -P FragmentPath journalwatch.service)" | grep XDG_CONFIG_HOME
  #   #=> /nix/store/vzfvp7kizycs6lnks6icmlw3h1x6p7la-journalwatch-config
  #   cat /nix/store/vzfvp7kizycs6lnks6icmlw3h1x6p7la-journalwatch-config/journalwatch/config
  #   cat /nix/store/vzfvp7kizycs6lnks6icmlw3h1x6p7la-journalwatch-config/journalwatch/patterns

  # Find a message and view its details
  #   journalctl -b -p5 -o json-pretty
  #   Then press "/", enter a pattern and press "Enter".

  assertions = [{
    assertion = config.services.opensmtpd.enable;
    message = "journalwatch requires a configured sendmail MTA, see sendmail-mta.nix.";
  }];

  services.journalwatch = {
    enable = true;
    # Same as configured by sendmail MTA.
    mailFrom = "langbein@mail.de";
    mailTo = "daniel+journalwatch@systemli.org";
    #interval = "hourly";

    # Lowest priority of message to be considered. A value between 7 (“debug”), and 0 (“emerg”). Defaults to 6 (“info”). If you don't care about anything with “info” priority, you can reduce this to e.g. 5 (“notice”) to considerably reduce the amount of messages without needing many filterBlocks.
    priority = 5;

    # Default patterns: https://github.com/The-Compiler/journalwatch/blob/363725ac4b8aa841d87654fa8a63403a59ad1275/journalwatch.py#L71
    # If the value of `match` starts and ends with a forward-slash, it is interpreted as a regular expression, if not, it's an exact match.
    # `filters` are always regular expressions.
    # All regular expressions have to match the full string!
    #
    # TODO: To insert `\` in the generated configuration, we have to use "\\" in double-quoted strings.
    # TODO: A ''\s'' in the multiline string `filter` is inserted as `\s` in the generated config file.
    #
    filterBlocks = [

      #
      # _TRANSPORT
      #

      { # yodaNas
        filters = ''
          booting system configuration /nix/store/\S+
        '';
        match = "_TRANSPORT = kernel";
      }

      #
      # _SYSTEMD_CGROUP
      #

      { # yodaYoga
        filters = ''
          parent not found! continent_id [0-9]+
        '';
        match = "_SYSTEMD_CGROUP = /system.slice/docker.service";
      }

      #
      # CONTAINER_NAME
      #

      { # yodaNas
        filters = ''
          .*
        '';
        match = "CONTAINER_NAME = /(nc_web_[^-_\\s]+|nginx_arch\\.p1st\\.de|recipe\\.privacy1st\\.de_nginx|nginx_privacy1st\\.de)/";
      }
      { # yodaNas
        # TODO: Open issue on GitHub https://github.com/nginx-proxy/nginx-proxy/issues/1256. Maybe set env variable RESOLVERS=1.1.1.1?
        filters = ''
          \S+ \S+ \[warn\] [0-9]+#[0-9]+: no resolver defined to resolve r3\.o\.lencr\.org while requesting certificate status, responder: r3\.o\.lencr\.org, certificate: "/etc/nginx/certs/\S+\.crt"
          \S+ \S+ \[error\] [0-9]+#[0-9]+: OCSP responder sent invalid "Content-Type" header: "text/html" while requesting certificate status, responder: r3\.o\.lencr\.org, peer: \S+, certificate: "/etc/nginx/certs/\S+\.crt"
          \S+ \S+ \[error\] [0-9]+#[0-9]+: recv\(\) failed \(111: Connection refused\) while requesting certificate status, responder: r3\.o\.lencr\.org, peer: \S+, certificate: "/etc/nginx/certs/\S+\.crt"
          \S+ \S+ \[error\] [0-9]+#[0-9]+: recv\(\) failed \(104: Connection reset by peer\) while requesting certificate status, responder: r3\.o\.lencr\.org, peer: \S+, certificate: "/etc/nginx/certs/\S+\.crt"
          \S+ \S+ \[error\] [0-9]+#[0-9]+: OCSP responder prematurely closed connection while requesting certificate status, responder: r3\.o\.lencr\.org, peer: \S+, certificate: "/etc/nginx/certs/\S+\.crt"
          #
          \S+ \S+ \[error\] [0-9]+#[0-9]+: \*[0-9]+ upstream timed out \(110: Operation timed out\) while reading upstream, client: .*"
          \S+ \S+ \[error\] [0-9]+#[0-9]+: \*[0-9]+ connect\(\) failed \(111: Connection refused\) while connecting to upstream, client: .*"
          \S+ \S+ \[error\] [0-9]+#[0-9]+: \*[0-9]+ upstream prematurely closed connection while reading response header from upstream, client: .*"
        '';
        match = "CONTAINER_NAME = nginx-proxy";
      }

      #
      # IMAGE_NAME
      #

      { # yodaNas, yodaYoga
        filters = ''
          .*
        '';
        match = "IMAGE_NAME = /(p1st/docker-gen:.+|nginxproxy/acme-companion|thetorproject/snowflake-proxy:.+|collabora/code|wordpress|danielszabo99/microbin|p1st/mastodon-toot-follower:.+|browserless/chrome|vabene1111/recipes)/";
      }
      { # yodaNas
        filters = ''
          \S+ \S+ \[error\] [0-9]+#[0-9]+: \S+ open\(\) "/usr/share/nginx/html/robots\.txt" failed \(2: No such file or directory\), client: \S+, server: localhost, request: "GET /robots\.txt HTTP/[^"]+", host: "[^"]+"
          #
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: signal 3 \(SIGQUIT\) received, shutting down
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: gracefully shutting down
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: exiting
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: exit
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: signal 17 \(SIGCHLD\) received from [0-9]+
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: worker process [0-9]+ exited with code 0
          #
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: signal 29 \(SIGIO\) received
          #
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: signal 1 \(SIGHUP\) received, reconfiguring
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: reconfiguring
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: using the "epoll" event method
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: nginx/\S+
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: built by gcc \S+ \S+ \(Alpine \S+\)
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: OS: Linux \S+
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: getrlimit\(RLIMIT_NOFILE\): [0-9]+:[0-9]+
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: start worker processes
          \S+ \S+ \[notice\] [0-9]+#[0-9]+: start worker process [0-9]+
        '';
        match = "IMAGE_NAME = /nginx:\\S+/";
      }
      { # yodaNas
        # TODO: logged IP is not the public one, but always 172.24.0.6.
        filters = ''
          Could not yet connect with DB\. Retrying in 10s \.\.\.
          Nextcloud or one of the apps require upgrade - only a limited number of commands are available
          You may use your browser or the occ upgrade command to do the upgrade
          \[\S+ \S+\] NOTICE: ready to handle connections
          #
          \s+Starting ...
          \s+[0-9]+/[0-9]+ \[[>=-]+\]   [0-9]+% Starting ...
          \s+[0-9]+/[0-9]+ \[[>=-]+\]   [0-9]+%
          \s+[0-9]+/[0-9]+ \[[>=-]+\]\s+[0-9]+%\s+[0-9]+ \[[>=-]+\]
          #
          \s+[0-9]+ \[[>=-]+\]    [0-9]+ \[[>=-]+\]
          \s+[0-9]+ \[[>=-]+\]
          \s+[0-9]+ \[[>=-]+\]\[\S+ \S+\] NOTICE: fpm is running, pid [0-9]+
          \[\S+ \S+\] NOTICE: fpm is running, pid [0-9]+
          mimetypelist\.js is updated
          \[\S+ \S+\] NOTICE: Finishing \.\.\.
          \[\S+ \S+\] NOTICE: exiting, bye-bye!
          # Ignore: At least one idle child was left.
          \[\S+ \S+\] WARNING: \[pool www\] seems busy \(you may need to increase pm\.start_servers, or pm\.min/max_spare_servers\), spawning [0-9]+ children, there are [1-9][0-9]* idle, and [0-9]+ total children
          \[\S+ \S+\] WARNING: \[pool www\] seems busy \(you may need to increase pm\.start_servers, or pm\.min/max_spare_servers\), spawning [1-9][0-9]* children, there are 0 idle, and [0-9]+ total children
          #
          # 1.1.1.1 -  28/Sep/2023:21:03:39 +0000 "GET /status.php" 200
          # 1.1.1.1 -  28/Sep/2023:21:12:16 +0000 "GET /index.php" 200
          # 1.1.1.1 - my-username 28/Sep/2023:21:20:16 +0000 "DELETE /index.php" 200
          # 1.1.1.1 - my-username 28/Sep/2023:21:10:18 +0000 "PROPFIND /remote\.php" 207
          # 1.1.1.1 - my-username 28/Sep/2023:21:11:48 +0000 "GET /ocs/v2.php" 304
          # 1.1.1.1 -  28/Sep/2023:21:13:10 +0000 "GET /ocs/v2.php" 304
          [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(DELETE|GET|HEAD|OPTIONS|PATCH|POST|PROPFIND|PUT) /(index|status)\.php" (200|201|204|206|302|303|304|400|401|403|404|405|409|412|422|423|500)
          [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(DELETE|GET|HEAD|MKCOL|MOVE|OPTIONS|PATCH|POST|PROPPATCH|PROPFIND|PUT|REPORT|SEARCH) /remote\.php" (200|201|204|206|207|400|401|404|405|412|415|416|423|500|501)
          [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(DELETE|GET|PATCH|POST|PUT) /ocs/(v1|v2)\.php" (200|201|202|204|304|401|403|404|412|500)
          [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(GET|HEAD) /(ocm|ocs)-provider/index\.php" 200
          [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(DELETE|GET|MKCOL|MOVE|PROPFIND|PUT) /public\.php" (200|201|204|207|401|403|404)
        '';
        match = "IMAGE_NAME = /p1st/nextcloud:(25|26|27|stable)-fpm-alpine/";
      }
      { # yodaNas
        filters = ''
          \s+[0-9]+ \[[>-]+\]    [0-9]+ \[[>-]+\]
          \s+[0-9]+ \[[>-]+\]
          \s+[0-9]+ \[[>-]+\]\[\S+ \S+\] NOTICE: fpm is running, pid [0-9]+
        '';
        match = "IMAGE_NAME = /nc_app_[^-_\\s]+/";
      }
      { # yodaNas
        filters = ''
          \S+ \S+ 0 \[Note\] Starting MariaDB \S+ source revision \S+ as process 1
          \S+ \S+ 0 \[Note\] InnoDB: Compressed tables use zlib \S+
          \S+ \S+ 0 \[Note\] InnoDB: Number of transaction pools: 1
          \S+ \S+ 0 \[Note\] InnoDB: Using crc32 \+ pclmulqdq instructions
          \S+ \S+ 0 \[Note\] InnoDB: Using liburing
          \S+ \S+ 0 \[Note\] InnoDB: Initializing buffer pool, total size = 1\.000GiB, chunk size = 16\.000MiB
          \S+ \S+ 0 \[Note\] InnoDB: Completed initialization of buffer pool
          \S+ \S+ 0 \[Note\] InnoDB: Buffered log writes \(block size=512 bytes\)
          \S+ \S+ 0 \[Note\] InnoDB: End of log at LSN=\S+
          \S+ \S+ 0 \[Note\] InnoDB: Opened 3 undo tablespaces
          \S+ \S+ 0 \[Note\] InnoDB: 128 rollback segments in 3 undo tablespaces are active\.
          \S+ \S+ 0 \[Note\] InnoDB: 128 rollback segments are active\.
          \S+ \S+ 0 \[Note\] InnoDB: Setting file '\./\S+' size to 12\.000MiB\. Physically writing the file full; Please wait \.\.\.
          \S+ \S+ 0 \[Note\] InnoDB: File '\./\S+' size is now 12\.000MiB\.
          \S+ \S+ 0 \[Note\] InnoDB: log sequence number \S+; transaction id \S+
          \S+ \S+ 0 \[Note\] InnoDB: Loading buffer pool\(s\) from /var/lib/mysql/ib_buffer_pool
          \S+ \S+ 0 \[Note\] InnoDB: Read redo log up to LSN=\S+
          \S+ \S+ 0 \[Note\] InnoDB: Buffer pool\(s\) load completed at \S+ \S+
          \S+ \S+ 0 \[Note\] Plugin 'FEEDBACK' is disabled\.
          \S+ \S+ 0 \[Note\] Server socket created on IP: '0.0.0.0'\.
          \S+ \S+ 0 \[Note\] Server socket created on IP: '::'\.
          \S+ \S+ 0 \[Note\] mariadbd: ready for connections\.
          Version: '\S+'  socket: '/run/mysqld/mysqld.sock'  port: \S+  mariadb\.org binary distribution
          #
          \S+ \S+ 0 \[Note\] mariadbd \(initiated by: unknown\): Normal shutdown
          \S+ \S+ 0 \[Note\] InnoDB: FTS optimize thread exiting\.
          \S+ \S+ 0 \[Note\] InnoDB: Starting shutdown\.\.\.
          \S+ \S+ 0 \[Note\] InnoDB: Dumping buffer pool\(s\) to /var/lib/mysql/ib_buffer_pool
          \S+ \S+ 0 \[Note\] InnoDB: Buffer pool\(s\) dump completed at \S+ \S+
          \S+ \S+ 0 \[Note\] InnoDB: Removed temporary tablespace data file: "\./ibtmp1"
          \S+ \S+ 0 \[Note\] InnoDB: Shutdown completed; log sequence number \S+; transaction id \S+
          \S+ \S+ 0 \[Note\] mariadbd: Shutdown complete
          \s*
        '';
        match = "IMAGE_NAME = /mariadb(:.+)?/";
      }
      { # yodaNas
        filters = ''
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+starting PostgreSQL \S+ on x86_64-pc-linux-musl, compiled by gcc \(Alpine \S+\) \S+ \S+, 64-bit
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+checkpoint starting: time
          #
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+listening on IPv4 address \S+, port 5432
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+listening on IPv6 address \S+, port 5432
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+listening on Unix socket \S+
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+database system was shut down at \S+ \S+ UTC
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+database system is ready to accept connections
          #
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+received fast shutdown request
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+aborting any active transactions
          \S+ \S+ UTC \[[0-9]+\] FATAL:\s+terminating connection due to administrator command
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+shutting down
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+database system is shut down
          #
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+checkpoint starting: shutdown immediate
          \S+ \S+ UTC \[[0-9]+\] LOG:\s+checkpoint complete: .*
          # background worker "logical replication launcher" (PID 28) exited with exit code 1 -> See https://postgrespro.com/list/thread-id/2498034
        '';
        match = "IMAGE_NAME = /postgres:.+/";
      }
      { # yodaNas
        filters = ''
          \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] MySQL Server - start\.
          \S+ [0-9]+ \[Warning\] \[MY-[0-9]+\] \[Server\] Plugin mysql_native_password reported: '''mysql_native_password' is deprecated and will be removed in a future release\. Please use caching_sha2_password instead'
          \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] /usr/sbin/mysqld \(mysqld \S+\) starting as process 1
          \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[InnoDB\] InnoDB initialization has started\.
          \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[InnoDB\] InnoDB initialization has ended\.
          \S+ [0-9]+ \[Warning\] \[MY-[0-9]+\] \[Server\] CA certificate ca\.pem is self signed\.
          \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] Channel mysql_main configured to support TLS\. Encrypted connections are now supported for this channel\.+
          \S+ [0-9]+ \[Warning\] \[MY-[0-9]+\] \[Server\] Insecure configuration for --pid-file: Location '/var/run/mysqld' in the path is accessible to all OS users\. Consider choosing a different directory\.+
          \S+ [0-9]+ \[Warning\] \[MY-[0-9]+\] \[Server\] The syntax '--skip-host-cache' is deprecated and will be removed in a future release\. Please use SET GLOBAL host_cache_size=0 instead\.
          \S+ [0-9]+ \[Warning\] \[MY-[0-9]+\] \[Server\] 'SET_USER_ID' \(granted to '\S+'\) is deprecated and will be removed in a future release\.
          \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] X Plugin ready for connections\. Bind-address: '::' port: 33060, socket: /var/run/mysqld/mysqlx\.sock
          \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] /usr/sbin/mysqld: ready for connections\. Version: '\S+'  socket: '/var/run/mysqld/mysqld\.sock'  port: 3306  MySQL Community Server - GPL\.
          #
          \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] Received SHUTDOWN from user <via user signal>\. Shutting down mysqld \(Version: \S+\)\.
          \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] /usr/sbin/mysqld: Shutdown complete \(mysqld \S+\)  MySQL Community Server - GPL\.
          \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] MySQL Server - end\.
        '';
        match = "IMAGE_NAME = /mysql:[0-9]+/";
      }
      { # yodaNas
        filters = ''
          crond: crond \(busybox \S+\) started, log level [0-9]+
          crond: USER root pid\s+[0-9]+ cmd wget -qO- http://money\.p1st\.de:8080/api/v1/cron/\S+ > /proc/1/fd/1 2>/proc/1/fd/2
        '';
        match = "IMAGE_NAME = busybox";
      }
      { # yodaNas
        filters = ''
          AH[0-9]+: apache2: Could not reliably determine the server's fully qualified domain name, using \S+\. Set the 'ServerName' directive globally to suppress this message
          \[\S+ \S+ \S+ \S+ \S+\] \[mpm_prefork:notice\] \[pid 1\] AH[0-9]+: Apache/\S+ \(Debian\) configured -- resuming normal operations
          \[\S+ \S+ \S+ \S+ \S+\] \[core:notice\] \[pid 1\] AH[0-9]+: Command line: 'apache2 -D FOREGROUND'
          > Illuminate\\Foundation\\ComposerScripts::postAutoloadDump
          wait-for-it\.sh: waiting 60 seconds for \S+:3306
          wait-for-it\.sh: \S+:3306 is available after [0-9]+ seconds
          #
          \[\S+ \S+ \S+ \S+ \S+\] \[mpm_prefork:notice\] \[pid 1\] AH[0-9]+: caught SIGWINCH, shutting down gracefully
        '';
        match = "IMAGE_NAME = fireflyiii/core:latest";
      }
      { # yodaNas
        filters = ''
          AH[0-9]+: apache2: Could not reliably determine the server's fully qualified domain name, using \S+\. Set the 'ServerName' directive globally to suppress this message
          \[\S+ \S+ \S+ \S+ \S+\] \[mpm_prefork:notice\] \[pid 1\] AH[0-9]+: Apache/\S+ \(Debian\) configured -- resuming normal operations
          \[\S+ \S+ \S+ \S+ \S+\] \[core:notice\] \[pid 1(:tid 1)?\] AH[0-9]+: Command line: 'apache2 -D FOREGROUND'
          #
          \[\S+ \S+ \S+ \S+ \S+\] \[mpm_prefork:notice\] \[pid 1(:tid 1)?\] AH[0-9]+: caught SIGWINCH, shutting down gracefully
        '';
        match = "IMAGE_NAME = fireflyiii/data-importer:latest";
      }
      { # yodaNas
        filters = ''
          .*
        '';
        match = "IMAGE_NAME = /(deluan/navidrome|ghcr\\.io/dgtlmoon/changedetection\\.io)/";
      }

      #
      # _SYSTEMD_UNIT
      #

      # TODO: Wait until issue is resolved
      # https://github.com/NixOS/nixpkgs/issues/267857
      # /etc/tmpfiles.d/tmp.conf:11: Duplicate line for path "/tmp", ignoring.
      { # yodaHedgehog
        filters = ''
          /etc/tmpfiles\.d/tmp\.conf:11: Duplicate line for path "/tmp", ignoring\.
        '';
        match = "_SYSTEMD_UNIT = systemd-tmpfiles-setup.service";
      }
      { # yodaTux
        filters = ''
          .*
        '';
        match = "_SYSTEMD_UNIT = /(bluetooth\\.service|cups\\.service)/";
      }
      { # yodaNas. TODO: This is maybe a bug. It occurs when deploying and switching to a NixOS config with colmena.
        filters = ''
          \S+: Process '/nix/store/\S+-bash-\S+/bin/sh -c 'echo 2 > /proc/sys/net/ipv6/conf/\S+/use_tempaddr''' failed with exit code 1.
        '';
        match = "_SYSTEMD_UNIT = systemd-udevd.service";
      }
      { # yodaTux, yodaYoga
        filters = ''
          The system will suspend now!
          The system will power off now!
          The system will reboot now!
          System is powering down\.
          System is rebooting\.
        '';
        match = "_SYSTEMD_UNIT = systemd-logind.service";
      }
      { # yodaTux, yodaNas
        filters = ''
          Reexecuting\.
          (finished )?switching to system configuration /nix/store/.+-nixos-system-.+-[0-9]+\.[0-9]+pre-git
        '';
        match = "_SYSTEMD_UNIT = /(user@[0-9]+\\.service|init.scope)/";
      }
      { # yodaTux
        filters = ''
          Acquired the name org\.freedesktop\.PolicyKit1 on the system bus
          Reloading rules
          Collecting garbage unconditionally\.\.\.
          Loading rules from directory /.+
          Finished loading, compiling and executing [0-9]+ rules
        '';
        match = "_SYSTEMD_UNIT = polkit.service";
      }
      { # yodaTux
        filters = ''
          .+ error name="org\.bluez\.MediaEndpoint1\.Error\.NotImplemented" .+
          # Open issue: https://github.com/NixOS/nixpkgs/issues/79220
          Unknown (username|group) .+ in message bus configuration file
        '';
        match = "_SYSTEMD_UNIT = dbus.service";
      }
      { # yodaTux
        filters = ''
          Mounted /dev/\S+ at /\S+ on behalf of uid [0-9]+
          Cleaning up mount point /\S+ \(device \S+ is not mounted\)
          Unmounted /dev/\S+ on behalf of uid [0-9]+
          Successfully sent SCSI command SYNCHRONIZE CACHE to /dev/\S+
          Successfully sent SCSI command START STOP UNIT to /dev/\S+
          Powered off /dev/\S+ - successfully wrote to sysfs path /sys/devices/\S+
        '';
        match = "_SYSTEMD_UNIT = udisks2.service";
      }
      { # yodaNas
        filters = ''
          \S Skipped - Image is already being pulled by \S
          \S Pulling
          \S Pulled
          Container \S  Running
          Container \S  Waiting
          Container \S  Healthy
        '';
        match = "_SYSTEMD_UNIT = Jinja-Compose.service";
      }

      #
      # SYSLOG_IDENTIFIER
      #

      { # yodaYoga, yodaNas
        filters = ''
          # Somebody evil ...
          error: kex_exchange_identification: banner line contains invalid characters
          # error: kex_exchange_identification: client sent invalid protocol identifier "MGLNDD_188.194.209.73_2222"
          # error: kex_exchange_identification: client sent invalid protocol identifier "GET / HTTP/1.1"
          error: kex_exchange_identification: client sent invalid protocol identifier "[^"]*"
          error: kex_exchange_identification: Connection closed by remote host
          error: kex_exchange_identification: read: Connection reset by peer
          error: kex_protocol_error: type [0-9]+ seq [0-9]+ \[preauth\]
          error: kex protocol error: type [0-9]+ seq [0-9]+ \[preauth\]
          error: PAM: Authentication failure for \S+ from \S+
          error: PAM: Authentication failure for illegal user \S+ from \S+
          error: Protocol major versions differ: 2 vs\. 1
          error: beginning MaxStartups throttling
          fatal: Timeout before authentication for \S+ port [0-9]+
        '';
        match = "SYSLOG_IDENTIFIER = sshd";
      }
      {
        filters = ''
          # yodaTux. If the user `yoda` runs a command with `sudo`.
          \s+yoda : TTY=pts/[0-9] ; PWD=/\S+ ; USER=root ; COMMAND=/.+
          # yodaNas. If the btrbk service is run.
          \s+btrbk : PWD=/ ; USER=root ; COMMAND=/.+
          \s+root : PWD=(/|/root) ; USER=root ; COMMAND=(/run/current-system/sw|/nix/store/[a-z0-9]+-btrfs-progs-[0-9\.]+)/bin/btrfs (subvolume list|subvolume show|subvolume delete|send|receive) .+
          \s+root : PWD=(/|/root) ; USER=root ; COMMAND=(/run/current-system/sw|/nix/store/[a-z0-9]+-coreutils-[0-9\.]+)/bin/readlink -v -e .+
        '';
        match = "SYSLOG_IDENTIFIER = sudo";
      }
      { # yodaYoga
        filters = ''
          (finished )?switching to system configuration /nix/store/.+-nixos-system-.+-[0-9]+\.[0-9]+pre-git
        '';
        match = "SYSLOG_IDENTIFIER = nixos";
      }
      { # yodaYoga
        filters = ''
          \S+\.(service|scope|slice|mount): Consumed .+ CPU time, read .+ from disk, written .+ to disk.+
          \S+\.(service|scope|slice|mount): Consumed .+ CPU time, received .+ IP traffic, sent .+ IP traffic\.
          \S+\.(service|scope|slice|mount): Consumed .+ CPU time, no IP traffic\.
          #
          Shutting down\.
        '';
        match = "SYSLOG_IDENTIFIER = systemd";
      }
      { # yodaTux
        filters = ''
          .*
        '';
        match = "SYSLOG_IDENTIFIER = //nix/store/.+/libexec/gdm-x-session/";
      }
      {
        filters = (''
          # Ignore.
          systemd\[[0-9]\]: memfd_create\(\) called without MFD_EXEC or MFD_NOEXEC_SEAL set
          # Ignore.
          Unknown kernel command line parameters "ip=dhcp", will be passed to user space.
          # Ignore.
          random: crng reseeded on system resumption
          random: crng init done
          # Ignore.
          BTRFS warning \(device dm-[0-9]+\): skipping scrub of block group [0-9]+ due to active swapfile
          # Ignore.
          ata[0-9]\.00: Entering standby power mode
          ata[0-9]\.00: Entering active power mode
          #
          # yodaTux
          #
          # Ignore.
          audit: type=2000 audit\([0-9]+\.[0-9]+:[0-9]+\): state=initialized audit_enabled=0 res=1
          ENERGY_PERF_BIAS: Set to 'normal', was 'performance'
          Kernel command line: initrd=\S+ init=\S+ ip=dhcp mds=full,nosmt loglevel=[0-9]
          Kernel command line: initrd=\S+ init=\S+ ip=dhcp loglevel=[0-9]
          # Linux version 6.1.61 (nixbld@localhost) (gcc (GCC) 12.2.0, GNU ld (GNU Binutils) 2.40) #1-NixOS SMP PREEMPT_DYNAMIC Thu Nov  2 08:35:33 UTC 2023
          Linux version \S+ \(nixbld@localhost\) \(gcc \(GCC\) \S+, GNU ld \(GNU Binutils\) \S+\) #1-NixOS SMP PREEMPT_DYNAMIC \S+ \S+\s+\S+ \S+ UTC \S+
          sd [0-9]:0:0:0: \[sd[a-z]\] [0-9]+ 512-byte logical blocks: \(\S+ (GB|TB)/\S+ (GiB|TiB)\)
          sd [0-9]:0:0:0: \[sd[a-z]\] [0-9]+-byte physical blocks
          sd [0-9]:0:0:0: \[sd[a-z]\] Write Protect is off
          sd [0-9]:0:0:0: \[sd[a-z]\] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA
          sd [0-9]:0:0:0: \[sd[a-z]\] Optimal transfer size [0-9]+ bytes not a multiple of preferred minimum block size ([0-9] bytes)
          sd [0-9]:0:0:0: \[sd[a-z]\] Attached SCSI (removable )?disk
          sd [0-9]:0:0:0: \[sd[a-z]\] Starting disk
          sd [0-9]:0:0:0: \[sd[a-z]\] Stopping disk
          sd [0-9]:0:0:0: \[sd[a-z]\] Synchronizing SCSI cache
          sd [0-9]:0:0:0: \[sd[a-z]\] supports TCG Opal
          \#3
          #
          # yodaTab
          #
          # Ignore.
          mmc0: cannot verify signal voltage switch
          Initialise system trusted keyrings
          Key type asymmetric registered
          Asymmetric key parser 'x509' registered
          Loading compiled-in X\.509 certificates
          Key type \.fscrypt registered
          Key type fscrypt-provisioning registered
          Key type encrypted registered
          Bridge firewalling registered
          SCSI subsystem initialized
          scsi [0-9]:0:0:0: Direct-Access\s+ATA.+PQ: 0 ANSI: 5
          scsi [0-9]:0:0:0: Direct-Access.+PQ: 0 ANSI: 6
          thinkpad_acpi: Disabling thinkpad-acpi brightness events by default\.\.\.
          VFS: Disk quotas dquot_[0-9]+\.[0-9]+\.[0-9]+
          ata1\.00: supports DRM functions and may not be fully accessible
          #
          done\.
        '' + (if (config.networking.hostName == "yodaHedgehog") then ''
          #
          # Not sure about this.
          #
          hpet: number irqs doesn't agree with number of timers
          gpio gpiochip0: \(INT0002 Virtual GPIO\): not an immutable chip, please consider fixing it!
          at24 0-0050: supply vcc not found, using dummy regulator
          intel-spi: probe of intel-spi failed with error -22
          #
          # Ignore.
          #
          ACPI BIOS Warning \(bug\): .*
          ata[0-9]: link is slow to respond, please be patient \(ready=0\)
          # Not sure about this.
          ata4\.00: qc timeout after 10000 msecs \(cmd 0x40\)
          ata4\.00: VERIFY failed \(err_mask=0x4\)
          ata4\.00: failed to IDENTIFY \(I/O error, err_mask=0x40\)
          ata4\.00: revalidation failed \(errno=-5\)
          #
          # Ignore very frequent kernel stack traces
          #
          ------------\[ cut here \]------------
          i915 0000:00:02\.0: drm_WARN_ON\(\(intel_uncore_read\(&dev_priv->uncore, \(\(const i915_reg_t\){ \.reg = \(0x130090\) }\)\) & mask\) != mask\)
          WARNING: CPU: [0-9]+ PID: [0-9]+ at drivers/gpu/drm/i915/vlv_suspend\.c:.*
          Modules linked in:( \S+)+ ?
          ( dmi_sysfs| efivarfs| nfnetlink| configfs| efi_pstore| ip_tables| x_tables| autofs4| dm_crypt| cbc| irqbypass| drivetemp| drm| fuse| deflate){5}( \S+)+
          CPU: [0-9]+ PID: [0-9]+ Comm: kworker/\S+ Tainted: G        W          [0-9]+\.[0-9]+\.[0-9]+ #1-NixOS
          CPU: [0-9]+ PID: [0-9]+ Comm: kworker/\S+ Not tainted [0-9]+\.[0-9]+\.[0-9]+ #1-NixOS
          Hardware name: To Be Filled By O\.E\.M\. To Be Filled By O\.E\.M\./N3150-ITX, BIOS P1\.10 05/20/2015
          Workqueue: pm pm_runtime_work
          Workqueue: events_unbound async_run_entry_fn
          RIP: .*:vlv_suspend_complete\+0x[0-9abcdef]+/0x[0-9abcdef]+ \[i915\]
          Code: 8b 7b 08 4c 8b 67 50 4d 85 e4 74 25 e8 ([0-9a-f][0-9a-f] )+<0f> 0b e9 [0-9a-f][0-9a-f] f9 ff ff 4c 8b 27 eb d6( [0-9a-f][0-9a-f])+
          RSP: [0-9]+:[0-9a-f]+ EFLAGS: [0-9a-f]+
          RAX: 0000000000000000 RBX: [0-9a-f]+ RCX: [0-9a-f]+
          RDX: [0-9a-f]+ RSI: [0-9a-f]+ RDI: [0-9a-f]+
          RBP: [0-9a-f]+ R08: [0-9a-f]+ R09: [0-9a-f]+
          R10: [0-9a-f]+ R11: [0-9a-f]+ R12: [0-9a-f]+
          R13: [0-9a-f]+ R14: [0-9a-f]+ R15: [0-9a-f]+
          FS:  0000000000000000\(0000\) GS:[0-9a-f]+\(0000\) knlGS:0000000000000000
          CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
          CR2: [0-9a-f]+ CR3: [0-9a-f]+ CR4: [0-9a-f]+
          Call Trace:
           <TASK>
           \? \S+\+0x[0-9abcdef]+/0x[0-9abcdef]+( \[i915\])?
           (intel_runtime_suspend|pci_pm_runtime_suspend|__rpm_callback|rpm_callback|rpm_suspend|pm_runtime_work|process_one_work|worker_thread|kthread|ret_from_fork|ret_from_fork_asm|async_run_entry_fn|async_suspend_late|__device_suspend_late|dpm_run_callback|i915_drm_suspend_late)\+0x[0-9abcdef]+/0x[0-9abcdef]+( \[i915\])?
           </TASK>
          ---\[ end trace 0000000000000000 \]---
        '' else if (config.networking.hostName == "yodaYoga") then ''
          # Ignore BIOS bug.
          ACPI: \[Firmware Bug\]: .*
          ACPI BIOS Error \(bug\): .*
          ACPI Error: .*
          ACPI Warning: .*
          # Ignore.
          usb: port power management may be unreliable
        '' else if (config.networking.hostName == "yodaTux") then ''
          # Ignore BIOS bug.
          ACPI: FW issue: working around C-state latencies out of order
          # Kernel WiFi driver bug.
          #iwlwifi 0000:01:00\.0: .*
          iwlwifi 0000:01:00\.0: Unhandled alg: 0x707
          iwlwifi 0000:01:00\.0: Not associated and the session protection is over already...
          iwlwifi 0000:01:00\.0: api flags index 2 larger than supported by driver
        '' else if (config.networking.hostName == "yodaNas") then ''
          # Probably safe to ignore.
          resource: resource sanity check: requesting \S+, which spans more than Reserved \S+
          # Ingore: Probably just copying of boot graphics (.bmp) failed. https://patchwork.kernel.org/project/linux-fbdev/patch/20180617153235.16219-3-hdegoede@redhat.com/
          efifb: Ignoring BGRT: unexpected or invalid BMP data
          # Not sure about this.
          i915 0000:00:02\.0: \[drm\] \[ENCODER:.*\] is disabled/in DSI mode with an ungated DDI clock, gate it
          # Not sure about this. Probably BIOS related.
          # https://wiki.archlinux.org/title/improving_performance#Enabling_PCI_resizable_BAR
          caller (pmc_core_probe|get_primary_reg_base)\+0x\S+ \[intel_pmc_core\] mapping multiple BARs
        '' else ""));
        match = "SYSLOG_IDENTIFIER = kernel";
      }
      { # yodaTux
        filters = ''
          .*
        '';
        match = "SYSLOG_IDENTIFIER = simple-scan";
      }

      #
      # _SYSTEMD_USER_UNIT
      #

      { # yodaTux, yodaTab
        filters = ''
          .+ Setting AttentionNeeded to FALSE because EnsureCredentials\(\) succeded
          Connecting to org\.freedesktop\.Tracker3\.Miner\.Files
        '';
        match = "_SYSTEMD_USER_UNIT = dbus.service";
      }
      { # yodaTux
        filters = ''
          .*
        '';
        match = "_SYSTEMD_USER_UNIT = /(org\\.gnome\\..+\\.service|pipewire\\.service|wireplumber\\.service|app-gnome-org\\.gnome\\.Software-[0-9]+\\.scope)/";
      }
   ];
  };
}