{ config, pkgs, ... }: { # Systemd Journal Monitoring. # Alternative: # journal-biref # https://github.com/twaugh/journal-brief # https://opensource.com/article/20/7/systemd-journals-email # Write to Systemd Journal: # echo 'hello' | systemd-cat -p emerg # echo 'hello' | systemd-cat -t someapp -p emerg # View Systemd Journal. # Output similar to dmesg # journalctl -b -k # Filter by app: # journalctl -b -t someapp # Filter by priority: # journalctl -b -p 5 # Manually execute journalwatch timer: # sudo systemctl start journalwatch.service # View generated journalwatch config: # cat "$(systemctl show -P FragmentPath journalwatch.service)" | grep XDG_CONFIG_HOME # #=> /nix/store/vzfvp7kizycs6lnks6icmlw3h1x6p7la-journalwatch-config # cat /nix/store/vzfvp7kizycs6lnks6icmlw3h1x6p7la-journalwatch-config/journalwatch/config # cat /nix/store/vzfvp7kizycs6lnks6icmlw3h1x6p7la-journalwatch-config/journalwatch/patterns # Find a message and view its details # journalctl -b -p5 -o json-pretty # Then press "/", enter a pattern and press "Enter". assertions = [{ assertion = config.services.opensmtpd.enable; message = "journalwatch requires a configured sendmail MTA, see sendmail-mta.nix."; }]; services.journalwatch = { enable = true; # Same as configured by sendmail MTA. mailFrom = "langbein@mail.de"; mailTo = "daniel+journalwatch@systemli.org"; #interval = "hourly"; # Lowest priority of message to be considered. # A value between 7 (“debug”), and 0 (“emerg”). # Defaults to 6 (“info”). If you don't care about anything with “info” priority, you can reduce this to e.g. 5 (“notice”) to considerably reduce the amount of messages without needing many filterBlocks. priority = 6; # Default patterns: https://github.com/The-Compiler/journalwatch/blob/363725ac4b8aa841d87654fa8a63403a59ad1275/journalwatch.py#L71 # If the value of `match` starts and ends with a forward-slash, it is interpreted as a regular expression, if not, it's an exact match. # `filters` are always regular expressions. # All regular expressions have to match the full string! # # TODO: To insert `\` in the generated configuration, we have to use "\\" in double-quoted strings. # TODO: A ''\s'' in the multiline string `filter` is inserted as `\s` in the generated config file. # filterBlocks = [ # # _TRANSPORT # { # yodaNas filters = '' booting system configuration /nix/store/\S+ ''; match = "_TRANSPORT = kernel"; } # # _SYSTEMD_CGROUP # { # yodaYoga filters = '' parent not found! continent_id [0-9]+ ''; match = "_SYSTEMD_CGROUP = /system.slice/docker.service"; } # # CONTAINER_NAME # { # yodaNas filters = '' .* ''; match = "CONTAINER_NAME = /(nc_web_[^-_\\s]+|nginx_arch\\.p1st\\.de|recipe\\.privacy1st\\.de_nginx|nginx_privacy1st\\.de)/"; } { # yodaNas # TODO: Open issue on GitHub https://github.com/nginx-proxy/nginx-proxy/issues/1256. Maybe set env variable RESOLVERS=1.1.1.1? filters = '' \S+ \S+ \[warn\] [0-9]+#[0-9]+: no resolver defined to resolve r3\.o\.lencr\.org while requesting certificate status, responder: r3\.o\.lencr\.org, certificate: "/etc/nginx/certs/\S+\.crt" \S+ \S+ \[error\] [0-9]+#[0-9]+: OCSP responder sent invalid "Content-Type" header: "text/html" while requesting certificate status, responder: r3\.o\.lencr\.org, peer: \S+, certificate: "/etc/nginx/certs/\S+\.crt" \S+ \S+ \[error\] [0-9]+#[0-9]+: recv\(\) failed \(111: Connection refused\) while requesting certificate status, responder: r3\.o\.lencr\.org, peer: \S+, certificate: "/etc/nginx/certs/\S+\.crt" \S+ \S+ \[error\] [0-9]+#[0-9]+: recv\(\) failed \(104: Connection reset by peer\) while requesting certificate status, responder: r3\.o\.lencr\.org, peer: \S+, certificate: "/etc/nginx/certs/\S+\.crt" \S+ \S+ \[error\] [0-9]+#[0-9]+: OCSP responder prematurely closed connection while requesting certificate status, responder: r3\.o\.lencr\.org, peer: \S+, certificate: "/etc/nginx/certs/\S+\.crt" # \S+ \S+ \[error\] [0-9]+#[0-9]+: \*[0-9]+ upstream timed out \(110: Operation timed out\) while reading upstream, client: .*" \S+ \S+ \[error\] [0-9]+#[0-9]+: \*[0-9]+ connect\(\) failed \(111: Connection refused\) while connecting to upstream, client: .*" \S+ \S+ \[error\] [0-9]+#[0-9]+: \*[0-9]+ upstream prematurely closed connection while reading response header from upstream, client: .*" ''; match = "CONTAINER_NAME = nginx-proxy"; } { # yodaNas filters = '' 127.0.0.1 - - \[\S+ \S+\] "GET \S+ \S+" [0-9]+ [0-9]+ "-" "[^"]" ''; match = "CONTAINER_NAME = money.p1st.de"; } # # IMAGE_NAME # { # yodaNas, yodaYoga filters = '' .* ''; match = "IMAGE_NAME = /(redis:alpine|p1st/docker-gen:.+|nginxproxy/acme-companion|thetorproject/snowflake-proxy:.+|collabora/code|wordpress|danielszabo99/microbin|p1st/mastodon-toot-follower:.+|browserless/chrome|vabene1111/recipes)/"; } { # yodaNas filters = '' \S+ \S+ \[error\] [0-9]+#[0-9]+: \S+ open\(\) "/usr/share/nginx/html/robots\.txt" failed \(2: No such file or directory\), client: \S+, server: localhost, request: "GET /robots\.txt HTTP/[^"]+", host: "[^"]+" # \S+ \S+ \[notice\] [0-9]+#[0-9]+: signal 3 \(SIGQUIT\) received, shutting down \S+ \S+ \[notice\] [0-9]+#[0-9]+: gracefully shutting down \S+ \S+ \[notice\] [0-9]+#[0-9]+: exiting \S+ \S+ \[notice\] [0-9]+#[0-9]+: exit \S+ \S+ \[notice\] [0-9]+#[0-9]+: signal 17 \(SIGCHLD\) received from [0-9]+ \S+ \S+ \[notice\] [0-9]+#[0-9]+: worker process [0-9]+ exited with code 0 # \S+ \S+ \[notice\] [0-9]+#[0-9]+: signal 29 \(SIGIO\) received # \S+ \S+ \[notice\] [0-9]+#[0-9]+: signal 1 \(SIGHUP\) received, reconfiguring \S+ \S+ \[notice\] [0-9]+#[0-9]+: reconfiguring \S+ \S+ \[notice\] [0-9]+#[0-9]+: using the "epoll" event method \S+ \S+ \[notice\] [0-9]+#[0-9]+: nginx/\S+ \S+ \S+ \[notice\] [0-9]+#[0-9]+: built by gcc \S+ \S+ \(Alpine \S+\) \S+ \S+ \[notice\] [0-9]+#[0-9]+: OS: Linux \S+ \S+ \S+ \[notice\] [0-9]+#[0-9]+: getrlimit\(RLIMIT_NOFILE\): [0-9]+:[0-9]+ \S+ \S+ \[notice\] [0-9]+#[0-9]+: start worker processes \S+ \S+ \[notice\] [0-9]+#[0-9]+: start worker process [0-9]+ ''; match = "IMAGE_NAME = /nginx:\\S+/"; } { # yodaNas filters = '' 127\.0\.0\.1 - - \[\S+ \+0200\] "GET /health HTTP/1\.1" 200 1741 "-" "Firefly III Health Checker/[0-9\.]+" ''; match = "IMAGE_NAME = fireflyiii/data-importer:latest"; } { # yodaNas # TODO: logged IP is not the public one, but always 172.24.0.6. filters = '' Could not yet connect with DB\. Retrying in 10s \.\.\. Nextcloud or one of the apps require upgrade - only a limited number of commands are available You may use your browser or the occ upgrade command to do the upgrade \[\S+ \S+\] NOTICE: ready to handle connections # \s+Starting ... \s+[0-9]+/[0-9]+ \[[>=-]+\] [0-9]+% Starting ... \s+[0-9]+/[0-9]+ \[[>=-]+\] [0-9]+% \s+[0-9]+/[0-9]+ \[[>=-]+\]\s+[0-9]+%\s+[0-9]+ \[[>=-]+\] # \s+[0-9]+ \[[>=-]+\] [0-9]+ \[[>=-]+\] \s+[0-9]+ \[[>=-]+\] \s+[0-9]+ \[[>=-]+\]\[\S+ \S+\] NOTICE: fpm is running, pid [0-9]+ \[\S+ \S+\] NOTICE: fpm is running, pid [0-9]+ mimetypelist\.js is updated \[\S+ \S+\] NOTICE: Finishing \.\.\. \[\S+ \S+\] NOTICE: exiting, bye-bye! # Ignore: At least one idle child was left. # TODO: Occasionally check how often this warning is logged. \[\S+ \S+\] WARNING: \[pool www\] seems busy \(you may need to increase pm\.start_servers, or pm\.min/max_spare_servers\), spawning [0-9]+ children, there are [1-9][0-9]* idle, and [0-9]+ total children \[\S+ \S+\] WARNING: \[pool www\] seems busy \(you may need to increase pm\.start_servers, or pm\.min/max_spare_servers\), spawning [1-9][0-9]* children, there are 0 idle, and [0-9]+ total children # # 1.1.1.1 - 28/Sep/2023:21:03:39 +0000 "GET /status.php" 200 # 1.1.1.1 - 28/Sep/2023:21:12:16 +0000 "GET /index.php" 200 # 1.1.1.1 - my-username 28/Sep/2023:21:20:16 +0000 "DELETE /index.php" 200 # 1.1.1.1 - my-username 28/Sep/2023:21:10:18 +0000 "PROPFIND /remote\.php" 207 # 1.1.1.1 - my-username 28/Sep/2023:21:11:48 +0000 "GET /ocs/v2.php" 304 # 1.1.1.1 - 28/Sep/2023:21:13:10 +0000 "GET /ocs/v2.php" 304 [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(DELETE|GET|HEAD|OPTIONS|PATCH|POST|PROPFIND|PUT) /(index|status)\.php" (200|201|204|206|302|303|304|400|401|403|404|405|409|412|422|423|500) [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(COPY|DELETE|GET|HEAD|MKCOL|MOVE|OPTIONS|PATCH|POST|PROPPATCH|PROPFIND|PUT|REPORT|SEARCH) /remote\.php" (200|201|204|206|207|400|401|404|405|409|412|415|416|423|500|501) [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(DELETE|GET|PATCH|POST|PUT) /ocs/(v1|v2)\.php" (200|201|202|204|304|401|403|404|412|500) [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(GET|HEAD) /(ocm|ocs)-provider/index\.php" 200 [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ - \S* \S+ \+0000 "(DELETE|GET|MKCOL|MOVE|PROPFIND|PUT) /public\.php" (200|201|204|207|401|403|404) ''; match = "IMAGE_NAME = /p1st/nextcloud:(25|26|27|stable)-fpm-alpine/"; } { # yodaNas filters = '' \s+[0-9]+ \[[>-]+\] [0-9]+ \[[>-]+\] \s+[0-9]+ \[[>-]+\] \s+[0-9]+ \[[>-]+\]\[\S+ \S+\] NOTICE: fpm is running, pid [0-9]+ # Configuring Redis as session handler => Searching for scripts \(\*\.sh\) to run, located in the folder: /docker-entrypoint-hooks\.d/before-starting Executing /occ_entrypoint\.sh Waiting for DB connection \.\.\. installed and image version: \s+[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ Nextcloud is installed\. occ command /env/001: upgrade Nextcloud is already latest version occ command /env/002: user:setting \S+ settings email \S+ occ command /env/003: config:system:delete trusted_domains System config value trusted_domains deleted occ command /env/004: config:system:set trusted_domains [0-9]+ --value=\S+ System config value trusted_domains => [0-9]+ set to string \S+ occ command /env/007: config:system:set --value '172\.16\.0\.0/12' 'trusted_proxies' '0' System config value trusted_proxies => 0 set to string 172\.16\.0\.0/12 occ command /env/009: config:system:set --type=string --value 'https' 'overwriteprotocol' System config value overwriteprotocol set to string https occ command /env/010: config:system:set --type=string --value '\S+' 'overwrite\.cli\.url' System config value overwrite\.cli\.url set to string https://\S+ occ command /env/011: db:add-missing-columns Done\. occ command /env/011: db:add-missing-primary-keys occ command /env/012: db:add-missing-indices occ command /env/014: db:convert-filecache-bigint --no-interaction All tables already up to date! occ command /env/015: encryption:migrate-key-storage-format Updating key storage format Start to update the keys: Key storage format successfully updated occ command /env/016: maintenance:repair --include-expensive - Repair MySQL collation - All tables already have the correct collation -> nothing to do - Clean tags and favorites - 0 tags of deleted users have been removed\. - 0 tags for delete files have been removed\. - 0 tag entries for deleted tags have been removed\. - 0 tags with no entries have been removed\. - Repair invalid shares - Move \.step file of updater to backup location - Add move avatar background job - Repair step already executed - Add preview cleanup background jobs - Migrate oauth2_clients table to nextcloud schema - Update the oauth2_access_tokens table schema\. - Update the oauth2_clients table schema\. - Delete clients \(and their related access tokens\) with the redirect_uri starting with oc:// or ending with \* - Fix potential broken mount points - No mounts updated - Repair language codes - Add log rotate job - Clear frontend caches - Image cache cleared - JS cache cleared - Clear every generated avatar - Add preview background cleanup job - Queue a one-time job to cleanup old backups of the updater - Cleanup invalid photocache files for carddav - Add background job to cleanup login flow v2 tokens - Remove potentially over exposing share links - No need to remove link shares\. - Clear access cache of projects - Reset generated avatar flag - Keep legacy encryption enabled - Check encryption key format - Remove old dashboard app config data - Add job to cleanup the bruteforce entries - Queue a one-time job to check for user uploaded certificates - Repair DAV shares - Add background job to set the lookup server share state for users - Add token cleanup job - Clean up abandoned apps - Add possibly missing system config - Add AI tasks cleanup job - Queue a job to generate metadata - migrate lazy config values - Cache logo dimension to fix size in emails on Outlook - Logo dimensions are already known - Remove shares of old group memberships - Repair mime types - Validate the phone number and store it in a known format for search - Handle outdated scheduling events - Cleaning up old scheduling events - Deduplicate shared bookmark folders - Removed 0 duplicate shares - Remove superfluous shared bookmark folders - Removed 0 superfluous shares - Remove orphaned bookmark shares - Removed 0 orphaned shares - Removed 0 orphaned public links - Remove orphaned bookmark tree items - Removed 0 orphaned \S+ entries - Reinserted 0 orphaned children entries - Reinserted 0 orphaned bookmarks - Update bookmark group shares - Removed 0 users and added 0 users to 0 groups - Removed 0 shares - Upgrading Circles App - Fix component of birthday calendars - 8 birthday calendars updated\. - Regenerating birthday calendars to use new icons and fix old birthday events without year - Fix broken values of calendar objects - Registering building of calendar search index as background job - Register building of social profile search index as background job - Registering background jobs to update cache for webcal calendars - Added 0 background jobs to update webcal calendars - Registering building of calendar reminder index as background job - Clean up orphan event and contact data - 0 \S+ without a calendar have been cleaned up - Remove activity entries of private events - Removed 0 activity entries - Clean up old calendar subscriptions from deleted users that were not cleaned-up - 0 calendar subscriptions without an user have been cleaned up - Remove invalid object properties - 0 invalid object properties removed\. - 0 invalid object properties removed. - Copy the share password into the dedicated column - Set existing shares as accepted - Migrate timestamp values to integer to store unix epoch - Remove the unused News update job - Job does not exist, all good - Update OAuth token expiration times - init metadata - Polls - Drop orphaned tables - No orphaned tables found - Polls - Drop orphaned columns - Polls - Create hashes for votes and options - Updated [0-9]+ option hashes - Updated [0-9]+ vote hashes - Polls - Delete duplicates and orphaned records - Polls - Create indices and foreign key constraints - Added oc_polls_polls\['poll_id'\] <- oc_polls_\S+\['id'\] - Index \S+ already exists in oc_polls_\S+ - Polls - Foreign key contraints created\. - Polls - Indices created\. - Create help command - Invalidate access cache for projects conversation provider - Invalidation not required - Cache the user display names - Send an admin notification if monthly report is disabled - Force-reset all Text document sessions - Initialize migration of background images from dashboard to theming app - Add background job to check for backup codes - Populating added database structures for workflows occ command /env/017: app:update --all # (drawio|memories|metadata|gpoddersync|deck|bookmarks|recommendations) new version available: [0-9]+\.[0-9]+\.[0-9]+ (drawio|memories|metadata|gpoddersync|deck|bookmarks|recommendations) updated No such app enabled: (drawio|memories|metadata|gpoddersync|deck|bookmarks|recommendations) # occ command /env/018: config:system:set --type=string --value '/shared' 'share_folder' System config value share_folder set to string /shared occ command /env/019: config:system:delete 'skeletondirectory' System config value skeletondirectory deleted occ command /env/020: config:system:set --type=string --value 'DE' 'default_phone_region' System config value default_phone_region set to string DE occ command /env/021: config:system:set --type=integer --value '30' 'activity_expire_days' System config value activity_expire_days set to integer 30 occ command /env/022: config:system:set --type=boolean --value 'false' 'activity_use_cached_mountpoints' System config value activity_use_cached_mountpoints set to boolean false ''; match = "IMAGE_NAME = /nc_app_[^-_\\s]+/"; } { # yodaNas filters = '' \S+ \S+ 0 \[Note\] Starting MariaDB \S+ source revision \S+ as process 1 \S+ \S+ 0 \[Note\] InnoDB: Compressed tables use zlib \S+ \S+ \S+ 0 \[Note\] InnoDB: Number of transaction pools: 1 \S+ \S+ 0 \[Note\] InnoDB: Using crc32 \+ pclmulqdq instructions \S+ \S+ 0 \[Note\] InnoDB: Using liburing \S+ \S+ 0 \[Note\] InnoDB: Initializing buffer pool, total size = 1\.000GiB, chunk size = 16\.000MiB \S+ \S+ 0 \[Note\] InnoDB: Completed initialization of buffer pool \S+ \S+ 0 \[Note\] InnoDB: Buffered log writes \(block size=512 bytes\) \S+ \S+ 0 \[Note\] InnoDB: End of log at LSN=\S+ \S+ \S+ 0 \[Note\] InnoDB: Opened 3 undo tablespaces \S+ \S+ 0 \[Note\] InnoDB: 128 rollback segments in 3 undo tablespaces are active\. \S+ \S+ 0 \[Note\] InnoDB: 128 rollback segments are active\. \S+ \S+ 0 \[Note\] InnoDB: Setting file '\./\S+' size to 12\.000MiB\. Physically writing the file full; Please wait \.\.\. \S+ \S+ 0 \[Note\] InnoDB: File '\./\S+' size is now 12\.000MiB\. \S+ \S+ 0 \[Note\] InnoDB: log sequence number \S+; transaction id \S+ \S+ \S+ 0 \[Note\] InnoDB: Loading buffer pool\(s\) from /var/lib/mysql/ib_buffer_pool \S+ \S+ 0 \[Note\] InnoDB: Read redo log up to LSN=\S+ \S+ \S+ 0 \[Note\] InnoDB: Buffer pool\(s\) load completed at \S+ \S+ \S+ \S+ 0 \[Note\] Plugin 'FEEDBACK' is disabled\. \S+ \S+ 0 \[Note\] Server socket created on IP: '0.0.0.0'\. \S+ \S+ 0 \[Note\] Server socket created on IP: '::'\. \S+ \S+ 0 \[Note\] mariadbd: ready for connections\. Version: '\S+' socket: '/run/mysqld/mysqld.sock' port: \S+ mariadb\.org binary distribution # \S+ \S+ 0 \[Note\] mariadbd \(initiated by: unknown\): Normal shutdown \S+ \S+ 0 \[Note\] InnoDB: FTS optimize thread exiting\. \S+ \S+ 0 \[Note\] InnoDB: Starting shutdown\.\.\. \S+ \S+ 0 \[Note\] InnoDB: Dumping buffer pool\(s\) to /var/lib/mysql/ib_buffer_pool \S+ \S+ 0 \[Note\] InnoDB: Buffer pool\(s\) dump completed at \S+ \S+ \S+ \S+ 0 \[Note\] InnoDB: Removed temporary tablespace data file: "\./ibtmp1" \S+ \S+ 0 \[Note\] InnoDB: Shutdown completed; log sequence number \S+; transaction id \S+ \S+ \S+ 0 \[Note\] mariadbd: Shutdown complete \s* ''; match = "IMAGE_NAME = /mariadb(:.+)?/"; } { # yodaNas filters = '' \S+ \S+ UTC \[[0-9]+\] LOG:\s+starting PostgreSQL \S+ on x86_64-pc-linux-musl, compiled by gcc \(Alpine \S+\) \S+ \S+, 64-bit \S+ \S+ UTC \[[0-9]+\] LOG:\s+checkpoint starting: time # \S+ \S+ UTC \[[0-9]+\] LOG:\s+listening on IPv4 address \S+, port 5432 \S+ \S+ UTC \[[0-9]+\] LOG:\s+listening on IPv6 address \S+, port 5432 \S+ \S+ UTC \[[0-9]+\] LOG:\s+listening on Unix socket \S+ \S+ \S+ UTC \[[0-9]+\] LOG:\s+database system was shut down at \S+ \S+ UTC \S+ \S+ UTC \[[0-9]+\] LOG:\s+database system is ready to accept connections # \S+ \S+ UTC \[[0-9]+\] LOG:\s+received fast shutdown request \S+ \S+ UTC \[[0-9]+\] LOG:\s+aborting any active transactions \S+ \S+ UTC \[[0-9]+\] FATAL:\s+terminating connection due to administrator command \S+ \S+ UTC \[[0-9]+\] LOG:\s+shutting down \S+ \S+ UTC \[[0-9]+\] LOG:\s+database system is shut down # \S+ \S+ UTC \[[0-9]+\] LOG:\s+checkpoint starting: shutdown immediate \S+ \S+ UTC \[[0-9]+\] LOG:\s+checkpoint complete: .* # background worker "logical replication launcher" (PID 28) exited with exit code 1 -> See https://postgrespro.com/list/thread-id/2498034 ''; match = "IMAGE_NAME = /postgres:.+/"; } { # yodaNas filters = '' \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] MySQL Server - start\. \S+ [0-9]+ \[Warning\] \[MY-[0-9]+\] \[Server\] Plugin mysql_native_password reported: '''mysql_native_password' is deprecated and will be removed in a future release\. Please use caching_sha2_password instead' \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] /usr/sbin/mysqld \(mysqld \S+\) starting as process 1 \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[InnoDB\] InnoDB initialization has started\. \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[InnoDB\] InnoDB initialization has ended\. \S+ [0-9]+ \[Warning\] \[MY-[0-9]+\] \[Server\] CA certificate ca\.pem is self signed\. \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] Channel mysql_main configured to support TLS\. Encrypted connections are now supported for this channel\.+ \S+ [0-9]+ \[Warning\] \[MY-[0-9]+\] \[Server\] Insecure configuration for --pid-file: Location '/var/run/mysqld' in the path is accessible to all OS users\. Consider choosing a different directory\.+ \S+ [0-9]+ \[Warning\] \[MY-[0-9]+\] \[Server\] The syntax '--skip-host-cache' is deprecated and will be removed in a future release\. Please use SET GLOBAL host_cache_size=0 instead\. \S+ [0-9]+ \[Warning\] \[MY-[0-9]+\] \[Server\] 'SET_USER_ID' \(granted to '\S+'\) is deprecated and will be removed in a future release\. \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] X Plugin ready for connections\. Bind-address: '::' port: 33060, socket: /var/run/mysqld/mysqlx\.sock \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] /usr/sbin/mysqld: ready for connections\. Version: '\S+' socket: '/var/run/mysqld/mysqld\.sock' port: 3306 MySQL Community Server - GPL\. # \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] Received SHUTDOWN from user \. Shutting down mysqld \(Version: \S+\)\. \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] /usr/sbin/mysqld: Shutdown complete \(mysqld \S+\) MySQL Community Server - GPL\. \S+ [0-9]+ \[System\] \[MY-[0-9]+\] \[Server\] MySQL Server - end\. ''; match = "IMAGE_NAME = /mysql:[0-9]+/"; } { # yodaNas filters = '' crond: crond \(busybox \S+\) started, log level [0-9]+ crond: USER root pid\s+[0-9]+ cmd wget -qO- http://money\.p1st\.de:8080/api/v1/cron/\S+ > /proc/1/fd/1 2>/proc/1/fd/2 ''; match = "IMAGE_NAME = busybox"; } { # yodaNas filters = '' AH[0-9]+: apache2: Could not reliably determine the server's fully qualified domain name, using \S+\. Set the 'ServerName' directive globally to suppress this message \[\S+ \S+ \S+ \S+ \S+\] \[mpm_prefork:notice\] \[pid 1(:tid 1)?\] AH[0-9]+: Apache/\S+ \(Debian\) configured -- resuming normal operations \[\S+ \S+ \S+ \S+ \S+\] \[core:notice\] \[pid 1\] AH[0-9]+: Command line: 'apache2 -D FOREGROUND' > Illuminate\\Foundation\\ComposerScripts::postAutoloadDump wait-for-it\.sh: waiting 60 seconds for \S+:3306 wait-for-it\.sh: \S+:3306 is available after [0-9]+ seconds # \[\S+ \S+ \S+ \S+ \S+\] \[mpm_prefork:notice\] \[pid 1\] AH[0-9]+: caught SIGWINCH, shutting down gracefully ''; match = "IMAGE_NAME = fireflyiii/core:latest"; } { # yodaNas filters = '' AH[0-9]+: apache2: Could not reliably determine the server's fully qualified domain name, using \S+\. Set the 'ServerName' directive globally to suppress this message \[\S+ \S+ \S+ \S+ \S+\] \[mpm_prefork:notice\] \[pid 1(:tid 1)?\] AH[0-9]+: Apache/\S+ \(Debian\) configured -- resuming normal operations \[\S+ \S+ \S+ \S+ \S+\] \[core:notice\] \[pid 1(:tid 1)?\] AH[0-9]+: Command line: 'apache2 -D FOREGROUND' # \[\S+ \S+ \S+ \S+ \S+\] \[mpm_prefork:notice\] \[pid 1(:tid 1)?\] AH[0-9]+: caught SIGWINCH, shutting down gracefully ''; match = "IMAGE_NAME = fireflyiii/data-importer:latest"; } { # yodaNas filters = '' .* ''; match = "IMAGE_NAME = /(deluan/navidrome|ghcr\\.io/dgtlmoon/changedetection\\.io)/"; } # # _SYSTEMD_UNIT # { # yodaNas filters = '' \s*The \S+ (A|AAAA) record points already to \S+ ''; match = "_SYSTEMD_UNIT = netcup-dns.service"; } { # yodaNas filters = '' \[\S+ \S+\] \[NOTICE\] Server with the lowest initial latency: \S+ \(rtt: [0-9]+ms\) \[\S+ \S+\] \[NOTICE\] -\s+[0-9]+ms \S+ \[\S+ \S+\] \[NOTICE\] Anonymizing queries for \[\S+\] via \[\S+\] ''; match = "_SYSTEMD_UNIT = dnscrypt-proxy2.service"; } { # yodaNas filters = '' pam_unix\(sudo:session\): session opened for user root\(uid=0\) by \(uid=994\) pam_unix\(sudo:session\): session closed for user root -+ Backup Summary \(btrbk command line client, version \S+\) \s+Date:\s+\S+ \S+ \S+ \S+ \S+ \s+Config: \S+ Legend: \s+=== up-to-date subvolume \(source snapshot\) \s+\+\+\+ created subvolume \(source snapshot\) \s+\+\+\+ deleted subvolume \s+\*\*\* received subvolume \(non-incremental\) \s+>>> received subvolume \(incremental\) \s+ # # BTRFS subvolume paths # /jc-data/\S+ /mnt/data/\S+/\S+ \+\+\+ /mnt/data/\S+/\S+ \+\+\+ /snap/\S+ ''; match = "_SYSTEMD_UNIT = /btrbk-local-snapshot-(hdd|ssd)\\.service/"; } { # yodaHedgehog filters = '' info: OpenSMTPD \S+-portable starting \S+ smtp connected address=local host=${config.networking.hostName} \S+ smtp message msgid=\S+ size=\S+ nrcpt=1 proto=ESMTP \S+ smtp envelope evpid=[0-9a-f]+ from= to=\S+ \S+ smtp disconnected reason=quit \S+ mta connecting address=smtps://\S+ host=\S+ \S+ mta connected \S+ mta tls ciphers=TLSv1.3:TLS_AES_256_GCM_SHA384:256 #\S+ mta cert-check result=\\"valid\\" fingerprint=\S+ \S+ mta cert-check result="valid" fingerprint="SHA256:[0-9a-f]+" \S+ mta delivery evpid=\S+ from=\S+ to=\S+ rcpt=<-> source=\S+ relay="\S+ \(\S+\)" delay=\S+ result="Ok" stat="250 2.0.0 Ok: queued as \S+" \S+ mta disconnected reason=quit messages=1 Exiting ''; match = "_SYSTEMD_UNIT = opensmtpd.service"; } { # yodaYoga, yodaNas filters = '' Accepted publickey for root from \S+ port \S+ ssh2: RSA SHA256:\S+ pam_unix\(sshd:session\): session opened for user \S+ by \S+ Received disconnect from \S+ port \S+:11: disconnected by user Disconnected from user \S+ \S+ port \S+ pam_unix\(sshd:session\): session closed for user \S+ # # Somebody evil ... # Failed keyboard-interactive/pam for invalid user \S+ from \S+ port \S+ ssh2 Unable to negotiate with \S+ port \S+: no matching MAC found. Their offer: \S+ \[preauth\] Invalid user \S+ from \S+ port \S+ Disconnected from invalid user \S+ \S+ port \S+ \[preauth\] Disconnected from authenticating user root \S+ port \S+ \[preauth\] Received disconnect from \S+ port \S+:11: Client disconnecting normally \[preauth\] Received disconnect from \S+ port \S+:11: Bye Bye \[preauth\] Connection closed by \S+ port \S+ \[preauth\] Connection closed by authenticating user root \S+ port \S+ \[preauth\] error: kex_exchange_identification: banner line contains invalid characters # error: kex_exchange_identification: client sent invalid protocol identifier "MGLNDD_188.194.209.73_2222" # error: kex_exchange_identification: client sent invalid protocol identifier "GET / HTTP/1.1" error: kex_exchange_identification: client sent invalid protocol identifier "[^"]*" error: kex_exchange_identification: Connection closed by remote host error: kex_exchange_identification: read: Connection reset by peer error: kex_protocol_error: type [0-9]+ seq [0-9]+ \[preauth\] error: kex protocol error: type [0-9]+ seq [0-9]+ \[preauth\] error: PAM: Authentication failure for \S+ from \S+ error: PAM: Authentication failure for illegal user \S+ from \S+ error: Protocol major versions differ: 2 vs\. 1 error: beginning MaxStartups throttling fatal: Timeout before authentication for \S+ port [0-9]+ ''; match = "_SYSTEMD_UNIT = sshd.service"; } # TODO: Wait until issue is resolved # https://github.com/NixOS/nixpkgs/issues/267857 # /etc/tmpfiles.d/tmp.conf:11: Duplicate line for path "/tmp", ignoring. { # yodaHedgehog filters = '' /etc/tmpfiles\.d/tmp\.conf:11: Duplicate line for path "/tmp", ignoring\. ''; match = "_SYSTEMD_UNIT = systemd-tmpfiles-setup.service"; } { # yodaTux filters = '' .* ''; match = "_SYSTEMD_UNIT = /(bluetooth\\.service|cups\\.service)/"; } { # yodaNas. TODO: This is maybe a bug. It occurs when deploying and switching to a NixOS config with colmena. filters = '' \S+: Process '/nix/store/\S+-bash-\S+/bin/sh -c 'echo 2 > /proc/sys/net/ipv6/conf/\S+/use_tempaddr''' failed with exit code 1. ''; match = "_SYSTEMD_UNIT = systemd-udevd.service"; } { # yodaTux, yodaYoga filters = '' The system will suspend now! The system will power off now! The system will reboot now! System is powering down\. System is rebooting\. ''; match = "_SYSTEMD_UNIT = systemd-logind.service"; } { # yodaTux, yodaNas filters = '' Starting \S+\.service\.\.\. \S+\.service: Deactivated successfully\. Finished \S+\.service\. # Reexecuting\. (finished )?switching to system configuration /nix/store/.+-nixos-system-.+-[0-9]+\.[0-9]+pre-git ''; match = "_SYSTEMD_UNIT = /(user@[0-9]+\\.service|init.scope)/"; } { # yodaTux filters = '' Acquired the name org\.freedesktop\.PolicyKit1 on the system bus Reloading rules Collecting garbage unconditionally\.\.\. Loading rules from directory /.+ Finished loading, compiling and executing [0-9]+ rules ''; match = "_SYSTEMD_UNIT = polkit.service"; } { # yodaTux filters = '' .+ error name="org\.bluez\.MediaEndpoint1\.Error\.NotImplemented" .+ # Open issue: https://github.com/NixOS/nixpkgs/issues/79220 Unknown (username|group) .+ in message bus configuration file ''; match = "_SYSTEMD_UNIT = dbus.service"; } { # yodaTux filters = '' Mounted /dev/\S+ at /\S+ on behalf of uid [0-9]+ Cleaning up mount point /\S+ \(device \S+ is not mounted\) Unmounted /dev/\S+ on behalf of uid [0-9]+ Successfully sent SCSI command SYNCHRONIZE CACHE to /dev/\S+ Successfully sent SCSI command START STOP UNIT to /dev/\S+ Powered off /dev/\S+ - successfully wrote to sysfs path /sys/devices/\S+ ''; match = "_SYSTEMD_UNIT = udisks2.service"; } { # yodaNas filters = '' \s+\S+ Skipped - Image is already being pulled by \S+ \s+\S+ Pulling \s+\S+ Already exists \s+\S+ Waiting \s+\S+ (Downloading|Extracting) \[=*> *\]\s+([0-9]+\.)?[0-9]+(B|kB|MB|GB)/([0-9]+\.)?[0-9]+(B|kB|MB|GB) \s+\S+ Download complete \s+\S+ Verifying Checksum \s+\S+ Pull complete \s+\S+ Pulled \s+Network \S+ (Creating|Created) \s+Container \S+ (Creating|Created|Recreate|Recreated|Waiting|Starting|Started|Running|Healthy) ''; match = "_SYSTEMD_UNIT = Jinja-Compose.service"; } # # SYSLOG_IDENTIFIER # { filters = '' # yodaTux. If the user `yoda` runs a command with `sudo`. \s+yoda : TTY=pts/[0-9] ; PWD=/\S+ ; USER=root ; COMMAND=/.+ # yodaNas. If the btrbk service is run. \s+btrbk : PWD=/ ; USER=root ; COMMAND=/.+ \s+root : PWD=(/|/root) ; USER=root ; COMMAND=(/run/current-system/sw|/nix/store/[a-z0-9]+-btrfs-progs-[0-9\.]+)/bin/btrfs (subvolume list|subvolume show|subvolume delete|send|receive) .+ \s+root : PWD=(/|/root) ; USER=root ; COMMAND=(/run/current-system/sw|/nix/store/[a-z0-9]+-coreutils-[0-9\.]+)/bin/readlink -v -e .+ ''; match = "SYSLOG_IDENTIFIER = sudo"; } { # yodaYoga filters = '' (finished )?switching to system configuration /nix/store/.+-nixos-system-.+-[0-9]+\.[0-9]+pre-git ''; match = "SYSLOG_IDENTIFIER = nixos"; } { # yodaYoga, yodaNas filters = '' \S+\.(service|scope|slice|mount): Consumed .+ CPU time, read .+ from disk, written .+ to disk(, .+|\.) \S+\.(service|scope|slice|mount): Consumed .+ CPU time(, .+)?, received .+ IP traffic, sent .+ IP traffic\. \S+\.(service|scope|slice|mount): Consumed .+ CPU time, no IP traffic\. # Starting Takes BTRFS snapshots and maintains retention policies\.\.\.\. Finished Takes BTRFS snapshots and maintains retention policies\.\. # Started Logrotate Service\. \S+\.(service|mount): Deactivated successfully\. # Deactivated successfully\. Shutting down\. ''; match = "SYSLOG_IDENTIFIER = systemd"; } { # yodaTux filters = '' .* ''; match = "SYSLOG_IDENTIFIER = //nix/store/.+/libexec/gdm-x-session/"; } { filters = ('' # Somebody evil iterating through different ports refused connection: IN=\S+ OUT= MAC=\S+ SRC=\S+ DST=\S+ LEN=\S+ TC=0 HOPLIMIT=255 FLOWLBL=\S+ PROTO=TCP SPT=\S+ DPT=\S+ WINDOW=\S+ RES=0x00 SYN URGP=0 # Ignore. systemd\[[0-9]\]: memfd_create\(\) called without MFD_EXEC or MFD_NOEXEC_SEAL set # Ignore. Unknown kernel command line parameters "ip=dhcp", will be passed to user space. # Ignore. random: crng reseeded on system resumption random: crng init done # Ignore. BTRFS warning \(device dm-[0-9]+\): skipping scrub of block group [0-9]+ due to active swapfile # Ignore. ata[0-9]\.00: Entering standby power mode ata[0-9]\.00: Entering active power mode # # yodaTux # # Ignore. audit: type=2000 audit\([0-9]+\.[0-9]+:[0-9]+\): state=initialized audit_enabled=0 res=1 ENERGY_PERF_BIAS: Set to 'normal', was 'performance' Kernel command line: initrd=\S+ init=\S+ ip=dhcp mds=full,nosmt loglevel=[0-9] Kernel command line: initrd=\S+ init=\S+ ip=dhcp loglevel=[0-9] # Linux version 6.1.61 (nixbld@localhost) (gcc (GCC) 12.2.0, GNU ld (GNU Binutils) 2.40) #1-NixOS SMP PREEMPT_DYNAMIC Thu Nov 2 08:35:33 UTC 2023 Linux version \S+ \(nixbld@localhost\) \(gcc \(GCC\) \S+, GNU ld \(GNU Binutils\) \S+\) #1-NixOS SMP PREEMPT_DYNAMIC \S+ \S+\s+\S+ \S+ UTC \S+ sd [0-9]:0:0:0: \[sd[a-z]\] [0-9]+ 512-byte logical blocks: \(\S+ (GB|TB)/\S+ (GiB|TiB)\) sd [0-9]:0:0:0: \[sd[a-z]\] [0-9]+-byte physical blocks sd [0-9]:0:0:0: \[sd[a-z]\] Write Protect is off sd [0-9]:0:0:0: \[sd[a-z]\] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA sd [0-9]:0:0:0: \[sd[a-z]\] Optimal transfer size [0-9]+ bytes not a multiple of preferred minimum block size ([0-9] bytes) sd [0-9]:0:0:0: \[sd[a-z]\] Attached SCSI (removable )?disk sd [0-9]:0:0:0: \[sd[a-z]\] Starting disk sd [0-9]:0:0:0: \[sd[a-z]\] Stopping disk sd [0-9]:0:0:0: \[sd[a-z]\] Synchronizing SCSI cache sd [0-9]:0:0:0: \[sd[a-z]\] supports TCG Opal \#3 # # yodaTab # # Ignore. mmc0: cannot verify signal voltage switch Initialise system trusted keyrings Key type asymmetric registered Asymmetric key parser 'x509' registered Loading compiled-in X\.509 certificates Key type \.fscrypt registered Key type fscrypt-provisioning registered Key type encrypted registered Bridge firewalling registered SCSI subsystem initialized scsi [0-9]:0:0:0: Direct-Access\s+ATA.+PQ: 0 ANSI: 5 scsi [0-9]:0:0:0: Direct-Access.+PQ: 0 ANSI: 6 thinkpad_acpi: Disabling thinkpad-acpi brightness events by default\.\.\. VFS: Disk quotas dquot_[0-9]+\.[0-9]+\.[0-9]+ ata1\.00: supports DRM functions and may not be fully accessible # done\. '' + (if (config.networking.hostName == "yodaHedgehog") then '' # # Not sure about this. # hpet: number irqs doesn't agree with number of timers gpio gpiochip0: \(INT0002 Virtual GPIO\): not an immutable chip, please consider fixing it! at24 0-0050: supply vcc not found, using dummy regulator intel-spi: probe of intel-spi failed with error -22 # # Ignore. # ACPI BIOS Warning \(bug\): .* ata[0-9]: link is slow to respond, please be patient \(ready=0\) # Not sure about this. ata4\.00: qc timeout after 10000 msecs \(cmd 0x40\) ata4\.00: VERIFY failed \(err_mask=0x4\) ata4\.00: failed to IDENTIFY \(I/O error, err_mask=0x40\) ata4\.00: revalidation failed \(errno=-5\) # # Ignore very frequent kernel stack traces # ------------\[ cut here \]------------ i915 0000:00:02\.0: drm_WARN_ON\(\(intel_uncore_read\(&dev_priv->uncore, \(\(const i915_reg_t\){ \.reg = \(0x130090\) }\)\) & mask\) != mask\) WARNING: CPU: [0-9]+ PID: [0-9]+ at drivers/gpu/drm/i915/vlv_suspend\.c:.* Modules linked in:( \S+)+ ? ( dmi_sysfs| efivarfs| nfnetlink| configfs| efi_pstore| ip_tables| x_tables| autofs4| dm_crypt| cbc| irqbypass| drivetemp| drm| fuse| deflate){5}( \S+)+ CPU: [0-9]+ PID: [0-9]+ Comm: kworker/\S+ Tainted: G W [0-9]+\.[0-9]+\.[0-9]+ #1-NixOS CPU: [0-9]+ PID: [0-9]+ Comm: kworker/\S+ Not tainted [0-9]+\.[0-9]+\.[0-9]+ #1-NixOS Hardware name: To Be Filled By O\.E\.M\. To Be Filled By O\.E\.M\./N3150-ITX, BIOS P1\.10 05/20/2015 Workqueue: pm pm_runtime_work Workqueue: events_unbound async_run_entry_fn RIP: .*:vlv_suspend_complete\+0x[0-9abcdef]+/0x[0-9abcdef]+ \[i915\] Code: 8b 7b 08 4c 8b 67 50 4d 85 e4 74 25 e8 ([0-9a-f][0-9a-f] )+<0f> 0b e9 [0-9a-f][0-9a-f] f9 ff ff 4c 8b 27 eb d6( [0-9a-f][0-9a-f])+ RSP: [0-9]+:[0-9a-f]+ EFLAGS: [0-9a-f]+ RAX: 0000000000000000 RBX: [0-9a-f]+ RCX: [0-9a-f]+ RDX: [0-9a-f]+ RSI: [0-9a-f]+ RDI: [0-9a-f]+ RBP: [0-9a-f]+ R08: [0-9a-f]+ R09: [0-9a-f]+ R10: [0-9a-f]+ R11: [0-9a-f]+ R12: [0-9a-f]+ R13: [0-9a-f]+ R14: [0-9a-f]+ R15: [0-9a-f]+ FS: 0000000000000000\(0000\) GS:[0-9a-f]+\(0000\) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: [0-9a-f]+ CR3: [0-9a-f]+ CR4: [0-9a-f]+ Call Trace: \? \S+\+0x[0-9abcdef]+/0x[0-9abcdef]+( \[i915\])? (intel_runtime_suspend|pci_pm_runtime_suspend|__rpm_callback|rpm_callback|rpm_suspend|pm_runtime_work|process_one_work|worker_thread|kthread|ret_from_fork|ret_from_fork_asm|async_run_entry_fn|async_suspend_late|__device_suspend_late|dpm_run_callback|i915_drm_suspend_late)\+0x[0-9abcdef]+/0x[0-9abcdef]+( \[i915\])? ---\[ end trace 0000000000000000 \]--- '' else if (config.networking.hostName == "yodaYoga") then '' # Ignore BIOS bug. ACPI: \[Firmware Bug\]: .* ACPI BIOS Error \(bug\): .* ACPI Error: .* ACPI Warning: .* # Ignore. usb: port power management may be unreliable '' else if (config.networking.hostName == "yodaTux") then '' # Ignore BIOS bug. ACPI: FW issue: working around C-state latencies out of order # Kernel WiFi driver bug. #iwlwifi 0000:01:00\.0: .* iwlwifi 0000:01:00\.0: Unhandled alg: 0x707 iwlwifi 0000:01:00\.0: Not associated and the session protection is over already... iwlwifi 0000:01:00\.0: api flags index 2 larger than supported by driver '' else if (config.networking.hostName == "yodaNas") then '' # Probably safe to ignore. resource: resource sanity check: requesting \[mem 0x[0-9a-f]+-0x[0-9a-f]+\], which spans more than Reserved \[mem 0x[0-9a-f]+-0x[0-9a-f]+\] # Ingore: Probably just copying of boot graphics (.bmp) failed. https://patchwork.kernel.org/project/linux-fbdev/patch/20180617153235.16219-3-hdegoede@redhat.com/ efifb: Ignoring BGRT: unexpected or invalid BMP data # Not sure about this. i915 0000:00:02\.0: \[drm\] \[ENCODER:.*\] is disabled/in DSI mode with an ungated DDI clock, gate it # Not sure about this. Probably BIOS related. # https://wiki.archlinux.org/title/improving_performance#Enabling_PCI_resizable_BAR caller (pmc_core_probe|get_primary_reg_base)\+0x\S+ \[intel_pmc_core\] mapping multiple BARs '' else "")); match = "SYSLOG_IDENTIFIER = kernel"; } { # yodaTux filters = '' .* ''; match = "SYSLOG_IDENTIFIER = simple-scan"; } # # _SYSTEMD_USER_UNIT # { # yodaTux, yodaTab filters = '' .+ Setting AttentionNeeded to FALSE because EnsureCredentials\(\) succeded Connecting to org\.freedesktop\.Tracker3\.Miner\.Files ''; match = "_SYSTEMD_USER_UNIT = dbus.service"; } { # yodaTux filters = '' .* ''; match = "_SYSTEMD_USER_UNIT = /(org\\.gnome\\..+\\.service|pipewire\\.service|wireplumber\\.service|app-gnome-org\\.gnome\\.Software-[0-9]+\\.scope)/"; } ]; }; }