diff --git a/modules/journalwatch.nix b/modules/journalwatch.nix index f46f679..16d4905 100644 --- a/modules/journalwatch.nix +++ b/modules/journalwatch.nix @@ -1,6 +1,40 @@ +# Systemd Journal Monitoring. +# +# Alternative: +# journal-biref +# https://github.com/twaugh/journal-brief +# https://opensource.com/article/20/7/systemd-journals-email + +# Write to Systemd Journal: +# echo 'hello' | systemd-cat -p emerg +# echo 'hello' | systemd-cat -t someapp -p emerg + +# View Systemd Journal. +# Output similar to dmesg +# journalctl -b -k +# Filter by app: +# journalctl -b -t someapp +# Filter by priority: +# journalctl -b -p 5 + +# Manually execute journalwatch timer: +# sudo systemctl start journalwatch.service + +# View generated journalwatch config: +# cat "$(systemctl show -P FragmentPath journalwatch.service)" | grep XDG_CONFIG_HOME +# #=> /nix/store/vzfvp7kizycs6lnks6icmlw3h1x6p7la-journalwatch-config +# cat /nix/store/vzfvp7kizycs6lnks6icmlw3h1x6p7la-journalwatch-config/journalwatch/config +# cat /nix/store/vzfvp7kizycs6lnks6icmlw3h1x6p7la-journalwatch-config/journalwatch/patterns + +# Find a message and view its details +# journalctl -b -p5 -o json-pretty +# Then press "/", enter a pattern and press "Enter". + { config, pkgs, ... }: let sshd_filter = '' + Accepted publickey for (root|yoda) from \S+ port \S+ ssh2: ED25519 SHA256:\S+ + # TODO: Change RSA key of yodaHedgehog to ED25519 Accepted publickey for (root|yoda) from \S+ port \S+ ssh2: RSA SHA256:\S+ pam_unix\(sshd:session\): session opened for user \S+ by \S+ Received disconnect from \S+ port \S+:11: disconnected by user @@ -39,37 +73,6 @@ let ''; in { - # Systemd Journal Monitoring. - # Alternative: - # journal-biref - # https://github.com/twaugh/journal-brief - # https://opensource.com/article/20/7/systemd-journals-email - - # Write to Systemd Journal: - # echo 'hello' | systemd-cat -p emerg - # echo 'hello' | systemd-cat -t someapp -p emerg - - # View Systemd Journal. - # Output similar to dmesg - # journalctl -b -k - # Filter by app: - # journalctl -b -t someapp - # Filter by priority: - # journalctl -b -p 5 - - # Manually execute journalwatch timer: - # sudo systemctl start journalwatch.service - - # View generated journalwatch config: - # cat "$(systemctl show -P FragmentPath journalwatch.service)" | grep XDG_CONFIG_HOME - # #=> /nix/store/vzfvp7kizycs6lnks6icmlw3h1x6p7la-journalwatch-config - # cat /nix/store/vzfvp7kizycs6lnks6icmlw3h1x6p7la-journalwatch-config/journalwatch/config - # cat /nix/store/vzfvp7kizycs6lnks6icmlw3h1x6p7la-journalwatch-config/journalwatch/patterns - - # Find a message and view its details - # journalctl -b -p5 -o json-pretty - # Then press "/", enter a pattern and press "Enter". - assertions = [{ assertion = config.services.opensmtpd.enable; message = "journalwatch requires a configured sendmail MTA, see sendmail-mta.nix."; @@ -112,15 +115,8 @@ in # _SYSTEMD_CGROUP # - { # yodaYoga - filters = '' - parent not found! continent_id [0-9]+ - ''; - match = "_SYSTEMD_CGROUP = /system.slice/docker.service"; - } - # - # CONTAINER_NAME + # CONTAINER_NAME # TODO use IMAGE_NAME instead # { # yodaNas @@ -534,6 +530,12 @@ in ''; match = "_SYSTEMD_UNIT = /(systemd-logind|syncthing)\\.service/"; } + { # yodaTux + filters = '' + .* + ''; + match = "_SYSTEMD_UNIT = /(bluetooth|cups|systemd-timesyncd)\\.service/"; + } { # yodaNas filters = '' @@ -553,12 +555,11 @@ in } { # yodaNas filters = '' - time="[^"]+" level=error msg="\[resolver\] failed to query DNS server: 127.0.0.1:53, query: ;[a-z0-9\.]+\tIN\t A" error="read udp 127.0.0.1:[0-9]+->127.0.0.1:53: i/o timeout" + time="[^"]+" level=error msg="\[resolver\] failed to query DNS server: 127.0.0.1:53, query: ;[a-z0-9\.]+\\tIN\\t A" error="read udp 127.0.0.1:[0-9]+->127.0.0.1:53: i/o timeout" time="[^"]+" level=info msg="Attempting next endpoint for pull after error: Head "[^"]+": Get "[^"]+": net/http: request canceled while waiting for connection \(Client\.Timeout exceeded while awaiting headers\)" ''; match = "_SYSTEMD_UNIT = docker.service"; } - { # yodaNas filters = '' pam_unix\(sudo:session\): session opened for user root\(uid=0\) by \(uid=994\) @@ -584,7 +585,6 @@ in ''; match = "_SYSTEMD_UNIT = /btrbk-local-snapshot-(hdd|ssd)\\.service/"; } - { # yodaHedgehog filters = '' info: OpenSMTPD \S+-portable starting @@ -612,11 +612,11 @@ in ''; match = "_SYSTEMD_UNIT = systemd-tmpfiles-setup.service"; } - { # yodaTux + { # yodaHedgehog filters = '' - .* + sd[a-z] spinup ''; - match = "_SYSTEMD_UNIT = /(bluetooth\\.service|cups\\.service)/"; + match = "_SYSTEMD_UNIT = hd-idle.service"; } { # yodaNas. TODO: This is maybe a bug. It occurs when deploying and switching to a NixOS config with colmena. filters = '' @@ -636,14 +636,89 @@ in } { # yodaTux, yodaNas filters = '' - Starting \S+\.service\.\.\. - \S+\.service: Deactivated successfully\. - Finished \S+\.service\. + \S+\.(service|scope|slice|mount): Consumed [0-9\.]+(s|ms) CPU time(, .+)?\. # - Reexecuting\. - (finished )?switching to system configuration /nix/store/.+-nixos-system-.+-[0-9]+\.[0-9]+pre-git + Starting \S+\.service\.\.\. + \S+\.service: Scheduled restart job, restart counter is at 1\. + Finished \S+\.service\. + \S+\.(service|mount): Deactivated successfully\. + # + Starting User Runtime Directory /run/user/(0|1000)\.\.\. + Starting User Manager for UID (0|1000)\.\.\. + Starting Spin down inactive HDD \S+\.\.\. + Starting Takes BTRFS snapshots and maintains retention policies\.\.\.\. + Starting Load Kernel Module efi_pstore\.\.\. + Starting Create SUID/SGID Wrappers\.\.\. + Starting Update and start Jinja-Compose project\.\.\. + Started \S+\.service\. + Started Logrotate Service\. + Started User Manager for UID (0|1000)\. + Started Session [0-9]+ of User root\. + Finished Takes BTRFS snapshots and maintains retention policies\.\. + Finished User Runtime Directory /run/user/(0|1000)\. + Finished Spin down inactive HDD \S+\. + Finished Load Kernel Module efi_pstore\. + Finished Create SUID/SGID Wrappers\. + Finished Update and start Jinja-Compose project\. + Stopping Reactivate sysinit units\.\.\. + Stopping User Manager for UID (0|1000)\.\.\. + Stopping User Runtime Directory /run/user/(0|1000)\.\.\. + Stopped target Local File Systems\. + Stopped target Remote File Systems\. + Stopped target Reactivate sysinit units\. + Stopped User Manager for UID (0|1000)\. + Stopped User Runtime Directory /run/user/(0|1000)\.\.\. + Reached target Reactivate sysinit units\. + Reached target Local File Systems\. + Reached target Remote File Systems\. + Reloading requested from client PID [0-9]+ \('systemctl'\) \(unit session-[0-9]+\.scope\)\.\.\. + Reloading\.\.\. + Reloading finished in [0-9]+ ms\. + session-[0-9]+\.scope: Deactivated successfully\. + # + (\S+ )+was skipped because of an unmet condition check \([^)]+\)\. + File System Check on Root Device was skipped because of an unmet condition check (ConditionPathIsReadWrite=!/). + # + # TODO bug + smtpd-key\.service: Found left-over process [0-9]+ \(smtpd-key-pre-s\) in control group while starting unit\. Ignoring\. + smtpd-key\.service: This usually indicates unclean termination of a previous run, or service implementation deficiencies\. ''; - match = "_SYSTEMD_UNIT = /(user@[0-9]+\\.service|init.scope)/"; + match = "_SYSTEMD_UNIT = init.scope"; + } + { # yodaTux, yodaNas + filters = '' + Activating special unit Exit the Session\.\.\. + Starting D-Bus User Message Bus Socket\.\.\. + Starting Run user-specific NixOS activation\.\.\. + Listening on D-Bus User Message Bus Socket\. + Stopped target Main User Target\. + Stopped target Basic System\. + Stopped target Paths\. + Stopped target Sockets\. + Stopped target Timers\. + Closed D-Bus User Message Bus Socket\. + Removed slice User Application Slice\. + Reached target Shutdown\. + Reached target Paths\. + Reached target Timers\. + Reached target Sockets\. + Reached target Basic System\. + Reached target Exit the Session\. + Reached target Main User Target\. + Finished Exit the Session\. + Finished Run user-specific NixOS activation\. + # + Startup finished in [0-9]+ms\. + # + Created slice User Application Slice\. + Queued start job for default target Main User Target\. + pam_unix\(systemd-user:session\): session opened for user root\(uid=0\) by \(uid=0\) + pam_unix\(systemd-user:session\): session closed for user root + Reexecuting requested from client PID [0-9]+ \('systemctl'\)\.\.\. + Reexecuting\. + Run \S+ was skipped because of an unmet condition check \([^)]+\)\. + ''; + match = "_SYSTEMD_UNIT = /user@(0|1000)\\.service/"; } { # yodaTux filters = '' @@ -652,6 +727,10 @@ in Collecting garbage unconditionally\.\.\. Loading rules from directory /.+ Finished loading, compiling and executing [0-9]+ rules + # + Registered Authentication Agent for unix-process:[0-9]+:[0-9]+ \(system bus name :[0-9\.]+ \[/run/current-system/sw/bin/pkttyagent --notify-fd 5 --fallback\], object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale en_US.UTF-8\) + Operator of unix-process:[0-9]+:[0-9]+ successfully authenticated as unix-user:yoda to gain ONE-SHOT authorization for action org\.freedesktop\.systemd1\.manage-units for system-bus-name::[0-9\.]+ \[systemctl start journalwatch\.service\] \(owned by unix-user:yoda\) + Unregistered Authentication Agent for unix-process:[0-9]+:[0-9]+ \(system bus name :[0-9\.]+, object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale en_US.UTF-8\) \(disconnected from bus\) ''; match = "_SYSTEMD_UNIT = polkit.service"; } @@ -674,6 +753,12 @@ in ''; match = "_SYSTEMD_UNIT = udisks2.service"; } + { # yodaNas + filters = '' + Disk is in standby or sleeping\. + ''; + match = "_SYSTEMD_UNIT = /hdparm-ata-.*\\.service/"; + } { # yodaNas filters = '' \s+\S+ Skipped - Image is already being pulled by \S+ @@ -723,51 +808,6 @@ in ''; match = "SYSLOG_IDENTIFIER = nixos"; } - { # yodaYoga, yodaNas - filters = '' - \S+\.(service|scope|slice|mount): Consumed .+ CPU time, read .+ from disk, written .+ to disk(, .+|\.) - \S+\.(service|scope|slice|mount): Consumed .+ CPU time(, .+)?, received .+ IP traffic, sent .+ IP traffic\. - \S+\.(service|scope|slice|mount): Consumed .+ CPU time(, .+)?, no IP traffic\. - # - .*smtpd-key\.service.* - # - Starting User Runtime Directory /run/user/(0|1000)\.\.\. - Finished User Runtime Directory /run/user/(0|1000)\. - Stopping User Runtime Directory /run/user/(0|1000)\.\.\. - Stopped User Runtime Directory /run/user/(0|1000)\. - Starting User Manager for UID (0|1000)\.\.\. - Started User Manager for UID (0|1000)\. - Stopping User Manager for UID (0|1000)\.\.\. - Stopped User Manager for UID (0|1000)\. - Started Session [0-9]+ of User (root|yoda)\. - session-[0-9]+\.scope: Deactivated successfully\. - # - Starting Load Kernel Module efi_pstore\.\.\. - Starting Create SUID/SGID Wrappers\.\.\. - Stopped target Reactivate sysinit units\. - Stopping Reactivate sysinit units\.\.\. - Reached target Reactivate sysinit units\. - Reached target Local File Systems\. - Reached target Remote File Systems\. - Finished Load Kernel Module efi_pstore\. - Finished Create SUID/SGID Wrappers\. - [a-zA-Z ]+ was skipped because of an unmet condition check \([^\)]+\)\. - Update is Completed was skipped because no trigger condition checks were met\. - # - Starting Update and start Jinja-Compose project\.\.\. - Finished Update and start Jinja-Compose project\. - # - Starting Takes BTRFS snapshots and maintains retention policies\.\.\.\. - Finished Takes BTRFS snapshots and maintains retention policies\.\. - # - Started Logrotate Service\. - \S+\.(service|mount): Deactivated successfully\. - # - Deactivated successfully\. - Shutting down\. - ''; - match = "SYSLOG_IDENTIFIER = systemd"; - } { # yodaTux filters = '' .* @@ -921,6 +961,12 @@ in # _SYSTEMD_USER_UNIT # + { # yodaTux + filters = '' + .* + ''; + match = "_SYSTEMD_USER_UNIT = /(org\\.gnome\\..+\\.service|pipewire\\.service|wireplumber\\.service|app-gnome-org\\.gnome\\.Software-[0-9]+\\.scope)/"; + } { # yodaTux, yodaTab filters = '' .+ Setting AttentionNeeded to FALSE because EnsureCredentials\(\) succeded @@ -928,12 +974,6 @@ in ''; match = "_SYSTEMD_USER_UNIT = dbus.service"; } - { # yodaTux - filters = '' - .* - ''; - match = "_SYSTEMD_USER_UNIT = /(org\\.gnome\\..+\\.service|pipewire\\.service|wireplumber\\.service|app-gnome-org\\.gnome\\.Software-[0-9]+\\.scope)/"; - } ]; }; }