diff --git a/README.md b/README.md index 17d9e19..da9307f 100644 --- a/README.md +++ b/README.md @@ -2,33 +2,19 @@ ## Usage -Usage: ` | ./main.py` +Redirect your Snowflake logs to this python script. -- Example: Pipe docker log from remote computer into `main.py` +Add argument `-p` if they are "plain" Snowflake logs or `-s` if Snowflakes output was logged by systemd. - ```bash - (venv-310) [user@linux snoflake-stats]$ ssh root_at_my_server 'docker logs snowflake-proxy' 2>&1 | ./main.py - ``` +Examples: -- Example: Pipe logfile into `main.py` +```shell +ssh root@snowflake-host 'docker logs snowflake-proxy' 2>&1 | ./main.py -p +``` - ```bash - (venv-310) [user@linux snoflake-stats]$ cat snowflake.log | ./main.py - ``` - -Usage: `./main.py []` - -- Example: Specify name of local docker container - - ```bash - (venv-310) [user@linux snoflake-stats]$ ./main.py snowflake-proxy - ``` - -- Example: Docker container name and ssh hostname - - ```bash - (venv-310) [user@linux snoflake-stats]$ ./main.py snowflake-proxy root_at_my_server - ``` +```shell +ssh user@snowflake-host 'journalctl -u snowflake-proxy.service' | ./main.py -s +``` ## Example output diff --git a/exec.py b/exec.py deleted file mode 100644 index 34e7015..0000000 --- a/exec.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 -from typing import List -import sys -import subprocess -import shlex - - -class NonZeroExitCode(Exception): - def __init__(self, exit_code, stdout, stderr): - super(NonZeroExitCode, self).__init__(f'exit_code: {exit_code}, stdout: {stdout}, stderr: {stderr}') - - -def wrap_command_with_ssh(command: List[str], ssh_host: str) -> List[str]: - return ['ssh', ssh_host, ' '.join([shlex.quote(arg) for arg in command])] - - -def execute(command: List[str], ssh_host: str = None) -> None: - """ - https://docs.python.org/3.10/library/subprocess.html#frequently-used-arguments - - Executes the given command using stdin, stdout, stderr of the parent process. - - The output is not captured. - - :raises subprocess.CalledProcessError: In case of non-zero exit code. - """ - if ssh_host is not None: - command = wrap_command_with_ssh(command, ssh_host) - - print(f'[DEBUG] execute: {command}') - - subprocess.run( - command, - stdin=sys.stdin, - stdout=sys.stdout, - stderr=sys.stderr, - check=True, - ) - - -def execute_and_capture(command: List[str], ssh_host: str = None, file='stdout') -> str: - """ - Executes the given command and captures it stdout. - - :return: Stdout of subprocess. - :raises NonZeroExitCode: In case of non-zero exit code. This exception includes the exit_code, stdout and stderr in it's message. - """ - if ssh_host is not None: - command = wrap_command_with_ssh(command, ssh_host) - - print(f'[DEBUG] execute_and_capture: {command}') - - completed: subprocess.CompletedProcess = subprocess.run( - command, - capture_output=True, - text=True, - ) - if completed.returncode != 0: - raise NonZeroExitCode(completed.returncode, completed.stdout, completed.stderr) - - if file == 'stdout': - return completed.stdout - if file == 'stderr': - return completed.stderr - raise ValueError('invalid argument') diff --git a/main.py b/main.py index 0a8d205..5ebf960 100755 --- a/main.py +++ b/main.py @@ -5,24 +5,55 @@ import re from datetime import datetime import sys -import exec - def test() -> None: - log = example_log() + log = example_log().splitlines() + parse_log(log) + + log = example_systemd_log().splitlines() + log = [remove_systemd_prefix(line) for line in log] parse_log(log) def main() -> None: - if len(sys.argv) > 1: - log = get_docker_log() - else: - log = sys.stdin.read() + systemd = parse_args() + log = sys.stdin.read().splitlines() + if systemd: + log = [remove_systemd_prefix(line) for line in log] parse_log(log) -def parse_log(log: str) -> None: - tps = [Throughput.from_str(line) for line in log.splitlines()] +def parse_args() -> bool: + usage = (f'usage: {sys.argv[0]} -p|-s\n' + f' -p: parse plain Snowflake output\n' + f' -p: parse Snowflake output logged by systemd\n') + + if len(sys.argv) != 2 or sys.argv[0] in ['--help', '-h']: + print(usage, file=sys.stderr) + exit(1) + + arg1 = sys.argv[1] + if arg1 == '-p': + return False + elif arg1 == '-s': + return True + else: + raise Exception(usage) + + +def remove_systemd_prefix(line: str) -> str: + pattern_str = r'[A-Z][a-z]+ [0-9]+ [0-9][0-9]:[0-9][0-9]:[0-9][0-9] \S+ \S+\[[0-9]+\]: (.+)' + pattern = re.compile(pattern_str) + match = pattern.match(line) + + if not match: + raise Exception(match) + + return match.group(1) + + +def parse_log(log: list[str]) -> None: + tps = [Throughput.from_str(line) for line in log] tps = [tp for tp in tps if tp] if len(tps) > 0: print() @@ -50,31 +81,14 @@ def example_log() -> str: [ '2022/09/13 15:08:36 Proxy starting', '2022/09/13 15:08:43 NAT type: unrestricted', - '2022/09/27 02:02:26 In the last 1h0m0s, there were 1 connections. Traffic Relayed ↑ 708 KB, ↓ 328 KB.', - '2022/09/28 02:02:26 In the last 1h0m0s, there were 0 connections. Traffic Relayed ↑ 0 B, ↓ 0 B.', - '2022/09/29 05:02:26 In the last 1h0m0s, there were 5 connections. Traffic Relayed ↑ 6 MB, ↓ 787 KB.', - '2022/09/29 11:02:26 In the last 1h0m0s, there were 26 connections. Traffic Relayed ↑ 16 MB, ↓ 10 MB.', + '2025/02/19 18:24:29 In the last 1h0m0s, there were 31 completed connections. Traffic Relayed ↓ 46719 KB, ↑ 3229 KB.' 'sctp ERROR: 2022/09/29 23:00:53 [0xc00006e000] stream 1 not found)', ] ) -def get_docker_log() -> str: - if len(sys.argv) < 2: - print(f'usage: {sys.argv[0]} []', file=sys.stderr) - exit(1) - - container_name = sys.argv[1] - - ssh_hostname = None - if len(sys.argv) > 2: - ssh_hostname = sys.argv[2] - - return docker_logs(container_name, ssh_hostname) - - -def docker_logs(container_name: str, ssh_host: str = None) -> str: - return exec.execute_and_capture(['docker', 'logs', container_name], ssh_host, 'stderr') +def example_systemd_log() -> str: + return 'Feb 19 19:24:29 yodaNas proxy[1318]: 2025/02/19 18:24:29 In the last 1h0m0s, there were 3 completed connections. Traffic Relayed ↓ 46719 KB, ↑ 3229 KB.\n' class Throughput: @@ -93,8 +107,8 @@ class Throughput: @classmethod def from_str(cls, line: str) -> Throughput | None: pattern_str = r'(\d\d\d\d/\d\d/\d\d \d\d:\d\d:\d\d)' \ - r' In the last 1h0m0s, there were (\d+) connections\. ' \ - r'Traffic Relayed ↑ (\d+ [A-Z]+), ↓ (\d+ [A-Z]+)\.' + r' In the last 1h0m0s, there were (\d+) completed connections\. ' \ + r'Traffic Relayed ↓ (\d+ [A-Z]+), ↑ (\d+ [A-Z]+)\.' pattern = re.compile(pattern_str) match = pattern.match(line) @@ -109,8 +123,8 @@ class Throughput: dt = datetime.strptime(match.group(1), Throughput.DATE_FORMAT_STR) connections = int(match.group(2)) - bytes_up = cls._split_to_bytes(match.group(3)) - bytes_down = cls._split_to_bytes(match.group(4)) + bytes_down = cls._split_to_bytes(match.group(3)) + bytes_up = cls._split_to_bytes(match.group(4)) return cls(dt, bytes_up, bytes_down, connections)