From 13bbb37b1936e9c04ee5c7ca84b1c012d5cdbf43 Mon Sep 17 00:00:00 2001 From: Daniel Langbein Date: Wed, 6 Apr 2022 14:42:27 +0200 Subject: [PATCH] init --- .gitignore | 1 + README.md | 24 ++++++++++- exec.py | 65 ++++++++++++++++++++++++++++ main.py | 124 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 exec.py create mode 100755 main.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..85e7c1d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/.idea/ diff --git a/README.md b/README.md index 4deb5ed..3f0d1f8 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,24 @@ -# snoflake-stats +# snowflake-stats +## Usage + +```bash +(venv-310) [user@linux snoflake-stats]$ ./main.py +``` + +## Example output + +``` +[DEBUG] execute_and_capture: ['ssh', 'root_at_my_server', 'docker logs snowflake-proxy'] +From 2022-03-28 11:58:25 until 2022-04-06 12:31:40: +2022-03-28: 1.8 GB, 0.1 GB -- (335199 OnMessages, 1799508 sends, 151390 seconds) +2022-03-29: 3.2 GB, 0.2 GB -- (509488 OnMessages, 2925767 sends, 117434 seconds) +2022-03-30: 3.5 GB, 0.2 GB -- (815323 OnMessages, 3441369 sends, 907236 seconds) +2022-03-31: 1.7 GB, 0.1 GB -- (350893 OnMessages, 1801615 sends, 222564 seconds) +2022-04-01: 7.0 GB, 0.4 GB -- (1358655 OnMessages, 6719817 sends, 239774 seconds) +2022-04-02: 1.8 GB, 0.1 GB -- (340389 OnMessages, 1702678 sends, 92243 seconds) +2022-04-03: 4.9 GB, 0.3 GB -- (972402 OnMessages, 4976213 sends, 1249335 seconds) +2022-04-04: 4.8 GB, 0.3 GB -- (824929 OnMessages, 5106120 sends, 265337 seconds) +2022-04-05: 2.1 GB, 0.2 GB -- (475139 OnMessages, 2322166 sends, 298093 seconds) +2022-04-06: 0.5 GB, 0.2 GB -- (553513 OnMessages, 894773 sends, 206981 seconds) +``` diff --git a/exec.py b/exec.py new file mode 100644 index 0000000..34e7015 --- /dev/null +++ b/exec.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +from typing import List +import sys +import subprocess +import shlex + + +class NonZeroExitCode(Exception): + def __init__(self, exit_code, stdout, stderr): + super(NonZeroExitCode, self).__init__(f'exit_code: {exit_code}, stdout: {stdout}, stderr: {stderr}') + + +def wrap_command_with_ssh(command: List[str], ssh_host: str) -> List[str]: + return ['ssh', ssh_host, ' '.join([shlex.quote(arg) for arg in command])] + + +def execute(command: List[str], ssh_host: str = None) -> None: + """ + https://docs.python.org/3.10/library/subprocess.html#frequently-used-arguments + + Executes the given command using stdin, stdout, stderr of the parent process. + + The output is not captured. + + :raises subprocess.CalledProcessError: In case of non-zero exit code. + """ + if ssh_host is not None: + command = wrap_command_with_ssh(command, ssh_host) + + print(f'[DEBUG] execute: {command}') + + subprocess.run( + command, + stdin=sys.stdin, + stdout=sys.stdout, + stderr=sys.stderr, + check=True, + ) + + +def execute_and_capture(command: List[str], ssh_host: str = None, file='stdout') -> str: + """ + Executes the given command and captures it stdout. + + :return: Stdout of subprocess. + :raises NonZeroExitCode: In case of non-zero exit code. This exception includes the exit_code, stdout and stderr in it's message. + """ + if ssh_host is not None: + command = wrap_command_with_ssh(command, ssh_host) + + print(f'[DEBUG] execute_and_capture: {command}') + + completed: subprocess.CompletedProcess = subprocess.run( + command, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + raise NonZeroExitCode(completed.returncode, completed.stdout, completed.stderr) + + if file == 'stdout': + return completed.stdout + if file == 'stderr': + return completed.stderr + raise ValueError('invalid argument') diff --git a/main.py b/main.py new file mode 100755 index 0000000..1f70feb --- /dev/null +++ b/main.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +from __future__ import annotations +from typing import List +from datetime import datetime + +import exec + + +def main(): + log: str = docker_logs(container_name='snowflake-proxy', ssh_host='rootnas') + filtered: List[str] = [line for line in log.splitlines() + if Throughput.PATTERN in line] + # filtered = filtered_example() + + tps = [Throughput.from_str(line) for line in filtered] + print(f'From {tps[0].dt} until {tps[-1].dt}:') + + grouped_by_day = {} + for tp in tps: + if tp.dt.date() in grouped_by_day.keys(): + grouped_by_day[tp.dt.date()].append(tp) + else: + grouped_by_day[tp.dt.date()] = [tp] + + for day, tp_list in grouped_by_day.items(): + tps_sum = sum(tp_list, Throughput.zero()) + print(f'{day}: {tps_sum}') + + +def filtered_example() -> List[str]: + return [ + '2022/04/04 15:08:10 Traffic throughput (up|down): 4 MB|259 KB -- (691 OnMessages, 3886 Sends, over 269 seconds)', + '2022/04/04 16:00:06 Traffic throughput (up|down): 13 MB|15 MB -- (46326 OnMessages, 32325 Sends, over 36634 seconds)', + '2022/04/04 15:57:04 Traffic throughput (up|down): 61 KB|8 KB -- (69 OnMessages, 91 Sends, over 157 seconds)', + ] + + +def docker_logs(container_name: str, ssh_host: str = None) -> str: + return exec.execute_and_capture(['docker', 'logs', container_name], ssh_host, 'stderr') + + +class Throughput: + FORMAT_EXAMPLE = '2022/04/06 10:37:42' + FORMAT_STR = '%Y/%m/%d %H:%M:%S' + FORMAT_LENGTH = len(FORMAT_EXAMPLE) + + PATTERN = ' Traffic throughput (up|down): ' + + _unit_dict = { + 'B': 1, + 'KB': 10 ** 3, + 'MB': 10 ** 6, + 'GB': 10 ** 9, + 'TB': 10 ** 12, + } + + @classmethod + def from_str(cls, line: str) -> Throughput: + dt_str = line[0:Throughput.FORMAT_LENGTH] + dt = datetime.strptime(dt_str, Throughput.FORMAT_STR) + + _, tail = line.split(Throughput.PATTERN) + up, tail = tail.split('|') + down, tail = tail.split(' -- (') + on_messages, tail = tail.split(', ', maxsplit=1) + sends, tail = tail.split(', ') + seconds, tail = tail.split(')') + + bytes_up = cls._split_to_bytes(up) + bytes_down = cls._split_to_bytes(down) + on_messages = int(on_messages.split(' ')[0]) + sends = int(sends.split(' ')[0]) + seconds = int(seconds.split(' ')[1]) + + return cls(dt, bytes_up, bytes_down, on_messages, sends, seconds) + + @classmethod + def from_args(cls, dt, bytes_up, bytes_down, on_messages, sends, seconds) -> Throughput: + return cls(dt, bytes_up, bytes_down, on_messages, sends, seconds) + + @classmethod + def zero(cls) -> Throughput: + return cls(None, 0, 0, 0, 0, 0) + + def __init__(self, dt, bytes_up, bytes_down, on_messages, sends, seconds): + self.dt = dt + self.bytes_up = bytes_up + self.bytes_down = bytes_down + self.on_messages = on_messages + self.sends = sends + self.seconds = seconds + + def __add__(self, other): + if not isinstance(other, Throughput): + raise ValueError('invalid argument') + return Throughput.from_args( + self.dt, + self.bytes_up + other.bytes_up, + self.bytes_down + other.bytes_down, + self.on_messages + other.on_messages, + self.sends + other.sends, + self.seconds + other.seconds, + ) + + def __str__(self) -> str: + return f'{Throughput._to_gb(self.bytes_up)} GB, {Throughput._to_gb(self.bytes_down)} GB -- ({self.on_messages} OnMessages, {self.sends} sends, {self.seconds} seconds)' + + @classmethod + def _split_to_bytes(cls, num_unit: str) -> int: + num, unit = num_unit.split(' ') + num = int(num) + return cls._to_bytes(num, unit) + + @classmethod + def _to_bytes(cls, num: int, unit: str) -> int: + return num * cls._unit_dict[unit] + + @classmethod + def _to_gb(cls, num_bytes: int) -> int: + return round(num_bytes / cls._unit_dict['GB'], 1) + + +if __name__ == '__main__': + main()