From 32d815892121a65629cc89d857f836c2ec4bfe56 Mon Sep 17 00:00:00 2001 From: Daniel Langbein Date: Sat, 28 Nov 2020 21:11:21 +0100 Subject: [PATCH] plot data --- Dockerfile | 4 +- docker-compose.yml | 5 +- plot/dummy | 0 src/langfingaz/__init__.py | 0 src/{ => langfingaz}/bbbRequest.py | 10 ++-- src/langfingaz/loadData.py | 12 ++++ src/{ => langfingaz}/logMeetingData.py | 17 +++--- src/{ => langfingaz}/parseMeetings.py | 11 +++- src/langfingaz/plotMeetings.py | 52 ++++++++++++++++ .../saveData.py} | 24 ++++---- src/langfingaz/util/__init__.py | 0 src/langfingaz/util/fileUtil.py | 51 ++++++++++++++++ src/{ => langfingaz/util}/util.py | 15 ----- src/langfingaz/util/utilTest.py | 60 +++++++++++++++++++ src/utilTest.py | 24 -------- 15 files changed, 214 insertions(+), 71 deletions(-) create mode 100644 plot/dummy create mode 100644 src/langfingaz/__init__.py rename src/{ => langfingaz}/bbbRequest.py (78%) create mode 100644 src/langfingaz/loadData.py rename src/{ => langfingaz}/logMeetingData.py (68%) rename src/{ => langfingaz}/parseMeetings.py (87%) create mode 100644 src/langfingaz/plotMeetings.py rename src/{saveMeetings.py => langfingaz/saveData.py} (52%) create mode 100644 src/langfingaz/util/__init__.py create mode 100644 src/langfingaz/util/fileUtil.py rename src/{ => langfingaz/util}/util.py (75%) create mode 100644 src/langfingaz/util/utilTest.py delete mode 100644 src/utilTest.py diff --git a/Dockerfile b/Dockerfile index 7771e32..8fc91a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY requirements.txt ./ RUN pip3 install --no-cache-dir -r requirements.txt COPY ./src/ ./ -RUN chmod +x ./logMeetingData.py +RUN chmod +x ./langfingaz/logMeetingData.py # unbuffered output option otherwise script sleeps before any output appears -CMD [ "python", "-u", "./logMeetingData.py" ] +CMD [ "python", "-u", "./langfingaz/logMeetingData.py" ] diff --git a/docker-compose.yml b/docker-compose.yml index 300b2ef..a0c279d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,8 +2,9 @@ version: '3.7' services: dk-gen: build: . -# environment: -# - PYTHONPATH=/usr/src/ + environment: + # pythonpath is required to import from self created modules (langfingaz) + - PYTHONPATH=/usr/src/ volumes: - ./secret:/usr/secret - ./data:/usr/data diff --git a/plot/dummy b/plot/dummy new file mode 100644 index 0000000..e69de29 diff --git a/src/langfingaz/__init__.py b/src/langfingaz/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/bbbRequest.py b/src/langfingaz/bbbRequest.py similarity index 78% rename from src/bbbRequest.py rename to src/langfingaz/bbbRequest.py index 5490b0a..11e1349 100644 --- a/src/bbbRequest.py +++ b/src/langfingaz/bbbRequest.py @@ -2,7 +2,7 @@ import hashlib import requests from xml.etree import ElementTree -import util +import langfingaz.util.fileUtil as fileUtil def requestMeetingData() -> str: @@ -31,16 +31,16 @@ def getRequestUrl(api_method: str = 'getMeetings', query_string: str = '') -> st def getUrl() -> str: - filepath = util.getWorkingDir().joinpath("../secret").joinpath("url.txt") - url = util.readFirstLine(filepath).strip() + filepath = fileUtil.getProjectBaseDir().joinpath("secret").joinpath("url.txt") + url = fileUtil.readFirstLine(filepath).strip() if not url.endswith("/"): raise ValueError("url should end with '/'") return url def getSecret() -> str: - filepath = util.getWorkingDir().joinpath("../secret").joinpath("secret.txt") - secret = util.readFirstLine(filepath).strip() + filepath = fileUtil.getProjectBaseDir().joinpath("secret").joinpath("secret.txt") + secret = fileUtil.readFirstLine(filepath).strip() min_length = 12 if len(secret) <= min_length: raise ValueError("secret should be longer than {} characters!".format(min_length)) diff --git a/src/langfingaz/loadData.py b/src/langfingaz/loadData.py new file mode 100644 index 0000000..98c7e57 --- /dev/null +++ b/src/langfingaz/loadData.py @@ -0,0 +1,12 @@ +from typing import Tuple +from datetime import datetime +from pathlib import Path + +from langfingaz.util import fileUtil + + +def loadData(file: Path) -> Tuple[str, datetime]: + dataStr: str = file.read_text() + t: datetime = fileUtil.getDatetimePrefix(file) + + return dataStr, t diff --git a/src/logMeetingData.py b/src/langfingaz/logMeetingData.py similarity index 68% rename from src/logMeetingData.py rename to src/langfingaz/logMeetingData.py index dc23d39..96fdb26 100755 --- a/src/logMeetingData.py +++ b/src/langfingaz/logMeetingData.py @@ -1,11 +1,10 @@ #!/usr/bin/python3 import time +from pathlib import Path -import saveMeetings -import bbbRequest -import parseMeetings -import util +from langfingaz import parseMeetings, bbbRequest, saveData +from langfingaz.util import util as util def sleepFiveMin(verbose=False): @@ -17,13 +16,12 @@ def sleepFiveMin(verbose=False): time.sleep(fiveMinutes) -def v2(): +def v2(folder: Path = saveData.getDefaultFolder()): print("BBB meetingData logger started!") while True: meetingsStr = bbbRequest.requestMeetingData() - saveDir = util.getWorkingDir().joinpath("../data") - savedFile = saveMeetings.saveMeetingsData(meetingsStr, saveDir) + savedFile = saveData.saveMeetingsData(meetingsStr, folder) print("Saved meetings at {}".format(savedFile)) meetings = parseMeetings.parseMeetingsData(meetingsStr) @@ -34,10 +32,9 @@ def v2(): sleepFiveMin(verbose=True) -def v1(): +def v1(folder: Path = saveData.getDefaultFolder()): while True: - saveDir = util.getWorkingDir().joinpath("../data") - saveMeetings.requestAndSaveMeetingData(saveDir) + saveData.requestAndSaveMeetingData(folder) print('.', end='') sleepFiveMin() diff --git a/src/parseMeetings.py b/src/langfingaz/parseMeetings.py similarity index 87% rename from src/parseMeetings.py rename to src/langfingaz/parseMeetings.py index b987b49..ba02eda 100644 --- a/src/parseMeetings.py +++ b/src/langfingaz/parseMeetings.py @@ -1,7 +1,8 @@ from typing import List from xml.etree import ElementTree +from datetime import datetime -import util +from langfingaz.util import util class Meeting(object): @@ -11,7 +12,8 @@ class Meeting(object): self.internalMeetingID: str = xml_meeting.find('internalMeetingID').text self.isRunning: bool = util.asBoolean(xml_meeting.find('running').text) - self.startTime = xml_meeting.find('startTime').text + self.startTime: int = int(xml_meeting.find('startTime').text) + self.createTime: int = int(xml_meeting.find('createTime').text) if self.isRunning: self.endTime = None else: @@ -56,11 +58,14 @@ class BbbStatus(object): and (TODO) it's CPU utilization and network usage. """ - def __init__(self, meetings: List[Meeting]): + def __init__(self, meetings: List[Meeting], pointOfTime: datetime = None): """ :param meetings: All current meetings + :param pointOfTime: The date and time at which the information about + the running meetings has been captured / requested from the BBB instance """ self.meetings = meetings + self.pointOfTime = pointOfTime self.recordingCount = 0 self.participantCount = 0 diff --git a/src/langfingaz/plotMeetings.py b/src/langfingaz/plotMeetings.py new file mode 100644 index 0000000..43afb12 --- /dev/null +++ b/src/langfingaz/plotMeetings.py @@ -0,0 +1,52 @@ +from pathlib import Path +from typing import List +import matplotlib.pyplot as plt # TODO +from datetime import datetime + +from langfingaz import loadData +from langfingaz import parseMeetings +from langfingaz.parseMeetings import BbbStatus, Meeting +from langfingaz.util import fileUtil + + +def plotMeetings(folder: Path): + bbbStati: List[BbbStatus] = [] + + for file in folder.iterdir(): + if file.name.endswith(".xml"): + dataStr, t = loadData.loadData(file) + meetings: List[Meeting] = parseMeetings.parseMeetingsData(dataStr) + bbbStati.append(parseMeetings.BbbStatus(meetings, t)) + + doPlotMeetings(bbbStati) + + +def doPlotMeetings(bbbStati: List[BbbStatus]): + time = [] # x-axis: time + participants = [] # yAxis (1) + videos = [] # yAxis (2) + voices = [] # yAxis (3) + + for bbbStatus in bbbStati: + time.append(bbbStatus.pointOfTime) + participants.append(bbbStatus.participantCount) + videos.append(bbbStatus.videoCount) + voices.append(bbbStatus.voiceParticipantCount) + + + # Note that even in the OO-style, we use `.pyplot.figure` to create the figure. + fig, ax = plt.subplots() # Create a figure and an axes. + ax.plot(time, participants, label='participants') # Plot some data on the axes. + ax.plot(time, videos, label='video') # Plot more data on the axes... + ax.plot(time, voices, label='voice') # ... and some more. + ax.set_xlabel('time') # Add an x-label to the axes. + ax.set_ylabel('numbers') # Add a y-label to the axes. + ax.set_title("BigBlueButton Statistics") # Add a title to the axes. + ax.legend() # Add a legend. + + fig.savefig(fileUtil.setDatetimePrefix(fileUtil.getProjectBaseDir().joinpath("plot"), datetime.now())) + plt.show() + + +if __name__ == '__main__': + plotMeetings(fileUtil.getProjectBaseDir().joinpath("data")) diff --git a/src/saveMeetings.py b/src/langfingaz/saveData.py similarity index 52% rename from src/saveMeetings.py rename to src/langfingaz/saveData.py index f0e987a..c743391 100644 --- a/src/saveMeetings.py +++ b/src/langfingaz/saveData.py @@ -1,10 +1,16 @@ from datetime import datetime from pathlib import Path -import bbbRequest +import langfingaz.util.fileUtil as fileUtil +# import util.util as util +from langfingaz import bbbRequest -def requestAndSaveMeetingData(folder: Path) -> Path: +def getDefaultFolder() -> Path: + return fileUtil.getProjectBaseDir().joinpath("data") + + +def requestAndSaveMeetingData(folder: Path = getDefaultFolder()) -> Path: """ save a new xml file in the given folder @@ -16,7 +22,7 @@ def requestAndSaveMeetingData(folder: Path) -> Path: return saveMeetingsData(bbbRequest.requestMeetingData(), folder) -def saveMeetingsData(dataStr: str, folder: Path) -> Path: +def saveMeetingsData(dataStr: str, folder: Path = getDefaultFolder()) -> Path: """ save a new xml file in the given folder @@ -28,7 +34,7 @@ def saveMeetingsData(dataStr: str, folder: Path) -> Path: return doSaveData(dataStr, folder, 'meetings') -def doSaveData(dataStr: str, folder: Path, dataType: str = 'meetings') -> Path: +def doSaveData(dataStr: str, folder: Path, dataType: str) -> Path: """ save a new xml file in the given folder @@ -38,11 +44,9 @@ def doSaveData(dataStr: str, folder: Path, dataType: str = 'meetings') -> Path: :return: Path to created file """ - # time_str = datetime.utcnow().strftime('%Y%m%d_%H%M%S') - time_str = datetime.now().strftime('%Y%m%d_%H%M%S') - filename = '{}_{}.xml'.format(time_str, dataType) - filepath = folder.joinpath(filename) + fileWithoutDate = folder.joinpath(dataType + '.xml') + prefixedFile = fileUtil.setDatetimePrefix(fileWithoutDate, datetime.now()) - with open(filepath, "w") as xml_file: + with open(prefixedFile, "w") as xml_file: xml_file.write(dataStr) - return filepath + return prefixedFile diff --git a/src/langfingaz/util/__init__.py b/src/langfingaz/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/langfingaz/util/fileUtil.py b/src/langfingaz/util/fileUtil.py new file mode 100644 index 0000000..e256e2a --- /dev/null +++ b/src/langfingaz/util/fileUtil.py @@ -0,0 +1,51 @@ +from datetime import datetime +from pathlib import Path +import os + + +def getWorkingDir() -> Path: + return Path(os.path.dirname(os.path.realpath(__file__))) + + +def getProjectBaseDir() -> Path: + return getWorkingDir().parent.parent.parent + + +def readFirstLine(file: Path) -> str: + """ + :param file: Path to file + :return: first line of file + """ + with open(file, "r") as f: + return f.readline() + + +def __getDatetimePrefixLength() -> int: + return len("20200101_120030") # 1st January 2020, 12:00 and 30 seconds + + +def setDatetimePrefix(file: Path, t: datetime) -> Path: + # filename = file.name + prefix = t.strftime('%Y%m%d_%H%M%S') + filename = prefix + "_" + file.name + return file.parent.joinpath(filename) + + +def removeDatetimePrefix(file: Path) -> Path: + prefixLen = __getDatetimePrefixLength() + + # prefixlen + 1 to remove the underline! + return file.parent.joinpath(file.name[prefixLen + 1:]) + + +def getDatetimePrefix(file: Path) -> datetime: + """ + :param file: some file which filename is prefixed with a date in the form %Y%m%d_%H%M%S + :return: date from filename + """ + prefixLen = __getDatetimePrefixLength() + + if len(file.name) < prefixLen: + raise ValueError("Given file does not seem to contain a datetime prefix!") + prefix = file.name[:prefixLen] + return datetime.strptime(prefix, '%Y%m%d_%H%M%S') diff --git a/src/util.py b/src/langfingaz/util/util.py similarity index 75% rename from src/util.py rename to src/langfingaz/util/util.py index 98cdece..8edd337 100644 --- a/src/util.py +++ b/src/langfingaz/util/util.py @@ -1,18 +1,3 @@ -from pathlib import Path -import os - -def getWorkingDir() -> Path: - return Path(os.path.dirname(os.path.realpath(__file__))) - -def readFirstLine(file: Path) -> str: - """ - :param file: Path to file - :return: first line of file - """ - with open(file, "r") as f: - return f.readline() - - def indentMultilineStr(s: str, indentWith='\t'): indented = indentWith + s.replace('\n', '\n' + indentWith) if s.endswith('\n'): diff --git a/src/langfingaz/util/utilTest.py b/src/langfingaz/util/utilTest.py new file mode 100644 index 0000000..7032146 --- /dev/null +++ b/src/langfingaz/util/utilTest.py @@ -0,0 +1,60 @@ +import unittest +from langfingaz.util import util as util +import langfingaz.util.fileUtil as fileUtil +from pathlib import Path +from datetime import datetime + + +class UtilTestCase(unittest.TestCase): + def test_indentString(self): + unindented = "Hello\nWorld!" + expected = "\tHello\n\tWorld!" + + actual = util.indentMultilineStr(unindented) + + self.assertEqual(expected, actual) + + def test_indentString2(self): + unindented = "Hello\nWorld!\n" + expected = "\tHello\n\tWorld!\n" + + actual = util.indentMultilineStr(unindented) + + self.assertEqual(expected, actual) + + def test_addPrefixDate(self): + file: Path = Path("/foo/bar") + t: datetime = datetime.strptime('20200101_120030', '%Y%m%d_%H%M%S') + expectedFile: Path = Path("/foo/20200101_120030_bar") + + # ACT + prefixedFile: Path = fileUtil.setDatetimePrefix(file, t) + + # ASSERT + self.assertEqual(type(expectedFile), type(prefixedFile)) + self.assertEqual(expectedFile, prefixedFile) + + def test_readPrefixDate(self): + file: Path = Path("/foo/20200101_120030_bar") + expectedDate = datetime.replace(year=2020, month=1, day=1, hour=12, minute=0, second=1) + + # ACT + dateFromFile: datetime = fileUtil.getDatetimePrefix(file) + + # ASSERT + self.assertEqual(type(expectedDate), type(dateFromFile)) + self.assertEqual(expectedDate, dateFromFile) + + def test_setAndRemoveDatetimePrefix(self): + t: datetime = datetime.strptime('20200101_120030', '%Y%m%d_%H%M%S') + + file: Path = Path("/foo/bar") + prefixed: Path = fileUtil.setDatetimePrefix(file, t) + actual: Path = fileUtil.removeDatetimePrefix(prefixed) + + self.assertEqual(type(file), type(actual)) + self.assertEqual(file, actual) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/utilTest.py b/src/utilTest.py deleted file mode 100644 index 5a1c00b..0000000 --- a/src/utilTest.py +++ /dev/null @@ -1,24 +0,0 @@ -import unittest -import util - - -class UtilTestCase(unittest.TestCase): - def test_indentString(self): - unindented = "Hello\nWorld!" - expected = "\tHello\n\tWorld!" - - actual = util.indentMultilineStr(unindented) - - self.assertEqual(expected, actual) - - def test_indentString2(self): - unindented = "Hello\nWorld!\n" - expected = "\tHello\n\tWorld!\n" - - actual = util.indentMultilineStr(unindented) - - self.assertEqual(expected, actual) - - -if __name__ == '__main__': - unittest.main()