From 3e8b4dfcf150c854780c1abe2e6ca8a9da4a03fc Mon Sep 17 00:00:00 2001 From: Christopher Usher Date: Mon, 13 Nov 2023 13:15:05 -0800 Subject: [PATCH] Starting integration of donation graphs --- graphs/Dockerfile | 9 +++ graphs/graphs/__init__.py | 0 graphs/graphs/__main__.py | 15 +++++ graphs/graphs/main.py | 127 ++++++++++++++++++++++++++++++++++++++ graphs/setup.py | 13 ++++ 5 files changed, 164 insertions(+) create mode 100644 graphs/Dockerfile create mode 100644 graphs/graphs/__init__.py create mode 100644 graphs/graphs/__main__.py create mode 100644 graphs/graphs/main.py create mode 100644 graphs/setup.py diff --git a/graphs/Dockerfile b/graphs/Dockerfile new file mode 100644 index 0000000..16b424a --- /dev/null +++ b/graphs/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.11 + +RUN pip install bokeh + +COPY donation_graphs /tmp/donation_graphs +RUN pip install /tmp/donation_graphs && rm -r /tmp/donation_graphs + +LABEL org.opencontainers.image.source https://github.com/dbvideostriketeam/wubloader +ENTRYPOINT ["python3", "-m", "donation_graphs"] diff --git a/graphs/graphs/__init__.py b/graphs/graphs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/graphs/graphs/__main__.py b/graphs/graphs/__main__.py new file mode 100644 index 0000000..27210eb --- /dev/null +++ b/graphs/graphs/__main__.py @@ -0,0 +1,15 @@ +import gevent.monkey +gevent.monkey.patch_all() + +import logging +import os + +import argh + +from graphs.main import main +# from main import main + +LOG_FORMAT = "[%(asctime)s] %(levelname)8s %(name)s(%(module)s:%(lineno)d): %(message)s" + +logging.basicConfig(level='INFO', format=LOG_FORMAT) +argh.dispatch_command(main) diff --git a/graphs/graphs/main.py b/graphs/graphs/main.py new file mode 100644 index 0000000..481efab --- /dev/null +++ b/graphs/graphs/main.py @@ -0,0 +1,127 @@ +import gevent.monkey +gevent.monkey.patch_all() + +import datetime +import logging +import json + +import requests + +import bokeh.plotting +import bokeh.models +import bokeh.palettes + +import numpy as np + +def parse_json(json_donations, start_date, end_hour=np.inf, every_five=True): + + end_hour = float(end_hour) + times = [] + donations = [] + for entry in json_donations: + times.append(datetime.datetime(*entry[:5]).isoformat()) + donations.append(entry[5]) + + times = np.array(times, dtype=np.datetime64) + donations = np.asarray(donations) + + start_time = np.datetime64(start_date) + bustimes = np.array(times - start_time, dtype=np.int_) + + trimmed_bustimes = bustimes[(bustimes <= 60 * 60 * end_hour) & (bustimes >= 0)] + trimmed_donations = donations[(bustimes <= 60 * 60 * end_hour) & (bustimes >= 0)] + + if every_five: + five_bustimes = trimmed_bustimes[::5] + five_donations = trimmed_donations[::5] + return five_bustimes, five_donations + else: + return trimmed_bustimes, trimmed_donations + +def load_previous_donations(start_end_times, timeout): + + all_years = {} + for year in start_end_times: + start, end = start_end_times[year] + if not end: + current_year = year + continue + + logging.info('Loading year {}'.format(year)) + url = 'https://vst.ninja/DB{}/graphs/jsons/DB{}.json'.format(year, year) + year_json = requests.get(url, timeout=timeout).json() + all_years[year] = parse_json(year_json, start, end, year >= 5) + + return all_years, current_year + +def main(): + + stopping = gevent.event.Event() + + delay = 60 * 1 + timeout = 15 + start_end_file = 'start_end_times.json' + logging.info('Loading start and end times') + start_end_times = json.load(open(start_end_file)) + start_end_times = {int(year):start_end_times[year] for year in start_end_times} + all_years, current_year = load_previous_donations(start_end_times, timeout) + + current_url = 'http://example.com/{}/{}'.format(current_year, current_year) + + + while not stopping.is_set(): + + try: + + logging.info('Loading current data') + current_json = requests.get(current_url, timeout=timeout).json() + + p = bokeh.plotting.figure(x_axis_label='Bus Time', y_axis_label='Donations', x_range=(0, 60 * 60 * 172), + width=1280, height=720, active_scroll='wheel_zoom', + tools='pan,wheel_zoom,box_zoom,reset') + + p.add_tools(bokeh.models.HoverTool(tooltips=[('', '$name'), ('Bustime', '@Bustime{00:00:00}'), + ('Donations', '$@Donations{0,0.00}')])) + for year in start_end_times: + label_year = year + if year > 10: + label_year += 2006 + label = 'DBfH {}'.format(label_year) + if year != current_year: + times, donations = all_years[year] + line_width = 2 + else: + times, donations = parse_json(current_json, start_end_times[year][0], every_five=False) + line_width = 3 + model = bokeh.models.ColumnDataSource(data={'Bustime':times, 'Donations':donations}) + p.line(x='Bustime', y='Donations', source=model, line_width=line_width, + line_color=bokeh.palettes.Category20[20][current_year - year], + legend_label=label, name=label) + + + p.xaxis.ticker = bokeh.models.AdaptiveTicker(mantissas=[60, 120, 300, 600, 1200, 3600, 7200, 10800, 43200, 86400], base=10000000) + p.xaxis.formatter = bokeh.models.NumeralTickFormatter(format='00:00:00') + p.yaxis.formatter = bokeh.models.NumeralTickFormatter(format='$0,0') + + p.legend.location = "top_left" + p.legend.click_policy="hide" + + logging.info('Saving plot') + bokeh.plotting.save(p, filename='dbfh_donations.html', title='DBfH Donations') + except Exception: + logging.exception('Plotting failed. Retrying') + + stopping.wait(delay) + +# logging.info('Starting Graph Generator') +# generator = GraphGenerator(current_url, start_time, previous_years:) +# manager = gevent.spawn(generator.run) + +# def stop(): +# manager.stop() + +# gevent.signal_handler(signal.SIGTERM, stop) + +# stop() +# logging.info('Gracefully stopped') + diff --git a/graphs/setup.py b/graphs/setup.py new file mode 100644 index 0000000..6281a58 --- /dev/null +++ b/graphs/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup, find_packages + +setup( + name = 'graphs', + version = '0.0.0', + packages = find_packages(), + install_requires = [ + 'argh', + 'bokeh', + 'gevent', + 'request' + ], +)