From 8bbc72184c2396b1b9cfc0dfebe74850ef9762fd Mon Sep 17 00:00:00 2001 From: ZeldaZach Date: Sat, 20 Jul 2024 23:51:47 +0200 Subject: [PATCH] Support hot reload of Zulip Schedule - Move sheets API into common dir, since multi use - Live download from Google Sheets using Config - Falls back on old schedule if new one can't be downloaded for some reason --- .../sheetsync => common/common}/sheets.py | 2 +- sheetsync/sheetsync/main.py | 2 +- zulip_bots/Dockerfile | 4 ++ zulip_bots/zulip_bots/schedulebot.py | 70 +++++++++++++------ 4 files changed, 54 insertions(+), 24 deletions(-) rename {sheetsync/sheetsync => common/common}/sheets.py (97%) diff --git a/sheetsync/sheetsync/sheets.py b/common/common/sheets.py similarity index 97% rename from sheetsync/sheetsync/sheets.py rename to common/common/sheets.py index 3294bf6..7090409 100644 --- a/sheetsync/sheetsync/sheets.py +++ b/common/common/sheets.py @@ -1,7 +1,7 @@ import logging -from common.googleapis import GoogleAPIClient +from .googleapis import GoogleAPIClient class Sheets(object): diff --git a/sheetsync/sheetsync/main.py b/sheetsync/sheetsync/main.py index 1c9568c..5c2a217 100644 --- a/sheetsync/sheetsync/main.py +++ b/sheetsync/sheetsync/main.py @@ -18,7 +18,7 @@ import common import common.dateutil from common.database import DBManager, query, get_column_placeholder -from .sheets import Sheets +from common.sheets import Sheets sheets_synced = prom.Counter( 'sheets_synced', diff --git a/zulip_bots/Dockerfile b/zulip_bots/Dockerfile index b82f1d6..ae142fc 100644 --- a/zulip_bots/Dockerfile +++ b/zulip_bots/Dockerfile @@ -15,6 +15,10 @@ RUN pip install gevent==22.10.2 COPY chat_archiver/girc /tmp/girc RUN pip install /tmp/girc && rm -r /tmp/girc +# Install common lib first as it changes less +COPY common /tmp/common +RUN pip install /tmp/common && rm -r /tmp/common + # Actual application COPY zulip_bots /tmp/zulip_bots RUN pip install /tmp/zulip_bots && rm -r /tmp/zulip_bots diff --git a/zulip_bots/zulip_bots/schedulebot.py b/zulip_bots/zulip_bots/schedulebot.py index 638151e..01ab4ca 100644 --- a/zulip_bots/zulip_bots/schedulebot.py +++ b/zulip_bots/zulip_bots/schedulebot.py @@ -2,7 +2,6 @@ import gevent.monkey gevent.monkey.patch_all() -import csv import logging import time from calendar import timegm @@ -13,6 +12,7 @@ import argh from .zulip import Client from .config import get_config +from common.sheets import Sheets logging.basicConfig(level='INFO') @@ -200,28 +200,33 @@ def post_schedule(client, send_client, start_time, schedule, stream, hour, no_me send_client.send_to_stream(stream, "Schedule", "\n".join(lines)) -def parse_schedule(user_ids, schedule_file): +def parse_schedule(sheets_client, user_ids, schedule_sheet_id, schedule_sheet_name): schedule = {} user_ids = {user.lower(): id for user, id in user_ids.items()} - with open(schedule_file) as f: - for row in csv.reader(f): - name = row[0].lower() - if name in ["", "Chat Member", "Hour of the Run"] or name.startswith("-") or name.startswith("["): - continue - if name not in user_ids: - logging.warning(f"No user id known for user {name}") - continue - user_id = user_ids[name] - if user_id in schedule: - logging.warning(f"Multiple rows for user {name}, merging") - _, old_hours = schedule[user_id] - merged = [ - old or new - for old, new in zip(old_hours, row[1:]) - ] - schedule[user_id] = name, merged - else: - schedule[user_id] = name, row[1:] + + try: + raw_schedule = sheets_client.get_rows(schedule_sheet_id, schedule_sheet_name) + except Exception: + return None + + for row in raw_schedule: + name = row[0].lower() + if name in ["", "Chat Member", "Hour of the Run"] or name.startswith("-") or name.startswith("["): + continue + if name not in user_ids: + logging.warning(f"No user id known for user {name}") + continue + user_id = user_ids[name] + if user_id in schedule: + logging.warning(f"Multiple rows for user {name}, merging") + _, old_hours = schedule[user_id] + merged = [ + old or new + for old, new in zip(old_hours, row[1:]) + ] + schedule[user_id] = name, merged + else: + schedule[user_id] = name, row[1:] return schedule @@ -250,7 +255,17 @@ def main(conf_file, hour=-1, no_groups=False, stream="General", no_mentions=Fals send_auth = config.get("send_user", config["api_user"]) send_client = Client(config["url"], send_auth["email"], send_auth["api_key"]) group_ids = config["groups"] - schedule = parse_schedule(config["members"], config["schedule"]) + sheets_client = Sheets( + config["google_credentials"]["client_id"], + config["google_credentials"]["client_secret"], + config["google_credentials"]["refresh_token"], + ) + reload_schedule = lambda: parse_schedule( + sheets_client, + config["members"], + config["schedule_sheet_id"], + config["schedule_sheet_name"] + ) groups_by_shift = {int(id): shifts for id, shifts in config["groups_by_shift"].items()} # Accept start time timestamp with or without trailing "Z" indicating UTC. @@ -259,14 +274,25 @@ def main(conf_file, hour=-1, no_groups=False, stream="General", no_mentions=Fals start_time = start_time[:-1] start_time = timegm(time.strptime(start_time, "%Y-%m-%dT%H:%M:%S")) + # Attempt to download the schedule + schedule = reload_schedule() + if schedule is None: + raise Exception("Schedule failed to download") + if hour >= 0: if not no_groups: update_groups(client, group_ids, groups_by_shift, schedule, hour, start_time, last) if stream: post_schedule(client, send_client, start_time, schedule, stream, hour, no_mentions, last, omega) return + while True: hour = int((time.time() - start_time) / 3600) + # Download a new schedule or use the old one if there's a failure + new_schedule = reload_schedule() + if new_schedule is not None: + schedule = new_schedule + if not no_initial: if not no_groups: update_groups(client, group_ids, groups_by_shift, schedule, hour, start_time, last)