From 17af1c4e891ed7ba2ac4200a589967f3fed5f3d0 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Sun, 27 Oct 2019 01:19:54 -0700 Subject: [PATCH] cutter, sheetsync: Wait for DB to connect on startup This is a nicer error than crashing in the depths of some error handler (which is what will happen if the DB goes unavailable while they're running), and it's a far more common case (eg. the DB is misconfigured) than having it fail halfway through. Neither of these services can do anything meaningful without the DB, so crashing without it is acceptable behaviour. --- cutter/cutter/main.py | 11 +++++++++-- sheetsync/sheetsync/main.py | 14 +++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index 549539e..b399724 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -565,13 +565,20 @@ def main( # We want to error if either errors, and shut down if either exits. dbmanager = None stopping = gevent.event.Event() - while dbmanager is None: + dbmanager = DBManager(dsn=dbconnect) + while True: try: - dbmanager = DBManager(dsn=dbconnect) + # Get a test connection so we know the database is up, + # this produces a clearer error in cases where there's a connection problem. + conn = dbmanager.get_conn() except Exception: delay = common.jitter(10) logging.info('Cannot connect to database. Retrying in {:.0f} s'.format(delay)) stop.wait(delay) + else: + # put it back so it gets reused on next get_conn() + dbmanager.put_conn(conn) + break with open(creds_file) as f: credentials = json.load(f) diff --git a/sheetsync/sheetsync/main.py b/sheetsync/sheetsync/main.py index 5ffa076..27b612d 100644 --- a/sheetsync/sheetsync/main.py +++ b/sheetsync/sheetsync/main.py @@ -130,6 +130,8 @@ class SheetSync(object): sync_errors.inc() # To ensure a fresh slate and clear any DB-related errors, get a new conn on error. # This is heavy-handed but simple and effective. + # If we can't re-connect, the program will crash from here, + # then restart and wait until it can connect again. self.conn = self.dbmanager.get_conn() self.wait(self.ERROR_RETRY_INTERVAL) else: @@ -293,14 +295,20 @@ def main(dbconnect, sheets_creds_file, edit_url, bustime_start, sheet_id, worksh logging.info("Starting up") - dbmanager = None - while dbmanager is None: + dbmanager = DBManager(dsn=dbconnect) + while True: try: - dbmanager = DBManager(dsn=dbconnect) + # Get a test connection so we know the database is up, + # this produces a clearer error in cases where there's a connection problem. + conn = dbmanager.get_conn() except Exception: delay = common.jitter(10) logging.info('Cannot connect to database. Retrying in {:.0f} s'.format(delay)) stop.wait(delay) + else: + # put it back so it gets reused on next get_conn() + dbmanager.put_conn(conn) + break sheets_creds = json.load(open(sheets_creds_file))