retry database connection if it fails

pull/73/head
Christopher Usher 5 years ago
parent 57962c8104
commit 1dbe585837

@ -494,7 +494,16 @@ def main(dbconnect, youtube_creds_file, name=None, base_dir=".", metrics_port=80
# We have two independent jobs to do - to perform cut jobs (cutter),
# and to check the status of transcoding videos to see if they're done (transcode checker).
# We want to error if either errors, and shut down if either exits.
dbmanager = DBManager(dsn=dbconnect)
dbmanager = None
stopping = gevent.event.Event()
while dbmanager is None:
try:
dbmanager = DBManager(dsn=dbconnect)
except Exception:
delay = common.jitter(10)
logging.info('Cannot connect to database. Retrying in {:.0f} s'.format(delay))
stopping.wait(delay)
youtube_creds = json.load(open(youtube_creds_file))
youtube = Youtube(
client_id=youtube_creds['client_id'],

@ -280,7 +280,16 @@ def main(dbconnect, sheets_creds_file, edit_url, bustime_start, sheet_id, worksh
logging.info("Starting up")
dbmanager = DBManager(dsn=dbconnect)
dbmanager = None
stopping = gevent.event.Event()
while dbmanager is None:
try:
dbmanager = DBManager(dsn=dbconnect)
except Exception:
delay = common.jitter(10)
logging.info('Cannot connect to database. Retrying in {:.0f} s'.format(delay))
stopping.wait(delay)
sheets_creds = json.load(open(sheets_creds_file))
sheets = Sheets(

@ -12,7 +12,8 @@ import prometheus_client
import psycopg2
from psycopg2 import sql
from common import database, PromLogCountsHandler, install_stacksampler
import common
from common import database
from common.flask_stats import request_stats, after_request
psycopg2.extras.register_uuid()
@ -219,15 +220,25 @@ def reset_row(ident):
def main(connection_string, host='0.0.0.0', port=8004, backdoor_port=0):
"""Thrimshim service."""
server = WSGIServer((host, port), cors(app))
app.db_manager = database.DBManager(dsn=connection_string)
app.no_authentication = no_authentication
app.db_manager = None
stopping = gevent.event.Event()
while app.db_manager is None:
try:
app.db_manager = database.DBManager(dsn=connection_string)
except Exception:
delay = common.jitter(10)
logging.info('Cannot connect to database. Retrying in {:.0f} s'.format(delay))
stopping.wait(delay)
def stop():
logging.info("Shutting down")
server.stop()
gevent.signal(signal.SIGTERM, stop)
PromLogCountsHandler.install()
install_stacksampler()
common.PromLogCountsHandler.install()
common.install_stacksampler()
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start()

Loading…
Cancel
Save