retry database connection if it fails

cutter/cutter/main.py

@@ -494,7 +494,16 @@ def main(dbconnect, youtube_creds_file, name=None, base_dir=".", metrics_port=80
 	# We have two independent jobs to do - to perform cut jobs (cutter),
 	# and to check the status of transcoding videos to see if they're done (transcode checker).
 	# We want to error if either errors, and shut down if either exits.
-	dbmanager = DBManager(dsn=dbconnect)
+	dbmanager = None
+	stopping = gevent.event.Event()
+	while dbmanager is None:
+		try:
+			dbmanager = DBManager(dsn=dbconnect)
+		except Exception:
+			delay = common.jitter(10)
+			logging.info('Cannot connect to database. Retrying in {:.0f} s'.format(delay))
+			stopping.wait(delay)
+
 	youtube_creds = json.load(open(youtube_creds_file))
 	youtube = Youtube(
 		client_id=youtube_creds['client_id'],

sheetsync/sheetsync/main.py

@@ -280,7 +280,16 @@ def main(dbconnect, sheets_creds_file, edit_url, bustime_start, sheet_id, worksh
 
 	logging.info("Starting up")
 
-	dbmanager = DBManager(dsn=dbconnect)
+	dbmanager = None
+	stopping = gevent.event.Event()
+	while dbmanager is None:
+		try:
+			dbmanager = DBManager(dsn=dbconnect)
+		except Exception:
+			delay = common.jitter(10)
+			logging.info('Cannot connect to database. Retrying in {:.0f} s'.format(delay))
+			stopping.wait(delay)
+
 	sheets_creds = json.load(open(sheets_creds_file))
 
 	sheets = Sheets(

thrimshim/thrimshim/main.py

@@ -12,7 +12,8 @@ import prometheus_client
 import psycopg2
 from psycopg2 import sql
 
-from common import database, PromLogCountsHandler, install_stacksampler
+import common
+from common import database
 from common.flask_stats import request_stats, after_request
 
 psycopg2.extras.register_uuid()
@@ -219,15 +220,25 @@ def reset_row(ident):
 def main(connection_string, host='0.0.0.0', port=8004, backdoor_port=0):
 	"""Thrimshim service."""
 	server = WSGIServer((host, port), cors(app))
-	app.db_manager = database.DBManager(dsn=connection_string)
 
+	app.no_authentication = no_authentication
+	app.db_manager = None
+	stopping = gevent.event.Event()
+	while app.db_manager is None:
+		try:
+			app.db_manager = database.DBManager(dsn=connection_string)
+		except Exception:
+			delay = common.jitter(10)
+			logging.info('Cannot connect to database. Retrying in {:.0f} s'.format(delay))
+			stopping.wait(delay)
+	
 	def stop():
 		logging.info("Shutting down")
 		server.stop()
 	gevent.signal(signal.SIGTERM, stop)
 
-	PromLogCountsHandler.install()
+	common.PromLogCountsHandler.install()
-	install_stacksampler()
+	common.install_stacksampler()
 
 	if backdoor_port:
 		gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start()