Merge branch 'mike/sheetsync/fix-db-error'

backfiller/backfiller/main.py

@@ -154,6 +154,8 @@ class BackfillerManager(object):
 		self.run_once = run_once
 		self.node_file = node_file
 		self.node_database = node_database
+		if self.node_database is not None:
+			self.db_manager =  database.DBManager(dsn=self.node_database)
 		self.localhost = localhost
 		self.stopping = gevent.event.Event()
 		self.logger = logging.getLogger("BackfillerManager({})".format(stream))
@@ -189,14 +191,17 @@ class BackfillerManager(object):
 		stop will exit the loop."""
 		self.logger.info('Starting')
 		if self.node_database is not None:
-			self.connection = database.DBManager(dsn=self.node_database).get_conn()
+			self.connection = self.db_manager.get_conn()
-
 		failures = 0
 
 		while not self.stopping.is_set():
 			try:
 				new_nodes = set(self.get_nodes())
 			except Exception:
+				# To ensure a fresh slate and clear any DB-related errors, get a new conn on error.
+				# This is heavy-handed but simple and effective.
+				if self.node_database is not None:
+					self.connection = self.db_manager.get_conn()
 				if failures < MAX_BACKOFF:
 					failures += 1
 				delay = common.jitter(TIMEOUT * 2**failures)

cutter/cutter/main.py

@@ -389,14 +389,14 @@ class TranscodeChecker(object):
 	NO_VIDEOS_RETRY_INTERVAL = 5
 	ERROR_RETRY_INTERVAL = 5
 
-	def __init__(self, youtube, conn, stop):
+	def __init__(self, youtube, dbmanager, stop):
 		"""
 		youtube is an authenticated and initialized youtube api client.
 		Conn is a database connection.
 		Stop is an Event triggering graceful shutdown when set.
 		"""
 		self.youtube = youtube
-		self.conn = conn
+		self.dbmanager = dbmanager
 		self.stop = stop
 		self.logger = logging.getLogger(type(self).__name__)
 
@@ -405,6 +405,7 @@ class TranscodeChecker(object):
 		self.stop.wait(common.jitter(interval))
 
 	def run(self):
+		self.conn = self.dbmanager.get_conn()
 		while not self.stop.is_set():
 			try:
 				ids = self.get_ids_to_check()
@@ -421,6 +422,9 @@ class TranscodeChecker(object):
 				self.logger.info("Marked {} videos as done".format(done))
 			except Exception:
 				self.logger.exception("Error in TranscodeChecker")
+				# To ensure a fresh slate and clear any DB-related errors, get a new conn on error.
+				# This is heavy-handed but simple and effective.
+				self.conn = self.dbmanager.get_conn()
 				self.wait(self.ERROR_RETRY_INTERVAL)
 
 	def get_ids_to_check(self):

sheetsync/sheetsync/main.py

@@ -27,7 +27,7 @@ class SheetSync(object):
 
 	def __init__(self, stop, dbmanager, sheets, sheet_id, worksheets, edit_url, bustime_start, allocate_ids=False):
 		self.stop = stop
-		self.conn = dbmanager.get_conn()
+		self.dbmanager = dbmanager
 		self.sheets = sheets
 		self.sheet_id = sheet_id
 		self.worksheets = worksheets
@@ -90,6 +90,8 @@ class SheetSync(object):
 		self.stop.wait(common.jitter(interval))
 
 	def run(self):
+		self.conn = self.dbmanager.get_conn()
+
 		while not self.stop.is_set():
 			
 			try:
@@ -109,6 +111,9 @@ class SheetSync(object):
 						self.sync_row(worksheet, row_index, row, events.get(row['id']))
 			except Exception:
 				logging.exception("Failed to sync")
+				# To ensure a fresh slate and clear any DB-related errors, get a new conn on error.
+				# This is heavy-handed but simple and effective.
+				self.conn = self.dbmanager.get_conn()
 				self.wait(self.ERROR_RETRY_INTERVAL)
 			else:
 				logging.info("Successful sync")