sheetsync: Remove pick_worksheets() from middleware api

Instead, get_rows() makes that decision internally if needed.

sheetsync/sheetsync/main.py

@@ -104,10 +104,8 @@ class SheetSync(object):
 				# and comparing locally.
 				events = self.get_events()
 
-				worksheets = self.middleware.pick_worksheets()
+				for row in self.middleware.get_rows():
-				for worksheet in worksheets:
+					self.sync_row(row, events.get(row['id']))
-					for row in self.middleware.get_rows(worksheet):
-						self.sync_row(row, events.get(row['id']))
 
 			except Exception as e:
 				# for HTTPErrors, http response body includes the more detailed error
@@ -123,7 +121,7 @@ class SheetSync(object):
 				self.conn = self.dbmanager.get_conn()
 				wait(self.stop, sync_start, self.ERROR_RETRY_INTERVAL)
 			else:
-				logging.info("Successful sync of worksheets: {}".format(", ".join(worksheets)))
+				logging.info("Successful sync")
 				sheets_synced.inc()
 				sheet_sync_duration.observe(monotonic() - sync_start)
 				wait(self.stop, sync_start, self.RETRY_INTERVAL)

sheetsync/sheetsync/sheets.py

@@ -153,50 +153,51 @@ class SheetsMiddleware():
 		self.sync_count += 1
 		return worksheets
 
-	def get_rows(self, worksheet):
+	def get_rows(self):
 		"""Fetch all rows of worksheet, parsed into a list of dicts."""
-		rows = self.sheets.get_rows(self.sheet_id, worksheet)
+		for worksheet in self.pick_worksheets():
-		for row_index, row in enumerate(rows):
+			rows = self.sheets.get_rows(self.sheet_id, worksheet)
-			# Skip first row (ie. the column titles).
+			for row_index, row in enumerate(rows):
-			# Need to do it inside the loop and not eg. use rows[1:],
+				# Skip first row (ie. the column titles).
-			# because then row_index won't be correct.
+				# Need to do it inside the loop and not eg. use rows[1:],
-			if row_index == 0:
+				# because then row_index won't be correct.
-				continue
+				if row_index == 0:
-			row = self.parse_row(worksheet, row_index, row)
-
-			# Handle rows without an allocated id
-			if row['id'] is None:
-				# If a row is all empty (including no id), ignore it.
-				# Ignore the tags column for this check since it is never non-empty due to implicit tags
-				# (and even if there's other tags, we don't care if there's nothing else in the row).
-				if not any(row[col] for col in self.input_columns if col != 'tags'):
 					continue
-				# If we can't allocate ids, warn and ignore.
+				row = self.parse_row(worksheet, row_index, row)
-				if not self.allocate_ids:
+
-					logging.warning(f"Row {worksheet!r}:{row['index']} has no valid id, skipping")
+				# Handle rows without an allocated id
-					continue
+				if row['id'] is None:
-				# Otherwise, allocate id for a new row.
+					# If a row is all empty (including no id), ignore it.
-				row['id'] = str(uuid.uuid4())
+					# Ignore the tags column for this check since it is never non-empty due to implicit tags
-				logging.info(f"Allocating id for row {worksheet!r}:{row['index']} = {row['id']}")
+					# (and even if there's other tags, we don't care if there's nothing else in the row).
-				self.sheets.write_value(
+					if not any(row[col] for col in self.input_columns if col != 'tags'):
-					self.sheet_id, worksheet,
+						continue
-					row["index"], self.column_map['id'],
+					# If we can't allocate ids, warn and ignore.
-					str(row['id']),
+					if not self.allocate_ids:
-				)
+						logging.warning(f"Row {worksheet!r}:{row['index']} has no valid id, skipping")
-
+						continue
-			# Set edit link if marked for editing and start/end set.
+					# Otherwise, allocate id for a new row.
-			# This prevents accidents / clicking the wrong row and provides
+					row['id'] = str(uuid.uuid4())
-			# feedback that sheet sync is still working.
+					logging.info(f"Allocating id for row {worksheet!r}:{row['index']} = {row['id']}")
-			# Also clear it if it shouldn't be set.
+					self.sheets.write_value(
-			# We do this here instead of in sync_row() because it's Sheets-specific logic
+						self.sheet_id, worksheet,
-			# that doesn't depend on the DB event in any way.
+						row["index"], self.column_map['id'],
-			edit_link = self.edit_url.format(row['id']) if row['marked_for_edit'] == '[+] Marked' else ''
+						str(row['id']),
-			if row['edit_link'] != edit_link:
+					)
-				logging.info("Updating sheet row {} with edit link {}".format(row['id'], edit_link))
+
-				self.write_value(row, "edit_link", edit_link)
+				# Set edit link if marked for editing and start/end set.
-				self.mark_modified(row)
+				# This prevents accidents / clicking the wrong row and provides
-
+				# feedback that sheet sync is still working.
-			yield row
+				# Also clear it if it shouldn't be set.
+				# We do this here instead of in sync_row() because it's Sheets-specific logic
+				# that doesn't depend on the DB event in any way.
+				edit_link = self.edit_url.format(row['id']) if row['marked_for_edit'] == '[+] Marked' else ''
+				if row['edit_link'] != edit_link:
+					logging.info("Updating sheet row {} with edit link {}".format(row['id'], edit_link))
+					self.write_value(row, "edit_link", edit_link)
+					self.mark_modified(row)
+
+				yield row
 
 	def parse_row(self, worksheet, row_index, row):
 		"""Take a row as a sequence of columns, and return a dict {column: value}"""

sheetsync/sheetsync/streamlog.py

@@ -72,10 +72,6 @@ class StreamLogMiddleware:
 			'state': lambda v: v[0].upper() + v[1:].lower(), # Titlecase
 		}
 
-	def pick_worksheets(self):
-		# We don't have a concept of seperate worksheets, so just use a generic name
-		return "streamlog"
-
 	def get_rows(self):
 		for row in self.client.get_rows():
 			yield self.parse_row(row)