sheetsync: Remove pick_worksheets() from middleware api

Instead, get_rows() makes that decision internally if needed.
pull/401/head
Mike Lang 1 year ago committed by Mike Lang
parent 17463d70fe
commit 85de9757f7

@ -104,10 +104,8 @@ class SheetSync(object):
# and comparing locally.
events = self.get_events()
worksheets = self.middleware.pick_worksheets()
for worksheet in worksheets:
for row in self.middleware.get_rows(worksheet):
self.sync_row(row, events.get(row['id']))
for row in self.middleware.get_rows():
self.sync_row(row, events.get(row['id']))
except Exception as e:
# for HTTPErrors, http response body includes the more detailed error
@ -123,7 +121,7 @@ class SheetSync(object):
self.conn = self.dbmanager.get_conn()
wait(self.stop, sync_start, self.ERROR_RETRY_INTERVAL)
else:
logging.info("Successful sync of worksheets: {}".format(", ".join(worksheets)))
logging.info("Successful sync")
sheets_synced.inc()
sheet_sync_duration.observe(monotonic() - sync_start)
wait(self.stop, sync_start, self.RETRY_INTERVAL)

@ -153,50 +153,51 @@ class SheetsMiddleware():
self.sync_count += 1
return worksheets
def get_rows(self, worksheet):
def get_rows(self):
"""Fetch all rows of worksheet, parsed into a list of dicts."""
rows = self.sheets.get_rows(self.sheet_id, worksheet)
for row_index, row in enumerate(rows):
# Skip first row (ie. the column titles).
# Need to do it inside the loop and not eg. use rows[1:],
# because then row_index won't be correct.
if row_index == 0:
continue
row = self.parse_row(worksheet, row_index, row)
# Handle rows without an allocated id
if row['id'] is None:
# If a row is all empty (including no id), ignore it.
# Ignore the tags column for this check since it is never non-empty due to implicit tags
# (and even if there's other tags, we don't care if there's nothing else in the row).
if not any(row[col] for col in self.input_columns if col != 'tags'):
for worksheet in self.pick_worksheets():
rows = self.sheets.get_rows(self.sheet_id, worksheet)
for row_index, row in enumerate(rows):
# Skip first row (ie. the column titles).
# Need to do it inside the loop and not eg. use rows[1:],
# because then row_index won't be correct.
if row_index == 0:
continue
# If we can't allocate ids, warn and ignore.
if not self.allocate_ids:
logging.warning(f"Row {worksheet!r}:{row['index']} has no valid id, skipping")
continue
# Otherwise, allocate id for a new row.
row['id'] = str(uuid.uuid4())
logging.info(f"Allocating id for row {worksheet!r}:{row['index']} = {row['id']}")
self.sheets.write_value(
self.sheet_id, worksheet,
row["index"], self.column_map['id'],
str(row['id']),
)
# Set edit link if marked for editing and start/end set.
# This prevents accidents / clicking the wrong row and provides
# feedback that sheet sync is still working.
# Also clear it if it shouldn't be set.
# We do this here instead of in sync_row() because it's Sheets-specific logic
# that doesn't depend on the DB event in any way.
edit_link = self.edit_url.format(row['id']) if row['marked_for_edit'] == '[+] Marked' else ''
if row['edit_link'] != edit_link:
logging.info("Updating sheet row {} with edit link {}".format(row['id'], edit_link))
self.write_value(row, "edit_link", edit_link)
self.mark_modified(row)
yield row
row = self.parse_row(worksheet, row_index, row)
# Handle rows without an allocated id
if row['id'] is None:
# If a row is all empty (including no id), ignore it.
# Ignore the tags column for this check since it is never non-empty due to implicit tags
# (and even if there's other tags, we don't care if there's nothing else in the row).
if not any(row[col] for col in self.input_columns if col != 'tags'):
continue
# If we can't allocate ids, warn and ignore.
if not self.allocate_ids:
logging.warning(f"Row {worksheet!r}:{row['index']} has no valid id, skipping")
continue
# Otherwise, allocate id for a new row.
row['id'] = str(uuid.uuid4())
logging.info(f"Allocating id for row {worksheet!r}:{row['index']} = {row['id']}")
self.sheets.write_value(
self.sheet_id, worksheet,
row["index"], self.column_map['id'],
str(row['id']),
)
# Set edit link if marked for editing and start/end set.
# This prevents accidents / clicking the wrong row and provides
# feedback that sheet sync is still working.
# Also clear it if it shouldn't be set.
# We do this here instead of in sync_row() because it's Sheets-specific logic
# that doesn't depend on the DB event in any way.
edit_link = self.edit_url.format(row['id']) if row['marked_for_edit'] == '[+] Marked' else ''
if row['edit_link'] != edit_link:
logging.info("Updating sheet row {} with edit link {}".format(row['id'], edit_link))
self.write_value(row, "edit_link", edit_link)
self.mark_modified(row)
yield row
def parse_row(self, worksheet, row_index, row):
"""Take a row as a sequence of columns, and return a dict {column: value}"""

@ -72,10 +72,6 @@ class StreamLogMiddleware:
'state': lambda v: v[0].upper() + v[1:].lower(), # Titlecase
}
def pick_worksheets(self):
# We don't have a concept of seperate worksheets, so just use a generic name
return "streamlog"
def get_rows(self):
for row in self.client.get_rows():
yield self.parse_row(row)

Loading…
Cancel
Save