Update sheetsync CLI interface to allow any combination of types and backends

docker-compose.jsonnet

@@ -441,27 +441,42 @@
     },
 
     [if $.enabled.sheetsync then "sheetsync"]: {
-      local sync_sheet = {
+      local sync_sheet_base = {
-          type: "sheets",
+        name: if $.sheet_reverse_sync then "reverse-" else "",
+        backend: "sheets",
         creds: "/etc/sheet-creds.json",
         sheet_id: $.sheet_id,
-          worksheets: $.worksheets,
         allocate_ids: true,
+        reverse_sync: $.sheet_reverse_sync,
+      },
+      local sync_sheet = [
+        sync_sheet_base + {
+          name+: "sheet-events",
+          type: "events",
+          worksheets: $.worksheets,
           edit_url: $.edit_url,
           bustime_start: $.bustime_start,
-          playlist_worksheet: $.playlist_worksheet,
-          reverse_sync: $.sheet_reverse_sync,
         },
-      local sync_streamlog = {
+        sync_sheet_base + {
-          type: "streamlog",
+          name+: "sheet-playlists",
+          type: "playlists",
+          worksheets: [$.playlist_worksheet],
+        },
+      ],
+      local sync_streamlog_base = {
+        backend: "streamlog",
         creds: "/etc/streamlog-token.txt",
         url: $.streamlog_url,
         event_id: $.streamlog_event,
       },
-      local config = std.prune([
+      local sync_streamlog = [
-          if $.sheet_id != null then sync_sheet,
+        sync_streamlog_base + {name: "streamlog-events", type: "events"},
-          if $.streamlog_url != null then sync_streamlog,
+        sync_streamlog_base + {name: "streamlog-playlists", type: "playlists"},
-      ]),
+      ],
+      local config = (
+          (if $.sheet_id != null then sync_sheet else [])
+          + (if $.streamlog_url != null then sync_streamlog else [])
+      ),
       image: $.get_image("sheetsync"),
       // Args for the sheetsync
       command: [

sheetsync/sheetsync/main.py

@@ -18,7 +18,7 @@ from common.database import DBManager, query, get_column_placeholder
 from common.sheets import Sheets as SheetsClient
 
 from .sheets import SheetsEventsMiddleware, SheetsPlaylistsMiddleware
-from .streamlog import StreamLogClient, StreamLogMiddleware
+from .streamlog import StreamLogClient, StreamLogEventsMiddleware, StreamLogPlaylistsMiddleware
 
 sheets_synced = prom.Counter(
 	'sheets_synced',
@@ -325,25 +325,30 @@ class PlaylistsSync(SheetSync):
 	metavar="SYNC-CONFIG",
 	nargs="+",
 	type=json.loads,
-	help="\n".join([
+	help="""
-		'A JSON object describing a sync operation to perform. One of:',
+		A JSON object describing a sync operation to perform.
-		'  type: "sheets"',
+		Always present:
-		'  creds: path to credentials JSON file containing "client_id", "client_secret" and "refresh_token"',
+			name: A human identifier for this sync operation
-		'  sheet_id: The id of the Google Sheet to use',
+			backend: The data source. One of "sheets" or "streamlog"
-		'  worksheets: List of worksheets within that sheet to sync',
+			type: What kind of data is being synced. One of "events" or "playlists"
-		'  edit_url: a format string for edit links, with {} as a placeholder for id',
+		When backend is "sheets":
-		'  bustime_start: Timestamp string at which bustime is 00:00',
+			creds: path to credentials JSON file containing "client_id", "client_secret" and "refresh_token"
-		'  allocate_ids: Boolean, optional. When true, will give rows without ids an id.',
+			sheet_id: The id of the Google Sheet to use
-		'    Only one sheetsync acting on the same sheet should have this enabled.',
+			worksheets: List of worksheets within that sheet to sync
-		'  reverse_sync: Boolean, optional. When true, enables an alternate mode',
+			allocate_ids: Boolean, optional. When true, will give rows without ids an id.
-		'    where all data is synced from the database to the sheet',
+				Only one sheetsync acting on the same sheet should have this enabled.
-		'  playlist_worksheet: An optional additional worksheet name that holds playlist tag definitions',
+			reverse_sync: Boolean, optional. When true, enables an alternate mode
-		'or:',
+				where all data is synced from the database to the sheet.
-		'  type: streamlog',
+				Only one sheetsync acting on the same sheet should have this enabled.
-		'  creds: path to a file containing the auth key',
+			When type is "events":
-		'  url: The URL of the streamlog server',
+				edit_url: a format string for edit links, with {} as a placeholder for id
-		'  event_id: The id of the streamlog event to sync',
+				bustime_start: Timestamp string at which bustime is 00:00
-	]),
+		When backend is "streamlog":
+			type: streamlog
+			creds: path to a file containing the auth key
+			url: The URL of the streamlog server
+			event_id: The id of the streamlog event to sync
+	""",
 )
 def main(dbconnect, sync_configs, metrics_port=8005, backdoor_port=0):
 	"""
@@ -383,44 +388,55 @@ def main(dbconnect, sync_configs, metrics_port=8005, backdoor_port=0):
 	workers = []
 
 	for config in sync_configs:
-		if config["type"] == "sheets":
+		if config["backend"] == "sheets":
 			creds = json.load(open(config["creds"]))
 			client = SheetsClient(
 				client_id=creds['client_id'],
 				client_secret=creds['client_secret'],
 				refresh_token=creds['refresh_token'],
 			)
+			allocate_ids = config.get("allocate_ids", False)
+			if config["type"] == "sheets":
 				middleware = SheetsEventsMiddleware(
 					client,
 					config["sheet_id"],
 					config["worksheets"],
 					common.dateutil.parse(config["bustime_start"]),
 					config["edit_url"],
-				config.get("allocate_ids", False),
+					allocate_ids,
 				)
-			if "playlist_worksheet" in config:
+			elif config["type"] == "playlists":
 				middleware = SheetsPlaylistsMiddleware(
 					client,
 					config["sheet_id"],
 					[config["playlist_worksheet"]],
 					config.get("allocate_ids", False),
 				)
-				workers.append(
+			else:
-					PlaylistsSync("playlists", middleware, stop, dbmanager, config.get("reverse_sync", False))
+				raise ValueError("Unknown type {!r}".format(config["type"]))
-				)
+		elif config["backend"] == "streamlog":
-		elif config["type"] == "streamlog":
 			auth_token = open(config["creds"]).read().strip()
 			client = StreamLogClient(
 				config["url"],
 				config["event_id"],
 				auth_token,
 			)
-			middleware = StreamLogMiddleware(client)
+			if config["type"] == "events":
+				middleware = StreamLogEventsMiddleware(client)
+			elif config["type"] == "playlists":
+				middleware = StreamLogPlaylistsMiddleware(client)
 			else:
 				raise ValueError("Unknown type {!r}".format(config["type"]))
-		workers.append(
+		else:
-			EventsSync(config["type"], middleware, stop, dbmanager, config.get("reverse_sync", False)),
+			raise ValueError("Unknown backend {!r}".format(config["backend"]))
-		)
+
+		sync_class = {
+			"events": EventsSync,
+			"playlists": PlaylistsSync,
+		}[config["type"]]
+		reverse_sync = config.get("reverse_sync", False)
+		sync = sync_class(config["name"], middleware, stop, dbmanager, reverse_sync)
+		workers.append(sync)
 
 	jobs = [gevent.spawn(worker.run) for worker in workers]
 	# Block until any one exits