Update sheetsync CLI interface to allow any combination of types and backends

pull/401/head
Mike Lang 3 months ago committed by Mike Lang
parent bb4cc8c668
commit 73eaac16f3

@ -441,27 +441,42 @@
},
[if $.enabled.sheetsync then "sheetsync"]: {
local sync_sheet = {
type: "sheets",
creds: "/etc/sheet-creds.json",
sheet_id: $.sheet_id,
local sync_sheet_base = {
name: if $.sheet_reverse_sync then "reverse-" else "",
backend: "sheets",
creds: "/etc/sheet-creds.json",
sheet_id: $.sheet_id,
allocate_ids: true,
reverse_sync: $.sheet_reverse_sync,
},
local sync_sheet = [
sync_sheet_base + {
name+: "sheet-events",
type: "events",
worksheets: $.worksheets,
allocate_ids: true,
edit_url: $.edit_url,
bustime_start: $.bustime_start,
playlist_worksheet: $.playlist_worksheet,
reverse_sync: $.sheet_reverse_sync,
},
local sync_streamlog = {
type: "streamlog",
creds: "/etc/streamlog-token.txt",
url: $.streamlog_url,
event_id: $.streamlog_event,
},
sync_sheet_base + {
name+: "sheet-playlists",
type: "playlists",
worksheets: [$.playlist_worksheet],
},
],
local sync_streamlog_base = {
backend: "streamlog",
creds: "/etc/streamlog-token.txt",
url: $.streamlog_url,
event_id: $.streamlog_event,
},
local config = std.prune([
if $.sheet_id != null then sync_sheet,
if $.streamlog_url != null then sync_streamlog,
]),
local sync_streamlog = [
sync_streamlog_base + {name: "streamlog-events", type: "events"},
sync_streamlog_base + {name: "streamlog-playlists", type: "playlists"},
],
local config = (
(if $.sheet_id != null then sync_sheet else [])
+ (if $.streamlog_url != null then sync_streamlog else [])
),
image: $.get_image("sheetsync"),
// Args for the sheetsync
command: [

@ -18,7 +18,7 @@ from common.database import DBManager, query, get_column_placeholder
from common.sheets import Sheets as SheetsClient
from .sheets import SheetsEventsMiddleware, SheetsPlaylistsMiddleware
from .streamlog import StreamLogClient, StreamLogMiddleware
from .streamlog import StreamLogClient, StreamLogEventsMiddleware, StreamLogPlaylistsMiddleware
sheets_synced = prom.Counter(
'sheets_synced',
@ -325,25 +325,30 @@ class PlaylistsSync(SheetSync):
metavar="SYNC-CONFIG",
nargs="+",
type=json.loads,
help="\n".join([
'A JSON object describing a sync operation to perform. One of:',
' type: "sheets"',
' creds: path to credentials JSON file containing "client_id", "client_secret" and "refresh_token"',
' sheet_id: The id of the Google Sheet to use',
' worksheets: List of worksheets within that sheet to sync',
' edit_url: a format string for edit links, with {} as a placeholder for id',
' bustime_start: Timestamp string at which bustime is 00:00',
' allocate_ids: Boolean, optional. When true, will give rows without ids an id.',
' Only one sheetsync acting on the same sheet should have this enabled.',
' reverse_sync: Boolean, optional. When true, enables an alternate mode',
' where all data is synced from the database to the sheet',
' playlist_worksheet: An optional additional worksheet name that holds playlist tag definitions',
'or:',
' type: streamlog',
' creds: path to a file containing the auth key',
' url: The URL of the streamlog server',
' event_id: The id of the streamlog event to sync',
]),
help="""
A JSON object describing a sync operation to perform.
Always present:
name: A human identifier for this sync operation
backend: The data source. One of "sheets" or "streamlog"
type: What kind of data is being synced. One of "events" or "playlists"
When backend is "sheets":
creds: path to credentials JSON file containing "client_id", "client_secret" and "refresh_token"
sheet_id: The id of the Google Sheet to use
worksheets: List of worksheets within that sheet to sync
allocate_ids: Boolean, optional. When true, will give rows without ids an id.
Only one sheetsync acting on the same sheet should have this enabled.
reverse_sync: Boolean, optional. When true, enables an alternate mode
where all data is synced from the database to the sheet.
Only one sheetsync acting on the same sheet should have this enabled.
When type is "events":
edit_url: a format string for edit links, with {} as a placeholder for id
bustime_start: Timestamp string at which bustime is 00:00
When backend is "streamlog":
type: streamlog
creds: path to a file containing the auth key
url: The URL of the streamlog server
event_id: The id of the streamlog event to sync
""",
)
def main(dbconnect, sync_configs, metrics_port=8005, backdoor_port=0):
"""
@ -383,44 +388,55 @@ def main(dbconnect, sync_configs, metrics_port=8005, backdoor_port=0):
workers = []
for config in sync_configs:
if config["type"] == "sheets":
if config["backend"] == "sheets":
creds = json.load(open(config["creds"]))
client = SheetsClient(
client_id=creds['client_id'],
client_secret=creds['client_secret'],
refresh_token=creds['refresh_token'],
)
middleware = SheetsEventsMiddleware(
client,
config["sheet_id"],
config["worksheets"],
common.dateutil.parse(config["bustime_start"]),
config["edit_url"],
config.get("allocate_ids", False),
)
if "playlist_worksheet" in config:
allocate_ids = config.get("allocate_ids", False)
if config["type"] == "sheets":
middleware = SheetsEventsMiddleware(
client,
config["sheet_id"],
config["worksheets"],
common.dateutil.parse(config["bustime_start"]),
config["edit_url"],
allocate_ids,
)
elif config["type"] == "playlists":
middleware = SheetsPlaylistsMiddleware(
client,
config["sheet_id"],
[config["playlist_worksheet"]],
config.get("allocate_ids", False),
)
workers.append(
PlaylistsSync("playlists", middleware, stop, dbmanager, config.get("reverse_sync", False))
)
elif config["type"] == "streamlog":
else:
raise ValueError("Unknown type {!r}".format(config["type"]))
elif config["backend"] == "streamlog":
auth_token = open(config["creds"]).read().strip()
client = StreamLogClient(
config["url"],
config["event_id"],
auth_token,
)
middleware = StreamLogMiddleware(client)
if config["type"] == "events":
middleware = StreamLogEventsMiddleware(client)
elif config["type"] == "playlists":
middleware = StreamLogPlaylistsMiddleware(client)
else:
raise ValueError("Unknown type {!r}".format(config["type"]))
else:
raise ValueError("Unknown type {!r}".format(config["type"]))
workers.append(
EventsSync(config["type"], middleware, stop, dbmanager, config.get("reverse_sync", False)),
)
raise ValueError("Unknown backend {!r}".format(config["backend"]))
sync_class = {
"events": EventsSync,
"playlists": PlaylistsSync,
}[config["type"]]
reverse_sync = config.get("reverse_sync", False)
sync = sync_class(config["name"], middleware, stop, dbmanager, reverse_sync)
workers.append(sync)
jobs = [gevent.spawn(worker.run) for worker in workers]
# Block until any one exits

Loading…
Cancel
Save