From ecf58dfbfc302fdedbc0c471e7255d16d8f9a2cb Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Wed, 16 Oct 2019 14:11:07 +1100 Subject: [PATCH] cutter: Add ability to configure multiple upload locations Cutter now takes a 'config' arg which is a json blob with detail on each upload location. This is a bit nasty if you're trying to run it manually but was the easiest way to transfer the config data from docker-compose.jsonnet to the actual application. --- cutter/cutter/main.py | 51 +++++++++++++++++++++----------- cutter/cutter/upload_backends.py | 16 ++++++---- docker-compose.jsonnet | 7 +++++ 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index 75c3e7e..02eadbe 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -500,13 +500,17 @@ class TranscodeChecker(object): return result.rowcount -def main(dbconnect, youtube_creds_file, name=None, base_dir=".", metrics_port=8003, backdoor_port=0, upload_location=""): +def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=8003, backdoor_port=0): """dbconnect should be a postgres connection string, which is either a space-separated list of key=value pairs, or a URI like: postgresql://USER:PASSWORD@HOST/DBNAME?KEY=VALUE - youtube_creds_file should be a json file containing keys 'client_id', 'client_secret' and 'refresh_token'. - upload_location is the name of the upload location this youtube creds file gives access to. + config should be a json blob mapping upload location names to a config object + for that location. This config object should contain the keys: + type: the name of the upload backend type + along with any additional config options defined for that backend type. + + creds_file should contain any required credentials for the upload backends, as JSON. name defaults to hostname. """ @@ -538,25 +542,36 @@ def main(dbconnect, youtube_creds_file, name=None, base_dir=".", metrics_port=80 logging.info('Cannot connect to database. Retrying in {:.0f} s'.format(delay)) stop.wait(delay) - youtube_creds = json.load(open(youtube_creds_file)) - youtube = Youtube( - client_id=youtube_creds['client_id'], - client_secret=youtube_creds['client_secret'], - refresh_token=youtube_creds['refresh_token'], - ) - cutter = Cutter({upload_location: youtube}, dbmanager, stop, name, base_dir) - transcode_checker = TranscodeChecker(youtube, dbmanager, stop) - jobs = [ - gevent.spawn(cutter.run), - gevent.spawn(transcode_checker.run), + with open(creds_file) as f: + credentials = json.load(f) + + config = json.loads(config) + upload_locations = {} + for location, backend_config in config.items(): + backend_type = backend_config.pop('type') + if type == 'youtube': + backend_type = Youtube + else: + raise ValueError("Unknown upload backend type: {!r}".format(type)) + upload_locations[location] = backend_type(credentials, **backend_config) + + cutter = Cutter(upload_locations, dbmanager, stop, name, base_dir) + transcode_checkers = [ + TranscodeChecker(backend, dbmanager, stop) + for backend in upload_locations.values() + if backend.needs_transcode + ] + jobs = [gevent.spawn(cutter.run)] + [ + gevent.spawn(transcode_checker.run) + for transcode_checker in transcode_checkers ] - # Block until either exits + # Block until any one exits gevent.wait(jobs, count=1) - # Stop the other if it isn't stopping already + # Stop the others if they aren't stopping already stop.set() - # Block until both have exited + # Block until all have exited gevent.wait(jobs) - # Call get() for each to re-raise if either errored + # Call get() for each one to re-raise if any errored for job in jobs: job.get() diff --git a/cutter/cutter/upload_backends.py b/cutter/cutter/upload_backends.py index 15dd336..838b6a4 100644 --- a/cutter/cutter/upload_backends.py +++ b/cutter/cutter/upload_backends.py @@ -8,7 +8,8 @@ class UploadBackend(object): """Represents a place a video can be uploaded, and maintains any state needed to perform uploads. - Config args for the backend are passed into __init__ as kwargs. + Config args for the backend are passed into __init__ as kwargs, + along with credentials as the first arg. Should have a method upload_video(title, description, tags, data). Title, description and tags may have backend-specific meaning. @@ -41,16 +42,21 @@ class UploadBackend(object): class Youtube(UploadBackend): """Represents a youtube channel to upload to, and settings for doing so. - Besides credentials, config args: - hidden: If false or not given, video is public. If true, video is unlisted. + Config args besides credentials: + hidden: + If false, video is public. If true, video is unlisted. Default false. """ needs_transcode = True encoding_settings = [] # TODO youtube's recommended settings - def __init__(self, client_id, client_secret, refresh_token, hidden=False): + def __init__(self, credentials, hidden=False): self.logger = logging.getLogger(type(self).__name__) - self.client = GoogleAPIClient(client_id, client_secret, refresh_token) + self.client = GoogleAPIClient( + credentials['client_id'], + credentials['client_secret'], + credentials['refresh_token'], + ) self.hidden = hidden def upload_video(self, title, description, tags, data): diff --git a/docker-compose.jsonnet b/docker-compose.jsonnet index d181570..10afc4f 100644 --- a/docker-compose.jsonnet +++ b/docker-compose.jsonnet @@ -89,6 +89,12 @@ // 'client_id', 'client_secret' and 'refresh_token'. cutter_creds_file:: "./google_creds.json", + // Config for cutter upload locations. See cutter docs for full detail. + cutter_config:: { + desertbus: {type: "youtube"}, + unlisted: {type: "youtube", hidden: true}, + }, + // Path to a JSON file containing google credentials for sheetsync as keys // 'client_id', 'client_secret' and 'refresh_token'. // May be the same as cutter_creds_file. @@ -178,6 +184,7 @@ "--base-dir", "/mnt", "--backdoor-port", std.toString($.backdoor_port), $.db_connect, + std.manifestJson($.cutter_config), "/etc/wubloader-creds.json", ], volumes: [