cutter: Add ability to configure multiple upload locations

Cutter now takes a 'config' arg which is a json blob with detail
on each upload location. This is a bit nasty if you're trying to run it manually
but was the easiest way to transfer the config data from docker-compose.jsonnet
to the actual application.

cutter/cutter/main.py

@@ -500,13 +500,17 @@ class TranscodeChecker(object):
 		return result.rowcount
 
 
-def main(dbconnect, youtube_creds_file, name=None, base_dir=".", metrics_port=8003, backdoor_port=0, upload_location=""):
+def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=8003, backdoor_port=0):
 	"""dbconnect should be a postgres connection string, which is either a space-separated
 	list of key=value pairs, or a URI like:
 		postgresql://USER:PASSWORD@HOST/DBNAME?KEY=VALUE
 
-	youtube_creds_file should be a json file containing keys 'client_id', 'client_secret' and 'refresh_token'.
+	config should be a json blob mapping upload location names to a config object
-	upload_location is the name of the upload location this youtube creds file gives access to.
+	for that location. This config object should contain the keys:
+		type: the name of the upload backend type
+	along with any additional config options defined for that backend type.
+
+	creds_file should contain any required credentials for the upload backends, as JSON.
 
 	name defaults to hostname.
 	"""
@@ -538,25 +542,36 @@ def main(dbconnect, youtube_creds_file, name=None, base_dir=".", metrics_port=80
 			logging.info('Cannot connect to database. Retrying in {:.0f} s'.format(delay))
 			stop.wait(delay)
 
-	youtube_creds = json.load(open(youtube_creds_file))
+	with open(creds_file) as f:
-	youtube = Youtube(
+		credentials = json.load(f)
-		client_id=youtube_creds['client_id'],
+
-		client_secret=youtube_creds['client_secret'],
+	config = json.loads(config)
-		refresh_token=youtube_creds['refresh_token'],
+	upload_locations = {}
-	)
+	for location, backend_config in config.items():
-	cutter = Cutter({upload_location: youtube}, dbmanager, stop, name, base_dir)
+		backend_type = backend_config.pop('type')
-	transcode_checker = TranscodeChecker(youtube, dbmanager, stop)
+		if type == 'youtube':
-	jobs = [
+			backend_type = Youtube
-		gevent.spawn(cutter.run),
+		else:
-		gevent.spawn(transcode_checker.run),
+			raise ValueError("Unknown upload backend type: {!r}".format(type))
+		upload_locations[location] = backend_type(credentials, **backend_config)
+
+	cutter = Cutter(upload_locations, dbmanager, stop, name, base_dir)
+	transcode_checkers = [
+		TranscodeChecker(backend, dbmanager, stop)
+		for backend in upload_locations.values()
+		if backend.needs_transcode
+	]
+	jobs = [gevent.spawn(cutter.run)] + [
+		gevent.spawn(transcode_checker.run)
+		for transcode_checker in transcode_checkers
 	]
-	# Block until either exits
+	# Block until any one exits
 	gevent.wait(jobs, count=1)
-	# Stop the other if it isn't stopping already
+	# Stop the others if they aren't stopping already
 	stop.set()
-	# Block until both have exited
+	# Block until all have exited
 	gevent.wait(jobs)
-	# Call get() for each to re-raise if either errored
+	# Call get() for each one to re-raise if any errored
 	for job in jobs:
 		job.get()
 

cutter/cutter/upload_backends.py

@@ -8,7 +8,8 @@ class UploadBackend(object):
 	"""Represents a place a video can be uploaded,
 	and maintains any state needed to perform uploads.
 
-	Config args for the backend are passed into __init__ as kwargs.
+	Config args for the backend are passed into __init__ as kwargs,
+	along with credentials as the first arg.
 
 	Should have a method upload_video(title, description, tags, data).
 	Title, description and tags may have backend-specific meaning.
@@ -41,16 +42,21 @@ class UploadBackend(object):
 
 class Youtube(UploadBackend):
 	"""Represents a youtube channel to upload to, and settings for doing so.
-	Besides credentials, config args:
+	Config args besides credentials:
-		hidden: If false or not given, video is public. If true, video is unlisted.
+		hidden:
+			If false, video is public. If true, video is unlisted. Default false.
 	"""
 
 	needs_transcode = True
 	encoding_settings = [] # TODO youtube's recommended settings
 
-	def __init__(self, client_id, client_secret, refresh_token, hidden=False):
+	def __init__(self, credentials, hidden=False):
 		self.logger = logging.getLogger(type(self).__name__)
-		self.client = GoogleAPIClient(client_id, client_secret, refresh_token)
+		self.client = GoogleAPIClient(
+			credentials['client_id'],
+			credentials['client_secret'],
+			credentials['refresh_token'],
+		)
 		self.hidden = hidden
 
 	def upload_video(self, title, description, tags, data):

docker-compose.jsonnet

@@ -89,6 +89,12 @@
   // 'client_id', 'client_secret' and 'refresh_token'.
   cutter_creds_file:: "./google_creds.json",
 
+  // Config for cutter upload locations. See cutter docs for full detail.
+  cutter_config:: {
+    desertbus: {type: "youtube"},
+    unlisted: {type: "youtube", hidden: true},
+  },
+
   // Path to a JSON file containing google credentials for sheetsync as keys
   // 'client_id', 'client_secret' and 'refresh_token'.
   // May be the same as cutter_creds_file.
@@ -178,6 +184,7 @@
         "--base-dir", "/mnt",
         "--backdoor-port", std.toString($.backdoor_port),
         $.db_connect,
+        std.manifestJson($.cutter_config),
         "/etc/wubloader-creds.json",
       ],
       volumes: [