cutter: Replace youtube client with generic upload backend and specific youtube implementaton

Still to go: Actually constructing the correct backend based on given config,
switching behaviour based on needs_transcode.

DATABASE.md

@@ -130,7 +130,7 @@ columns                    | type                               | role        |
 `description`              | `TEXT NOT NULL DEFAULT ''`         | sheet input | Event description. Provides the default title and description for editors, and displayed on the public sheet.
 `submitter_winner`         | `TEXT NOT NULL DEFAULT ''`         | sheet input | A column detailing challenge submitter, auction winner, or other "associated person" data. This shouldn't be relied on in any processing but should be displayed on the public sheet.
 `poster_moment`            | `BOOLEAN NOT NULL DEFAULT FALSE`   | sheet input | Whether or not the event was featured on the poster. Used for building the postermap and also displayed on the public sheet.
-`image_links`              | `TEXT[] NOT NULL`                 | sheet input | Any additional gif or image links associated with the event. Displayed on the public sheet.
+`image_links`              | `TEXT[] NOT NULL`                  | sheet input | Any additional gif or image links associated with the event. Displayed on the public sheet.
 `notes`                    | `TEXT NOT NULL DEFAULT ''`         | sheet input | Private notes on this event, used eg. to leave messages or special instructions. Displayed to the editor during editing, but otherwise unused.
 `allow_holes`              | `BOOLEAN NOT NULL DEFAULT FALSE`   | edit input  | If false, any missing segments encountered while cutting will cause the cut to fail. Setting this to true should be done by an operator to indicate that holes are expected in this range. It is also the operator's responsibility to ensure that all allowed cutters have all segments that they can get, since there is no guarentee that only the cutter with the least missing segments will get the cut job.
 `uploader_whitelist`       | `TEXT[]`                           | edit input  | List of uploaders which are allowed to cut this entry, or NULL to indicate no restriction. This is useful if you are allowing holes and the amount of missing data differs between nodes (this shouldn't happen - this would mean replication is also failing), or if an operator is investigating a problem with a specific node.

cutter/cutter/main.py

@@ -18,7 +18,7 @@ import common
 from common.database import DBManager, query
 from common.segments import get_best_segments, cut_segments, ContainsHoles
 
-from .youtube import Youtube
+from .upload_locations import Youtube
 
 
 videos_uploaded  = prom.Counter(
@@ -435,13 +435,14 @@ class TranscodeChecker(object):
 	FOUND_VIDEOS_RETRY_INTERVAL = 20
 	ERROR_RETRY_INTERVAL = 20
 
-	def __init__(self, youtube, dbmanager, stop):
+	def __init__(self, backend, dbmanager, stop):
 		"""
-		youtube is an authenticated and initialized youtube upload backend.
+		backend is an upload backend that supports transcoding
+		and defines check_status().
 		Conn is a database connection.
 		Stop is an Event triggering graceful shutdown when set.
 		"""
-		self.youtube = youtube
+		self.backend = backend
 		self.dbmanager = dbmanager
 		self.stop = stop
 		self.logger = logging.getLogger(type(self).__name__)
@@ -483,10 +484,10 @@ class TranscodeChecker(object):
 	def check_ids(self, ids):
 		# Future work: Set error in DB if video id is not present,
 		# and/or try to get more info from yt about what's wrong.
-		statuses = self.youtube.get_video_status(ids.values())
+		done = self.backend.check_status(ids.values())
 		return {
 			id: video_id for id, video_id in ids.items()
-			if statuses.get(video_id) == 'processed'
+			if video_id in done
 		}
 
 	def mark_done(self, ids):

cutter/cutter/upload_backends.py

@@ -0,0 +1,99 @@
+
+import logging
+
+from common.googleapis import GoogleAPIClient
+
+
+class UploadBackend(object):
+	"""Represents a place a video can be uploaded,
+	and maintains any state needed to perform uploads.
+
+	Config args for the backend are passed into __init__ as kwargs.
+
+	Should have a method upload_video(title, description, tags, data).
+	Title, description and tags may have backend-specific meaning.
+	Tags is a list of string.
+	Data may be a string, file-like object or iterator of strings.
+	It should return (video_id, video_link).
+
+	If the video must undergo additional processing before it's available
+	(ie. it should go into the TRANSCODING state), then the backend should
+	define the 'needs_transcode' attribute as True.
+	If it does, it should also have a method check_status(ids) which takes a
+	list of video ids and returns a list of the ones who have finished processing.
+
+	The upload backend also determines the encoding settings for the cutting
+	process, this is given as a list of ffmpeg args
+	under the 'encoding_settings' attribute.
+	"""
+
+	needs_transcode = False
+
+	# reasonable default if settings don't otherwise matter
+	encoding_settings = [] # TODO
+
+	def upload_video(self, title, description, tags, data):
+		raise NotImplementedError
+
+	def check_status(self, ids):
+		raise NotImplementedError
+
+
+class Youtube(UploadBackend):
+	"""Represents a youtube channel to upload to, and settings for doing so.
+	Besides credentials, config args:
+		hidden: If false or not given, video is public. If true, video is unlisted.
+	"""
+
+	needs_transcode = True
+	encoding_settings = [] # TODO youtube's recommended settings
+
+	def __init__(self, client_id, client_secret, refresh_token, hidden=False):
+		self.logger = logging.getLogger(type(self).__name__)
+		self.client = GoogleAPIClient(client_id, client_secret, refresh_token)
+		self.hidden = hidden
+
+	def upload_video(self, title, description, tags, data):
+		json = {
+			'snippet': {
+				'title': title,
+				'description': description,
+				'tags': tags,
+			},
+		}
+		if self.hidden:
+			json['status'] = {
+				'privacyStatus': 'unlisted',
+			}
+		resp = self.client.request('POST',
+			'https://www.googleapis.com/upload/youtube/v3/videos',
+			params={
+				'part': 'snippet,status' if self.hidden else 'snippet',
+				'uploadType': 'resumable',
+			},
+			json=json,
+		)
+		resp.raise_for_status()
+		upload_url = resp.headers['Location']
+		resp = self.client.request('POST', upload_url, data=data)
+		resp.raise_for_status()
+		id = resp.json()['id']
+		return id, 'https://youtu.be/{}'.format(id)
+
+	def check_status(self, ids):
+		output = []
+		# Break up into groups of 10 videos. I'm not sure what the limit is so this is reasonable.
+		for i in range(0, len(ids), 10):
+			group = ids[i:i+10]
+			resp = self.client.request('GET',
+				'https://www.googleapis.com/youtube/v3/videos',
+				params={
+					'part': 'id,status',
+					'id': ','.join(group),
+				},
+			)
+			resp.raise_for_status()
+			for item in resp.json()['items']:
+				if item['status']['uploadStatus'] == 'processed':
+					output.append(item['id'])
+		return output

cutter/cutter/youtube.py

@@ -1,61 +0,0 @@
-
-import logging
-
-from common.googleapis import GoogleAPIClient
-
-
-class Youtube(object):
-	"""Manages youtube API operations"""
-
-	def __init__(self, client_id, client_secret, refresh_token):
-		self.logger = logging.getLogger(type(self).__name__)
-		self.client = GoogleAPIClient(client_id, client_secret, refresh_token)
-
-	def upload_video(self, title, description, tags, data, hidden=False):
-		"""Data may be a string, file-like object or iterator. Returns id."""
-		json = {
-			'snippet': {
-				'title': title,
-				'description': description,
-				'tags': tags,
-			},
-		}
-		if hidden:
-			json['status'] = {
-				'privacyStatus': 'unlisted',
-			}
-		resp = self.client.request('POST',
-			'https://www.googleapis.com/upload/youtube/v3/videos',
-			params={
-				'part': 'snippet,status' if hidden else 'snippet',
-				'uploadType': 'resumable',
-			},
-			json=json,
-		)
-		resp.raise_for_status()
-		upload_url = resp.headers['Location']
-		resp = self.client.request('POST', upload_url, data=data)
-		resp.raise_for_status()
-		return resp.json()['id']
-
-	def get_video_status(self, ids):
-		"""For a list of video ids, returns a dict {id: upload status}.
-		A video is fully processed when upload status is 'processed'.
-		NOTE: Video ids may be missing from the result, this probably indicates
-		the video is errored.
-		"""
-		output = {}
-		# Break up into groups of 10 videos. I'm not sure what the limit is so this is reasonable.
-		for i in range(0, len(ids), 10):
-			group = ids[i:i+10]
-			resp = self.client.request('GET',
-				'https://www.googleapis.com/youtube/v3/videos',
-				params={
-					'part': 'id,status',
-					'id': ','.join(group),
-				},
-			)
-			resp.raise_for_status()
-			for item in resp.json()['items']:
-				output[item['id']] = item['status']['uploadStatus']
-		return output