cutter: Replace youtube client with generic upload backend and specific youtube implementaton

Still to go: Actually constructing the correct backend based on given config,
switching behaviour based on needs_transcode.

cutter/cutter/main.py

@@ -18,7 +18,7 @@ import common
 from common.database import DBManager, query
 from common.segments import get_best_segments, cut_segments, ContainsHoles
 
-from .youtube import Youtube
+from .upload_locations import Youtube
 
 
 videos_uploaded  = prom.Counter(
@@ -435,13 +435,14 @@ class TranscodeChecker(object):
 	FOUND_VIDEOS_RETRY_INTERVAL = 20
 	ERROR_RETRY_INTERVAL = 20
 
-	def __init__(self, youtube, dbmanager, stop):
+	def __init__(self, backend, dbmanager, stop):
 		"""
-		youtube is an authenticated and initialized youtube upload backend.
+		backend is an upload backend that supports transcoding
+		and defines check_status().
 		Conn is a database connection.
 		Stop is an Event triggering graceful shutdown when set.
 		"""
-		self.youtube = youtube
+		self.backend = backend
 		self.dbmanager = dbmanager
 		self.stop = stop
 		self.logger = logging.getLogger(type(self).__name__)
@@ -483,10 +484,10 @@ class TranscodeChecker(object):
 	def check_ids(self, ids):
 		# Future work: Set error in DB if video id is not present,
 		# and/or try to get more info from yt about what's wrong.
-		statuses = self.youtube.get_video_status(ids.values())
+		done = self.backend.check_status(ids.values())
 		return {
 			id: video_id for id, video_id in ids.items()
-			if statuses.get(video_id) == 'processed'
+			if video_id in done
 		}
 
 	def mark_done(self, ids):

cutter/cutter/upload_backends.py

@@ -0,0 +1,99 @@
+
+import logging
+
+from common.googleapis import GoogleAPIClient
+
+
+class UploadBackend(object):
+	"""Represents a place a video can be uploaded,
+	and maintains any state needed to perform uploads.
+
+	Config args for the backend are passed into __init__ as kwargs.
+
+	Should have a method upload_video(title, description, tags, data).
+	Title, description and tags may have backend-specific meaning.
+	Tags is a list of string.
+	Data may be a string, file-like object or iterator of strings.
+	It should return (video_id, video_link).
+
+	If the video must undergo additional processing before it's available
+	(ie. it should go into the TRANSCODING state), then the backend should
+	define the 'needs_transcode' attribute as True.
+	If it does, it should also have a method check_status(ids) which takes a
+	list of video ids and returns a list of the ones who have finished processing.
+
+	The upload backend also determines the encoding settings for the cutting
+	process, this is given as a list of ffmpeg args
+	under the 'encoding_settings' attribute.
+	"""
+
+	needs_transcode = False
+
+	# reasonable default if settings don't otherwise matter
+	encoding_settings = [] # TODO
+
+	def upload_video(self, title, description, tags, data):
+		raise NotImplementedError
+
+	def check_status(self, ids):
+		raise NotImplementedError
+
+
+class Youtube(UploadBackend):
+	"""Represents a youtube channel to upload to, and settings for doing so.
+	Besides credentials, config args:
+		hidden: If false or not given, video is public. If true, video is unlisted.
+	"""
+
+	needs_transcode = True
+	encoding_settings = [] # TODO youtube's recommended settings
+
+	def __init__(self, client_id, client_secret, refresh_token, hidden=False):
+		self.logger = logging.getLogger(type(self).__name__)
+		self.client = GoogleAPIClient(client_id, client_secret, refresh_token)
+		self.hidden = hidden
+
+	def upload_video(self, title, description, tags, data):
+		json = {
+			'snippet': {
+				'title': title,
+				'description': description,
+				'tags': tags,
+			},
+		}
+		if self.hidden:
+			json['status'] = {
+				'privacyStatus': 'unlisted',
+			}
+		resp = self.client.request('POST',
+			'https://www.googleapis.com/upload/youtube/v3/videos',
+			params={
+				'part': 'snippet,status' if self.hidden else 'snippet',
+				'uploadType': 'resumable',
+			},
+			json=json,
+		)
+		resp.raise_for_status()
+		upload_url = resp.headers['Location']
+		resp = self.client.request('POST', upload_url, data=data)
+		resp.raise_for_status()
+		id = resp.json()['id']
+		return id, 'https://youtu.be/{}'.format(id)
+
+	def check_status(self, ids):
+		output = []
+		# Break up into groups of 10 videos. I'm not sure what the limit is so this is reasonable.
+		for i in range(0, len(ids), 10):
+			group = ids[i:i+10]
+			resp = self.client.request('GET',
+				'https://www.googleapis.com/youtube/v3/videos',
+				params={
+					'part': 'id,status',
+					'id': ','.join(group),
+				},
+			)
+			resp.raise_for_status()
+			for item in resp.json()['items']:
+				if item['status']['uploadStatus'] == 'processed':
+					output.append(item['id'])
+		return output

cutter/cutter/youtube.py

@@ -1,61 +0,0 @@
-
-import logging
-
-from common.googleapis import GoogleAPIClient
-
-
-class Youtube(object):
-	"""Manages youtube API operations"""
-
-	def __init__(self, client_id, client_secret, refresh_token):
-		self.logger = logging.getLogger(type(self).__name__)
-		self.client = GoogleAPIClient(client_id, client_secret, refresh_token)
-
-	def upload_video(self, title, description, tags, data, hidden=False):
-		"""Data may be a string, file-like object or iterator. Returns id."""
-		json = {
-			'snippet': {
-				'title': title,
-				'description': description,
-				'tags': tags,
-			},
-		}
-		if hidden:
-			json['status'] = {
-				'privacyStatus': 'unlisted',
-			}
-		resp = self.client.request('POST',
-			'https://www.googleapis.com/upload/youtube/v3/videos',
-			params={
-				'part': 'snippet,status' if hidden else 'snippet',
-				'uploadType': 'resumable',
-			},
-			json=json,
-		)
-		resp.raise_for_status()
-		upload_url = resp.headers['Location']
-		resp = self.client.request('POST', upload_url, data=data)
-		resp.raise_for_status()
-		return resp.json()['id']
-
-	def get_video_status(self, ids):
-		"""For a list of video ids, returns a dict {id: upload status}.
-		A video is fully processed when upload status is 'processed'.
-		NOTE: Video ids may be missing from the result, this probably indicates
-		the video is errored.
-		"""
-		output = {}
-		# Break up into groups of 10 videos. I'm not sure what the limit is so this is reasonable.
-		for i in range(0, len(ids), 10):
-			group = ids[i:i+10]
-			resp = self.client.request('GET',
-				'https://www.googleapis.com/youtube/v3/videos',
-				params={
-					'part': 'id,status',
-					'id': ','.join(group),
-				},
-			)
-			resp.raise_for_status()
-			for item in resp.json()['items']:
-				output[item['id']] = item['status']['uploadStatus']
-		return output