Merge pull request #155 from ekimekim/mike/manual-uploads

manual upload

DATABASE.md

@@ -40,6 +40,9 @@ finalizing the upload to make it official. If a cutter dies and leaves an event
 it is indeterminate whether the upload actually occurred - in this unlikely scenario, an operator
 should manually inspect things and decide on further action.
 
+* `MANUAL_UPLOAD`: A hack because youtube APIs aren't usable. An event for which a video has been
+created but not uploaded. It must be uploaded manually, at which point it will be moved to transcoding.
+
 * `TRANSCODING`: An event which has been succesfully uploaded, but is not yet ready for public consumption.
 The upload is no longer cancellable. If further re-edits need to be applied,
 an operator should manually delete or unlist the video then set the state back to `UNEDITED`.
@@ -76,16 +79,23 @@ we are certain the upload didn't actually go through, and the cut can be immedia
 * `FINALIZING -> UNEDITED`: When the finalization failed with an unknown error,
 we are certain the upload didn't actually go through, and operator intervention is required.
 
+* `FINALIZING -> UPLOAD_PENDING`: When you cannot use the youtube API, and need to designate
+the video for manual uploading, or to be uploaded later.
+
 * `FINALIZING -> TRANSCODING`: When the cutter has successfully finalized the upload,
 but the upload location requires further processing before the video is done.
 
 * `FINALIZING -> DONE`: When the cutter has successfully finalized the upload,
 and the upload location requires no further processing.
 
+* `UPLOAD_PENDING -> TRANSCODING`: When an operator has uploaded the video.
+
 * `TRANSCODING -> DONE`: When any cutter detects that the upload location is finished
 transcoding the video, and it is ready for public consumption.
 
-This is summarised in the below graph:
+* Missing some states here around rollbacks of upload pending
+
+This is summarised in the below graph: (missing `UPLOAD_PENDING`)
 
 ```
                                             retry

cutter/cutter/main.py

@@ -19,7 +19,7 @@ from common.database import DBManager, query
 from common.segments import get_best_segments, fast_cut_segments, full_cut_segments, ContainsHoles
 from common.stats import timed
 
-from .upload_backends import Youtube, Local, UploadError
+from .upload_backends import Youtube, Local, Manual, UploadError
 
 
 videos_uploaded = prom.Counter(
@@ -443,12 +443,17 @@ class Cutter(object):
 				gevent.sleep(self.RETRYABLE_UPLOAD_ERROR_WAIT_INTERVAL)
 			return
 
-		# Success! Set TRANSCODING or DONE and clear any previous error.
+		# Success! Set UPLOAD_PENDING, TRANSCODING or DONE and clear any previous error.
-		success_state = 'TRANSCODING' if upload_backend.needs_transcode else 'DONE'
+		if upload_backend.needs_manual_upload:
+			success_state = 'UPLOAD_PENDING'
+		elif upload_backend.needs_transcode:
+			success_state = 'TRANSCODING'
+		else:
+			success_state = 'DONE'
 		maybe_upload_time = {"upload_time": datetime.datetime.utcnow()} if success_state == 'DONE' else {}
 		set_row(state=success_state, video_id=video_id, video_link=video_link, error=None, **maybe_upload_time)
 
-		self.logger.info("Successfully cut and uploaded job {} as {}".format(format_job(job), video_link))
+		self.logger.info("Successfully cut and put into {} job {} as {}".format(success_state, format_job(job), video_link))
 		videos_uploaded.labels(video_channel=job.video_channel,
 				video_quality=job.video_quality,
 				upload_location=job.upload_location).inc()
@@ -640,6 +645,8 @@ def main(
 			backend_type = Youtube
 		elif backend_type == 'local':
 			backend_type = Local
+		elif backend_type == 'manual':
+			backend_type = Manual
 		else:
 			raise ValueError("Unknown upload backend type: {!r}".format(backend_type))
 		backend = backend_type(credentials, **backend_config)

cutter/cutter/upload_backends.py

@@ -62,6 +62,10 @@ class UploadBackend(object):
 	If it does, it should also have a method check_status(ids) which takes a
 	list of video ids and returns a list of the ones who have finished processing.
 
+	If the upload backend cannot actually upload the file but needs to leave it
+	in a state for further manual processing, it should set needs_manual_upload.
+	This will put the video into the UPLOAD_PENDING state.
+
 	The upload backend also determines the encoding settings for the cutting
 	process, this is given as a list of ffmpeg args
 	under the 'encoding_settings' attribute.
@@ -73,6 +77,7 @@ class UploadBackend(object):
 	"""
 
 	needs_transcode = False
+	needs_manual_upload = False
 
 	# reasonable default if settings don't otherwise matter:
 	# high-quality mpegts, without wasting too much cpu on encoding
@@ -250,3 +255,7 @@ class Local(UploadBackend):
 		else:
 			url = 'file://{}'.format(filepath)
 		return video_id, url
+
+
+class Manual(Local):
+	needs_manual_upload = True

postgres/setup.sh

@@ -42,6 +42,7 @@ CREATE TYPE event_state as ENUM (
 	'EDITED',
 	'CLAIMED',
 	'FINALIZING',
+	'UPLOAD_PENDING',
 	'TRANSCODING',
 	'DONE'
 );

thrimshim/thrimshim/main.py

@@ -2,6 +2,7 @@ import datetime
 from functools import wraps
 import json
 import logging
+import re
 import signal
 import sys
 
@@ -272,9 +273,19 @@ def update_row(ident, editor=None):
 @request_stats
 @authenticate
 def manual_link(ident, editor=None):
-	"""Manually set a video_link if the state is 'UNEDITED' or 'DONE' and the 
+	"""Manually set a video_link if the state is 'UNEDITED', 'UPLOAD_PENDING' or 'DONE'"""
-	upload_location is 'manual'."""
 	link = flask.request.json['link']
+	new_state = flask.request.json.get('state', 'DONE')
+	if new_state == 'DONE':
+		video_id = link
+	else:
+		# Attempt to parse from youtube URL https://www.youtube.com/watch?v=XXXXXXXXXXX or https://youtu.be/XXXXXXXXXXX
+		match = re.search(r'[0-9A-Za-z_-]{11}$', link)
+		if not match:
+			return 'Could not parse youtube id from url {!r}'.format(link), 400
+		video_id = match.group(0)
+		# Normalize URL
+		link = 'https://youtu.be/{}'.format(video_id)
 	conn = app.db_manager.get_conn()
 	results = database.query(conn, """
 		SELECT id, state, upload_location 
@@ -283,16 +294,16 @@ def manual_link(ident, editor=None):
 	old_row = results.fetchone()
 	if old_row is None:
 		return 'Row {} not found'.format(ident), 404
-	if old_row.state != 'UNEDITED' and not (old_row.state == 'DONE' and old_row.upload_location == 'manual'):
+	if old_row.state not in ('UNEDITED', 'UPLOAD_PENDING') and not (old_row.state == 'DONE' and old_row.upload_location == 'manual'):
 		return 'Invalid state {} for manual video link'.format(old_row.state), 403		
 	now = datetime.datetime.utcnow()
 	results = database.query(conn, """
 		UPDATE events 
-		SET state='DONE', upload_location = 'manual', video_link = %s,
+		SET state=%s, upload_location = 'manual', video_link = %s,
-			editor = %s, edit_time = %s, upload_time = %s
+			upload_time = %s, video_id = %s
-		WHERE id = %s AND (state = 'UNEDITED' OR (state = 'DONE' AND
+		WHERE id = %s AND (state IN ('UNEDITED', 'UPLOAD_PENDING') OR (state = 'DONE' AND
-			upload_location = 'manual'))""", link, editor, now, now, ident)
+			upload_location = 'manual'))""", new_state, link, now, video_id, ident)
-	logging.info("Row {} video_link set to {}".format(ident, link))
+	logging.info("Row {} {} and video_link set to {}".format(ident, new_state, link))
 	return ''