From 4bbcc8bc0660d211188439da53954ad87dec4efd Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Fri, 8 Nov 2019 08:45:29 -0800 Subject: [PATCH] Revert "Merge pull request #155 from ekimekim/mike/manual-uploads" This reverts commit 99de5863536266e395f2e870a9fd98a7997c4f8a, reversing changes made to 4b04f70b6f56af9f451543a08856a44aa6be9ab8. We don't need this feature and it complicates things and adds bugs. --- DATABASE.md | 12 +----------- cutter/cutter/main.py | 15 ++++----------- cutter/cutter/upload_backends.py | 9 --------- postgres/setup.sh | 1 - thrimshim/thrimshim/main.py | 27 ++++++++------------------- 5 files changed, 13 insertions(+), 51 deletions(-) diff --git a/DATABASE.md b/DATABASE.md index 312db7c..01946c9 100644 --- a/DATABASE.md +++ b/DATABASE.md @@ -40,9 +40,6 @@ finalizing the upload to make it official. If a cutter dies and leaves an event it is indeterminate whether the upload actually occurred - in this unlikely scenario, an operator should manually inspect things and decide on further action. -* `MANUAL_UPLOAD`: A hack because youtube APIs aren't usable. An event for which a video has been -created but not uploaded. It must be uploaded manually, at which point it will be moved to transcoding. - * `TRANSCODING`: An event which has been succesfully uploaded, but is not yet ready for public consumption. The upload is no longer cancellable. If further re-edits need to be applied, an operator should manually delete or unlist the video then set the state back to `UNEDITED`. @@ -79,23 +76,16 @@ we are certain the upload didn't actually go through, and the cut can be immedia * `FINALIZING -> UNEDITED`: When the finalization failed with an unknown error, we are certain the upload didn't actually go through, and operator intervention is required. -* `FINALIZING -> UPLOAD_PENDING`: When you cannot use the youtube API, and need to designate -the video for manual uploading, or to be uploaded later. - * `FINALIZING -> TRANSCODING`: When the cutter has successfully finalized the upload, but the upload location requires further processing before the video is done. * `FINALIZING -> DONE`: When the cutter has successfully finalized the upload, and the upload location requires no further processing. -* `UPLOAD_PENDING -> TRANSCODING`: When an operator has uploaded the video. - * `TRANSCODING -> DONE`: When any cutter detects that the upload location is finished transcoding the video, and it is ready for public consumption. -* Missing some states here around rollbacks of upload pending - -This is summarised in the below graph: (missing `UPLOAD_PENDING`) +This is summarised in the below graph: ``` retry diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index 0ef102f..cc048c5 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -19,7 +19,7 @@ from common.database import DBManager, query from common.segments import get_best_segments, fast_cut_segments, full_cut_segments, ContainsHoles from common.stats import timed -from .upload_backends import Youtube, Local, Manual, UploadError +from .upload_backends import Youtube, Local, UploadError videos_uploaded = prom.Counter( @@ -443,17 +443,12 @@ class Cutter(object): gevent.sleep(self.RETRYABLE_UPLOAD_ERROR_WAIT_INTERVAL) return - # Success! Set UPLOAD_PENDING, TRANSCODING or DONE and clear any previous error. - if upload_backend.needs_manual_upload: - success_state = 'UPLOAD_PENDING' - elif upload_backend.needs_transcode: - success_state = 'TRANSCODING' - else: - success_state = 'DONE' + # Success! Set TRANSCODING or DONE and clear any previous error. + success_state = 'TRANSCODING' if upload_backend.needs_transcode else 'DONE' maybe_upload_time = {"upload_time": datetime.datetime.utcnow()} if success_state == 'DONE' else {} set_row(state=success_state, video_id=video_id, video_link=video_link, error=None, **maybe_upload_time) - self.logger.info("Successfully cut and put into {} job {} as {}".format(success_state, format_job(job), video_link)) + self.logger.info("Successfully cut and uploaded job {} as {}".format(format_job(job), video_link)) videos_uploaded.labels(video_channel=job.video_channel, video_quality=job.video_quality, upload_location=job.upload_location).inc() @@ -645,8 +640,6 @@ def main( backend_type = Youtube elif backend_type == 'local': backend_type = Local - elif backend_type == 'manual': - backend_type = Manual else: raise ValueError("Unknown upload backend type: {!r}".format(backend_type)) backend = backend_type(credentials, **backend_config) diff --git a/cutter/cutter/upload_backends.py b/cutter/cutter/upload_backends.py index 4d83f3f..a2fba77 100644 --- a/cutter/cutter/upload_backends.py +++ b/cutter/cutter/upload_backends.py @@ -62,10 +62,6 @@ class UploadBackend(object): If it does, it should also have a method check_status(ids) which takes a list of video ids and returns a list of the ones who have finished processing. - If the upload backend cannot actually upload the file but needs to leave it - in a state for further manual processing, it should set needs_manual_upload. - This will put the video into the UPLOAD_PENDING state. - The upload backend also determines the encoding settings for the cutting process, this is given as a list of ffmpeg args under the 'encoding_settings' attribute. @@ -77,7 +73,6 @@ class UploadBackend(object): """ needs_transcode = False - needs_manual_upload = False # reasonable default if settings don't otherwise matter: # high-quality mpegts, without wasting too much cpu on encoding @@ -255,7 +250,3 @@ class Local(UploadBackend): else: url = 'file://{}'.format(filepath) return video_id, url - - -class Manual(Local): - needs_manual_upload = True diff --git a/postgres/setup.sh b/postgres/setup.sh index 97fdd72..4f63c7e 100644 --- a/postgres/setup.sh +++ b/postgres/setup.sh @@ -42,7 +42,6 @@ CREATE TYPE event_state as ENUM ( 'EDITED', 'CLAIMED', 'FINALIZING', - 'UPLOAD_PENDING', 'TRANSCODING', 'DONE' ); diff --git a/thrimshim/thrimshim/main.py b/thrimshim/thrimshim/main.py index fe5b1d1..9eb188a 100644 --- a/thrimshim/thrimshim/main.py +++ b/thrimshim/thrimshim/main.py @@ -2,7 +2,6 @@ import datetime from functools import wraps import json import logging -import re import signal import sys @@ -273,19 +272,9 @@ def update_row(ident, editor=None): @request_stats @authenticate def manual_link(ident, editor=None): - """Manually set a video_link if the state is 'UNEDITED', 'UPLOAD_PENDING' or 'DONE'""" + """Manually set a video_link if the state is 'UNEDITED' or 'DONE' and the + upload_location is 'manual'.""" link = flask.request.json['link'] - new_state = flask.request.json.get('state', 'DONE') - if new_state == 'DONE': - video_id = link - else: - # Attempt to parse from youtube URL https://www.youtube.com/watch?v=XXXXXXXXXXX or https://youtu.be/XXXXXXXXXXX - match = re.search(r'[0-9A-Za-z_-]{11}$', link) - if not match: - return 'Could not parse youtube id from url {!r}'.format(link), 400 - video_id = match.group(0) - # Normalize URL - link = 'https://youtu.be/{}'.format(video_id) conn = app.db_manager.get_conn() results = database.query(conn, """ SELECT id, state, upload_location @@ -294,16 +283,16 @@ def manual_link(ident, editor=None): old_row = results.fetchone() if old_row is None: return 'Row {} not found'.format(ident), 404 - if old_row.state not in ('UNEDITED', 'UPLOAD_PENDING') and not (old_row.state == 'DONE' and old_row.upload_location == 'manual'): + if old_row.state != 'UNEDITED' and not (old_row.state == 'DONE' and old_row.upload_location == 'manual'): return 'Invalid state {} for manual video link'.format(old_row.state), 403 now = datetime.datetime.utcnow() results = database.query(conn, """ UPDATE events - SET state=%s, upload_location = 'manual', video_link = %s, - upload_time = %s, video_id = %s - WHERE id = %s AND (state IN ('UNEDITED', 'UPLOAD_PENDING') OR (state = 'DONE' AND - upload_location = 'manual'))""", new_state, link, now, video_id, ident) - logging.info("Row {} {} and video_link set to {}".format(ident, new_state, link)) + SET state='DONE', upload_location = 'manual', video_link = %s, + editor = %s, edit_time = %s, upload_time = %s + WHERE id = %s AND (state = 'UNEDITED' OR (state = 'DONE' AND + upload_location = 'manual'))""", link, editor, now, now, ident) + logging.info("Row {} video_link set to {}".format(ident, link)) return ''