henine
/
wubloader
mirror of https://github.com/ekimekim/wubloader
1
0
Fork 0
Code Issues Releases Wiki Activity

Revert "Merge pull request #155 from ekimekim/mike/manual-uploads"

Browse Source 
This reverts commit 99de586353, reversing
changes made to 4b04f70b6f.

We don't need this feature and it complicates things and adds bugs.
mike/testing
Mike Lang 5 years ago
parent 845744cbf6
commit 4bbcc8bc06
5 changed files with 13 additions and 51 deletions
Show Stats Download Patch File Download Diff File

12
DATABASE.md
Unescape Escape View File

@ -40,9 +40,6 @@ finalizing the upload to make it official. If a cutter dies and leaves an event
it is indeterminate whether the upload actually occurred - in this unlikely scenario, an operator
should manually inspect things and decide on further action.
* `MANUAL_UPLOAD`: A hack because youtube APIs aren't usable. An event for which a video has been
created but not uploaded. It must be uploaded manually, at which point it will be moved to transcoding.
* `TRANSCODING`: An event which has been succesfully uploaded, but is not yet ready for public consumption.
The upload is no longer cancellable. If further re-edits need to be applied,
an operator should manually delete or unlist the video then set the state back to `UNEDITED`.
@ -79,23 +76,16 @@ we are certain the upload didn't actually go through, and the cut can be immedia
* `FINALIZING -> UNEDITED`: When the finalization failed with an unknown error,
we are certain the upload didn't actually go through, and operator intervention is required.
* `FINALIZING -> UPLOAD_PENDING`: When you cannot use the youtube API, and need to designate
the video for manual uploading, or to be uploaded later.
* `FINALIZING -> TRANSCODING`: When the cutter has successfully finalized the upload,
but the upload location requires further processing before the video is done.
* `FINALIZING -> DONE`: When the cutter has successfully finalized the upload,
and the upload location requires no further processing.
* `UPLOAD_PENDING -> TRANSCODING`: When an operator has uploaded the video.
* `TRANSCODING -> DONE`: When any cutter detects that the upload location is finished
transcoding the video, and it is ready for public consumption.
* Missing some states here around rollbacks of upload pending
This is summarised in the below graph: (missing `UPLOAD_PENDING`)
This is summarised in the below graph:
```
retry

15
cutter/cutter/main.py
Unescape Escape View File

@ -19,7 +19,7 @@ from common.database import DBManager, query
from common.segments import get_best_segments, fast_cut_segments, full_cut_segments, ContainsHoles
from common.stats import timed
from .upload_backends import Youtube, Local, Manual, UploadError
from .upload_backends import Youtube, Local, UploadError
videos_uploaded = prom.Counter(
@ -443,17 +443,12 @@ class Cutter(object):
gevent.sleep(self.RETRYABLE_UPLOAD_ERROR_WAIT_INTERVAL)
return
# Success! Set UPLOAD_PENDING, TRANSCODING or DONE and clear any previous error.
if upload_backend.needs_manual_upload:
success_state = 'UPLOAD_PENDING'
elif upload_backend.needs_transcode:
success_state = 'TRANSCODING'
else:
success_state = 'DONE'
# Success! Set TRANSCODING or DONE and clear any previous error.
success_state = 'TRANSCODING' if upload_backend.needs_transcode else 'DONE'
maybe_upload_time = {"upload_time": datetime.datetime.utcnow()} if success_state == 'DONE' else {}
set_row(state=success_state, video_id=video_id, video_link=video_link, error=None, **maybe_upload_time)
self.logger.info("Successfully cut and put into {} job {} as {}".format(success_state, format_job(job), video_link))
self.logger.info("Successfully cut and uploaded job {} as {}".format(format_job(job), video_link))
videos_uploaded.labels(video_channel=job.video_channel,
video_quality=job.video_quality,
upload_location=job.upload_location).inc()
@ -645,8 +640,6 @@ def main(
backend_type = Youtube
elif backend_type == 'local':
backend_type = Local
elif backend_type == 'manual':
backend_type = Manual
else:
raise ValueError("Unknown upload backend type: {!r}".format(backend_type))
backend = backend_type(credentials, **backend_config)

9
cutter/cutter/upload_backends.py
Unescape Escape View File

@ -62,10 +62,6 @@ class UploadBackend(object):
If it does, it should also have a method check_status(ids) which takes a
list of video ids and returns a list of the ones who have finished processing.
If the upload backend cannot actually upload the file but needs to leave it
in a state for further manual processing, it should set needs_manual_upload.
This will put the video into the UPLOAD_PENDING state.
The upload backend also determines the encoding settings for the cutting
process, this is given as a list of ffmpeg args
under the 'encoding_settings' attribute.
@ -77,7 +73,6 @@ class UploadBackend(object):
"""
needs_transcode = False
needs_manual_upload = False
# reasonable default if settings don't otherwise matter:
# high-quality mpegts, without wasting too much cpu on encoding
@ -255,7 +250,3 @@ class Local(UploadBackend):
else:
url = 'file://{}'.format(filepath)
return video_id, url
class Manual(Local):
needs_manual_upload = True

1
postgres/setup.sh
Unescape Escape View File

@ -42,7 +42,6 @@ CREATE TYPE event_state as ENUM (
'EDITED',
'CLAIMED',
'FINALIZING',
'UPLOAD_PENDING',
'TRANSCODING',
'DONE'
);

27
thrimshim/thrimshim/main.py
Unescape Escape View File

@ -2,7 +2,6 @@ import datetime
from functools import wraps
import json
import logging
import re
import signal
import sys
@ -273,19 +272,9 @@ def update_row(ident, editor=None):
@request_stats
@authenticate
def manual_link(ident, editor=None):
"""Manually set a video_link if the state is 'UNEDITED', 'UPLOAD_PENDING' or 'DONE'"""
"""Manually set a video_link if the state is 'UNEDITED' or 'DONE' and the
upload_location is 'manual'."""
link = flask.request.json['link']
new_state = flask.request.json.get('state', 'DONE')
if new_state == 'DONE':
video_id = link
else:
# Attempt to parse from youtube URL https://www.youtube.com/watch?v=XXXXXXXXXXX or https://youtu.be/XXXXXXXXXXX
match = re.search(r'[0-9A-Za-z_-]{11}$', link)
if not match:
return 'Could not parse youtube id from url {!r}'.format(link), 400
video_id = match.group(0)
# Normalize URL
link = 'https://youtu.be/{}'.format(video_id)
conn = app.db_manager.get_conn()
results = database.query(conn, """
SELECT id, state, upload_location
@ -294,16 +283,16 @@ def manual_link(ident, editor=None):
old_row = results.fetchone()
if old_row is None:
return 'Row {} not found'.format(ident), 404
if old_row.state not in ('UNEDITED', 'UPLOAD_PENDING') and not (old_row.state == 'DONE' and old_row.upload_location == 'manual'):
if old_row.state != 'UNEDITED' and not (old_row.state == 'DONE' and old_row.upload_location == 'manual'):
return 'Invalid state {} for manual video link'.format(old_row.state), 403
now = datetime.datetime.utcnow()
results = database.query(conn, """
UPDATE events
SET state=%s, upload_location = 'manual', video_link = %s,
upload_time = %s, video_id = %s
WHERE id = %s AND (state IN ('UNEDITED', 'UPLOAD_PENDING') OR (state = 'DONE' AND
upload_location = 'manual'))""", new_state, link, now, video_id, ident)
logging.info("Row {} {} and video_link set to {}".format(ident, new_state, link))
SET state='DONE', upload_location = 'manual', video_link = %s,
editor = %s, edit_time = %s, upload_time = %s
WHERE id = %s AND (state = 'UNEDITED' OR (state = 'DONE' AND
upload_location = 'manual'))""", link, editor, now, now, ident)
logging.info("Row {} video_link set to {}".format(ident, link))
return ''

Write Preview
Loading…
Cancel
Save