From ba746ff6e65d131964dfc9d647f2bccddefbacc6 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Sat, 26 Oct 2019 04:33:24 -0700 Subject: [PATCH] Add title and description header/footer in thrimshim instead of cutter This accomplishes two things: 1. It allows thrimshim to properly validate length restrictions (not implemented yet) 2. It means that the database has a record of the values actually written for each of these rows, instead of that information depending on how the cutter was configured at the time. --- cutter/cutter/main.py | 29 ++++------------------------- docker-compose.jsonnet | 4 ++-- thrimshim/thrimshim/main.py | 27 ++++++++++++++++++++++++++- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index 733b391..549539e 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -73,7 +73,7 @@ class Cutter(object): ERROR_RETRY_INTERVAL = 5 RETRYABLE_UPLOAD_ERROR_WAIT_INTERVAL = 5 - def __init__(self, upload_locations, dbmanager, stop, name, segments_path, tags, title_header, description_footer): + def __init__(self, upload_locations, dbmanager, stop, name, segments_path, tags): """upload_locations is a map {location name: upload location backend} Conn is a database connection. Stop is an Event triggering graceful shutdown when set. @@ -86,8 +86,6 @@ class Cutter(object): self.stop = stop self.segments_path = segments_path self.tags = tags - self.title_header = title_header - self.description_footer = description_footer self.logger = logging.getLogger(type(self).__name__) self.refresh_conn() @@ -338,14 +336,8 @@ class Cutter(object): # a second try around the whole thing. try: video_id, video_link = upload_backend.upload_video( - title=( - "{} - {}".format(self.title_header, job.video_title) - if self.title_header else job.video_title - ), - description=( - "{}\n\n{}".format(job.video_description, self.description_footer) - if self.description_footer else job.video_description - ), + title=job.video_title, + description=job.video_description, # Add category and sheet_name as tags tags=self.tags + [job.category, job.sheet_name], data=upload_wrapper(), @@ -524,8 +516,6 @@ def main( name=None, base_dir=".", tags='', - title_header="", - description_footer="", metrics_port=8003, backdoor_port=0, ): @@ -552,17 +542,6 @@ def main( name defaults to hostname. tags should be a comma-seperated list of tags to attach to all videos. - - title_header will be prepended to all video titles, seperated by a " - ". - description_footer will be added as a seperate paragraph at the end of all video descriptions. - For example, with --title-header foo --description-footer 'A video of foo.', - then a video with title 'bar' and a description 'Bar with baz' would actually have: - title: foo - bar - description: - Bar with baz - - A video of foo. - """ common.PromLogCountsHandler.install() common.install_stacksampler() @@ -620,7 +599,7 @@ def main( if backend.needs_transcode and not no_transcode_check: needs_transcode_check.append(backend) - cutter = Cutter(upload_locations, dbmanager, stop, name, base_dir, tags, title_header, description_footer) + cutter = Cutter(upload_locations, dbmanager, stop, name, base_dir, tags) transcode_checkers = [ TranscodeChecker(backend, dbmanager, stop) for backend in needs_transcode_check diff --git a/docker-compose.jsonnet b/docker-compose.jsonnet index 7638241..33d2f6e 100644 --- a/docker-compose.jsonnet +++ b/docker-compose.jsonnet @@ -194,8 +194,6 @@ "--base-dir", "/mnt", "--backdoor-port", std.toString($.backdoor_port), "--tags", std.join(",", $.video_tags), - "--title-header", $.title_header, - "--description-footer", $.description_footer, $.db_connect, std.manifestJson($.cutter_config), "/etc/wubloader-creds.json", @@ -219,6 +217,8 @@ // Args for the thrimshim: database connection string command: [ "--backdoor-port", std.toString($.backdoor_port), + "--title-header", $.title_header, + "--description-footer", $.description_footer, $.db_connect, $.channel, $.bustime_start, diff --git a/thrimshim/thrimshim/main.py b/thrimshim/thrimshim/main.py index 4460382..1d5ae9d 100644 --- a/thrimshim/thrimshim/main.py +++ b/thrimshim/thrimshim/main.py @@ -141,6 +141,21 @@ def get_row(ident): if response["video_channel"] is None: response["video_channel"] = app.default_channel response["bustime_start"] = app.bustime_start + + # remove any added headers or footers so round-tripping is a no-op + if ( + app.title_header is not None + and response["video_title"] is not None + and response["video_title"].startswith(app.title_header + ' - ') + ): + response["video_title"] = response["video_title"][len(app.title_header + ' - '):] + if ( + app.description_footer is not None + and response["video_description"] is not None + and response["video_description"].endswith('\n\n' + app.description_footer) + ): + response["video_description"] = response["video_description"][:-len('\n\n' + app.description_footer)] + logging.info('Row {} fetched'.format(ident)) return json.dumps(response) @@ -205,6 +220,12 @@ def update_row(ident, editor=None): new_row['editor'] = editor new_row['edit_time'] = datetime.datetime.utcnow() + # Include headers and footers + if app.title_header is not None and 'video_title' in new_row: + new_row['video_title'] = '{} - {}'.format(app.title_header, new_row['video_title']) + if app.description_footer is not None and 'video_description' in new_row: + new_row['video_description'] = '{}\n\n{}'.format(new_row['video_description'], app.description_footer) + # actually update database build_query = sql.SQL(""" UPDATE events @@ -275,14 +296,18 @@ def reset_row(ident, editor=None): @argh.arg('bustime-start', help='The start time in UTC for the event, for UTC-Bustime conversion') @argh.arg('--backdoor-port', help='Port for gevent.backdoor access. By default disabled.') @argh.arg('--no-authentication', help='Do not authenticate') +@argh.arg('--title-header', help='A header to prefix all titles with, seperated from the submitted title by " - "') +@argh.arg('--description-footer', help='A footer to suffix all descriptions with, seperated from the submitted description by a blank line.') def main(connection_string, default_channel, bustime_start, host='0.0.0.0', port=8004, backdoor_port=0, - no_authentication=False): + no_authentication=False, title_header=None, description_footer=None): """Thrimshim service.""" server = WSGIServer((host, port), cors(app)) app.no_authentication = no_authentication app.default_channel = default_channel app.bustime_start = bustime_start + app.title_header = title_header + app.description_footer = description_footer stopping = gevent.event.Event() def stop():