From 48593e2b0645ab4a7e933083a31fe31d283e868c Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Wed, 16 Oct 2019 17:13:32 +1100 Subject: [PATCH 1/4] database, sheetsync: Add worksheet name column 'sheet_name' This tells us which sheet a row came from (so we don't need to scan every sheet to find it if we're trying to do lookups in that direction). It is also needed in order to tag the videos with the Day number. --- DATABASE.md | 1 + postgres/setup.sh | 1 + sheetsync/sheetsync/main.py | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/DATABASE.md b/DATABASE.md index e689b83..01946c9 100644 --- a/DATABASE.md +++ b/DATABASE.md @@ -125,6 +125,7 @@ Edit input values are initially NULL, but must not be NULL once the state is no columns | type | role | description -------------------------- | ---------------------------------- | :---------: | ----------- `id` | `UUID PRIMARY KEY` | sheet input | Generated and attached to rows in the sheet to uniquely identify them even in the face of added, deleted or moved rows. +`sheet_name` | `TEXT NOT NULL` | sheet input | The name of the worksheet that the row is on. This is used to tag videos, and can be used to narrow down the range to look for an id in for more efficient lookup (though we never do that right now). `event_start`, `event_end` | `TIMESTAMP` | sheet input | Start and end time of the event. Parsed from the sheet into timestamps or NULL. Used to set the editor time span, and displayed on the public sheet. The start time also determines what "day" the event lies on, for video tagging and other purposes. `category` | `TEXT NOT NULL DEFAULT ''` | sheet input | The kind of event. By convention selected from a small list of categories, but stored as an arbitrary string because there's little to no benefit to using an enum here, it just makes our job harder when adding a new category. Used to tag videos, and for display on the public sheet. `description` | `TEXT NOT NULL DEFAULT ''` | sheet input | Event description. Provides the default title and description for editors, and displayed on the public sheet. diff --git a/postgres/setup.sh b/postgres/setup.sh index 256783c..20d8141 100644 --- a/postgres/setup.sh +++ b/postgres/setup.sh @@ -49,6 +49,7 @@ CREATE TYPE event_state as ENUM ( CREATE TABLE events ( id UUID PRIMARY KEY, + sheet_name TEXT NOT NULL, event_start TIMESTAMP, event_end TIMESTAMP, category TEXT NOT NULL DEFAULT '', diff --git a/sheetsync/sheetsync/main.py b/sheetsync/sheetsync/main.py index 25231e2..5ffa076 100644 --- a/sheetsync/sheetsync/main.py +++ b/sheetsync/sheetsync/main.py @@ -190,7 +190,7 @@ class SheetSync(object): logging.info("Inserting new event {}".format(row['id'])) # Insertion conflict just means that another sheet sync beat us to the insert. # We can ignore it. - insert_cols = ['id'] + self.input_columns + insert_cols = ['id', 'sheet_name'] + self.input_columns built_query = sql.SQL(""" INSERT INTO events ({}) VALUES ({}) @@ -199,7 +199,7 @@ class SheetSync(object): sql.SQL(", ").join(sql.Identifier(col) for col in insert_cols), sql.SQL(", ").join(sql.Placeholder(col) for col in insert_cols), ) - query(self.conn, built_query, **row) + query(self.conn, built_query, sheet_name=worksheet, **row) return # Update database with any changed inputs From cbd0ef3d9e318daaffb6f8ed7e95eb05c5f3c30a Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Wed, 16 Oct 2019 17:26:21 +1100 Subject: [PATCH 2/4] cutter: Add title header, description footer and static tags These are pre-canned parts of the video metadata that we want to be configurable. --- cutter/cutter/main.py | 48 ++++++++++++++++++++++++++++++++++++------ docker-compose.jsonnet | 13 ++++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index 36c12bf..9096812 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -72,7 +72,7 @@ class Cutter(object): ERROR_RETRY_INTERVAL = 5 RETRYABLE_UPLOAD_ERROR_WAIT_INTERVAL = 5 - def __init__(self, upload_locations, dbmanager, stop, name, segments_path): + def __init__(self, upload_locations, dbmanager, stop, name, segments_path, tags, title_header, description_footer): """upload_locations is a map {location name: upload location backend} Conn is a database connection. Stop is an Event triggering graceful shutdown when set. @@ -84,6 +84,9 @@ class Cutter(object): self.dbmanager = dbmanager self.stop = stop self.segments_path = segments_path + self.tags = tags + self.title_header = title_header + self.description_footer = description_footer self.logger = logging.getLogger(type(self).__name__) self.refresh_conn() @@ -341,11 +344,16 @@ class Cutter(object): try: video_id = upload_backend.upload_video( - title=job.video_title, - description=job.video_description, - tags=[], # TODO + title=( + "{} - {}".format(self.title_header, job.video_title) + if self.title_header else job.video_title + ), + description=( + "{}\n\n{}".format(job.video_description, self.description_footer) + if self.description_footer else job.video_description + ), + tags=self.tags, data=upload_wrapper(), - hidden=True, # TODO remove when not testing ) except JobConsistencyError: raise # this ensures it's not caught in the next except block @@ -515,7 +523,18 @@ class TranscodeChecker(object): return result.rowcount -def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=8003, backdoor_port=0): +def main( + dbconnect, + config, + creds_file, + name=None, + base_dir=".", + tags='', + title_header="", + description_footer="", + metrics_port=8003, + backdoor_port=0, +): """dbconnect should be a postgres connection string, which is either a space-separated list of key=value pairs, or a URI like: postgresql://USER:PASSWORD@HOST/DBNAME?KEY=VALUE @@ -534,6 +553,19 @@ def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=80 creds_file should contain any required credentials for the upload backends, as JSON. name defaults to hostname. + + tags should be a comma-seperated list of tags to attach to all videos. + + title_header will be prepended to all video titles, seperated by a " - ". + description_footer will be added as a seperate paragraph at the end of all video descriptions. + For example, with --title-header foo --description-footer 'A video of foo.', + then a video with title 'bar' and a description 'Bar with baz' would actually have: + title: foo - bar + description: + Bar with baz + + A video of foo. + """ common.PromLogCountsHandler.install() common.install_stacksampler() @@ -545,6 +577,8 @@ def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=80 if name is None: name = socket.gethostname() + tags = tags.split(',') if tags else [] + stop = gevent.event.Event() gevent.signal(signal.SIGTERM, stop.set) # shut down on sigterm @@ -583,7 +617,7 @@ def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=80 if backend.needs_transcode and not no_transcode_check: needs_transcode_check.append(backend) - cutter = Cutter(upload_locations, dbmanager, stop, name, base_dir) + cutter = Cutter(upload_locations, dbmanager, stop, name, base_dir, tags, title_header, description_footer) transcode_checkers = [ TranscodeChecker(backend, dbmanager, stop) for backend in needs_transcode_check diff --git a/docker-compose.jsonnet b/docker-compose.jsonnet index c5a96b7..0d7fc06 100644 --- a/docker-compose.jsonnet +++ b/docker-compose.jsonnet @@ -95,6 +95,16 @@ unlisted: {type: "youtube", hidden: true, no_transcode_check: true}, }, + // Fixed tags to add to all videos + video_tags:: ["DB13", "DB2019", "2019", "Desert Bus", "Desert Bus for Hope", "Child's Play Charity", "Child's Play", "Charity Fundraiser"], + + // The header to put at the front of video titles, eg. a video with a title + // of "hello world" with title header "foo" becomes: "foo - hello world". + title_header:: "DB2019", + + // The footer to put at the bottom of descriptions, in its own paragraph. + description_footer:: "Uploaded by the Desert Bus Video Strike Team", + // Path to a JSON file containing google credentials for sheetsync as keys // 'client_id', 'client_secret' and 'refresh_token'. // May be the same as cutter_creds_file. @@ -183,6 +193,9 @@ command: [ "--base-dir", "/mnt", "--backdoor-port", std.toString($.backdoor_port), + "--tags", std.join(",", $.video_tags), + "--title-header", $.title_header, + "--description-footer", $.description_footer, $.db_connect, std.manifestJson($.cutter_config), "/etc/wubloader-creds.json", From 5e0d5b9ddc5dc1c21e50d88d9fa7ac4ef91c791c Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Wed, 16 Oct 2019 17:32:57 +1100 Subject: [PATCH 3/4] cutter: Add category and sheet_name as video tags This gives us tags on categories, and on which day's sheet the event came from. We use these to make automatic playlists. --- cutter/cutter/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index 9096812..5dfe7c1 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -35,6 +35,7 @@ upload_errors = prom.Counter( # A list of all the DB column names in CutJob CUT_JOB_PARAMS = [ + "sheet_name", "category", "allow_holes", "uploader_whitelist", @@ -352,7 +353,8 @@ class Cutter(object): "{}\n\n{}".format(job.video_description, self.description_footer) if self.description_footer else job.video_description ), - tags=self.tags, + # Add category and sheet_name as tags + tags=self.tags + [job.category, job.sheet_name], data=upload_wrapper(), ) except JobConsistencyError: From 40c4baef0f6bdd44835f3be692ee0c69895cdc91 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Wed, 16 Oct 2019 18:08:45 +1100 Subject: [PATCH 4/4] youtube upload: Set category and language settings configured on a per-location basis. --- cutter/cutter/upload_backends.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cutter/cutter/upload_backends.py b/cutter/cutter/upload_backends.py index 8c11b6c..042d2b5 100644 --- a/cutter/cutter/upload_backends.py +++ b/cutter/cutter/upload_backends.py @@ -50,12 +50,18 @@ class Youtube(UploadBackend): Config args besides credentials: hidden: If false, video is public. If true, video is unlisted. Default false. + category_id: + The numeric category id to set as the youtube category of all videos. + Default is 23, which is the id for "Comedy". Set to null to not set. + language: + The language code to describe all videos as. + Default is "en", ie. English. Set to null to not set. """ needs_transcode = True encoding_settings = [] # TODO youtube's recommended settings - def __init__(self, credentials, hidden=False): + def __init__(self, credentials, hidden=False, category_id=23, language="en"): self.logger = logging.getLogger(type(self).__name__) self.client = GoogleAPIClient( credentials['client_id'], @@ -63,6 +69,8 @@ class Youtube(UploadBackend): credentials['refresh_token'], ) self.hidden = hidden + self.category_id = category_id + self.language = language def upload_video(self, title, description, tags, data): json = { @@ -72,6 +80,11 @@ class Youtube(UploadBackend): 'tags': tags, }, } + if self.category_id is not None: + json['snippet']['categoryId'] = self.category_id + if self.language is not None: + json['snippet']['defaultLanguage'] = self.language + json['snippet']['defaultAudioLanguage'] = self.language if self.hidden: json['status'] = { 'privacyStatus': 'unlisted',