diff --git a/DATABASE.md b/DATABASE.md index e689b83..01946c9 100644 --- a/DATABASE.md +++ b/DATABASE.md @@ -125,6 +125,7 @@ Edit input values are initially NULL, but must not be NULL once the state is no columns | type | role | description -------------------------- | ---------------------------------- | :---------: | ----------- `id` | `UUID PRIMARY KEY` | sheet input | Generated and attached to rows in the sheet to uniquely identify them even in the face of added, deleted or moved rows. +`sheet_name` | `TEXT NOT NULL` | sheet input | The name of the worksheet that the row is on. This is used to tag videos, and can be used to narrow down the range to look for an id in for more efficient lookup (though we never do that right now). `event_start`, `event_end` | `TIMESTAMP` | sheet input | Start and end time of the event. Parsed from the sheet into timestamps or NULL. Used to set the editor time span, and displayed on the public sheet. The start time also determines what "day" the event lies on, for video tagging and other purposes. `category` | `TEXT NOT NULL DEFAULT ''` | sheet input | The kind of event. By convention selected from a small list of categories, but stored as an arbitrary string because there's little to no benefit to using an enum here, it just makes our job harder when adding a new category. Used to tag videos, and for display on the public sheet. `description` | `TEXT NOT NULL DEFAULT ''` | sheet input | Event description. Provides the default title and description for editors, and displayed on the public sheet. diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index 36c12bf..5dfe7c1 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -35,6 +35,7 @@ upload_errors = prom.Counter( # A list of all the DB column names in CutJob CUT_JOB_PARAMS = [ + "sheet_name", "category", "allow_holes", "uploader_whitelist", @@ -72,7 +73,7 @@ class Cutter(object): ERROR_RETRY_INTERVAL = 5 RETRYABLE_UPLOAD_ERROR_WAIT_INTERVAL = 5 - def __init__(self, upload_locations, dbmanager, stop, name, segments_path): + def __init__(self, upload_locations, dbmanager, stop, name, segments_path, tags, title_header, description_footer): """upload_locations is a map {location name: upload location backend} Conn is a database connection. Stop is an Event triggering graceful shutdown when set. @@ -84,6 +85,9 @@ class Cutter(object): self.dbmanager = dbmanager self.stop = stop self.segments_path = segments_path + self.tags = tags + self.title_header = title_header + self.description_footer = description_footer self.logger = logging.getLogger(type(self).__name__) self.refresh_conn() @@ -341,11 +345,17 @@ class Cutter(object): try: video_id = upload_backend.upload_video( - title=job.video_title, - description=job.video_description, - tags=[], # TODO + title=( + "{} - {}".format(self.title_header, job.video_title) + if self.title_header else job.video_title + ), + description=( + "{}\n\n{}".format(job.video_description, self.description_footer) + if self.description_footer else job.video_description + ), + # Add category and sheet_name as tags + tags=self.tags + [job.category, job.sheet_name], data=upload_wrapper(), - hidden=True, # TODO remove when not testing ) except JobConsistencyError: raise # this ensures it's not caught in the next except block @@ -515,7 +525,18 @@ class TranscodeChecker(object): return result.rowcount -def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=8003, backdoor_port=0): +def main( + dbconnect, + config, + creds_file, + name=None, + base_dir=".", + tags='', + title_header="", + description_footer="", + metrics_port=8003, + backdoor_port=0, +): """dbconnect should be a postgres connection string, which is either a space-separated list of key=value pairs, or a URI like: postgresql://USER:PASSWORD@HOST/DBNAME?KEY=VALUE @@ -534,6 +555,19 @@ def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=80 creds_file should contain any required credentials for the upload backends, as JSON. name defaults to hostname. + + tags should be a comma-seperated list of tags to attach to all videos. + + title_header will be prepended to all video titles, seperated by a " - ". + description_footer will be added as a seperate paragraph at the end of all video descriptions. + For example, with --title-header foo --description-footer 'A video of foo.', + then a video with title 'bar' and a description 'Bar with baz' would actually have: + title: foo - bar + description: + Bar with baz + + A video of foo. + """ common.PromLogCountsHandler.install() common.install_stacksampler() @@ -545,6 +579,8 @@ def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=80 if name is None: name = socket.gethostname() + tags = tags.split(',') if tags else [] + stop = gevent.event.Event() gevent.signal(signal.SIGTERM, stop.set) # shut down on sigterm @@ -583,7 +619,7 @@ def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=80 if backend.needs_transcode and not no_transcode_check: needs_transcode_check.append(backend) - cutter = Cutter(upload_locations, dbmanager, stop, name, base_dir) + cutter = Cutter(upload_locations, dbmanager, stop, name, base_dir, tags, title_header, description_footer) transcode_checkers = [ TranscodeChecker(backend, dbmanager, stop) for backend in needs_transcode_check diff --git a/cutter/cutter/upload_backends.py b/cutter/cutter/upload_backends.py index 8c11b6c..042d2b5 100644 --- a/cutter/cutter/upload_backends.py +++ b/cutter/cutter/upload_backends.py @@ -50,12 +50,18 @@ class Youtube(UploadBackend): Config args besides credentials: hidden: If false, video is public. If true, video is unlisted. Default false. + category_id: + The numeric category id to set as the youtube category of all videos. + Default is 23, which is the id for "Comedy". Set to null to not set. + language: + The language code to describe all videos as. + Default is "en", ie. English. Set to null to not set. """ needs_transcode = True encoding_settings = [] # TODO youtube's recommended settings - def __init__(self, credentials, hidden=False): + def __init__(self, credentials, hidden=False, category_id=23, language="en"): self.logger = logging.getLogger(type(self).__name__) self.client = GoogleAPIClient( credentials['client_id'], @@ -63,6 +69,8 @@ class Youtube(UploadBackend): credentials['refresh_token'], ) self.hidden = hidden + self.category_id = category_id + self.language = language def upload_video(self, title, description, tags, data): json = { @@ -72,6 +80,11 @@ class Youtube(UploadBackend): 'tags': tags, }, } + if self.category_id is not None: + json['snippet']['categoryId'] = self.category_id + if self.language is not None: + json['snippet']['defaultLanguage'] = self.language + json['snippet']['defaultAudioLanguage'] = self.language if self.hidden: json['status'] = { 'privacyStatus': 'unlisted', diff --git a/docker-compose.jsonnet b/docker-compose.jsonnet index c5a96b7..0d7fc06 100644 --- a/docker-compose.jsonnet +++ b/docker-compose.jsonnet @@ -95,6 +95,16 @@ unlisted: {type: "youtube", hidden: true, no_transcode_check: true}, }, + // Fixed tags to add to all videos + video_tags:: ["DB13", "DB2019", "2019", "Desert Bus", "Desert Bus for Hope", "Child's Play Charity", "Child's Play", "Charity Fundraiser"], + + // The header to put at the front of video titles, eg. a video with a title + // of "hello world" with title header "foo" becomes: "foo - hello world". + title_header:: "DB2019", + + // The footer to put at the bottom of descriptions, in its own paragraph. + description_footer:: "Uploaded by the Desert Bus Video Strike Team", + // Path to a JSON file containing google credentials for sheetsync as keys // 'client_id', 'client_secret' and 'refresh_token'. // May be the same as cutter_creds_file. @@ -183,6 +193,9 @@ command: [ "--base-dir", "/mnt", "--backdoor-port", std.toString($.backdoor_port), + "--tags", std.join(",", $.video_tags), + "--title-header", $.title_header, + "--description-footer", $.description_footer, $.db_connect, std.manifestJson($.cutter_config), "/etc/wubloader-creds.json", diff --git a/postgres/setup.sh b/postgres/setup.sh index 256783c..20d8141 100644 --- a/postgres/setup.sh +++ b/postgres/setup.sh @@ -49,6 +49,7 @@ CREATE TYPE event_state as ENUM ( CREATE TABLE events ( id UUID PRIMARY KEY, + sheet_name TEXT NOT NULL, event_start TIMESTAMP, event_end TIMESTAMP, category TEXT NOT NULL DEFAULT '', diff --git a/sheetsync/sheetsync/main.py b/sheetsync/sheetsync/main.py index 25231e2..5ffa076 100644 --- a/sheetsync/sheetsync/main.py +++ b/sheetsync/sheetsync/main.py @@ -190,7 +190,7 @@ class SheetSync(object): logging.info("Inserting new event {}".format(row['id'])) # Insertion conflict just means that another sheet sync beat us to the insert. # We can ignore it. - insert_cols = ['id'] + self.input_columns + insert_cols = ['id', 'sheet_name'] + self.input_columns built_query = sql.SQL(""" INSERT INTO events ({}) VALUES ({}) @@ -199,7 +199,7 @@ class SheetSync(object): sql.SQL(", ").join(sql.Identifier(col) for col in insert_cols), sql.SQL(", ").join(sql.Placeholder(col) for col in insert_cols), ) - query(self.conn, built_query, **row) + query(self.conn, built_query, sheet_name=worksheet, **row) return # Update database with any changed inputs