Merge pull request #102 from ekimekim/mike/cutter/tags-and-things

tags, other video presentation tweaks and things

DATABASE.md

@@ -125,6 +125,7 @@ Edit input values are initially NULL, but must not be NULL once the state is no
 columns                    | type                               | role        | description
 -------------------------- | ---------------------------------- | :---------: | -----------
 `id`                       | `UUID PRIMARY KEY`                 | sheet input | Generated and attached to rows in the sheet to uniquely identify them even in the face of added, deleted or moved rows.
+`sheet_name`               | `TEXT NOT NULL`                    | sheet input | The name of the worksheet that the row is on. This is used to tag videos, and can be used to narrow down the range to look for an id in for more efficient lookup (though we never do that right now).
 `event_start`, `event_end` | `TIMESTAMP`                        | sheet input | Start and end time of the event. Parsed from the sheet into timestamps or NULL. Used to set the editor time span, and displayed on the public sheet. The start time also determines what "day" the event lies on, for video tagging and other purposes.
 `category`                 | `TEXT NOT NULL DEFAULT ''`         | sheet input | The kind of event. By convention selected from a small list of categories, but stored as an arbitrary string because there's little to no benefit to using an enum here, it just makes our job harder when adding a new category. Used to tag videos, and for display on the public sheet.
 `description`              | `TEXT NOT NULL DEFAULT ''`         | sheet input | Event description. Provides the default title and description for editors, and displayed on the public sheet.

cutter/cutter/main.py

@@ -35,6 +35,7 @@ upload_errors  = prom.Counter(
 
 # A list of all the DB column names in CutJob
 CUT_JOB_PARAMS = [
+	"sheet_name",
 	"category",
 	"allow_holes",
 	"uploader_whitelist",
@@ -72,7 +73,7 @@ class Cutter(object):
 	ERROR_RETRY_INTERVAL = 5
 	RETRYABLE_UPLOAD_ERROR_WAIT_INTERVAL = 5
 
-	def __init__(self, upload_locations, dbmanager, stop, name, segments_path):
+	def __init__(self, upload_locations, dbmanager, stop, name, segments_path, tags, title_header, description_footer):
 		"""upload_locations is a map {location name: upload location backend}
 		Conn is a database connection.
 		Stop is an Event triggering graceful shutdown when set.
@@ -84,6 +85,9 @@ class Cutter(object):
 		self.dbmanager = dbmanager
 		self.stop = stop
 		self.segments_path = segments_path
+		self.tags = tags
+		self.title_header = title_header
+		self.description_footer = description_footer
 		self.logger = logging.getLogger(type(self).__name__)
 		self.refresh_conn()
 
@@ -341,11 +345,17 @@ class Cutter(object):
 
 		try:
 			video_id = upload_backend.upload_video(
-				title=job.video_title,
-				description=job.video_description,
-				tags=[], # TODO
+				title=(
+					"{} - {}".format(self.title_header, job.video_title)
+					if self.title_header else job.video_title
+				),
+				description=(
+					"{}\n\n{}".format(job.video_description, self.description_footer)
+					if self.description_footer else job.video_description
+				),
+				# Add category and sheet_name as tags
+				tags=self.tags + [job.category, job.sheet_name],
 				data=upload_wrapper(),
-				hidden=True, # TODO remove when not testing
 			)
 		except JobConsistencyError:
 			raise # this ensures it's not caught in the next except block
@@ -515,7 +525,18 @@ class TranscodeChecker(object):
 		return result.rowcount
 
 
-def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=8003, backdoor_port=0):
+def main(
+	dbconnect,
+	config,
+	creds_file,
+	name=None,
+	base_dir=".",
+	tags='',
+	title_header="",
+	description_footer="",
+	metrics_port=8003,
+	backdoor_port=0,
+):
 	"""dbconnect should be a postgres connection string, which is either a space-separated
 	list of key=value pairs, or a URI like:
 		postgresql://USER:PASSWORD@HOST/DBNAME?KEY=VALUE
@@ -534,6 +555,19 @@ def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=80
 	creds_file should contain any required credentials for the upload backends, as JSON.
 
 	name defaults to hostname.
+
+	tags should be a comma-seperated list of tags to attach to all videos.
+
+	title_header will be prepended to all video titles, seperated by a " - ".
+	description_footer will be added as a seperate paragraph at the end of all video descriptions.
+	For example, with --title-header foo --description-footer 'A video of foo.',
+	then a video with title 'bar' and a description 'Bar with baz' would actually have:
+		title: foo - bar
+		description:
+			Bar with baz
+
+			A video of foo.
+
 	"""
 	common.PromLogCountsHandler.install()
 	common.install_stacksampler()
@@ -545,6 +579,8 @@ def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=80
 	if name is None:
 		name = socket.gethostname()
 
+	tags = tags.split(',') if tags else []
+
 	stop = gevent.event.Event()
 	gevent.signal(signal.SIGTERM, stop.set) # shut down on sigterm
 
@@ -583,7 +619,7 @@ def main(dbconnect, config, creds_file, name=None, base_dir=".", metrics_port=80
 		if backend.needs_transcode and not no_transcode_check:
 			needs_transcode_check.append(backend)
 
-	cutter = Cutter(upload_locations, dbmanager, stop, name, base_dir)
+	cutter = Cutter(upload_locations, dbmanager, stop, name, base_dir, tags, title_header, description_footer)
 	transcode_checkers = [
 		TranscodeChecker(backend, dbmanager, stop)
 		for backend in needs_transcode_check

cutter/cutter/upload_backends.py

@@ -50,12 +50,18 @@ class Youtube(UploadBackend):
 	Config args besides credentials:
 		hidden:
 			If false, video is public. If true, video is unlisted. Default false.
+		category_id:
+			The numeric category id to set as the youtube category of all videos.
+			Default is 23, which is the id for "Comedy". Set to null to not set.
+		language:
+			The language code to describe all videos as.
+			Default is "en", ie. English. Set to null to not set.
 	"""
 
 	needs_transcode = True
 	encoding_settings = [] # TODO youtube's recommended settings
 
-	def __init__(self, credentials, hidden=False):
+	def __init__(self, credentials, hidden=False, category_id=23, language="en"):
 		self.logger = logging.getLogger(type(self).__name__)
 		self.client = GoogleAPIClient(
 			credentials['client_id'],
@@ -63,6 +69,8 @@ class Youtube(UploadBackend):
 			credentials['refresh_token'],
 		)
 		self.hidden = hidden
+		self.category_id = category_id
+		self.language = language
 
 	def upload_video(self, title, description, tags, data):
 		json = {
@@ -72,6 +80,11 @@ class Youtube(UploadBackend):
 				'tags': tags,
 			},
 		}
+		if self.category_id is not None:
+			json['snippet']['categoryId'] = self.category_id
+		if self.language is not None:
+			json['snippet']['defaultLanguage'] = self.language
+			json['snippet']['defaultAudioLanguage'] = self.language
 		if self.hidden:
 			json['status'] = {
 				'privacyStatus': 'unlisted',

docker-compose.jsonnet

@@ -95,6 +95,16 @@
     unlisted: {type: "youtube", hidden: true, no_transcode_check: true},
   },
 
+  // Fixed tags to add to all videos
+  video_tags:: ["DB13", "DB2019", "2019", "Desert Bus", "Desert Bus for Hope", "Child's Play Charity", "Child's Play", "Charity Fundraiser"],
+
+  // The header to put at the front of video titles, eg. a video with a title
+  // of "hello world" with title header "foo" becomes: "foo - hello world".
+  title_header:: "DB2019",
+
+  // The footer to put at the bottom of descriptions, in its own paragraph.
+  description_footer:: "Uploaded by the Desert Bus Video Strike Team",
+
   // Path to a JSON file containing google credentials for sheetsync as keys
   // 'client_id', 'client_secret' and 'refresh_token'.
   // May be the same as cutter_creds_file.
@@ -183,6 +193,9 @@
       command: [
         "--base-dir", "/mnt",
         "--backdoor-port", std.toString($.backdoor_port),
+        "--tags", std.join(",", $.video_tags),
+        "--title-header", $.title_header,
+        "--description-footer", $.description_footer,
         $.db_connect,
         std.manifestJson($.cutter_config),
         "/etc/wubloader-creds.json",

postgres/setup.sh

@@ -49,6 +49,7 @@ CREATE TYPE event_state as ENUM (
 
 CREATE TABLE events (
 	id UUID PRIMARY KEY,
+	sheet_name TEXT NOT NULL,
 	event_start TIMESTAMP,
 	event_end TIMESTAMP,
 	category TEXT NOT NULL DEFAULT '',

sheetsync/sheetsync/main.py

@@ -190,7 +190,7 @@ class SheetSync(object):
 			logging.info("Inserting new event {}".format(row['id']))
 			# Insertion conflict just means that another sheet sync beat us to the insert.
 			# We can ignore it.
-			insert_cols = ['id'] + self.input_columns
+			insert_cols = ['id', 'sheet_name'] + self.input_columns
 			built_query = sql.SQL("""
 				INSERT INTO events ({})
 				VALUES ({})
@@ -199,7 +199,7 @@ class SheetSync(object):
 				sql.SQL(", ").join(sql.Identifier(col) for col in insert_cols),
 				sql.SQL(", ").join(sql.Placeholder(col) for col in insert_cols),
 			)
-			query(self.conn, built_query, **row)
+			query(self.conn, built_query, sheet_name=worksheet, **row)
 			return
 
 		# Update database with any changed inputs