Add "public" column to control if videos are unlisted

This column is modifyable.
This replaces the old "hidden" argument to youtube upload backends.

DATABASE.md

@@ -148,6 +148,7 @@ columns                    | type                                 | role
 `allow_holes`              | `BOOLEAN NOT NULL DEFAULT FALSE`     | edit input  | If false, any missing segments encountered while cutting will cause the cut to fail. Setting this to true should be done by an operator to indicate that holes are expected in this range. It is also the operator's responsibility to ensure that all allowed cutters have all segments that they can get, since there is no guarentee that only the cutter with the least missing segments will get the cut job.
 `uploader_whitelist`       | `TEXT[]`                             | edit input  | List of uploaders which are allowed to cut this entry, or NULL to indicate no restriction. This is useful if you are allowing holes and the amount of missing data differs between nodes (this shouldn't happen - this would mean replication is also failing), or if an operator is investigating a problem with a specific node.
 `upload_location`          | `TEXT`                               | edit input  | The upload location to upload the cut video to. This is used by the cutter, and must match one of the cutter's configured upload locations. If it does not, the cutter will not claim the event.
+`public`                   | `BOOLEAN NOT NULL DEFAULT TRUE`      | edit input  | Whether the uploaded video should be public or not, if the upload location supports that distinction. For example, on youtube, non-public videos are "unlisted".
 `video_ranges`             | `{start TIMESTAMP, end TIMESTAMP}[]` | edit input  | A non-zero number of start and end times, describing the ranges of video to cut. They will be cut back-to-back in the given order, with the transitions between as per `video_transitions`. If already set, used as the default range settings when editing.
 `video_transitions`        | `{type TEXT, duration INTERVAL}[]`   | edit input  | Defines how to transition between each range defined in `video_ranges`, and must be exactly the length of `video_ranges` minus 1. Each index in `video_transitions` defines the transition between the range with the same index in `video_ranges` and the next one. Transitions either specify a transition type as understood by `ffmpeg`'s `xfade` filter and a duration (amount of overlap), or can be NULL to indicate a hard cut.
 `video_title`              | `TEXT`                               | edit input  | The title of the video. If already set, used as the default title when editing instead of `description`.

cutter/cutter/main.py

@@ -58,6 +58,7 @@ CUT_JOB_PARAMS = [
 	"allow_holes",
 	"uploader_whitelist",
 	"upload_location",
+	"public",
 	"video_ranges",
 	"video_transitions",
 	"video_title",
@@ -417,6 +418,7 @@ class Cutter(object):
 					description=job.video_description,
 					# Merge static and video-specific tags
 					tags=list(set(self.tags + job.video_tags)),
+					public=job.public,
 					data=upload_wrapper(),
 				)
 			except (JobConsistencyError, JobCancelled, UploadError):
@@ -615,7 +617,7 @@ class VideoUpdater(object):
 			try:
 				videos = list(self.get_videos())
 				self.logger.info("Found {} videos in MODIFIED".format(len(videos)))
-				for id, video_id, title, description, tags in videos:
+				for id, video_id, title, description, tags, public in videos:
 					# NOTE: Since we aren't claiming videos, it's technically possible for this
 					# to happen:
 					# 1. we get MODIFIED video with title A
@@ -625,12 +627,12 @@ class VideoUpdater(object):
 					# 5. it appears to be successfully updated with B, but the title is actually A.
 					# This is unlikely and not a disaster, so we'll just live with it.
 					try:
-						self.backend.update_video(video_id, title, description, tags)
+						self.backend.update_video(video_id, title, description, tags, public)
 					except Exception as ex:
 						self.logger.exception("Failed to update video")
 						self.mark_errored(id, "Failed to update video: {}".format(ex))
 						continue
-					marked = self.mark_done(id, video_id, title, description, tags)
+					marked = self.mark_done(id, video_id, title, description, tags, public)
 					if marked:
 						assert marked == 1
 						self.logger.info("Updated video {}".format(id))
@@ -648,15 +650,15 @@ class VideoUpdater(object):
 		# To avoid exhausting API quota, errors aren't retryable.
 		# We ignore any rows where error is not null.
 		return query(self.conn, """
-			SELECT id, video_id, video_title, video_description, video_tags
+			SELECT id, video_id, video_title, video_description, video_tags, public
 			FROM events
 			WHERE state = 'MODIFIED' AND error IS NULL
 		""")
 
-	def mark_done(self, id, video_id, title, description, tags):
+	def mark_done(self, id, video_id, title, description, tags, public):
 		"""We don't want to set to DONE if the video has been modified *again* since
 		we saw it."""
-		args = dict(id=id, video_id=video_id, video_title=title, video_description=description, video_tags=tags)
+		args = dict(id=id, video_id=video_id, video_title=title, video_description=description, video_tags=tags, public=public)
 		built_query = sql.SQL("""
 			UPDATE events
 			SET state = 'DONE'

cutter/cutter/upload_backends.py

@@ -51,9 +51,10 @@ class UploadBackend(object):
 	Config args for the backend are passed into __init__ as kwargs,
 	along with credentials as the first arg.
 
-	Should have a method upload_video(title, description, tags, data).
+	Should have a method upload_video(title, description, tags, public, data).
-	Title, description and tags may have backend-specific meaning.
+	Title, description, tags and public may have backend-specific meaning.
 	Tags is a list of string.
+	Public is a boolean.
 	Data is an iterator of bytes.
 	It should return (video_id, video_link).
 
@@ -64,7 +65,7 @@ class UploadBackend(object):
 	list of video ids and returns a list of the ones who have finished processing.
 
 	If updating existing videos is supported, the backend should also define a method
-	update_video(video_id, title, description, tags).
+	update_video(video_id, title, description, tags, public).
 	Fields which cannot be updated may be ignored.
 	Must not change the video id or link. Returns nothing.
 
@@ -85,21 +86,19 @@ class UploadBackend(object):
 	encoding_settings = ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '0', '-f', 'mpegts']
 	encoding_streamable = True
 
-	def upload_video(self, title, description, tags, data):
+	def upload_video(self, title, description, tags, public, data):
 		raise NotImplementedError
 
 	def check_status(self, ids):
 		raise NotImplementedError
 
-	def update_video(self, video_id, title, description, tags):
+	def update_video(self, video_id, title, description, tags, public):
 		raise NotImplementedError
 
 
 class Youtube(UploadBackend):
 	"""Represents a youtube channel to upload to, and settings for doing so.
 	Config args besides credentials:
-		hidden:
-			If false, video is public. If true, video is unlisted. Default false.
 		category_id:
 			The numeric category id to set as the youtube category of all videos.
 			Default is 23, which is the id for "Comedy". Set to null to not set.
@@ -128,7 +127,7 @@ class Youtube(UploadBackend):
 		'-movflags', 'faststart', # put MOOV atom at the front of the file, as requested
 	]
 
-	def __init__(self, credentials, hidden=False, category_id=23, language="en", use_yt_recommended_encoding=False,
+	def __init__(self, credentials, category_id=23, language="en", use_yt_recommended_encoding=False,
 		mime_type='video/MP2T'):
 		self.logger = logging.getLogger(type(self).__name__)
 		self.client = GoogleAPIClient(
@@ -136,7 +135,6 @@ class Youtube(UploadBackend):
 			credentials['client_secret'],
 			credentials['refresh_token'],
 		)
-		self.hidden = hidden
 		self.category_id = category_id
 		self.language = language
 		self.mime_type = mime_type
@@ -144,28 +142,27 @@ class Youtube(UploadBackend):
 			self.encoding_settings = self.recommended_settings
 			self.encoding_streamable = False
 
-	def upload_video(self, title, description, tags, data):
+	def upload_video(self, title, description, tags, public, data):
 		json = {
 			'snippet': {
 				'title': title,
 				'description': description,
 				'tags': tags,
 			},
+			'status': {
+				'privacyStatus': 'public' if public else 'unlisted',
+			},
 		}
 		if self.category_id is not None:
 			json['snippet']['categoryId'] = self.category_id
 		if self.language is not None:
 			json['snippet']['defaultLanguage'] = self.language
 			json['snippet']['defaultAudioLanguage'] = self.language
-		if self.hidden:
-			json['status'] = {
-				'privacyStatus': 'unlisted',
-			}
 		resp = self.client.request('POST',
 			'https://www.googleapis.com/upload/youtube/v3/videos',
 			headers={'X-Upload-Content-Type': self.mime_type},
 			params={
-				'part': 'snippet,status' if self.hidden else 'snippet',
+				'part': 'snippet,status',
 				'uploadType': 'resumable',
 			},
 			json=json,
@@ -208,13 +205,13 @@ class Youtube(UploadBackend):
 					output.append(item['id'])
 		return output
 
-	def update_video(self, video_id, title, description, tags):
+	def update_video(self, video_id, title, description, tags, public):
 		# Any values we don't give will be deleted on PUT, so we need to first
 		# get all the existing values then merge in our updates.
 		resp = self.client.request('GET',
 			'https://www.googleapis.com/youtube/v3/videos',
 			params={
-				'part': 'id,snippet',
+				'part': 'id,snippet,status',
 				'id': video_id,
 			},
 			metric_name='get_video',
@@ -226,24 +223,27 @@ class Youtube(UploadBackend):
 		assert len(data) == 1
 		data = data[0]
 		snippet = data['snippet'].copy()
+		status = data['status'].copy()
 
 		snippet['title'] = title
 		snippet['description'] = description
 		snippet['tags'] = tags
+		status['privacyStatus'] = 'public' if public else 'unlisted'
 		# Since we're fetching this data anyway, we can save some quota by avoiding repeated work.
 		# We could still race and do the same update twice, but that's fine.
-		if snippet == data['snippet']:
+		if snippet == data['snippet'] and status == data['status']:
 			self.logger.info("Skipping update for video {}: No changes".format(video_id))
 			return
 
 		resp = self.client.request('PUT',
 			'https://www.googleapis.com/youtube/v3/videos',
 			params={
-				'part': 'id,snippet',
+				'part': 'id,snippet,status',
 			},
 			json={
 				'id': video_id,
 				'snippet': snippet,
+				'status': status,
 			},
 			metric_name='update_video',
 		)
@@ -264,10 +264,10 @@ class Local(UploadBackend):
 			If not given, returns a file:// url with the full path.
 		write_info:
 			If true, writes a json file alongside the video file containing
-			the video title, description and tags.
+			the video title, description, tags and public setting.
 			This is intended primarily for testing purposes.
 	Saves files under their title, plus a random video id to avoid conflicts.
-	Ignores description and tags.
+	Ignores other parameters.
 	"""
 
 	def __init__(self, credentials, path, url_prefix=None, write_info=False):
@@ -282,7 +282,7 @@ class Local(UploadBackend):
 				raise
 			# ignore already-exists errors
 
-	def upload_video(self, title, description, tags, data):
+	def upload_video(self, title, description, tags, public, data):
 		video_id = str(uuid.uuid4())
 		# make title safe by removing offending characters, replacing with '-'
 		safe_title = re.sub('[^A-Za-z0-9_]', '-', title)
@@ -296,6 +296,7 @@ class Local(UploadBackend):
 						'title': title,
 						'description': description,
 						'tags': tags,
+						'public': public,
 					}) + '\n')
 			with open(filepath, 'wb') as f:
 				for chunk in data:
@@ -310,7 +311,7 @@ class Local(UploadBackend):
 			url = 'file://{}'.format(filepath)
 		return video_id, url
 
-	def update_video(self, video_id, title, description, tags):
+	def update_video(self, video_id, title, description, tags, public):
 		if not self.write_info:
 			return
 		safe_title = re.sub('[^A-Za-z0-9_]', '-', title)
@@ -319,4 +320,5 @@ class Local(UploadBackend):
 				'title': title,
 				'description': description,
 				'tags': tags,
+				'public': public,
 			}) + '\n')

docker-compose.jsonnet

@@ -117,7 +117,6 @@
   // Config for cutter upload locations. See cutter docs for full detail.
   cutter_config:: {
     desertbus: {type: "youtube"},
-    unlisted: {type: "youtube", hidden: true, no_transcode_check: true},
   },
   default_location:: "desertbus",
 

postgres/setup.sh

@@ -72,6 +72,7 @@ CREATE TABLE events (
 	allow_holes BOOLEAN NOT NULL DEFAULT FALSE,
 	uploader_whitelist TEXT[],
 	upload_location TEXT CHECK (state = 'UNEDITED' OR upload_location IS NOT NULL),
+	public BOOLEAN NOT NULL DEFAULT TRUE,
 	video_ranges video_range[] CHECK (state IN ('UNEDITED', 'DONE') OR video_ranges IS NOT NULL),
 	video_transitions video_transition[] CHECK (state IN ('UNEDITED', 'DONE') OR video_transitions IS NOT NULL),
 	CHECK (