diff --git a/DATABASE.md b/DATABASE.md index c2bf010..acf220d 100644 --- a/DATABASE.md +++ b/DATABASE.md @@ -148,6 +148,7 @@ columns | type | role `allow_holes` | `BOOLEAN NOT NULL DEFAULT FALSE` | edit input | If false, any missing segments encountered while cutting will cause the cut to fail. Setting this to true should be done by an operator to indicate that holes are expected in this range. It is also the operator's responsibility to ensure that all allowed cutters have all segments that they can get, since there is no guarentee that only the cutter with the least missing segments will get the cut job. `uploader_whitelist` | `TEXT[]` | edit input | List of uploaders which are allowed to cut this entry, or NULL to indicate no restriction. This is useful if you are allowing holes and the amount of missing data differs between nodes (this shouldn't happen - this would mean replication is also failing), or if an operator is investigating a problem with a specific node. `upload_location` | `TEXT` | edit input | The upload location to upload the cut video to. This is used by the cutter, and must match one of the cutter's configured upload locations. If it does not, the cutter will not claim the event. +`public` | `BOOLEAN NOT NULL DEFAULT TRUE` | edit input | Whether the uploaded video should be public or not, if the upload location supports that distinction. For example, on youtube, non-public videos are "unlisted". `video_ranges` | `{start TIMESTAMP, end TIMESTAMP}[]` | edit input | A non-zero number of start and end times, describing the ranges of video to cut. They will be cut back-to-back in the given order, with the transitions between as per `video_transitions`. If already set, used as the default range settings when editing. `video_transitions` | `{type TEXT, duration INTERVAL}[]` | edit input | Defines how to transition between each range defined in `video_ranges`, and must be exactly the length of `video_ranges` minus 1. Each index in `video_transitions` defines the transition between the range with the same index in `video_ranges` and the next one. Transitions either specify a transition type as understood by `ffmpeg`'s `xfade` filter and a duration (amount of overlap), or can be NULL to indicate a hard cut. `video_title` | `TEXT` | edit input | The title of the video. If already set, used as the default title when editing instead of `description`. diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index 40809f2..3968de0 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -58,6 +58,7 @@ CUT_JOB_PARAMS = [ "allow_holes", "uploader_whitelist", "upload_location", + "public", "video_ranges", "video_transitions", "video_title", @@ -417,6 +418,7 @@ class Cutter(object): description=job.video_description, # Merge static and video-specific tags tags=list(set(self.tags + job.video_tags)), + public=job.public, data=upload_wrapper(), ) except (JobConsistencyError, JobCancelled, UploadError): @@ -615,7 +617,7 @@ class VideoUpdater(object): try: videos = list(self.get_videos()) self.logger.info("Found {} videos in MODIFIED".format(len(videos))) - for id, video_id, title, description, tags in videos: + for id, video_id, title, description, tags, public in videos: # NOTE: Since we aren't claiming videos, it's technically possible for this # to happen: # 1. we get MODIFIED video with title A @@ -625,12 +627,12 @@ class VideoUpdater(object): # 5. it appears to be successfully updated with B, but the title is actually A. # This is unlikely and not a disaster, so we'll just live with it. try: - self.backend.update_video(video_id, title, description, tags) + self.backend.update_video(video_id, title, description, tags, public) except Exception as ex: self.logger.exception("Failed to update video") self.mark_errored(id, "Failed to update video: {}".format(ex)) continue - marked = self.mark_done(id, video_id, title, description, tags) + marked = self.mark_done(id, video_id, title, description, tags, public) if marked: assert marked == 1 self.logger.info("Updated video {}".format(id)) @@ -648,15 +650,15 @@ class VideoUpdater(object): # To avoid exhausting API quota, errors aren't retryable. # We ignore any rows where error is not null. return query(self.conn, """ - SELECT id, video_id, video_title, video_description, video_tags + SELECT id, video_id, video_title, video_description, video_tags, public FROM events WHERE state = 'MODIFIED' AND error IS NULL """) - def mark_done(self, id, video_id, title, description, tags): + def mark_done(self, id, video_id, title, description, tags, public): """We don't want to set to DONE if the video has been modified *again* since we saw it.""" - args = dict(id=id, video_id=video_id, video_title=title, video_description=description, video_tags=tags) + args = dict(id=id, video_id=video_id, video_title=title, video_description=description, video_tags=tags, public=public) built_query = sql.SQL(""" UPDATE events SET state = 'DONE' diff --git a/cutter/cutter/upload_backends.py b/cutter/cutter/upload_backends.py index e0c8975..60d9303 100644 --- a/cutter/cutter/upload_backends.py +++ b/cutter/cutter/upload_backends.py @@ -51,9 +51,10 @@ class UploadBackend(object): Config args for the backend are passed into __init__ as kwargs, along with credentials as the first arg. - Should have a method upload_video(title, description, tags, data). - Title, description and tags may have backend-specific meaning. + Should have a method upload_video(title, description, tags, public, data). + Title, description, tags and public may have backend-specific meaning. Tags is a list of string. + Public is a boolean. Data is an iterator of bytes. It should return (video_id, video_link). @@ -64,7 +65,7 @@ class UploadBackend(object): list of video ids and returns a list of the ones who have finished processing. If updating existing videos is supported, the backend should also define a method - update_video(video_id, title, description, tags). + update_video(video_id, title, description, tags, public). Fields which cannot be updated may be ignored. Must not change the video id or link. Returns nothing. @@ -85,21 +86,19 @@ class UploadBackend(object): encoding_settings = ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '0', '-f', 'mpegts'] encoding_streamable = True - def upload_video(self, title, description, tags, data): + def upload_video(self, title, description, tags, public, data): raise NotImplementedError def check_status(self, ids): raise NotImplementedError - def update_video(self, video_id, title, description, tags): + def update_video(self, video_id, title, description, tags, public): raise NotImplementedError class Youtube(UploadBackend): """Represents a youtube channel to upload to, and settings for doing so. Config args besides credentials: - hidden: - If false, video is public. If true, video is unlisted. Default false. category_id: The numeric category id to set as the youtube category of all videos. Default is 23, which is the id for "Comedy". Set to null to not set. @@ -128,7 +127,7 @@ class Youtube(UploadBackend): '-movflags', 'faststart', # put MOOV atom at the front of the file, as requested ] - def __init__(self, credentials, hidden=False, category_id=23, language="en", use_yt_recommended_encoding=False, + def __init__(self, credentials, category_id=23, language="en", use_yt_recommended_encoding=False, mime_type='video/MP2T'): self.logger = logging.getLogger(type(self).__name__) self.client = GoogleAPIClient( @@ -136,7 +135,6 @@ class Youtube(UploadBackend): credentials['client_secret'], credentials['refresh_token'], ) - self.hidden = hidden self.category_id = category_id self.language = language self.mime_type = mime_type @@ -144,28 +142,27 @@ class Youtube(UploadBackend): self.encoding_settings = self.recommended_settings self.encoding_streamable = False - def upload_video(self, title, description, tags, data): + def upload_video(self, title, description, tags, public, data): json = { 'snippet': { 'title': title, 'description': description, 'tags': tags, }, + 'status': { + 'privacyStatus': 'public' if public else 'unlisted', + }, } if self.category_id is not None: json['snippet']['categoryId'] = self.category_id if self.language is not None: json['snippet']['defaultLanguage'] = self.language json['snippet']['defaultAudioLanguage'] = self.language - if self.hidden: - json['status'] = { - 'privacyStatus': 'unlisted', - } resp = self.client.request('POST', 'https://www.googleapis.com/upload/youtube/v3/videos', headers={'X-Upload-Content-Type': self.mime_type}, params={ - 'part': 'snippet,status' if self.hidden else 'snippet', + 'part': 'snippet,status', 'uploadType': 'resumable', }, json=json, @@ -208,13 +205,13 @@ class Youtube(UploadBackend): output.append(item['id']) return output - def update_video(self, video_id, title, description, tags): + def update_video(self, video_id, title, description, tags, public): # Any values we don't give will be deleted on PUT, so we need to first # get all the existing values then merge in our updates. resp = self.client.request('GET', 'https://www.googleapis.com/youtube/v3/videos', params={ - 'part': 'id,snippet', + 'part': 'id,snippet,status', 'id': video_id, }, metric_name='get_video', @@ -226,24 +223,27 @@ class Youtube(UploadBackend): assert len(data) == 1 data = data[0] snippet = data['snippet'].copy() + status = data['status'].copy() snippet['title'] = title snippet['description'] = description snippet['tags'] = tags + status['privacyStatus'] = 'public' if public else 'unlisted' # Since we're fetching this data anyway, we can save some quota by avoiding repeated work. # We could still race and do the same update twice, but that's fine. - if snippet == data['snippet']: + if snippet == data['snippet'] and status == data['status']: self.logger.info("Skipping update for video {}: No changes".format(video_id)) return resp = self.client.request('PUT', 'https://www.googleapis.com/youtube/v3/videos', params={ - 'part': 'id,snippet', + 'part': 'id,snippet,status', }, json={ 'id': video_id, 'snippet': snippet, + 'status': status, }, metric_name='update_video', ) @@ -264,10 +264,10 @@ class Local(UploadBackend): If not given, returns a file:// url with the full path. write_info: If true, writes a json file alongside the video file containing - the video title, description and tags. + the video title, description, tags and public setting. This is intended primarily for testing purposes. Saves files under their title, plus a random video id to avoid conflicts. - Ignores description and tags. + Ignores other parameters. """ def __init__(self, credentials, path, url_prefix=None, write_info=False): @@ -282,7 +282,7 @@ class Local(UploadBackend): raise # ignore already-exists errors - def upload_video(self, title, description, tags, data): + def upload_video(self, title, description, tags, public, data): video_id = str(uuid.uuid4()) # make title safe by removing offending characters, replacing with '-' safe_title = re.sub('[^A-Za-z0-9_]', '-', title) @@ -296,6 +296,7 @@ class Local(UploadBackend): 'title': title, 'description': description, 'tags': tags, + 'public': public, }) + '\n') with open(filepath, 'wb') as f: for chunk in data: @@ -310,7 +311,7 @@ class Local(UploadBackend): url = 'file://{}'.format(filepath) return video_id, url - def update_video(self, video_id, title, description, tags): + def update_video(self, video_id, title, description, tags, public): if not self.write_info: return safe_title = re.sub('[^A-Za-z0-9_]', '-', title) @@ -319,4 +320,5 @@ class Local(UploadBackend): 'title': title, 'description': description, 'tags': tags, + 'public': public, }) + '\n') diff --git a/docker-compose.jsonnet b/docker-compose.jsonnet index 0778d69..de17759 100644 --- a/docker-compose.jsonnet +++ b/docker-compose.jsonnet @@ -117,7 +117,6 @@ // Config for cutter upload locations. See cutter docs for full detail. cutter_config:: { desertbus: {type: "youtube"}, - unlisted: {type: "youtube", hidden: true, no_transcode_check: true}, }, default_location:: "desertbus", diff --git a/postgres/setup.sh b/postgres/setup.sh index 308b8db..39e9bc5 100644 --- a/postgres/setup.sh +++ b/postgres/setup.sh @@ -72,6 +72,7 @@ CREATE TABLE events ( allow_holes BOOLEAN NOT NULL DEFAULT FALSE, uploader_whitelist TEXT[], upload_location TEXT CHECK (state = 'UNEDITED' OR upload_location IS NOT NULL), + public BOOLEAN NOT NULL DEFAULT TRUE, video_ranges video_range[] CHECK (state IN ('UNEDITED', 'DONE') OR video_ranges IS NOT NULL), video_transitions video_transition[] CHECK (state IN ('UNEDITED', 'DONE') OR video_transitions IS NOT NULL), CHECK (