From bd519369bdf7753f0f002a4ace758b6e31e614c9 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Mon, 3 Oct 2022 19:18:26 +1100 Subject: [PATCH] thumbnail fixes --- cutter/cutter/main.py | 27 +++++++++++++++++---------- cutter/cutter/upload_backends.py | 5 +++-- thrimshim/thrimshim/main.py | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index 162840b..56ae5dc 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -692,9 +692,6 @@ UPDATE_JOB_PARAMS = [ "thumbnail_image", "thumbnail_last_written", ] -UpdateJob = namedtuple('UpdateJob', [ - "id", -] + UPDATE_JOB_PARAMS) class VideoUpdater(object): CHECK_INTERVAL = 10 # this is slow to reduce the chance of multiple cutters updating the same row @@ -755,12 +752,20 @@ class VideoUpdater(object): assert False, "Bad thumbnail mode: {}".format(job.thumbnail_mode) updates['thumbnail_image'] = thumbnail_image new_hash = hashlib.sha256(thumbnail_image).digest() - if job.thumbnail_last_written != new_hash: + if bytes(job.thumbnail_last_written) != new_hash: self.logger.info("Setting thumbnail for {}".format(job.id)) - self.backend.set_thumbnail(job.video_id, job.thumbnail_image) + self.backend.set_thumbnail(job.video_id, thumbnail_image) updates['thumbnail_last_written'] = new_hash + else: + self.logger.info("No change in thumbnail image for {}".format(job.id)) except Exception as ex: - self.logger.exception("Failed to update video") + # for HTTPErrors, getting http response body is also useful + if isinstance(ex, requests.HTTPError): + self.logger.exception("Failed to update video: {}".format(ex.response.content)) + ex = "{}: {}".format(ex, ex.response.content) + else: + self.logger.exception("Failed to update video") + self.mark_errored(job.id, "Failed to update video: {}".format(ex)) continue @@ -788,14 +793,15 @@ class VideoUpdater(object): """).format( sql.SQL(", ").join(sql.Identifier(key) for key in UPDATE_JOB_PARAMS) ) - return [UpdateJob(**row) for row in query(self.conn, built_query)] + return list(query(self.conn, built_query)) def mark_done(self, job, updates): """We don't want to set to DONE if the video has been modified *again* since we saw it.""" + updates['state'] = 'DONE' built_query = sql.SQL(""" UPDATE events - SET state = 'DONE', {} + SET {} WHERE state = 'MODIFIED' AND {} """).format( sql.SQL(", ").join( @@ -804,12 +810,13 @@ class VideoUpdater(object): ) for key in updates ), sql.SQL(" AND ").join( - sql.SQL("{} = {}").format(sql.Identifier(key), get_column_placeholder(key)) + # NULL != NULL, so we need "IS NOT DISTINCT FROM" to mean "equal, even if they're null" + sql.SQL("{} IS NOT DISTINCT FROM {}").format(sql.Identifier(key), get_column_placeholder(key)) for key in UPDATE_JOB_PARAMS ) ) updates = {"new_{}".format(key): value for key, value in updates.items()} - return query(self.conn, built_query, **job, **updates).rowcount + return query(self.conn, built_query, **job._asdict(), **updates).rowcount def mark_errored(self, id, error): # We don't overwrite any existing error, it is most likely from another attempt to update diff --git a/cutter/cutter/upload_backends.py b/cutter/cutter/upload_backends.py index b4b2968..bb4de2f 100644 --- a/cutter/cutter/upload_backends.py +++ b/cutter/cutter/upload_backends.py @@ -258,9 +258,10 @@ class Youtube(UploadBackend): def set_thumbnail(self, video_id, thumbnail): resp = self.client.request('POST', - 'https://www.googleapis.com/youtube/v3/thumbnails/set', + 'https://www.googleapis.com/upload/youtube/v3/thumbnails/set', params={'videoId': video_id}, - body=thumbnail, + headers={'Content-Type': 'image/png'}, + data=thumbnail, ) resp.raise_for_status() diff --git a/thrimshim/thrimshim/main.py b/thrimshim/thrimshim/main.py index d671531..ccdc2f8 100644 --- a/thrimshim/thrimshim/main.py +++ b/thrimshim/thrimshim/main.py @@ -210,6 +210,8 @@ def get_row(ident): return value.isoformat() if isinstance(value, datetime.timedelta): return value.total_seconds() + if isinstance(value, memoryview) or isinstance(value, bytes): + return base64.b64encode(bytes(value)).decode() raise TypeError(f"Can't convert object of type {value.__class__.__name__} to JSON: {value}") return json.dumps(response, default=convert) @@ -288,6 +290,18 @@ def update_row(ident, editor=None): for transition in new_row['video_transitions'] ] + # Convert binary fields from base64 and do basic validation of contents + if new_row.get('thumbnail_image') is not None: + if new_row['thumbnail_mode'] != 'CUSTOM': + return 'Can only upload custom image when thumbnail_mode = "CUSTOM"', 400 + try: + new_row['thumbnail_image'] = base64.b64decode(new_row['thumbnail_image']) + except binascii.Error: + return 'thumbnail_image must be valid base64', 400 + # check for PNG file header + if not new_row['thumbnail_image'].startswith(b'\x89PNG\r\n\x1a\n'): + return 'thumbnail_image must be a PNG', 400 + conn = app.db_manager.get_conn() # Check a row with id = ident is in the database built_query = sql.SQL("""