From e0fc1eaf08c58578b1f8b694f4b1010d635d10bd Mon Sep 17 00:00:00 2001 From: Christopher Usher Date: Mon, 21 Oct 2024 18:03:26 -0700 Subject: [PATCH] Refactor get_thumbnail and other fixes and improvments --- common/common/images.py | 46 +++++++---- cutter/cutter/main.py | 21 +---- postgres/schema.sql | 16 +++- restreamer/restreamer/main.py | 71 +++++++++------- thrimshim/thrimshim/main.py | 149 +++++++++++++++------------------- 5 files changed, 149 insertions(+), 154 deletions(-) diff --git a/common/common/images.py b/common/common/images.py index 32f2ecc..36e4d77 100644 --- a/common/common/images.py +++ b/common/common/images.py @@ -4,25 +4,43 @@ from io import BytesIO from PIL import Image +from common import database -""" -A thumbnail template consists of a byte stream representation of a PNG template image along with two 4-tuples [left x, top y, right x, bottom y] describing the crop and location of the frame. -To create a thumbnail, the input frame is first cropped to the bounds of the "crop" box, -then resized to the size of the "location" box, then pasted underneath the template image at that -location within the template image. +def get_template(dbmanager, name, crop=None, location=None): + """Fetch the thumbnail template and any missing parameters from the database""" + with dbmanager.get_conn() as conn: + query = """ + SELECT image, crop, location FROM templates WHERE name = %s + """ + results = database.query(conn, query, name) + row = results.fetchone() + if row is None: + raise ValueError('Template {} not found'.format(name)) + row = row._asdict() + if not crop: + crop = row['crop'] + if not location: + location = row['location'] -For example -crop = (50, 100, 1870, 980) -location = (320, 180, 1600, 900) -would crop the input frame from (50, 100) to (1870, 980), resize it to 720x1280, -and place it at (320, 180). + return row['image'], crop, location -If the original frame and the template differ in size, the frame is first resized to the template. -This allows you to work with a consistent coordinate system regardless of the input frame size. -""" def compose_thumbnail_template(template_data, frame_data, crop, location): + """ + A thumbnail template consists of a byte stream representation of a PNG template image along with two 4-tuples [left x, top y, right x, bottom y] describing the crop and location of the frame. + + To create a thumbnail, the input frame is first cropped to the bounds of the "crop" box, then resized to the size of the "location" box, then pasted underneath the template image at that +location within the template image. + + For example + crop = (50, 100, 1870, 980) + location = (320, 180, 1600, 900) + would crop the input frame from (50, 100) to (1870, 980), resize it to 720x1280, and place it at (320, 180). + + If the original frame and the template differ in size, the frame is first resized to the template. + This allows you to work with a consistent coordinate system regardless of the input frame size. + """ # PIL can't load an image from a byte string directly, we have to pretend to be a file template = Image.open(BytesIO(template_data)) @@ -66,4 +84,4 @@ def cli(template, frame, crop, location): if __name__ == '__main__': import argh - argh.dispatch_command(cli) + argh.dispatch_command(cli) diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index aff1366..9de8067 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -18,7 +18,7 @@ from psycopg2 import sql import common from common.database import DBManager, query, get_column_placeholder from common.segments import get_best_segments, archive_cut_segments, fast_cut_segments, full_cut_segments, smart_cut_segments, extract_frame, ContainsHoles, get_best_segments_for_frame -from common.images import compose_thumbnail_template +from common.images import compose_thumbnail_template, get_template from common.stats import timed from .upload_backends import Youtube, Local, LocalArchive, UploadError @@ -84,25 +84,6 @@ CutJob = namedtuple('CutJob', [ # params which map directly from DB columns ] + CUT_JOB_PARAMS) -def get_template(dbmanager, name, crop, location): - """Fetch the thumbnail template and any missing parameters from the database""" - with dbmanager.get_conn() as conn: - query = """ - SELECT image, crop, location FROM templates WHERE name = %s - """ - results = database.query(conn, query, name) - row = results.fetchone() - if row is None: - raise ValueError('Template {} not found'.format(name)) - row = row._asdict() - if not crop: - crop = row['crop'] - if not location: - location = row['location'] - - return row['image'], crop, location - - def get_duration(job): """Get total video duration of a job, in seconds""" # Due to ranges and transitions, this is actually non-trivial to calculate. diff --git a/postgres/schema.sql b/postgres/schema.sql index e951405..97d4977 100644 --- a/postgres/schema.sql +++ b/postgres/schema.sql @@ -89,8 +89,14 @@ CREATE TABLE events ( OR thumbnail_mode = 'NONE' OR thumbnail_last_written IS NOT NULL ), - thumbnail_crop INTEGER[], -- left, upper, right, and lower pixel coordinates to crop the selected frame - thumbnail_location INTEGER[], -- left, top, right, bottom pixel coordinates to position the cropped frame + thumbnail_crop INTEGER[] CHECK ( + cardinality(thumbnail_crop) = 4 + OR thumbnail_crop IS NULL + ), -- left, upper, right, and lower pixel coordinates to crop the selected frame + thumbnail_location INTEGER[] CHECK ( + cardinality(thumbnail_crop) = 4 + OR thumbnail_crop IS NULL + ), -- left, top, right, bottom pixel coordinates to position the cropped frame state event_state NOT NULL DEFAULT 'UNEDITED', uploader TEXT CHECK (state IN ('UNEDITED', 'EDITED', 'DONE') OR uploader IS NOT NULL), @@ -185,6 +191,8 @@ CREATE TABLE templates ( image BYTEA NOT NULL, description TEXT NOT NULL DEFAULT '', attribution TEXT NOT NULL DEFAULT '', - crop INTEGER[] NOT NULL, - location INTEGER[] NOT NULL + crop INTEGER[] NOT NULL CHECK ( + cardinality(thumbnail_crop) = 4), + location INTEGER[] NOT NULL CHECK ( + cardinality(thumbnail_crop) = 4), ); diff --git a/restreamer/restreamer/main.py b/restreamer/restreamer/main.py index df34160..eaa2e31 100644 --- a/restreamer/restreamer/main.py +++ b/restreamer/restreamer/main.py @@ -8,7 +8,6 @@ import os import subprocess from uuid import uuid4 -import base64 import gevent import gevent.backdoor import gevent.event @@ -18,7 +17,7 @@ from gevent.pywsgi import WSGIServer from common import database, dateutil, get_best_segments, rough_cut_segments, fast_cut_segments, full_cut_segments, PromLogCountsHandler, install_stacksampler, serve_with_graceful_shutdown from common.flask_stats import request_stats, after_request -from common.images import compose_thumbnail_template +from common.images import compose_thumbnail_template, get_template from common.segments import smart_cut_segments, feed_input, render_segments_waveform, extract_frame, list_segment_files, get_best_segments_for_frame from common.chat import get_batch_file_range, merge_messages from common.cached_iterator import CachedIterator @@ -480,43 +479,53 @@ def get_frame(channel, quality): @app.route('/thumbnail//.png') @request_stats -def get_thumbnail(channel, quality): +@has_path_args +def get_thumbnail_named_template(channel, quality): """ Returns a PNG image which is a preview of how a thumbnail will be generated. + Params: + timestamp: Required. The frame to use as the thumbnail image. + Must be in ISO 8601 format (ie. yyyy-mm-ddTHH:MM:SS) and UTC. + template: Required. The template name to use. + Must be one of the template names as returned by GET /thrimshim/templates + crop: Left, upper, right, and lower pixel coordinates to crop the selected frame. + Default is to use the crop in the database. + location: Left, top, right, bottom pixel coordinates to position the cropped frame. + Default is to use the location in the databse. """ + crop = request.args.get('crop', None) + location = request.args.get('location', None) + if app.db_manager is None: + return 'A database connection is required to generate thumbnails', 501 + try: + template, crop, location = get_template(app.db_manager, request.args['template'], crop, location) + except ValueError: + return 'Template {} not found'.format(request.args['template']), 404 + return get_thumbnail(channel, quality, request.args['timestamp'], template, crop, location) - template_params = request.json - if template_params['image']: - template = base64.b64decode(template_params['image']) +@app.route('/thumbnail//.png', methods=['POST']) +@request_stats +@has_path_args +def get_thumbnail_uploaded_template(channel, quality): + """ + Returns a PNG image which is a preview of how a thumbnail will be generated. + Params: + timestamp: Required. The frame to use as the thumbnail image. + Must be in ISO 8601 format (ie. yyyy-mm-ddTHH:MM:SS) and UTC. + crop: Required. Left, upper, right, and lower pixel coordinates to crop the selected frame. + location: Required. Left, top, right, bottom pixel coordinates to position the cropped frame. + """ + template = request.body + return get_thumbnail(channel, quality, request.args['timestamp'], template, request.args['crop'], request.args['location']) - crop = template_params['crop'] - location = template_params['location'] - else: - if app.db_manager is None: - return 'A database connection is required to generate thumbnails', 501 - - with app.db_manager.get_conn() as conn: - query = """ - SELECT image, crop, location FROM templates WHERE name = %s - """ - results = database.query(conn, query, template_params['name']) - row = results.fetchone() - if row is None: - return 'Template {} not found'.format(template_params['name']), 404 - row = row._asdict() - template = row['image'] - if not template_params['crop']: - crop = row['crop'] - else: - crop = template_params['crop'] - if not template_params['location']: - location = row['location'] - else: - location = template_params['location'] +def get_thumbnail(channel, quality, timestamp, template, crop, location): + """ + Generates a PNG thumbnail by combining a frame at timestamp with the template. + """ - timestamp = dateutil.parse_utc_only(template_params['timestamp']) + timestamp = dateutil.parse_utc_only(timestamp) hours_path = os.path.join(app.static_folder, channel, quality) if not os.path.isdir(hours_path): diff --git a/thrimshim/thrimshim/main.py b/thrimshim/thrimshim/main.py index 99b911d..9a90d84 100644 --- a/thrimshim/thrimshim/main.py +++ b/thrimshim/thrimshim/main.py @@ -46,77 +46,66 @@ def cors(app): return app(environ, _start_response) return handle -def authenticate_artist(f): - """"Authenticate a token against the database to authenticate an artist + +def check_user(request, role): + """"Authenticate a token against the database to authenticate a user. Reference: https://developers.google.com/identity/sign-in/web/backend-auth""" - + try: + userToken = request.json['token'] + except (KeyError, TypeError): + return 'User token required', 401 + # check whether token is valid + try: + idinfo = google.oauth2.id_token.verify_oauth2_token(userToken, google.auth.transport.requests.Request(), None) + if idinfo['iss'] not in ['accounts.google.com', 'https://accounts.google.com']: + raise ValueError('Wrong issuer.') + except ValueError: + return 'Invalid token. Access denied.', 403 + + # check whether user is in the database + email = idinfo['email'].lower() + conn = app.db_manager.get_conn() + query = """ + SELECT email, %(role)s + FROM roles + WHERE lower(email) = %(email)s AND %(role) + """ + results = database.query(conn, query, email=email, role=role) + row = results.fetchone() + if row is None: + return 'Unknown user. Access denied.', 403 + return email, 200 + + +def authenticate_artist(f): + """"Authenticate an artist.""" @wraps(f) def artist_auth_wrapper(*args, **kwargs): if app.no_authentication: return f(*args, editor='NOT_AUTH', **kwargs) - try: - userToken = flask.request.json['token'] - except (KeyError, TypeError): - return 'User token required', 401 - # check whether token is valid - try: - idinfo = google.oauth2.id_token.verify_oauth2_token(userToken, google.auth.transport.requests.Request(), None) - if idinfo['iss'] not in ['accounts.google.com', 'https://accounts.google.com']: - raise ValueError('Wrong issuer.') - except ValueError: - return 'Invalid token. Access denied.', 403 - - # check whether user is in the database - email = idinfo['email'].lower() - conn = app.db_manager.get_conn() - results = database.query(conn, """ - SELECT email, artist - FROM roles - WHERE lower(email) = %s AND artist""", email) - row = results.fetchone() - if row is None: - return 'Unknown user. Access denied.', 403 + message, code = check_user(flask.request, 'artist') + if code != 200: + return message, code - return f(*args, editor=email, **kwargs) + return f(*args, artist=message, **kwargs) return artist_auth_wrapper def authenticate_editor(f): - """Authenticate a token against the database to authenticate an editor. - - Reference: https://developers.google.com/identity/sign-in/web/backend-auth""" + """Authenticate an editor.""" @wraps(f) def editor_auth_wrapper(*args, **kwargs): if app.no_authentication: return f(*args, editor='NOT_AUTH', **kwargs) - try: - userToken = flask.request.json['token'] - except (KeyError, TypeError): - return 'User token required', 401 - # check whether token is valid - try: - idinfo = google.oauth2.id_token.verify_oauth2_token(userToken, google.auth.transport.requests.Request(), None) - if idinfo['iss'] not in ['accounts.google.com', 'https://accounts.google.com']: - raise ValueError('Wrong issuer.') - except ValueError: - return 'Invalid token. Access denied.', 403 - - # check whether user is in the database - email = idinfo['email'].lower() - conn = app.db_manager.get_conn() - results = database.query(conn, """ - SELECT email, editor - FROM roles - WHERE lower(email) = %s AND editor""", email) - row = results.fetchone() - if row is None: - return 'Unknown user. Access denied.', 403 + message, code = check_user(flask.request, 'editor') + if code != 200: + return message, code - return f(*args, editor=email, **kwargs) + return f(*args, editor=message, **kwargs) return editor_auth_wrapper @@ -515,7 +504,7 @@ def update_row(ident, editor=None): _write_audit_log(conn, ident, "update-row", editor, old_row, new_row) logging.info('Row {} updated to state {}'.format(ident, new_row['state'])) - return '' + return '', 201 @app.route('/thrimshim/manual-link/', methods=['POST']) @@ -651,18 +640,13 @@ def get_template_metadata(name): return 'Template {} not found'.format(name), 404 return json.dumps(row._asdict()) - -@app.route('/thrimshim/add-template', methods=['POST']) -@request_stats -def add_template(artist=None): - """Add a template to the database""" - new_template = flask.request.json +def validate_template(new_template): columns = ['name', 'image', 'description', 'attribution', 'crop', 'location'] #check for missing fields missing = set(columns) - set(new_template) if missing: - return 'Fields missing in JSON: {}'.format(', '.join(missing)), 400 + return None, 'Fields missing in JSON: {}'.format(', '.join(missing)), 400 # delete any extras extras = set(new_template) - set(columns) for extra in extras: @@ -672,13 +656,24 @@ def add_template(artist=None): try: new_template['image'] = base64.b64decode(new_template['image']) except binascii.Error: - return 'Template image must be valid base64', 400 + return None, 'Template image must be valid base64', 400 # check for PNG file header if not new_template['thumbnail_image'].startswith(b'\x89PNG\r\n\x1a\n'): - return 'Template image must be a PNG', 400 + return None, 'Template image must be a PNG', 400 - with app.db_manager.get_conn() as conn: + return columns, new_template, 200 + + +@app.route('/thrimshim/add-template', methods=['POST']) +@request_stats +def add_template(artist=None): + """Add a template to the database""" + columns, message, code = validate_template(flask.request.json) + if code != 200: + return message, code + new_template = message + with app.db_manager.get_conn() as conn: #check if name is already in the database query = sql.SQL(""" SELECT name FROM events WHERE name = %s @@ -696,32 +691,16 @@ def add_template(artist=None): ) database.query(conn, query, **new_template) - return '' + return '', 201 @app.route('/thrimshim/update-template/', methods=['POST']) @request_stats def update_template(name, artist=None): """Update a template in the database""" - new_template = flask.request.json - - columns = ['name', 'image', 'description', 'attribution', 'crop', 'location'] - #check for missing fields - missing = set(columns) - set(new_template) - if missing: - return 'Fields missing in JSON: {}'.format(', '.join(missing)), 400 - # delete any extras - extras = set(new_template) - set(columns) - for extra in extras: - del new_template[extra] - - #convert and validate template image - try: - new_template['image'] = base64.b64decode(new_template['image']) - except binascii.Error: - return 'Template image must be valid base64', 400 - # check for PNG file header - if not new_template['thumbnail_image'].startswith(b'\x89PNG\r\n\x1a\n'): - return 'Template image must be a PNG', 400 + columns, message, code = validate_template(flask.request.json) + if code != 200: + return message, code + new_template = message with app.db_manage.get_conn() as conn: #check if template is in database @@ -769,7 +748,7 @@ def get_thumbnail(ident): if event['thumbnail_mode'] != 'NONE' and event['thumbnail_image']: return flask.Response(event['thumbnail_image'], mimetype='image/png') else: - return '', 200 + return '', 404