diff --git a/common/common/__init__.py b/common/common/__init__.py index 85529ba..f2ff9fa 100644 --- a/common/common/__init__.py +++ b/common/common/__init__.py @@ -7,7 +7,7 @@ import errno import os import random -from .segments import get_best_segments, fast_cut_segments, full_cut_segments, parse_segment_path, SegmentInfo +from .segments import get_best_segments, rough_cut_segments, fast_cut_segments, full_cut_segments, parse_segment_path, SegmentInfo from .stats import timed, PromLogCountsHandler, install_stacksampler diff --git a/common/common/segments.py b/common/common/segments.py index 67b9fc7..34283ae 100644 --- a/common/common/segments.py +++ b/common/common/segments.py @@ -333,7 +333,19 @@ def read_chunks(fileobj, chunk_size=16*1024): yield chunk -@timed('cut', type='fast', normalize=lambda _, segments, start, end: (end - start).total_seconds()) +@timed('cut', cut_type='rough', normalize=lambda _, segments, start, end: (end - start).total_seconds()) +def rough_cut_segments(segments, start, end): + """Yields chunks of a MPEGTS video file covering at least the timestamp range, + likely with a few extra seconds on either side. + This method works by simply concatenating all the segments, without any re-encoding. + """ + for segment in segments: + with open(segment.path) as f: + for chunk in read_chunks(f): + yield chunk + + +@timed('cut', cut_type='fast', normalize=lambda _, segments, start, end: (end - start).total_seconds()) def fast_cut_segments(segments, start, end): """Yields chunks of a MPEGTS video file covering the exact timestamp range. segments should be a list of segments as returned by get_best_segments(). @@ -421,7 +433,7 @@ def feed_input(segments, pipe): @timed('cut', - type=lambda _, segments, start, end, encode_args, stream=False: ("full-streamed" if stream else "full-buffered"), + cut_type=lambda _, segments, start, end, encode_args, stream=False: ("full-streamed" if stream else "full-buffered"), normalize=lambda _, segments, start, end, *a, **k: (end - start).total_seconds(), ) def full_cut_segments(segments, start, end, encode_args, stream=False): diff --git a/common/common/stats.py b/common/common/stats.py index bd3a854..5b1ddc4 100644 --- a/common/common/stats.py +++ b/common/common/stats.py @@ -11,6 +11,10 @@ from monotonic import monotonic import prometheus_client as prom +# need to keep global track of what metrics we've registered +# because we're not allowed to re-register +metrics = {} + def timed(name=None, buckets=[10.**x for x in range(-9, 5)], normalized_buckets=None, @@ -25,6 +29,7 @@ def timed(name=None, if you're using gevent and the wrapped function blocks) and do not include subprocesses. NAME defaults to the wrapped function's name. + NAME must be unique OR have the exact same labels as other timed() calls with that name. Any labels passed in are included. Given label values may be callable, in which case they are passed the input and result from the wrapped function and should return a label value. @@ -86,31 +91,40 @@ def timed(name=None, # can't safely assign to name inside closure, we use a new _name variable instead _name = fn.__name__ if name is None else name - latency = prom.Histogram( - "{}_latency".format(_name), - "Wall clock time taken to execute {}".format(_name), - labels.keys() + ['error'], - buckets=buckets, - ) - cputime = prom.Histogram( - "{}_cputime".format(_name), - "Process-wide consumed CPU time during execution of {}".format(_name), - labels.keys() + ['error', 'type'], - buckets=buckets, - ) - if normalize: - normal_latency = prom.Histogram( - "{}_latency_normalized".format(_name), - "Wall clock time taken to execute {} per unit of work".format(_name), + if name in metrics: + latency, cputime = metrics[name] + else: + latency = prom.Histogram( + "{}_latency".format(_name), + "Wall clock time taken to execute {}".format(_name), labels.keys() + ['error'], - buckets=normalized_buckets, + buckets=buckets, ) - normal_cputime = prom.Histogram( - "{}_cputime_normalized".format(_name), - "Process-wide consumed CPU time during execution of {} per unit of work".format(_name), + cputime = prom.Histogram( + "{}_cputime".format(_name), + "Process-wide consumed CPU time during execution of {}".format(_name), labels.keys() + ['error', 'type'], - buckets=normalized_buckets, + buckets=buckets, ) + metrics[name] = latency, cputime + if normalize: + normname = '{} normalized'.format(name) + if normname in metrics: + normal_latency, normal_cputime = metrics[normname] + else: + normal_latency = prom.Histogram( + "{}_latency_normalized".format(_name), + "Wall clock time taken to execute {} per unit of work".format(_name), + labels.keys() + ['error'], + buckets=normalized_buckets, + ) + normal_cputime = prom.Histogram( + "{}_cputime_normalized".format(_name), + "Process-wide consumed CPU time during execution of {} per unit of work".format(_name), + labels.keys() + ['error', 'type'], + buckets=normalized_buckets, + ) + metrics[normname] = normal_latency, normal_cputime @functools.wraps(fn) def wrapper(*args, **kwargs): diff --git a/restreamer/restreamer/main.py b/restreamer/restreamer/main.py index b74d7b7..8ec8060 100644 --- a/restreamer/restreamer/main.py +++ b/restreamer/restreamer/main.py @@ -13,7 +13,7 @@ import prometheus_client as prom from flask import Flask, url_for, request, abort, Response from gevent.pywsgi import WSGIServer -from common import dateutil, get_best_segments, fast_cut_segments, full_cut_segments, PromLogCountsHandler, install_stacksampler +from common import dateutil, get_best_segments, rough_cut_segments, fast_cut_segments, full_cut_segments, PromLogCountsHandler, install_stacksampler from common.flask_stats import request_stats, after_request import generate_hls @@ -235,8 +235,15 @@ def cut(channel, quality): if any holes are detected, rather than producing a video with missing parts. Set to true by passing "true" (case insensitive). Even if holes are allowed, a 406 may result if the resulting video would be empty. - type: One of "fast" or "full". Default to "fast". - A fast cut is much faster but minor artifacting may be present near the start and end. + type: One of: + "rough": A direct concat, like a fast cut but without any ffmpeg. + It may extend beyond the requested start and end times by a few seconds. + "fast": Very fast but with minor artifacting where the first and last segments join + the other segments. + "mpegts": A full cut to a streamable mpegts format. This consumes signifigant server + resources, so please use sparingly. + "mp4": As mpegts, but encodes as MP4. This format must be buffered to disk before + sending so it's a bit slower. """ start = dateutil.parse_utc_only(request.args['start']) if 'start' in request.args else None end = dateutil.parse_utc_only(request.args['end']) if 'end' in request.args else None @@ -268,14 +275,17 @@ def cut(channel, quality): return "We have no content available within the requested time range.", 406 type = request.args.get('type', 'fast') + if type == 'rough': + return Response(rough_cut_segments(segments, start, end), mimetype='video/MP2T') if type == 'fast': return Response(fast_cut_segments(segments, start, end), mimetype='video/MP2T') - elif type == 'full': - # output as high-quality mpegts, without wasting too much cpu on encoding - encoding_args = ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '0', '-f', 'mpegts'] - return Response(full_cut_segments(segments, start, end, encoding_args, stream=True), mimetype='video/MP2T') + elif type in ('mpegts', 'mp4'): + # encode as high-quality, without wasting too much cpu on encoding + stream, muxer, mimetype = (True, 'mpegts', 'video/MP2T') if type == 'mpegts' else (False, 'mp4', 'video/mp4') + encoding_args = ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '0', '-f', muxer] + return Response(full_cut_segments(segments, start, end, encoding_args, stream=stream), mimetype=mimetype) else: - return "Unknown type {!r}. Must be 'fast' or 'full'.".format(type), 400 + return "Unknown type {!r}".format(type), 400 def main(host='0.0.0.0', port=8000, base_dir='.', backdoor_port=0): diff --git a/thrimbletrimmer/index.html b/thrimbletrimmer/index.html index 159db15..482ab03 100644 --- a/thrimbletrimmer/index.html +++ b/thrimbletrimmer/index.html @@ -101,6 +101,14 @@ Reset Edits Help Ultrawide +
+ Download type: +