Merge pull request #139 from ekimekim/mike/restreamer/cut-types

restreamer: Add more options for fetching cuts

common/common/__init__.py

@@ -7,7 +7,7 @@ import errno
 import os
 import random
 
-from .segments import get_best_segments, fast_cut_segments, full_cut_segments, parse_segment_path, SegmentInfo
+from .segments import get_best_segments, rough_cut_segments, fast_cut_segments, full_cut_segments, parse_segment_path, SegmentInfo
 from .stats import timed, PromLogCountsHandler, install_stacksampler
 
 

common/common/segments.py

@@ -333,7 +333,19 @@ def read_chunks(fileobj, chunk_size=16*1024):
 		yield chunk
 
 
-@timed('cut', type='fast', normalize=lambda _, segments, start, end: (end - start).total_seconds())
+@timed('cut', cut_type='rough', normalize=lambda _, segments, start, end: (end - start).total_seconds())
+def rough_cut_segments(segments, start, end):
+	"""Yields chunks of a MPEGTS video file covering at least the timestamp range,
+	likely with a few extra seconds on either side.
+	This method works by simply concatenating all the segments, without any re-encoding.
+	"""
+	for segment in segments:
+		with open(segment.path) as f:
+			for chunk in read_chunks(f):
+				yield chunk
+
+
+@timed('cut', cut_type='fast', normalize=lambda _, segments, start, end: (end - start).total_seconds())
 def fast_cut_segments(segments, start, end):
 	"""Yields chunks of a MPEGTS video file covering the exact timestamp range.
 	segments should be a list of segments as returned by get_best_segments().
@@ -421,7 +433,7 @@ def feed_input(segments, pipe):
 
 
 @timed('cut',
-	type=lambda _, segments, start, end, encode_args, stream=False: ("full-streamed" if stream else "full-buffered"),
+	cut_type=lambda _, segments, start, end, encode_args, stream=False: ("full-streamed" if stream else "full-buffered"),
 	normalize=lambda _, segments, start, end, *a, **k: (end - start).total_seconds(),
 )
 def full_cut_segments(segments, start, end, encode_args, stream=False):

common/common/stats.py

@@ -11,6 +11,10 @@ from monotonic import monotonic
 import prometheus_client as prom
 
 
+# need to keep global track of what metrics we've registered
+# because we're not allowed to re-register
+metrics = {}
+
 
 def timed(name=None,
 	buckets=[10.**x for x in range(-9, 5)], normalized_buckets=None,
@@ -25,6 +29,7 @@ def timed(name=None,
 	if you're using gevent and the wrapped function blocks) and do not include subprocesses.
 
 	NAME defaults to the wrapped function's name.
+	NAME must be unique OR have the exact same labels as other timed() calls with that name.
 
 	Any labels passed in are included. Given label values may be callable, in which case
 	they are passed the input and result from the wrapped function and should return a label value.
@@ -86,31 +91,40 @@ def timed(name=None,
 		# can't safely assign to name inside closure, we use a new _name variable instead
 		_name = fn.__name__ if name is None else name
 
-		latency = prom.Histogram(
+		if name in metrics:
-			"{}_latency".format(_name),
+			latency, cputime = metrics[name]
-			"Wall clock time taken to execute {}".format(_name),
+		else:
-			labels.keys() + ['error'],
+			latency = prom.Histogram(
-			buckets=buckets,
+				"{}_latency".format(_name),
-		)
+				"Wall clock time taken to execute {}".format(_name),
-		cputime = prom.Histogram(
-			"{}_cputime".format(_name),
-			"Process-wide consumed CPU time during execution of {}".format(_name),
-			labels.keys() + ['error', 'type'],
-			buckets=buckets,
-		)
-		if normalize:
-			normal_latency = prom.Histogram(
-				"{}_latency_normalized".format(_name),
-				"Wall clock time taken to execute {} per unit of work".format(_name),
 				labels.keys() + ['error'],
-				buckets=normalized_buckets,
+				buckets=buckets,
 			)
-			normal_cputime = prom.Histogram(
+			cputime = prom.Histogram(
-				"{}_cputime_normalized".format(_name),
+				"{}_cputime".format(_name),
-				"Process-wide consumed CPU time during execution of {} per unit of work".format(_name),
+				"Process-wide consumed CPU time during execution of {}".format(_name),
 				labels.keys() + ['error', 'type'],
-				buckets=normalized_buckets,
+				buckets=buckets,
 			)
+			metrics[name] = latency, cputime
+		if normalize:
+			normname = '{} normalized'.format(name)
+			if normname in metrics:
+				normal_latency, normal_cputime = metrics[normname]
+			else:
+				normal_latency = prom.Histogram(
+					"{}_latency_normalized".format(_name),
+					"Wall clock time taken to execute {} per unit of work".format(_name),
+					labels.keys() + ['error'],
+					buckets=normalized_buckets,
+				)
+				normal_cputime = prom.Histogram(
+					"{}_cputime_normalized".format(_name),
+					"Process-wide consumed CPU time during execution of {} per unit of work".format(_name),
+					labels.keys() + ['error', 'type'],
+					buckets=normalized_buckets,
+				)
+				metrics[normname] = normal_latency, normal_cputime
 
 		@functools.wraps(fn)
 		def wrapper(*args, **kwargs):

restreamer/restreamer/main.py

@@ -13,7 +13,7 @@ import prometheus_client as prom
 from flask import Flask, url_for, request, abort, Response
 from gevent.pywsgi import WSGIServer
 
-from common import dateutil, get_best_segments, fast_cut_segments, full_cut_segments, PromLogCountsHandler, install_stacksampler
+from common import dateutil, get_best_segments, rough_cut_segments, fast_cut_segments, full_cut_segments, PromLogCountsHandler, install_stacksampler
 from common.flask_stats import request_stats, after_request
 
 import generate_hls
@@ -235,8 +235,15 @@ def cut(channel, quality):
 			if any holes are detected, rather than producing a video with missing parts.
 			Set to true by passing "true" (case insensitive).
 			Even if holes are allowed, a 406 may result if the resulting video would be empty.
-		type: One of "fast" or "full". Default to "fast".
+		type: One of:
-			A fast cut is much faster but minor artifacting may be present near the start and end.
+			"rough": A direct concat, like a fast cut but without any ffmpeg.
+				It may extend beyond the requested start and end times by a few seconds.
+			"fast": Very fast but with minor artifacting where the first and last segments join
+				the other segments.
+			"mpegts": A full cut to a streamable mpegts format. This consumes signifigant server
+				resources, so please use sparingly.
+			"mp4": As mpegts, but encodes as MP4. This format must be buffered to disk before
+				sending so it's a bit slower.
 	"""
 	start = dateutil.parse_utc_only(request.args['start']) if 'start' in request.args else None
 	end = dateutil.parse_utc_only(request.args['end']) if 'end' in request.args else None
@@ -268,14 +275,17 @@ def cut(channel, quality):
 		return "We have no content available within the requested time range.", 406
 
 	type = request.args.get('type', 'fast')
+	if type == 'rough':
+		return Response(rough_cut_segments(segments, start, end), mimetype='video/MP2T')
 	if type == 'fast':
 		return Response(fast_cut_segments(segments, start, end), mimetype='video/MP2T')
-	elif type == 'full':
+	elif type in ('mpegts', 'mp4'):
-		# output as high-quality mpegts, without wasting too much cpu on encoding
+		# encode as high-quality, without wasting too much cpu on encoding
-		encoding_args = ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '0', '-f', 'mpegts']
+		stream, muxer, mimetype = (True, 'mpegts', 'video/MP2T') if type == 'mpegts' else (False, 'mp4', 'video/mp4')
-		return Response(full_cut_segments(segments, start, end, encoding_args, stream=True), mimetype='video/MP2T')
+		encoding_args = ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '0', '-f', muxer]
+		return Response(full_cut_segments(segments, start, end, encoding_args, stream=stream), mimetype=mimetype)
 	else:
-		return "Unknown type {!r}. Must be 'fast' or 'full'.".format(type), 400
+		return "Unknown type {!r}".format(type), 400
 
 
 def main(host='0.0.0.0', port=8000, base_dir='.', backdoor_port=0):

thrimbletrimmer/index.html

@@ -101,6 +101,14 @@
             <a id="ResetButton" href="JavaScript:thrimbletrimmerResetLink();">Reset Edits</a>
             <a id="HelpButton" style="float:right;" href="JavaScript:toggleHiddenPane('HelpPane');">Help</a>
             <a id="UltrawideButton" style="float:right;margin-right:10px;" href="JavaScript:toggleUltrawide();">Ultrawide</a>
+			<br/>
+			Download type:
+			<select id="DownloadType">
+				<option value="rough" selected>Rough (fastest, pads start and end by a few seconds)</option>
+				<option value="fast">Fast (may have artifacts a few seconds in from start and end)</option>
+				<option value="mpegts">MPEG-TS (slow, consumes server resources)</option>
+				<option value="mp4">MP4 (slower, consumes server resources, output is MP4 file)</option>
+			</select>
         </div>
         <div id="ManualLinkPane" style="display:none">
             <input id="ManualLink" /> <input type="button" id="ManualButton" onclick="thrimbletrimmerManualLink()" value="Set Link" />

thrimbletrimmer/scripts/IO.js

@@ -274,6 +274,11 @@ thrimbletrimmerDownload = function(isEditor) {
         "?" + buildQuery({
             start: range.start,
             end: range.end,
+			// In non-editor, always use rough cut. They don't have the edit controls to do
+			// fine time selection anyway.
+			type: (isEditor) ? (
+				document.getElementById('DownloadType').options[document.getElementById('DownloadType').options.selectedIndex].value
+			) : "rough",
             // Always allow holes in non-editor, accidentially including holes isn't important
             allow_holes: (isEditor) ? String(document.getElementById('AllowHoles').checked) : "true",
         });