From 3bbe1ed32d353c917f0db6cc9b91f473cf9de5d9 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Wed, 26 Dec 2018 20:07:07 -0800 Subject: [PATCH 1/6] Prefer longer duration on multiple segments --- common/common.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/common/common.py b/common/common.py index 83455ed..17f9b6b 100644 --- a/common/common.py +++ b/common/common.py @@ -240,8 +240,12 @@ def best_segments_by_start(hour): logging.warning("Multiple versions of full segment at start_time {}: {}".format( start_time, ", ".join(map(str, segments)) )) - # we have to pick one, so might as well make it consistent by sorting by path - full_segments.sort(key=lambda segment: segment.path) + # We've observed some cases where the same segment (with the same hash) will be reported + # with different durations (generally at stream end). Prefer the longer duration, + # as this will ensure that if hashes are different we get the most data, and if they + # are the same it should keep holes to a minimum. + # If same duration, we have to pick one, so pick highest-sorting hash just so we're consistent. + full_segments = [max(full_segments, key=lambda segment: (segment.duration, segment.hash))] yield full_segments[0] continue # no full segments, fall back to measuring partials. From 8f5a98a90690595fa35a168558818794913ebd06 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Wed, 26 Dec 2018 21:03:15 -0800 Subject: [PATCH 2/6] restreamer: Don't offer a variant on the master playlist if it's outside requested time range This prevents clients from picking a variant that they then can't play any content for. In general we expect the same content to be available on all variants being captured, but if the set of captured variants changes we still want to handle that gracefully. --- restreamer/restreamer/main.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/restreamer/restreamer/main.py b/restreamer/restreamer/main.py index c5bc873..7f32ad5 100644 --- a/restreamer/restreamer/main.py +++ b/restreamer/restreamer/main.py @@ -1,4 +1,5 @@ +import datetime import errno import functools import json @@ -85,6 +86,17 @@ def list_segments(stream, variant, hour): return json.dumps(listdir(path, error=False)) +def time_range_for_variant(stream, variant): + """Returns earliest and latest times that the given variant has segments for + (up to hour resolution), or 404 if it doesn't exist / is empty.""" + hours = listdir(os.path.join(app.static_folder, stream, variant)) + if not hours: + abort(404) + first, last = min(hours), max(hours) + # note last hour parses to _start_ of that hour, so we add 1h to go to end of that hour + return dateutil.parser.parse(first), dateutil.parser.parse(last) + datetime.timedelta(hours=1) + + @app.route('/playlist/.m3u8') @has_path_args def generate_master_playlist(stream): @@ -93,11 +105,24 @@ def generate_master_playlist(stream): start, end: The time to begin and end the stream at. See generate_media_playlist for details. """ + start = dateutil.parser.parse(request.args['start']) if 'start' in request.args else None + end = dateutil.parser.parse(request.args['end']) if 'end' in request.args else None variants = listdir(os.path.join(app.static_folder, stream)) - playlists = { - variant: url_for('generate_media_playlist', stream=stream, variant=variant, **request.args) - for variant in variants - } + + playlists = {} + for variant in variants: + # If start or end are given, try to restrict offered variants to ones which exist for that + # time range. + if start is not None or end is not None: + first, last = time_range_for_variant(stream, variant) + if start is not None and last < start: + continue # last time for variant is before our start time, don't offer variant + if end is not None and end < first: + continue # our end time is before first time for variant, don't offer variant + playlists[variant] = url_for( + 'generate_media_playlist', stream=stream, variant=variant, **request.args + ) + return generate_hls.generate_master(playlists) From 6fa74608fbf50042467967ad9c561b154d8e669b Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Wed, 26 Dec 2018 21:07:43 -0800 Subject: [PATCH 3/6] common: Improve some docs to note types of things that are ambiguous --- common/common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/common.py b/common/common.py index 17f9b6b..be6f71b 100644 --- a/common/common.py +++ b/common/common.py @@ -73,7 +73,8 @@ class SegmentInfo( 'path', 'stream', 'variant', 'start', 'duration', 'is_partial', 'hash' ]) ): - """Info parsed from a segment path, including original path.""" + """Info parsed from a segment path, including original path. + Note that start time is a datetime and duration is a timedelta, and hash is a decoded binary string.""" @property def end(self): return self.start + self.duration @@ -116,6 +117,7 @@ def parse_segment_path(path): def get_best_segments(hours_path, start, end): """Return a list of the best sequence of non-overlapping segments we have for a given time range. Hours path should be the directory containing hour directories. + Time args start and end should be given as datetime objects. The first segment may start before the time range, and the last may end after it. The returned list contains items that are either: SegmentInfo: a segment From e34f04cf57f1ae788e3547898a1ee82df1d74010 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Wed, 26 Dec 2018 21:29:42 -0800 Subject: [PATCH 4/6] restreamer: Harden generate_media_playlist to handle weird inputs and defaults --- restreamer/restreamer/main.py | 36 +++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/restreamer/restreamer/main.py b/restreamer/restreamer/main.py index 7f32ad5..41adfd0 100644 --- a/restreamer/restreamer/main.py +++ b/restreamer/restreamer/main.py @@ -129,13 +129,37 @@ def generate_master_playlist(stream): @app.route('/playlist//.m3u8') @has_path_args def generate_media_playlist(stream, variant): - #TODO handle no start/end - #TODO error handling of args - # TODO lots of other stuff - start = dateutil.parser.parse(request.args['start']) - end = dateutil.parser.parse(request.args['end']) + """Returns a HLS media playlist for the given stream and variant. + Takes optional params: + start, end: The time to begin and end the stream at. + Must be in ISO 8601 format (ie. yyyy-mm-ddTHH:MM:SS). + If not given, effectively means "infinity", ie. no start means + any time ago, no end means any time in the future. + """ + hours_path = os.path.join(app.static_folder, stream, variant) - segments = get_best_segments(hours_path, start, end) + if not os.path.isdir(hours_path): + abort(404) + + start = dateutil.parser.parse(request.args['start']) if 'start' in request.args else None + end = dateutil.parser.parse(request.args['end']) if 'end' in request.args else None + if start is None or end is None: + # If start or end are not given, use the earliest/latest time available + first, last = time_range_for_variant(stream, variant) + if start is None: + start = first + if end is None: + end = last + + # get_best_segments requires start be before end, special case that as no segments + # (not an error because someone might ask for a specific start, no end, but we ended up with + # end before start because that's the latest time we have) + if start < end: + segments = get_best_segments(hours_path, start, end) + else: + # Note the None to indicate there was a "hole" at both start and end + segments = [None] + return generate_hls.generate_media(segments, os.path.join(app.static_url_path, stream, variant)) From 201959888ab71904a95e61d6c4e3eb28ecef835c Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Wed, 26 Dec 2018 21:44:51 -0800 Subject: [PATCH 5/6] restreamer: More accurate target duration in playlist --- restreamer/restreamer/generate_hls.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/restreamer/restreamer/generate_hls.py b/restreamer/restreamer/generate_hls.py index 8e9d09e..8db0b14 100644 --- a/restreamer/restreamer/generate_hls.py +++ b/restreamer/restreamer/generate_hls.py @@ -1,5 +1,7 @@ +import datetime import os import urllib +from collections import Counter def generate_master(playlists): @@ -22,9 +24,19 @@ def generate_media(segments, base_url): """Generate a media playlist from a list of segments as returned by common.get_best_segments(). Segments are specified as hour/name.ts relative to base_url. """ + # We have to pick a "target duration". in most circumstances almost all segments + # will be of that duration, so we get the most common duration out of all the segments + # and use that. + # If we have no segments, default to 6 seconds. + non_none_segments = [segment for segment in segments if segment is not None] + if non_none_segments: + # Note most_common returns [(value, count)] so we unpack. + ((target_duration, _),) = Counter(segment.duration for segment in non_none_segments).most_common(1) + else: + target_duration = datetime.timedelta(seconds=6) lines = [ "#EXTM3U", - "#EXT-X-TARGETDURATION:6", + "#EXT-X-TARGETDURATION:{:.3f}".format(target_duration.total_seconds()), ] for segment in segments: # TODO handle missing bits, stream endings, other stuff From b4e627f382d6b6c8d73e1ea36134fef4457c7368 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Wed, 26 Dec 2018 22:06:28 -0800 Subject: [PATCH 6/6] restreamer: When generating playlists, include discontinuities, timestamps and endlist This fills out the incomplete playlist generation functionality to handle holes and communicate extra information. See comments for details. --- restreamer/restreamer/generate_hls.py | 36 +++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/restreamer/restreamer/generate_hls.py b/restreamer/restreamer/generate_hls.py index 8db0b14..835daa2 100644 --- a/restreamer/restreamer/generate_hls.py +++ b/restreamer/restreamer/generate_hls.py @@ -24,6 +24,7 @@ def generate_media(segments, base_url): """Generate a media playlist from a list of segments as returned by common.get_best_segments(). Segments are specified as hour/name.ts relative to base_url. """ + # We have to pick a "target duration". in most circumstances almost all segments # will be of that duration, so we get the most common duration out of all the segments # and use that. @@ -34,14 +35,45 @@ def generate_media(segments, base_url): ((target_duration, _),) = Counter(segment.duration for segment in non_none_segments).most_common(1) else: target_duration = datetime.timedelta(seconds=6) + lines = [ "#EXTM3U", "#EXT-X-TARGETDURATION:{:.3f}".format(target_duration.total_seconds()), ] + + # Note and remove any trailing None from the segment list - this indicates there is a hole + # at the end, which means we should mark the stream as incomplete but not include a discontinuity. + if segments and segments[-1] is None: + incomplete = True + segments = segments[:-1] + else: + incomplete = False + + # Remove any leading None from the segment list - this indicates there is a hole at the start, + # which isn't actually meaningful in any way to us. + # Note that in the case of segments = [None], we already removed it when we removed the trailing + # None, and segments is now []. This is fine. + if segments and segments[0] is None: + segments = segments[1:] + for segment in segments: - # TODO handle missing bits, stream endings, other stuff - if segment is not None: + if segment is None: + # Discontinuity. Adding this tag tells the client that we've missed something + # and it should start decoding fresh on the next segment. This is required when + # someone stops/starts a stream and a good idea if we're missing a segment in a + # continuous stream. + lines.append("#EXT-X-DISCONTINUITY") + else: + # Each segment has two prefixes: timestamp and duration. + # This tells the client exactly what time the segment represents, which is important + # for the editor since it needs to describe cut points in these times. path = '/'.join(segment.path.split('/')[-2:]) + lines.append("#EXT-X-PROGRAM-DATE-TIME:{}".format(segment.start.strftime("%Y-%m-%dT%H:%M:%S.%fZ"))) lines.append("#EXTINF:{:.3f},live".format(segment.duration.total_seconds())) lines.append(urllib.quote(os.path.join(base_url, path))) + + # If stream is complete, add an ENDLIST marker to show this. + if not incomplete: + lines.append("#EXT-X-ENDLIST") + return "\n".join(lines) + '\n'