From 5e7904dab3dc682023bc552d543ee67e1c1e4dc3 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Tue, 24 Oct 2023 00:06:26 +1100 Subject: [PATCH] wip: archive cut --- common/common/segments.py | 61 ++++++++++++++++++++++++++++++++++++++- cutter/cutter/main.py | 18 ++++++------ 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/common/common/segments.py b/common/common/segments.py index 83712e1..68bfeb5 100644 --- a/common/common/segments.py +++ b/common/common/segments.py @@ -13,6 +13,7 @@ import shutil from collections import namedtuple from contextlib import closing from tempfile import TemporaryFile +from uuid import uuid4 import gevent from gevent import subprocess @@ -590,7 +591,7 @@ def feed_input(segments, pipe): pipe.close() -@timed('cut', +@timed('cut_range', cut_type=lambda _, segments, start, end, encode_args, stream=False: ("full-streamed" if stream else "full-buffered"), normalize=lambda _, segments, start, end, *a, **k: (end - start).total_seconds(), ) @@ -649,6 +650,64 @@ def full_cut_segments(segments, start, end, encode_args, stream=False): pass +@timed('cut', cut_type='archive', normalize=lambda ret, sr, ranges: range_total(ranges)) +def archive_cut_segments(segment_ranges, ranges, tempdir): + """ + Archive cuts are special in a few ways. + Like a rough cut, they do not take explicit start/end times but instead + use the entire segment range. + Like a full cut, they are passed entirely through ffmpeg. + They explicitly use ffmpeg arguments to copy the video without re-encoding, + but are placed into an MKV container. + They are split at each discontinuity into seperate videos. + Finally, because the files are expected to be very large and non-streamable, + instead of streaming the data back to the caller, we return a list of temporary filenames + which the caller should then do something with (probably either read then delete, or rename). + """ + # don't re-encode anything, just put it into an MKV container + encode_args = ["-c", "copy", "-f", "mkv"] + # We treat multiple segment ranges as having an explicit discontinuity between them. + # So we apply split_contiguous() to each range, then flatten. + contiguous_ranges = [] + for segments in segment_ranges: + contiguous_ranges += list(split_contiguous(segments)) + for segments in contiguous_ranges: + ffmpeg = None + input_feeder = None + tempfile_name = os.path.join(tempdir, "archive-temp-{}.mkv".format(uuid4())) + try: + tempfile = open(tempfile_name, "wb") + + ffmpeg = ffmpeg_cut_stdin(tempfile, None, None, encode_args) + input_feeder = gevent.spawn(feed_input, segments, ffmpeg.stdin) + + # since we've now handed off the tempfile fd to ffmpeg, close ours + tempfile.close() + + # check if any errors occurred in input writing, or if ffmpeg exited non-success. + if ffmpeg.wait() != 0: + raise Exception("Error while streaming cut: ffmpeg exited {}".format(ffmpeg.returncode)) + input_feeder.get() # re-raise any errors from feed_input() + except: + # if something goes wrong, try to clean up ignoring errors + if input_feeder is not None: + input_feeder.kill() + if ffmpeg is not None and ffmpeg.poll() is None: + for action in (ffmpeg.kill, ffmpeg.stdin.close, ffmpeg.stdout.close): + try: + action() + except (OSError, IOError): + pass + try: + os.remove(tempfile_name) + except (OSError, IOError): + pass + raise + else: + # Success, inform caller of tempfile. It's now their responsibility to delete. + yield tempfile + + @timed('waveform') def render_segments_waveform(segments, size=(1024, 128), scale='sqrt', color='#000000'): """ diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index 5de70c3..784425a 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -396,23 +396,23 @@ class Cutter(object): nonlocal upload_finished try: - if upload_backend.encoding_settings in ("fast", "smart"): - self.logger.debug("No encoding settings, using fast cut") + if upload_backend.encoding_settings in ("fast", "smart", "archive"): + self.logger.debug(f"Using {upload_backend.encoding_settings} cut") if any(transition is not None for transition in job.video_transitions): raise ValueError("Fast cuts do not support complex transitions") cut_fn = { "fast": fast_cut_segments, "smart": smart_cut_segments, + # Note archive cuts return a list of filenames instead of data chunks. + # We assume the upload location expects this. + # We use segments_path as a tempdir path under the assumption that: + # a) it has plenty of space + # b) for a Local upload location, it will be on the same filesystem as the + # final desired path. + "archive": lambda sr, vr: archive_cut_segments(sr, vr, self.segments_path), }[upload_backend.encoding_settings] cut = cut_fn(job.segment_ranges, job.video_ranges) - elif upload_backend.encoding_settings in ("rough", "split"): - # A rough cut copies the segments byte-for-byte with no processing. - # A split cut is a rough cut where the video is split into contiguous ranges - # seperated by discontinuities. We communicate these discontinuities to the uploader - # by inserting a None into the stream of chunks. It is expected a split-cut-capable upload - # location will detect these Nones and do some special behaviour (eg. making a seperate video). - else: self.logger.debug("Using encoding settings for {} cut: {}".format( "streamable" if upload_backend.encoding_streamable else "non-streamable",