diff --git a/backfiller/backfiller/main.py b/backfiller/backfiller/main.py index 701a34c..0e3d20b 100644 --- a/backfiller/backfiller/main.py +++ b/backfiller/backfiller/main.py @@ -330,6 +330,11 @@ class BackfillerWorker(object): except ValueError as e: self.logger.warning('File {} invaid: {}'.format(path, e)) continue + + # Ignore temp segments as they may go away by the time we fetch them. + if segment.type == "temp": + self.logger.debug('Skipping {} as it is a temp segment'.format(path)) + continue # to avoid getting in the downloader's way ignore segments # less than recent_cutoff old diff --git a/common/common/segments.py b/common/common/segments.py index ddb3d75..3fe5b15 100644 --- a/common/common/segments.py +++ b/common/common/segments.py @@ -28,7 +28,7 @@ def unpadded_b64_decode(s): class SegmentInfo( namedtuple('SegmentInfoBase', [ - 'path', 'stream', 'variant', 'start', 'duration', 'is_partial', 'hash' + 'path', 'stream', 'variant', 'start', 'duration', 'type', 'hash' ]) ): """Info parsed from a segment path, including original path. @@ -36,6 +36,9 @@ class SegmentInfo( @property def end(self): return self.start + self.duration + @property + def is_partial(self): + return self.type != "full" def parse_segment_path(path): @@ -64,7 +67,7 @@ def parse_segment_path(path): variant = variant, start = datetime.datetime.strptime("{}:{}".format(hour, time), "%Y-%m-%dT%H:%M:%S.%f"), duration = datetime.timedelta(seconds=float(duration)), - is_partial = type != "full", + type = type, hash = hash, ) except ValueError as e: @@ -219,7 +222,9 @@ def best_segments_by_start(hour): # but is easy enough to do, so we might as well. parsed = (parse_segment_path(os.path.join(hour, name)) for name in segment_paths) for start_time, segments in itertools.groupby(parsed, key=lambda segment: segment.start): - segments = list(segments) + # ignore temp segments as they might go away by the time we want to use them + segments = [segment for segment in segments if segment.type != "temp"] + full_segments = [segment for segment in segments if not segment.is_partial] if full_segments: if len(full_segments) != 1: diff --git a/cutter/cutter/youtube.py b/cutter/cutter/youtube.py index 2b2a23e..0bbe336 100644 --- a/cutter/cutter/youtube.py +++ b/cutter/cutter/youtube.py @@ -26,7 +26,6 @@ class Youtube(object): } resp = self.client.request('POST', 'https://www.googleapis.com/upload/youtube/v3/videos', - headers=self.auth_headers(), params={ 'part': 'snippet,status' if hidden else 'snippet', 'uploadType': 'resumable', @@ -35,7 +34,7 @@ class Youtube(object): ) resp.raise_for_status() upload_url = resp.headers['Location'] - resp = self.client.request('POST', upload_url, headers=self.auth_headers(), data=data) + resp = self.client.request('POST', upload_url, data=data) resp.raise_for_status() return resp.json()['id'] @@ -51,7 +50,6 @@ class Youtube(object): group = ids[i:i+10] resp = self.client.request('GET', 'https://www.googleapis.com/youtube/v3/videos', - headers=self.auth_headers(), params={ 'part': 'id,status', 'id': ','.join(group),