diff --git a/backfiller/backfiller/main.py b/backfiller/backfiller/main.py index dadbef5..f4ecc85 100644 --- a/backfiller/backfiller/main.py +++ b/backfiller/backfiller/main.py @@ -147,15 +147,19 @@ def backfill_node(base_dir, node, stream, variants, hours=None, start=None, seconds to prioritise letting the downloader grab these segments.""" if hours is None: - hours = list_remote_hours(node, stream, variant) + # gather all available hours from all variants and take the union + hours = set().union(*[ + list_remote_hours(node, stream, variant) + for variant in variants + ]) elif is_iterable(hours): - pass # hours already in desired format + hours = list(hours) # coerce to list so it can be sorted else: n_hours = hours if n_hours < 1: raise ValueError('Number of hours has to be 1 or greater') now = datetime.datetime.utcnow() - hours = [(now - i * timedelta(hours=1)).strftime(HOUR_FMT) for i in range(n_hours)] + hours = [(now - i * datetime.timedelta(hours=1)).strftime(HOUR_FMT) for i in range(n_hours)] if start is not None: hours = [hour for hour in hours if hour >= start] @@ -168,9 +172,9 @@ def backfill_node(base_dir, node, stream, variants, hours=None, start=None, if order == 'random': random.shuffle(hours) elif order == 'forward': - sort(hours) + hours.sort() elif order == 'reverse': - sort(hours, reverse=True) + hours.sort(reverse=True) for variant in variants: diff --git a/common/common.py b/common/common.py index acb6b08..ba8f48d 100644 --- a/common/common.py +++ b/common/common.py @@ -8,6 +8,7 @@ import errno import itertools import logging import os +import random import sys from collections import namedtuple diff --git a/downloader/downloader/main.py b/downloader/downloader/main.py index d06cc50..0a068a7 100644 --- a/downloader/downloader/main.py +++ b/downloader/downloader/main.py @@ -4,7 +4,6 @@ import errno import hashlib import logging import os -import random import signal import sys import uuid