From 48d11045d47ac688c6e3e3b1930d43ff27f7ddca Mon Sep 17 00:00:00 2001 From: Christopher Usher Date: Tue, 1 Jan 2019 16:26:25 -0800 Subject: [PATCH] Change to backfiller.main to backfill the last 3 hours on start up before doing a full backfill --- backfiller/backfiller/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backfiller/backfiller/main.py b/backfiller/backfiller/main.py index 32b2466..bc7df4d 100644 --- a/backfiller/backfiller/main.py +++ b/backfiller/backfiller/main.py @@ -233,7 +233,7 @@ def backfill_node(base_dir, node, stream, variants, hours=None, start=None, def main(base_dir, stream, variants, fill_wait=5, full_fill_wait=180, sleep_time=1): """Prototype backfiller service. - Do a full backfill of stream/variants from all nodes initially. Then every sleep_time minutes check to see if more than fill_wait minutes have passed since the last backfill. If so do a backfill of the last 3 hours. Also check whether it has been more than full_fill_wait minutes since the last full backfill; if so, do a full backfill.""" + Do a backfill of the last 3 hours from stream/variants from all nodes initially before doing a full backfill from all nodes. Then every sleep_time minutes check to see if more than fill_wait minutes have passed since the last backfill. If so do a backfill of the last 3 hours. Also check whether it has been more than full_fill_wait minutes since the last full backfill; if so, do a full backfill.""" # TODO replace this with a more robust event based service and backfill from multiple nodes in parallel # stretch goal: provide an interface to trigger backfills manually # stretch goal: use the backfiller to monitor the restreamer @@ -241,6 +241,7 @@ def main(base_dir, stream, variants, fill_wait=5, full_fill_wait=180, sleep_time fill_start = datetime.datetime.now() full_fill_start = fill_start + backfill(base_dir, stream, variants) backfill(base_dir, stream, variants, order='random') # I'm sure there is a module that does this in a more robust way