From bae039977b8ea28e3c54a97e82ff0459d701b60a Mon Sep 17 00:00:00 2001 From: Christopher Usher Date: Tue, 1 Jan 2019 20:22:14 -0800 Subject: [PATCH] trying getting the backfiller to actually start --- backfiller/Dockerfile | 2 +- backfiller/backfiller/main.py | 2 +- backfiller/setup.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backfiller/Dockerfile b/backfiller/Dockerfile index c2f603c..c7a58cf 100644 --- a/backfiller/Dockerfile +++ b/backfiller/Dockerfile @@ -7,7 +7,7 @@ COPY common /tmp/common RUN pip install /tmp/common && rm -r /tmp/common # Install actual application -COPY downloader /tmp/backfiller +COPY backfiller /tmp/backfiller RUN pip install /tmp/backfiller && rm -r /tmp/backfiller ENTRYPOINT ["python2", "-m", "backfiller", "--base-dir", "/mnt"] diff --git a/backfiller/backfiller/main.py b/backfiller/backfiller/main.py index bc7df4d..0a0b88a 100644 --- a/backfiller/backfiller/main.py +++ b/backfiller/backfiller/main.py @@ -230,7 +230,7 @@ def backfill_node(base_dir, node, stream, variants, hours=None, start=None, logging.info('Finished backfilling from {}'.format(node)) -def main(base_dir, stream, variants, fill_wait=5, full_fill_wait=180, sleep_time=1): +def main(base_dir='/mnt', stream='desertbus', variants=['source'], fill_wait=5, full_fill_wait=180, sleep_time=1): """Prototype backfiller service. Do a backfill of the last 3 hours from stream/variants from all nodes initially before doing a full backfill from all nodes. Then every sleep_time minutes check to see if more than fill_wait minutes have passed since the last backfill. If so do a backfill of the last 3 hours. Also check whether it has been more than full_fill_wait minutes since the last full backfill; if so, do a full backfill.""" diff --git a/backfiller/setup.py b/backfiller/setup.py index ee0fa28..1b4ee32 100644 --- a/backfiller/setup.py +++ b/backfiller/setup.py @@ -5,6 +5,8 @@ setup( version = "0.0.0", packages = find_packages(), install_requires = [ + "argh", + "gevent", "requests", "wubloader-common", ],