trying getting the backfiller to actually start

pull/24/head
Christopher Usher 6 years ago
parent 1fcd9b5b36
commit bae039977b

@ -7,7 +7,7 @@ COPY common /tmp/common
RUN pip install /tmp/common && rm -r /tmp/common RUN pip install /tmp/common && rm -r /tmp/common
# Install actual application # Install actual application
COPY downloader /tmp/backfiller COPY backfiller /tmp/backfiller
RUN pip install /tmp/backfiller && rm -r /tmp/backfiller RUN pip install /tmp/backfiller && rm -r /tmp/backfiller
ENTRYPOINT ["python2", "-m", "backfiller", "--base-dir", "/mnt"] ENTRYPOINT ["python2", "-m", "backfiller", "--base-dir", "/mnt"]

@ -230,7 +230,7 @@ def backfill_node(base_dir, node, stream, variants, hours=None, start=None,
logging.info('Finished backfilling from {}'.format(node)) logging.info('Finished backfilling from {}'.format(node))
def main(base_dir, stream, variants, fill_wait=5, full_fill_wait=180, sleep_time=1): def main(base_dir='/mnt', stream='desertbus', variants=['source'], fill_wait=5, full_fill_wait=180, sleep_time=1):
"""Prototype backfiller service. """Prototype backfiller service.
Do a backfill of the last 3 hours from stream/variants from all nodes initially before doing a full backfill from all nodes. Then every sleep_time minutes check to see if more than fill_wait minutes have passed since the last backfill. If so do a backfill of the last 3 hours. Also check whether it has been more than full_fill_wait minutes since the last full backfill; if so, do a full backfill.""" Do a backfill of the last 3 hours from stream/variants from all nodes initially before doing a full backfill from all nodes. Then every sleep_time minutes check to see if more than fill_wait minutes have passed since the last backfill. If so do a backfill of the last 3 hours. Also check whether it has been more than full_fill_wait minutes since the last full backfill; if so, do a full backfill."""

@ -5,6 +5,8 @@ setup(
version = "0.0.0", version = "0.0.0",
packages = find_packages(), packages = find_packages(),
install_requires = [ install_requires = [
"argh",
"gevent",
"requests", "requests",
"wubloader-common", "wubloader-common",
], ],

Loading…
Cancel
Save