Chipping away at backfiller

I'm going to have to learn to write better commit messages
pull/18/head
Christopher Usher 6 years ago committed by Mike Lang
parent 2fb17fff59
commit 5615c1bdb0

@ -7,20 +7,18 @@
# more frequently, backfill the last couple hours # more frequently, backfill the last couple hours
# (last three hour directories so always at least two hours). # (last three hour directories so always at least two hours).
import glob
import requests import requests
import os import os
hour_fmt = '%Y-%m-%dT%H' hour_fmt = '%Y-%m-%dT%H'
def is_localhost(node):
return None
def get_nodes(): def get_nodes():
# either read a config file or query the database to get the addresses # either read a config file or query the database to get the addresses
# of the other nodes # of the other nodes
# figure out some way that the local machine isn't in the list of returned
# nodes so that
# as a prototype can just hardcode some addresses. # as a prototype can just hardcode some addresses.
@ -31,7 +29,9 @@ def get_nodes():
def list_local_segments(base_dir, stream, variant, hour): def list_local_segments(base_dir, stream, variant, hour):
path = os.path.join(base_dir, stream, variant, hour) path = os.path.join(base_dir, stream, variant, hour)
return [name for name in os.listdir(path) if not name.startswith('.')] local_segments = [name for name in os.listdir(path) if not
name.startswith('.')]
return local_segments
def get_hours(node, stream, variant): def get_hours(node, stream, variant):
@ -51,16 +51,22 @@ def list_remote_segments(node, stream, variant, hour):
#based on _get_segment in downloader/main #based on _get_segment in downloader/main
def get_remote_segment(base_dir, node, stream, variant, hour, missing_segment): def get_remote_segment(base_dir, node, stream, variant, hour, missing_segment):
resp = requests.get('https://{}/segments/{}/{}/{}/{}'.format(node, stream, variant, resp = requests.get('https://{}/segments/{}/{}/{}/{}'.format(node, stream,
hour, missing_segment), stream=True) variant, hour, missing_segment), stream=True)
if resp.status_code != 200:
return False
temp_name = 'temp_backfill'
with open('temp_backfill', 'w') as f: with open(temp_name, 'w') as f:
for chunk in resp.iter_content(8192): for chunk in resp.iter_content(8192):
f.write(chunk) f.write(chunk)
path = os.path.join(base_dir, stream, variant, hour, missing_segment) path = os.path.join(base_dir, stream, variant, hour, missing_segment)
os.rename(temp, segment) os.rename(temp_name, segment)
return True
def back_fill(static_folder, stream, variants, hours=None, nodes=None, def back_fill(static_folder, stream, variants, hours=None, nodes=None,

Loading…
Cancel
Save