From 557ddddc31c54f11a68b9178dfef5bd196902485 Mon Sep 17 00:00:00 2001 From: Chris Usher Date: Mon, 30 Sep 2019 00:12:16 +0100 Subject: [PATCH] better logging for the backfiller --- backfiller/backfiller/main.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/backfiller/backfiller/main.py b/backfiller/backfiller/main.py index 412b791..f2a23f1 100644 --- a/backfiller/backfiller/main.py +++ b/backfiller/backfiller/main.py @@ -67,8 +67,8 @@ def list_remote_segments(node, channel, quality, hour, timeout=TIMEOUT): return common.encode_strings(resp.json()) -def get_remote_segment(base_dir, node, channel, quality, hour, missing_segment, - timeout=TIMEOUT): +def get_remote_segment(base_dir, node, channel, quality, hour, missing_segment, + logger, timeout=TIMEOUT): """Get a segment from a node. Fetches channel/quality/hour/missing_segment from node and puts it in @@ -107,7 +107,7 @@ def get_remote_segment(base_dir, node, channel, quality, hour, missing_segment, logging.debug('Saving completed segment {} as {}'.format(temp_path, path)) common.rename(temp_path, path) segments_backfilled.labels(remote=node, channel=channel, quality=quality, hour=hour).inc() - + logger.info('Segment {}/{}/{} backfilled'.format(quality, hour, missing_segment)) def list_hours(node, channel, qualities, start=None): """Return a list of all available hours from a node. @@ -270,6 +270,7 @@ class BackfillerManager(object): for row in results: nodes[row.name] = row.url nodes.pop(self.localhost, None) + self.logger.info('Nodes fetched: {}'.format(nodes.keys())) return nodes.values() class BackfillerWorker(object): @@ -317,7 +318,7 @@ class BackfillerWorker(object): for hour in hours: - self.logger.debug('Backfilling {}/{}'.format(quality, hour)) + self.logger.info('Backfilling {}/{}'.format(quality, hour)) local_segments = set(list_local_segments(self.base_dir, self.channel, quality, hour)) remote_segments = set(list_remote_segments(self.node, self.channel, quality, hour)) @@ -358,7 +359,7 @@ class BackfillerWorker(object): # start segment as soon as a pool slot opens up, then track it in workers workers.append(pool.spawn( get_remote_segment, - self.base_dir, self.node, self.channel, quality, hour, missing_segment + self.base_dir, self.node, self.channel, quality, hour, missing_segment, self.logger )) # verify that all the workers succeeded. if any failed, raise the exception from @@ -375,7 +376,9 @@ class BackfillerWorker(object): while not self.stopping.is_set(): try: + self.logger.info('Starting backfill') self.backfill(list_hours(self.node, self.channel, self.qualities, self.start)) + self.logger.info('Backfill complete') failures = 0 #reset failure count on a successful backfill if not self.run_once: self.stopping.wait(common.jitter(self.WAIT_INTERVAL))