From 46b7c7a3b69529ff81fc36253d415fd2462ef4e2 Mon Sep 17 00:00:00 2001 From: Christopher Usher Date: Fri, 18 Oct 2019 02:23:45 +0100 Subject: [PATCH] new plotting --- segment_coverage/segment_coverage/main.py | 146 +++++++++++----------- 1 file changed, 75 insertions(+), 71 deletions(-) diff --git a/segment_coverage/segment_coverage/main.py b/segment_coverage/segment_coverage/main.py index 324c448..42b97d5 100644 --- a/segment_coverage/segment_coverage/main.py +++ b/segment_coverage/segment_coverage/main.py @@ -6,20 +6,13 @@ import signal import argh import gevent.backdoor -import matplotlib.pyplot as plt -from matplotlib import colors +import matplotlib import numpy as np import prometheus_client as prom import common -total_segment_count_gauge = prom.Gauge( - 'total_segment_count', - 'Total number of segments in an hour', - ['channel', 'quality', 'hour'], -) - full_segment_count_gauge = prom.Gauge( 'full_segment_count', 'Number of full segments in an hour', @@ -32,12 +25,6 @@ partial_segment_count_gauge = prom.Gauge( ['channel', 'quality', 'hour'], ) -total_segment_duration_gauge = prom.Gauge( - 'total_segment_count', - 'Total segment duration in an hour', - ['channel', 'quality', 'hour'], -) - full_segment_duration_gauge = prom.Gauge( 'full_segment_count', 'Full segment duration in an hour', @@ -122,6 +109,56 @@ class CoverageChecker(object): self.logger.info('Stopping') self.stopping.set() + def create_coverage_map(self, quality, all_hour_holes, all_hour_partials): + """Create a PNG show segment coverage for the last week.""" + + latest_hour = datetime.datetime.strptime(max(all_hour_holes.keys()), HOUR_FMT) + hours = [latest_hour - datetime.timedelta(hours=i) for i in range(167, -1, -1)] + + pixel_starts = np.arange(0, 3600, 2) + pixel_ends = np.arange(2, 3601, 2) + + coverage_mask = np.zeros(168 * 1800, dtype=np.bool_) + partial_mask = np.zeros(168 * 1800, dtype=np.bool_) + for i in range(len(hours)): + hour = hours[i] + if hour not in all_hour_holes: + continue + + else: + hour_str = hour.strftime(HOUR_FMT) + + hour_coverage = np.ones(1800, dtype=np.bool_) + hour_partial = np.zeros(1800, dtype=np.bool_) + + for hole in all_hour_holes[hour_str]: + hole_start = np.floor((hole[0] - hour).seconds) + hole_end = np.ceil((hole[1] - hour).seconds) + hour_coverage = hour_coverage & ((pixel_starts < hole_start) | (pixel_ends > hole_end)) + + for partial in all_hour_partials[hour_str]: + partial_start = np.floor((partial[0] - hour).seconds) + partial_end = np.ceil((partial[1] - hour).seconds) + hour_partial = hour_partial | ((pixel_starts >= partial_start) & (pixel_ends <= partial_end)) + + coverage_mask[i * 1800:(i + 1) * 1800] = hour_coverage + partial_mask[i * 1800:(i + 1) * 1800] = hour_partial + + columns = 300 + rows = coverage_mask.size / columns + + coverage_mask = coverage_mask.reshape((rows, columns)).T + partial_mask = partial_mask.reshape((rows, columns)).T + + colours = np.ones((columns, rows, 3)) + colours[coverage_mask] = matplotlib.colors.to_rgb('tab:blue') + colours[coverage_mask & partial_mask] = matplotlib.colors.to_rgb('tab:orange') + + final_path = os.path.join(self.basedir, 'coverage-maps', '{}_{}_coverage.png'.format(self.channel, quality)) + temp_path = final_path.replace('_coverage', '_temp') + common.ensure_directory(temp_path) + matplotlib.image.imsave(temp_path, colours) + os.rename(temp_path, final_path) def run(self): """Loop over available hours for each quality, checking segment coverage.""" @@ -133,11 +170,12 @@ class CoverageChecker(object): if self.stopping.is_set(): break - coverage_map_dir = os.path.join('/', 'coverage', self.channel, quality) path = os.path.join(self.base_dir, self.channel, quality) hours = [name for name in os.listdir(path) if not name.startswith('.')] hours.sort() previous_hour_segments = None + all_hour_holes = {} + all_hour_partials = {} for hour in hours: if self.stopping.is_set(): break @@ -246,13 +284,6 @@ class CoverageChecker(object): hole_duration = end - start - coverage editable_hole_duration = end - start - editable_coverage - pixel_starts = np.arange(3600) - pixel_ends = np.arange(1, 3601) - - runtime_mask = np.ones(3600, dtype=np.bool_) - coverage_mask = np.ones(3600, dtype=np.bool_) - partial_mask = np.zeros(3600, dtype=np.bool_) - hour_start = datetime.datetime.strptime(hour, HOUR_FMT) hour_end = hour_start + datetime.timedelta(hours=1) # handle the case when there is a hole between the last segment of the previous hour and the first of this @@ -262,56 +293,16 @@ class CoverageChecker(object): holes.append((hour_start, start)) hole_duration += start - hour_start editable_holes.append((hour_start, start)) - editable_hole_duration += start - hour_start - # if it is the first hour, instead ignore anything before the first segment - else: - start_seconds = start.minute * 60 + start.second - runtime_mask = runtime_mask & (start_seconds <= pixel_starts) - - #handle the case when there is a hole between the last segment and the end of the hour - if hour != hours[-1]: - if end < hour_end: - holes.append((end, hour_end)) - hole_duration += hour_end - end - editable_holes.append((end, hour_end)) - editable_hole_duration += hour_end - end - # if it is the last hour, ignore anything after the last segment - else: - end_seconds = end.minute * 60 + end.second - runtime_mask = runtime_mask & (end_seconds > pixel_starts) - - for hole in holes: - hole_start = np.floor((hole[0] - hour_start).seconds) - hole_end = np.ceil((hole[1] - hour_start).seconds) - coverage_mask = coverage_mask & ((pixel_starts < hole_start) | (pixel_ends > hole_end)) - - for partial in only_partials: - partial_start = np.floor((partial[0] - hour_start).seconds) - partial_end = np.ceil((partial[1] - hour_start).seconds) - partial_mask = partial_mask | ((pixel_starts >= partial_start) & (pixel_ends <= partial_end)) - - runtime_mask = runtime_mask.reshape((60, 60)) - coverage_mask = coverage_mask.reshape((60, 60)) - partial_mask = partial_mask.reshape((60, 60)) - - colours = np.ones((60, 60, 3)) - - colours[runtime_mask & coverage_mask] = colors.to_rgb('tab:green') - colours[runtime_mask & partial_mask] = colors.to_rgb('tab:orange') - colours[runtime_mask & ~coverage_mask] = colors.to_rgb('tab:red') - - fig = plt.figure(figsize=(3, 3)) - fig.add_axes([0, 0, 1, 1]) - plt.imshow(colours) - plt.axis('off') - - tmp_path = os.path.join(coverage_map_dir, 'tmp.png') - common.ensure_directory(tmp_path) - final_path = os.path.join(coverage_map_dir, '{}.png'.format(hour)) - plt.savefig(tmp_path, dpi=60) - os.rename(tmp_path, final_path) - + editable_hole_duration += start - hour_start + + #handle the case when there is a hole between the last segment and the end of the hour if not the last hour + if hour != hours[-1] and end < hour_end: + holes.append((end, hour_end)) + hole_duration += hour_end - end + editable_holes.append((end, hour_end)) + editable_hole_duration += hour_end - end + #put all of the guages here self.logger.info('{}/{}: Start: {} End: {} ({} s)'.format(quality, hour, start, end, (end - start).seconds)) self.logger.info('{}/{}: {} full segments totalling {} s'.format(quality, hour, full_segment_count, full_segment_duration.seconds)) @@ -323,8 +314,21 @@ class CoverageChecker(object): self.logger.info('{}/{}: {} editable holes totalling {} s '.format(quality, hour, len(editable_holes), editable_hole_duration.seconds)) self.logger.info('Checking {}/{} complete'.format(quality, hour)) + # add holes for the start and end hours for the coverage map + # do this after updating gauges and logging as these aren't likely real holes, just the start and end of the stream. + if previous_hour_segments is None: + holes.append((hour_start, start)) + if hour == hours[-1]: + holes.append((end, hour_end)) + + + all_hour_holes[hour] = holes + all_hour_partials[hour] = only_partials + previous_hour_segments = best_segments + self.create_coverage_map(quality, all_hour_holes, all_hour_partials) + self.stopping.wait(common.jitter(self.CHECK_INTERVAL))