diff --git a/segment_coverage/Dockerfile b/segment_coverage/Dockerfile index f6af41c..d7924af 100644 --- a/segment_coverage/Dockerfile +++ b/segment_coverage/Dockerfile @@ -1,7 +1,12 @@ FROM alpine:3.7 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras libpng +# freetype-dev and libpng-dev are required for matplotlib +RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras freetype-dev libpng-dev build-base libstdc++ +#need to install these manually +RUN ln -s /usr/include/locale.h /usr/include/xlocale.h \ + && pip install numpy \ + && pip install matplotlib # Install common lib first as it changes less COPY common /tmp/common diff --git a/segment_coverage/segment_coverage/main.py b/segment_coverage/segment_coverage/main.py index de07454..58df487 100644 --- a/segment_coverage/segment_coverage/main.py +++ b/segment_coverage/segment_coverage/main.py @@ -7,18 +7,19 @@ import signal import argh import gevent.backdoor import matplotlib.pyplot as plt +from matplotlib import colors import numpy as np import prometheus_client as prom import common +HOUR_FMT = '%Y-%m-%dT%H' class CoverageChecker(object): """Checks the segment coverage for a given channel in a a given directoy.""" CHECK_INTERVAL = 60 #seconds between checking coverage - CHECK_INTERVAL = 6 #seconds between checking coverage def __init__(self, channel, qualities, base_dir): """Constructor for CoverageChecker. @@ -41,6 +42,8 @@ class CoverageChecker(object): """Loop over available hours for each quality, checking segment coverage.""" self.logger.info('Starting') + os.mkdir('/coverage_maps') + while not self.stopping.is_set(): for quality in self.qualities: @@ -59,6 +62,11 @@ class CoverageChecker(object): # but more complicated to capture more detailed metrics hour_path = os.path.join(self.base_dir, self.channel, quality, hour) segment_names = [name for name in os.listdir(hour_path) if not name.startswith('.')] + + if not segment_names: + self.logger.info('{}/{} is empty'.format(quality, hour)) + continue + segment_names.sort() parsed = (common.parse_segment_path(os.path.join(hour_path, name)) for name in segment_names) @@ -118,6 +126,8 @@ class CoverageChecker(object): self.logger.debug(best_segment.path.split('/')[-1]) best_segments.append(best_segment) + # now update coverage, overlaps and holes + if previous is None: coverage += best_segment.duration editable_coverage += best_segment.duration @@ -148,6 +158,87 @@ class CoverageChecker(object): end = best_segments[-1].start + best_segments[-1].duration hole_duration = end - start - coverage editable_hole_duration = end - start - editable_coverage + + pixel_starts = np.arange(3600) + pixel_ends = np.arange(1, 3601) + + runtime_mask = np.ones(3600, dtype=np.bool_) + coverage_mask = np.ones(3600, dtype=np.bool_) + partial_mask = np.zeros(3600, dtype=np.bool_) + + hour_start = datetime.datetime.strptime(hour, HOUR_FMT) + hour_end = hour_start + datetime.timedelta(hours=1) + # handle the case when there is a hole between the last segment of the previous hour and the first of this + if previous_hour_segments: + last_segment = previous_hour_segments[-1] + if best_segments[0].start > last_segment.start + last_segment.duration: + holes.append((hour_start, start)) + hole_duration += start - hour_start + editable_holes.append((hour_start, start)) + editable_hole_duration += start - hour_start + # if it is the first hour, instead ignore anything before the first segment + else: + start_seconds = start.minute * 60 + start.second + self.logger.info(start_seconds) + runtime_mask = runtime_mask & (start_seconds <= pixel_starts) + + #handle the case when there is a hole between the last segment and the end of the hour + if hour != hours[-1]: + if end < hour_end: + holes.append((end, hour_end)) + hole_duration += hour_end - end + editable_holes.append((end, hour_end)) + editable_hole_duration += hour_end - end + # if it is the last hour, ignore anything after the last segment + else: + end_seconds = end.minute * 60 + end.second + self.logger.info(end_seconds) + runtime_mask = runtime_mask & (end_seconds > pixel_starts) + + for hole in holes: + hole_start = np.floor((hole[0] - hour_start).seconds) + hole_end = np.ceil((hole[1] - hour_start).seconds) + self.logger.info('hole {} {}'.format(hole_start, hole_end)) + coverage_mask = coverage_mask & ((pixel_starts < hole_start) | (pixel_ends > hole_end)) + + for partial in only_partials: + partial_start = np.floor((partial[0] - hour_start).seconds) + partial_end = np.ceil((partial[1] - hour_start).seconds) + self.logger.info('partial {} {}'.format(partial_start, partial_end)) + partial_mask = partial_mask | ((pixel_starts >= partial_start) & (pixel_ends <= partial_end)) + + runtime_mask = runtime_mask.reshape((60, 60)) + coverage_mask = coverage_mask.reshape((60, 60)) + partial_mask = partial_mask.reshape((60, 60)) + + colours = np.ones((60, 60, 3)) + + colours[runtime_mask & coverage_mask] = colors.to_rgb('tab:green') + colours[runtime_mask & partial_mask] = colors.to_rgb('tab:orange') + colours[runtime_mask & ~coverage_mask] = colors.to_rgb('tab:red') + + fig = plt.figure(figsize=(3, 3)) + fig.add_axes([0, 0, 1, 1]) + plt.imshow(colours) + plt.axis('off') + plt.savefig('/coverage_maps/{}.png'.format(hour), dpi=60) + + +# plt.figure() +# plt.imshow(runtime_mask.astype('d')) +# plt.colorbar() +# plt.savefig('/coverage_maps/runtime_{}.png'.format(hour)) +# +# plt.figure() +# plt.imshow(coverage_mask.astype('d')) +# plt.colorbar() +# plt.savefig('/coverage_maps/coverage_{}.png'.format(hour)) +# +# plt.figure() +# plt.imshow(partial_mask.astype('d')) +# plt.colorbar() +# plt.savefig('/coverage_maps/partial_{}.png'.format(hour)) + self.logger.info('{}/{}: Start: {} End: {} ({} s)'.format(quality, hour, start, end, (end - start).seconds)) self.logger.info('{}/{}: {} full segments totalling {} s'.format(quality, hour, full_segment_count, full_segment_duration.seconds)) self.logger.info('{}/{}: {} full segment repeats totalling {} s'.format(quality, hour, full_repeats, full_repeat_duration.seconds)) @@ -160,9 +251,6 @@ class CoverageChecker(object): self.logger.info('{}/{}: {} overlapping segments, {} s overlapping'.format(quality, hour, overlap_count, overlap_duration.seconds)) - self.logger.info(holes) - self.logger.info(only_partials) - previous_hour_segments = best_segments self.stopping.wait(common.jitter(self.CHECK_INTERVAL))