working!

segment_coverage/Dockerfile

@@ -1,7 +1,12 @@
 FROM alpine:3.7
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras libpng
+# freetype-dev and libpng-dev are required for matplotlib
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras freetype-dev libpng-dev build-base libstdc++
+#need to install these manually 
+RUN ln -s /usr/include/locale.h /usr/include/xlocale.h \
+	&& pip install numpy \
+	&& pip install matplotlib
 
 # Install common lib first as it changes less
 COPY common /tmp/common

segment_coverage/segment_coverage/main.py

@@ -7,18 +7,19 @@ import signal
 import argh
 import gevent.backdoor
 import matplotlib.pyplot as plt
+from matplotlib import colors
 import numpy as np
 import prometheus_client as prom
 
 import common
 
 
+HOUR_FMT = '%Y-%m-%dT%H'
 
 class CoverageChecker(object):
 	"""Checks the segment coverage for a given channel in a a given directoy."""
 
 	CHECK_INTERVAL = 60 #seconds between checking coverage
-	CHECK_INTERVAL = 6 #seconds between checking coverage
 
 	def __init__(self, channel, qualities, base_dir):
 		"""Constructor for CoverageChecker.
@@ -41,6 +42,8 @@ class CoverageChecker(object):
 		"""Loop over available hours for each quality, checking segment coverage."""
 		self.logger.info('Starting')
 
+		os.mkdir('/coverage_maps')
+
 		while not self.stopping.is_set():
 
 			for quality in self.qualities:
@@ -59,6 +62,11 @@ class CoverageChecker(object):
 					# but more complicated to capture more detailed metrics
 					hour_path = os.path.join(self.base_dir, self.channel, quality, hour)
 					segment_names = [name for name in os.listdir(hour_path) if not name.startswith('.')]
+
+					if not segment_names:
+						self.logger.info('{}/{} is empty'.format(quality, hour))
+						continue
+
 					segment_names.sort()
 					parsed = (common.parse_segment_path(os.path.join(hour_path, name)) for name in segment_names)
 
@@ -118,6 +126,8 @@ class CoverageChecker(object):
 						self.logger.debug(best_segment.path.split('/')[-1])
 						best_segments.append(best_segment)
 
+						# now update coverage, overlaps and holes
+
 						if previous is None:
 							coverage += best_segment.duration
 							editable_coverage += best_segment.duration
@@ -148,6 +158,87 @@ class CoverageChecker(object):
 					end = best_segments[-1].start + best_segments[-1].duration
 					hole_duration = end - start - coverage
 					editable_hole_duration = end - start - editable_coverage
+
+					pixel_starts = np.arange(3600)
+					pixel_ends = np.arange(1, 3601)
+
+					runtime_mask = np.ones(3600, dtype=np.bool_)
+					coverage_mask = np.ones(3600, dtype=np.bool_)
+					partial_mask = np.zeros(3600, dtype=np.bool_)
+
+					hour_start = datetime.datetime.strptime(hour, HOUR_FMT)
+					hour_end = hour_start + datetime.timedelta(hours=1)
+					# handle the case when there is a hole between the last segment of the previous hour and the first of this
+					if previous_hour_segments:
+						last_segment = previous_hour_segments[-1]
+						if best_segments[0].start > last_segment.start + last_segment.duration:
+							holes.append((hour_start, start))
+							hole_duration += start - hour_start
+							editable_holes.append((hour_start, start))
+							editable_hole_duration += start - hour_start							
+					# if it is the first hour, instead ignore anything before the first segment
+					else:
+						start_seconds = start.minute * 60 + start.second
+						self.logger.info(start_seconds)
+						runtime_mask = runtime_mask & (start_seconds <= pixel_starts)
+
+					#handle the case when there is a hole between the last segment and the end of the hour
+					if hour != hours[-1]:
+						if end < hour_end:
+							holes.append((end, hour_end))
+							hole_duration += hour_end - end
+							editable_holes.append((end, hour_end))
+							editable_hole_duration += hour_end - end							
+					# if it is the last hour, ignore anything after the last segment
+					else:
+						end_seconds = end.minute * 60 + end.second
+						self.logger.info(end_seconds)
+						runtime_mask = runtime_mask & (end_seconds > pixel_starts)
+
+					for hole in holes:
+						hole_start = np.floor((hole[0] - hour_start).seconds)
+						hole_end = np.ceil((hole[1] - hour_start).seconds)
+						self.logger.info('hole {} {}'.format(hole_start, hole_end))
+						coverage_mask = coverage_mask & ((pixel_starts < hole_start) | (pixel_ends > hole_end))
+
+					for partial in only_partials:
+						partial_start = np.floor((partial[0] - hour_start).seconds)
+						partial_end = np.ceil((partial[1] - hour_start).seconds)
+						self.logger.info('partial {} {}'.format(partial_start, partial_end))
+						partial_mask = partial_mask | ((pixel_starts >= partial_start) & (pixel_ends <= partial_end))
+
+					runtime_mask = runtime_mask.reshape((60, 60))
+					coverage_mask = coverage_mask.reshape((60, 60))
+					partial_mask = partial_mask.reshape((60, 60))
+
+					colours = np.ones((60, 60, 3))
+
+					colours[runtime_mask & coverage_mask] = colors.to_rgb('tab:green')
+					colours[runtime_mask & partial_mask] = colors.to_rgb('tab:orange')
+					colours[runtime_mask & ~coverage_mask] = colors.to_rgb('tab:red')
+
+					fig = plt.figure(figsize=(3, 3))
+					fig.add_axes([0, 0, 1, 1])
+					plt.imshow(colours)
+					plt.axis('off')
+					plt.savefig('/coverage_maps/{}.png'.format(hour), dpi=60)
+
+
+#					plt.figure()
+#					plt.imshow(runtime_mask.astype('d'))
+#					plt.colorbar()
+#					plt.savefig('/coverage_maps/runtime_{}.png'.format(hour))
+#
+#					plt.figure()
+#					plt.imshow(coverage_mask.astype('d'))
+#					plt.colorbar()
+#					plt.savefig('/coverage_maps/coverage_{}.png'.format(hour))
+#
+#					plt.figure()
+#					plt.imshow(partial_mask.astype('d'))
+#					plt.colorbar()
+#					plt.savefig('/coverage_maps/partial_{}.png'.format(hour))						
+
 					self.logger.info('{}/{}: Start: {} End: {} ({} s)'.format(quality, hour, start, end, (end - start).seconds))
 					self.logger.info('{}/{}: {} full segments totalling {} s'.format(quality, hour, full_segment_count, full_segment_duration.seconds))
 					self.logger.info('{}/{}: {} full segment repeats totalling {} s'.format(quality, hour, full_repeats, full_repeat_duration.seconds))
@@ -160,9 +251,6 @@ class CoverageChecker(object):
 					self.logger.info('{}/{}: {} overlapping segments, {} s overlapping'.format(quality, hour, overlap_count, overlap_duration.seconds))
 
 
-					self.logger.info(holes)
-					self.logger.info(only_partials)
-
 					previous_hour_segments = best_segments
 
 			self.stopping.wait(common.jitter(self.CHECK_INTERVAL))