henine
/
wubloader
mirror of https://github.com/ekimekim/wubloader
1
0
Fork 0
Code Issues Releases Wiki Activity

working!

Browse Source
pull/128/head
Christopher Usher 5 years ago
parent ac72f775c9
commit 20a8a214d6
2 changed files with 98 additions and 5 deletions
Show Stats Download Patch File Download Diff File

7
segment_coverage/Dockerfile
Unescape Escape View File

@ -1,7 +1,12 @@
FROM alpine:3.7
# dependencies needed for compiling c extensions
# also busybox-extras for telnet for easier use of backdoor
RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras libpng
# freetype-dev and libpng-dev are required for matplotlib
RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras freetype-dev libpng-dev build-base libstdc++
#need to install these manually
RUN ln -s /usr/include/locale.h /usr/include/xlocale.h \
&& pip install numpy \
&& pip install matplotlib
# Install common lib first as it changes less
COPY common /tmp/common

96
segment_coverage/segment_coverage/main.py
Unescape Escape View File

@ -7,18 +7,19 @@ import signal
import argh
import gevent.backdoor
import matplotlib.pyplot as plt
from matplotlib import colors
import numpy as np
import prometheus_client as prom
import common
HOUR_FMT = '%Y-%m-%dT%H'
class CoverageChecker(object):
"""Checks the segment coverage for a given channel in a a given directoy."""
CHECK_INTERVAL = 60 #seconds between checking coverage
CHECK_INTERVAL = 6 #seconds between checking coverage
def __init__(self, channel, qualities, base_dir):
"""Constructor for CoverageChecker.
@ -41,6 +42,8 @@ class CoverageChecker(object):
"""Loop over available hours for each quality, checking segment coverage."""
self.logger.info('Starting')
os.mkdir('/coverage_maps')
while not self.stopping.is_set():
for quality in self.qualities:
@ -59,6 +62,11 @@ class CoverageChecker(object):
# but more complicated to capture more detailed metrics
hour_path = os.path.join(self.base_dir, self.channel, quality, hour)
segment_names = [name for name in os.listdir(hour_path) if not name.startswith('.')]
if not segment_names:
self.logger.info('{}/{} is empty'.format(quality, hour))
continue
segment_names.sort()
parsed = (common.parse_segment_path(os.path.join(hour_path, name)) for name in segment_names)
@ -118,6 +126,8 @@ class CoverageChecker(object):
self.logger.debug(best_segment.path.split('/')[-1])
best_segments.append(best_segment)
# now update coverage, overlaps and holes
if previous is None:
coverage += best_segment.duration
editable_coverage += best_segment.duration
@ -148,6 +158,87 @@ class CoverageChecker(object):
end = best_segments[-1].start + best_segments[-1].duration
hole_duration = end - start - coverage
editable_hole_duration = end - start - editable_coverage
pixel_starts = np.arange(3600)
pixel_ends = np.arange(1, 3601)
runtime_mask = np.ones(3600, dtype=np.bool_)
coverage_mask = np.ones(3600, dtype=np.bool_)
partial_mask = np.zeros(3600, dtype=np.bool_)
hour_start = datetime.datetime.strptime(hour, HOUR_FMT)
hour_end = hour_start + datetime.timedelta(hours=1)
# handle the case when there is a hole between the last segment of the previous hour and the first of this
if previous_hour_segments:
last_segment = previous_hour_segments[-1]
if best_segments[0].start > last_segment.start + last_segment.duration:
holes.append((hour_start, start))
hole_duration += start - hour_start
editable_holes.append((hour_start, start))
editable_hole_duration += start - hour_start
# if it is the first hour, instead ignore anything before the first segment
else:
start_seconds = start.minute * 60 + start.second
self.logger.info(start_seconds)
runtime_mask = runtime_mask & (start_seconds <= pixel_starts)
#handle the case when there is a hole between the last segment and the end of the hour
if hour != hours[-1]:
if end < hour_end:
holes.append((end, hour_end))
hole_duration += hour_end - end
editable_holes.append((end, hour_end))
editable_hole_duration += hour_end - end
# if it is the last hour, ignore anything after the last segment
else:
end_seconds = end.minute * 60 + end.second
self.logger.info(end_seconds)
runtime_mask = runtime_mask & (end_seconds > pixel_starts)
for hole in holes:
hole_start = np.floor((hole[0] - hour_start).seconds)
hole_end = np.ceil((hole[1] - hour_start).seconds)
self.logger.info('hole {} {}'.format(hole_start, hole_end))
coverage_mask = coverage_mask & ((pixel_starts < hole_start) | (pixel_ends > hole_end))
for partial in only_partials:
partial_start = np.floor((partial[0] - hour_start).seconds)
partial_end = np.ceil((partial[1] - hour_start).seconds)
self.logger.info('partial {} {}'.format(partial_start, partial_end))
partial_mask = partial_mask | ((pixel_starts >= partial_start) & (pixel_ends <= partial_end))
runtime_mask = runtime_mask.reshape((60, 60))
coverage_mask = coverage_mask.reshape((60, 60))
partial_mask = partial_mask.reshape((60, 60))
colours = np.ones((60, 60, 3))
colours[runtime_mask & coverage_mask] = colors.to_rgb('tab:green')
colours[runtime_mask & partial_mask] = colors.to_rgb('tab:orange')
colours[runtime_mask & ~coverage_mask] = colors.to_rgb('tab:red')
fig = plt.figure(figsize=(3, 3))
fig.add_axes([0, 0, 1, 1])
plt.imshow(colours)
plt.axis('off')
plt.savefig('/coverage_maps/{}.png'.format(hour), dpi=60)
# plt.figure()
# plt.imshow(runtime_mask.astype('d'))
# plt.colorbar()
# plt.savefig('/coverage_maps/runtime_{}.png'.format(hour))
#
# plt.figure()
# plt.imshow(coverage_mask.astype('d'))
# plt.colorbar()
# plt.savefig('/coverage_maps/coverage_{}.png'.format(hour))
#
# plt.figure()
# plt.imshow(partial_mask.astype('d'))
# plt.colorbar()
# plt.savefig('/coverage_maps/partial_{}.png'.format(hour))
self.logger.info('{}/{}: Start: {} End: {} ({} s)'.format(quality, hour, start, end, (end - start).seconds))
self.logger.info('{}/{}: {} full segments totalling {} s'.format(quality, hour, full_segment_count, full_segment_duration.seconds))
self.logger.info('{}/{}: {} full segment repeats totalling {} s'.format(quality, hour, full_repeats, full_repeat_duration.seconds))
@ -160,9 +251,6 @@ class CoverageChecker(object):
self.logger.info('{}/{}: {} overlapping segments, {} s overlapping'.format(quality, hour, overlap_count, overlap_duration.seconds))
self.logger.info(holes)
self.logger.info(only_partials)
previous_hour_segments = best_segments
self.stopping.wait(common.jitter(self.CHECK_INTERVAL))

Write Preview
Loading…
Cancel
Save