From 997c1242b2468568fcd5ae4f5e14014e9e10bf36 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Tue, 8 Jan 2019 00:49:49 -0800 Subject: [PATCH] get_best_segments: Let other things run get_best_segments can sometimes take a very long time, we don't want to stop other work from happening while it's ongoing. So we ask gevent to run other things until there's no other work to do, then we do one hour, then check back with gevent again. In combination with the performance improvements, this should mean we don't block other things from running for more than a few hundred ms at most. --- common/common/segments.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/common/common/segments.py b/common/common/segments.py index 65b7e20..5816cf9 100644 --- a/common/common/segments.py +++ b/common/common/segments.py @@ -11,6 +11,8 @@ import os import sys from collections import namedtuple +import gevent + from .stats import timed @@ -109,6 +111,11 @@ def get_best_segments(hours_path, start, end): result = [] for hour in hour_paths_for_range(hours_path, start, end): + # Especially when processing multiple hours, this routine can take a signifigant amount + # of time with no blocking. To ensure other stuff is still completed in a timely fashion, + # we yield to let other things run. + gevent.idle() + # best_segments_by_start will give us the best available segment for each unique start time for segment in best_segments_by_start(hour):