henine
/
wubloader
mirror of https://github.com/ekimekim/wubloader
1
0
Fork 0
Code Issues Releases Wiki Activity

refactor reviews for extra info reporting even on happy path

Browse Source
condor-scripts
Mike Lang 4 years ago
parent c1465f49a6
commit e4700e6e4f
2 changed files with 89 additions and 59 deletions
Show Stats Download Patch File Download Diff File

51
restreamer/restreamer/main.py
Unescape Escape View File

@ -10,7 +10,7 @@ import signal
import gevent import gevent
import gevent.backdoor import gevent.backdoor
import prometheus_client as prom import prometheus_client as prom
from flask import Flask, url_for, request, abort, redirect, Response from flask import Flask, url_for, request, abort, Response
from gevent.pywsgi import WSGIServer from gevent.pywsgi import WSGIServer
from common import dateutil, get_best_segments, rough_cut_segments, smart_cut_segments, fast_cut_segments, full_cut_segments, PromLogCountsHandler, install_stacksampler from common import dateutil, get_best_segments, rough_cut_segments, smart_cut_segments, fast_cut_segments, full_cut_segments, PromLogCountsHandler, install_stacksampler
@ -373,8 +373,12 @@ def review_race(match_id, race_number):
finish_range = map(float, request.args.get('finish_range', '-5,10').split(',')) finish_range = map(float, request.args.get('finish_range', '-5,10').split(','))
racer1_start = float(request.args['racer1_start']) if 'racer1_start' in request.args else None racer1_start = float(request.args['racer1_start']) if 'racer1_start' in request.args else None
racer2_start = float(request.args['racer2_start']) if 'racer2_start' in request.args else None racer2_start = float(request.args['racer2_start']) if 'racer2_start' in request.args else None
path_to_url = lambda path: os.path.join(app.static_url_path, os.path.relpath(path, app.static_folder))
link = lambda url, text: '<a href="{}">{}</a>'.format(url, text)
try: try:
review_path, suspect_starts = review( info = review(
match_id, race_number, app.static_folder, app.condor_db, start_range, finish_range, match_id, race_number, app.static_folder, app.condor_db, start_range, finish_range,
racer1_start, racer2_start, racer1_start, racer2_start,
) )
@ -389,32 +393,29 @@ def review_race(match_id, race_number):
"Please check start video and adjust start_range or set racer{}_start: {}\n" "Please check start video and adjust start_range or set racer{}_start: {}\n"
"Note timestamps in that video are only valid for the current start_range.\n" "Note timestamps in that video are only valid for the current start_range.\n"
).format( ).format(
e, e.racer_number, os.path.join(app.static_url_path, os.path.relpath(e.path, app.static_folder)) e, e.racer_number, path_to_url(e.path),
), 400 ), 400
relative_path = os.path.relpath(review_path, app.static_folder) body = []
review_url = os.path.join(app.static_url_path, relative_path) for racer_info in info["racers"]:
if "start_path" in racer_info:
if suspect_starts: body.append("Detected {info[name]} start at {info[offset]} of {link}".format(
# warn and link via html info=racer_info,
return "\n".join([ link=link(path_to_url(racer_info["start_path"]), "start video"),
"<html>", ))
"<body>", if racer_info["start_holes"]:
"Review succeeded, but start times are uncertain. Please check start videos:</br>", body.append("WARNING: Start video for {} was missing some data and may be inaccurate".format(racer_info["name"]))
"\n".join('<a href="{}">{}</a></br>'.format( if len(racer_info["starts"]) > 1:
os.path.join(app.static_url_path, os.path.relpath(e.path, app.static_folder)), body.append("WARNING: Detected multiple possible start times for {}: {}".format(
e, racer_info["name"], ", ".join(racer_info["starts"])
) for e in suspect_starts), ))
"If all is well, the review is available",
'<a href="{}">HERE</a>'.format(review_url),
"</body>",
"</html>",
])
else: else:
# happy path, redirect body.append("Using given start of {info[offset]} for {info[name]}".format(info=racer_info))
response = redirect(review_url) if racer_info["finish_holes"]:
response.autocorrect_location_header = False body.append("WARNING: Result video for {} was missing some data and may be inaccurate".format(racer_info["name"]))
return response body.append(link(path_to_url(info["result_path"]), "Result video available here"))
return "<html><body>{}</body></html>".format("</br>\n".join(body))
def main(host='0.0.0.0', port=8000, base_dir='.', backdoor_port=0, condor_db=None): def main(host='0.0.0.0', port=8000, base_dir='.', backdoor_port=0, condor_db=None):

87
restreamer/restreamer/review.py
Unescape Escape View File

@ -1,5 +1,6 @@
import datetime import datetime
import json
import logging import logging
import os import os
import re import re
@ -24,15 +25,11 @@ class RaceNotFound(Exception):
class CantFindStart(Exception): class CantFindStart(Exception):
def __init__(self, racer, racer_number, found, path): def __init__(self, racer, racer_number, path):
self.racer = racer self.racer = racer
self.racer_number = racer_number self.racer_number = racer_number
self.found = found
self.path = path self.path = path
def __str__(self): def __str__(self):
if self.found > 0:
return "Found multiple ({}) possible start points for racer {} ({})".format(self.found, self.racer_number, self.racer)
else:
return "Failed to find start point for racer {} ({})".format(self.racer_number, self.racer) return "Failed to find start point for racer {} ({})".format(self.racer_number, self.racer)
@ -41,12 +38,14 @@ def ts(dt):
def cut_to_file(logger, filename, base_dir, stream, start, end, variant='source', frame_counter=False): def cut_to_file(logger, filename, base_dir, stream, start, end, variant='source', frame_counter=False):
"""Returns boolean of whether cut video contained holes"""
logger.info("Cutting {}".format(filename)) logger.info("Cutting {}".format(filename))
segments = get_best_segments( segments = get_best_segments(
os.path.join(base_dir, stream, variant).lower(), os.path.join(base_dir, stream, variant).lower(),
start, end, start, end,
) )
if None in segments: contains_holes = None in segments
if contains_holes:
logger.warning("Cutting {} ({} to {}) but it contains holes".format(filename, ts(start), ts(end))) logger.warning("Cutting {} ({} to {}) but it contains holes".format(filename, ts(start), ts(end)))
if not segments or set(segments) == {None}: if not segments or set(segments) == {None}:
raise NoSegments("Can't cut {} ({} to {}): No segments".format(filename, ts(start), ts(end))) raise NoSegments("Can't cut {} ({} to {}): No segments".format(filename, ts(start), ts(end)))
@ -66,6 +65,7 @@ def cut_to_file(logger, filename, base_dir, stream, start, end, variant='source'
with open(filename, 'w') as f: with open(filename, 'w') as f:
for chunk in full_cut_segments(segments, start, end, filter_args + encoding_args): for chunk in full_cut_segments(segments, start, end, filter_args + encoding_args):
f.write(chunk) f.write(chunk)
return contains_holes
def add_range(base, range): def add_range(base, range):
@ -86,6 +86,21 @@ def review(
match_id, race_number, base_dir, db_url, start_range=(0, 10), finish_range=(-5, 10), match_id, race_number, base_dir, db_url, start_range=(0, 10), finish_range=(-5, 10),
racer1_start=None, racer2_start=None, racer1_start=None, racer2_start=None,
): ):
"""Cuts a review, returning the following structure:
{
racers: [
{
name: racer name
start_path: path to start video, omitted if start given
start_holes: bool, whether the start video contained holes, omitted if start given
starts: [start times within video], omitted if start given
offset: final time offset used
finish_holes: bool, whether the finish video contained holes
} for each racer
]
result_path: path to result video
}
"""
logger = logging.getLogger("review").getChild("{}-{}".format(match_id, race_number)) logger = logging.getLogger("review").getChild("{}-{}".format(match_id, race_number))
conn = conn_from_url(db_url) conn = conn_from_url(db_url)
@ -129,22 +144,29 @@ def review(
os.makedirs(output_dir) os.makedirs(output_dir)
result_name = "review_{}.mp4".format(cache_str) result_name = "review_{}.mp4".format(cache_str)
result_path = os.path.join(output_dir, result_name) result_path = os.path.join(output_dir, result_name)
if os.path.exists(result_path): cache_path = os.path.join(output_dir, "cache_{}.json".format(cache_str))
if os.path.exists(result_path) and os.path.exists(cache_path):
logger.info("Result already exists for {}, reusing".format(result_path)) logger.info("Result already exists for {}, reusing".format(result_path))
return result_path, [] with open(cache_path) as f:
return json.load(f)
finish_paths = [] finish_paths = []
suspect_starts = [] result_info = {
"result_path": result_path
}
for racer_index, (racer, time_offset) in enumerate(((racer1, racer1_start), (racer2, racer2_start))): for racer_index, (racer, time_offset) in enumerate(((racer1, racer1_start), (racer2, racer2_start))):
nonce = str(uuid4()) nonce = str(uuid4())
racer_number = racer_index + 1 racer_number = racer_index + 1
racer_info = {"name": racer}
result_info.setdefault("racers", []).append(racer_info)
if time_offset is None: if time_offset is None:
start_path = os.path.join(output_dir, "start-{}-{}.mp4".format(racer_number, nonce)) start_path = os.path.join(output_dir, "start-{}-{}-{}.mp4".format(racer_number, cache_str, nonce))
racer_info["start_path"] = start_path
logger.info("Cutting start for racer {} ({})".format(racer_number, racer)) logger.info("Cutting start for racer {} ({})".format(racer_number, racer))
start_start, start_end = add_range(start, start_range) start_start, start_end = add_range(start, start_range)
cut_to_file(logger, start_path, base_dir, racer, start_start, start_end) racer_info["start_holes"] = cut_to_file(logger, start_path, base_dir, racer, start_start, start_end)
logger.info("Running blackdetect") logger.info("Running blackdetect")
args = [ args = [
@ -161,19 +183,25 @@ def review(
line for line in re.split('[\r\n]', err.strip()) line for line in re.split('[\r\n]', err.strip())
if line.startswith('[blackdetect @ ') if line.startswith('[blackdetect @ ')
] ]
if len(lines) != 1: starts = []
found = len(lines) racer_info["starts"] = starts
logger.warning("Unable to detect start (expected 1 black interval, but found {}), re-cutting with timestamps".format(found)) for line in lines:
cut_to_file(logger, start_path, base_dir, racer, start_start, start_end, frame_counter=True)
error = CantFindStart(racer, racer_number, found, start_path)
if not lines:
raise error
# try to continue by picking first
suspect_starts.append(error)
line = lines[0]
black_end = line.split(' ')[4] black_end = line.split(' ')[4]
assert black_end.startswith('black_end:') assert black_end.startswith('black_end:')
time_offset = float(black_end.split(':')[1]) starts.append(float(black_end.split(':')[1]))
# unconditionally re-cut a start, this time with frame counter.
# TODO avoid the repeated work and do cut + blackdetect + frame counter all in one pass
cut_to_file(logger, start_path, base_dir, racer, start_start, start_end, frame_counter=True)
if not starts:
raise CantFindStart(racer, racer_number, start_path)
if len(starts) > 1:
logging.warning("Found multiple starts, picking first: {}".format(starts))
time_offset = starts[0]
racer_info["offset"] = time_offset
time_offset = datetime.timedelta(seconds=start_range[0] + time_offset) time_offset = datetime.timedelta(seconds=start_range[0] + time_offset)
# start each racer's finish video at TIME_OFFSET later, so they are the same # start each racer's finish video at TIME_OFFSET later, so they are the same
@ -183,7 +211,7 @@ def review(
finish_path = os.path.join(output_dir, "finish-{}-{}.mp4".format(racer_number, nonce)) finish_path = os.path.join(output_dir, "finish-{}-{}.mp4".format(racer_number, nonce))
finish_paths.append(finish_path) finish_paths.append(finish_path)
logger.info("Got time offset of {}, cutting finish at finish_base {}".format(time_offset, finish_base)) logger.info("Got time offset of {}, cutting finish at finish_base {}".format(time_offset, finish_base))
cut_to_file(logger, finish_path, base_dir, racer, finish_start, finish_end) racer_info["finish_holes"] = cut_to_file(logger, finish_path, base_dir, racer, finish_start, finish_end)
temp_path = "{}.{}.mp4".format(result_path, str(uuid4())) temp_path = "{}.{}.mp4".format(result_path, str(uuid4()))
args = ['ffmpeg'] args = ['ffmpeg']
@ -195,13 +223,14 @@ def review(
'-y', temp_path, '-y', temp_path,
] ]
cache_temp = "{}.{}.json".format(cache_path, str(uuid4()))
with open(cache_temp, 'w') as f:
f.write(json.dumps(result_info))
os.rename(cache_temp, cache_path)
logger.info("Cutting final result") logger.info("Cutting final result")
subprocess.check_call(args) subprocess.check_call(args)
# atomic rename so that if result_path exists at all, we know it is complete and correct # atomic rename so that if result_path exists at all, we know it is complete and correct
# don't do this if we have a suspect start though, as the cached result wouldn't know.
if suspect_starts:
result_path = temp_path
else:
os.rename(temp_path, result_path) os.rename(temp_path, result_path)
logger.info("Review done, suspect starts = {}".format(len(suspect_starts))) logger.info("Review done: {}".format(result_info))
return result_path, suspect_starts return result_info

Write Preview
Loading…
Cancel
Save