henine
/
wubloader
mirror of https://github.com/ekimekim/wubloader
1
0
Fork 0
Code Issues Releases Wiki Activity

scripts for doing mass cutting of seasons

Browse Source
condor-scripts
Mike Lang 3 years ago
parent 0c52f653b1
commit c9003fd540
3 changed files with 206 additions and 103 deletions
Show Stats Download Patch File Download Diff File

106
cut_races.py
Unescape Escape View File

@ -1,38 +1,21 @@
"""
Attempts to cut every race for a league from local segments.
Database info:
matches maps to multiple races via match_races (on match_id)
matches links to racers and cawmentator:
matches.racer_{1,2}_id
matches.cawmentator_id
races contains start time:
races.timestamp
races maps to multiple runs via race_runs
race_runs contains time for each racer
race_runs.time: centiseconds
race_runs.rank: 1 for fastest time
"""
import datetime
import logging
import os
import sys
from getpass import getpass
import tempfile
import shutil
from uuid import uuid4
import argh
import mysql.connector
from common.segments import get_best_segments, cut_segments
import cut_sync_race
INFO_QUERY = """
SELECT
match_info.racer_1_name as racer_1,
match_info.racer_2_name as racer_2,
match_info.cawmentator_name as cawmentator,
match_info.league_tag as league,
match_info.match_id as match_id,
match_races.race_number as race_number,
races.timestamp as start,
@ -42,46 +25,21 @@ INFO_QUERY = """
JOIN races ON (match_races.race_id = races.race_id)
JOIN race_runs ON (races.race_id = race_runs.race_id)
WHERE match_info.completed AND race_runs.rank = 1
ORDER BY start ASC
"""
def ts(dt):
return dt.strftime("%FT%T")
class NoSegments(Exception):
pass
def cut_to_file(filename, base_dir, stream, start, end, variant='source'):
if os.path.exists(filename):
return
logging.info("Cutting {}".format(filename))
segments = get_best_segments(
os.path.join(base_dir, stream, variant).lower(),
start, end,
)
if None in segments:
logging.warning("Cutting {} ({} to {}) but it contains holes".format(filename, ts(start), ts(end)))
if not segments or set(segments) == {None}:
raise NoSegments("Can't cut {} ({} to {}): No segments".format(filename, ts(start), ts(end)))
with open(filename, 'w') as f:
for chunk in cut_segments(segments, start, end):
f.write(chunk)
def main(host='condor.live', user='necrobot-read', password=None, database='season_8', base_dir='.', output_dir='.', find=None):
def main(
output_dir,
host='condor.live', user='necrobot-read', password='necrobot-read', database='condor_x2',
base_dir='/srv/wubloader', start_range='0,10', non_interactive=False,
):
logging.basicConfig(level=logging.INFO)
start_range = map(int, start_range.split(","))
if password is None:
password = getpass("Password? ")
conn = mysql.connector.connect(
host=host, user=user, password=password, database=database,
)
if find:
find = tuple(find.split('-'))
cur = conn.cursor()
cur.execute(INFO_QUERY)
@ -93,28 +51,40 @@ def main(host='condor.live', user='necrobot-read', password=None, database='seas
logging.info("Got info on {} races".format(len(data)))
for racer1, racer2, cawmentator, match_id, race_number, start, duration in data:
if find and (racer1.lower(), racer2.lower()) != find:
continue
end = start + datetime.timedelta(seconds=duration/100.)
base_name = "-".join(map(str, [racer1, racer2, match_id, race_number]))
for racer1, racer2, cawmentator, league, match_id, race_number, start, duration in data:
base_name = "-".join(map(str, [league, match_id, race_number, racer1, racer2]))
items = [(racer1, racer1), (racer2, racer2)]
if cawmentator:
items.append(("cawmentary", cawmentator))
items.append(("caw-{}".format(cawmentator), cawmentator))
for name, stream in items:
output_path = os.path.join(output_dir, "{}-{}.mp4".format(base_name, name))
if os.path.exists(output_path):
continue
logging.info("Cutting {}, starting at {}".format(output_path, start))
output_temp = "{}.tmp{}.mp4".format(output_path, uuid4())
temp_dir = tempfile.mkdtemp()
caw_kwargs = {
# bypass start checks, cut a longer range instead
"output_range": (-5, 30),
"time_offset": 0,
} if name.startswith("caw-") else {}
try:
cut_to_file(
os.path.join(output_dir, "{}-{}.ts".format(base_name, name)),
base_dir, stream, start, end,
cut_sync_race.cut_race(
base_dir, output_temp, temp_dir, stream, start, duration,
start_range=start_range, non_interactive=non_interactive,
**caw_kwargs
)
except NoSegments as e:
except cut_sync_race.NoSegments as e:
logging.warning(e)
except Exception as e:
logging.exception("Failed to cut {}-{}.ts ({} to {})".format(
base_name, name, ts(start), ts(end),
), exc_info=True)
logging.exception("Failed to cut {}".format(output_path), exc_info=True)
if not non_interactive:
raw_input("Press enter to continue ")
else:
os.rename(output_temp, output_path)
finally:
shutil.rmtree(temp_dir, ignore_errors=True)
if __name__ == '__main__':

78
cut_sync_race.py
Unescape Escape View File

@ -81,12 +81,13 @@ def add_range(base, range):
def main(match_id, race_number, output_path,
host='condor.live', user='necrobot-read', password='necrobot-read', database='condor_x2',
base_dir='/srv/wubloader',
start_range=(0, 10), non_interactive=False,
start_range="0,10", non_interactive=False, racer=0,
):
logging.basicConfig(level=logging.INFO)
match_id = int(match_id)
race_number = int(race_number)
start_range = map(int, start_range.split(","))
if password is None:
password = getpass("Password? ")
@ -122,53 +123,60 @@ def main(match_id, race_number, output_path,
assert len(data) == 1, repr(data)
(racer1, racer2, start, duration, winner), = data
racer = [racer1, racer2][winner - 1]
if racer == 0:
racer = winner
racer = [racer1, racer2][racer - 1]
if racer == 'smokepipe_':
racer = 'smokepipetwitch'
temp_dir = tempfile.mkdtemp()
try:
cut_race(output_path, racer, start, duration, start_range=start_range, non_interactive=non_interactive)
cut_race(base_dir, output_path, temp_dir, racer, start, duration, start_range=start_range, non_interactive=non_interactive)
finally:
shutil.rmtree(temp_dir, ignore_errors=True)
def cut_race(base_dir, output_path, temp_dir, racer, start, duration, start_range=(0, 10), non_interactive=False):
def cut_race(
base_dir, output_path, temp_dir, racer, start, duration,
start_range=(0, 10), non_interactive=False, output_range=(-1, 5),
time_offset=None
):
start_path = os.path.join(temp_dir, "start-{}.mp4".format(uuid4()))
end = start + datetime.timedelta(seconds=duration/100.)
output_range = [datetime.timedelta(seconds=n) for n in (-1, 5)]
output_range = [datetime.timedelta(seconds=n) for n in output_range]
start_start, start_end = add_range(start, start_range)
cut_to_file(start_path, base_dir, racer, start_start, start_end, fast_encode=True)
args = [
'ffmpeg', '-hide_banner',
'-i', start_path,
'-vf', 'blackdetect=d=0.1',
'-f', 'null', '/dev/null'
]
proc = subprocess.Popen(args, stderr=subprocess.PIPE)
out, err = proc.communicate()
if proc.wait() != 0:
raise Exception("ffmpeg exited {}\n{}".format(proc.wait(), err))
lines = [
line for line in re.split('[\r\n]', err.strip())
if line.startswith('[blackdetect @ ')
]
if len(lines) > 0:
line = lines[0] # take first
black_end = line.split(' ')[4]
assert black_end.startswith('black_end:')
time_offset = float(black_end.split(':')[1])
os.remove(start_path) # clean up
elif non_interactive:
raise Exception("Unable to detect start (expected 1 black interval, but found {}).".format(len(lines)))
else:
print "Unable to detect start (expected 1 black interval, but found {}).".format(len(lines))
print "Cutting file {} for manual detection.".format(start_path)
cut_to_file(start_path, base_dir, racer, start_start, start_end, frame_counter=True, fast_encode=True)
time_offset = float(raw_input("What timestamp of this video do we start at? "))
time_offset = datetime.timedelta(seconds=time_offset)
if time_offset is None:
cut_to_file(start_path, base_dir, racer, start_start, start_end, fast_encode=True)
args = [
'ffmpeg', '-hide_banner',
'-i', start_path,
'-vf', 'blackdetect=d=0.1',
'-f', 'null', '/dev/null'
]
proc = subprocess.Popen(args, stderr=subprocess.PIPE)
out, err = proc.communicate()
if proc.wait() != 0:
raise Exception("ffmpeg exited {}\n{}".format(proc.wait(), err))
lines = [
line for line in re.split('[\r\n]', err.strip())
if line.startswith('[blackdetect @ ')
]
if len(lines) > 0:
line = lines[0] # take first
black_end = line.split(' ')[4]
assert black_end.startswith('black_end:')
time_offset = float(black_end.split(':')[1])
os.remove(start_path) # clean up
elif non_interactive:
raise Exception("Unable to detect start (expected 1 black interval, but found {}).".format(len(lines)))
else:
print "Unable to detect start (expected 1 black interval, but found {}).".format(len(lines))
print "Cutting file {} for manual detection.".format(start_path)
cut_to_file(start_path, base_dir, racer, start_start, start_end, frame_counter=True, fast_encode=True)
time_offset = float(raw_input("What timestamp of this video do we start at? "))
time_offset = datetime.timedelta(seconds=time_offset)
output_start = start_start + time_offset + output_range[0]
cut_to_file(output_path, base_dir, racer, output_start, end + output_range[1], fast_encode=False)

125
delete_old_segments.py
Unescape Escape View File

@ -0,0 +1,125 @@
import os
import shutil
import argh
RACERS = [
"abu__yazan",
"amellensatz",
"amitai_",
"ancalagor",
"arborelia",
"artq",
"asher__rose",
"autoclaves",
"azumak1",
"bgm_",
"biggiemac42",
"bookworm52",
"botwman",
"cavalca",
"cheekyhummus",
"chef_mayhem",
"crate3333",
"cryss__",
"cyber_1",
"definitely_not_him",
"dlkurosh_",
"echaen",
"eladdifficult",
"eleriz",
"evtet",
"faba_nocturne",
"flamehaze0",
"focusingface",
"g4mmadelta",
"gamingprosadventure",
"gekke__",
"gfitty",
"gleeokenspiel",
"hallowedcrow",
"hating_mirror",
"hummingbee__",
"itsmedash2",
"jackofgames",
"jo_jo909",
"jped",
"kailaria",
"kupioala",
"ladyljl",
"lakehope2",
"linforcer",
"macnd",
"mahpete_",
"minhs2",
"missmists",
"mizmy",
"monster_racer",
"nbtelethia",
"nd_tea",
"nomkey",
"notester_82",
"ocre307",
"pancelor",
"paratroopa1",
"parkerpng",
"pedrom664",
"pleasantbreeze",
"poanclr2",
"priw8",
"probablyathena",
"ptar124",
"ratracing",
"raviolinguini",
"ribbongraph",
"rivs218",
"rolamni",
"royalgoof",
"sailo93",
"sailormint",
"sanga_5_3",
"scc127",
"shandee_",
"shortcakesweets",
"sivcria",
"siveure",
"slimo",
"smokepipetwitch",
"speedfrog",
"speedgaming",
"speedgaming2",
"spootybiscuit",
"squega",
"squidy1226",
"studkid",
"supervillain_joe",
"szprychaaa",
"tang_pls",
"teki_b0t",
"tictacfoe",
"tr0gd0rburns",
"tufwfo",
"valkhiya",
"vhatenav",
"waskilledby",
"wooperpng",
"wow_tomato",
"yjalexis",
"yuka34",
"yuka_nd",
"zelllooo",
]
def main(keep_hour, base_dir='/srv/wubloader', dry_run=False):
for racer in RACERS:
hours_dir = os.path.join(base_dir, racer, "source")
for hour in os.listdir(hours_dir):
if hour < keep_hour:
hour_path = os.path.join(hours_dir, hour)
print "Deleting hour: {}".format(hour_path)
if not dry_run:
shutil.rmtree(hour_path)
if __name__ == '__main__':
argh.dispatch_command(main)
Write Preview
Loading…
Cancel
Save