scripts for doing mass cutting of seasons

cut_races.py

@@ -1,38 +1,21 @@
 
-"""
-Attempts to cut every race for a league from local segments.
-
-Database info:
-
-matches maps to multiple races via match_races (on match_id)
-matches links to racers and cawmentator:
-	matches.racer_{1,2}_id
-	matches.cawmentator_id
-races contains start time:
-	races.timestamp
-races maps to multiple runs via race_runs
-race_runs contains time for each racer
-	race_runs.time: centiseconds
-	race_runs.rank: 1 for fastest time
-
-"""
-
-import datetime
 import logging
 import os
-import sys
+import tempfile
-from getpass import getpass
+import shutil
+from uuid import uuid4
 
 import argh
 import mysql.connector
 
-from common.segments import get_best_segments, cut_segments
+import cut_sync_race
 
 INFO_QUERY = """
 	SELECT
 		match_info.racer_1_name as racer_1,
 		match_info.racer_2_name as racer_2,
 		match_info.cawmentator_name as cawmentator,
+		match_info.league_tag as league,
 		match_info.match_id as match_id,
 		match_races.race_number as race_number,
 		races.timestamp as start,
@@ -42,46 +25,21 @@ INFO_QUERY = """
 	JOIN races ON (match_races.race_id = races.race_id)
 	JOIN race_runs ON (races.race_id = race_runs.race_id)
 	WHERE match_info.completed AND race_runs.rank = 1
+	ORDER BY start ASC
 """
 
-
+def main(
-def ts(dt):
+	output_dir,
-	return dt.strftime("%FT%T")
+	host='condor.live', user='necrobot-read', password='necrobot-read', database='condor_x2',
-
+	base_dir='/srv/wubloader', start_range='0,10', non_interactive=False,
-
+):
-class NoSegments(Exception):
-	pass
-
-
-def cut_to_file(filename, base_dir, stream, start, end, variant='source'):
-	if os.path.exists(filename):
-		return
-	logging.info("Cutting {}".format(filename))
-	segments = get_best_segments(
-		os.path.join(base_dir, stream, variant).lower(),
-		start, end,
-	)
-	if None in segments:
-		logging.warning("Cutting {} ({} to {}) but it contains holes".format(filename, ts(start), ts(end)))
-	if not segments or set(segments) == {None}:
-		raise NoSegments("Can't cut {} ({} to {}): No segments".format(filename, ts(start), ts(end)))
-	with open(filename, 'w') as f:
-		for chunk in cut_segments(segments, start, end):
-			f.write(chunk)
-
-
-def main(host='condor.live', user='necrobot-read', password=None, database='season_8', base_dir='.', output_dir='.', find=None):
 	logging.basicConfig(level=logging.INFO)
+	start_range = map(int, start_range.split(","))
 
-	if password is None:
-		password = getpass("Password? ")
 	conn = mysql.connector.connect(
 		host=host, user=user, password=password, database=database,
 	)
 
-	if find:
-		find = tuple(find.split('-'))
-
 	cur = conn.cursor()
 	cur.execute(INFO_QUERY)
 
@@ -93,28 +51,40 @@ def main(host='condor.live', user='necrobot-read', password=None, database='seas
 
 	logging.info("Got info on {} races".format(len(data)))
 
-	for racer1, racer2, cawmentator, match_id, race_number, start, duration in data:
+	for racer1, racer2, cawmentator, league, match_id, race_number, start, duration in data:
-		if find and (racer1.lower(), racer2.lower()) != find:
+		base_name = "-".join(map(str, [league, match_id, race_number, racer1, racer2]))
-			continue
-
-		end = start + datetime.timedelta(seconds=duration/100.)
-		base_name = "-".join(map(str, [racer1, racer2, match_id, race_number]))
 
 		items = [(racer1, racer1), (racer2, racer2)]
 		if cawmentator:
-			items.append(("cawmentary", cawmentator))
+			items.append(("caw-{}".format(cawmentator), cawmentator))
 		for name, stream in items:
+			output_path = os.path.join(output_dir, "{}-{}.mp4".format(base_name, name))
+			if os.path.exists(output_path):
+				continue
+			logging.info("Cutting {}, starting at {}".format(output_path, start))
+			output_temp = "{}.tmp{}.mp4".format(output_path, uuid4())
+			temp_dir = tempfile.mkdtemp()
+			caw_kwargs = {
+				# bypass start checks, cut a longer range instead
+				"output_range": (-5, 30),
+				"time_offset": 0,
+			} if name.startswith("caw-") else {}
 			try:
-				cut_to_file(
+				cut_sync_race.cut_race(
-					os.path.join(output_dir, "{}-{}.ts".format(base_name, name)),
+					base_dir, output_temp, temp_dir, stream, start, duration,
-					base_dir, stream, start, end,
+					start_range=start_range, non_interactive=non_interactive,
+					**caw_kwargs
 				)
-			except NoSegments as e:
+			except cut_sync_race.NoSegments as e:
 				logging.warning(e)
 			except Exception as e:
-				logging.exception("Failed to cut {}-{}.ts ({} to {})".format(
+				logging.exception("Failed to cut {}".format(output_path), exc_info=True)
-					base_name, name, ts(start), ts(end),
+				if not non_interactive:
-				), exc_info=True)
+					raw_input("Press enter to continue ")
+			else:
+				os.rename(output_temp, output_path)
+			finally:
+				shutil.rmtree(temp_dir, ignore_errors=True)
 
 
 if __name__ == '__main__':

cut_sync_race.py

@@ -81,12 +81,13 @@ def add_range(base, range):
 def main(match_id, race_number, output_path,
 	host='condor.live', user='necrobot-read', password='necrobot-read', database='condor_x2',
 	base_dir='/srv/wubloader',
-	start_range=(0, 10), non_interactive=False,
+	start_range="0,10", non_interactive=False, racer=0,
 ):
 	logging.basicConfig(level=logging.INFO)
 
 	match_id = int(match_id)
 	race_number = int(race_number)
+	start_range = map(int, start_range.split(","))
 
 	if password is None:
 		password = getpass("Password? ")
@@ -122,53 +123,60 @@ def main(match_id, race_number, output_path,
 	assert len(data) == 1, repr(data)
 
 	(racer1, racer2, start, duration, winner), = data
-	racer = [racer1, racer2][winner - 1]
+	if racer == 0:
+		racer = winner
+	racer = [racer1, racer2][racer - 1]
 	if racer == 'smokepipe_':
 		racer = 'smokepipetwitch'
 
 	temp_dir = tempfile.mkdtemp()
 	try:
-		cut_race(output_path, racer, start, duration, start_range=start_range, non_interactive=non_interactive)
+		cut_race(base_dir, output_path, temp_dir, racer, start, duration, start_range=start_range, non_interactive=non_interactive)
 	finally:
 		shutil.rmtree(temp_dir, ignore_errors=True)
 
-def cut_race(base_dir, output_path, temp_dir, racer, start, duration, start_range=(0, 10), non_interactive=False):
+def cut_race(
+	base_dir, output_path, temp_dir, racer, start, duration,
+	start_range=(0, 10), non_interactive=False, output_range=(-1, 5),
+	time_offset=None
+):
 	start_path = os.path.join(temp_dir, "start-{}.mp4".format(uuid4()))
 	end = start + datetime.timedelta(seconds=duration/100.)
-	output_range = [datetime.timedelta(seconds=n) for n in (-1, 5)]
+	output_range = [datetime.timedelta(seconds=n) for n in output_range]
-
 	start_start, start_end = add_range(start, start_range)
-	cut_to_file(start_path, base_dir, racer, start_start, start_end, fast_encode=True)
 
-	args = [
+	if time_offset is None:
-		'ffmpeg', '-hide_banner',
+		cut_to_file(start_path, base_dir, racer, start_start, start_end, fast_encode=True)
-		'-i', start_path,
-		'-vf', 'blackdetect=d=0.1',
-		'-f', 'null', '/dev/null'
-	]
-	proc = subprocess.Popen(args, stderr=subprocess.PIPE)
-	out, err = proc.communicate()
-	if proc.wait() != 0:
-		raise Exception("ffmpeg exited {}\n{}".format(proc.wait(), err))
-	lines = [
-		line for line in re.split('[\r\n]', err.strip())
-		if line.startswith('[blackdetect @ ')
-	]
-	if len(lines) > 0:
-		line = lines[0] # take first
-		black_end = line.split(' ')[4]
-		assert black_end.startswith('black_end:')
-		time_offset = float(black_end.split(':')[1])
-		os.remove(start_path) # clean up
-	elif non_interactive:
-		raise Exception("Unable to detect start (expected 1 black interval, but found {}).".format(len(lines)))
-	else:
-		print "Unable to detect start (expected 1 black interval, but found {}).".format(len(lines))
-		print "Cutting file {} for manual detection.".format(start_path)
-		cut_to_file(start_path, base_dir, racer, start_start, start_end, frame_counter=True, fast_encode=True)
-		time_offset = float(raw_input("What timestamp of this video do we start at? "))
-	time_offset = datetime.timedelta(seconds=time_offset)
 
+		args = [
+			'ffmpeg', '-hide_banner',
+			'-i', start_path,
+			'-vf', 'blackdetect=d=0.1',
+			'-f', 'null', '/dev/null'
+		]
+		proc = subprocess.Popen(args, stderr=subprocess.PIPE)
+		out, err = proc.communicate()
+		if proc.wait() != 0:
+			raise Exception("ffmpeg exited {}\n{}".format(proc.wait(), err))
+		lines = [
+			line for line in re.split('[\r\n]', err.strip())
+			if line.startswith('[blackdetect @ ')
+		]
+		if len(lines) > 0:
+			line = lines[0] # take first
+			black_end = line.split(' ')[4]
+			assert black_end.startswith('black_end:')
+			time_offset = float(black_end.split(':')[1])
+			os.remove(start_path) # clean up
+		elif non_interactive:
+			raise Exception("Unable to detect start (expected 1 black interval, but found {}).".format(len(lines)))
+		else:
+			print "Unable to detect start (expected 1 black interval, but found {}).".format(len(lines))
+			print "Cutting file {} for manual detection.".format(start_path)
+			cut_to_file(start_path, base_dir, racer, start_start, start_end, frame_counter=True, fast_encode=True)
+			time_offset = float(raw_input("What timestamp of this video do we start at? "))
+
+	time_offset = datetime.timedelta(seconds=time_offset)
 	output_start = start_start + time_offset + output_range[0]
 	cut_to_file(output_path, base_dir, racer, output_start, end + output_range[1], fast_encode=False)
 

delete_old_segments.py

@@ -0,0 +1,125 @@
+
+import os
+import shutil
+
+import argh
+
+RACERS = [
+	"abu__yazan",
+	"amellensatz",
+	"amitai_",
+	"ancalagor",
+	"arborelia",
+	"artq",
+	"asher__rose",
+	"autoclaves",
+	"azumak1",
+	"bgm_",
+	"biggiemac42",
+	"bookworm52",
+	"botwman",
+	"cavalca",
+	"cheekyhummus",
+	"chef_mayhem",
+	"crate3333",
+	"cryss__",
+	"cyber_1",
+	"definitely_not_him",
+	"dlkurosh_",
+	"echaen",
+	"eladdifficult",
+	"eleriz",
+	"evtet",
+	"faba_nocturne",
+	"flamehaze0",
+	"focusingface",
+	"g4mmadelta",
+	"gamingprosadventure",
+	"gekke__",
+	"gfitty",
+	"gleeokenspiel",
+	"hallowedcrow",
+	"hating_mirror",
+	"hummingbee__",
+	"itsmedash2",
+	"jackofgames",
+	"jo_jo909",
+	"jped",
+	"kailaria",
+	"kupioala",
+	"ladyljl",
+	"lakehope2",
+	"linforcer",
+	"macnd",
+	"mahpete_",
+	"minhs2",
+	"missmists",
+	"mizmy",
+	"monster_racer",
+	"nbtelethia",
+	"nd_tea",
+	"nomkey",
+	"notester_82",
+	"ocre307",
+	"pancelor",
+	"paratroopa1",
+	"parkerpng",
+	"pedrom664",
+	"pleasantbreeze",
+	"poanclr2",
+	"priw8",
+	"probablyathena",
+	"ptar124",
+	"ratracing",
+	"raviolinguini",
+	"ribbongraph",
+	"rivs218",
+	"rolamni",
+	"royalgoof",
+	"sailo93",
+	"sailormint",
+	"sanga_5_3",
+	"scc127",
+	"shandee_",
+	"shortcakesweets",
+	"sivcria",
+	"siveure",
+	"slimo",
+	"smokepipetwitch",
+	"speedfrog",
+	"speedgaming",
+	"speedgaming2",
+	"spootybiscuit",
+	"squega",
+	"squidy1226",
+	"studkid",
+	"supervillain_joe",
+	"szprychaaa",
+	"tang_pls",
+	"teki_b0t",
+	"tictacfoe",
+	"tr0gd0rburns",
+	"tufwfo",
+	"valkhiya",
+	"vhatenav",
+	"waskilledby",
+	"wooperpng",
+	"wow_tomato",
+	"yjalexis",
+	"yuka34",
+	"yuka_nd",
+	"zelllooo",
+]
+
+def main(keep_hour, base_dir='/srv/wubloader', dry_run=False):
+	for racer in RACERS:
+		hours_dir = os.path.join(base_dir, racer, "source")
+		for hour in os.listdir(hours_dir):
+			if hour < keep_hour:
+				hour_path = os.path.join(hours_dir, hour)
+				print "Deleting hour: {}".format(hour_path)
+				if not dry_run:
+					shutil.rmtree(hour_path)
+
+if __name__ == '__main__':
+	argh.dispatch_command(main)