henine
/
wubloader
mirror of https://github.com/ekimekim/wubloader
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #142 from ekimekim/mike/fixes

Browse Source 
Misc fixes and improvements
pull/146/head
Mike Lang 5 years ago committed by GitHub
parent c1cb7966af 07b93f779c
commit 48d4dddb1c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 80 additions and 87 deletions
Show Stats Download Patch File Download Diff File

79
backfiller/backfiller/main.py
Unescape Escape View File

@ -3,6 +3,7 @@
import datetime import datetime
import errno import errno
import hashlib import hashlib
import itertools
import logging import logging
import os import os
import random import random
@ -45,13 +46,12 @@ hash_mismatches = prom.Counter(
node_list_errors = prom.Counter( node_list_errors = prom.Counter(
'node_list_errors', 'node_list_errors',
'Number of errors fetching a list of nodes', 'Number of errors fetching a list of nodes',
['filename', 'database'],
) )
backfill_errors = prom.Counter( backfill_errors = prom.Counter(
'backfill_errors', 'backfill_errors',
'Number of errors backfilling', 'Number of errors backfilling',
['remote', 'channel'], ['remote'],
) )
segments_deleted = prom.Counter( segments_deleted = prom.Counter(
@ -170,10 +170,10 @@ def get_remote_segment(base_dir, node, channel, quality, hour, missing_segment,
logger.info('Segment {}/{}/{} backfilled'.format(quality, hour, missing_segment)) logger.info('Segment {}/{}/{} backfilled'.format(quality, hour, missing_segment))
def list_hours(node, channel, qualities, start=None): def list_hours(node, channel, quality, start=None):
"""Return a list of all available hours from a node. """Return a list of all available hours from a node.
List all hours available from node/channel for each quality in qualities List all hours available from node/channel
ordered from newest to oldest. ordered from newest to oldest.
Keyword arguments: Keyword arguments:
@ -181,8 +181,7 @@ def list_hours(node, channel, qualities, start=None):
return hours more recent than that number of hours ago. If None (default), return hours more recent than that number of hours ago. If None (default),
all hours are returned.""" all hours are returned."""
hour_lists = [list_remote_hours(node, channel, quality) for quality in qualities] hours = list_remote_hours(node, channel, quality)
hours = list(set().union(*hour_lists))
hours.sort(reverse=True) #latest hour first hours.sort(reverse=True) #latest hour first
if start is not None: if start is not None:
@ -205,7 +204,7 @@ class BackfillerManager(object):
NODE_INTERVAL = 300 #seconds between updating list of nodes NODE_INTERVAL = 300 #seconds between updating list of nodes
def __init__(self, base_dir, channel, qualities, static_nodes=[], def __init__(self, base_dir, channels, qualities, static_nodes=[],
start=None, delete_old=False, run_once=False, node_file=None, start=None, delete_old=False, run_once=False, node_file=None,
node_database=None, localhost=None, download_concurrency=5, node_database=None, localhost=None, download_concurrency=5,
recent_cutoff=120): recent_cutoff=120):
@ -213,7 +212,7 @@ class BackfillerManager(object):
Creates a manager for a given channel with specified qualities.""" Creates a manager for a given channel with specified qualities."""
self.base_dir = base_dir self.base_dir = base_dir
self.channel = channel self.channels = channels
self.qualities = qualities self.qualities = qualities
self.static_nodes = static_nodes self.static_nodes = static_nodes
self.start = start self.start = start
@ -226,7 +225,7 @@ class BackfillerManager(object):
self.download_concurrency = download_concurrency self.download_concurrency = download_concurrency
self.recent_cutoff = recent_cutoff self.recent_cutoff = recent_cutoff
self.stopping = gevent.event.Event() self.stopping = gevent.event.Event()
self.logger = logging.getLogger("BackfillerManager({})".format(channel)) self.logger = logging.getLogger("BackfillerManager")
self.workers = {} # {node url: worker} self.workers = {} # {node url: worker}
def stop(self): def stop(self):
@ -258,8 +257,8 @@ class BackfillerManager(object):
else: else:
self.logger.info('Deleting hours older than {} hours ago'.format(self.start)) self.logger.info('Deleting hours older than {} hours ago'.format(self.start))
for quality in self.qualities: for channel, quality in itertools.product(self.channels, self.qualities):
hours = list_local_hours(self.base_dir, self.channel, quality) hours = list_local_hours(self.base_dir, channel, quality)
if not isinstance(self.start, datetime.datetime): if not isinstance(self.start, datetime.datetime):
cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=self.start) cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=self.start)
else: else:
@ -271,13 +270,13 @@ class BackfillerManager(object):
# deleting segments can take a bit time but is less important # deleting segments can take a bit time but is less important
# than the actually backfilling so we yield # than the actually backfilling so we yield
gevent.idle() gevent.idle()
path = os.path.join(self.base_dir, self.channel, quality, hour) path = os.path.join(self.base_dir, channel, quality, hour)
self.logger.info('Deleting {}'.format(path)) self.logger.info('Deleting {}'.format(path))
segments = list_local_segments(self.base_dir, self.channel, quality, hour) segments = list_local_segments(self.base_dir, channel, quality, hour)
for segment in segments: for segment in segments:
try: try:
os.remove(os.path.join(path, segment)) os.remove(os.path.join(path, segment))
segments_deleted.labels(channel=self.channel, quality=quality, hour=hour).inc() segments_deleted.labels(channel=channel, quality=quality, hour=hour).inc()
except OSError as e: except OSError as e:
# ignore error when the file is already gone # ignore error when the file is already gone
if e.errno != errno.ENOENT: if e.errno != errno.ENOENT:
@ -321,7 +320,7 @@ class BackfillerManager(object):
failures += 1 failures += 1
delay = common.jitter(TIMEOUT * 2**failures) delay = common.jitter(TIMEOUT * 2**failures)
self.logger.exception('Getting nodes failed. Retrying in {:.0f} s'.format(delay)) self.logger.exception('Getting nodes failed. Retrying in {:.0f} s'.format(delay))
node_list_errors.labels(filename=self.node_file).inc() node_list_errors.inc()
self.stopping.wait(delay) self.stopping.wait(delay)
continue continue
exisiting_nodes = set(self.workers.keys()) exisiting_nodes = set(self.workers.keys())
@ -412,7 +411,7 @@ class BackfillerWorker(object):
self.base_dir = manager.base_dir self.base_dir = manager.base_dir
self.node = node self.node = node
self.download_concurrency = manager.download_concurrency self.download_concurrency = manager.download_concurrency
self.channel = manager.channel self.channels = manager.channels
self.qualities = manager.qualities self.qualities = manager.qualities
self.start = manager.start self.start = manager.start
self.run_once = manager.run_once self.run_once = manager.run_once
@ -421,7 +420,7 @@ class BackfillerWorker(object):
self.done = gevent.event.Event() self.done = gevent.event.Event()
def __repr__(self): def __repr__(self):
return '<{} at 0x{:x} for {!r}/{!r}>'.format(type(self).__name__, id(self), self.node, self.channel) return '<{} at 0x{:x} for {!r}>'.format(type(self).__name__, id(self), self.node)
__str__ = __repr__ __str__ = __repr__
def stop(self): def stop(self):
@ -429,15 +428,14 @@ class BackfillerWorker(object):
self.logger.info('Stopping') self.logger.info('Stopping')
self.stopping.set() self.stopping.set()
def backfill(self, hours): def backfill(self):
"""Backfill from remote node. """Backfill from remote node.
Backfill from node/channel/qualities to base_dir/channel/qualities for Backfill from node/channel/qualities to base_dir/channel/qualities for
each hour in hours. each hour in hours.
""" """
for quality in self.qualities: for channel, quality in itertools.product(self.channels, self.qualities):
for hour in list_hours(self.node, channel, quality, self.start):
for hour in hours:
# since backfilling can take a long time, recheck whether this # since backfilling can take a long time, recheck whether this
# hour is after the start # hour is after the start
if self.start is not None: if self.start is not None:
@ -450,8 +448,8 @@ class BackfillerWorker(object):
self.logger.info('Backfilling {}/{}'.format(quality, hour)) self.logger.info('Backfilling {}/{}'.format(quality, hour))
local_segments = set(list_local_segments(self.base_dir, self.channel, quality, hour)) local_segments = set(list_local_segments(self.base_dir, channel, quality, hour))
remote_segments = set(list_remote_segments(self.node, self.channel, quality, hour)) remote_segments = set(list_remote_segments(self.node, channel, quality, hour))
missing_segments = list(remote_segments - local_segments) missing_segments = list(remote_segments - local_segments)
# randomise the order of the segments to reduce the chance that # randomise the order of the segments to reduce the chance that
@ -466,7 +464,7 @@ class BackfillerWorker(object):
if self.stopping.is_set(): if self.stopping.is_set():
return return
path = os.path.join(self.channel, quality, hour, missing_segment) path = os.path.join(channel, quality, hour, missing_segment)
# test to see if file is a segment and get the segments start time # test to see if file is a segment and get the segments start time
try: try:
@ -489,7 +487,7 @@ class BackfillerWorker(object):
# start segment as soon as a pool slot opens up, then track it in workers # start segment as soon as a pool slot opens up, then track it in workers
workers.append(pool.spawn( workers.append(pool.spawn(
get_remote_segment, get_remote_segment,
self.base_dir, self.node, self.channel, quality, hour, missing_segment, self.logger self.base_dir, self.node, channel, quality, hour, missing_segment, self.logger
)) ))
# verify that all the workers succeeded. if any failed, raise the exception from # verify that all the workers succeeded. if any failed, raise the exception from
@ -498,7 +496,7 @@ class BackfillerWorker(object):
worker.get() # re-raise error, if any worker.get() # re-raise error, if any
self.logger.info('{} segments in {}/{} backfilled'.format(len(workers), quality, hour)) self.logger.info('{} segments in {}/{} backfilled'.format(len(workers), quality, hour))
hours_backfilled.labels(remote=self.node, channel=self.channel, quality=quality).inc() hours_backfilled.labels(remote=self.node, channel=channel, quality=quality).inc()
def run(self): def run(self):
@ -508,7 +506,7 @@ class BackfillerWorker(object):
while not self.stopping.is_set(): while not self.stopping.is_set():
try: try:
self.logger.info('Starting backfill') self.logger.info('Starting backfill')
self.backfill(list_hours(self.node, self.channel, self.qualities, self.start)) self.backfill()
self.logger.info('Backfill complete') self.logger.info('Backfill complete')
failures = 0 #reset failure count on a successful backfill failures = 0 #reset failure count on a successful backfill
if not self.run_once: if not self.run_once:
@ -519,7 +517,7 @@ class BackfillerWorker(object):
failures += 1 failures += 1
delay = common.jitter(TIMEOUT * 2**failures) delay = common.jitter(TIMEOUT * 2**failures)
self.logger.exception('Backfill failed. Retrying in {:.0f} s'.format(delay)) self.logger.exception('Backfill failed. Retrying in {:.0f} s'.format(delay))
backfill_errors.labels(remote=self.node, channel=self.channel).inc() backfill_errors.labels(remote=self.node).inc()
self.stopping.wait(delay) self.stopping.wait(delay)
if self.run_once: if self.run_once:
@ -567,18 +565,12 @@ def main(channels, base_dir='.', qualities='source', metrics_port=8002,
common.install_stacksampler() common.install_stacksampler()
prom.start_http_server(metrics_port) prom.start_http_server(metrics_port)
managers = [] logging.info('Starting backfilling {} with {} as qualities to {}'.format(', '.join(channels), ', '.join(qualities), base_dir))
workers = [] manager = BackfillerManager(base_dir, channels, qualities, static_nodes,
for channel in channels:
logging.info('Starting backfilling {} with {} as qualities to {}'.format(channel, ', '.join(qualities), base_dir))
manager = BackfillerManager(base_dir, channel, qualities, static_nodes,
start, delete_old, run_once, node_file, node_database, start, delete_old, run_once, node_file, node_database,
localhost, download_concurrency, recent_cutoff) localhost, download_concurrency, recent_cutoff)
managers.append(manager)
workers.append(gevent.spawn(manager.run))
def stop(): def stop():
for manager in managers:
manager.stop() manager.stop()
gevent.signal(signal.SIGTERM, stop) gevent.signal(signal.SIGTERM, stop)
@ -586,19 +578,6 @@ def main(channels, base_dir='.', qualities='source', metrics_port=8002,
if backdoor_port: if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start()
# Wait for any to die manager.run()
gevent.wait(workers, count=1)
# If one has stopped, either:
# 1. stop() was called and all are stopping
# 2. one errored and we should stop all remaining and report the error
# Our behaviour in both cases is the same:
# 1. Tell all managers to gracefully stop
stop()
# 2. Wait (with timeout) until they've stopped
gevent.wait(workers)
# 3. Check if any of them failed. If they did, report it. If mulitple
# failed, we report one arbitrarily.
for worker in workers:
worker.get()
logging.info('Gracefully stopped') logging.info('Gracefully stopped')

2
build
Unescape Escape View File

@ -3,7 +3,7 @@
set -eu set -eu
# cd to location of script # cd to location of script
cd "$(dirname "$(readlink -f "$0")")" cd "$(dirname "$(realpath "$0")")"
# Builds the docker images. # Builds the docker images.
# Usage: ./build {COMPONENTS}, or just ./build to build all. # Usage: ./build {COMPONENTS}, or just ./build to build all.

17
common/common/requests.py
Unescape Escape View File

@ -1,9 +1,12 @@
"""Code for instrumenting requests calls. Requires requests, obviously.""" """Code for instrumenting requests calls. Requires requests, obviously."""
# absolute_import prevents "import requests" in this module from just importing itself
from __future__ import absolute_import
import urlparse import urlparse
import requests import requests.sessions
import prometheus_client as prom import prometheus_client as prom
from monotonic import monotonic from monotonic import monotonic
@ -27,7 +30,7 @@ request_concurrency = prom.Gauge(
['name', 'method', 'domain'], ['name', 'method', 'domain'],
) )
class InstrumentedSession(requests.Session): class InstrumentedSession(requests.sessions.Session):
"""A requests Session that automatically records metrics on requests made. """A requests Session that automatically records metrics on requests made.
Users may optionally pass a 'metric_name' kwarg that will be included as the 'name' label. Users may optionally pass a 'metric_name' kwarg that will be included as the 'name' label.
""" """
@ -45,7 +48,11 @@ class InstrumentedSession(requests.Session):
request_latency.labels(name, method, domain, "error").observe(latency) request_latency.labels(name, method, domain, "error").observe(latency)
raise raise
request_latency.labels(name, method, domain, response.status_code).observe(response.elapsed) request_latency.labels(name, method, domain, response.status_code).observe(response.elapsed.total_seconds())
if 'content-length' in response.headers: try:
response_size.labels(name, method, domain, response.status_code).observe(response.headers['content-length']) content_length = int(response.headers['content-length'])
except (KeyError, ValueError):
pass # either not present or not valid
else:
response_size.labels(name, method, domain, response.status_code).observe(content_length)
return response return response

8
common/common/stats.py
Unescape Escape View File

@ -91,8 +91,8 @@ def timed(name=None,
# can't safely assign to name inside closure, we use a new _name variable instead # can't safely assign to name inside closure, we use a new _name variable instead
_name = fn.__name__ if name is None else name _name = fn.__name__ if name is None else name
if name in metrics: if _name in metrics:
latency, cputime = metrics[name] latency, cputime = metrics[_name]
else: else:
latency = prom.Histogram( latency = prom.Histogram(
"{}_latency".format(_name), "{}_latency".format(_name),
@ -106,9 +106,9 @@ def timed(name=None,
labels.keys() + ['error', 'type'], labels.keys() + ['error', 'type'],
buckets=buckets, buckets=buckets,
) )
metrics[name] = latency, cputime metrics[_name] = latency, cputime
if normalize: if normalize:
normname = '{} normalized'.format(name) normname = '{} normalized'.format(_name)
if normname in metrics: if normname in metrics:
normal_latency, normal_cputime = metrics[normname] normal_latency, normal_cputime = metrics[normname]
else: else:

7
cutter/cutter/main.py
Unescape Escape View File

@ -341,7 +341,7 @@ class Cutter(object):
if result.rowcount != 1: if result.rowcount != 1:
# If we hadn't yet set finalizing, then this means an operator cancelled the job # If we hadn't yet set finalizing, then this means an operator cancelled the job
# while we were cutting it. This isn't a problem. # while we were cutting it. This isn't a problem.
if not finalize_begun: if not finalize_begun[0]:
raise JobCancelled() raise JobCancelled()
raise JobConsistencyError("No job with id {} and uploader {} when setting: {}".format( raise JobConsistencyError("No job with id {} and uploader {} when setting: {}".format(
job.id, self.name, ", ".join("{} = {!r}".format(k, v) for k, v in kwargs.items()) job.id, self.name, ", ".join("{} = {!r}".format(k, v) for k, v in kwargs.items())
@ -444,7 +444,8 @@ class Cutter(object):
# Success! Set TRANSCODING or DONE and clear any previous error. # Success! Set TRANSCODING or DONE and clear any previous error.
success_state = 'TRANSCODING' if upload_backend.needs_transcode else 'DONE' success_state = 'TRANSCODING' if upload_backend.needs_transcode else 'DONE'
set_row(state=success_state, video_id=video_id, video_link=video_link, error=None) maybe_upload_time = {"upload_time": datetime.datetime.utcnow()} if success_state == 'DONE' else {}
set_row(state=success_state, video_id=video_id, video_link=video_link, error=None, **maybe_upload_time)
self.logger.info("Successfully cut and uploaded job {} as {}".format(format_job(job), video_link)) self.logger.info("Successfully cut and uploaded job {} as {}".format(format_job(job), video_link))
videos_uploaded.labels(video_channel=job.video_channel, videos_uploaded.labels(video_channel=job.video_channel,
@ -617,7 +618,7 @@ def main(
conn = dbmanager.get_conn() conn = dbmanager.get_conn()
except Exception: except Exception:
delay = common.jitter(10) delay = common.jitter(10)
logging.info('Cannot connect to database. Retrying in {:.0f} s'.format(delay)) logging.warning('Cannot connect to database. Retrying in {:.0f} s'.format(delay), exc_info=True)
stop.wait(delay) stop.wait(delay)
else: else:
# put it back so it gets reused on next get_conn() # put it back so it gets reused on next get_conn()

7
cutter/cutter/upload_backends.py
Unescape Escape View File

@ -76,7 +76,7 @@ class UploadBackend(object):
# reasonable default if settings don't otherwise matter: # reasonable default if settings don't otherwise matter:
# high-quality mpegts, without wasting too much cpu on encoding # high-quality mpegts, without wasting too much cpu on encoding
encoding_args = ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '0', '-f', 'mpegts'] encoding_settings = ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '0', '-f', 'mpegts']
encoding_streamable = True encoding_streamable = True
def upload_video(self, title, description, tags, data): def upload_video(self, title, description, tags, data):
@ -224,11 +224,10 @@ class Local(UploadBackend):
# ignore already-exists errors # ignore already-exists errors
def upload_video(self, title, description, tags, data): def upload_video(self, title, description, tags, data):
video_id = uuid.uuid4() video_id = str(uuid.uuid4())
# make title safe by removing offending characters, replacing with '-' # make title safe by removing offending characters, replacing with '-'
safe_title = re.sub('[^A-Za-z0-9_]', '-', title) safe_title = re.sub('[^A-Za-z0-9_]', '-', title)
# If fast cut enabled, use .ts, otherwise use .mp4 ext = 'ts'
ext = 'ts' if self.encoding_settings is None else 'mp4'
filename = '{}-{}.{}'.format(safe_title, video_id, ext) filename = '{}-{}.{}'.format(safe_title, video_id, ext)
filepath = os.path.join(self.path, filename) filepath = os.path.join(self.path, filename)
try: try:

5
docker-compose.jsonnet
Unescape Escape View File

@ -127,13 +127,13 @@
// The spreadsheet id and worksheet names for sheet sync to act on // The spreadsheet id and worksheet names for sheet sync to act on
sheet_id:: "your_id_here", sheet_id:: "your_id_here",
worksheets:: ["Tech Test & Pre-Show"] + ["Day %d" % n for n in std.range(1, 7)], worksheets:: ["Tech Test & Preshow"] + ["Day %d" % n for n in std.range(1, 7)],
// Now for the actual docker-compose config // Now for the actual docker-compose config
// The connection string for the database. Constructed from db_args. // The connection string for the database. Constructed from db_args.
db_connect:: std.join(" ", [ db_connect:: std.join(" ", [
"%s=%s" % [key, $.db_args[key]] "%s='%s'" % [key, $.db_args[key]]
for key in std.objectFields($.db_args) for key in std.objectFields($.db_args)
]), ]),
@ -206,6 +206,7 @@
"--base-dir", "/mnt", "--base-dir", "/mnt",
"--backdoor-port", std.toString($.backdoor_port), "--backdoor-port", std.toString($.backdoor_port),
"--tags", std.join(",", $.video_tags), "--tags", std.join(",", $.video_tags),
"--name", $.localhost,
$.db_connect, $.db_connect,
std.manifestJson($.cutter_config), std.manifestJson($.cutter_config),
"/etc/wubloader-creds.json", "/etc/wubloader-creds.json",

8
downloader/downloader/main.py
Unescape Escape View File

@ -126,6 +126,7 @@ class StreamsManager(object):
self.refresh_needed = gevent.event.Event() # set to tell main loop to refresh now self.refresh_needed = gevent.event.Event() # set to tell main loop to refresh now
self.stopping = gevent.event.Event() # set to tell main loop to stop self.stopping = gevent.event.Event() # set to tell main loop to stop
self.important = important self.important = important
self.master_playlist_log_level = logging.INFO if important else logging.DEBUG
if self.important: if self.important:
self.FETCH_MIN_INTERVAL = self.IMPORTANT_FETCH_MIN_INTERVAL self.FETCH_MIN_INTERVAL = self.IMPORTANT_FETCH_MIN_INTERVAL
@ -203,7 +204,7 @@ class StreamsManager(object):
"""Re-fetch master playlist and start new workers if needed""" """Re-fetch master playlist and start new workers if needed"""
try: try:
# Fetch playlist. On soft timeout, retry. # Fetch playlist. On soft timeout, retry.
self.logger.info("Fetching master playlist") self.logger.log(self.master_playlist_log_level, "Fetching master playlist")
fetch_time = monotonic() fetch_time = monotonic()
with soft_hard_timeout(self.logger, "fetching master playlist", self.FETCH_TIMEOUTS, self.trigger_refresh): with soft_hard_timeout(self.logger, "fetching master playlist", self.FETCH_TIMEOUTS, self.trigger_refresh):
master_playlist = twitch.get_master_playlist(self.channel) master_playlist = twitch.get_master_playlist(self.channel)
@ -226,8 +227,7 @@ class StreamsManager(object):
except Exception as e: except Exception as e:
if isinstance(e, requests.HTTPError) and e.response is not None and e.response.status_code == 404: if isinstance(e, requests.HTTPError) and e.response is not None and e.response.status_code == 404:
# Log about important streams being down at info, but others at debug. # Log about important streams being down at info, but others at debug.
level = logging.INFO if self.important else logging.DEBUG self.logger.log(self.master_playlist_log_level, "Stream is not up. Retrying.")
self.logger.log(level, "Stream is not up. Retrying.")
self.trigger_refresh() self.trigger_refresh()
else: else:
self.logger.exception("Failed to fetch master playlist") self.logger.exception("Failed to fetch master playlist")
@ -243,7 +243,7 @@ class StreamsManager(object):
self.MAX_WORKER_AGE - workers[-1].age() self.MAX_WORKER_AGE - workers[-1].age()
for workers in self.stream_workers.values() if workers for workers in self.stream_workers.values() if workers
] or [0])) ] or [0]))
self.logger.info("Next master playlist refresh in at most {} sec".format(time_to_next_max_age)) self.logger.log(self.master_playlist_log_level, "Next master playlist refresh in at most {} sec".format(time_to_next_max_age))
# wait until refresh triggered, next max age reached, or we're stopping (whichever happens first) # wait until refresh triggered, next max age reached, or we're stopping (whichever happens first)
gevent.wait([self.stopping, self.refresh_needed], timeout=time_to_next_max_age, count=1) gevent.wait([self.stopping, self.refresh_needed], timeout=time_to_next_max_age, count=1)
if not self.stopping.is_set(): if not self.stopping.is_set():

2
generate-docker-compose
Unescape Escape View File

@ -3,7 +3,7 @@
set -eu set -eu
# cd to location of script # cd to location of script
cd "$(dirname "$(readlink -f "$0")")" cd "$(dirname "$(realpath "$0")")"
# We generate first, and capture the output, to avoid overwriting the file on error. # We generate first, and capture the output, to avoid overwriting the file on error.
# To avoid jsonnet needing to exist locally, we run it in a container. # To avoid jsonnet needing to exist locally, we run it in a container.

2
get-build-tag
Unescape Escape View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
# cd to location of script # cd to location of script
cd "$(dirname "$(readlink -f "$0")")" cd "$(dirname "$(realpath "$0")")"
TAG=$(git rev-parse --short HEAD) TAG=$(git rev-parse --short HEAD)
if [ -n "$(git status --porcelain --untracked-files=no)" ]; then if [ -n "$(git status --porcelain --untracked-files=no)" ]; then

2
segment_coverage/segment_coverage/main.py
Unescape Escape View File

@ -192,7 +192,7 @@ class CoverageChecker(object):
hours = [name for name in os.listdir(path) if not name.startswith('.')] hours = [name for name in os.listdir(path) if not name.startswith('.')]
except OSError as e: except OSError as e:
if e.errno == errno.ENOENT: if e.errno == errno.ENOENT:
self.logger.warning('{} does not exist'.format(path)) self.logger.info('{} does not exist, skipping'.format(path))
continue continue
hours.sort() hours.sort()

10
sheetsync/sheetsync/main.py
Unescape Escape View File

@ -214,11 +214,11 @@ class SheetSync(object):
sql.SQL(", ").join(sql.Placeholder(col) for col in insert_cols), sql.SQL(", ").join(sql.Placeholder(col) for col in insert_cols),
) )
query(self.conn, built_query, sheet_name=worksheet, **row) query(self.conn, built_query, sheet_name=worksheet, **row)
rows_found(worksheet).inc() rows_found.labels(worksheet).inc()
rows_changed('insert', worksheet).inc() rows_changed.labels('insert', worksheet).inc()
return return
rows_found(worksheet).inc() rows_found.labels(worksheet).inc()
# Update database with any changed inputs # Update database with any changed inputs
changed = [col for col in self.input_columns if row[col] != getattr(event, col)] changed = [col for col in self.input_columns if row[col] != getattr(event, col)]
@ -236,7 +236,7 @@ class SheetSync(object):
) for col in changed ) for col in changed
)) ))
query(self.conn, built_query, **row) query(self.conn, built_query, **row)
rows_changed('input', worksheet).inc() rows_changed.labels('input', worksheet).inc()
# Update sheet with any changed outputs # Update sheet with any changed outputs
format_output = lambda v: '' if v is None else v # cast nulls to empty string format_output = lambda v: '' if v is None else v # cast nulls to empty string
@ -251,7 +251,7 @@ class SheetSync(object):
row_index, self.column_map[col], row_index, self.column_map[col],
format_output(getattr(event, col)), format_output(getattr(event, col)),
) )
rows_changed('output', worksheet).inc() rows_changed.labels('output', worksheet).inc()
# Set edit link if marked for editing and start/end set. # Set edit link if marked for editing and start/end set.
# This prevents accidents / clicking the wrong row and provides # This prevents accidents / clicking the wrong row and provides

7
thrimbletrimmer/scripts/IO.js
Unescape Escape View File

@ -40,7 +40,11 @@ pageSetup = function(isEditor) {
document.getElementById("AllowHoles").checked = data.allow_holes; document.getElementById("AllowHoles").checked = data.allow_holes;
document.getElementById("uploaderWhitelist").value = (!!data.uploader_whitelist) ? data.uploader_whitelist.join(",") : ""; document.getElementById("uploaderWhitelist").value = (!!data.uploader_whitelist) ? data.uploader_whitelist.join(",") : "";
if ( if (
(data.upload_locations.length > 0 && data.upload_location != data.upload_locations[0]) (
data.upload_locations.length > 0
&& data.upload_location != null
&& data.upload_location != data.upload_locations[0]
)
|| data.allow_holes || data.allow_holes
|| !!data.uploader_whitelist || !!data.uploader_whitelist
) { ) {
@ -253,6 +257,7 @@ thrimbletrimmerSubmit = function(state) {
setTimeout(() => { window.location.href = '/thrimbletrimmer/dashboard.html'; }, 500); setTimeout(() => { window.location.href = '/thrimbletrimmer/dashboard.html'; }, 500);
return return
} }
alert("Draft saved");
document.getElementById('SubmitButton').disabled = false; document.getElementById('SubmitButton').disabled = false;
})); }));
}; };

2
thrimbletrimmer/stream.html
Unescape Escape View File

@ -57,8 +57,6 @@
To fix this, re-load the video in the desired time range (default: the last 10 minutes) by clicking Load Playlist.<br/> To fix this, re-load the video in the desired time range (default: the last 10 minutes) by clicking Load Playlist.<br/>
Download Quality: <select id="qualityLevel"></select> Download Quality: <select id="qualityLevel"></select>
<input type="button" id="DownloadButton" value="Download this time range" onclick="thrimbletrimmerDownload()"/> <input type="button" id="DownloadButton" value="Download this time range" onclick="thrimbletrimmerDownload()"/>
<a href="/thrimbletrimmer/dashboard.html">Go To Dashboard</a> |
<a href="/thrimbletrimmer/dashboard.html">Go To Editor</a>
<a id="HelpButton" style="float:right;" href="JavaScript:toggleHiddenPane('HelpPane');">Help</a> <a id="HelpButton" style="float:right;" href="JavaScript:toggleHiddenPane('HelpPane');">Help</a>
<a id="UltrawideButton" style="float:right;margin-right:10px;" href="JavaScript:toggleUltrawide();">Ultrawide</a> <a id="UltrawideButton" style="float:right;margin-right:10px;" href="JavaScript:toggleUltrawide();">Ultrawide</a>
</div> </div>

3
thrimbletrimmer/styles/style.css
Unescape Escape View File

@ -24,6 +24,9 @@ body.ultrawide .my-player-dimensions { width:100% !important; }
.vjs-menu-button-popup .vjs-menu { .vjs-menu-button-popup .vjs-menu {
bottom:-3px; bottom:-3px;
} }
.video-js .vjs-time-display {
width: 100px;
}
.video-js .vjs-picture-in-picture-control { .video-js .vjs-picture-in-picture-control {
display: none; display: none;
} }

Write Preview
Loading…
Cancel
Save