cutter: Add initial outline

The cutter has two jobs:
* To cut videos, taking them through states EDITED -> TRANSCODING
* To monitor TRANSCODING videos for when they're complete

We run these as separate greenlets with their own DB connections,
and if either dies we gracefully shut down the other.

build

@@ -8,7 +8,7 @@ set -eu
 # Pass PUSH=true to also push the resulting images, or PUSH=latest to push them as :latest tag
 
 # The different images we can build
-COMPONENTS=(downloader restreamer backfiller thrimshim)
+COMPONENTS=(downloader restreamer backfiller thrimshim cutter)
 
 # Define push if not already defined
 PUSH=${PUSH:-}

cutter/Dockerfile

@@ -0,0 +1,16 @@
+FROM alpine:3.7
+# dependencies needed for compiling c extensions
+# also busybox-extras for telnet for easier use of backdoor
+# and postgresql-dev as a dependency of psycopg2.
+# Add postgresql-client for easier debugging of DB issues.
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-client
+
+# Install common lib first as it changes less
+COPY common /tmp/common
+RUN pip install /tmp/common && rm -r /tmp/common
+
+# Install actual application
+COPY cutter /tmp/cutter
+RUN pip install /tmp/cutter && rm -r /tmp/cutter
+
+ENTRYPOINT ["python2", "-m", "cutter", "--base-dir", "/mnt"]

cutter/cutter/__main__.py

@@ -0,0 +1,16 @@
+
+import gevent.monkey
+gevent.monkey.patch_all()
+
+import logging
+import os
+
+import argh
+
+from cutter.main import main
+
+LOG_FORMAT = "[%(asctime)s] %(levelname)8s %(name)s(%(module)s:%(lineno)d): %(message)s"
+
+level = os.environ.get('WUBLOADER_LOG_LEVEL', 'INFO').upper()
+logging.basicConfig(level=level, format=LOG_FORMAT)
+argh.dispatch_command(main)

cutter/cutter/main.py

@@ -0,0 +1,67 @@
+
+import logging
+import signal
+
+import gevent.backdoor
+import gevent.event
+import prometheus_client as prom
+
+
+class Cutter(object):
+	def __init__(self, stop):
+		"""Stop is an Event triggering graceful shutdown when set."""
+		self.stop = stop
+		self.logger = logging.getLogger(type(self).__name__)
+
+
+	def run(self):
+		while not self.stop.is_set():
+			pass
+
+
+class TranscodeChecker(object):
+	def __init__(self, stop):
+		"""
+		Stop is an Event triggering graceful shutdown when set.
+		"""
+		self.stop = stop
+		self.logger = logging.getLogger(type(self).__name__)
+
+	def run(self):
+		while not self.stop.is_set():
+			pass
+
+
+def main(metrics_port=8003, backdoor_port=0):
+	common.PromLogCountsHandler.install()
+	common.install_stacksampler()
+	prom.start_http_server(metrics_port)
+
+	if backdoor_port:
+		gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start()
+
+	stop = gevent.event.Event()
+	gevent.signal(signal.SIGTERM, stop.set) # shut down on sigterm
+
+	logging.info("Starting up")
+
+	# We have two independent jobs to do - to perform cut jobs (cutter),
+	# and to check the status of transcoding videos to see if they're done (transcode checker).
+	# We want to error if either errors, and shut down if either exits.
+	cutter = Cutter(stop)
+	transcode_checker = TranscodeChecker(stop)
+	jobs = [
+		gevent.spawn(cutter.run),
+		gevent.spawn(transcode_checker.run),
+	]
+	# Block until either exits
+	gevent.wait(jobs, count=1)
+	# Stop the other if it isn't stopping already
+	stop.set()
+	# Block until both have exited
+	gevent.wait(jobs)
+	# Call get() for each to re-raise if either errored
+	for job in jobs:
+		job.get()
+
+	logging.info("Gracefully stopped")

cutter/setup.py

@@ -0,0 +1,16 @@
+from setuptools import setup, find_packages
+
+setup(
+	name = "wubloader-cutter",
+	version = "0.0.0",
+	packages = find_packages(),
+	install_requires = [
+		"argh",
+		"gevent",
+		"prometheus-client",
+		"psycogreen",
+		"psycopg2",
+		"requests",
+		"wubloader-common",
+	],
+)