diff --git a/build b/build index 3b438f6..5096c03 100755 --- a/build +++ b/build @@ -8,7 +8,7 @@ set -eu # Pass PUSH=true to also push the resulting images, or PUSH=latest to push them as :latest tag # The different images we can build -COMPONENTS=(downloader restreamer backfiller thrimshim) +COMPONENTS=(downloader restreamer backfiller thrimshim cutter) # Define push if not already defined PUSH=${PUSH:-} diff --git a/cutter/Dockerfile b/cutter/Dockerfile new file mode 100644 index 0000000..f44e16f --- /dev/null +++ b/cutter/Dockerfile @@ -0,0 +1,16 @@ +FROM alpine:3.7 +# dependencies needed for compiling c extensions +# also busybox-extras for telnet for easier use of backdoor +# and postgresql-dev as a dependency of psycopg2. +# Add postgresql-client for easier debugging of DB issues. +RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-client + +# Install common lib first as it changes less +COPY common /tmp/common +RUN pip install /tmp/common && rm -r /tmp/common + +# Install actual application +COPY cutter /tmp/cutter +RUN pip install /tmp/cutter && rm -r /tmp/cutter + +ENTRYPOINT ["python2", "-m", "cutter", "--base-dir", "/mnt"] diff --git a/cutter/cutter/__init__.py b/cutter/cutter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cutter/cutter/__main__.py b/cutter/cutter/__main__.py new file mode 100644 index 0000000..e30e676 --- /dev/null +++ b/cutter/cutter/__main__.py @@ -0,0 +1,16 @@ + +import gevent.monkey +gevent.monkey.patch_all() + +import logging +import os + +import argh + +from cutter.main import main + +LOG_FORMAT = "[%(asctime)s] %(levelname)8s %(name)s(%(module)s:%(lineno)d): %(message)s" + +level = os.environ.get('WUBLOADER_LOG_LEVEL', 'INFO').upper() +logging.basicConfig(level=level, format=LOG_FORMAT) +argh.dispatch_command(main) diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py new file mode 100644 index 0000000..83262c4 --- /dev/null +++ b/cutter/cutter/main.py @@ -0,0 +1,67 @@ + +import logging +import signal + +import gevent.backdoor +import gevent.event +import prometheus_client as prom + + +class Cutter(object): + def __init__(self, stop): + """Stop is an Event triggering graceful shutdown when set.""" + self.stop = stop + self.logger = logging.getLogger(type(self).__name__) + + + def run(self): + while not self.stop.is_set(): + pass + + +class TranscodeChecker(object): + def __init__(self, stop): + """ + Stop is an Event triggering graceful shutdown when set. + """ + self.stop = stop + self.logger = logging.getLogger(type(self).__name__) + + def run(self): + while not self.stop.is_set(): + pass + + +def main(metrics_port=8003, backdoor_port=0): + common.PromLogCountsHandler.install() + common.install_stacksampler() + prom.start_http_server(metrics_port) + + if backdoor_port: + gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() + + stop = gevent.event.Event() + gevent.signal(signal.SIGTERM, stop.set) # shut down on sigterm + + logging.info("Starting up") + + # We have two independent jobs to do - to perform cut jobs (cutter), + # and to check the status of transcoding videos to see if they're done (transcode checker). + # We want to error if either errors, and shut down if either exits. + cutter = Cutter(stop) + transcode_checker = TranscodeChecker(stop) + jobs = [ + gevent.spawn(cutter.run), + gevent.spawn(transcode_checker.run), + ] + # Block until either exits + gevent.wait(jobs, count=1) + # Stop the other if it isn't stopping already + stop.set() + # Block until both have exited + gevent.wait(jobs) + # Call get() for each to re-raise if either errored + for job in jobs: + job.get() + + logging.info("Gracefully stopped") diff --git a/cutter/setup.py b/cutter/setup.py new file mode 100644 index 0000000..0483c07 --- /dev/null +++ b/cutter/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup, find_packages + +setup( + name = "wubloader-cutter", + version = "0.0.0", + packages = find_packages(), + install_requires = [ + "argh", + "gevent", + "prometheus-client", + "psycogreen", + "psycopg2", + "requests", + "wubloader-common", + ], +)