diff --git a/build b/build index 5096c03..e231e7d 100755 --- a/build +++ b/build @@ -8,7 +8,7 @@ set -eu # Pass PUSH=true to also push the resulting images, or PUSH=latest to push them as :latest tag # The different images we can build -COMPONENTS=(downloader restreamer backfiller thrimshim cutter) +COMPONENTS=(downloader restreamer backfiller thrimshim cutter sheetsync) # Define push if not already defined PUSH=${PUSH:-} diff --git a/sheetsync/Dockerfile b/sheetsync/Dockerfile new file mode 100644 index 0000000..31648ad --- /dev/null +++ b/sheetsync/Dockerfile @@ -0,0 +1,16 @@ +FROM alpine:3.7 +# dependencies needed for compiling c extensions +# also busybox-extras for telnet for easier use of backdoor +# and postgresql-dev as a dependency of psycopg2. +# Add postgresql-client for easier debugging of DB issues. +RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-client + +# Install common lib first as it changes less +COPY common /tmp/common +RUN pip install /tmp/common && rm -r /tmp/common + +# Install actual application +COPY sheetsync /tmp/sheetsync +RUN pip install /tmp/sheetsync && rm -r /tmp/sheetsync + +ENTRYPOINT ["python2", "-m", "sheetsync", "--base-dir", "/mnt"] diff --git a/sheetsync/setup.py b/sheetsync/setup.py new file mode 100644 index 0000000..05b5bbf --- /dev/null +++ b/sheetsync/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +setup( + name = "wubloader-cutter", + version = "0.0.0", + packages = find_packages(), + install_requires = [ + "argh", + "gevent", + "prometheus-client", + "psycogreen", + "psycopg2", + "wubloader-common", + ], +) diff --git a/sheetsync/sheetsync/__init__.py b/sheetsync/sheetsync/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sheetsync/sheetsync/__main__.py b/sheetsync/sheetsync/__main__.py new file mode 100644 index 0000000..68ac65a --- /dev/null +++ b/sheetsync/sheetsync/__main__.py @@ -0,0 +1,16 @@ + +import gevent.monkey +gevent.monkey.patch_all() + +import logging +import os + +import argh + +from sheetsync.main import main + +LOG_FORMAT = "[%(asctime)s] %(levelname)8s %(name)s(%(module)s:%(lineno)d): %(message)s" + +level = os.environ.get('WUBLOADER_LOG_LEVEL', 'INFO').upper() +logging.basicConfig(level=level, format=LOG_FORMAT) +argh.dispatch_command(main) diff --git a/sheetsync/sheetsync/main.py b/sheetsync/sheetsync/main.py new file mode 100644 index 0000000..703f87b --- /dev/null +++ b/sheetsync/sheetsync/main.py @@ -0,0 +1,40 @@ + +import json +import logging +import signal + +import argh +import gevent.backdoor +import gevent.event +import prometheus_client as prom + +import common +from common.database import DBManager + + +@argh.arg('worksheet-names', nargs='+', help="The names of the individual worksheets within the sheet to operate on.") +def main(dbconnect, sheets_creds_file, sheet_id, worksheet_names, metrics_port=8004, backdoor_port=0): + """dbconnect should be a postgres connection string, which is either a space-separated + list of key=value pairs, or a URI like: + postgresql://USER:PASSWORD@HOST/DBNAME?KEY=VALUE + + sheets_creds_file should be a json file containing keys 'client_id', 'client_secret' and 'refresh_token'. + """ + common.PromLogCountsHandler.install() + common.install_stacksampler() + prom.start_http_server(metrics_port) + + if backdoor_port: + gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() + + stop = gevent.event.Event() + gevent.signal(signal.SIGTERM, stop.set) # shut down on sigterm + + logging.info("Starting up") + + dbmanager = DBManager(dsn=dbconnect) + sheets_creds = json.load(open(sheets_creds_file)) + + # TODO the thing + + logging.info("Gracefully stopped")