sheet sync: Basic skeleton

build

@@ -8,7 +8,7 @@ set -eu
 # Pass PUSH=true to also push the resulting images, or PUSH=latest to push them as :latest tag
 
 # The different images we can build
-COMPONENTS=(downloader restreamer backfiller thrimshim cutter)
+COMPONENTS=(downloader restreamer backfiller thrimshim cutter sheetsync)
 
 # Define push if not already defined
 PUSH=${PUSH:-}

sheetsync/Dockerfile

@@ -0,0 +1,16 @@
+FROM alpine:3.7
+# dependencies needed for compiling c extensions
+# also busybox-extras for telnet for easier use of backdoor
+# and postgresql-dev as a dependency of psycopg2.
+# Add postgresql-client for easier debugging of DB issues.
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-client
+
+# Install common lib first as it changes less
+COPY common /tmp/common
+RUN pip install /tmp/common && rm -r /tmp/common
+
+# Install actual application
+COPY sheetsync /tmp/sheetsync
+RUN pip install /tmp/sheetsync && rm -r /tmp/sheetsync
+
+ENTRYPOINT ["python2", "-m", "sheetsync", "--base-dir", "/mnt"]

sheetsync/setup.py

@@ -0,0 +1,15 @@
+from setuptools import setup, find_packages
+
+setup(
+	name = "wubloader-cutter",
+	version = "0.0.0",
+	packages = find_packages(),
+	install_requires = [
+		"argh",
+		"gevent",
+		"prometheus-client",
+		"psycogreen",
+		"psycopg2",
+		"wubloader-common",
+	],
+)

sheetsync/sheetsync/__main__.py

@@ -0,0 +1,16 @@
+
+import gevent.monkey
+gevent.monkey.patch_all()
+
+import logging
+import os
+
+import argh
+
+from sheetsync.main import main
+
+LOG_FORMAT = "[%(asctime)s] %(levelname)8s %(name)s(%(module)s:%(lineno)d): %(message)s"
+
+level = os.environ.get('WUBLOADER_LOG_LEVEL', 'INFO').upper()
+logging.basicConfig(level=level, format=LOG_FORMAT)
+argh.dispatch_command(main)

sheetsync/sheetsync/main.py

@@ -0,0 +1,40 @@
+
+import json
+import logging
+import signal
+
+import argh
+import gevent.backdoor
+import gevent.event
+import prometheus_client as prom
+
+import common
+from common.database import DBManager
+
+
+@argh.arg('worksheet-names', nargs='+', help="The names of the individual worksheets within the sheet to operate on.")
+def main(dbconnect, sheets_creds_file, sheet_id, worksheet_names, metrics_port=8004, backdoor_port=0):
+	"""dbconnect should be a postgres connection string, which is either a space-separated
+	list of key=value pairs, or a URI like:
+		postgresql://USER:PASSWORD@HOST/DBNAME?KEY=VALUE
+
+	sheets_creds_file should be a json file containing keys 'client_id', 'client_secret' and 'refresh_token'.
+	"""
+	common.PromLogCountsHandler.install()
+	common.install_stacksampler()
+	prom.start_http_server(metrics_port)
+
+	if backdoor_port:
+		gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start()
+
+	stop = gevent.event.Event()
+	gevent.signal(signal.SIGTERM, stop.set) # shut down on sigterm
+
+	logging.info("Starting up")
+
+	dbmanager = DBManager(dsn=dbconnect)
+	sheets_creds = json.load(open(sheets_creds_file))
+
+	# TODO the thing
+
+	logging.info("Gracefully stopped")