Add gevent.backdoor as an optional arg to all services

Backdoor allows the operator to telnet into the given port, and get a python shell
running inside the process, from which you can debug, modify state (eg. set the log level),
or whatever. This is extremely useful for debugging weird states that you encounter randomly
but can't easily reproduce, without restarting the process and needing to wait until it happens again.

backfiller/backfiller/main.py

@@ -9,8 +9,9 @@ import random
 import time
 import uuid
 
-import requests
+import gevent.backdoor
 import prometheus_client as prom
+import requests
 
 import common
 
@@ -228,7 +229,7 @@ def backfill_node(base_dir, node, stream, variants, hours=None, segment_order='r
 	logging.info('Finished backfilling from {}'.format(node))
 
 							
-def main(base_dir='.', stream='', variants='', fill_wait=5, full_fill_wait=180, sleep_time=1, metrics_port=8002, nodes=None):
+def main(base_dir='.', stream='', variants='', fill_wait=5, full_fill_wait=180, sleep_time=1, metrics_port=8002, nodes=None, backdoor_port=0):
 	"""Prototype backfiller service.
 
 	Do a backfill of the last 3 hours from stream/variants from all nodes
@@ -248,6 +249,9 @@ def main(base_dir='.', stream='', variants='', fill_wait=5, full_fill_wait=180,
 	common.PromLogCountsHandler.install()
 	prom.start_http_server(metrics_port)
 
+	if backdoor_port:
+		gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start()
+
 	logging.info('Starting backfilling {} with {} as variants to {}'.format(stream, ', '.join(variants), base_dir))
 
 	fill_start = datetime.datetime.now()

downloader/downloader/main.py

@@ -12,6 +12,7 @@ from contextlib import contextmanager
 
 import dateutil.parser
 import gevent
+import gevent.backdoor
 import gevent.event
 import prometheus_client as prom
 import requests
@@ -500,12 +501,14 @@ class SegmentGetter(object):
 			segments_downloaded.labels(partial="False", stream=self.channel, variant=self.stream).inc()
 
 
-def main(channel, base_dir=".", qualities="source", metrics_port=8001):
+def main(channel, base_dir=".", qualities="source", metrics_port=8001, backdoor_port=0):
 	qualities = qualities.split(",") if qualities else []
 	manager = StreamsManager(channel, base_dir, qualities)
 	gevent.signal(signal.SIGTERM, manager.stop) # shut down on sigterm
 	common.PromLogCountsHandler.install()
 	prom.start_http_server(metrics_port)
+	if backdoor_port:
+		gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start()
 	logging.info("Starting up")
 	manager.run()
 	logging.info("Gracefully stopped")

restreamer/restreamer/main.py

@@ -11,6 +11,7 @@ from contextlib import closing
 
 import dateutil.parser
 import gevent
+import gevent.backdoor
 import prometheus_client as prom
 from flask import Flask, url_for, request, abort, Response
 from gevent import subprocess
@@ -388,7 +389,7 @@ def cut_experimental(segments, cut_start, cut_end):
 
 
 
-def main(host='0.0.0.0', port=8000, base_dir='.'):
+def main(host='0.0.0.0', port=8000, base_dir='.', backdoor_port=0):
 	app.static_folder = base_dir
 	server = WSGIServer((host, port), cors(app))
 
@@ -399,6 +400,9 @@ def main(host='0.0.0.0', port=8000, base_dir='.'):
 
 	PromLogCountsHandler.install()
 
+	if backdoor_port:
+		gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start()
+
 	logging.info("Starting up")
 	server.serve_forever()
 	logging.info("Gracefully shut down")