From 9af7795f3413a22b46ae46d7b80f8cd71fd6cd04 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Sun, 6 Jan 2019 22:35:30 -0800 Subject: [PATCH] Add gevent.backdoor as an optional arg to all services Backdoor allows the operator to telnet into the given port, and get a python shell running inside the process, from which you can debug, modify state (eg. set the log level), or whatever. This is extremely useful for debugging weird states that you encounter randomly but can't easily reproduce, without restarting the process and needing to wait until it happens again. --- backfiller/backfiller/main.py | 8 ++++++-- downloader/downloader/main.py | 5 ++++- restreamer/restreamer/main.py | 6 +++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/backfiller/backfiller/main.py b/backfiller/backfiller/main.py index c120668..2b2288f 100644 --- a/backfiller/backfiller/main.py +++ b/backfiller/backfiller/main.py @@ -9,8 +9,9 @@ import random import time import uuid -import requests +import gevent.backdoor import prometheus_client as prom +import requests import common @@ -228,7 +229,7 @@ def backfill_node(base_dir, node, stream, variants, hours=None, segment_order='r logging.info('Finished backfilling from {}'.format(node)) -def main(base_dir='.', stream='', variants='', fill_wait=5, full_fill_wait=180, sleep_time=1, metrics_port=8002, nodes=None): +def main(base_dir='.', stream='', variants='', fill_wait=5, full_fill_wait=180, sleep_time=1, metrics_port=8002, nodes=None, backdoor_port=0): """Prototype backfiller service. Do a backfill of the last 3 hours from stream/variants from all nodes @@ -248,6 +249,9 @@ def main(base_dir='.', stream='', variants='', fill_wait=5, full_fill_wait=180, common.PromLogCountsHandler.install() prom.start_http_server(metrics_port) + if backdoor_port: + gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() + logging.info('Starting backfilling {} with {} as variants to {}'.format(stream, ', '.join(variants), base_dir)) fill_start = datetime.datetime.now() diff --git a/downloader/downloader/main.py b/downloader/downloader/main.py index 1d98606..50989ac 100644 --- a/downloader/downloader/main.py +++ b/downloader/downloader/main.py @@ -12,6 +12,7 @@ from contextlib import contextmanager import dateutil.parser import gevent +import gevent.backdoor import gevent.event import prometheus_client as prom import requests @@ -502,12 +503,14 @@ class SegmentGetter(object): segments_downloaded.labels(partial="False", stream=self.channel, variant=self.stream).inc() -def main(channel, base_dir=".", qualities="source", metrics_port=8001): +def main(channel, base_dir=".", qualities="source", metrics_port=8001, backdoor_port=0): qualities = qualities.split(",") if qualities else [] manager = StreamsManager(channel, base_dir, qualities) gevent.signal(signal.SIGTERM, manager.stop) # shut down on sigterm common.PromLogCountsHandler.install() prom.start_http_server(metrics_port) + if backdoor_port: + gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() logging.info("Starting up") manager.run() logging.info("Gracefully stopped") diff --git a/restreamer/restreamer/main.py b/restreamer/restreamer/main.py index c08a902..93af7cc 100644 --- a/restreamer/restreamer/main.py +++ b/restreamer/restreamer/main.py @@ -11,6 +11,7 @@ from contextlib import closing import dateutil.parser import gevent +import gevent.backdoor import prometheus_client as prom from flask import Flask, url_for, request, abort, Response from gevent import subprocess @@ -388,7 +389,7 @@ def cut_experimental(segments, cut_start, cut_end): -def main(host='0.0.0.0', port=8000, base_dir='.'): +def main(host='0.0.0.0', port=8000, base_dir='.', backdoor_port=0): app.static_folder = base_dir server = WSGIServer((host, port), cors(app)) @@ -399,6 +400,9 @@ def main(host='0.0.0.0', port=8000, base_dir='.'): PromLogCountsHandler.install() + if backdoor_port: + gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() + logging.info("Starting up") server.serve_forever() logging.info("Gracefully shut down")