diff --git a/backfiller/backfiller/main.py b/backfiller/backfiller/main.py index bec75a7..cb51ede 100644 --- a/backfiller/backfiller/main.py +++ b/backfiller/backfiller/main.py @@ -16,6 +16,7 @@ import argh import gevent.backdoor import gevent.pool import prometheus_client as prom +from requests.adapters import HTTPAdapter import common from common import dateutil @@ -23,8 +24,10 @@ from common import database from common.requests import InstrumentedSession from common.segments import list_segment_files, unpadded_b64_decode -# Wraps all requests in some metric collection +# Wraps all requests in some metric collection and connection pooling requests = InstrumentedSession() +adapter = HTTPAdapter(pool_maxsize=100) +requests.mount('https://', adapter) segments_backfilled = prom.Counter( 'segments_backfilled', diff --git a/downloader/downloader/main.py b/downloader/downloader/main.py index f36d70b..5186945 100644 --- a/downloader/downloader/main.py +++ b/downloader/downloader/main.py @@ -15,6 +15,7 @@ import gevent.backdoor import gevent.event import prometheus_client as prom import requests +import requests.adapters from monotonic import monotonic import common @@ -335,6 +336,8 @@ class StreamWorker(object): # This worker's SegmentGetters will use its session by default for performance, # but will fall back to a new one if something goes wrong. self.session = common.requests.InstrumentedSession() + adapter = requests.adapters.HTTPAdapter(pool_maxsize=100) + self.session.mount('https://', adapter) # Map cache is a simple cache to avoid re-downloading the same map URI for every segment, # since it's generally the same but may occasionally change. # We expect the map data to be very small so there is no eviction here.