Compare commits

..

2 Commits

Author SHA1 Message Date
Mike Lang 871a2e1909 Fix connection pool warnings by increasing pool size
in backfiller and downloader, the things making lots of outgoing http requests.

We want these larger sizes anyway to improve performance in downloader and backfiller.
2 months ago
Mike Lang 8eec87355c backfiller: Don't backfill segments that only differ from existing segment by 1ms
In the wild we've seen different servers get timestamps that differ by 1ms for segments
that are otherwise identical - same content, same duration.

The allowable fudge factor for segments is already 10ms, so having timing be 1ms different
between servers shouldn't cause any problems.
Worst case, there's a slight chance you'll get an adjacent frame when picking a cut point / thumbnail.
2 months ago

@ -16,6 +16,7 @@ import argh
import gevent.backdoor import gevent.backdoor
import gevent.pool import gevent.pool
import prometheus_client as prom import prometheus_client as prom
from requests.adapters import HTTPAdapter
import common import common
from common import dateutil from common import dateutil
@ -23,8 +24,10 @@ from common import database
from common.requests import InstrumentedSession from common.requests import InstrumentedSession
from common.segments import list_segment_files, unpadded_b64_decode from common.segments import list_segment_files, unpadded_b64_decode
# Wraps all requests in some metric collection # Wraps all requests in some metric collection and connection pooling
requests = InstrumentedSession() requests = InstrumentedSession()
adapter = HTTPAdapter(pool_maxsize=100)
requests.mount('https://', adapter)
segments_backfilled = prom.Counter( segments_backfilled = prom.Counter(
'segments_backfilled', 'segments_backfilled',
@ -515,6 +518,7 @@ class BackfillerWorker(object):
EPSILON = 0.001 EPSILON = 0.001
local_infos = [] local_infos = []
for path in local_segments: for path in local_segments:
path = os.path.join(channel, quality, hour, path)
try: try:
local_infos.append(common.parse_segment_path(path)) local_infos.append(common.parse_segment_path(path))
except ValueError as e: except ValueError as e:

@ -15,6 +15,7 @@ import gevent.backdoor
import gevent.event import gevent.event
import prometheus_client as prom import prometheus_client as prom
import requests import requests
import requests.adapters
from monotonic import monotonic from monotonic import monotonic
import common import common
@ -335,6 +336,8 @@ class StreamWorker(object):
# This worker's SegmentGetters will use its session by default for performance, # This worker's SegmentGetters will use its session by default for performance,
# but will fall back to a new one if something goes wrong. # but will fall back to a new one if something goes wrong.
self.session = common.requests.InstrumentedSession() self.session = common.requests.InstrumentedSession()
adapter = requests.adapters.HTTPAdapter(pool_maxsize=100)
self.session.mount('https://', adapter)
# Map cache is a simple cache to avoid re-downloading the same map URI for every segment, # Map cache is a simple cache to avoid re-downloading the same map URI for every segment,
# since it's generally the same but may occasionally change. # since it's generally the same but may occasionally change.
# We expect the map data to be very small so there is no eviction here. # We expect the map data to be very small so there is no eviction here.

Loading…
Cancel
Save