From a56f6859bb44ecf9a641bf720983c046858e8b5e Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Sun, 12 Sep 2021 12:29:35 +1000 Subject: [PATCH] more py3 fixes --- backfiller/backfiller/main.py | 6 +++--- common/common/requests.py | 4 ++-- cutter/cutter/main.py | 2 +- downloader/downloader/hls_playlist.py | 2 +- downloader/downloader/main.py | 12 +++++++----- downloader/downloader/twitch.py | 4 ++-- segment_coverage/segment_coverage/main.py | 2 +- 7 files changed, 17 insertions(+), 15 deletions(-) diff --git a/backfiller/backfiller/main.py b/backfiller/backfiller/main.py index 936b425..fe4cc2e 100644 --- a/backfiller/backfiller/main.py +++ b/backfiller/backfiller/main.py @@ -231,7 +231,7 @@ class BackfillerManager(object): def stop(self): """Shut down all workers and stop backfilling.""" self.logger.info('Stopping') - for node in self.workers.keys(): + for node in list(self.workers.keys()): self.stop_worker(node) self.stopping.set() @@ -344,7 +344,7 @@ class BackfillerManager(object): self.stopping.wait(common.jitter(self.NODE_INTERVAL)) #wait for all workers to finish - for worker in self.workers.values(): + for worker in list(self.workers.values()): worker.done.wait() def get_nodes(self): @@ -390,7 +390,7 @@ class BackfillerManager(object): for row in results: nodes[row.name] = row.url nodes.pop(self.localhost, None) - self.logger.info('Nodes fetched: {}'.format(nodes.keys())) + self.logger.info('Nodes fetched: {}'.format(list(nodes.keys()))) return list(nodes.values()) class BackfillerWorker(object): diff --git a/common/common/requests.py b/common/common/requests.py index 372c790..194dafb 100644 --- a/common/common/requests.py +++ b/common/common/requests.py @@ -1,7 +1,7 @@ """Code for instrumenting requests calls. Requires requests, obviously.""" -import urlparse +import urllib.parse import requests.sessions import prometheus_client as prom @@ -33,7 +33,7 @@ class InstrumentedSession(requests.sessions.Session): """ def request(self, method, url, *args, **kwargs): - _, domain, _, _, _ = urlparse.urlsplit(url) + _, domain, _, _, _ = urllib.parse.urlsplit(url) name = kwargs.pop('metric_name', '') start = monotonic() # we only use our own measured latency if an error occurs diff --git a/cutter/cutter/main.py b/cutter/cutter/main.py index 9815fc8..b3a6d31 100644 --- a/cutter/cutter/main.py +++ b/cutter/cutter/main.py @@ -544,7 +544,7 @@ class TranscodeChecker(object): def check_ids(self, ids): # Future work: Set error in DB if video id is not present, # and/or try to get more info from yt about what's wrong. - done = self.backend.check_status(ids.values()) + done = self.backend.check_status(list(ids.values())) return { id: video_id for id, video_id in ids.items() if video_id in done diff --git a/downloader/downloader/hls_playlist.py b/downloader/downloader/hls_playlist.py index e659044..b05f9f8 100644 --- a/downloader/downloader/hls_playlist.py +++ b/downloader/downloader/hls_playlist.py @@ -32,7 +32,7 @@ import re from binascii import unhexlify from collections import namedtuple from itertools import starmap -from urlparse import urljoin, urlparse +from urllib.parse import urljoin, urlparse # EXT-X-BYTERANGE diff --git a/downloader/downloader/main.py b/downloader/downloader/main.py index fddce06..337d157 100644 --- a/downloader/downloader/main.py +++ b/downloader/downloader/main.py @@ -17,11 +17,12 @@ import prometheus_client as prom import requests from monotonic import monotonic -import twitch import common import common.dateutil import common.requests +from . import twitch + segments_downloaded = prom.Counter( "segments_downloaded", @@ -328,7 +329,7 @@ class StreamWorker(object): else: self.logger.info("Worker stopped") finally: - for getter in self.getters.values(): + for getter in list(self.getters.values()): getter.done.wait() self.done.set() self.manager.stream_workers[self.quality].remove(self) @@ -392,8 +393,9 @@ class StreamWorker(object): if date is not None: date += datetime.timedelta(seconds=segment.duration) - # Clean up any old segment getters - for url, getter in self.getters.items(): + # Clean up any old segment getters. + # Note use of list() to make a copy to avoid modification-during-iteration + for url, getter in list(self.getters.items()): # If segment is done and wasn't in latest fetch if getter.done.is_set() and not any( segment.uri == url for segment in playlist.segments @@ -512,7 +514,7 @@ class SegmentGetter(object): partial: Segment is incomplete. Hash is included. temp: Segment has not been downloaded yet. A random uuid is added. """ - arg = str(uuid.uuid4()) if type == "temp" else b64encode(hash.digest(), b"-_").encode().rstrip("=") + arg = str(uuid.uuid4()) if type == "temp" else b64encode(hash.digest(), b"-_").decode().rstrip("=") return "{}-{}-{}.ts".format(self.prefix, type, arg) def exists(self): diff --git a/downloader/downloader/twitch.py b/downloader/downloader/twitch.py index e9032ef..eca1ee3 100644 --- a/downloader/downloader/twitch.py +++ b/downloader/downloader/twitch.py @@ -2,10 +2,10 @@ import logging import random -import hls_playlist - from common.requests import InstrumentedSession +from . import hls_playlist + logger = logging.getLogger(__name__) diff --git a/segment_coverage/segment_coverage/main.py b/segment_coverage/segment_coverage/main.py index d64968e..28523d3 100644 --- a/segment_coverage/segment_coverage/main.py +++ b/segment_coverage/segment_coverage/main.py @@ -195,7 +195,7 @@ class CoverageChecker(object): self.logger.exception('Getting nodes failed.', exc_info=True) return - self.logger.info('Nodes fetched: {}'.format(nodes.keys())) + self.logger.info('Nodes fetched: {}'.format(list(nodes.keys()))) html = """