From 7595f4479e16e2456d2dda056ed5756e46c52232 Mon Sep 17 00:00:00 2001 From: hyper750 Date: Mon, 24 Feb 2025 13:12:50 +0100 Subject: [PATCH] Be able to accept callable proxies, that change for each request --- yt_dlp/networking/common.py | 13 ++++++++++++- yt_dlp/utils/networking.py | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py index e8951c7e7d..a08c090f4f 100644 --- a/yt_dlp/networking/common.py +++ b/yt_dlp/networking/common.py @@ -267,7 +267,16 @@ class RequestHandler(abc.ABC): return self.cookiejar if cookiejar is None else cookiejar def _get_proxies(self, request): - return (request.proxies or self.proxies).copy() + if request.proxies: + return request.proxies.copy() + + proxies = dict() + for proxy_key, proxy_url in self.proxies.items(): + # Proxy url can be a callable + if isinstance(proxy_url, typing.Callable): + proxy_url = proxy_url() + proxies[proxy_key] = proxy_url + return proxies def _check_url_scheme(self, request: Request): scheme = urllib.parse.urlparse(request.url).scheme.lower() @@ -300,6 +309,8 @@ class RequestHandler(abc.ABC): # Skip proxy scheme checks continue + if isinstance(proxy_url, typing.Callable): + proxy_url = proxy_url() try: if urllib.request._parse_proxy(proxy_url)[0] is None: # Scheme-less proxies are not supported diff --git a/yt_dlp/utils/networking.py b/yt_dlp/utils/networking.py index 933b164be9..41edb80205 100644 --- a/yt_dlp/utils/networking.py +++ b/yt_dlp/utils/networking.py @@ -1,5 +1,6 @@ import collections import random +import typing import urllib.parse import urllib.request @@ -99,6 +100,8 @@ def clean_proxies(proxies: dict, headers: HTTPHeaderDict): if proxy_key == 'no': # special case continue if proxy_url is not None: + if isinstance(proxy_url, typing.Callable): + proxy_url = proxy_url() # Ensure proxies without a scheme are http. try: proxy_scheme = urllib.request._parse_proxy(proxy_url)[0]