|
|
|
@ -14,6 +14,8 @@ import errno
|
|
|
|
import gzip
|
|
|
|
import gzip
|
|
|
|
import hashlib
|
|
|
|
import hashlib
|
|
|
|
import hmac
|
|
|
|
import hmac
|
|
|
|
|
|
|
|
import html.entities
|
|
|
|
|
|
|
|
import html.parser
|
|
|
|
import importlib.util
|
|
|
|
import importlib.util
|
|
|
|
import io
|
|
|
|
import io
|
|
|
|
import itertools
|
|
|
|
import itertools
|
|
|
|
@ -29,6 +31,7 @@ import re
|
|
|
|
import shlex
|
|
|
|
import shlex
|
|
|
|
import socket
|
|
|
|
import socket
|
|
|
|
import ssl
|
|
|
|
import ssl
|
|
|
|
|
|
|
|
import struct
|
|
|
|
import subprocess
|
|
|
|
import subprocess
|
|
|
|
import sys
|
|
|
|
import sys
|
|
|
|
import tempfile
|
|
|
|
import tempfile
|
|
|
|
@ -36,35 +39,27 @@ import time
|
|
|
|
import traceback
|
|
|
|
import traceback
|
|
|
|
import types
|
|
|
|
import types
|
|
|
|
import urllib.parse
|
|
|
|
import urllib.parse
|
|
|
|
|
|
|
|
import urllib.request
|
|
|
|
import xml.etree.ElementTree
|
|
|
|
import xml.etree.ElementTree
|
|
|
|
import zlib
|
|
|
|
import zlib
|
|
|
|
|
|
|
|
import http.client
|
|
|
|
|
|
|
|
import http.cookiejar
|
|
|
|
|
|
|
|
|
|
|
|
from .compat import asyncio, functools # isort: split
|
|
|
|
from .compat import asyncio, functools # isort: split
|
|
|
|
from .compat import (
|
|
|
|
from .compat import (
|
|
|
|
compat_chr,
|
|
|
|
|
|
|
|
compat_cookiejar,
|
|
|
|
|
|
|
|
compat_etree_fromstring,
|
|
|
|
compat_etree_fromstring,
|
|
|
|
compat_expanduser,
|
|
|
|
compat_expanduser,
|
|
|
|
compat_html_entities,
|
|
|
|
|
|
|
|
compat_html_entities_html5,
|
|
|
|
|
|
|
|
compat_HTMLParseError,
|
|
|
|
compat_HTMLParseError,
|
|
|
|
compat_HTMLParser,
|
|
|
|
|
|
|
|
compat_http_client,
|
|
|
|
|
|
|
|
compat_HTTPError,
|
|
|
|
compat_HTTPError,
|
|
|
|
compat_os_name,
|
|
|
|
compat_os_name,
|
|
|
|
compat_parse_qs,
|
|
|
|
compat_parse_qs,
|
|
|
|
compat_shlex_quote,
|
|
|
|
compat_shlex_quote,
|
|
|
|
compat_str,
|
|
|
|
compat_str,
|
|
|
|
compat_struct_pack,
|
|
|
|
|
|
|
|
compat_struct_unpack,
|
|
|
|
|
|
|
|
compat_urllib_error,
|
|
|
|
|
|
|
|
compat_urllib_parse_unquote_plus,
|
|
|
|
|
|
|
|
compat_urllib_parse_urlencode,
|
|
|
|
compat_urllib_parse_urlencode,
|
|
|
|
compat_urllib_parse_urlparse,
|
|
|
|
compat_urllib_parse_urlparse,
|
|
|
|
compat_urllib_request,
|
|
|
|
|
|
|
|
compat_urlparse,
|
|
|
|
compat_urlparse,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
from .dependencies import brotli, certifi, websockets
|
|
|
|
from .dependencies import brotli, certifi, websockets, xattr
|
|
|
|
from .socks import ProxyType, sockssocket
|
|
|
|
from .socks import ProxyType, sockssocket
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -445,7 +440,7 @@ def get_elements_text_and_html_by_attribute(attribute, value, html, escape_value
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class HTMLBreakOnClosingTagParser(compat_HTMLParser):
|
|
|
|
class HTMLBreakOnClosingTagParser(html.parser.HTMLParser):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
HTML parser which raises HTMLBreakOnClosingTagException upon reaching the
|
|
|
|
HTML parser which raises HTMLBreakOnClosingTagException upon reaching the
|
|
|
|
closing tag for the first opening tag it has encountered, and can be used
|
|
|
|
closing tag for the first opening tag it has encountered, and can be used
|
|
|
|
@ -457,7 +452,7 @@ class HTMLBreakOnClosingTagParser(compat_HTMLParser):
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
def __init__(self):
|
|
|
|
self.tagstack = collections.deque()
|
|
|
|
self.tagstack = collections.deque()
|
|
|
|
compat_HTMLParser.__init__(self)
|
|
|
|
html.parser.HTMLParser.__init__(self)
|
|
|
|
|
|
|
|
|
|
|
|
def __enter__(self):
|
|
|
|
def __enter__(self):
|
|
|
|
return self
|
|
|
|
return self
|
|
|
|
@ -522,22 +517,22 @@ def get_element_text_and_html_by_tag(tag, html):
|
|
|
|
raise compat_HTMLParseError('unexpected end of html')
|
|
|
|
raise compat_HTMLParseError('unexpected end of html')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class HTMLAttributeParser(compat_HTMLParser):
|
|
|
|
class HTMLAttributeParser(html.parser.HTMLParser):
|
|
|
|
"""Trivial HTML parser to gather the attributes for a single element"""
|
|
|
|
"""Trivial HTML parser to gather the attributes for a single element"""
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
def __init__(self):
|
|
|
|
self.attrs = {}
|
|
|
|
self.attrs = {}
|
|
|
|
compat_HTMLParser.__init__(self)
|
|
|
|
html.parser.HTMLParser.__init__(self)
|
|
|
|
|
|
|
|
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
|
|
self.attrs = dict(attrs)
|
|
|
|
self.attrs = dict(attrs)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class HTMLListAttrsParser(compat_HTMLParser):
|
|
|
|
class HTMLListAttrsParser(html.parser.HTMLParser):
|
|
|
|
"""HTML parser to gather the attributes for the elements of a list"""
|
|
|
|
"""HTML parser to gather the attributes for the elements of a list"""
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
def __init__(self):
|
|
|
|
compat_HTMLParser.__init__(self)
|
|
|
|
html.parser.HTMLParser.__init__(self)
|
|
|
|
self.items = []
|
|
|
|
self.items = []
|
|
|
|
self._level = 0
|
|
|
|
self._level = 0
|
|
|
|
|
|
|
|
|
|
|
|
@ -763,7 +758,7 @@ def sanitized_Request(url, *args, **kwargs):
|
|
|
|
if auth_header is not None:
|
|
|
|
if auth_header is not None:
|
|
|
|
headers = args[1] if len(args) >= 2 else kwargs.setdefault('headers', {})
|
|
|
|
headers = args[1] if len(args) >= 2 else kwargs.setdefault('headers', {})
|
|
|
|
headers['Authorization'] = auth_header
|
|
|
|
headers['Authorization'] = auth_header
|
|
|
|
return compat_urllib_request.Request(url, *args, **kwargs)
|
|
|
|
return urllib.request.Request(url, *args, **kwargs)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def expand_path(s):
|
|
|
|
def expand_path(s):
|
|
|
|
@ -788,13 +783,13 @@ def _htmlentity_transform(entity_with_semicolon):
|
|
|
|
entity = entity_with_semicolon[:-1]
|
|
|
|
entity = entity_with_semicolon[:-1]
|
|
|
|
|
|
|
|
|
|
|
|
# Known non-numeric HTML entity
|
|
|
|
# Known non-numeric HTML entity
|
|
|
|
if entity in compat_html_entities.name2codepoint:
|
|
|
|
if entity in html.entities.name2codepoint:
|
|
|
|
return compat_chr(compat_html_entities.name2codepoint[entity])
|
|
|
|
return chr(html.entities.name2codepoint[entity])
|
|
|
|
|
|
|
|
|
|
|
|
# TODO: HTML5 allows entities without a semicolon. For example,
|
|
|
|
# TODO: HTML5 allows entities without a semicolon. For example,
|
|
|
|
# 'Éric' should be decoded as 'Éric'.
|
|
|
|
# 'Éric' should be decoded as 'Éric'.
|
|
|
|
if entity_with_semicolon in compat_html_entities_html5:
|
|
|
|
if entity_with_semicolon in html.entities.html5:
|
|
|
|
return compat_html_entities_html5[entity_with_semicolon]
|
|
|
|
return html.entities.html5[entity_with_semicolon]
|
|
|
|
|
|
|
|
|
|
|
|
mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
|
|
|
|
mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
|
|
|
|
if mobj is not None:
|
|
|
|
if mobj is not None:
|
|
|
|
@ -806,7 +801,7 @@ def _htmlentity_transform(entity_with_semicolon):
|
|
|
|
base = 10
|
|
|
|
base = 10
|
|
|
|
# See https://github.com/ytdl-org/youtube-dl/issues/7518
|
|
|
|
# See https://github.com/ytdl-org/youtube-dl/issues/7518
|
|
|
|
with contextlib.suppress(ValueError):
|
|
|
|
with contextlib.suppress(ValueError):
|
|
|
|
return compat_chr(int(numstr, base))
|
|
|
|
return chr(int(numstr, base))
|
|
|
|
|
|
|
|
|
|
|
|
# Unknown entity in name, return its literal representation
|
|
|
|
# Unknown entity in name, return its literal representation
|
|
|
|
return '&%s;' % entity
|
|
|
|
return '&%s;' % entity
|
|
|
|
@ -1015,7 +1010,7 @@ class YoutubeDLError(Exception):
|
|
|
|
super().__init__(self.msg)
|
|
|
|
super().__init__(self.msg)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
network_exceptions = [compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error]
|
|
|
|
network_exceptions = [urllib.error.URLError, http.client.HTTPException, socket.error]
|
|
|
|
if hasattr(ssl, 'CertificateError'):
|
|
|
|
if hasattr(ssl, 'CertificateError'):
|
|
|
|
network_exceptions.append(ssl.CertificateError)
|
|
|
|
network_exceptions.append(ssl.CertificateError)
|
|
|
|
network_exceptions = tuple(network_exceptions)
|
|
|
|
network_exceptions = tuple(network_exceptions)
|
|
|
|
@ -1267,7 +1262,7 @@ def handle_youtubedl_headers(headers):
|
|
|
|
return filtered_headers
|
|
|
|
return filtered_headers
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
|
|
|
|
class YoutubeDLHandler(urllib.request.HTTPHandler):
|
|
|
|
"""Handler for HTTP requests and responses.
|
|
|
|
"""Handler for HTTP requests and responses.
|
|
|
|
|
|
|
|
|
|
|
|
This class, when installed with an OpenerDirector, automatically adds
|
|
|
|
This class, when installed with an OpenerDirector, automatically adds
|
|
|
|
@ -1286,11 +1281,11 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self, params, *args, **kwargs):
|
|
|
|
def __init__(self, params, *args, **kwargs):
|
|
|
|
compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
|
|
|
|
urllib.request.HTTPHandler.__init__(self, *args, **kwargs)
|
|
|
|
self._params = params
|
|
|
|
self._params = params
|
|
|
|
|
|
|
|
|
|
|
|
def http_open(self, req):
|
|
|
|
def http_open(self, req):
|
|
|
|
conn_class = compat_http_client.HTTPConnection
|
|
|
|
conn_class = http.client.HTTPConnection
|
|
|
|
|
|
|
|
|
|
|
|
socks_proxy = req.headers.get('Ytdl-socks-proxy')
|
|
|
|
socks_proxy = req.headers.get('Ytdl-socks-proxy')
|
|
|
|
if socks_proxy:
|
|
|
|
if socks_proxy:
|
|
|
|
@ -1365,18 +1360,18 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
|
|
|
|
break
|
|
|
|
break
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
raise original_ioerror
|
|
|
|
raise original_ioerror
|
|
|
|
resp = compat_urllib_request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
|
|
|
|
resp = urllib.request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
|
|
|
|
resp.msg = old_resp.msg
|
|
|
|
resp.msg = old_resp.msg
|
|
|
|
del resp.headers['Content-encoding']
|
|
|
|
del resp.headers['Content-encoding']
|
|
|
|
# deflate
|
|
|
|
# deflate
|
|
|
|
if resp.headers.get('Content-encoding', '') == 'deflate':
|
|
|
|
if resp.headers.get('Content-encoding', '') == 'deflate':
|
|
|
|
gz = io.BytesIO(self.deflate(resp.read()))
|
|
|
|
gz = io.BytesIO(self.deflate(resp.read()))
|
|
|
|
resp = compat_urllib_request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
|
|
|
|
resp = urllib.request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
|
|
|
|
resp.msg = old_resp.msg
|
|
|
|
resp.msg = old_resp.msg
|
|
|
|
del resp.headers['Content-encoding']
|
|
|
|
del resp.headers['Content-encoding']
|
|
|
|
# brotli
|
|
|
|
# brotli
|
|
|
|
if resp.headers.get('Content-encoding', '') == 'br':
|
|
|
|
if resp.headers.get('Content-encoding', '') == 'br':
|
|
|
|
resp = compat_urllib_request.addinfourl(
|
|
|
|
resp = urllib.request.addinfourl(
|
|
|
|
io.BytesIO(self.brotli(resp.read())), old_resp.headers, old_resp.url, old_resp.code)
|
|
|
|
io.BytesIO(self.brotli(resp.read())), old_resp.headers, old_resp.url, old_resp.code)
|
|
|
|
resp.msg = old_resp.msg
|
|
|
|
resp.msg = old_resp.msg
|
|
|
|
del resp.headers['Content-encoding']
|
|
|
|
del resp.headers['Content-encoding']
|
|
|
|
@ -1399,7 +1394,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
|
|
|
|
|
|
|
|
|
|
|
|
def make_socks_conn_class(base_class, socks_proxy):
|
|
|
|
def make_socks_conn_class(base_class, socks_proxy):
|
|
|
|
assert issubclass(base_class, (
|
|
|
|
assert issubclass(base_class, (
|
|
|
|
compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
|
|
|
|
http.client.HTTPConnection, http.client.HTTPSConnection))
|
|
|
|
|
|
|
|
|
|
|
|
url_components = compat_urlparse.urlparse(socks_proxy)
|
|
|
|
url_components = compat_urlparse.urlparse(socks_proxy)
|
|
|
|
if url_components.scheme.lower() == 'socks5':
|
|
|
|
if url_components.scheme.lower() == 'socks5':
|
|
|
|
@ -1412,7 +1407,7 @@ def make_socks_conn_class(base_class, socks_proxy):
|
|
|
|
def unquote_if_non_empty(s):
|
|
|
|
def unquote_if_non_empty(s):
|
|
|
|
if not s:
|
|
|
|
if not s:
|
|
|
|
return s
|
|
|
|
return s
|
|
|
|
return compat_urllib_parse_unquote_plus(s)
|
|
|
|
return urllib.parse.unquote_plus(s)
|
|
|
|
|
|
|
|
|
|
|
|
proxy_args = (
|
|
|
|
proxy_args = (
|
|
|
|
socks_type,
|
|
|
|
socks_type,
|
|
|
|
@ -1430,7 +1425,7 @@ def make_socks_conn_class(base_class, socks_proxy):
|
|
|
|
self.sock.settimeout(self.timeout)
|
|
|
|
self.sock.settimeout(self.timeout)
|
|
|
|
self.sock.connect((self.host, self.port))
|
|
|
|
self.sock.connect((self.host, self.port))
|
|
|
|
|
|
|
|
|
|
|
|
if isinstance(self, compat_http_client.HTTPSConnection):
|
|
|
|
if isinstance(self, http.client.HTTPSConnection):
|
|
|
|
if hasattr(self, '_context'): # Python > 2.6
|
|
|
|
if hasattr(self, '_context'): # Python > 2.6
|
|
|
|
self.sock = self._context.wrap_socket(
|
|
|
|
self.sock = self._context.wrap_socket(
|
|
|
|
self.sock, server_hostname=self.host)
|
|
|
|
self.sock, server_hostname=self.host)
|
|
|
|
@ -1440,10 +1435,10 @@ def make_socks_conn_class(base_class, socks_proxy):
|
|
|
|
return SocksConnection
|
|
|
|
return SocksConnection
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
|
|
|
|
class YoutubeDLHTTPSHandler(urllib.request.HTTPSHandler):
|
|
|
|
def __init__(self, params, https_conn_class=None, *args, **kwargs):
|
|
|
|
def __init__(self, params, https_conn_class=None, *args, **kwargs):
|
|
|
|
compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
|
|
|
|
urllib.request.HTTPSHandler.__init__(self, *args, **kwargs)
|
|
|
|
self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
|
|
|
|
self._https_conn_class = https_conn_class or http.client.HTTPSConnection
|
|
|
|
self._params = params
|
|
|
|
self._params = params
|
|
|
|
|
|
|
|
|
|
|
|
def https_open(self, req):
|
|
|
|
def https_open(self, req):
|
|
|
|
@ -1470,7 +1465,7 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
|
|
|
|
raise
|
|
|
|
raise
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
|
|
|
|
class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
See [1] for cookie file format.
|
|
|
|
See [1] for cookie file format.
|
|
|
|
|
|
|
|
|
|
|
|
@ -1541,7 +1536,7 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
|
|
|
|
if self.filename is not None:
|
|
|
|
if self.filename is not None:
|
|
|
|
filename = self.filename
|
|
|
|
filename = self.filename
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
raise ValueError(compat_cookiejar.MISSING_FILENAME_TEXT)
|
|
|
|
raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
|
|
|
|
|
|
|
|
|
|
|
|
# Store session cookies with `expires` set to 0 instead of an empty string
|
|
|
|
# Store session cookies with `expires` set to 0 instead of an empty string
|
|
|
|
for cookie in self:
|
|
|
|
for cookie in self:
|
|
|
|
@ -1558,7 +1553,7 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
|
|
|
|
if self.filename is not None:
|
|
|
|
if self.filename is not None:
|
|
|
|
filename = self.filename
|
|
|
|
filename = self.filename
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
raise ValueError(compat_cookiejar.MISSING_FILENAME_TEXT)
|
|
|
|
raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
|
|
|
|
|
|
|
|
|
|
|
|
def prepare_line(line):
|
|
|
|
def prepare_line(line):
|
|
|
|
if line.startswith(self._HTTPONLY_PREFIX):
|
|
|
|
if line.startswith(self._HTTPONLY_PREFIX):
|
|
|
|
@ -1568,10 +1563,10 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
|
|
|
|
return line
|
|
|
|
return line
|
|
|
|
cookie_list = line.split('\t')
|
|
|
|
cookie_list = line.split('\t')
|
|
|
|
if len(cookie_list) != self._ENTRY_LEN:
|
|
|
|
if len(cookie_list) != self._ENTRY_LEN:
|
|
|
|
raise compat_cookiejar.LoadError('invalid length %d' % len(cookie_list))
|
|
|
|
raise http.cookiejar.LoadError('invalid length %d' % len(cookie_list))
|
|
|
|
cookie = self._CookieFileEntry(*cookie_list)
|
|
|
|
cookie = self._CookieFileEntry(*cookie_list)
|
|
|
|
if cookie.expires_at and not cookie.expires_at.isdigit():
|
|
|
|
if cookie.expires_at and not cookie.expires_at.isdigit():
|
|
|
|
raise compat_cookiejar.LoadError('invalid expires at %s' % cookie.expires_at)
|
|
|
|
raise http.cookiejar.LoadError('invalid expires at %s' % cookie.expires_at)
|
|
|
|
return line
|
|
|
|
return line
|
|
|
|
|
|
|
|
|
|
|
|
cf = io.StringIO()
|
|
|
|
cf = io.StringIO()
|
|
|
|
@ -1579,9 +1574,9 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
|
|
|
|
for line in f:
|
|
|
|
for line in f:
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
cf.write(prepare_line(line))
|
|
|
|
cf.write(prepare_line(line))
|
|
|
|
except compat_cookiejar.LoadError as e:
|
|
|
|
except http.cookiejar.LoadError as e:
|
|
|
|
if f'{line.strip()} '[0] in '[{"':
|
|
|
|
if f'{line.strip()} '[0] in '[{"':
|
|
|
|
raise compat_cookiejar.LoadError(
|
|
|
|
raise http.cookiejar.LoadError(
|
|
|
|
'Cookies file must be Netscape formatted, not JSON. See '
|
|
|
|
'Cookies file must be Netscape formatted, not JSON. See '
|
|
|
|
'https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl')
|
|
|
|
'https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl')
|
|
|
|
write_string(f'WARNING: skipping cookie file entry due to {e}: {line!r}\n')
|
|
|
|
write_string(f'WARNING: skipping cookie file entry due to {e}: {line!r}\n')
|
|
|
|
@ -1604,18 +1599,18 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
|
|
|
|
cookie.discard = True
|
|
|
|
cookie.discard = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
|
|
|
|
class YoutubeDLCookieProcessor(urllib.request.HTTPCookieProcessor):
|
|
|
|
def __init__(self, cookiejar=None):
|
|
|
|
def __init__(self, cookiejar=None):
|
|
|
|
compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
|
|
|
|
urllib.request.HTTPCookieProcessor.__init__(self, cookiejar)
|
|
|
|
|
|
|
|
|
|
|
|
def http_response(self, request, response):
|
|
|
|
def http_response(self, request, response):
|
|
|
|
return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
|
|
|
|
return urllib.request.HTTPCookieProcessor.http_response(self, request, response)
|
|
|
|
|
|
|
|
|
|
|
|
https_request = compat_urllib_request.HTTPCookieProcessor.http_request
|
|
|
|
https_request = urllib.request.HTTPCookieProcessor.http_request
|
|
|
|
https_response = http_response
|
|
|
|
https_response = http_response
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
|
|
|
|
class YoutubeDLRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
|
|
"""YoutubeDL redirect handler
|
|
|
|
"""YoutubeDL redirect handler
|
|
|
|
|
|
|
|
|
|
|
|
The code is based on HTTPRedirectHandler implementation from CPython [1].
|
|
|
|
The code is based on HTTPRedirectHandler implementation from CPython [1].
|
|
|
|
@ -1630,7 +1625,7 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
|
|
|
|
3. https://github.com/ytdl-org/youtube-dl/issues/28768
|
|
|
|
3. https://github.com/ytdl-org/youtube-dl/issues/28768
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
http_error_301 = http_error_303 = http_error_307 = http_error_308 = compat_urllib_request.HTTPRedirectHandler.http_error_302
|
|
|
|
http_error_301 = http_error_303 = http_error_307 = http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302
|
|
|
|
|
|
|
|
|
|
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
|
|
"""Return a Request or None in response to a redirect.
|
|
|
|
"""Return a Request or None in response to a redirect.
|
|
|
|
@ -1672,7 +1667,7 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
|
|
|
|
if code in (301, 302) and m == 'POST':
|
|
|
|
if code in (301, 302) and m == 'POST':
|
|
|
|
m = 'GET'
|
|
|
|
m = 'GET'
|
|
|
|
|
|
|
|
|
|
|
|
return compat_urllib_request.Request(
|
|
|
|
return urllib.request.Request(
|
|
|
|
newurl, headers=newheaders, origin_req_host=req.origin_req_host,
|
|
|
|
newurl, headers=newheaders, origin_req_host=req.origin_req_host,
|
|
|
|
unverifiable=True, method=m)
|
|
|
|
unverifiable=True, method=m)
|
|
|
|
|
|
|
|
|
|
|
|
@ -1967,7 +1962,7 @@ def bytes_to_intlist(bs):
|
|
|
|
def intlist_to_bytes(xs):
|
|
|
|
def intlist_to_bytes(xs):
|
|
|
|
if not xs:
|
|
|
|
if not xs:
|
|
|
|
return b''
|
|
|
|
return b''
|
|
|
|
return compat_struct_pack('%dB' % len(xs), *xs)
|
|
|
|
return struct.pack('%dB' % len(xs), *xs)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class LockingUnsupportedError(OSError):
|
|
|
|
class LockingUnsupportedError(OSError):
|
|
|
|
@ -2427,12 +2422,12 @@ def urljoin(base, path):
|
|
|
|
return compat_urlparse.urljoin(base, path)
|
|
|
|
return compat_urlparse.urljoin(base, path)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class HEADRequest(compat_urllib_request.Request):
|
|
|
|
class HEADRequest(urllib.request.Request):
|
|
|
|
def get_method(self):
|
|
|
|
def get_method(self):
|
|
|
|
return 'HEAD'
|
|
|
|
return 'HEAD'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PUTRequest(compat_urllib_request.Request):
|
|
|
|
class PUTRequest(urllib.request.Request):
|
|
|
|
def get_method(self):
|
|
|
|
def get_method(self):
|
|
|
|
return 'PUT'
|
|
|
|
return 'PUT'
|
|
|
|
|
|
|
|
|
|
|
|
@ -2484,7 +2479,7 @@ def url_or_none(url):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def request_to_url(req):
|
|
|
|
def request_to_url(req):
|
|
|
|
if isinstance(req, compat_urllib_request.Request):
|
|
|
|
if isinstance(req, urllib.request.Request):
|
|
|
|
return req.get_full_url()
|
|
|
|
return req.get_full_url()
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
return req
|
|
|
|
return req
|
|
|
|
@ -3037,7 +3032,7 @@ def update_Request(req, url=None, data=None, headers={}, query={}):
|
|
|
|
elif req_get_method == 'PUT':
|
|
|
|
elif req_get_method == 'PUT':
|
|
|
|
req_type = PUTRequest
|
|
|
|
req_type = PUTRequest
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
req_type = compat_urllib_request.Request
|
|
|
|
req_type = urllib.request.Request
|
|
|
|
new_req = req_type(
|
|
|
|
new_req = req_type(
|
|
|
|
req_url, data=req_data, headers=req_headers,
|
|
|
|
req_url, data=req_data, headers=req_headers,
|
|
|
|
origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
|
|
|
|
origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
|
|
|
|
@ -4636,20 +4631,20 @@ class GeoUtils:
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
block = code_or_block
|
|
|
|
block = code_or_block
|
|
|
|
addr, preflen = block.split('/')
|
|
|
|
addr, preflen = block.split('/')
|
|
|
|
addr_min = compat_struct_unpack('!L', socket.inet_aton(addr))[0]
|
|
|
|
addr_min = struct.unpack('!L', socket.inet_aton(addr))[0]
|
|
|
|
addr_max = addr_min | (0xffffffff >> int(preflen))
|
|
|
|
addr_max = addr_min | (0xffffffff >> int(preflen))
|
|
|
|
return compat_str(socket.inet_ntoa(
|
|
|
|
return compat_str(socket.inet_ntoa(
|
|
|
|
compat_struct_pack('!L', random.randint(addr_min, addr_max))))
|
|
|
|
struct.pack('!L', random.randint(addr_min, addr_max))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
|
|
|
|
class PerRequestProxyHandler(urllib.request.ProxyHandler):
|
|
|
|
def __init__(self, proxies=None):
|
|
|
|
def __init__(self, proxies=None):
|
|
|
|
# Set default handlers
|
|
|
|
# Set default handlers
|
|
|
|
for type in ('http', 'https'):
|
|
|
|
for type in ('http', 'https'):
|
|
|
|
setattr(self, '%s_open' % type,
|
|
|
|
setattr(self, '%s_open' % type,
|
|
|
|
lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
|
|
|
|
lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
|
|
|
|
meth(r, proxy, type))
|
|
|
|
meth(r, proxy, type))
|
|
|
|
compat_urllib_request.ProxyHandler.__init__(self, proxies)
|
|
|
|
urllib.request.ProxyHandler.__init__(self, proxies)
|
|
|
|
|
|
|
|
|
|
|
|
def proxy_open(self, req, proxy, type):
|
|
|
|
def proxy_open(self, req, proxy, type):
|
|
|
|
req_proxy = req.headers.get('Ytdl-request-proxy')
|
|
|
|
req_proxy = req.headers.get('Ytdl-request-proxy')
|
|
|
|
@ -4663,7 +4658,7 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
|
|
|
|
req.add_header('Ytdl-socks-proxy', proxy)
|
|
|
|
req.add_header('Ytdl-socks-proxy', proxy)
|
|
|
|
# yt-dlp's http/https handlers do wrapping the socket with socks
|
|
|
|
# yt-dlp's http/https handlers do wrapping the socket with socks
|
|
|
|
return None
|
|
|
|
return None
|
|
|
|
return compat_urllib_request.ProxyHandler.proxy_open(
|
|
|
|
return urllib.request.ProxyHandler.proxy_open(
|
|
|
|
self, req, proxy, type)
|
|
|
|
self, req, proxy, type)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -4683,7 +4678,7 @@ def long_to_bytes(n, blocksize=0):
|
|
|
|
s = b''
|
|
|
|
s = b''
|
|
|
|
n = int(n)
|
|
|
|
n = int(n)
|
|
|
|
while n > 0:
|
|
|
|
while n > 0:
|
|
|
|
s = compat_struct_pack('>I', n & 0xffffffff) + s
|
|
|
|
s = struct.pack('>I', n & 0xffffffff) + s
|
|
|
|
n = n >> 32
|
|
|
|
n = n >> 32
|
|
|
|
# strip off leading zeros
|
|
|
|
# strip off leading zeros
|
|
|
|
for i in range(len(s)):
|
|
|
|
for i in range(len(s)):
|
|
|
|
@ -4714,7 +4709,7 @@ def bytes_to_long(s):
|
|
|
|
s = b'\000' * extra + s
|
|
|
|
s = b'\000' * extra + s
|
|
|
|
length = length + extra
|
|
|
|
length = length + extra
|
|
|
|
for i in range(0, length, 4):
|
|
|
|
for i in range(0, length, 4):
|
|
|
|
acc = (acc << 32) + compat_struct_unpack('>I', s[i:i + 4])[0]
|
|
|
|
acc = (acc << 32) + struct.unpack('>I', s[i:i + 4])[0]
|
|
|
|
return acc
|
|
|
|
return acc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -4842,7 +4837,7 @@ def decode_png(png_data):
|
|
|
|
raise OSError('Not a valid PNG file.')
|
|
|
|
raise OSError('Not a valid PNG file.')
|
|
|
|
|
|
|
|
|
|
|
|
int_map = {1: '>B', 2: '>H', 4: '>I'}
|
|
|
|
int_map = {1: '>B', 2: '>H', 4: '>I'}
|
|
|
|
unpack_integer = lambda x: compat_struct_unpack(int_map[len(x)], x)[0]
|
|
|
|
unpack_integer = lambda x: struct.unpack(int_map[len(x)], x)[0]
|
|
|
|
|
|
|
|
|
|
|
|
chunks = []
|
|
|
|
chunks = []
|
|
|
|
|
|
|
|
|
|
|
|
@ -4954,7 +4949,6 @@ def write_xattr(path, key, value):
|
|
|
|
return
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
# UNIX Method 1. Use xattrs/pyxattrs modules
|
|
|
|
# UNIX Method 1. Use xattrs/pyxattrs modules
|
|
|
|
from .dependencies import xattr
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
setxattr = None
|
|
|
|
setxattr = None
|
|
|
|
if getattr(xattr, '_yt_dlp__identifier', None) == 'pyxattr':
|
|
|
|
if getattr(xattr, '_yt_dlp__identifier', None) == 'pyxattr':
|
|
|
|
|
pukkandan
3 years ago