[rh:requests] Work around partial read dropping data (#13599)

Authored by: Grub4K
4 weeks ago · c2ff2dbaec
parent ca5cce5b07
commit c2ff2dbaec
2 changed files with 16 additions and 5 deletions
--- a/test/test_networking.py
+++ b/test/test_networking.py
@ -22,7 +22,6 @@ import ssl
 import tempfile
 import threading
 import time
-import urllib.error
 import urllib.request
 import warnings
 import zlib
@ -223,10 +222,7 @@ class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
                if encoding == 'br' and brotli:
                    payload = brotli.compress(payload)
                elif encoding == 'gzip':
-                    buf = io.BytesIO()
-                    with gzip.GzipFile(fileobj=buf, mode='wb') as f:
-                        f.write(payload)
-                    payload = buf.getvalue()
+                    payload = gzip.compress(payload, mtime=0)
                elif encoding == 'deflate':
                    payload = zlib.compress(payload)
                elif encoding == 'unsupported':
@ -729,6 +725,17 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):

            assert 'X-test-heaDer: test' in res

+    def test_partial_read_then_full_read(self, handler):
+        with handler() as rh:
+            for encoding in ('', 'gzip', 'deflate'):
+                res = validate_and_send(rh, Request(
+                    f'http://127.0.0.1:{self.http_port}/content-encoding',
+                    headers={'ytdl-encoding': encoding}))
+                assert res.headers.get('Content-Encoding') == encoding
+                assert res.read(6) == b'<html>'
+                assert res.read(0) == b''
+                assert res.read() == b'<video src="/vid.mp4" /></html>'
+

@pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
 class TestClientCertificate:
--- a/yt_dlp/networking/_requests.py
+++ b/yt_dlp/networking/_requests.py
@ -140,6 +140,10 @@ class RequestsResponseAdapter(Response):

    def read(self, amt: int | None = None):
        try:
+            # Work around issue with `.read(amt)` then `.read()`
+            # See: https://github.com/urllib3/urllib3/issues/3636
+            if amt is None:
+                amt = (1 << 31) - 1
            # Interact with urllib3 response directly.
            return self.fp.read(amt, decode_content=True)