Skip to content

A feature-rich command-line audio/video downloader

[rh:requests] Work around partial read dropping data (#13599)

Simon Sawickic2ff2dbaec79
diff --git a/test/test_networking.py b/test/test_networking.py
index 2f441fced2..afdd0c7aa7 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -22,7 +22,6 @@ import ssl
import tempfile
import threading
import time
-import urllib.error
import urllib.request
import warnings
import zlib
@@ -223,10 +222,7 @@ class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
if encoding == 'br' and brotli:
payload = brotli.compress(payload)
elif encoding == 'gzip':
- buf = io.BytesIO()
- with gzip.GzipFile(fileobj=buf, mode='wb') as f:
- f.write(payload)
- payload = buf.getvalue()
+ payload = gzip.compress(payload, mtime=0)
elif encoding == 'deflate':
payload = zlib.compress(payload)
elif encoding == 'unsupported':
@@ -729,6 +725,17 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
assert 'X-test-heaDer: test' in res
+ def test_partial_read_then_full_read(self, handler):
+ with handler() as rh:
+ for encoding in ('', 'gzip', 'deflate'):
+ res = validate_and_send(rh, Request(
+ f'http://127.0.0.1:{self.http_port}/content-encoding',
+ headers={'ytdl-encoding': encoding}))
+ assert res.headers.get('Content-Encoding') == encoding
+ assert res.read(6) == b'<html>'
+ assert res.read(0) == b''
+ assert res.read() == b'<video src="/vid.mp4" /></html>'
+
@pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
class TestClientCertificate:
diff --git a/yt_dlp/networking/_requests.py b/yt_dlp/networking/_requests.py
index d02e976b57..2927ea7ffb 100644
--- a/yt_dlp/networking/_requests.py
+++ b/yt_dlp/networking/_requests.py
@@ -140,6 +140,10 @@ class RequestsResponseAdapter(Response):
def read(self, amt: int | None = None):
try:
+ # Work around issue with `.read(amt)` then `.read()`
+ # See: https://github.com/urllib3/urllib3/issues/3636
+ if amt is None:
+ amt = (1 << 31) - 1
# Interact with urllib3 response directly.
return self.fp.read(amt, decode_content=True)