Scrapy, a fast high-level web crawling & scraping framework for Python.
CHUNK_SIZE: 8 KiB → 32 KiB
Adrián Chaves62398e424c0a
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.pyindex b17fd7881..5610595d3 100644--- a/scrapy/utils/_compression.py+++ b/scrapy/utils/_compression.py@@ -12,6 +12,9 @@ except ImportError:pass+_CHUNK_SIZE = 65536 # 64 KiB++class _DecompressionMaxSizeExceeded(ValueError):pass@@ -23,9 +26,8 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes:output_stream = BytesIO()output_chunk = b"."decompressed_size = 0- CHUNK_SIZE = 8196while output_chunk:- input_chunk = input_stream.read(CHUNK_SIZE)+ input_chunk = input_stream.read(_CHUNK_SIZE)try:output_chunk = decompressor.decompress(input_chunk)except zlib.error:@@ -57,9 +59,8 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:output_stream = BytesIO()output_chunk = b"."decompressed_size = 0- CHUNK_SIZE = 8196while output_chunk:- input_chunk = input_stream.read(CHUNK_SIZE)+ input_chunk = input_stream.read(_CHUNK_SIZE)output_chunk = decompressor.process(input_chunk)decompressed_size += len(output_chunk)if max_size and decompressed_size > max_size:@@ -79,9 +80,8 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes:output_stream = BytesIO()output_chunk = b"."decompressed_size = 0- CHUNK_SIZE = 8196while output_chunk:- output_chunk = stream_reader.read(CHUNK_SIZE)+ output_chunk = stream_reader.read(_CHUNK_SIZE)decompressed_size += len(output_chunk)if max_size and decompressed_size > max_size:raise _DecompressionMaxSizeExceeded(diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.pyindex 5d23e8f05..cf7316e82 100644--- a/scrapy/utils/gz.py+++ b/scrapy/utils/gz.py@@ -4,7 +4,7 @@ from io import BytesIOfrom scrapy.http import Response-from ._compression import _DecompressionMaxSizeExceeded+from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceededdef gunzip(data: bytes, *, max_size: int = 0) -> bytes:@@ -18,7 +18,7 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes:decompressed_size = 0while chunk:try:- chunk = f.read1(8196)+ chunk = f.read1(_CHUNK_SIZE)except (OSError, EOFError, struct.error):# complete only if there is some data, otherwise re-raise# see issue 87 about catching struct.error