Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

CHUNK_SIZE: 8 KiB → 32 KiB

Adrián Chaves62398e424c0a
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index b17fd7881..5610595d3 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -12,6 +12,9 @@ except ImportError:
pass
+_CHUNK_SIZE = 65536 # 64 KiB
+
+
class _DecompressionMaxSizeExceeded(ValueError):
pass
@@ -23,9 +26,8 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes:
output_stream = BytesIO()
output_chunk = b"."
decompressed_size = 0
- CHUNK_SIZE = 8196
while output_chunk:
- input_chunk = input_stream.read(CHUNK_SIZE)
+ input_chunk = input_stream.read(_CHUNK_SIZE)
try:
output_chunk = decompressor.decompress(input_chunk)
except zlib.error:
@@ -57,9 +59,8 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:
output_stream = BytesIO()
output_chunk = b"."
decompressed_size = 0
- CHUNK_SIZE = 8196
while output_chunk:
- input_chunk = input_stream.read(CHUNK_SIZE)
+ input_chunk = input_stream.read(_CHUNK_SIZE)
output_chunk = decompressor.process(input_chunk)
decompressed_size += len(output_chunk)
if max_size and decompressed_size > max_size:
@@ -79,9 +80,8 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes:
output_stream = BytesIO()
output_chunk = b"."
decompressed_size = 0
- CHUNK_SIZE = 8196
while output_chunk:
- output_chunk = stream_reader.read(CHUNK_SIZE)
+ output_chunk = stream_reader.read(_CHUNK_SIZE)
decompressed_size += len(output_chunk)
if max_size and decompressed_size > max_size:
raise _DecompressionMaxSizeExceeded(
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 5d23e8f05..cf7316e82 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -4,7 +4,7 @@ from io import BytesIO
from scrapy.http import Response
-from ._compression import _DecompressionMaxSizeExceeded
+from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded
def gunzip(data: bytes, *, max_size: int = 0) -> bytes:
@@ -18,7 +18,7 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes:
decompressed_size = 0
while chunk:
try:
- chunk = f.read1(8196)
+ chunk = f.read1(_CHUNK_SIZE)
except (OSError, EOFError, struct.error):
# complete only if there is some data, otherwise re-raise
# see issue 87 about catching struct.error