Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Restore brotlipy support (#6261)

Yuri H. Galvao4cd94aa668c6
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 5610595d3..14531df3f 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -1,10 +1,40 @@
import zlib
from io import BytesIO
+from warnings import warn
+
+from scrapy.exceptions import ScrapyDeprecationWarning
try:
import brotli
except ImportError:
pass
+else:
+ try:
+ brotli.Decompressor.process
+ except AttributeError:
+
+ warn(
+ (
+ "You have brotlipy installed, and Scrapy will use it, but "
+ "Scrapy support for brotlipy is deprecated and will stop "
+ "working in a future version of Scrapy. brotlipy itself is "
+ "deprecated, it has been superseded by brotlicffi (not "
+ "currently supported by Scrapy). Please, uninstall brotlipy "
+ "and install brotli instead. brotlipy has the same import "
+ "name as brotli, so keeping both installed is strongly "
+ "discouraged."
+ ),
+ ScrapyDeprecationWarning,
+ )
+
+ def _brotli_decompress(decompressor, data):
+ return decompressor.decompress(data)
+
+ else:
+
+ def _brotli_decompress(decompressor, data):
+ return decompressor.process(data)
+
try:
import zstandard
@@ -61,7 +91,7 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:
decompressed_size = 0
while output_chunk:
input_chunk = input_stream.read(_CHUNK_SIZE)
- output_chunk = decompressor.process(input_chunk)
+ output_chunk = _brotli_decompress(decompressor, input_chunk)
decompressed_size += len(output_chunk)
if max_size and decompressed_size > max_size:
raise _DecompressionMaxSizeExceeded(
diff --git a/tox.ini b/tox.ini
index 4ed9b3bd7..237aa489c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -135,6 +135,7 @@ deps =
google-cloud-storage==1.29.0
Pillow==7.1.0
robotexclusionrulesparser==1.6.2
+ brotlipy
install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}