Scrapy, a fast high-level web crawling & scraping framework for Python.
Restore brotlipy support (#6261)
Yuri H. Galvao4cd94aa668c6
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.pyindex 5610595d3..14531df3f 100644--- a/scrapy/utils/_compression.py+++ b/scrapy/utils/_compression.py@@ -1,10 +1,40 @@import zlibfrom io import BytesIO+from warnings import warn++from scrapy.exceptions import ScrapyDeprecationWarningtry:import brotliexcept ImportError:pass+else:+ try:+ brotli.Decompressor.process+ except AttributeError:++ warn(+ (+ "You have brotlipy installed, and Scrapy will use it, but "+ "Scrapy support for brotlipy is deprecated and will stop "+ "working in a future version of Scrapy. brotlipy itself is "+ "deprecated, it has been superseded by brotlicffi (not "+ "currently supported by Scrapy). Please, uninstall brotlipy "+ "and install brotli instead. brotlipy has the same import "+ "name as brotli, so keeping both installed is strongly "+ "discouraged."+ ),+ ScrapyDeprecationWarning,+ )++ def _brotli_decompress(decompressor, data):+ return decompressor.decompress(data)++ else:++ def _brotli_decompress(decompressor, data):+ return decompressor.process(data)+try:import zstandard@@ -61,7 +91,7 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:decompressed_size = 0while output_chunk:input_chunk = input_stream.read(_CHUNK_SIZE)- output_chunk = decompressor.process(input_chunk)+ output_chunk = _brotli_decompress(decompressor, input_chunk)decompressed_size += len(output_chunk)if max_size and decompressed_size > max_size:raise _DecompressionMaxSizeExceeded(diff --git a/tox.ini b/tox.iniindex 4ed9b3bd7..237aa489c 100644--- a/tox.ini+++ b/tox.ini@@ -135,6 +135,7 @@ deps =google-cloud-storage==1.29.0Pillow==7.1.0robotexclusionrulesparser==1.6.2+ brotlipyinstall_command = {[pinned]install_command}setenv ={[pinned]setenv}