Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add brotlicffi support

Laerte Pereiracab1016bb6f7
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 7c40d0a02..7896f4c01 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -6,6 +6,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning
try:
import brotli
+except ImportError:
+ import brotlicffi as brotli
except ImportError:
pass
else:
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index ae5569d0a..7c36f748e 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -130,7 +130,10 @@ class HttpCompressionTest(TestCase):
def test_process_response_br(self):
try:
- import brotli # noqa: F401
+ try:
+ import brotli # noqa: F401
+ except ImportError:
+ import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
response = self._getresponse("br")
@@ -448,7 +451,10 @@ class HttpCompressionTest(TestCase):
def test_compression_bomb_setting_br(self):
try:
- import brotli # noqa: F401
+ try:
+ import brotli # noqa: F401
+ except ImportError:
+ import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_compression_bomb_setting("br")
@@ -486,7 +492,10 @@ class HttpCompressionTest(TestCase):
def test_compression_bomb_spider_attr_br(self):
try:
- import brotli # noqa: F401
+ try:
+ import brotli # noqa: F401
+ except ImportError:
+ import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_compression_bomb_spider_attr("br")
@@ -522,7 +531,10 @@ class HttpCompressionTest(TestCase):
def test_compression_bomb_request_meta_br(self):
try:
- import brotli # noqa: F401
+ try:
+ import brotli # noqa: F401
+ except ImportError:
+ import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_compression_bomb_request_meta("br")
@@ -568,7 +580,10 @@ class HttpCompressionTest(TestCase):
def test_download_warnsize_setting_br(self):
try:
- import brotli # noqa: F401
+ try:
+ import brotli # noqa: F401
+ except ImportError:
+ import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_download_warnsize_setting("br")
@@ -616,7 +631,10 @@ class HttpCompressionTest(TestCase):
def test_download_warnsize_spider_attr_br(self):
try:
- import brotli # noqa: F401
+ try:
+ import brotli # noqa: F401
+ except ImportError:
+ import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_download_warnsize_spider_attr("br")
@@ -662,7 +680,10 @@ class HttpCompressionTest(TestCase):
def test_download_warnsize_request_meta_br(self):
try:
- import brotli # noqa: F401
+ try:
+ import brotli # noqa: F401
+ except ImportError:
+ import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_download_warnsize_request_meta("br")
diff --git a/tox.ini b/tox.ini
index 237aa489c..6b804b78c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -162,6 +162,7 @@ basepython = {[testenv:pypy3]basepython}
deps =
{[pinned]deps}
PyPyDispatcher==2.1.0
+ brotlicffi
commands =
pytest --durations=10 scrapy tests
install_command = {[pinned]install_command}