Scrapy, a fast high-level web crawling & scraping framework for Python.
Add brotlicffi support
Laerte Pereiracab1016bb6f7
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.pyindex 7c40d0a02..7896f4c01 100644--- a/scrapy/utils/_compression.py+++ b/scrapy/utils/_compression.py@@ -6,6 +6,8 @@ from scrapy.exceptions import ScrapyDeprecationWarningtry:import brotli+except ImportError:+ import brotlicffi as brotliexcept ImportError:passelse:diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex ae5569d0a..7c36f748e 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -130,7 +130,10 @@ class HttpCompressionTest(TestCase):def test_process_response_br(self):try:- import brotli # noqa: F401+ try:+ import brotli # noqa: F401+ except ImportError:+ import brotlicffi # noqa: F401except ImportError:raise SkipTest("no brotli")response = self._getresponse("br")@@ -448,7 +451,10 @@ class HttpCompressionTest(TestCase):def test_compression_bomb_setting_br(self):try:- import brotli # noqa: F401+ try:+ import brotli # noqa: F401+ except ImportError:+ import brotlicffi # noqa: F401except ImportError:raise SkipTest("no brotli")self._test_compression_bomb_setting("br")@@ -486,7 +492,10 @@ class HttpCompressionTest(TestCase):def test_compression_bomb_spider_attr_br(self):try:- import brotli # noqa: F401+ try:+ import brotli # noqa: F401+ except ImportError:+ import brotlicffi # noqa: F401except ImportError:raise SkipTest("no brotli")self._test_compression_bomb_spider_attr("br")@@ -522,7 +531,10 @@ class HttpCompressionTest(TestCase):def test_compression_bomb_request_meta_br(self):try:- import brotli # noqa: F401+ try:+ import brotli # noqa: F401+ except ImportError:+ import brotlicffi # noqa: F401except ImportError:raise SkipTest("no brotli")self._test_compression_bomb_request_meta("br")@@ -568,7 +580,10 @@ class HttpCompressionTest(TestCase):def test_download_warnsize_setting_br(self):try:- import brotli # noqa: F401+ try:+ import brotli # noqa: F401+ except ImportError:+ import brotlicffi # noqa: F401except ImportError:raise SkipTest("no brotli")self._test_download_warnsize_setting("br")@@ -616,7 +631,10 @@ class HttpCompressionTest(TestCase):def test_download_warnsize_spider_attr_br(self):try:- import brotli # noqa: F401+ try:+ import brotli # noqa: F401+ except ImportError:+ import brotlicffi # noqa: F401except ImportError:raise SkipTest("no brotli")self._test_download_warnsize_spider_attr("br")@@ -662,7 +680,10 @@ class HttpCompressionTest(TestCase):def test_download_warnsize_request_meta_br(self):try:- import brotli # noqa: F401+ try:+ import brotli # noqa: F401+ except ImportError:+ import brotlicffi # noqa: F401except ImportError:raise SkipTest("no brotli")self._test_download_warnsize_request_meta("br")diff --git a/tox.ini b/tox.iniindex 237aa489c..6b804b78c 100644--- a/tox.ini+++ b/tox.ini@@ -162,6 +162,7 @@ basepython = {[testenv:pypy3]basepython}deps ={[pinned]deps}PyPyDispatcher==2.1.0+ brotlicfficommands =pytest --durations=10 scrapy testsinstall_command = {[pinned]install_command}