Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[httpcompression] import brotli when available

Pawel Miechf8f8bbe08016
diff --git a/requirements.txt b/requirements.txt
index 362d05013..f92603d3d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,4 +8,3 @@ six>=1.5.2
PyDispatcher>=2.0.5
service_identity
parsel>=1.1
-brotlipy==0.6
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 9202fd8da..04c9e355d 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,6 +1,5 @@
import zlib
-import brotli
from scrapy.utils.gz import gunzip, is_gzipped
from scrapy.http import Response, TextResponse
@@ -8,10 +7,18 @@ from scrapy.responsetypes import responsetypes
from scrapy.exceptions import NotConfigured
+ACCEPTED_ENCODINGS = [b'gzip', b'deflate']
+
+try:
+ import brotli
+ ACCEPTED_ENCODINGS.append(b'br')
+except ImportError:
+ pass
+
+
class HttpCompressionMiddleware(object):
"""This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites"""
-
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('COMPRESSION_ENABLED'):
@@ -19,7 +26,8 @@ class HttpCompressionMiddleware(object):
return cls()
def process_request(self, request, spider):
- request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br')
+ request.headers.setdefault('Accept-Encoding',
+ ",".join(ACCEPTED_ENCODINGS))
def process_response(self, request, response, spider):
@@ -57,7 +65,6 @@ class HttpCompressionMiddleware(object):
# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
# http://www.gzip.org/zlib/zlib_faq.html#faq38
body = zlib.decompress(body, -15)
- if encoding == b"br":
+ if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
body = brotli.decompress(body)
return body
-
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index c47de54ed..b47b267e2 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -1,6 +1,6 @@
from io import BytesIO
from unittest import TestCase
-from os.path import join, abspath, dirname
+from os.path import join
from gzip import GzipFile
from scrapy.spiders import Spider
@@ -20,6 +20,7 @@ FORMAT = {
'br': ('html-br.bin', 'br')
}
+
class HttpCompressionTest(TestCase):
def setUp(self):