Scrapy, a fast high-level web crawling & scraping framework for Python.
[httpcompression] import brotli when available
Pawel Miechf8f8bbe08016
diff --git a/requirements.txt b/requirements.txtindex 362d05013..f92603d3d 100644--- a/requirements.txt+++ b/requirements.txt@@ -8,4 +8,3 @@ six>=1.5.2PyDispatcher>=2.0.5service_identityparsel>=1.1-brotlipy==0.6diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 9202fd8da..04c9e355d 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -1,6 +1,5 @@import zlib-import brotlifrom scrapy.utils.gz import gunzip, is_gzippedfrom scrapy.http import Response, TextResponse@@ -8,10 +7,18 @@ from scrapy.responsetypes import responsetypesfrom scrapy.exceptions import NotConfigured+ACCEPTED_ENCODINGS = [b'gzip', b'deflate']++try:+ import brotli+ ACCEPTED_ENCODINGS.append(b'br')+except ImportError:+ pass++class HttpCompressionMiddleware(object):"""This middleware allows compressed (gzip, deflate) traffic to besent/received from web sites"""-@classmethoddef from_crawler(cls, crawler):if not crawler.settings.getbool('COMPRESSION_ENABLED'):@@ -19,7 +26,8 @@ class HttpCompressionMiddleware(object):return cls()def process_request(self, request, spider):- request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br')+ request.headers.setdefault('Accept-Encoding',+ ",".join(ACCEPTED_ENCODINGS))def process_response(self, request, response, spider):@@ -57,7 +65,6 @@ class HttpCompressionMiddleware(object):# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx# http://www.gzip.org/zlib/zlib_faq.html#faq38body = zlib.decompress(body, -15)- if encoding == b"br":+ if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:body = brotli.decompress(body)return body-diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex c47de54ed..b47b267e2 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -1,6 +1,6 @@from io import BytesIOfrom unittest import TestCase-from os.path import join, abspath, dirname+from os.path import joinfrom gzip import GzipFilefrom scrapy.spiders import Spider@@ -20,6 +20,7 @@ FORMAT = {'br': ('html-br.bin', 'br')}+class HttpCompressionTest(TestCase):def setUp(self):