Scrapy, a fast high-level web crawling & scraping framework for Python.
[httpcompression] add support for br - brotli content encoding
Pawel Miech16c4b4e184c1
diff --git a/requirements.txt b/requirements.txtindex f92603d3d..362d05013 100644--- a/requirements.txt+++ b/requirements.txt@@ -8,3 +8,4 @@ six>=1.5.2PyDispatcher>=2.0.5service_identityparsel>=1.1+brotlipy==0.6diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex bcf20f10c..9202fd8da 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -1,5 +1,7 @@import zlib+import brotli+from scrapy.utils.gz import gunzip, is_gzippedfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypes@@ -17,7 +19,7 @@ class HttpCompressionMiddleware(object):return cls()def process_request(self, request, spider):- request.headers.setdefault('Accept-Encoding', 'gzip,deflate')+ request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br')def process_response(self, request, response, spider):@@ -55,5 +57,7 @@ class HttpCompressionMiddleware(object):# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx# http://www.gzip.org/zlib/zlib_faq.html#faq38body = zlib.decompress(body, -15)+ if encoding == b"br":+ body = brotli.decompress(body)return bodydiff --git a/tests/sample_data/compressed/html-br.bin b/tests/sample_data/compressed/html-br.binnew file mode 100644index 000000000..c7eea4bb8Binary files /dev/null and b/tests/sample_data/compressed/html-br.bin differdiff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex b2426946d..c47de54ed 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -17,6 +17,7 @@ FORMAT = {'x-gzip': ('html-gzip.bin', 'gzip'),'rawdeflate': ('html-rawdeflate.bin', 'deflate'),'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),+ 'br': ('html-br.bin', 'br')}class HttpCompressionTest(TestCase):@@ -50,7 +51,8 @@ class HttpCompressionTest(TestCase):request = Request('http://scrapytest.org')assert 'Accept-Encoding' not in request.headersself.mw.process_request(request, self.spider)- self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate')+ self.assertEqual(request.headers.get('Accept-Encoding'),+ b'gzip,deflate,br')def test_process_response_gzip(self):response = self._getresponse('gzip')@@ -62,6 +64,15 @@ class HttpCompressionTest(TestCase):assert newresponse.body.startswith(b'<!DOCTYPE')assert 'Content-Encoding' not in newresponse.headers+ def test_process_response_br(self):+ response = self._getresponse('br')+ request = response.request+ self.assertEqual(response.headers['Content-Encoding'], b'br')+ newresponse = self.mw.process_response(request, response, self.spider)+ assert newresponse is not response+ assert newresponse.body.startswith(b"<!DOCTYPE")+ assert 'Content-Encoding' not in newresponse.headers+def test_process_response_rawdeflate(self):response = self._getresponse('rawdeflate')request = response.request