Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[httpcompression] add support for br - brotli content encoding

Pawel Miech16c4b4e184c1
diff --git a/requirements.txt b/requirements.txt
index f92603d3d..362d05013 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,3 +8,4 @@ six>=1.5.2
PyDispatcher>=2.0.5
service_identity
parsel>=1.1
+brotlipy==0.6
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index bcf20f10c..9202fd8da 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,5 +1,7 @@
import zlib
+import brotli
+
from scrapy.utils.gz import gunzip, is_gzipped
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
@@ -17,7 +19,7 @@ class HttpCompressionMiddleware(object):
return cls()
def process_request(self, request, spider):
- request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
+ request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br')
def process_response(self, request, response, spider):
@@ -55,5 +57,7 @@ class HttpCompressionMiddleware(object):
# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
# http://www.gzip.org/zlib/zlib_faq.html#faq38
body = zlib.decompress(body, -15)
+ if encoding == b"br":
+ body = brotli.decompress(body)
return body
diff --git a/tests/sample_data/compressed/html-br.bin b/tests/sample_data/compressed/html-br.bin
new file mode 100644
index 000000000..c7eea4bb8
Binary files /dev/null and b/tests/sample_data/compressed/html-br.bin differ
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index b2426946d..c47de54ed 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -17,6 +17,7 @@ FORMAT = {
'x-gzip': ('html-gzip.bin', 'gzip'),
'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
+ 'br': ('html-br.bin', 'br')
}
class HttpCompressionTest(TestCase):
@@ -50,7 +51,8 @@ class HttpCompressionTest(TestCase):
request = Request('http://scrapytest.org')
assert 'Accept-Encoding' not in request.headers
self.mw.process_request(request, self.spider)
- self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate')
+ self.assertEqual(request.headers.get('Accept-Encoding'),
+ b'gzip,deflate,br')
def test_process_response_gzip(self):
response = self._getresponse('gzip')
@@ -62,6 +64,15 @@ class HttpCompressionTest(TestCase):
assert newresponse.body.startswith(b'<!DOCTYPE')
assert 'Content-Encoding' not in newresponse.headers
+ def test_process_response_br(self):
+ response = self._getresponse('br')
+ request = response.request
+ self.assertEqual(response.headers['Content-Encoding'], b'br')
+ newresponse = self.mw.process_response(request, response, self.spider)
+ assert newresponse is not response
+ assert newresponse.body.startswith(b"<!DOCTYPE")
+ assert 'Content-Encoding' not in newresponse.headers
+
def test_process_response_rawdeflate(self):
response = self._getresponse('rawdeflate')
request = response.request