Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[httpcompression] skip test if no brotli

Pawel Miech3daf473686aa
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 04c9e355d..2fc1bb8eb 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -27,7 +27,7 @@ class HttpCompressionMiddleware(object):
def process_request(self, request, spider):
request.headers.setdefault('Accept-Encoding',
- ",".join(ACCEPTED_ENCODINGS))
+ b",".join(ACCEPTED_ENCODINGS))
def process_response(self, request, response, spider):
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 9d0c3c996..9baa4be21 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -6,6 +6,7 @@ pytest==2.9.2
pytest-twisted
pytest-cov==2.2.1
jmespath
+brotlipy==0.6
testfixtures
# optional for shell wrapper tests
bpython
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index b47b267e2..7924fb3b5 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -1,11 +1,12 @@
from io import BytesIO
-from unittest import TestCase
+from unittest import TestCase, SkipTest
from os.path import join
from gzip import GzipFile
from scrapy.spiders import Spider
from scrapy.http import Response, Request, HtmlResponse
-from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware
+from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \
+ ACCEPTED_ENCODINGS
from tests import tests_datadir
from w3lib.encoding import resolve_encoding
@@ -53,7 +54,7 @@ class HttpCompressionTest(TestCase):
assert 'Accept-Encoding' not in request.headers
self.mw.process_request(request, self.spider)
self.assertEqual(request.headers.get('Accept-Encoding'),
- b'gzip,deflate,br')
+ b','.join(ACCEPTED_ENCODINGS))
def test_process_response_gzip(self):
response = self._getresponse('gzip')
@@ -66,6 +67,10 @@ class HttpCompressionTest(TestCase):
assert 'Content-Encoding' not in newresponse.headers
def test_process_response_br(self):
+ try:
+ import brotli
+ except ImportError:
+ raise SkipTest("no brotli")
response = self._getresponse('br')
request = response.request
self.assertEqual(response.headers['Content-Encoding'], b'br')