Scrapy, a fast high-level web crawling & scraping framework for Python.
httocompression tests added
GeorgeA92c22e810658b2
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex b34f76f21..ca80e9444 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -18,11 +18,14 @@ except ImportError:class HttpCompressionMiddleware:"""This middleware allows compressed (gzip, deflate) traffic to besent/received from web sites"""+ def __init__(self, stats):+ self.stats = stats+@classmethoddef from_crawler(cls, crawler):if not crawler.settings.getbool('COMPRESSION_ENABLED'):raise NotConfigured- return cls()+ return cls(crawler.stats)def process_request(self, request, spider):request.headers.setdefault('Accept-Encoding',@@ -37,8 +40,8 @@ class HttpCompressionMiddleware:if content_encoding:encoding = content_encoding.pop()decoded_body = self._decode(response.body, encoding.lower())- spider.crawler.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)- spider.crawler.stats.inc_value('httpcompression/response_count', spider=spider)+ self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)+ self.stats.inc_value('httpcompression/response_count', spider=spider)respcls = responsetypes.from_args(headers=response.headers, url=response.url, body=decoded_body)diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex a806f55ce..998749c2e 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -8,6 +8,7 @@ from scrapy.http import Response, Request, HtmlResponsefrom scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGSfrom scrapy.responsetypes import responsetypesfrom scrapy.utils.gz import gunzip+from scrapy.utils.test import get_crawlerfrom tests import tests_datadirfrom w3lib.encoding import resolve_encoding@@ -26,8 +27,10 @@ FORMAT = {class HttpCompressionTest(TestCase):def setUp(self):- self.spider = Spider('foo')- self.mw = HttpCompressionMiddleware()+ self.crawler = get_crawler(Spider)+ self.spider = self.crawler._create_spider('scrapytest.org')+ self.mw = HttpCompressionMiddleware(self.crawler.stats)+ self.crawler.stats.open_spider(self.spider)def _getresponse(self, coding):if coding not in FORMAT:@@ -50,6 +53,13 @@ class HttpCompressionTest(TestCase):response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})return response+ def assertStatsEqual(self, key, value):+ self.assertEqual(+ self.crawler.stats.get_value(key, spider=self.spider),+ value,+ str(self.crawler.stats.get_stats(self.spider))+ )+def test_process_request(self):request = Request('http://scrapytest.org')assert 'Accept-Encoding' not in request.headers@@ -66,6 +76,7 @@ class HttpCompressionTest(TestCase):assert newresponse is not responseassert newresponse.body.startswith(b'<!DOCTYPE')assert 'Content-Encoding' not in newresponse.headers+ self.assertStatsEqual('httpcompression/response_count', 1)def test_process_response_br(self):try:@@ -79,6 +90,7 @@ class HttpCompressionTest(TestCase):assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert 'Content-Encoding' not in newresponse.headers+ self.assertStatsEqual('httpcompression/response_count', 1)def test_process_response_rawdeflate(self):response = self._getresponse('rawdeflate')@@ -89,6 +101,7 @@ class HttpCompressionTest(TestCase):assert newresponse is not responseassert newresponse.body.startswith(b'<!DOCTYPE')assert 'Content-Encoding' not in newresponse.headers+ self.assertStatsEqual('httpcompression/response_count', 1)def test_process_response_zlibdelate(self):response = self._getresponse('zlibdeflate')@@ -99,6 +112,7 @@ class HttpCompressionTest(TestCase):assert newresponse is not responseassert newresponse.body.startswith(b'<!DOCTYPE')assert 'Content-Encoding' not in newresponse.headers+ self.assertStatsEqual('httpcompression/response_count', 1)def test_process_response_plain(self):response = Response('http://scrapytest.org', body=b'<!DOCTYPE...')@@ -108,6 +122,7 @@ class HttpCompressionTest(TestCase):newresponse = self.mw.process_response(request, response, self.spider)assert newresponse is responseassert newresponse.body.startswith(b'<!DOCTYPE')+ self.assertStatsEqual('httpcompression/response_count', None)def test_multipleencodings(self):response = self._getresponse('gzip')@@ -135,6 +150,7 @@ class HttpCompressionTest(TestCase):assert isinstance(newresponse, HtmlResponse)self.assertEqual(newresponse.body, plainbody)self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))+ self.assertStatsEqual('httpcompression/response_count', 1)def test_process_response_force_recalculate_encoding(self):headers = {@@ -154,6 +170,7 @@ class HttpCompressionTest(TestCase):assert isinstance(newresponse, HtmlResponse)self.assertEqual(newresponse.body, plainbody)self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))+ self.assertStatsEqual('httpcompression/response_count', 1)def test_process_response_no_content_type_header(self):headers = {@@ -169,6 +186,7 @@ class HttpCompressionTest(TestCase):assert isinstance(newresponse, respcls)self.assertEqual(newresponse.body, plainbody)self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))+ self.assertStatsEqual('httpcompression/response_count', 1)def test_process_response_gzipped_contenttype(self):response = self._getresponse('gzip')@@ -179,6 +197,7 @@ class HttpCompressionTest(TestCase):self.assertIsNot(newresponse, response)self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))self.assertNotIn('Content-Encoding', newresponse.headers)+ self.assertStatsEqual('httpcompression/response_count', 1)def test_process_response_gzip_app_octetstream_contenttype(self):response = self._getresponse('gzip')@@ -189,6 +208,7 @@ class HttpCompressionTest(TestCase):self.assertIsNot(newresponse, response)self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))self.assertNotIn('Content-Encoding', newresponse.headers)+ self.assertStatsEqual('httpcompression/response_count', 1)def test_process_response_gzip_binary_octetstream_contenttype(self):response = self._getresponse('x-gzip')@@ -199,6 +219,7 @@ class HttpCompressionTest(TestCase):self.assertIsNot(newresponse, response)self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))self.assertNotIn('Content-Encoding', newresponse.headers)+ self.assertStatsEqual('httpcompression/response_count', 1)def test_process_response_gzipped_gzip_file(self):"""Test that a gzip Content-Encoded .gz file is gunzipped@@ -241,6 +262,7 @@ class HttpCompressionTest(TestCase):newresponse = self.mw.process_response(request, response, self.spider)self.assertEqual(gunzip(newresponse.body), plainbody)+ self.assertStatsEqual('httpcompression/response_count', 1)def test_process_response_head_request_no_decode_required(self):response = self._getresponse('gzip')@@ -251,3 +273,4 @@ class HttpCompressionTest(TestCase):newresponse = self.mw.process_response(request, response, self.spider)self.assertIs(newresponse, response)self.assertEqual(response.body, b'')+ self.assertStatsEqual('httpcompression/response_count', None)