Scrapy, a fast high-level web crawling & scraping framework for Python.
HttpErrorMiddleware stats
Mikhail Korobov26c1256f95e2
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.pyindex e34c265c1..def697c2b 100644--- a/scrapy/spidermiddlewares/httperror.py+++ b/scrapy/spidermiddlewares/httperror.py@@ -46,6 +46,10 @@ class HttpErrorMiddleware(object):def process_spider_exception(self, response, exception, spider):if isinstance(exception, HttpError):+ spider.crawler.stats.inc_value('httperror/response_ignored_count')+ spider.crawler.stats.inc_value(+ 'httperror/response_ignored_status_count/%s' % response.status+ )logger.info("Ignoring response %(response)r: HTTP status code is not handled or not allowed",{'response': response}, extra={'spider': spider},diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex 319746350..e1407e6b3 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -60,7 +60,8 @@ def _responses(request, status_codes):class TestHttpErrorMiddleware(TestCase):def setUp(self):- self.spider = Spider('foo')+ crawler = get_crawler(Spider)+ self.spider = Spider.from_crawler(crawler, name='foo')self.mw = HttpErrorMiddleware(Settings({}))self.req = Request('http://scrapytest.org')self.res200, self.res404 = _responses(self.req, [200, 404])@@ -73,10 +74,10 @@ class TestHttpErrorMiddleware(TestCase):def test_process_spider_exception(self):self.assertEquals([],- self.mw.process_spider_exception(self.res404, \+ self.mw.process_spider_exception(self.res404,HttpError(self.res404), self.spider))self.assertEquals(None,- self.mw.process_spider_exception(self.res404, \+ self.mw.process_spider_exception(self.res404,Exception(), self.spider))def test_handle_httpstatus_list(self):@@ -173,6 +174,12 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):self.assertEqual(crawler.spider.parsed, {'200'})self.assertEqual(crawler.spider.failed, {'404', '402', '500'})+ get_value = crawler.stats.get_value+ self.assertEqual(get_value('httperror/response_ignored_count'), 3)+ self.assertEqual(get_value('httperror/response_ignored_status_count/404'), 1)+ self.assertEqual(get_value('httperror/response_ignored_status_count/402'), 1)+ self.assertEqual(get_value('httperror/response_ignored_status_count/500'), 1)+@defer.inlineCallbacksdef test_logging(self):crawler = get_crawler(_HttpErrorSpider)