Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

HttpErrorMiddleware stats

Mikhail Korobov26c1256f95e2
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index e34c265c1..def697c2b 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -46,6 +46,10 @@ class HttpErrorMiddleware(object):
def process_spider_exception(self, response, exception, spider):
if isinstance(exception, HttpError):
+ spider.crawler.stats.inc_value('httperror/response_ignored_count')
+ spider.crawler.stats.inc_value(
+ 'httperror/response_ignored_status_count/%s' % response.status
+ )
logger.info(
"Ignoring response %(response)r: HTTP status code is not handled or not allowed",
{'response': response}, extra={'spider': spider},
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 319746350..e1407e6b3 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -60,7 +60,8 @@ def _responses(request, status_codes):
class TestHttpErrorMiddleware(TestCase):
def setUp(self):
- self.spider = Spider('foo')
+ crawler = get_crawler(Spider)
+ self.spider = Spider.from_crawler(crawler, name='foo')
self.mw = HttpErrorMiddleware(Settings({}))
self.req = Request('http://scrapytest.org')
self.res200, self.res404 = _responses(self.req, [200, 404])
@@ -73,10 +74,10 @@ class TestHttpErrorMiddleware(TestCase):
def test_process_spider_exception(self):
self.assertEquals([],
- self.mw.process_spider_exception(self.res404, \
+ self.mw.process_spider_exception(self.res404,
HttpError(self.res404), self.spider))
self.assertEquals(None,
- self.mw.process_spider_exception(self.res404, \
+ self.mw.process_spider_exception(self.res404,
Exception(), self.spider))
def test_handle_httpstatus_list(self):
@@ -173,6 +174,12 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
self.assertEqual(crawler.spider.parsed, {'200'})
self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
+ get_value = crawler.stats.get_value
+ self.assertEqual(get_value('httperror/response_ignored_count'), 3)
+ self.assertEqual(get_value('httperror/response_ignored_status_count/404'), 1)
+ self.assertEqual(get_value('httperror/response_ignored_status_count/402'), 1)
+ self.assertEqual(get_value('httperror/response_ignored_status_count/500'), 1)
+
@defer.inlineCallbacks
def test_logging(self):
crawler = get_crawler(_HttpErrorSpider)