Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve http status all on http error middleware

Jose Galdos13181ba7882f
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 375042340..bf908d2f7 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -32,7 +32,7 @@ class HttpErrorMiddleware:
if 200 <= response.status < 300: # common case
return
meta = response.meta
- if 'handle_httpstatus_all' in meta:
+ if meta.get('handle_httpstatus_all', False):
return
if 'handle_httpstatus_list' in meta:
allowed_statuses = meta['handle_httpstatus_list']
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index e032b247c..f3e5478c4 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -139,6 +139,19 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
+ def test_httperror_allow_all_false(self):
+ crawler = get_crawler(_HttpErrorSpider)
+ mw = HttpErrorMiddleware.from_crawler(crawler)
+ request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False})
+ request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True})
+ res404 = self.res404.copy()
+ res404.request = request_httpstatus_false
+ res402 = self.res402.copy()
+ res402.request = request_httpstatus_true
+
+ self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider)
+ self.assertIsNone(mw.process_spider_input(res402, self.spider))
+
class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
def setUp(self):