Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve http status all on http error middleware
Jose Galdos13181ba7882f
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.pyindex 375042340..bf908d2f7 100644--- a/scrapy/spidermiddlewares/httperror.py+++ b/scrapy/spidermiddlewares/httperror.py@@ -32,7 +32,7 @@ class HttpErrorMiddleware:if 200 <= response.status < 300: # common casereturnmeta = response.meta- if 'handle_httpstatus_all' in meta:+ if meta.get('handle_httpstatus_all', False):returnif 'handle_httpstatus_list' in meta:allowed_statuses = meta['handle_httpstatus_list']diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex e032b247c..f3e5478c4 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -139,6 +139,19 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):self.assertIsNone(self.mw.process_spider_input(res404, self.spider))self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)+ def test_httperror_allow_all_false(self):+ crawler = get_crawler(_HttpErrorSpider)+ mw = HttpErrorMiddleware.from_crawler(crawler)+ request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False})+ request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True})+ res404 = self.res404.copy()+ res404.request = request_httpstatus_false+ res402 = self.res402.copy()+ res402.request = request_httpstatus_true++ self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider)+ self.assertIsNone(mw.process_spider_input(res402, self.spider))+class TestHttpErrorMiddlewareIntegrational(TrialTestCase):def setUp(self):