Scrapy, a fast high-level web crawling & scraping framework for Python.
Better handling of method indexes
Eugenio Lacuesta20defa2e1662
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 96488806d..8607ed620 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -70,20 +70,19 @@ class SpiderMiddlewareManager(MiddlewareManager):return scrape_func(Failure(), request, spider)return scrape_func(response, request, spider)- def process_spider_exception(_failure, index=0):+ def process_spider_exception(_failure, start_index=0):exception = _failure.value# don't handle _InvalidOutput exceptionif isinstance(exception, _InvalidOutput):return _failure- for method in self.methods['process_spider_exception'][index:]:+ for method_index, method in enumerate(self.methods['process_spider_exception'][start_index:], start=start_index):if method is None:continue- index += 1result = method(response=response, exception=exception, spider=spider)if _isiterable(result):# stop exception handling by handing control over to the# process_spider_output chain if an iterable has been returned- return process_spider_output(result, index)+ return process_spider_output(result, method_index+1)elif result is None:continueelse:@@ -91,7 +90,7 @@ class SpiderMiddlewareManager(MiddlewareManager):.format(fname(method), type(result)))return _failure- def process_spider_output(result, index=0):+ def process_spider_output(result, start_index=0):# items in this iterable do not need to go through the process_spider_output# chain, they went through it already from the process_spider_exception methodrecovered = MutableChain()@@ -106,20 +105,19 @@ class SpiderMiddlewareManager(MiddlewareManager):raiserecovered.extend(exception_result)- for method in self.methods['process_spider_output'][index:]:+ for method_index, method in enumerate(self.methods['process_spider_output'][start_index:], start=start_index):if method is None:continue- index += 1# the following might fail directly if the output value is not a generatortry:result = method(response=response, result=result, spider=spider)except Exception as ex:- exception_result = process_spider_exception(Failure(ex), index+1)+ exception_result = process_spider_exception(Failure(ex), method_index+1)if exception_result is None or isinstance(exception_result, Failure):raisereturn exception_resultif _isiterable(result):- result = evaluate_iterable(result, index)+ result = evaluate_iterable(result, method_index)else:raise _InvalidOutput('Middleware {} must return an iterable, got {}' \.format(fname(method), type(result)))