Scrapy, a fast high-level web crawling & scraping framework for Python.
Untested experiment
Eugenio Lacuestac5fa0ae6bc53
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex c9dd8c91e..8ee42c2cf 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -3,6 +3,8 @@ Spider Middleware managerSee documentation in docs/topics/spider-middleware.rst"""+from itertools import chain+import sixfrom twisted.python.failure import Failurefrom scrapy.exceptions import _InvalidOutput@@ -10,9 +12,31 @@ from scrapy.middleware import MiddlewareManagerfrom scrapy.utils.defer import mustbe_deferredfrom scrapy.utils.conf import build_component_list+def _isiterable(possible_iterator):return hasattr(possible_iterator, '__iter__')++class MutableChain:+ """+ Thin wrapper around itertools.chain, allowing to add iterables "in-place"+ """+ def __init__(self, *args):+ self.data = chain(*args)++ def extend(self, *iterables):+ self.data = chain(self.data, *iterables)++ def __iter__(self):+ return self.data.__iter__()++ def __next__(self): # py3+ return self.data.__next__()++ def next(self): # py2+ return self.data.next()++class SpiderMiddlewareManager(MiddlewareManager):component_name = 'spider middleware'@@ -68,28 +92,32 @@ class SpiderMiddlewareManager(MiddlewareManager):return _failuredef process_spider_output(result, index):- def wrapper(result_iterable):+ # items in this iterable do not need to go through the process_spider_output+ # chain, they went through it already from the process_spider_exception method+ recovered = MutableChain()++ def evaluate_result(result_iterable, index):try:for r in result_iterable:yield rexcept Exception as ex:- # process the exception with the method from the next middlewareexception_result = process_spider_exception(Failure(ex), index)if exception_result is None or isinstance(exception_result, Failure):raise- for output in exception_result:- yield output+ recovered.extend(exception_result)+for i, method in enumerate(self.methods['process_spider_output']):if i < index or method is None:continueresult = method(response=response, result=result, spider=spider)index += 1if _isiterable(result):- result = wrapper(result)+ result = evaluate_result(result, index)else:raise _InvalidOutput('Middleware {} must return an iterable, got {}' \.format(fname(method), type(result)))- return result++ return chain(result, recovered)dfd = mustbe_deferred(process_spider_input, response)dfd.addErrback(process_spider_exception, index=0)