Scrapy, a fast high-level web crawling & scraping framework for Python.
Store output methods on the 'methods' dict
Eugenio Lacuesta985ab636cfa0
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 98e264bd3..c9dd8c91e 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -17,11 +17,6 @@ class SpiderMiddlewareManager(MiddlewareManager):component_name = 'spider middleware'- # List of dicts. Each dict represents a spider middleware and contains the- # 'process_spider_output', 'process_spider_exception' methods.- # The idea is to simulate the behaviour of a Twisted deferred's callback/errback chain- output_methods = []-@classmethoddef _get_mwlist_from_settings(cls, settings):return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))@@ -32,10 +27,8 @@ class SpiderMiddlewareManager(MiddlewareManager):self.methods['process_spider_input'].append(mw.process_spider_input)if hasattr(mw, 'process_start_requests'):self.methods['process_start_requests'].insert(0, mw.process_start_requests)- self.output_methods.insert(0, dict(- process_spider_output=getattr(mw, 'process_spider_output', None),- process_spider_exception=getattr(mw, 'process_spider_exception', None),- ))+ self.methods['process_spider_output'].insert(0, getattr(mw, 'process_spider_output', None))+ self.methods['process_spider_exception'].insert(0, getattr(mw, 'process_spider_exception', None))def scrape_response(self, scrape_func, response, request, spider):fname = lambda f:'%s.%s' % (@@ -53,54 +46,54 @@ class SpiderMiddlewareManager(MiddlewareManager):return scrape_func(Failure(), request, spider)return scrape_func(response, request, spider)- def process_spider_exception(_failure, mw_index):+ def process_spider_exception(_failure, index):exception = _failure.value# don't handle _InvalidOutput exceptionif isinstance(exception, _InvalidOutput):return _failure- for index, mw in enumerate(self.output_methods):- if index < mw_index or mw['process_spider_exception'] is None:+ for i, method in enumerate(self.methods['process_spider_exception']):+ if i < index or method is None:continue- result = mw['process_spider_exception'](response=response, exception=exception, spider=spider)- mw_index += 1+ result = method(response=response, exception=exception, spider=spider)+ index += 1if _isiterable(result):# stop exception handling by handing control over to the# process_spider_output chain if an iterable has been returned- return process_spider_output(result, mw_index)+ return process_spider_output(result, index)elif result is None:continueelse:raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \- .format(fname(mw['process_spider_exception']), type(result)))+ .format(fname(method), type(result)))return _failure- def process_spider_output(result, mw_index):+ def process_spider_output(result, index):def wrapper(result_iterable):try:for r in result_iterable:yield rexcept Exception as ex:# process the exception with the method from the next middleware- exception_result = process_spider_exception(Failure(ex), mw_index)+ exception_result = process_spider_exception(Failure(ex), index)if exception_result is None or isinstance(exception_result, Failure):raisefor output in exception_result:yield output- for index, mw in enumerate(self.output_methods):- if index < mw_index or mw['process_spider_output'] is None:+ for i, method in enumerate(self.methods['process_spider_output']):+ if i < index or method is None:continue- result = mw['process_spider_output'](response=response, result=result, spider=spider)- mw_index += 1+ result = method(response=response, result=result, spider=spider)+ index += 1if _isiterable(result):result = wrapper(result)else:raise _InvalidOutput('Middleware {} must return an iterable, got {}' \- .format(fname(mw['process_spider_output']), type(result)))+ .format(fname(method), type(result)))return resultdfd = mustbe_deferred(process_spider_input, response)- dfd.addErrback(process_spider_exception, mw_index=0)- dfd.addCallback(process_spider_output, mw_index=0)+ dfd.addErrback(process_spider_exception, index=0)+ dfd.addCallback(process_spider_output, index=0)return dfddef process_start_requests(self, start_requests, spider):