Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Store output methods on the 'methods' dict

Eugenio Lacuesta985ab636cfa0
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 98e264bd3..c9dd8c91e 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -17,11 +17,6 @@ class SpiderMiddlewareManager(MiddlewareManager):
component_name = 'spider middleware'
- # List of dicts. Each dict represents a spider middleware and contains the
- # 'process_spider_output', 'process_spider_exception' methods.
- # The idea is to simulate the behaviour of a Twisted deferred's callback/errback chain
- output_methods = []
-
@classmethod
def _get_mwlist_from_settings(cls, settings):
return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
@@ -32,10 +27,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
self.methods['process_spider_input'].append(mw.process_spider_input)
if hasattr(mw, 'process_start_requests'):
self.methods['process_start_requests'].insert(0, mw.process_start_requests)
- self.output_methods.insert(0, dict(
- process_spider_output=getattr(mw, 'process_spider_output', None),
- process_spider_exception=getattr(mw, 'process_spider_exception', None),
- ))
+ self.methods['process_spider_output'].insert(0, getattr(mw, 'process_spider_output', None))
+ self.methods['process_spider_exception'].insert(0, getattr(mw, 'process_spider_exception', None))
def scrape_response(self, scrape_func, response, request, spider):
fname = lambda f:'%s.%s' % (
@@ -53,54 +46,54 @@ class SpiderMiddlewareManager(MiddlewareManager):
return scrape_func(Failure(), request, spider)
return scrape_func(response, request, spider)
- def process_spider_exception(_failure, mw_index):
+ def process_spider_exception(_failure, index):
exception = _failure.value
# don't handle _InvalidOutput exception
if isinstance(exception, _InvalidOutput):
return _failure
- for index, mw in enumerate(self.output_methods):
- if index < mw_index or mw['process_spider_exception'] is None:
+ for i, method in enumerate(self.methods['process_spider_exception']):
+ if i < index or method is None:
continue
- result = mw['process_spider_exception'](response=response, exception=exception, spider=spider)
- mw_index += 1
+ result = method(response=response, exception=exception, spider=spider)
+ index += 1
if _isiterable(result):
# stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned
- return process_spider_output(result, mw_index)
+ return process_spider_output(result, index)
elif result is None:
continue
else:
raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
- .format(fname(mw['process_spider_exception']), type(result)))
+ .format(fname(method), type(result)))
return _failure
- def process_spider_output(result, mw_index):
+ def process_spider_output(result, index):
def wrapper(result_iterable):
try:
for r in result_iterable:
yield r
except Exception as ex:
# process the exception with the method from the next middleware
- exception_result = process_spider_exception(Failure(ex), mw_index)
+ exception_result = process_spider_exception(Failure(ex), index)
if exception_result is None or isinstance(exception_result, Failure):
raise
for output in exception_result:
yield output
- for index, mw in enumerate(self.output_methods):
- if index < mw_index or mw['process_spider_output'] is None:
+ for i, method in enumerate(self.methods['process_spider_output']):
+ if i < index or method is None:
continue
- result = mw['process_spider_output'](response=response, result=result, spider=spider)
- mw_index += 1
+ result = method(response=response, result=result, spider=spider)
+ index += 1
if _isiterable(result):
result = wrapper(result)
else:
raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
- .format(fname(mw['process_spider_output']), type(result)))
+ .format(fname(method), type(result)))
return result
dfd = mustbe_deferred(process_spider_input, response)
- dfd.addErrback(process_spider_exception, mw_index=0)
- dfd.addCallback(process_spider_output, mw_index=0)
+ dfd.addErrback(process_spider_exception, index=0)
+ dfd.addCallback(process_spider_output, index=0)
return dfd
def process_start_requests(self, start_requests, spider):