Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deferred-like process_output/process_exception chain

Eugenio Lacuesta4740dca8f260
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 50677670b..98e264bd3 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -17,6 +17,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
component_name = 'spider middleware'
+ # List of dicts. Each dict represents a spider middleware and contains the
+ # 'process_spider_output', 'process_spider_exception' methods.
+ # The idea is to simulate the behaviour of a Twisted deferred's callback/errback chain
+ output_methods = []
+
@classmethod
def _get_mwlist_from_settings(cls, settings):
return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
@@ -25,12 +30,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
super(SpiderMiddlewareManager, self)._add_middleware(mw)
if hasattr(mw, 'process_spider_input'):
self.methods['process_spider_input'].append(mw.process_spider_input)
- if hasattr(mw, 'process_spider_output'):
- self.methods['process_spider_output'].insert(0, mw.process_spider_output)
- if hasattr(mw, 'process_spider_exception'):
- self.methods['process_spider_exception'].insert(0, mw.process_spider_exception)
if hasattr(mw, 'process_start_requests'):
self.methods['process_start_requests'].insert(0, mw.process_start_requests)
+ self.output_methods.insert(0, dict(
+ process_spider_output=getattr(mw, 'process_spider_output', None),
+ process_spider_exception=getattr(mw, 'process_spider_exception', None),
+ ))
def scrape_response(self, scrape_func, response, request, spider):
fname = lambda f:'%s.%s' % (
@@ -48,45 +53,54 @@ class SpiderMiddlewareManager(MiddlewareManager):
return scrape_func(Failure(), request, spider)
return scrape_func(response, request, spider)
- def process_spider_exception(_failure):
+ def process_spider_exception(_failure, mw_index):
exception = _failure.value
# don't handle _InvalidOutput exception
if isinstance(exception, _InvalidOutput):
return _failure
- for method in self.methods['process_spider_exception']:
- result = method(response=response, exception=exception, spider=spider)
- if result is not None and not _isiterable(result):
- raise _InvalidOutput('Middleware {} must return None or an iterable ' \
- 'object, got {}'.format(fname(method), type(result)))
- # stop exception handling by handing control over to the
- # process_spider_output chain if an iterable has been returned
- if result is not None:
- return result
+ for index, mw in enumerate(self.output_methods):
+ if index < mw_index or mw['process_spider_exception'] is None:
+ continue
+ result = mw['process_spider_exception'](response=response, exception=exception, spider=spider)
+ mw_index += 1
+ if _isiterable(result):
+ # stop exception handling by handing control over to the
+ # process_spider_output chain if an iterable has been returned
+ return process_spider_output(result, mw_index)
+ elif result is None:
+ continue
+ else:
+ raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
+ .format(fname(mw['process_spider_exception']), type(result)))
return _failure
- def process_spider_output(result):
+ def process_spider_output(result, mw_index):
def wrapper(result_iterable):
try:
for r in result_iterable:
yield r
except Exception as ex:
- exception_result = process_spider_exception(Failure(ex))
+ # process the exception with the method from the next middleware
+ exception_result = process_spider_exception(Failure(ex), mw_index)
if exception_result is None or isinstance(exception_result, Failure):
raise
for output in exception_result:
yield output
- for method in self.methods['process_spider_output']:
- result = method(response=response, result=result, spider=spider)
+ for index, mw in enumerate(self.output_methods):
+ if index < mw_index or mw['process_spider_output'] is None:
+ continue
+ result = mw['process_spider_output'](response=response, result=result, spider=spider)
+ mw_index += 1
if _isiterable(result):
result = wrapper(result)
else:
- raise _InvalidOutput('Middleware {} must return an iterable object, ' \
- 'got {}'.format(fname(method), type(result)))
+ raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
+ .format(fname(mw['process_spider_output']), type(result)))
return result
dfd = mustbe_deferred(process_spider_input, response)
- dfd.addErrback(process_spider_exception)
- dfd.addCallback(process_spider_output)
+ dfd.addErrback(process_spider_exception, mw_index=0)
+ dfd.addCallback(process_spider_output, mw_index=0)
return dfd
def process_start_requests(self, start_requests, spider):