Scrapy, a fast high-level web crawling & scraping framework for Python.
Extract more common code.
Andrey Rakhmatullinc7b90c6e1e3d
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.pyindex 29634c3ad..4c923b1b3 100644--- a/scrapy/spidermiddlewares/depth.py+++ b/scrapy/spidermiddlewares/depth.py@@ -28,25 +28,22 @@ class DepthMiddleware:return cls(maxdepth, crawler.stats, verbose, prio)def process_spider_output(self, response, result, spider):- # base case (depth=0)- if 'depth' not in response.meta:- response.meta['depth'] = 0- if self.verbose_stats:- self.stats.inc_value('request_depth_count/0', spider=spider)-+ self._init_depth(response, spider)return (r for r in result or () if self._filter(r, response, spider))async def process_spider_output_async(self, response, result, spider):+ self._init_depth(response, spider)+ async for r in result or ():+ if self._filter(r, response, spider):+ yield r++ def _init_depth(self, response, spider):# base case (depth=0)if 'depth' not in response.meta:response.meta['depth'] = 0if self.verbose_stats:self.stats.inc_value('request_depth_count/0', spider=spider)- async for r in result or ():- if self._filter(r, response, spider):- yield r-def _filter(self, request, response, spider):if not isinstance(request, Request):return True