Scrapy, a fast high-level web crawling & scraping framework for Python.
Add a depth_reset request metadata key (#7913)
Adrian5270f3cf99d1
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rstindex db85906c1..1c9ee0c77 100644--- a/docs/topics/spider-middleware.rst+++ b/docs/topics/spider-middleware.rst@@ -228,25 +228,7 @@ DepthMiddleware.. module:: scrapy.spidermiddlewares.depth:synopsis: Depth Spider Middleware-.. class:: DepthMiddleware-- DepthMiddleware is used for tracking the depth of each Request inside the- site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever- there is no value previously set (usually just the first Request) and- incrementing it by 1 otherwise.-- It can be used to limit the maximum depth to scrape, control Request- priority based on their depth, and things like that.-- The :class:`DepthMiddleware` can be configured through the following- settings (see the settings documentation for more info):-- * :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to- crawl for any site. If zero, no limit will be imposed.- * :setting:`DEPTH_STATS_VERBOSE` - Whether to collect the number of- requests for each depth.- * :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on- their depth.+.. autoclass:: DepthMiddlewareHttpErrorMiddleware-------------------diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.pyindex 0131b62e7..1781543e3 100644--- a/scrapy/spidermiddlewares/depth.py+++ b/scrapy/spidermiddlewares/depth.py@@ -28,6 +28,27 @@ logger = logging.getLogger(__name__)class DepthMiddleware(BaseSpiderMiddleware):+ """Track the depth of each request within the site being scraped, setting+ ``request.meta["depth"]`` to 0 when there is no value previously set+ (usually just the first request) and incrementing it by 1 otherwise.++ It can be used to limit the maximum depth to scrape, control request+ priority based on their depth, and things like that, through the+ :setting:`DEPTH_LIMIT`, :setting:`DEPTH_STATS_VERBOSE` and+ :setting:`DEPTH_PRIORITY` settings.++ .. reqmeta:: depth_reset++ depth_reset+ -----------++ .. versionadded:: VERSION++ :attr:`~scrapy.Request.meta` key that, set to ``True``, gives a request+ depth 0 instead of the depth of its source response plus 1, e.g. to keep+ :setting:`DEPTH_LIMIT` from applying across a domain change.+ """+crawler: Crawlerdef __init__( # pylint: disable=super-init-not-called@@ -87,10 +108,13 @@ class DepthMiddleware(BaseSpiderMiddleware):def get_processed_request(self, request: Request, response: Response | None) -> Request | None:+ # Consumed here so that it cannot reach response.meta and, from there,+ # spread to further requests through a meta copy.+ depth_reset = request.meta.pop("depth_reset", False)if response is None:# start requestsreturn request- depth = response.meta["depth"] + 1+ depth = 0 if depth_reset else response.meta["depth"] + 1request.meta["depth"] = depthif self.prio:request.priority -= depth * self.priodiff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.pyindex 2aa76195e..bdacd7287 100644--- a/tests/test_spidermiddleware_depth.py+++ b/tests/test_spidermiddleware_depth.py@@ -60,6 +60,19 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> Noassert rdm == 1+def test_depth_reset(mw: DepthMiddleware, stats: StatsCollector) -> None:+ resp = Response("https://example.com")+ resp.request = Request("https://example.com", meta={"depth": 5})+ result = [Request("https://example.com", meta={"depth_reset": True})]++ out = list(mw.process_spider_output(resp, result))++ assert out == result+ assert out[0].meta["depth"] == 0+ assert "depth_reset" not in out[0].meta+ assert stats.get_value("request_depth_count/0") == 1++def test_process_spider_output_no_response(mw: DepthMiddleware, stats: StatsCollector) -> None: