Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add a depth_reset request metadata key (#7913)

Adrian5270f3cf99d1
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index db85906c1..1c9ee0c77 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -228,25 +228,7 @@ DepthMiddleware
.. module:: scrapy.spidermiddlewares.depth
:synopsis: Depth Spider Middleware
-.. class:: DepthMiddleware
-
- DepthMiddleware is used for tracking the depth of each Request inside the
- site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever
- there is no value previously set (usually just the first Request) and
- incrementing it by 1 otherwise.
-
- It can be used to limit the maximum depth to scrape, control Request
- priority based on their depth, and things like that.
-
- The :class:`DepthMiddleware` can be configured through the following
- settings (see the settings documentation for more info):
-
- * :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to
- crawl for any site. If zero, no limit will be imposed.
- * :setting:`DEPTH_STATS_VERBOSE` - Whether to collect the number of
- requests for each depth.
- * :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on
- their depth.
+.. autoclass:: DepthMiddleware
HttpErrorMiddleware
-------------------
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 0131b62e7..1781543e3 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -28,6 +28,27 @@ logger = logging.getLogger(__name__)
class DepthMiddleware(BaseSpiderMiddleware):
+ """Track the depth of each request within the site being scraped, setting
+ ``request.meta["depth"]`` to 0 when there is no value previously set
+ (usually just the first request) and incrementing it by 1 otherwise.
+
+ It can be used to limit the maximum depth to scrape, control request
+ priority based on their depth, and things like that, through the
+ :setting:`DEPTH_LIMIT`, :setting:`DEPTH_STATS_VERBOSE` and
+ :setting:`DEPTH_PRIORITY` settings.
+
+ .. reqmeta:: depth_reset
+
+ depth_reset
+ -----------
+
+ .. versionadded:: VERSION
+
+ :attr:`~scrapy.Request.meta` key that, set to ``True``, gives a request
+ depth 0 instead of the depth of its source response plus 1, e.g. to keep
+ :setting:`DEPTH_LIMIT` from applying across a domain change.
+ """
+
crawler: Crawler
def __init__( # pylint: disable=super-init-not-called
@@ -87,10 +108,13 @@ class DepthMiddleware(BaseSpiderMiddleware):
def get_processed_request(
self, request: Request, response: Response | None
) -> Request | None:
+ # Consumed here so that it cannot reach response.meta and, from there,
+ # spread to further requests through a meta copy.
+ depth_reset = request.meta.pop("depth_reset", False)
if response is None:
# start requests
return request
- depth = response.meta["depth"] + 1
+ depth = 0 if depth_reset else response.meta["depth"] + 1
request.meta["depth"] = depth
if self.prio:
request.priority -= depth * self.prio
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index 2aa76195e..bdacd7287 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -60,6 +60,19 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> No
assert rdm == 1
+def test_depth_reset(mw: DepthMiddleware, stats: StatsCollector) -> None:
+ resp = Response("https://example.com")
+ resp.request = Request("https://example.com", meta={"depth": 5})
+ result = [Request("https://example.com", meta={"depth_reset": True})]
+
+ out = list(mw.process_spider_output(resp, result))
+
+ assert out == result
+ assert out[0].meta["depth"] == 0
+ assert "depth_reset" not in out[0].meta
+ assert stats.get_value("request_depth_count/0") == 1
+
+
def test_process_spider_output_no_response(
mw: DepthMiddleware, stats: StatsCollector
) -> None: