Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add StickyMetaParamsMiddleware (#7711)

Adnan Awan744aaae60dab
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index aa08a0a14..46a2fc91f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -156,7 +156,8 @@ Request objects
:attr:`cb_kwargs` instead, see :ref:`callback-data`. However, request
metadata may be the right choice in certain scenarios, such as to
maintain some debugging data across all follow-up requests (e.g. the
- source URL).
+ source URL). To copy some metadata keys automatically into follow-up
+ requests, consider using the :setting:`STICKY_META_KEYS` setting.
A common use of request metadata is to define request-specific
parameters for Scrapy components (extensions, middlewares, etc.). For
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 9ef4d7dcb..320b64c8e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -2186,6 +2186,7 @@ Default:
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
+ "scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware": 1000,
}
A dict containing the spider middlewares enabled by default in Scrapy, and
@@ -2231,6 +2232,69 @@ finishes.
For more info see: :ref:`topics-stats`.
+.. setting:: STICKY_META_KEYS
+
+STICKY_META_KEYS
+----------------
+
+Default: ``[]`` (empty list)
+
+The :attr:`Request.meta <scrapy.http.Request.meta>` keys to copy automatically
+from a response into the follow-up requests yielded by its callback, handled by
+:class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware`.
+
+Metadata keys already set on a follow-up request are not overwritten.
+
+For example, the following spider:
+
+.. code-block:: python
+
+ import scrapy
+
+
+ class MySpider(scrapy.Spider):
+ name = "myspider"
+
+ async def start(self):
+ start_url = "https://toscrape.com/"
+ yield scrapy.Request(start_url, meta={"start_url": start_url})
+
+ def parse(self, response):
+ for a in response.css("a"):
+ yield response.follow(
+ a,
+ meta={"start_url": response.meta["start_url"]},
+ )
+ yield {
+ "url": response.url,
+ "start_url": response.meta["start_url"],
+ }
+
+can be rewritten as follows using the :setting:`STICKY_META_KEYS` setting:
+
+.. code-block:: python
+
+ import scrapy
+
+
+ class MySpider(scrapy.Spider):
+ name = "myspider"
+ custom_settings = {
+ "STICKY_META_KEYS": ["start_url"],
+ }
+
+ async def start(self):
+ start_url = "https://toscrape.com/"
+ yield scrapy.Request(start_url, meta={"start_url": start_url})
+
+ def parse(self, response):
+ for a in response.css("a"):
+ yield response.follow(a)
+ yield {
+ "url": response.url,
+ "start_url": response.meta["start_url"],
+ }
+
.. setting:: TELNETCONSOLE_ENABLED
TELNETCONSOLE_ENABLED
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 17516462b..07bc0b658 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -462,6 +462,12 @@ StartSpiderMiddleware
.. autoclass:: scrapy.spidermiddlewares.start.StartSpiderMiddleware
+StickyMetaParamsMiddleware
+--------------------------
+
+.. autoclass:: scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware
+
+
UrlLengthMiddleware
-------------------
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 31e326016..0c0216ed2 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -213,6 +213,7 @@ __all__ = [
"STATSMAILER_RCPTS",
"STATS_CLASS",
"STATS_DUMP",
+ "STICKY_META_KEYS",
"TELNETCONSOLE_ENABLED",
"TELNETCONSOLE_HOST",
"TELNETCONSOLE_PASSWORD",
@@ -584,7 +585,8 @@ SPIDER_MIDDLEWARES_BASE = {
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
- "scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware": 1000,
+ "scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware": 999,
+ "scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware": 1000,
# Spider side
}
@@ -595,6 +597,8 @@ STATS_DUMP = True
STATSMAILER_RCPTS: list[str] = []
+STICKY_META_KEYS: list[str] = []
+
TELNETCONSOLE_ENABLED = 1
TELNETCONSOLE_HOST = "127.0.0.1"
TELNETCONSOLE_PORT = [6023, 6073]
diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py
new file mode 100644
index 000000000..72d7c5752
--- /dev/null
+++ b/scrapy/spidermiddlewares/stickymeta.py
@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from scrapy.exceptions import NotConfigured
+from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
+
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
+ from scrapy.crawler import Crawler
+ from scrapy.http import Request, Response
+
+
+class StickyMetaParamsMiddleware(BaseSpiderMiddleware):
+ """Copy a configurable list of :attr:`Request.meta <scrapy.http.Request.meta>`
+ keys from a response into the follow-up requests of its callback.
+
+ The keys to copy are read from the :setting:`STICKY_META_KEYS` setting.
+ Keys already present in a follow-up request are not overwritten.
+ """
+
+ def __init__(self, crawler: Crawler, sticky_meta_keys: list[str]):
+ super().__init__(crawler)
+ self.sticky_meta_keys: list[str] = sticky_meta_keys
+
+ @classmethod
+ def from_crawler(cls, crawler: Crawler) -> Self:
+ sticky_meta_keys = crawler.settings.getlist("STICKY_META_KEYS")
+ if not sticky_meta_keys:
+ raise NotConfigured
+ return cls(crawler, sticky_meta_keys)
+
+ def get_processed_request(
+ self, request: Request, response: Response | None
+ ) -> Request | None:
+ if response is None:
+ return request
+ for key in self.sticky_meta_keys:
+ if key in response.meta:
+ request.meta.setdefault(key, response.meta[key])
+ return request
diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py
new file mode 100644
index 000000000..5f19704f1
--- /dev/null
+++ b/tests/test_spidermiddleware_stickymeta.py
@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from scrapy import Request, Spider
+from scrapy.exceptions import NotConfigured
+from scrapy.http import Response
+from scrapy.spidermiddlewares.stickymeta import StickyMetaParamsMiddleware
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+
+TEST_URL = "http://www.example.com"
+
+
+def _make_mw(sticky_meta_keys: Any) -> StickyMetaParamsMiddleware:
+ crawler = get_crawler(Spider, {"STICKY_META_KEYS": sticky_meta_keys})
+ return build_from_crawler(StickyMetaParamsMiddleware, crawler)
+
+
+async def _run_all_paths(
+ mw: StickyMetaParamsMiddleware, response: Response, spider_output: list[Any]
+) -> list[list[Any]]:
+ """Run the spider output through every processing path of the middleware."""
+ return [
+ list(mw.process_spider_output(response, spider_output)),
+ await collect_asyncgen(
+ mw.process_spider_output_async(response, as_async_generator(spider_output))
+ ),
+ ]
+
+
+def test_not_configured() -> None:
+ crawler = get_crawler(Spider)
+ with pytest.raises(NotConfigured):
+ build_from_crawler(StickyMetaParamsMiddleware, crawler)
+
+
+def test_comma_separated_string_setting() -> None:
+ mw = _make_mw("param1,param2")
+ assert mw.sticky_meta_keys == ["param1", "param2"]
+
+
+@coroutine_test
+async def test_sticky_params() -> None:
+ mw = _make_mw(["param2"])
+ request = Request(
+ TEST_URL, meta={"param": "Will not be stickied", "param2": "Stickied!"}
+ )
+ response = Response(TEST_URL, request=request)
+ spider_output = [Request(TEST_URL), {"name": "dummy"}]
+ for processed in await _run_all_paths(mw, response, spider_output):
+ assert processed[0].meta == {"param2": "Stickied!"}
+ assert processed[1] == {"name": "dummy"}
+
+
+@coroutine_test
+async def test_sticky_param_does_not_override_manually_configured_param() -> None:
+ mw = _make_mw(["param", "param2"])
+ request = Request(TEST_URL, meta={"param": "Stickied!", "param2": "Stickied!"})
+ response = Response(TEST_URL, request=request)
+ spider_output = [Request(TEST_URL, meta={"param": "Override stickied"})]
+ for processed in await _run_all_paths(mw, response, spider_output):
+ assert processed[0].meta == {
+ "param": "Override stickied",
+ "param2": "Stickied!",
+ }
+
+
+@coroutine_test
+async def test_sticky_key_missing_from_response_meta() -> None:
+ """Configured keys absent from the response meta are silently skipped."""
+ mw = _make_mw(["param", "missing"])
+ request = Request(TEST_URL, meta={"param": "Stickied!"})
+ response = Response(TEST_URL, request=request)
+ spider_output = [Request(TEST_URL)]
+ for processed in await _run_all_paths(mw, response, spider_output):
+ assert processed[0].meta == {"param": "Stickied!"}
+
+
+@coroutine_test
+async def test_start_requests_have_no_response() -> None:
+ """Start seeds are processed with ``response=None`` and are left untouched."""
+ mw = _make_mw(["param"])
+ start_request = Request(TEST_URL, meta={"param": "value"})
+ processed = await collect_asyncgen(
+ mw.process_start(as_async_generator([start_request]))
+ )
+ assert processed[0].meta == {"param": "value"}