Scrapy, a fast high-level web crawling & scraping framework for Python.
Add StickyMetaParamsMiddleware (#7711)
Adnan Awan744aaae60dab
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex aa08a0a14..46a2fc91f 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -156,7 +156,8 @@ Request objects:attr:`cb_kwargs` instead, see :ref:`callback-data`. However, requestmetadata may be the right choice in certain scenarios, such as tomaintain some debugging data across all follow-up requests (e.g. the- source URL).+ source URL). To copy some metadata keys automatically into follow-up+ requests, consider using the :setting:`STICKY_META_KEYS` setting.A common use of request metadata is to define request-specificparameters for Scrapy components (extensions, middlewares, etc.). Fordiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 9ef4d7dcb..320b64c8e 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -2186,6 +2186,7 @@ Default:"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,+ "scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware": 1000,}A dict containing the spider middlewares enabled by default in Scrapy, and@@ -2231,6 +2232,69 @@ finishes.For more info see: :ref:`topics-stats`.+.. setting:: STICKY_META_KEYS++STICKY_META_KEYS+----------------++Default: ``[]`` (empty list)++The :attr:`Request.meta <scrapy.http.Request.meta>` keys to copy automatically+from a response into the follow-up requests yielded by its callback, handled by+:class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware`.++Metadata keys already set on a follow-up request are not overwritten.++For example, the following spider:++.. code-block:: python++ import scrapy+++ class MySpider(scrapy.Spider):+ name = "myspider"++ async def start(self):+ start_url = "https://toscrape.com/"+ yield scrapy.Request(start_url, meta={"start_url": start_url})++ def parse(self, response):+ for a in response.css("a"):+ yield response.follow(+ a,+ meta={"start_url": response.meta["start_url"]},+ )+ yield {+ "url": response.url,+ "start_url": response.meta["start_url"],+ }++can be rewritten as follows using the :setting:`STICKY_META_KEYS` setting:++.. code-block:: python++ import scrapy+++ class MySpider(scrapy.Spider):+ name = "myspider"+ custom_settings = {+ "STICKY_META_KEYS": ["start_url"],+ }++ async def start(self):+ start_url = "https://toscrape.com/"+ yield scrapy.Request(start_url, meta={"start_url": start_url})++ def parse(self, response):+ for a in response.css("a"):+ yield response.follow(a)+ yield {+ "url": response.url,+ "start_url": response.meta["start_url"],+ }+.. setting:: TELNETCONSOLE_ENABLEDTELNETCONSOLE_ENABLEDdiff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rstindex 17516462b..07bc0b658 100644--- a/docs/topics/spider-middleware.rst+++ b/docs/topics/spider-middleware.rst@@ -462,6 +462,12 @@ StartSpiderMiddleware.. autoclass:: scrapy.spidermiddlewares.start.StartSpiderMiddleware+StickyMetaParamsMiddleware+--------------------------++.. autoclass:: scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware++UrlLengthMiddleware-------------------diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 31e326016..0c0216ed2 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -213,6 +213,7 @@ __all__ = ["STATSMAILER_RCPTS","STATS_CLASS","STATS_DUMP",+ "STICKY_META_KEYS","TELNETCONSOLE_ENABLED","TELNETCONSOLE_HOST","TELNETCONSOLE_PASSWORD",@@ -584,7 +585,8 @@ SPIDER_MIDDLEWARES_BASE = {"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,- "scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware": 1000,+ "scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware": 999,+ "scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware": 1000,# Spider side}@@ -595,6 +597,8 @@ STATS_DUMP = TrueSTATSMAILER_RCPTS: list[str] = []+STICKY_META_KEYS: list[str] = []+TELNETCONSOLE_ENABLED = 1TELNETCONSOLE_HOST = "127.0.0.1"TELNETCONSOLE_PORT = [6023, 6073]diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.pynew file mode 100644index 000000000..72d7c5752--- /dev/null+++ b/scrapy/spidermiddlewares/stickymeta.py@@ -0,0 +1,43 @@+from __future__ import annotations++from typing import TYPE_CHECKING++from scrapy.exceptions import NotConfigured+from scrapy.spidermiddlewares.base import BaseSpiderMiddleware++if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self++ from scrapy.crawler import Crawler+ from scrapy.http import Request, Response+++class StickyMetaParamsMiddleware(BaseSpiderMiddleware):+ """Copy a configurable list of :attr:`Request.meta <scrapy.http.Request.meta>`+ keys from a response into the follow-up requests of its callback.++ The keys to copy are read from the :setting:`STICKY_META_KEYS` setting.+ Keys already present in a follow-up request are not overwritten.+ """++ def __init__(self, crawler: Crawler, sticky_meta_keys: list[str]):+ super().__init__(crawler)+ self.sticky_meta_keys: list[str] = sticky_meta_keys++ @classmethod+ def from_crawler(cls, crawler: Crawler) -> Self:+ sticky_meta_keys = crawler.settings.getlist("STICKY_META_KEYS")+ if not sticky_meta_keys:+ raise NotConfigured+ return cls(crawler, sticky_meta_keys)++ def get_processed_request(+ self, request: Request, response: Response | None+ ) -> Request | None:+ if response is None:+ return request+ for key in self.sticky_meta_keys:+ if key in response.meta:+ request.meta.setdefault(key, response.meta[key])+ return requestdiff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.pynew file mode 100644index 000000000..5f19704f1--- /dev/null+++ b/tests/test_spidermiddleware_stickymeta.py@@ -0,0 +1,92 @@+from __future__ import annotations++from typing import Any++import pytest++from scrapy import Request, Spider+from scrapy.exceptions import NotConfigured+from scrapy.http import Response+from scrapy.spidermiddlewares.stickymeta import StickyMetaParamsMiddleware+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen+from scrapy.utils.misc import build_from_crawler+from scrapy.utils.test import get_crawler+from tests.utils.decorators import coroutine_test++TEST_URL = "http://www.example.com"+++def _make_mw(sticky_meta_keys: Any) -> StickyMetaParamsMiddleware:+ crawler = get_crawler(Spider, {"STICKY_META_KEYS": sticky_meta_keys})+ return build_from_crawler(StickyMetaParamsMiddleware, crawler)+++async def _run_all_paths(+ mw: StickyMetaParamsMiddleware, response: Response, spider_output: list[Any]+) -> list[list[Any]]:+ """Run the spider output through every processing path of the middleware."""+ return [+ list(mw.process_spider_output(response, spider_output)),+ await collect_asyncgen(+ mw.process_spider_output_async(response, as_async_generator(spider_output))+ ),+ ]+++def test_not_configured() -> None:+ crawler = get_crawler(Spider)+ with pytest.raises(NotConfigured):+ build_from_crawler(StickyMetaParamsMiddleware, crawler)+++def test_comma_separated_string_setting() -> None:+ mw = _make_mw("param1,param2")+ assert mw.sticky_meta_keys == ["param1", "param2"]+++@coroutine_test+async def test_sticky_params() -> None:+ mw = _make_mw(["param2"])+ request = Request(+ TEST_URL, meta={"param": "Will not be stickied", "param2": "Stickied!"}+ )+ response = Response(TEST_URL, request=request)+ spider_output = [Request(TEST_URL), {"name": "dummy"}]+ for processed in await _run_all_paths(mw, response, spider_output):+ assert processed[0].meta == {"param2": "Stickied!"}+ assert processed[1] == {"name": "dummy"}+++@coroutine_test+async def test_sticky_param_does_not_override_manually_configured_param() -> None:+ mw = _make_mw(["param", "param2"])+ request = Request(TEST_URL, meta={"param": "Stickied!", "param2": "Stickied!"})+ response = Response(TEST_URL, request=request)+ spider_output = [Request(TEST_URL, meta={"param": "Override stickied"})]+ for processed in await _run_all_paths(mw, response, spider_output):+ assert processed[0].meta == {+ "param": "Override stickied",+ "param2": "Stickied!",+ }+++@coroutine_test+async def test_sticky_key_missing_from_response_meta() -> None:+ """Configured keys absent from the response meta are silently skipped."""+ mw = _make_mw(["param", "missing"])+ request = Request(TEST_URL, meta={"param": "Stickied!"})+ response = Response(TEST_URL, request=request)+ spider_output = [Request(TEST_URL)]+ for processed in await _run_all_paths(mw, response, spider_output):+ assert processed[0].meta == {"param": "Stickied!"}+++@coroutine_test+async def test_start_requests_have_no_response() -> None:+ """Start seeds are processed with ``response=None`` and are left untouched."""+ mw = _make_mw(["param"])+ start_request = Request(TEST_URL, meta={"param": "value"})+ processed = await collect_asyncgen(+ mw.process_start(as_async_generator([start_request]))+ )+ assert processed[0].meta == {"param": "value"}