Scrapy, a fast high-level web crawling & scraping framework for Python.
Add a middleware to detect response.meta copies (#7588)
Adrian1157b3e235c9
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex e56be0a22..7468f1b64 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -238,6 +238,9 @@ Request objectsAlso mind that the :meth:`copy` and :meth:`replace` request methods:doc:`shallow-copy <library/copy>` request metadata.+ .. seealso:: :class:`~scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware`+ for a built-in middleware that warns about this issue at run time.+.. autoattribute:: dont_filter.. autoattribute:: Request.attributesdiff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rstindex 870adbc8c..aa14f6801 100644--- a/docs/topics/spider-middleware.rst+++ b/docs/topics/spider-middleware.rst@@ -316,6 +316,35 @@ Default: ``False``Pass all responses, regardless of its status code.+MetaCopyDetectionMiddleware+---------------------------++.. module:: scrapy.spidermiddlewares.metacopy+ :synopsis: Meta Copy Detection Spider Middleware++.. class:: MetaCopyDetectionMiddleware++ Warns when a spider yields a request that contains internal meta keys which+ should not be copied from :attr:`response.meta <scrapy.http.Response.meta>`+ into new requests. See :attr:`~scrapy.http.Request.meta` to learn why.++ Only 1 warning is emitted per crawl.++ MetaCopyDetectionMiddleware settings+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~++ .. setting:: META_COPY_WARN_SKIP_KEYS++ META_COPY_WARN_SKIP_KEYS+ ^^^^^^^^^^^^^^^^^^^^^^^^++ Default: ``[]``++ A list of internal meta key names to exclude from the internal-keys check.+ Use this when you intentionally copy one of the monitored keys and want to+ suppress the resulting warning without disabling the middleware entirely.++RefererMiddleware-----------------diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex d283336ab..993f2436d 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -554,6 +554,7 @@ SPIDER_MIDDLEWARES_BASE = {"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,+ "scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware": 1000,# Spider side}diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.pyindex 6c62dccb8..3b14d2b0a 100644--- a/scrapy/spidermiddlewares/base.py+++ b/scrapy/spidermiddlewares/base.py@@ -48,7 +48,10 @@ class BaseSpiderMiddleware:@_warn_spider_argdef process_spider_output(- self, response: Response, result: Iterable[Any], spider: Spider | None = None+ self,+ response: Response | None,+ result: Iterable[Any],+ spider: Spider | None = None,) -> Iterable[Any]:for o in result:if (o := self._get_processed(o, response)) is not None:@@ -57,7 +60,7 @@ class BaseSpiderMiddleware:@_warn_spider_argasync def process_spider_output_async(self,- response: Response,+ response: Response | None,result: AsyncIterator[Any],spider: Spider | None = None,) -> AsyncIterator[Any]:diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.pyindex 9c387530c..054804119 100644--- a/scrapy/spidermiddlewares/depth.py+++ b/scrapy/spidermiddlewares/depth.py@@ -55,19 +55,24 @@ class DepthMiddleware(BaseSpiderMiddleware):@_warn_spider_argdef process_spider_output(- self, response: Response, result: Iterable[Any], spider: Spider | None = None+ self,+ response: Response | None,+ result: Iterable[Any],+ spider: Spider | None = None,) -> Iterable[Any]:- self._init_depth(response)+ if response is not None:+ self._init_depth(response)yield from super().process_spider_output(response, result)@_warn_spider_argasync def process_spider_output_async(self,- response: Response,+ response: Response | None,result: AsyncIterator[Any],spider: Spider | None = None,) -> AsyncIterator[Any]:- self._init_depth(response)+ if response is not None:+ self._init_depth(response)async for o in super().process_spider_output_async(response, result):yield odiff --git a/scrapy/spidermiddlewares/metacopy.py b/scrapy/spidermiddlewares/metacopy.pynew file mode 100644index 000000000..aa5a120c6--- /dev/null+++ b/scrapy/spidermiddlewares/metacopy.py@@ -0,0 +1,65 @@+from __future__ import annotations++import logging+from typing import TYPE_CHECKING++from scrapy.spidermiddlewares.base import BaseSpiderMiddleware++if TYPE_CHECKING:+ from scrapy.crawler import Crawler+ from scrapy.http import Request, Response+++logger = logging.getLogger(__name__)+++class MetaCopyDetectionMiddleware(BaseSpiderMiddleware):+ """Warn when a spider yields a request with internal meta keys that should+ not be copied from response.meta, or when two requests share the same meta+ dict object.++ Each warning is emitted at most once per crawl.+ """++ _INTERNAL_KEYS: frozenset[str] = frozenset(+ {+ "_auth_proxy",+ "_dont_cache",+ "_scheme_proxy",+ "download_latency",+ "redirect_reasons",+ "redirect_times",+ "redirect_ttl",+ "redirect_urls",+ "retry_times",+ }+ )++ def __init__(self, crawler: Crawler) -> None:+ super().__init__(crawler)+ skip = frozenset(crawler.settings.getlist("META_COPY_WARN_SKIP_KEYS", []))+ self._keys: frozenset[str] = self._INTERNAL_KEYS - skip+ self._warned: bool = False++ def get_processed_request(+ self, request: Request, response: Response | None+ ) -> Request | None:+ if response is None:+ return request++ if not self._warned:+ found = self._keys & request.meta.keys()+ if found:+ spider_name = type(self.crawler.spider).__name__+ logger.warning(+ f"{spider_name} yielded a request containing internal "+ f"meta keys that were likely copied from response.meta "+ f"and should not be forwarded to new requests: "+ f"{sorted(found)}. See the MetaCopyDetectionMiddleware "+ f"documentation for more information. Source response: "+ f"{response}, target request: {request}",+ extra={"spider": self.crawler.spider},+ )+ self._warned = True++ return requestdiff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.pyindex 307ed2440..32a2ea8f2 100644--- a/tests/test_spidermiddleware_depth.py+++ b/tests/test_spidermiddleware_depth.py@@ -7,8 +7,10 @@ import pytestfrom scrapy.http import Request, Responsefrom scrapy.spidermiddlewares.depth import DepthMiddlewarefrom scrapy.spiders import Spider+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgenfrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawler+from tests.utils.decorators import coroutine_testif TYPE_CHECKING:from collections.abc import Generator@@ -58,6 +60,31 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> Noassert rdm == 1+def test_process_spider_output_no_response(+ mw: DepthMiddleware, stats: StatsCollector+) -> None:+ result = [Request("https://example.com")]++ out = list(mw.process_spider_output(None, result))+ assert out == result+ assert "depth" not in out[0].meta+ assert stats.get_value("request_depth_count/0") is None+++@coroutine_test+async def test_process_spider_output_async_no_response(+ mw: DepthMiddleware, stats: StatsCollector+) -> None:+ result = [Request("https://example.com")]++ out = await collect_asyncgen(+ mw.process_spider_output_async(None, as_async_generator(result))+ )+ assert out == result+ assert "depth" not in out[0].meta+ assert stats.get_value("request_depth_count/0") is None++def test_priority_and_non_verbose_stats() -> None:crawler = get_crawler(Spider,diff --git a/tests/test_spidermiddleware_metacopy.py b/tests/test_spidermiddleware_metacopy.pynew file mode 100644index 000000000..bfa27ac68--- /dev/null+++ b/tests/test_spidermiddleware_metacopy.py@@ -0,0 +1,121 @@+from __future__ import annotations++from logging import WARNING+from typing import TYPE_CHECKING++import pytest++from scrapy.http import Request, Response+from scrapy.spidermiddlewares.metacopy import MetaCopyDetectionMiddleware+from scrapy.spiders import Spider+from scrapy.utils.test import get_crawler++if TYPE_CHECKING:+ from scrapy.crawler import Crawler+++def make_response(url: str = "https://example.com") -> Response:+ response = Response(url)+ response.request = Request(url)+ return response+++def crawler() -> Crawler:+ return get_crawler(Spider)+++def mw(crawler: Crawler) -> MetaCopyDetectionMiddleware:+ return MetaCopyDetectionMiddleware.from_crawler(crawler)+++def process(+ mw: MetaCopyDetectionMiddleware,+ requests: list[Request],+ response: Response | None = None,+) -> list[Request]:+ if response is None:+ response = make_response()+ return list(mw.process_spider_output(response, requests))+++class TestInternalKeysCheck:+ def test_no_warning_for_clean_request(+ self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+ ) -> None:+ req = Request("https://example.com/1", meta={"my_key": "value"})+ with caplog.at_level(WARNING):+ process(mw, [req])+ assert not caplog.records++ def test_warns_on_internal_key(+ self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+ ) -> None:+ req = Request("https://example.com/1", meta={"retry_times": 1})+ with caplog.at_level(WARNING):+ process(mw, [req])+ assert len(caplog.records) == 1+ assert "retry_times" in caplog.text+ assert "https://example.com/1" in caplog.text++ def test_warns_once_across_multiple_requests(+ self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+ ) -> None:+ reqs = [+ Request("https://example.com/1", meta={"retry_times": 1}),+ Request("https://example.com/2", meta={"retry_times": 2}),+ ]+ with caplog.at_level(WARNING):+ process(mw, reqs)+ assert len(caplog.records) == 1++ def test_reports_all_found_keys(+ self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+ ) -> None:+ req = Request(+ "https://example.com/1",+ meta={"retry_times": 1, "redirect_times": 2},+ )+ with caplog.at_level(WARNING):+ process(mw, [req])+ assert "retry_times" in caplog.text+ assert "redirect_times" in caplog.text++ def test_includes_source_response_in_message(+ self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+ ) -> None:+ response = make_response("https://source.example.com")+ req = Request("https://example.com/1", meta={"retry_times": 1})+ with caplog.at_level(WARNING):+ process(mw, [req], response=response)+ assert "https://source.example.com" in caplog.text++ def test_skip_keys_setting(self, caplog: pytest.LogCaptureFixture) -> None:+ with caplog.at_level(WARNING):+ crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})+ mw = MetaCopyDetectionMiddleware.from_crawler(crawler)+ req = Request("https://example.com/1", meta={"retry_times": 1})+ process(mw, [req])+ assert not caplog.records++ def test_skip_keys_setting_partial(self, caplog: pytest.LogCaptureFixture) -> None:+ with caplog.at_level(WARNING):+ crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})+ mw = MetaCopyDetectionMiddleware.from_crawler(crawler)+ req = Request(+ "https://example.com/1",+ meta={"retry_times": 1, "redirect_times": 2},+ )+ process(mw, [req])+ assert len(caplog.records) == 1+ assert "retry_times" not in caplog.text+ assert "redirect_times" in caplog.text++ def test_no_warning_for_start_requests(+ self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+ ) -> None:+ req = Request("https://example.com/1", meta={"retry_times": 1})+ with caplog.at_level(WARNING):+ list(mw.process_spider_output(None, [req]))+ assert not caplog.records