Scrapy, a fast high-level web crawling & scraping framework for Python.
Let refcounting free responses after selector use or callback errors (#8245)
Adrianb4279e9e496d
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 4d82c9265..d910ec598 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -155,7 +155,13 @@ class SpiderMiddlewareManager(MiddlewareManager):# offering it to those middlewares again.with suppress(AttributeError):exception._spidermw_unhandled = True # type: ignore[attr-defined]- raise exception+ # The traceback of the exception includes this frame, so this local+ # variable would keep them in a reference cycle, and with them the+ # response.+ try:+ raise exception+ finally:+ del exceptiondef _process_spider_output(self,diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 411239800..1a0b5c317 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -8,6 +8,7 @@ See documentation in docs/topics/request-response.rstfrom __future__ import annotationsimport json+import weakreffrom contextlib import suppressfrom typing import TYPE_CHECKING, Any, castfrom urllib.parse import urljoin@@ -170,7 +171,11 @@ class TextResponse(Response):# circular importfrom scrapy.selector import Selector # noqa: PLC0415- self._cached_selector = Selector(self)+ selector = Selector(self)+ # The weak reference lets the response be freed as soon as nothing+ # else refers to it.+ selector._response = weakref.ref(self)+ self._cached_selector = selectorreturn self._cached_selectordef jmespath(self, query: str, **kwargs: Any) -> SelectorList:diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.pyindex 69344814b..d351ca84e 100644--- a/scrapy/selector/unified.py+++ b/scrapy/selector/unified.py@@ -1,5 +1,6 @@from __future__ import annotations+import weakreffrom typing import Any, Literalfrom parsel import Selector as _ParselSelector@@ -71,7 +72,7 @@ class Selector(_ParselSelector, object_ref):supported."""- __slots__ = ["response"]+ __slots__ = ["_response"]selectorlist_cls = SelectorListdef __init__(@@ -117,9 +118,15 @@ class Selector(_ParselSelector, object_ref):text = response.textkwargs.setdefault("base_url", get_base_url(response))- self.response = response+ self._response: TextResponse | weakref.ref[TextResponse] | None = responseif root is not _NOT_SET:kwargs["root"] = rootsuper().__init__(text=text, type=type, **kwargs)++ @property+ def response(self) -> TextResponse | None:+ if isinstance(self._response, weakref.ref):+ return self._response()+ return self._responsediff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.pyindex eb9283fe1..d3bf4d7c0 100644--- a/tests/test_http_response_text.py+++ b/tests/test_http_response_text.py@@ -1,6 +1,9 @@from __future__ import annotationsimport codecs+import gc+import platform+import weakreffrom typing import castfrom unittest import mock@@ -277,6 +280,22 @@ class TestTextResponse(TestResponseBase):assert response.selector.css("title::text").getall() == ["Some page"]assert response.selector.re("Some (.*)</title>") == ["page"]+ @pytest.mark.skipif(+ platform.python_implementation() != "CPython",+ reason="Relies on reference counting",+ )+ def test_selector_response_refcounting(self):+ response = self.response_class("http://www.example.com", body=b"<a>b</a>")+ selector = response.selector+ ref = weakref.ref(response)+ gc.disable()+ try:+ del response+ assert ref() is None+ finally:+ gc.enable()+ assert selector.response is None+def test_selector_shortcuts(self):body = b"<html><head><title>Some page</title><body></body></html>"response = self.response_class("http://www.example.com", body=body)diff --git a/tests/test_selector.py b/tests/test_selector.pyindex 482ca8407..802edfd67 100644--- a/tests/test_selector.py+++ b/tests/test_selector.py@@ -130,6 +130,10 @@ class TestSelector:f"{x.__class__.__name__} does not use __slots__")+ def test_response(self):+ response = TextResponse(url="http://example.com", body=b"")+ assert Selector(response).response is response+def test_selector_bad_args(self):with pytest.raises(ValueError, match="received both response and text"):Selector(TextResponse(url="http://example.com", body=b""), text="")diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex 62037234c..a2906613f 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -1,5 +1,8 @@from __future__ import annotations+import gc+import platform+import weakreffrom collections.abc import AsyncIterator, Iterablefrom inspect import isasyncgenfrom typing import TYPE_CHECKING, Any, cast@@ -104,6 +107,30 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):with pytest.raises(ZeroDivisionError):await collect_asyncgen(it)+ @pytest.mark.skipif(+ platform.python_implementation() != "CPython",+ reason="Relies on reference counting",+ )+ @coroutine_test+ async def test_response_refcounting(self):+ class RaiseExceptionProcessSpiderOutputMiddleware:+ async def process_spider_output(self, response, result):+ 1 / 0+ yield++ self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())+ it = await self._scrape_response()+ ref = weakref.ref(self.response)+ del self.response+ gc.disable()+ try:+ with pytest.raises(ZeroDivisionError):+ await collect_asyncgen(it)+ del it+ assert ref() is None+ finally:+ gc.enable()+class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):"""Helpers for testing sync, async and mixed middlewares.