Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Let refcounting free responses after selector use or callback errors (#8245)

Adrianb4279e9e496d
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 4d82c9265..d910ec598 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -155,7 +155,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
# offering it to those middlewares again.
with suppress(AttributeError):
exception._spidermw_unhandled = True # type: ignore[attr-defined]
- raise exception
+ # The traceback of the exception includes this frame, so this local
+ # variable would keep them in a reference cycle, and with them the
+ # response.
+ try:
+ raise exception
+ finally:
+ del exception
def _process_spider_output(
self,
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 411239800..1a0b5c317 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -8,6 +8,7 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
import json
+import weakref
from contextlib import suppress
from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urljoin
@@ -170,7 +171,11 @@ class TextResponse(Response):
# circular import
from scrapy.selector import Selector # noqa: PLC0415
- self._cached_selector = Selector(self)
+ selector = Selector(self)
+ # The weak reference lets the response be freed as soon as nothing
+ # else refers to it.
+ selector._response = weakref.ref(self)
+ self._cached_selector = selector
return self._cached_selector
def jmespath(self, query: str, **kwargs: Any) -> SelectorList:
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 69344814b..d351ca84e 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import weakref
from typing import Any, Literal
from parsel import Selector as _ParselSelector
@@ -71,7 +72,7 @@ class Selector(_ParselSelector, object_ref):
supported.
"""
- __slots__ = ["response"]
+ __slots__ = ["_response"]
selectorlist_cls = SelectorList
def __init__(
@@ -117,9 +118,15 @@ class Selector(_ParselSelector, object_ref):
text = response.text
kwargs.setdefault("base_url", get_base_url(response))
- self.response = response
+ self._response: TextResponse | weakref.ref[TextResponse] | None = response
if root is not _NOT_SET:
kwargs["root"] = root
super().__init__(text=text, type=type, **kwargs)
+
+ @property
+ def response(self) -> TextResponse | None:
+ if isinstance(self._response, weakref.ref):
+ return self._response()
+ return self._response
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index eb9283fe1..d3bf4d7c0 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -1,6 +1,9 @@
from __future__ import annotations
import codecs
+import gc
+import platform
+import weakref
from typing import cast
from unittest import mock
@@ -277,6 +280,22 @@ class TestTextResponse(TestResponseBase):
assert response.selector.css("title::text").getall() == ["Some page"]
assert response.selector.re("Some (.*)</title>") == ["page"]
+ @pytest.mark.skipif(
+ platform.python_implementation() != "CPython",
+ reason="Relies on reference counting",
+ )
+ def test_selector_response_refcounting(self):
+ response = self.response_class("http://www.example.com", body=b"<a>b</a>")
+ selector = response.selector
+ ref = weakref.ref(response)
+ gc.disable()
+ try:
+ del response
+ assert ref() is None
+ finally:
+ gc.enable()
+ assert selector.response is None
+
def test_selector_shortcuts(self):
body = b"<html><head><title>Some page</title><body></body></html>"
response = self.response_class("http://www.example.com", body=body)
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 482ca8407..802edfd67 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -130,6 +130,10 @@ class TestSelector:
f"{x.__class__.__name__} does not use __slots__"
)
+ def test_response(self):
+ response = TextResponse(url="http://example.com", body=b"")
+ assert Selector(response).response is response
+
def test_selector_bad_args(self):
with pytest.raises(ValueError, match="received both response and text"):
Selector(TextResponse(url="http://example.com", body=b""), text="")
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 62037234c..a2906613f 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,5 +1,8 @@
from __future__ import annotations
+import gc
+import platform
+import weakref
from collections.abc import AsyncIterator, Iterable
from inspect import isasyncgen
from typing import TYPE_CHECKING, Any, cast
@@ -104,6 +107,30 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
with pytest.raises(ZeroDivisionError):
await collect_asyncgen(it)
+ @pytest.mark.skipif(
+ platform.python_implementation() != "CPython",
+ reason="Relies on reference counting",
+ )
+ @coroutine_test
+ async def test_response_refcounting(self):
+ class RaiseExceptionProcessSpiderOutputMiddleware:
+ async def process_spider_output(self, response, result):
+ 1 / 0
+ yield
+
+ self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
+ it = await self._scrape_response()
+ ref = weakref.ref(self.response)
+ del self.response
+ gc.disable()
+ try:
+ with pytest.raises(ZeroDivisionError):
+ await collect_asyncgen(it)
+ del it
+ assert ref() is None
+ finally:
+ gc.enable()
+
class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
"""Helpers for testing sync, async and mixed middlewares.