Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Break built-in reference cycles that keep responses alive (#8460)

Adrian596523da9120
diff --git a/pyproject.toml b/pyproject.toml
index 7c3460a82..9e61bbf32 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,7 +8,7 @@ dynamic = ["version"]
description = "A high-level Web Crawling and Web Scraping framework"
dependencies = [
"PyDispatcher>=2.0.5",
- "Twisted>=23.8.0",
+ "Twisted>=23.10.0",
"aiodns>=3.3.0",
"aioftp>=0.26.1",
"aiohttp>=3.13.3",
@@ -79,7 +79,7 @@ ptpython = ["ptpython>=3.0.23"]
robotparser = ["robotexclusionrulesparser>=1.6.2"]
s3 = ["aioboto3>=9.0.0", "boto3>=1.35.2"]
socks = ["python-socks[asyncio]>=2.6.0"]
-twisted-http2 = ["Twisted[http2]>=23.8.0"]
+twisted-http2 = ["Twisted[http2]>=23.10.0"]
uvloop = [
"uvloop>=0.17.0; platform_system != 'Windows'",
]
diff --git a/scrapy/core/_http2/protocol.py b/scrapy/core/_http2/protocol.py
index 5037938ba..bcdbfe88c 100644
--- a/scrapy/core/_http2/protocol.py
+++ b/scrapy/core/_http2/protocol.py
@@ -410,7 +410,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
elif isinstance(event, SettingsAcknowledged):
self.settings_acknowledged(event)
elif isinstance(event, UnknownFrameReceived):
- logger.warning("Unknown frame received: %s", event.frame)
+ logger.warning(f"Unknown frame received: type {event.frame.type}")
# Event handler functions starts here
def connection_terminated(self, event: ConnectionTerminated) -> None:
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 2f7921eae..9ef420784 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -8,6 +8,7 @@ import warnings
from collections import deque
from collections.abc import AsyncIterator
from inspect import iscoroutine
+from traceback import clear_frames
from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
from twisted.internet.defer import Deferred, inlineCallbacks
@@ -335,6 +336,11 @@ class Scraper:
self._handle_spider_error(spider_exc, request, result)
else:
await self.handle_spider_output_async(output, request, result)
+ finally:
+ # Twisted keeps failures in frames of their own traceback, a
+ # reference cycle that would keep alive any response that those
+ # frames refer to.
+ clear_frames(result.tb)
async def _wait_for_processing(
self, result: Response | Failure, request: Request, queue_dfd: Deferred[None]
@@ -383,12 +389,24 @@ class Scraper:
else: # result is a Failure
cast("FailureWithRequest", result).request = request
parent_id = request.id
- if not request.errback:
- result.raiseException()
- warn_on_generator_with_return_value(self.crawler.spider, request.errback)
- output = request.errback(result)
+ if request.errback:
+ warn_on_generator_with_return_value(
+ self.crawler.spider, request.errback
+ )
+ output = request.errback(result)
+ else:
+ output = result
if isinstance(output, Failure):
- output.raiseException()
+ assert output.value is not None
+ exception = output.value.with_traceback(output.tb)
+ # The traceback of the exception includes this frame, so these
+ # local variables would keep them in a reference cycle, and
+ # with them the response.
+ del result, output
+ try:
+ raise exception
+ finally:
+ del exception
# else the errback returned actual output (like a callback),
# which needs to be passed to iterate_spider_output()
if isinstance(output, Deferred):
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 7da4d8d12..515c2ef8f 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -6,6 +6,7 @@ import logging
import warnings
from abc import ABC, abstractmethod
from collections import OrderedDict, defaultdict
+from traceback import clear_frames
from typing import TYPE_CHECKING, Any, Literal, Self, TypeAlias, TypedDict, cast
from twisted.internet.defer import Deferred, DeferredList
@@ -253,6 +254,10 @@ class MediaPipeline(ABC):
) -> None:
if isinstance(result, Failure):
# minimize cached information for failure
+ exception: BaseException | None = result.value
+ while exception is not None:
+ clear_frames(exception.__traceback__)
+ exception = exception.__context__
result.cleanFailure()
result.frames.clear()
if TWISTED_FAILURE_HAS_STACK:
diff --git a/tests/test_downloader.py b/tests/test_downloader.py
index ed123efef..7bdf9e129 100644
--- a/tests/test_downloader.py
+++ b/tests/test_downloader.py
@@ -6,11 +6,13 @@ from typing import Any
import pytest
from twisted.internet.defer import Deferred
+from twisted.python.failure import Failure
from scrapy import Request, Spider
from scrapy.crawler import Crawler
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Response
+from scrapy.pipelines.files import FilesPipeline
from scrapy.utils.asyncio import call_later, sleep
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler
@@ -80,6 +82,38 @@ def _raise(response: Response) -> None:
raise ValueError(response.url)
+def _ignore(failure: Failure) -> None:
+ pass
+
+
+class _RaiseOnSpiderInput:
+ def process_spider_input(self, response: Response) -> None:
+ raise ValueError(response.url)
+
+
+class _IgnoreResponse:
+ def process_response(self, request: Request, response: Response) -> Response:
+ raise IgnoreRequest
+
+
+async def _crawl_without_gc(crawler: Crawler) -> None:
+ # The log capture of pytest keeps the tracebacks of logged errors, and with
+ # them the responses they refer to.
+ logging.disable()
+ gc.disable()
+ try:
+ crawl_deferred = crawler.crawl()
+ # A stall would otherwise hang the test.
+ timeout = call_later(10, crawl_deferred.cancel)
+ try:
+ await maybe_deferred_to_future(crawl_deferred)
+ finally:
+ timeout.cancel()
+ finally:
+ gc.enable()
+ logging.disable(logging.NOTSET)
+
+
class gt:
__hash__ = None # type: ignore[assignment]
@@ -426,16 +460,40 @@ class TestRequestBackout:
assert _backout_stats(crawler) == expected_stats
@pytest.mark.parametrize(
- ("status", "callback"),
+ ("status", "callback", "errback", "settings"),
[
- (200, lambda response: response.css("a")),
- (200, _raise),
- (404, None),
+ (200, lambda response: response.css("a"), _ignore, {}),
+ (200, _raise, _ignore, {}),
+ (404, None, _ignore, {}),
+ (404, None, None, {}),
+ (404, None, lambda failure: failure, {}),
+ (
+ 200,
+ None,
+ None,
+ {"SPIDER_MIDDLEWARES": {_RaiseOnSpiderInput: 0}},
+ ),
+ (
+ 200,
+ None,
+ None,
+ {"DOWNLOADER_MIDDLEWARES": {_IgnoreResponse: 1}},
+ ),
+ ],
+ ids=[
+ "selector",
+ "callback-exception",
+ "errback",
+ "no-errback",
+ "errback-failure",
+ "spider-middleware-exception",
+ "downloader-middleware-ignore",
],
- ids=["selector", "callback-exception", "errback"],
)
@coroutine_test
- async def test_response_max_active_size_no_cycles(self, status, callback):
+ async def test_response_max_active_size_no_cycles(
+ self, status, callback, errback, settings
+ ):
"""Scrapy frees responses through reference counting alone, so a crawl
does not stall on RESPONSE_MAX_ACTIVE_SIZE without garbage
collection."""
@@ -447,37 +505,58 @@ class TestRequestBackout:
class TestSpider(Spider):
name = "test"
custom_settings = {
- "DOWNLOADER_MIDDLEWARES": {ResponseMiddleware: 0},
+ **settings,
+ "DOWNLOADER_MIDDLEWARES": {
+ ResponseMiddleware: 0,
+ **settings.get("DOWNLOADER_MIDDLEWARES", {}),
+ },
"RESPONSE_MAX_ACTIVE_SIZE": 1,
}
async def start(self):
for n in range(3):
yield Request(
- f"https://example.com/{n}",
- callback=callback,
- errback=lambda failure: None,
+ f"https://example.com/{n}", callback=callback, errback=errback
)
+ def parse(self, response):
+ pass
+
crawler = get_crawler(TestSpider)
- # The log capture of pytest keeps the tracebacks of logged errors, and
- # with them the responses they refer to.
- logging.disable()
- gc.disable()
- try:
- crawl_deferred = crawler.crawl()
- # A stall would otherwise hang the test.
- timeout = call_later(10, crawl_deferred.cancel)
- try:
- await maybe_deferred_to_future(crawl_deferred)
- finally:
- timeout.cancel()
- finally:
- gc.enable()
- logging.disable(logging.NOTSET)
+ await _crawl_without_gc(crawler)
+
+ assert crawler.stats
+ assert crawler.stats.get_value("downloader/response_count") == 3
+
+ @coroutine_test
+ async def test_response_max_active_size_no_cycles_media(self, tmp_path):
+
+ class ResponseMiddleware:
+ def process_request(self, request):
+ status = 404 if request.url.endswith("/file") else 200
+ return Response(request.url, status=status, body=b"a")
+
+ class TestSpider(Spider):
+ name = "test"
+ custom_settings = {
+ "DOWNLOADER_MIDDLEWARES": {ResponseMiddleware: 0},
+ "FILES_STORE": str(tmp_path),
+ "ITEM_PIPELINES": {FilesPipeline: 1},
+ "RESPONSE_MAX_ACTIVE_SIZE": 1,
+ }
+
+ async def start(self):
+ for n in range(3):
+ yield Request(f"https://example.com/{n}")
+
+ def parse(self, response):
+ yield {"file_urls": [f"{response.url}/file"]}
+
+ crawler = get_crawler(TestSpider)
+ await _crawl_without_gc(crawler)
assert crawler.stats
- assert crawler.stats.get_value("response_received_count") == 3
+ assert crawler.stats.get_value("item_scraped_count") == 3
@coroutine_test
async def test_response_size_process_request(self):
diff --git a/tox.ini b/tox.ini
index e030f07ce..2cee007ea 100644
--- a/tox.ini
+++ b/tox.ini
@@ -140,7 +140,7 @@ deps =
# pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0
pytest==8.4.0
Protego==0.2.0
- Twisted==23.8.0
+ Twisted==23.10.0
aiodns==3.3.0
aioftp==0.26.1
aiohttp==3.13.3
@@ -207,7 +207,7 @@ basepython = {[min]basepython}
deps =
{[min]deps}
Pillow==9.2.0
- Twisted[http2]==23.8.0
+ Twisted[http2]==23.10.0
h2==3.2.0
python-socks[asyncio]==2.6.0
siosocks==0.2.0 # SOCKS server for tests