Scrapy, a fast high-level web crawling & scraping framework for Python.
Break built-in reference cycles that keep responses alive (#8460)
Adrian596523da9120
diff --git a/pyproject.toml b/pyproject.tomlindex 7c3460a82..9e61bbf32 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -8,7 +8,7 @@ dynamic = ["version"]description = "A high-level Web Crawling and Web Scraping framework"dependencies = ["PyDispatcher>=2.0.5",- "Twisted>=23.8.0",+ "Twisted>=23.10.0","aiodns>=3.3.0","aioftp>=0.26.1","aiohttp>=3.13.3",@@ -79,7 +79,7 @@ ptpython = ["ptpython>=3.0.23"]robotparser = ["robotexclusionrulesparser>=1.6.2"]s3 = ["aioboto3>=9.0.0", "boto3>=1.35.2"]socks = ["python-socks[asyncio]>=2.6.0"]-twisted-http2 = ["Twisted[http2]>=23.8.0"]+twisted-http2 = ["Twisted[http2]>=23.10.0"]uvloop = ["uvloop>=0.17.0; platform_system != 'Windows'",]diff --git a/scrapy/core/_http2/protocol.py b/scrapy/core/_http2/protocol.pyindex 5037938ba..bcdbfe88c 100644--- a/scrapy/core/_http2/protocol.py+++ b/scrapy/core/_http2/protocol.py@@ -410,7 +410,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):elif isinstance(event, SettingsAcknowledged):self.settings_acknowledged(event)elif isinstance(event, UnknownFrameReceived):- logger.warning("Unknown frame received: %s", event.frame)+ logger.warning(f"Unknown frame received: type {event.frame.type}")# Event handler functions starts heredef connection_terminated(self, event: ConnectionTerminated) -> None:diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 2f7921eae..9ef420784 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -8,6 +8,7 @@ import warningsfrom collections import dequefrom collections.abc import AsyncIteratorfrom inspect import iscoroutine+from traceback import clear_framesfrom typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, castfrom twisted.internet.defer import Deferred, inlineCallbacks@@ -335,6 +336,11 @@ class Scraper:self._handle_spider_error(spider_exc, request, result)else:await self.handle_spider_output_async(output, request, result)+ finally:+ # Twisted keeps failures in frames of their own traceback, a+ # reference cycle that would keep alive any response that those+ # frames refer to.+ clear_frames(result.tb)async def _wait_for_processing(self, result: Response | Failure, request: Request, queue_dfd: Deferred[None]@@ -383,12 +389,24 @@ class Scraper:else: # result is a Failurecast("FailureWithRequest", result).request = requestparent_id = request.id- if not request.errback:- result.raiseException()- warn_on_generator_with_return_value(self.crawler.spider, request.errback)- output = request.errback(result)+ if request.errback:+ warn_on_generator_with_return_value(+ self.crawler.spider, request.errback+ )+ output = request.errback(result)+ else:+ output = resultif isinstance(output, Failure):- output.raiseException()+ assert output.value is not None+ exception = output.value.with_traceback(output.tb)+ # The traceback of the exception includes this frame, so these+ # local variables would keep them in a reference cycle, and+ # with them the response.+ del result, output+ try:+ raise exception+ finally:+ del exception# else the errback returned actual output (like a callback),# which needs to be passed to iterate_spider_output()if isinstance(output, Deferred):diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 7da4d8d12..515c2ef8f 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -6,6 +6,7 @@ import loggingimport warningsfrom abc import ABC, abstractmethodfrom collections import OrderedDict, defaultdict+from traceback import clear_framesfrom typing import TYPE_CHECKING, Any, Literal, Self, TypeAlias, TypedDict, castfrom twisted.internet.defer import Deferred, DeferredList@@ -253,6 +254,10 @@ class MediaPipeline(ABC):) -> None:if isinstance(result, Failure):# minimize cached information for failure+ exception: BaseException | None = result.value+ while exception is not None:+ clear_frames(exception.__traceback__)+ exception = exception.__context__result.cleanFailure()result.frames.clear()if TWISTED_FAILURE_HAS_STACK:diff --git a/tests/test_downloader.py b/tests/test_downloader.pyindex ed123efef..7bdf9e129 100644--- a/tests/test_downloader.py+++ b/tests/test_downloader.py@@ -6,11 +6,13 @@ from typing import Anyimport pytestfrom twisted.internet.defer import Deferred+from twisted.python.failure import Failurefrom scrapy import Request, Spiderfrom scrapy.crawler import Crawler-from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, Response+from scrapy.pipelines.files import FilesPipelinefrom scrapy.utils.asyncio import call_later, sleepfrom scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.test import get_crawler@@ -80,6 +82,38 @@ def _raise(response: Response) -> None:raise ValueError(response.url)+def _ignore(failure: Failure) -> None:+ pass+++class _RaiseOnSpiderInput:+ def process_spider_input(self, response: Response) -> None:+ raise ValueError(response.url)+++class _IgnoreResponse:+ def process_response(self, request: Request, response: Response) -> Response:+ raise IgnoreRequest+++async def _crawl_without_gc(crawler: Crawler) -> None:+ # The log capture of pytest keeps the tracebacks of logged errors, and with+ # them the responses they refer to.+ logging.disable()+ gc.disable()+ try:+ crawl_deferred = crawler.crawl()+ # A stall would otherwise hang the test.+ timeout = call_later(10, crawl_deferred.cancel)+ try:+ await maybe_deferred_to_future(crawl_deferred)+ finally:+ timeout.cancel()+ finally:+ gc.enable()+ logging.disable(logging.NOTSET)++class gt:__hash__ = None # type: ignore[assignment]@@ -426,16 +460,40 @@ class TestRequestBackout:assert _backout_stats(crawler) == expected_stats@pytest.mark.parametrize(- ("status", "callback"),+ ("status", "callback", "errback", "settings"),[- (200, lambda response: response.css("a")),- (200, _raise),- (404, None),+ (200, lambda response: response.css("a"), _ignore, {}),+ (200, _raise, _ignore, {}),+ (404, None, _ignore, {}),+ (404, None, None, {}),+ (404, None, lambda failure: failure, {}),+ (+ 200,+ None,+ None,+ {"SPIDER_MIDDLEWARES": {_RaiseOnSpiderInput: 0}},+ ),+ (+ 200,+ None,+ None,+ {"DOWNLOADER_MIDDLEWARES": {_IgnoreResponse: 1}},+ ),+ ],+ ids=[+ "selector",+ "callback-exception",+ "errback",+ "no-errback",+ "errback-failure",+ "spider-middleware-exception",+ "downloader-middleware-ignore",],- ids=["selector", "callback-exception", "errback"],)@coroutine_test- async def test_response_max_active_size_no_cycles(self, status, callback):+ async def test_response_max_active_size_no_cycles(+ self, status, callback, errback, settings+ ):"""Scrapy frees responses through reference counting alone, so a crawldoes not stall on RESPONSE_MAX_ACTIVE_SIZE without garbagecollection."""@@ -447,37 +505,58 @@ class TestRequestBackout:class TestSpider(Spider):name = "test"custom_settings = {- "DOWNLOADER_MIDDLEWARES": {ResponseMiddleware: 0},+ **settings,+ "DOWNLOADER_MIDDLEWARES": {+ ResponseMiddleware: 0,+ **settings.get("DOWNLOADER_MIDDLEWARES", {}),+ },"RESPONSE_MAX_ACTIVE_SIZE": 1,}async def start(self):for n in range(3):yield Request(- f"https://example.com/{n}",- callback=callback,- errback=lambda failure: None,+ f"https://example.com/{n}", callback=callback, errback=errback)+ def parse(self, response):+ pass+crawler = get_crawler(TestSpider)- # The log capture of pytest keeps the tracebacks of logged errors, and- # with them the responses they refer to.- logging.disable()- gc.disable()- try:- crawl_deferred = crawler.crawl()- # A stall would otherwise hang the test.- timeout = call_later(10, crawl_deferred.cancel)- try:- await maybe_deferred_to_future(crawl_deferred)- finally:- timeout.cancel()- finally:- gc.enable()- logging.disable(logging.NOTSET)+ await _crawl_without_gc(crawler)++ assert crawler.stats+ assert crawler.stats.get_value("downloader/response_count") == 3++ @coroutine_test+ async def test_response_max_active_size_no_cycles_media(self, tmp_path):++ class ResponseMiddleware:+ def process_request(self, request):+ status = 404 if request.url.endswith("/file") else 200+ return Response(request.url, status=status, body=b"a")++ class TestSpider(Spider):+ name = "test"+ custom_settings = {+ "DOWNLOADER_MIDDLEWARES": {ResponseMiddleware: 0},+ "FILES_STORE": str(tmp_path),+ "ITEM_PIPELINES": {FilesPipeline: 1},+ "RESPONSE_MAX_ACTIVE_SIZE": 1,+ }++ async def start(self):+ for n in range(3):+ yield Request(f"https://example.com/{n}")++ def parse(self, response):+ yield {"file_urls": [f"{response.url}/file"]}++ crawler = get_crawler(TestSpider)+ await _crawl_without_gc(crawler)assert crawler.stats- assert crawler.stats.get_value("response_received_count") == 3+ assert crawler.stats.get_value("item_scraped_count") == 3@coroutine_testasync def test_response_size_process_request(self):diff --git a/tox.ini b/tox.iniindex e030f07ce..2cee007ea 100644--- a/tox.ini+++ b/tox.ini@@ -140,7 +140,7 @@ deps =# pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0pytest==8.4.0Protego==0.2.0- Twisted==23.8.0+ Twisted==23.10.0aiodns==3.3.0aioftp==0.26.1aiohttp==3.13.3@@ -207,7 +207,7 @@ basepython = {[min]basepython}deps ={[min]deps}Pillow==9.2.0- Twisted[http2]==23.8.0+ Twisted[http2]==23.10.0h2==3.2.0python-socks[asyncio]==2.6.0siosocks==0.2.0 # SOCKS server for tests