Scrapy, a fast high-level web crawling & scraping framework for Python.
Bound the media pipeline result cache with MEDIA_CACHE_SIZE (#8193)
Adrian7ed269009cd7
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex a843e6f07..190f6bd84 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -586,6 +586,25 @@ To handle media redirections, set this setting to ``True``:MEDIA_ALLOW_REDIRECTS = True+Limiting the result cache+-------------------------++.. setting:: MEDIA_CACHE_SIZE++Media pipelines remember the outcome of every media request they handle, so+that a file referenced by several items is downloaded once. By default every+outcome is kept for the whole crawl, which takes about 600 bytes per distinct+media URL.++.. versionadded:: VERSION++To cap that memory, set :setting:`MEDIA_CACHE_SIZE` to the number of most+recently used outcomes to keep, or to ``0`` to keep none:++.. code-block:: python++ MEDIA_CACHE_SIZE = 10000+.. _topics-media-pipeline-override:Extending the Media Pipelinesdiff --git a/docs/topics/optimize.rst b/docs/topics/optimize.rstindex 1dbb5949f..e82d2f258 100644--- a/docs/topics/optimize.rst+++ b/docs/topics/optimize.rst@@ -295,6 +295,9 @@ Lowering memory usage- Set :setting:`JOBDIR` to offload all scheduled requests to disk.+- Set :setting:`MEDIA_CACHE_SIZE` if you use :ref:`media pipelines+ <topics-media-pipeline>`.+- Be on the lookout for :ref:`memory leaks <topics-leaks>`.diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 41b2e17c9..58cc21246 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -5,7 +5,7 @@ import functoolsimport loggingimport warningsfrom abc import ABC, abstractmethod-from collections import defaultdict+from collections import OrderedDict, defaultdictfrom typing import TYPE_CHECKING, Any, Literal, TypeAlias, TypedDict, castfrom twisted.internet.defer import Deferred, DeferredList@@ -80,7 +80,7 @@ class MediaPipeline(ABC):def __init__(self, spider: Spider):self.spider: Spider = spiderself.downloading: set[bytes] = set()- self.downloaded: dict[bytes, FileInfo | Failure] = {}+ self.downloaded: OrderedDict[bytes, FileInfo | Failure] = OrderedDict()self.waiting: defaultdict[bytes, list[Deferred[FileInfo]]] = defaultdict(list)@@ -111,6 +111,7 @@ class MediaPipeline(ABC):resolve("MEDIA_ALLOW_REDIRECTS"), False)self._handle_statuses(self.allow_redirects)+ self._cache_size: int = settings.getint(resolve("MEDIA_CACHE_SIZE"))def _handle_statuses(self, allow_redirects: bool) -> None:self.handle_httpstatus_list = None@@ -177,6 +178,7 @@ class MediaPipeline(ABC):# Return cached result if request was already seenif fp in info.downloaded:await _process_pending_io()+ info.downloaded.move_to_end(fp)cached_result = info.downloaded[fp]if isinstance(cached_result, Failure):if eb:@@ -276,6 +278,8 @@ class MediaPipeline(ABC):info.downloading.remove(fp)info.downloaded[fp] = result # cache result+ if 0 <= self._cache_size < len(info.downloaded):+ info.downloaded.popitem(last=False)for wad in info.waiting.pop(fp):if isinstance(result, Failure):wad.errback(result)diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 58cea3609..5995e143f 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -161,6 +161,7 @@ __all__ = ["MAIL_SSL","MAIL_TLS","MAIL_USER",+ "MEDIA_CACHE_SIZE","MEMDEBUG_ENABLED","MEMUSAGE_CHECK_INTERVAL_SECONDS","MEMUSAGE_ENABLED",@@ -497,6 +498,8 @@ MAIL_PASS = NoneMAIL_SSL = FalseMAIL_TLS = False+MEDIA_CACHE_SIZE = -1+MEMDEBUG_ENABLED = False # enable memory debuggingMEMUSAGE_ENABLED = Truediff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex fd377fb7f..09d470cc5 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -455,6 +455,32 @@ class TestMediaPipeline(TestBaseMediaPipeline):new_item = await self.pipe.process_item(item)assert new_item["results"] == [(True, {}), (True, {})]+ @pytest.mark.parametrize(+ ("size", "prefixed", "cached_urls"),+ [+ (2, False, ["http://url1", "http://url3"]),+ (2, True, ["http://url1", "http://url3"]),+ (0, False, []),+ (-1, False, ["http://url2", "http://url1", "http://url3"]),+ ],+ )+ @coroutine_test+ async def test_cache_size(self, size: int, prefixed: bool, cached_urls: list[str]):+ key = "MEDIA_CACHE_SIZE"+ if prefixed:+ key = f"{self.pipeline_class.__name__.upper()}_{key}"+ crawler = get_crawler(DefaultSpider, {key: size})+ crawler.spider = crawler._create_spider()+ crawler.engine = MagicMock(download_async=mocked_download_func)+ pipe = build_from_crawler(self.pipeline_class, crawler)+ pipe.open_spider()+ for url in ["http://url1", "http://url2", "http://url1", "http://url3"]:+ req = Request(url, meta={"response": Response(url)})+ await pipe.process_item({"requests": req})+ assert list(pipe.spiderinfo.downloaded) == [+ self.fingerprint(Request(url)) for url in cached_urls+ ]+@coroutine_testasync def test_use_media_to_download_result(self):req = Request("http://url", meta={"result": "ITSME"})