Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

tests: integration coverage for memusage extension (#7017)

Lions-1abbf3b95fcdf
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 01af02ba3..1444c8941 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -113,7 +113,7 @@ class MemoryUsage:
{"memusage": mem},
extra={"crawler": self.crawler},
)
- if self.notify_mails:
+ if self.notify_mails: # pragma: no cover
subj = (
f"{self.crawler.settings['BOT_NAME']} terminated: "
f"memory usage exceeded {mem}MiB at {socket.gethostname()}"
@@ -146,7 +146,7 @@ class MemoryUsage:
{"memusage": mem},
extra={"crawler": self.crawler},
)
- if self.notify_mails:
+ if self.notify_mails: # pragma: no cover
subj = (
f"{self.crawler.settings['BOT_NAME']} warning: "
f"memory usage reached {mem}MiB at {socket.gethostname()}"
@@ -155,7 +155,7 @@ class MemoryUsage:
self.crawler.stats.set_value("memusage/warning_notified", 1)
self.warned = True
- def _send_report(self, rcpts: list[str], subject: str) -> None:
+ def _send_report(self, rcpts: list[str], subject: str) -> None: # pragma: no cover
"""send notification mail with some additional useful info"""
assert self.crawler.engine
assert self.crawler.stats
diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py
new file mode 100644
index 000000000..a474725d8
--- /dev/null
+++ b/tests/test_extension_memusage.py
@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import logging
+import sys
+
+import pytest
+
+from scrapy import signals
+from scrapy.core import engine as engine_mod
+from scrapy.exceptions import NotConfigured
+from scrapy.extensions import memusage as memusage_mod
+from scrapy.extensions.memusage import MemoryUsage
+from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+from tests.utils import OneShotLoop
+from tests.utils.decorators import coroutine_test
+
+# MemoryUsage relies on the stdlib 'resource' module (not available on Windows)
+pytestmark = pytest.mark.skipif(
+ sys.platform.startswith("win"),
+ reason="MemoryUsage extension not available on Windows",
+)
+
+
+MB = 1024 * 1024
+
+
+class _LoopSpider(Spider):
+ name = "loop-data-spider"
+
+ def __init__(self, url: str, loops: int = 60, **kw):
+ super().__init__(**kw)
+ self.url = url
+ self.loops = loops
+ self.start_urls = [url]
+
+ def parse(self, response):
+ count = response.meta.get("count", 0)
+ if count + 1 < self.loops:
+ yield response.follow(
+ self.url, callback=self.parse, meta={"count": count + 1}
+ )
+
+
+def test_memusage_disabled() -> None:
+ settings = {
+ "MEMUSAGE_ENABLED": False,
+ }
+ with pytest.raises(NotConfigured):
+ MemoryUsage.from_crawler(get_crawler(settings_dict=settings))
+
+
+@coroutine_test
+async def test_memusage_limit_closes_spider_with_reason_and_error_log(
+ caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ settings = {
+ "MEMUSAGE_LIMIT_MB": 10,
+ "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01,
+ "TELNETCONSOLE_ENABLED": False,
+ "LOG_LEVEL": "INFO",
+ }
+
+ # Avoid background LoopingCall that can log after the test finishes.
+ monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop)
+ # Avoid engine start/stop races (the extension stops the engine in engine_started).
+ monkeypatch.setattr(engine_mod, "create_looping_call", OneShotLoop)
+ monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 250 * MB)
+
+ crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)
+
+ with caplog.at_level(logging.ERROR, logger="scrapy.extensions.memusage"):
+ await crawler.crawl_async(url="data:,", loops=100)
+
+ assert crawler.stats
+ assert crawler.stats.get_value("memusage/limit_reached") == 1
+ assert crawler.stats.get_value("finish_reason") == "memusage_exceeded"
+ assert any(
+ "memory usage exceeded" in r.getMessage().lower() for r in caplog.records
+ )
+
+
+@coroutine_test
+async def test_memusage_warning_logs_but_allows_normal_finish(
+ caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ settings = {
+ "MEMUSAGE_WARNING_MB": 50,
+ "MEMUSAGE_LIMIT_MB": 0, # no hard limit
+ "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01,
+ "TELNETCONSOLE_ENABLED": False,
+ "LOG_LEVEL": "INFO",
+ }
+
+ # Avoid background LoopingCall that can log after the test finishes.
+ monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop)
+ monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB)
+
+ crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)
+
+ warning_signals: list[int] = []
+
+ def on_warning_reached() -> None:
+ warning_signals.append(1)
+
+ crawler.signals.connect(on_warning_reached, signal=signals.memusage_warning_reached)
+
+ with caplog.at_level(logging.WARNING, logger="scrapy.extensions.memusage"):
+ await crawler.crawl_async(url="data:,", loops=60)
+
+ assert warning_signals == [1]
+ assert crawler.stats
+ assert crawler.stats.get_value("memusage/warning_reached") == 1
+ assert crawler.stats.get_value("finish_reason") == "finished"
+ assert any("memory usage reached" in r.getMessage().lower() for r in caplog.records)
diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py
index 5b7848d54..73c0e3776 100644
--- a/tests/utils/__init__.py
+++ b/tests/utils/__init__.py
@@ -1,5 +1,6 @@
import asyncio
import os
+from collections.abc import Callable
from pathlib import Path
from twisted.internet.defer import Deferred
@@ -31,3 +32,19 @@ def get_script_run_env() -> dict[str, str]:
env = os.environ.copy()
env["PYTHONPATH"] = pythonpath
return env
+
+
+class OneShotLoop:
+ """Test stub for create_looping_call: run once immediately, no background task."""
+
+ def __init__(self, func: Callable[[], None]):
+ self.func = func
+ self.running = False
+
+ def start(self, _interval: float, now: bool = True) -> None:
+ self.running = True
+ if now:
+ self.func()
+
+ def stop(self) -> None:
+ self.running = False