Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support MemUsage on Windows (#8062)

Adrianc2c890856cf5
diff --git a/conftest.py b/conftest.py
index 4c0eca896..1f6b7c7b3 100644
--- a/conftest.py
+++ b/conftest.py
@@ -209,6 +209,7 @@ def pytest_runtest_setup(item):
"uvloop",
"botocore",
"boto3",
+ "psutil",
]
for module in optional_deps:
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 605c7fe87..2efdf9f10 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -125,6 +125,9 @@ The following extras are available:
- :ref:`Images pipeline <images-pipeline>`
* - ``ipython``
- :ref:`IPython shell <shell-config>`
+ * - ``memusage``
+ - :ref:`Memory usage extension <topics-extensions-ref-memusage>` on
+ Windows
* - ``ptpython``
- :ref:`ptpython shell <shell-config>`
* - ``robotparser``
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 5cb85d863..77209b462 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -148,7 +148,8 @@ Memory usage extension
.. class:: scrapy.extensions.memusage.MemoryUsage
-.. note:: This extension does not work in Windows.
+.. note:: On Windows, this extension requires the ``memusage``
+ :ref:`extra <extras>`.
Monitors the memory used by the Scrapy process that runs the spider and:
diff --git a/pyproject.toml b/pyproject.toml
index cd7dbac7f..952c741c2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -76,6 +76,7 @@ gcs = ["google-cloud-storage>=1.30.0"]
httpx = ["httpx2[http2,socks]>=2.0.0"]
images = ["Pillow>=9.2.0"]
ipython = ["ipython>=8.15.0"]
+memusage = ["psutil>=5.9.0"]
ptpython = ["ptpython>=3.0.23"]
robotparser = ["robotexclusionrulesparser>=1.6.2"]
s3 = ["boto3>=1.20.0"]
@@ -146,6 +147,7 @@ module = [
"brotlicffi",
"google.*",
"ipdb",
+ "psutil",
"pydispatch.*",
"pyftpdlib.*",
"pyperclip",
@@ -293,6 +295,7 @@ markers = [
"requires_uvloop: marks tests as only enabled when uvloop is known to be working",
"requires_botocore: marks tests that need botocore (but not boto3)",
"requires_boto3: marks tests that need botocore and boto3",
+ "requires_psutil: marks tests that need psutil",
"requires_mitmproxy: marks tests that need a mitmdump executable",
"requires_internet: marks tests that need real Internet access",
"walltime: marks benchmarks that need CodSpeed's walltime mode to be meaningful",
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 59126893d..f371c4a78 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -22,6 +22,8 @@ from scrapy.utils.engine import get_engine_status
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
+ from types import ModuleType
+
from twisted.internet.task import LoopingCall
from scrapy.crawler import Crawler
@@ -35,11 +37,23 @@ class MemoryUsage:
def __init__(self, crawler: Crawler):
if not crawler.settings.getbool("MEMUSAGE_ENABLED"):
raise NotConfigured
+ self.resource: ModuleType | None
try:
# stdlib's resource module is only available on unix platforms.
self.resource = import_module("resource")
- except ImportError as exc:
- raise NotConfigured from exc
+ except ImportError:
+ self.resource = None
+ try:
+ import psutil # noqa: PLC0415
+ except ImportError as exc:
+ raise NotConfigured(
+ "The MemoryUsage extension requires either the stdlib "
+ "'resource' module (not available on Windows) or the "
+ "'psutil' package. Install it with the 'memusage' extra "
+ "(pip install scrapy[memusage]), or disable this "
+ "extension by setting MEMUSAGE_ENABLED to False."
+ ) from exc
+ self._process = psutil.Process()
self.crawler: Crawler = crawler
self._stats: StatsCollector = crawler.stats
@@ -70,11 +84,15 @@ class MemoryUsage:
return cls(crawler)
def get_virtual_size(self) -> int:
- size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
- if sys.platform != "darwin":
- # on macOS ru_maxrss is in bytes, on Linux it is in KB
- size *= 1024
- return size
+ if self.resource is not None:
+ size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
+ if sys.platform != "darwin":
+ # on macOS ru_maxrss is in bytes, on Linux it is in KB
+ size *= 1024
+ return size
+ # The psutil fallback is only used where 'resource' is unavailable,
+ # i.e. on Windows, where psutil tracks the peak working set size.
+ return int(self._process.memory_info().peak_wset)
def engine_started(self) -> None:
self._stats.set_value("memusage/startup", self.get_virtual_size())
diff --git a/tests/AsyncCrawlerProcess/reactorless_datauri.py b/tests/AsyncCrawlerProcess/reactorless_datauri.py
index 5270b7b83..a10a6ee4f 100644
--- a/tests/AsyncCrawlerProcess/reactorless_datauri.py
+++ b/tests/AsyncCrawlerProcess/reactorless_datauri.py
@@ -12,7 +12,9 @@ class DataSpider(Spider):
return {"data": response.text}
-process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+process = AsyncCrawlerProcess(
+ settings={"TWISTED_REACTOR_ENABLED": False, "MEMUSAGE_ENABLED": False}
+)
process.crawl(DataSpider)
process.start()
diff --git a/tests/AsyncCrawlerProcess/reactorless_simple.py b/tests/AsyncCrawlerProcess/reactorless_simple.py
index 33b4e8cb2..b4a63de0a 100644
--- a/tests/AsyncCrawlerProcess/reactorless_simple.py
+++ b/tests/AsyncCrawlerProcess/reactorless_simple.py
@@ -12,7 +12,9 @@ class NoRequestsSpider(scrapy.Spider):
yield
-process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+process = AsyncCrawlerProcess(
+ settings={"TWISTED_REACTOR_ENABLED": False, "MEMUSAGE_ENABLED": False}
+)
process.crawl(NoRequestsSpider)
process.start()
diff --git a/tests/AsyncCrawlerRunner/reactorless_datauri.py b/tests/AsyncCrawlerRunner/reactorless_datauri.py
index d964f9c1c..9131ad76b 100644
--- a/tests/AsyncCrawlerRunner/reactorless_datauri.py
+++ b/tests/AsyncCrawlerRunner/reactorless_datauri.py
@@ -17,7 +17,9 @@ class DataSpider(Spider):
async def main() -> None:
configure_logging()
- runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
+ runner = AsyncCrawlerRunner(
+ settings={"TWISTED_REACTOR_ENABLED": False, "MEMUSAGE_ENABLED": False}
+ )
await runner.crawl(DataSpider)
diff --git a/tests/AsyncCrawlerRunner/reactorless_simple.py b/tests/AsyncCrawlerRunner/reactorless_simple.py
index e91c7c89b..8e0aefada 100644
--- a/tests/AsyncCrawlerRunner/reactorless_simple.py
+++ b/tests/AsyncCrawlerRunner/reactorless_simple.py
@@ -17,7 +17,9 @@ class NoRequestsSpider(Spider):
async def main() -> None:
configure_logging()
- runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
+ runner = AsyncCrawlerRunner(
+ settings={"TWISTED_REACTOR_ENABLED": False, "MEMUSAGE_ENABLED": False}
+ )
await runner.crawl(NoRequestsSpider)
diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py
index e8065f35b..3b0698334 100644
--- a/tests/test_extension_memusage.py
+++ b/tests/test_extension_memusage.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import logging
import sys
+from types import SimpleNamespace
from typing import TYPE_CHECKING, Any
from unittest.mock import AsyncMock, MagicMock
@@ -64,6 +65,34 @@ def test_memusage_disabled() -> None:
build_from_crawler(MemoryUsage, get_crawler(settings_dict=settings))
+def test_memusage_missing_resource_and_psutil(monkeypatch: pytest.MonkeyPatch) -> None:
+ def raise_import_error(name: str) -> None:
+ raise ImportError(name)
+
+ monkeypatch.setattr(memusage_mod, "import_module", raise_import_error)
+ monkeypatch.setitem(sys.modules, "psutil", None)
+ with pytest.raises(NotConfigured, match=r"resource.*psutil"):
+ build_from_crawler(
+ MemoryUsage, get_crawler(settings_dict={"MEMUSAGE_ENABLED": True})
+ )
+
+
+def test_memusage_falls_back_to_psutil(monkeypatch: pytest.MonkeyPatch) -> None:
+ def raise_import_error(name: str) -> None:
+ raise ImportError(name)
+
+ monkeypatch.setattr(memusage_mod, "import_module", raise_import_error)
+ ext = build_from_crawler(
+ MemoryUsage, get_crawler(settings_dict={"MEMUSAGE_ENABLED": True})
+ )
+ assert ext.resource is None
+ monkeypatch.setattr(
+ ext._process, "memory_info", lambda: SimpleNamespace(peak_wset=456)
+ )
+ assert ext.get_virtual_size() == 456
+
+
def test_memusage_limit_stops_crawler_without_spider(
monkeypatch: pytest.MonkeyPatch,
) -> None:
diff --git a/tests/test_spidermiddleware_metacopy.py b/tests/test_spidermiddleware_metacopy.py
index dac0d6b70..bd66f3e34 100644
--- a/tests/test_spidermiddleware_metacopy.py
+++ b/tests/test_spidermiddleware_metacopy.py
@@ -96,24 +96,30 @@ def test_includes_source_response_in_message(
assert "https://source.example.com" in caplog.text
-def test_skip_keys_setting(caplog: pytest.LogCaptureFixture) -> None:
+def skip_keys_mw() -> MetaCopyDetectionMiddleware:
+ crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
+ return build_from_crawler(MetaCopyDetectionMiddleware, crawler)
+
+
+def test_skip_keys_setting(
+ skip_keys_mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+) -> None:
+ req = Request("https://example.com/1", meta={"retry_times": 1})
with caplog.at_level(WARNING):
- crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
- mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)
- req = Request("https://example.com/1", meta={"retry_times": 1})
- process(mw, [req])
+ process(skip_keys_mw, [req])
assert not caplog.records
-def test_skip_keys_setting_partial(caplog: pytest.LogCaptureFixture) -> None:
+def test_skip_keys_setting_partial(
+ skip_keys_mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+) -> None:
+ req = Request(
+ "https://example.com/1",
+ meta={"retry_times": 1, "redirect_times": 2},
+ )
with caplog.at_level(WARNING):
- crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
- mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)
- req = Request(
- "https://example.com/1",
- meta={"retry_times": 1, "redirect_times": 2},
- )
- process(mw, [req])
+ process(skip_keys_mw, [req])
assert len(caplog.records) == 1
assert "retry_times" not in caplog.text
assert "redirect_times" in caplog.text
diff --git a/tox.ini b/tox.ini
index fb16942a3..934c8fcd9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -189,6 +189,7 @@ deps =
google-cloud-storage
httpx2[http2,socks]
ipython
+ psutil
ptpython # optional for shell wrapper tests
robotexclusionrulesparser
uvloop; platform_system != "Windows" and implementation_name != "pypy"
@@ -205,6 +206,7 @@ deps =
google-cloud-storage==1.30.0
httpx2[http2,socks]==2.0.0
ipython==8.15.0
+ psutil==5.9.0
ptpython==3.0.23
robotexclusionrulesparser==1.6.2
uvloop==0.17.0; platform_system != "Windows" and implementation_name != "pypy"