Scrapy, a fast high-level web crawling & scraping framework for Python.
Support MemUsage on Windows (#8062)
Adrianc2c890856cf5
diff --git a/conftest.py b/conftest.pyindex 4c0eca896..1f6b7c7b3 100644--- a/conftest.py+++ b/conftest.py@@ -209,6 +209,7 @@ def pytest_runtest_setup(item):"uvloop","botocore","boto3",+ "psutil",]for module in optional_deps:diff --git a/docs/intro/install.rst b/docs/intro/install.rstindex 605c7fe87..2efdf9f10 100644--- a/docs/intro/install.rst+++ b/docs/intro/install.rst@@ -125,6 +125,9 @@ The following extras are available:- :ref:`Images pipeline <images-pipeline>`* - ``ipython``- :ref:`IPython shell <shell-config>`+ * - ``memusage``+ - :ref:`Memory usage extension <topics-extensions-ref-memusage>` on+ Windows* - ``ptpython``- :ref:`ptpython shell <shell-config>`* - ``robotparser``diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rstindex 5cb85d863..77209b462 100644--- a/docs/topics/extensions.rst+++ b/docs/topics/extensions.rst@@ -148,7 +148,8 @@ Memory usage extension.. class:: scrapy.extensions.memusage.MemoryUsage-.. note:: This extension does not work in Windows.+.. note:: On Windows, this extension requires the ``memusage``+ :ref:`extra <extras>`.Monitors the memory used by the Scrapy process that runs the spider and:diff --git a/pyproject.toml b/pyproject.tomlindex cd7dbac7f..952c741c2 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -76,6 +76,7 @@ gcs = ["google-cloud-storage>=1.30.0"]httpx = ["httpx2[http2,socks]>=2.0.0"]images = ["Pillow>=9.2.0"]ipython = ["ipython>=8.15.0"]+memusage = ["psutil>=5.9.0"]ptpython = ["ptpython>=3.0.23"]robotparser = ["robotexclusionrulesparser>=1.6.2"]s3 = ["boto3>=1.20.0"]@@ -146,6 +147,7 @@ module = ["brotlicffi","google.*","ipdb",+ "psutil","pydispatch.*","pyftpdlib.*","pyperclip",@@ -293,6 +295,7 @@ markers = ["requires_uvloop: marks tests as only enabled when uvloop is known to be working","requires_botocore: marks tests that need botocore (but not boto3)","requires_boto3: marks tests that need botocore and boto3",+ "requires_psutil: marks tests that need psutil","requires_mitmproxy: marks tests that need a mitmdump executable","requires_internet: marks tests that need real Internet access","walltime: marks benchmarks that need CodSpeed's walltime mode to be meaningful",diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.pyindex 59126893d..f371c4a78 100644--- a/scrapy/extensions/memusage.py+++ b/scrapy/extensions/memusage.py@@ -22,6 +22,8 @@ from scrapy.utils.engine import get_engine_statusfrom scrapy.utils.misc import build_from_crawlerif TYPE_CHECKING:+ from types import ModuleType+from twisted.internet.task import LoopingCallfrom scrapy.crawler import Crawler@@ -35,11 +37,23 @@ class MemoryUsage:def __init__(self, crawler: Crawler):if not crawler.settings.getbool("MEMUSAGE_ENABLED"):raise NotConfigured+ self.resource: ModuleType | Nonetry:# stdlib's resource module is only available on unix platforms.self.resource = import_module("resource")- except ImportError as exc:- raise NotConfigured from exc+ except ImportError:+ self.resource = None+ try:+ import psutil # noqa: PLC0415+ except ImportError as exc:+ raise NotConfigured(+ "The MemoryUsage extension requires either the stdlib "+ "'resource' module (not available on Windows) or the "+ "'psutil' package. Install it with the 'memusage' extra "+ "(pip install scrapy[memusage]), or disable this "+ "extension by setting MEMUSAGE_ENABLED to False."+ ) from exc+ self._process = psutil.Process()self.crawler: Crawler = crawlerself._stats: StatsCollector = crawler.stats@@ -70,11 +84,15 @@ class MemoryUsage:return cls(crawler)def get_virtual_size(self) -> int:- size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss- if sys.platform != "darwin":- # on macOS ru_maxrss is in bytes, on Linux it is in KB- size *= 1024- return size+ if self.resource is not None:+ size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss+ if sys.platform != "darwin":+ # on macOS ru_maxrss is in bytes, on Linux it is in KB+ size *= 1024+ return size+ # The psutil fallback is only used where 'resource' is unavailable,+ # i.e. on Windows, where psutil tracks the peak working set size.+ return int(self._process.memory_info().peak_wset)def engine_started(self) -> None:self._stats.set_value("memusage/startup", self.get_virtual_size())diff --git a/tests/AsyncCrawlerProcess/reactorless_datauri.py b/tests/AsyncCrawlerProcess/reactorless_datauri.pyindex 5270b7b83..a10a6ee4f 100644--- a/tests/AsyncCrawlerProcess/reactorless_datauri.py+++ b/tests/AsyncCrawlerProcess/reactorless_datauri.py@@ -12,7 +12,9 @@ class DataSpider(Spider):return {"data": response.text}-process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})+process = AsyncCrawlerProcess(+ settings={"TWISTED_REACTOR_ENABLED": False, "MEMUSAGE_ENABLED": False}+)process.crawl(DataSpider)process.start()diff --git a/tests/AsyncCrawlerProcess/reactorless_simple.py b/tests/AsyncCrawlerProcess/reactorless_simple.pyindex 33b4e8cb2..b4a63de0a 100644--- a/tests/AsyncCrawlerProcess/reactorless_simple.py+++ b/tests/AsyncCrawlerProcess/reactorless_simple.py@@ -12,7 +12,9 @@ class NoRequestsSpider(scrapy.Spider):yield-process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})+process = AsyncCrawlerProcess(+ settings={"TWISTED_REACTOR_ENABLED": False, "MEMUSAGE_ENABLED": False}+)process.crawl(NoRequestsSpider)process.start()diff --git a/tests/AsyncCrawlerRunner/reactorless_datauri.py b/tests/AsyncCrawlerRunner/reactorless_datauri.pyindex d964f9c1c..9131ad76b 100644--- a/tests/AsyncCrawlerRunner/reactorless_datauri.py+++ b/tests/AsyncCrawlerRunner/reactorless_datauri.py@@ -17,7 +17,9 @@ class DataSpider(Spider):async def main() -> None:configure_logging()- runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})+ runner = AsyncCrawlerRunner(+ settings={"TWISTED_REACTOR_ENABLED": False, "MEMUSAGE_ENABLED": False}+ )await runner.crawl(DataSpider)diff --git a/tests/AsyncCrawlerRunner/reactorless_simple.py b/tests/AsyncCrawlerRunner/reactorless_simple.pyindex e91c7c89b..8e0aefada 100644--- a/tests/AsyncCrawlerRunner/reactorless_simple.py+++ b/tests/AsyncCrawlerRunner/reactorless_simple.py@@ -17,7 +17,9 @@ class NoRequestsSpider(Spider):async def main() -> None:configure_logging()- runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})+ runner = AsyncCrawlerRunner(+ settings={"TWISTED_REACTOR_ENABLED": False, "MEMUSAGE_ENABLED": False}+ )await runner.crawl(NoRequestsSpider)diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.pyindex e8065f35b..3b0698334 100644--- a/tests/test_extension_memusage.py+++ b/tests/test_extension_memusage.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport loggingimport sys+from types import SimpleNamespacefrom typing import TYPE_CHECKING, Anyfrom unittest.mock import AsyncMock, MagicMock@@ -64,6 +65,34 @@ def test_memusage_disabled() -> None:build_from_crawler(MemoryUsage, get_crawler(settings_dict=settings))+def test_memusage_missing_resource_and_psutil(monkeypatch: pytest.MonkeyPatch) -> None:+ def raise_import_error(name: str) -> None:+ raise ImportError(name)++ monkeypatch.setattr(memusage_mod, "import_module", raise_import_error)+ monkeypatch.setitem(sys.modules, "psutil", None)+ with pytest.raises(NotConfigured, match=r"resource.*psutil"):+ build_from_crawler(+ MemoryUsage, get_crawler(settings_dict={"MEMUSAGE_ENABLED": True})+ )++[email protected]_psutil+def test_memusage_falls_back_to_psutil(monkeypatch: pytest.MonkeyPatch) -> None:+ def raise_import_error(name: str) -> None:+ raise ImportError(name)++ monkeypatch.setattr(memusage_mod, "import_module", raise_import_error)+ ext = build_from_crawler(+ MemoryUsage, get_crawler(settings_dict={"MEMUSAGE_ENABLED": True})+ )+ assert ext.resource is None+ monkeypatch.setattr(+ ext._process, "memory_info", lambda: SimpleNamespace(peak_wset=456)+ )+ assert ext.get_virtual_size() == 456++def test_memusage_limit_stops_crawler_without_spider(monkeypatch: pytest.MonkeyPatch,) -> None:diff --git a/tests/test_spidermiddleware_metacopy.py b/tests/test_spidermiddleware_metacopy.pyindex dac0d6b70..bd66f3e34 100644--- a/tests/test_spidermiddleware_metacopy.py+++ b/tests/test_spidermiddleware_metacopy.py@@ -96,24 +96,30 @@ def test_includes_source_response_in_message(assert "https://source.example.com" in caplog.text-def test_skip_keys_setting(caplog: pytest.LogCaptureFixture) -> None:+def skip_keys_mw() -> MetaCopyDetectionMiddleware:+ crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})+ return build_from_crawler(MetaCopyDetectionMiddleware, crawler)+++def test_skip_keys_setting(+ skip_keys_mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+) -> None:+ req = Request("https://example.com/1", meta={"retry_times": 1})with caplog.at_level(WARNING):- crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})- mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)- req = Request("https://example.com/1", meta={"retry_times": 1})- process(mw, [req])+ process(skip_keys_mw, [req])assert not caplog.records-def test_skip_keys_setting_partial(caplog: pytest.LogCaptureFixture) -> None:+def test_skip_keys_setting_partial(+ skip_keys_mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+) -> None:+ req = Request(+ "https://example.com/1",+ meta={"retry_times": 1, "redirect_times": 2},+ )with caplog.at_level(WARNING):- crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})- mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)- req = Request(- "https://example.com/1",- meta={"retry_times": 1, "redirect_times": 2},- )- process(mw, [req])+ process(skip_keys_mw, [req])assert len(caplog.records) == 1assert "retry_times" not in caplog.textassert "redirect_times" in caplog.textdiff --git a/tox.ini b/tox.iniindex fb16942a3..934c8fcd9 100644--- a/tox.ini+++ b/tox.ini@@ -189,6 +189,7 @@ deps =google-cloud-storagehttpx2[http2,socks]ipython+ psutilptpython # optional for shell wrapper testsrobotexclusionrulesparseruvloop; platform_system != "Windows" and implementation_name != "pypy"@@ -205,6 +206,7 @@ deps =google-cloud-storage==1.30.0httpx2[http2,socks]==2.0.0ipython==8.15.0+ psutil==5.9.0ptpython==3.0.23robotexclusionrulesparser==1.6.2uvloop==0.17.0; platform_system != "Windows" and implementation_name != "pypy"