Scrapy, a fast high-level web crawling & scraping framework for Python.
Speed up the PyPy CI job and the tests that wait on the engine heartbeat (#8020)
Adrian916039ca4ca7
diff --git a/conftest.py b/conftest.pyindex ad11cedcc..1a1fcf053 100644--- a/conftest.py+++ b/conftest.py@@ -8,6 +8,7 @@ from typing import TYPE_CHECKINGimport pytestfrom twisted.web.http import H2_ENABLED+from scrapy.core.engine import ExecutionEnginefrom scrapy.utils.reactor import set_asyncio_event_loop_policyfrom scrapy.utils.reactorless import install_reactor_import_hookfrom tests.keys import generate_keys@@ -69,6 +70,17 @@ def pytest_addoption(parser, pluginmanager):)[email protected](autouse=True)+def fast_engine_heartbeat(monkeypatch: pytest.MonkeyPatch) -> None:+ """Shorten the interval at which the engine checks for work while idle.++ Whenever nothing else wakes the engine up, e.g. while a component sleeps,+ it stays idle until its next heartbeat, so tests that go through that wait+ pay the whole interval.+ """+ monkeypatch.setattr(ExecutionEngine, "_SLOT_HEARTBEAT_INTERVAL", 0.1)++@pytest.fixture(scope="session")def mockserver() -> Generator[MockServer]:with MockServer() as mockserver:diff --git a/pyproject.toml b/pyproject.tomlindex f84b40768..b84980c09 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -198,8 +198,12 @@ branch = trueinclude = ["scrapy/*"]omit = ["tests/*"]disable_warnings = ["include-ignored"]+# Starting a coverage session in every spawned interpreter is what covers the+# code paths of the command-line interface. It is also the bulk of the run time+# of the tests that spawn one, so SCRAPY_COVERAGE_PATCH switches it for a cheap+# patch where that cost is not worth paying; see tox.ini.patch = [- "subprocess",+ "${SCRAPY_COVERAGE_PATCH-subprocess}",][tool.coverage.paths]diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 4cf4cf2b1..d4b0ea436 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -54,6 +54,17 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):self.times[slot].append(time.time())+class NoDelayDownloaderSlotsSpider(DownloaderSlotsSettingsTestSpider):+ custom_settings = {+ "DOWNLOAD_SLOTS": {+ slot: {}+ for slot in DownloaderSlotsSettingsTestSpider.custom_settings[+ "DOWNLOAD_SLOTS"+ ]+ },+ }++@inline_callbacks_testdef test_delay(mockserver: MockServer):crawler = get_crawler(DownloaderSlotsSettingsTestSpider)@@ -128,11 +139,11 @@ async def test_none_slot_with_priority_queue() -> None:"""Test specific cases for None slot handling with different priority queues."""crawler = get_crawler(- DownloaderSlotsSettingsTestSpider,+ NoDelayDownloaderSlotsSpider,settings_dict={"SCHEDULER_PRIORITY_QUEUE": priority_queue_class},)await crawler.crawl_async(mockserver=mockserver)- assert isinstance(crawler.spider, DownloaderSlotsSettingsTestSpider)+ assert isinstance(crawler.spider, NoDelayDownloaderSlotsSpider)assert hasattr(crawler.spider, "times")assert None not in crawler.spider.timesdiff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.pyindex 5c3c92d3b..f2bc2af42 100644--- a/tests/test_engine_loop.py+++ b/tests/test_engine_loop.py@@ -75,12 +75,14 @@ class SyncStartSpider(Spider):class TestMain:@coroutine_test- async def test_sleep(self):+ async def test_sleep(self) -> None:"""Neither asynchronous sleeps on Spider.start() nor the equivalent onthe scheduler (returning no requests while also returning True fromthe has_pending_requests() method) should cause the spider to miss theprocessing of any later requests."""- seconds = 2+ # Long enough to span several engine heartbeats, which are what+ # wake the engine up while the scheduler reports no requests.+ seconds = 0.5class TestSpider(Spider):name = "test"diff --git a/tox.ini b/tox.iniindex 9ac273c8f..b20cb30f2 100644--- a/tox.ini+++ b/tox.ini@@ -290,6 +290,13 @@ commands =basepython = pypy3deps ={[testenv:extra-deps]deps}+setenv =+ # Spawning an interpreter is slow on PyPy, and having each one start a+ # coverage session on top makes the tests that do it take 2-3 times as+ # long. The CPython jobs cover those code paths, so here coverage only+ # measures the test process itself, and _exit is a no-op stand-in for the+ # subprocess patch, which coverage offers no way to turn off.+ SCRAPY_COVERAGE_PATCH=_exit[testenv:min-pypy3]basepython = pypy3.11