Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Speed up the PyPy CI job and the tests that wait on the engine heartbeat (#8020)

Adrian916039ca4ca7
diff --git a/conftest.py b/conftest.py
index ad11cedcc..1a1fcf053 100644
--- a/conftest.py
+++ b/conftest.py
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING
import pytest
from twisted.web.http import H2_ENABLED
+from scrapy.core.engine import ExecutionEngine
from scrapy.utils.reactor import set_asyncio_event_loop_policy
from scrapy.utils.reactorless import install_reactor_import_hook
from tests.keys import generate_keys
@@ -69,6 +70,17 @@ def pytest_addoption(parser, pluginmanager):
)
[email protected](autouse=True)
+def fast_engine_heartbeat(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Shorten the interval at which the engine checks for work while idle.
+
+ Whenever nothing else wakes the engine up, e.g. while a component sleeps,
+ it stays idle until its next heartbeat, so tests that go through that wait
+ pay the whole interval.
+ """
+ monkeypatch.setattr(ExecutionEngine, "_SLOT_HEARTBEAT_INTERVAL", 0.1)
+
+
@pytest.fixture(scope="session")
def mockserver() -> Generator[MockServer]:
with MockServer() as mockserver:
diff --git a/pyproject.toml b/pyproject.toml
index f84b40768..b84980c09 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -198,8 +198,12 @@ branch = true
include = ["scrapy/*"]
omit = ["tests/*"]
disable_warnings = ["include-ignored"]
+# Starting a coverage session in every spawned interpreter is what covers the
+# code paths of the command-line interface. It is also the bulk of the run time
+# of the tests that spawn one, so SCRAPY_COVERAGE_PATCH switches it for a cheap
+# patch where that cost is not worth paying; see tox.ini.
patch = [
- "subprocess",
+ "${SCRAPY_COVERAGE_PATCH-subprocess}",
]
[tool.coverage.paths]
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 4cf4cf2b1..d4b0ea436 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -54,6 +54,17 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):
self.times[slot].append(time.time())
+class NoDelayDownloaderSlotsSpider(DownloaderSlotsSettingsTestSpider):
+ custom_settings = {
+ "DOWNLOAD_SLOTS": {
+ slot: {}
+ for slot in DownloaderSlotsSettingsTestSpider.custom_settings[
+ "DOWNLOAD_SLOTS"
+ ]
+ },
+ }
+
+
@inline_callbacks_test
def test_delay(mockserver: MockServer):
crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
@@ -128,11 +139,11 @@ async def test_none_slot_with_priority_queue(
) -> None:
"""Test specific cases for None slot handling with different priority queues."""
crawler = get_crawler(
- DownloaderSlotsSettingsTestSpider,
+ NoDelayDownloaderSlotsSpider,
settings_dict={"SCHEDULER_PRIORITY_QUEUE": priority_queue_class},
)
await crawler.crawl_async(mockserver=mockserver)
- assert isinstance(crawler.spider, DownloaderSlotsSettingsTestSpider)
+ assert isinstance(crawler.spider, NoDelayDownloaderSlotsSpider)
assert hasattr(crawler.spider, "times")
assert None not in crawler.spider.times
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index 5c3c92d3b..f2bc2af42 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -75,12 +75,14 @@ class SyncStartSpider(Spider):
class TestMain:
@coroutine_test
- async def test_sleep(self):
+ async def test_sleep(self) -> None:
"""Neither asynchronous sleeps on Spider.start() nor the equivalent on
the scheduler (returning no requests while also returning True from
the has_pending_requests() method) should cause the spider to miss the
processing of any later requests."""
- seconds = 2
+ # Long enough to span several engine heartbeats, which are what
+ # wake the engine up while the scheduler reports no requests.
+ seconds = 0.5
class TestSpider(Spider):
name = "test"
diff --git a/tox.ini b/tox.ini
index 9ac273c8f..b20cb30f2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -290,6 +290,13 @@ commands =
basepython = pypy3
deps =
{[testenv:extra-deps]deps}
+setenv =
+ # Spawning an interpreter is slow on PyPy, and having each one start a
+ # coverage session on top makes the tests that do it take 2-3 times as
+ # long. The CPython jobs cover those code paths, so here coverage only
+ # measures the test process itself, and _exit is a no-op stand-in for the
+ # subprocess patch, which coverage offers no way to turn off.
+ SCRAPY_COVERAGE_PATCH=_exit
[testenv:min-pypy3]
basepython = pypy3.11