Scrapy, a fast high-level web crawling & scraping framework for Python.
Reactorless tests 1: the test env (#6952)
Andrey Rakhmatullin3186ccf5d5ad
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlindex 31ca6b2e9..585cdd307 100644--- a/.github/workflows/tests-ubuntu.yml+++ b/.github/workflows/tests-ubuntu.yml@@ -32,6 +32,9 @@ jobs:- python-version: "3.13"env:TOXENV: default-reactor+ - python-version: "3.13"+ env:+ TOXENV: no-reactor- python-version: pypy3.11env:TOXENV: pypy3@@ -43,6 +46,9 @@ jobs:- python-version: "3.10.19"env:TOXENV: default-reactor-pinned+ - python-version: "3.10.19"+ env:+ TOXENV: no-reactor-pinned- python-version: pypy3.11env:TOXENV: pypy3-pinneddiff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.ymlindex 5f4b383fd..9b03b594d 100644--- a/.github/workflows/tests-windows.yml+++ b/.github/workflows/tests-windows.yml@@ -32,6 +32,9 @@ jobs:- python-version: "3.13"env:TOXENV: default-reactor+ - python-version: "3.13"+ env:+ TOXENV: no-reactor# pinned deps- python-version: "3.10.11"diff --git a/conftest.py b/conftest.pyindex 55b5a28c1..b17789b3a 100644--- a/conftest.py+++ b/conftest.py@@ -8,6 +8,7 @@ import pytestfrom twisted.web.http import H2_ENABLEDfrom scrapy.utils.reactor import set_asyncio_event_loop_policy+from scrapy.utils.reactorless import install_reactor_import_hookfrom tests.keys import generate_keysfrom tests.mockserver.http import MockServer@@ -25,10 +26,6 @@ collect_ignore = [# not a test, but looks like a test"scrapy/utils/testproc.py","scrapy/utils/testsite.py",- "tests/ftpserver.py",- "tests/mockserver.py",- "tests/pipelines.py",- "tests/spiders.py",# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess*_py_files("tests/AsyncCrawlerProcess"),# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess@@ -56,6 +53,17 @@ if not H2_ENABLED:)+def pytest_addoption(parser, pluginmanager):+ if pluginmanager.hasplugin("twisted"):+ return+ # add the full choice set so that pytest doesn't complain about invalid choices in some cases+ parser.addoption(+ "--reactor",+ default="none",+ choices=["asyncio", "default", "none"],+ )++@pytest.fixture(scope="session")def mockserver() -> Generator[MockServer]:with MockServer() as mockserver:@@ -72,17 +80,26 @@ def pytest_configure(config):# Needed on Windows to switch from proactor to selector for Twisted reactor compatibility.# If we decide to run tests with both, we will need to add a new option and check it here.set_asyncio_event_loop_policy()+ elif config.getoption("--reactor") == "none":+ install_reactor_import_hook()def pytest_runtest_setup(item):# Skip tests based on reactor markersreactor = item.config.getoption("--reactor")- if item.get_closest_marker("only_asyncio") and reactor != "asyncio":- pytest.skip("This test is only run with --reactor=asyncio")+ if item.get_closest_marker("requires_reactor") and reactor == "none":+ pytest.skip('This test is only run when the --reactor value is not "none"')- if item.get_closest_marker("only_not_asyncio") and reactor == "asyncio":- pytest.skip("This test is only run without --reactor=asyncio")+ if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}:+ pytest.skip(+ 'This test is only run when the --reactor value is "asyncio" (default) or "none"'+ )++ if item.get_closest_marker("only_not_asyncio") and reactor in {"asyncio", "none"}:+ pytest.skip(+ 'This test is only run when the --reactor value is not "asyncio" (default) or "none"'+ )# Skip tests requiring optional dependenciesoptional_deps = [diff --git a/pyproject.toml b/pyproject.tomlindex 7982b43b8..a2ace0ec7 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -227,8 +227,9 @@ addopts = [xfail_strict = truepython_files = ["test_*.py", "test_*/__init__.py"]markers = [- "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed",- "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed",+ "only_asyncio: marks tests that require the asyncio loop to be used",+ "only_not_asyncio: marks tests that require the asyncio loop to not be used",+ "requires_reactor: marks tests that require a reactor","requires_uvloop: marks tests as only enabled when uvloop is known to be working","requires_botocore: marks tests that need botocore (but not boto3)","requires_boto3: marks tests that need botocore and boto3",diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 4e0356350..7d81868a9 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -17,7 +17,7 @@ from unittest import mockfrom twisted.trial.unittest import SkipTestfrom twisted.web.client import Agent-from scrapy.crawler import CrawlerRunner+from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner, CrawlerRunnerBasefrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.deprecate import create_deprecated_class@@ -116,16 +116,24 @@ def get_reactor_settings() -> dict[str, Any]:``Crawler._apply_settings()`` checks that the installed reactor matches thesettings, so tests that run the crawler in the current process may need to- pass a correct ``"TWISTED_REACTOR"`` setting value when creating it.+ pass a correct :setting:`TWISTED_REACTOR` setting value when creating it."""- if not is_reactor_installed():- raise RuntimeError(- "get_reactor_settings() called without an installed reactor,"- " you may need to install a reactor explicitly when running your tests."- )settings: dict[str, Any] = {}- if not is_asyncio_reactor_installed():- settings["TWISTED_REACTOR"] = None+ if is_reactor_installed():+ if not is_asyncio_reactor_installed():+ settings["TWISTED_REACTOR"] = None+ else:+ # We are either running Scrapy tests for the reactorless mode, or+ # running some 3rd-party library tests for the reactorless mode, or+ # running some 3rd-party library tests without initializing a reactor+ # properly. The first two cases are fine, but we cannot distinguish the+ # last one from them.+ settings["TWISTED_ENABLED"] = False+ settings["DOWNLOAD_HANDLERS"] = {+ "ftp": None,+ "http": None,+ "https": None,+ }return settings@@ -144,7 +152,11 @@ def get_crawler(**get_reactor_settings(),**(settings_dict or {}),}- runner = CrawlerRunner(settings)+ runner: CrawlerRunnerBase+ if is_reactor_installed():+ runner = CrawlerRunner(settings)+ else:+ runner = AsyncCrawlerRunner(settings)crawler = runner.create_crawler(spidercls or DefaultSpider)crawler._apply_settings()return crawlerdiff --git a/tests/test_addons.py b/tests/test_addons.pyindex 457945ea5..fd673b2a5 100644--- a/tests/test_addons.py+++ b/tests/test_addons.py@@ -2,13 +2,12 @@ import itertoolsfrom typing import Anyfrom unittest.mock import patch-from twisted.internet.defer import inlineCallbacks-from scrapy import Spider-from scrapy.crawler import Crawler, CrawlerRunner+from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunnerfrom scrapy.exceptions import NotConfiguredfrom scrapy.settings import BaseSettings, Settingsfrom scrapy.utils.test import get_crawler, get_reactor_settings+from tests.utils.decorators import inlineCallbacksclass SimpleAddon:@@ -109,9 +108,15 @@ class TestAddonManager:crawler = get_crawler(settings_dict=settings_dict)assert crawler.settings.getint("KEY") == 15+ runner_cls = (+ CrawlerRunner+ if settings_dict.get("TWISTED_ENABLED", True)+ else AsyncCrawlerRunner+ )+settings = Settings(settings_dict)settings.set("KEY", 0, priority="default")- runner = CrawlerRunner(settings)+ runner = runner_cls(settings)crawler = runner.create_crawler(Spider)crawler._apply_settings()assert crawler.settings.getint("KEY") == 15@@ -123,44 +128,39 @@ class TestAddonManager:}settings = Settings(settings_dict)settings.set("KEY", 0, priority="default")- runner = CrawlerRunner(settings)+ runner = runner_cls(settings)crawler = runner.create_crawler(Spider)assert crawler.settings.getint("KEY") == 20def test_fallback_workflow(self):- FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"+ FALLBACK_SETTING = "MY_FALLBACK_SCHEDULER"class AddonWithFallback:def update_settings(self, settings):if not settings.get(FALLBACK_SETTING):settings.set(FALLBACK_SETTING,- settings.getwithbase("DOWNLOAD_HANDLERS")["https"],+ settings.get("SCHEDULER"),"addon",)- settings["DOWNLOAD_HANDLERS"]["https"] = "AddonHandler"+ settings["SCHEDULER"] = "AddonScheduler"settings_dict = {"ADDONS": {AddonWithFallback: 1},}crawler = get_crawler(settings_dict=settings_dict)+ assert crawler.settings.get("SCHEDULER") == "AddonScheduler"assert (- crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"] == "AddonHandler"- )- assert (- crawler.settings.get(FALLBACK_SETTING)- == "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler"+ crawler.settings.get(FALLBACK_SETTING) == "scrapy.core.scheduler.Scheduler")settings_dict = {"ADDONS": {AddonWithFallback: 1},- "DOWNLOAD_HANDLERS": {"https": "UserHandler"},+ "SCHEDULER": "UserScheduler",}crawler = get_crawler(settings_dict=settings_dict)- assert (- crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"] == "AddonHandler"- )- assert crawler.settings.get(FALLBACK_SETTING) == "UserHandler"+ assert crawler.settings.get("SCHEDULER") == "AddonScheduler"+ assert crawler.settings.get(FALLBACK_SETTING) == "UserScheduler"def test_logging_message(self):class LoggedAddon:diff --git a/tests/test_closespider.py b/tests/test_closespider.pyindex 2243cd371..623ed89c8 100644--- a/tests/test_closespider.py+++ b/tests/test_closespider.py@@ -1,5 +1,3 @@-from twisted.internet.defer import inlineCallbacks-from scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.spiders import (@@ -9,6 +7,7 @@ from tests.spiders import (MaxItemsAndRequestsSpider,SlowSpider,)+from tests.utils.decorators import inlineCallbacksclass TestCloseSpider:diff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex 326be570e..505960685 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -1,7 +1,6 @@from unittest import TextTestResultimport pytest-from twisted.internet.defer import inlineCallbacksfrom twisted.python import failurefrom scrapy import FormRequest@@ -19,6 +18,7 @@ from scrapy.spidermiddlewares.httperror import HttpErrorfrom scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer+from tests.utils.decorators import inlineCallbacksclass DemoItem(Item):diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex a35bfd590..d7940842a 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -18,13 +18,14 @@ from scrapy.core.downloader.contextfactory import (from scrapy.core.downloader.handlers.http11 import _RequestBodyProducerfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.settings import Settings-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytesfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http_resources import PayloadResourcefrom tests.mockserver.utils import ssl_context_factory+from tests.utils.decorators import deferred_f_from_coro_fif TYPE_CHECKING:from twisted.internet.defer import Deferreddiff --git a/tests/test_core_scraper.py b/tests/test_core_scraper.pyindex 47dee1271..3986656f0 100644--- a/tests/test_core_scraper.py+++ b/tests/test_core_scraper.py@@ -2,9 +2,9 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.test import get_crawlerfrom tests.spiders import SimpleSpider+from tests.utils.decorators import deferred_f_from_coro_fif TYPE_CHECKING:import pytestdiff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 1d64c2f39..c88526eea 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -9,7 +9,7 @@ from urllib.parse import urlencode, urlparseimport pytestfrom testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacks, succeed+from twisted.internet.defer import succeedfrom twisted.internet.ssl import Certificatefrom twisted.python.failure import Failure@@ -18,7 +18,7 @@ from scrapy.crawler import CrawlerRunnerfrom scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownloadfrom scrapy.http import Requestfrom scrapy.http.response import Response-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.engine import format_engine_status, get_engine_statusfrom scrapy.utils.python import to_unicodefrom scrapy.utils.test import get_crawler, get_reactor_settings@@ -55,6 +55,7 @@ from tests.spiders import (StartGoodAndBadOutput,StartItemSpider,)+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacksif TYPE_CHECKING:from scrapy.statscollectors import StatsCollectordiff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 6d0c82825..9235ef885 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -14,7 +14,7 @@ from typing import Anyimport pytestfrom packaging.version import parse as parse_versionfrom pexpect.popen_spawn import PopenSpawn-from twisted.internet.defer import Deferred, inlineCallbacks+from twisted.internet.defer import Deferredfrom w3lib import __version__ as w3lib_versionfrom zope.interface.exceptions import MultipleInvalid@@ -31,11 +31,7 @@ from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions.throttle import AutoThrottlefrom scrapy.settings import Settings, default_settingsfrom scrapy.utils.asyncio import call_later-from scrapy.utils.defer import (- deferred_f_from_coro_f,- deferred_from_coro,- maybe_deferred_to_future,-)+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_futurefrom scrapy.utils.log import (_uninstall_scrapy_root_handler,configure_logging,@@ -45,6 +41,7 @@ from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler, get_reactor_settingsfrom tests.mockserver.http import MockServerfrom tests.utils import get_script_run_env+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacksBASE_SETTINGS: dict[str, Any] = {}diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.pyindex 6cda607cc..60d2dfd12 100644--- a/tests/test_downloader_handler_twisted_ftp.py+++ b/tests/test_downloader_handler_twisted_ftp.py@@ -25,6 +25,9 @@ if TYPE_CHECKING:from twisted.protocols.ftp import FTPFactory+pytestmark = pytest.mark.requires_reactor++class TestFTPBase(ABC):username = "scrapy"password = "passwd"diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.pyindex 939e2c8ad..99f3697f8 100644--- a/tests/test_downloader_handler_twisted_http10.py+++ b/tests/test_downloader_handler_twisted_http10.py@@ -16,6 +16,9 @@ if TYPE_CHECKING:from tests.mockserver.http import MockServer+pytestmark = pytest.mark.requires_reactor++class HTTP10DownloadHandlerMixin:@propertydef download_handler_cls(self) -> type[DownloadHandlerProtocol]:diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.pyindex eb2735c4d..ffaa2c569 100644--- a/tests/test_downloader_handler_twisted_http11.py+++ b/tests/test_downloader_handler_twisted_http11.py@@ -4,6 +4,8 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Any+import pytest+from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandlerfrom tests.test_downloader_handlers_http_base import (TestHttp11Base,@@ -21,6 +23,9 @@ if TYPE_CHECKING:from scrapy.core.downloader.handlers import DownloadHandlerProtocol+pytestmark = pytest.mark.requires_reactor++class HTTP11DownloadHandlerMixin:@propertydef download_handler_cls(self) -> type[DownloadHandlerProtocol]:diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 5ae35a461..22eb56a94 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -27,9 +27,13 @@ if TYPE_CHECKING:from tests.mockserver.http import MockServerfrom tests.mockserver.proxy_echo import ProxyEchoMockServer-pytestmark = pytest.mark.skipif(- not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"-)++pytestmark = [+ pytest.mark.requires_reactor,+ pytest.mark.skipif(+ not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"+ ),+]class H2DownloadHandlerMixin:diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 602b2186f..7844415f5 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -19,9 +19,9 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Requestfrom scrapy.responsetypes import responsetypesfrom scrapy.utils.boto import is_botocore_available-from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawler+from tests.utils.decorators import deferred_f_from_coro_fclass DummyDH:diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex 7ad787346..4d4b4ecaa 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -26,11 +26,7 @@ from scrapy.exceptions import (UnsupportedURLSchemeError,)from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse-from scrapy.utils.defer import (- deferred_f_from_coro_f,- deferred_from_coro,- maybe_deferred_to_future,-)+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler@@ -38,6 +34,7 @@ from tests import NON_EXISTING_RESOLVABLEfrom tests.mockserver.proxy_echo import ProxyEchoMockServerfrom tests.mockserver.simple_https import SimpleMockServerfrom tests.spiders import SingleRequestSpider+from tests.utils.decorators import deferred_f_from_coro_fif TYPE_CHECKING:from collections.abc import AsyncGenerator, Generatordiff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex bc1748dfb..10ac77aa0 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -13,9 +13,10 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManagerfrom scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutputfrom scrapy.http import Request, Responsefrom scrapy.spiders import Spider-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.python import to_bytesfrom scrapy.utils.test import get_crawler, get_from_asyncio_queue+from tests.utils.decorators import deferred_f_from_coro_fif TYPE_CHECKING:from collections.abc import AsyncGeneratordiff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex 1e99d5162..9b4b2750b 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -14,12 +14,9 @@ from scrapy.http import Request, Response, TextResponsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.settings import Settingsfrom scrapy.utils.asyncio import call_later-from scrapy.utils.defer import (- deferred_f_from_coro_f,- deferred_from_coro,- maybe_deferred_to_future,-)+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_futurefrom tests.test_robotstxt_interface import rerp_available+from tests.utils.decorators import deferred_f_from_coro_fif TYPE_CHECKING:from scrapy.crawler import Crawlerdiff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex dffe95406..66dafd9b6 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -2,17 +2,16 @@ import timefrom typing import Anyimport pytest-from twisted.internet.defer import inlineCallbacksfrom scrapy import Requestfrom scrapy.core.downloader import Downloader, Slotfrom scrapy.crawler import CrawlerRunnerfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.spiders import MetaSpider+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacksclass DownloaderSlotsSettingsTestSpider(MetaSpider):@@ -85,6 +84,7 @@ class TestCrawl:assert max(list(error_delta.values())) < tolerance[email protected]_reactor # needs a reactor or an event loop for Downloader._slot_gc_loopdef test_params():params = {"concurrency": 1,@@ -109,6 +109,7 @@ def test_params():)[email protected]_reactor # needs a reactor or an event loop for Downloader._slot_gc_loopdef test_get_slot_deprecated_spider_arg():crawler = get_crawler(DefaultSpider)crawler.spider = crawler._create_spider()diff --git a/tests/test_engine.py b/tests/test_engine.pyindex d3485e344..d4f9fd794 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -17,7 +17,6 @@ from itemadapter import ItemAdapterfrom pydispatch import dispatcherfrom testfixtures import LogCapturefrom twisted.internet import defer-from twisted.internet.defer import inlineCallbacksfrom scrapy import signalsfrom scrapy.core.engine import ExecutionEngine, _Slot@@ -29,7 +28,6 @@ from scrapy.linkextractors import LinkExtractorfrom scrapy.spiders import Spiderfrom scrapy.utils.defer import (_schedule_coro,- deferred_f_from_coro_f,deferred_from_coro,maybe_deferred_to_future,)@@ -37,6 +35,7 @@ from scrapy.utils.signal import disconnect_allfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests import get_testdata+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacksif TYPE_CHECKING:from scrapy.core.scheduler import Scheduler@@ -599,6 +598,7 @@ class TestEngineDownload(TestEngineDownloadAsync):return await maybe_deferred_to_future(engine.download(request))[email protected]_reactor # needs a reactor or an event loop for _Slot.heartbeatdef test_request_scheduled_signal(caplog):class TestScheduler(BaseScheduler):def __init__(self):diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.pyindex c219a0f6f..f9cd437f7 100644--- a/tests/test_engine_loop.py+++ b/tests/test_engine_loop.py@@ -7,10 +7,11 @@ from typing import TYPE_CHECKINGfrom twisted.internet.defer import Deferredfrom scrapy import Request, Spider, signals-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.test_scheduler import MemoryScheduler+from tests.utils.decorators import deferred_f_from_coro_fif TYPE_CHECKING:import pytestdiff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.pyindex b46ebe2d9..e330adb3f 100644--- a/tests/test_engine_stop_download_bytes.py+++ b/tests/test_engine_stop_download_bytes.py@@ -3,7 +3,6 @@ from __future__ import annotationsfrom typing import TYPE_CHECKINGfrom scrapy.exceptions import StopDownload-from scrapy.utils.defer import deferred_f_from_coro_ffrom tests.test_engine import (AttrsItemsSpider,CrawlerRun,@@ -12,6 +11,7 @@ from tests.test_engine import (MySpider,TestEngineBase,)+from tests.utils.decorators import deferred_f_from_coro_fif TYPE_CHECKING:import pytestdiff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.pyindex 76cfbe8ed..f46f46bd7 100644--- a/tests/test_engine_stop_download_headers.py+++ b/tests/test_engine_stop_download_headers.py@@ -3,7 +3,6 @@ from __future__ import annotationsfrom typing import TYPE_CHECKINGfrom scrapy.exceptions import StopDownload-from scrapy.utils.defer import deferred_f_from_coro_ffrom tests.test_engine import (AttrsItemsSpider,CrawlerRun,@@ -12,6 +11,7 @@ from tests.test_engine import (MySpider,TestEngineBase,)+from tests.utils.decorators import deferred_f_from_coro_fif TYPE_CHECKING:import pytestdiff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.pyindex 8d6c22ad7..4f368d5ec 100644--- a/tests/test_extension_periodic_log.py+++ b/tests/test_extension_periodic_log.py@@ -3,6 +3,8 @@ from __future__ import annotationsimport datetimefrom typing import TYPE_CHECKING, Any+import pytest+from scrapy.extensions.periodic_log import PeriodicLogfrom scrapy.utils.test import get_crawler@@ -86,6 +88,7 @@ class TestPeriodicLog:assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})+ @pytest.mark.requires_reactor # needs a reactor or an event loop for PeriodicLog.taskdef test_log_delta(self):def emulate(settings=None):spider = MetaSpider()@@ -143,6 +146,7 @@ class TestPeriodicLog:and ("downloader/" in k and "bytes" not in k),)+ @pytest.mark.requires_reactor # needs a reactor or an event loop for PeriodicLog.taskdef test_log_stats(self):def emulate(settings=None):spider = MetaSpider()diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.pyindex 6b4ad450f..15e5aacce 100644--- a/tests/test_extension_telnet.py+++ b/tests/test_extension_telnet.py@@ -1,10 +1,10 @@import pytestfrom twisted.conch.telnet import ITelnetProtocolfrom twisted.cred import credentials-from twisted.internet.defer import inlineCallbacksfrom scrapy.extensions.telnet import TelnetConsolefrom scrapy.utils.test import get_crawler+from tests.utils.decorators import inlineCallbacksclass TestTelnetExtension:diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex a9d608e52..914d9a950 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -29,7 +29,6 @@ import lxml.etreeimport pytestfrom packaging.version import Versionfrom testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacksfrom w3lib.url import file_uri_to_path, path_to_file_urifrom zope.interface import implementerfrom zope.interface.verify import verifyObject@@ -50,12 +49,13 @@ from scrapy.extensions.feedexport import (StdoutFeedStorage,)from scrapy.settings import Settings-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.python import to_unicodefrom scrapy.utils.test import get_crawlerfrom tests.mockserver.ftp import MockFTPServerfrom tests.mockserver.http import MockServerfrom tests.spiders import ItemSpider+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacksif TYPE_CHECKING:from collections.abc import Callable, Iterablediff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 46ef75064..944a67920 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -39,9 +39,12 @@ if TYPE_CHECKING:from scrapy.core.http2.protocol import H2ClientProtocol-pytestmark = pytest.mark.skipif(- not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"-)+pytestmark = [+ pytest.mark.requires_reactor,+ pytest.mark.skipif(+ not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"+ ),+]def generate_random_string(size: int) -> str:diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex 426be13a7..5cd816d76 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -2,7 +2,6 @@ import loggingimport pytestfrom testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacksfrom twisted.python.failure import Failurefrom scrapy.exceptions import DropItem@@ -13,6 +12,7 @@ from scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.spiders import ItemSpider+from tests.utils.decorators import inlineCallbacksclass CustomItem(Item):diff --git a/tests/test_logstats.py b/tests/test_logstats.pyindex d25bc0c93..c2e6d3456 100644--- a/tests/test_logstats.py+++ b/tests/test_logstats.py@@ -16,6 +16,7 @@ class TestLogStats:self.stats.set_value("response_received_count", 4802)self.stats.set_value("item_scraped_count", 3201)+ @pytest.mark.requires_reactor # needs a reactor or an event loop for LogStats.taskdef test_stats_calculations(self):logstats = LogStats.from_crawler(self.crawler)diff --git a/tests/test_mail.py b/tests/test_mail.pyindex 4e3523208..c845ba320 100644--- a/tests/test_mail.py+++ b/tests/test_mail.py@@ -1,12 +1,14 @@from email.charset import Charsetfrom io import BytesIO+import pytestfrom twisted.internet import deferfrom twisted.internet._sslverify import ClientTLSOptionsfrom scrapy.mail import MailSender[email protected]_reactorclass TestMailSender:def test_send(self):mailsender = MailSender(debug=True)diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex 494833fe1..f29bc90e8 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Anyimport pytestfrom testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacksfrom w3lib.url import add_or_replace_parameterfrom scrapy import Spider, signals@@ -15,6 +14,7 @@ from scrapy.utils.misc import load_objectfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.spiders import SimpleSpider+from tests.utils.decorators import inlineCallbacksif TYPE_CHECKING:from scrapy.crawler import Crawlerdiff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 6ab5d9a8e..18fa6cdf2 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -19,7 +19,6 @@ from urllib.parse import urlparseimport attrimport pytestfrom itemadapter import ItemAdapter-from twisted.internet.defer import inlineCallbacksfrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Request, Response@@ -32,10 +31,10 @@ from scrapy.pipelines.files import (S3FilesStore,)from scrapy.settings import Settings-from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.ftp import MockFTPServer+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacksfrom .test_pipeline_media import _mocked_download_funcdiff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 05ad43d16..37e87f1a9 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -13,11 +13,12 @@ from scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.files import FileExceptionfrom scrapy.pipelines.media import MediaPipeline-from scrapy.utils.defer import _defer_sleep_async, deferred_f_from_coro_f+from scrapy.utils.defer import _defer_sleep_asyncfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.signal import disconnect_allfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler+from tests.utils.decorators import deferred_f_from_coro_fasync def _mocked_download_func(request):diff --git a/tests/test_pipelines.py b/tests/test_pipelines.pyindex acf8dd412..5bdca3035 100644--- a/tests/test_pipelines.py+++ b/tests/test_pipelines.py@@ -10,14 +10,11 @@ from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.pipelines import ItemPipelineManagerfrom scrapy.utils.asyncio import call_laterfrom scrapy.utils.conf import build_component_list-from scrapy.utils.defer import (- deferred_f_from_coro_f,- deferred_to_future,- maybe_deferred_to_future,-)+from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_futurefrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler, get_from_asyncio_queuefrom tests.mockserver.http import MockServer+from tests.utils.decorators import deferred_f_from_coro_fclass SimplePipeline:@@ -251,6 +248,9 @@ class TestPipeline:class TestCustomPipelineManager:+ # needs a reactor or an event loop for is_asyncio_available()+ # (for ItemPipelineManager.process_item())+ @pytest.mark.requires_reactordef test_deprecated_process_item_spider_arg(self) -> None:class CustomPipelineManager(ItemPipelineManager):def process_item(self, item, spider): # pylint: disable=useless-parent-delegationdiff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pyindex 27a78f802..4071224d6 100644--- a/tests/test_proxy_connect.py+++ b/tests/test_proxy_connect.py@@ -8,12 +8,12 @@ from urllib.parse import urlsplit, urlunsplitimport pytestfrom testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacksfrom scrapy.http import Requestfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.spiders import SimpleSpider, SingleRequestSpider+from tests.utils.decorators import inlineCallbacksclass MitmProxy:diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.pyindex e2c1668d8..4065a5a2c 100644--- a/tests/test_request_attribute_binding.py+++ b/tests/test_request_attribute_binding.py@@ -1,11 +1,11 @@from testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacksfrom scrapy import Request, signalsfrom scrapy.http.response import Responsefrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.spiders import SingleRequestSpider+from tests.utils.decorators import inlineCallbacksOVERRIDDEN_URL = "https://example.org"diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex 67ad4ba1e..56d7264da 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -1,10 +1,10 @@from testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacksfrom scrapy.http import Requestfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.spiders import MockServerSpider+from tests.utils.decorators import inlineCallbacksclass InjectArgumentsDownloaderMiddleware:diff --git a/tests/test_request_left.py b/tests/test_request_left.pyindex 721bfc76e..a5d5efffa 100644--- a/tests/test_request_left.py+++ b/tests/test_request_left.py@@ -1,9 +1,8 @@-from twisted.internet.defer import inlineCallbacks-from scrapy.signals import request_left_downloaderfrom scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer+from tests.utils.decorators import inlineCallbacksclass SignalCatcherSpider(Spider):diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex ef1da9b3a..8730489f7 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -8,7 +8,6 @@ from collections import dequefrom typing import Any, NamedTupleimport pytest-from twisted.internet.defer import inlineCallbacksfrom scrapy.core.downloader import Downloaderfrom scrapy.core.scheduler import BaseScheduler, Scheduler@@ -20,6 +19,7 @@ from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import load_objectfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer+from tests.utils.decorators import inlineCallbacksclass MemoryScheduler(BaseScheduler):@@ -99,6 +99,9 @@ class MockCrawler(Crawler):self.stats = load_object(self.settings["STATS_CLASS"])(self)+# needs a reactor or an event loop for is_asyncio_available()+# (for _schedule_coro())[email protected]_reactorclass SchedulerHandler(ABC):jobdir = None@@ -243,6 +246,9 @@ _URLS_WITH_SLOTS = [class TestMigration:+ # needs a reactor or an event loop for is_asyncio_available()+ # (for _schedule_coro())+ @pytest.mark.requires_reactordef test_migration(self, tmpdir):class PrevSchedulerHandler(SchedulerHandler):jobdir = tmpdirdiff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.pyindex 5dd2de8d8..538464788 100644--- a/tests/test_scheduler_base.py+++ b/tests/test_scheduler_base.py@@ -5,7 +5,6 @@ from urllib.parse import urljoinimport pytestfrom testfixtures import LogCapturefrom twisted.internet import defer-from twisted.internet.defer import inlineCallbacksfrom scrapy.core.scheduler import BaseSchedulerfrom scrapy.http import Request@@ -14,6 +13,7 @@ from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.request import fingerprintfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer+from tests.utils.decorators import inlineCallbacksPATHS = ["/a", "/b", "/c"]URLS = [urljoin("https://example.org", p) for p in PATHS]diff --git a/tests/test_signals.py b/tests/test_signals.pyindex 89b7a74dc..048af00ab 100644--- a/tests/test_signals.py+++ b/tests/test_signals.py@@ -1,10 +1,9 @@import pytest-from twisted.internet.defer import inlineCallbacksfrom scrapy import Request, Spider, signals-from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.test import get_crawler, get_from_asyncio_queuefrom tests.mockserver.http import MockServer+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacksclass ItemSpider(Spider):diff --git a/tests/test_spider.py b/tests/test_spider.pyindex a1649aa6f..dd1cc15b0 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -12,7 +12,6 @@ from unittest import mockimport pytestfrom testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacksfrom w3lib.url import safe_url_stringfrom scrapy import signals@@ -29,9 +28,9 @@ from scrapy.spiders import (XMLFeedSpider,)from scrapy.spiders.init import InitSpider-from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.test import get_crawler, get_reactor_settingsfrom tests import get_testdata, tests_datadir+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacksclass TestSpider:diff --git a/tests/test_spider_start.py b/tests/test_spider_start.pyindex e608d8975..d6767d74c 100644--- a/tests/test_spider_start.py+++ b/tests/test_spider_start.py@@ -9,10 +9,11 @@ from testfixtures import LogCapturefrom scrapy import Spider, signalsfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.test import get_crawlerfrom .utils import twisted_sleep+from .utils.decorators import deferred_f_from_coro_fSLEEP_SECONDS = 0.1diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex 3ac6d9af7..17d897526 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -15,9 +15,10 @@ from scrapy.http import Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.asyncgen import collect_asyncgenfrom scrapy.utils.asyncio import call_later-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler+from tests.utils.decorators import deferred_f_from_coro_fif TYPE_CHECKING:from twisted.python.failure import Failurediff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex 9899a0cd7..c0284c5ed 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -4,7 +4,6 @@ import loggingimport pytestfrom testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacksfrom scrapy.http import Request, Responsefrom scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware@@ -12,6 +11,7 @@ from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.spiders import MockServerSpider+from tests.utils.decorators import inlineCallbacksclass _HttpErrorSpider(MockServerSpider):diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 97d9d6470..b35a4c704 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -1,9 +1,9 @@from testfixtures import LogCapturefrom scrapy import Request, Spider-from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer+from tests.utils.decorators import deferred_f_from_coro_fclass _BaseSpiderMiddleware:diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.pyindex 18868bf8c..250b58544 100644--- a/tests/test_spidermiddleware_process_start.py+++ b/tests/test_spidermiddleware_process_start.py@@ -5,11 +5,12 @@ import pytestfrom scrapy import Spider, signalsfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.test import get_crawlerfrom tests.test_spider_start import SLEEP_SECONDSfrom .utils import twisted_sleep+from .utils.decorators import deferred_f_from_coro_fITEM_A = {"id": "a"}ITEM_B = {"id": "b"}diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.pyindex c2efe47c9..1f2209f60 100644--- a/tests/test_spidermiddleware_start.py+++ b/tests/test_spidermiddleware_start.py@@ -1,9 +1,9 @@from scrapy.http import Requestfrom scrapy.spidermiddlewares.start import StartSpiderMiddlewarefrom scrapy.spiders import Spider-from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawler+from tests.utils.decorators import deferred_f_from_coro_fclass TestMiddleware:diff --git a/tests/test_stats.py b/tests/test_stats.pyindex b5ab06901..187d240a8 100644--- a/tests/test_stats.py+++ b/tests/test_stats.py@@ -10,9 +10,9 @@ from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions.corestats import CoreStatsfrom scrapy.spiders import Spiderfrom scrapy.statscollectors import DummyStatsCollector, StatsCollector-from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.test import get_crawlerfrom tests.spiders import SimpleSpider+from tests.utils.decorators import deferred_f_from_coro_fif TYPE_CHECKING:from scrapy.crawler import Crawlerdiff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.pyindex b2d2b04c7..dba1ac5da 100644--- a/tests/test_utils_asyncgen.py+++ b/tests/test_utils_asyncgen.py@@ -1,5 +1,5 @@from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen-from scrapy.utils.defer import deferred_f_from_coro_f+from tests.utils.decorators import deferred_f_from_coro_fclass TestAsyncgenUtils:diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex b6b77c9e3..6706c1c80 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -14,13 +14,14 @@ from scrapy.utils.asyncio import (_parallel_asyncio,is_asyncio_available,)-from scrapy.utils.defer import deferred_f_from_coro_f+from tests.utils.decorators import deferred_f_from_coro_fif TYPE_CHECKING:from collections.abc import AsyncGeneratorclass TestAsyncio:+ @pytest.mark.requires_reactor # needs a reactor or an event loop for is_asyncio_available()def test_is_asyncio_available(self, reactor_pytest: str) -> None:# the result should depend only on the pytest --reactor argumentassert is_asyncio_available() == (reactor_pytest == "asyncio")@@ -103,6 +104,7 @@ class TestParallelAsyncio:assert max_parallel_count[0] <= self.CONCURRENT_ITEMS[email protected]_reactor # needs a running event loop for AsyncioLoopingCall.start()@pytest.mark.only_asyncioclass TestAsyncioLoopingCall:def test_looping_call(self):diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex be623c3fb..ad317f3ae 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -6,7 +6,8 @@ from asyncio import Futurefrom typing import TYPE_CHECKING, Anyimport pytest-from twisted.internet.defer import Deferred, inlineCallbacks, succeed+from twisted.internet.defer import Deferred, succeed+from twisted.internet.defer import inlineCallbacks as inlineCallbacks_origfrom scrapy.utils.asyncgen import as_async_generator, collect_asyncgenfrom scrapy.utils.defer import (@@ -19,6 +20,7 @@ from scrapy.utils.defer import (mustbe_deferred,parallel_async,)+from tests.utils.decorators import inlineCallbacksif TYPE_CHECKING:from collections.abc import AsyncGenerator, Awaitable, Callable, Generator@@ -107,6 +109,7 @@ class TestIterErrback:assert isinstance(errors[0].value, ZeroDivisionError)[email protected]_reactorclass TestAiterErrback:@deferred_f_from_coro_fasync def test_aiter_errback_good(self):@@ -134,6 +137,7 @@ class TestAiterErrback:assert isinstance(errors[0].value, ZeroDivisionError)[email protected]_reactorclass TestAsyncDefTestsuite:@deferred_f_from_coro_fasync def test_deferred_f_from_coro_f(self):@@ -304,7 +308,7 @@ class TestDeferredFromCoro:class TestDeferredFFromCoroF:- @inlineCallbacks+ @inlineCallbacks_origdef _assert_result(self, c_f: Callable[[], Awaitable[int]]) -> Generator[Deferred[Any], Any, None]:@@ -342,6 +346,7 @@ class TestDeferredFFromCoroF:@pytest.mark.only_asyncio[email protected]_reactorclass TestDeferredToFuture:@deferred_f_from_coro_fasync def test_deferred(self):@@ -377,6 +382,9 @@ class TestDeferredToFuture:@pytest.mark.only_asyncio+# needs a reactor or an event loop for is_asyncio_available()+# (for maybe_deferred_to_future())[email protected]_reactorclass TestMaybeDeferredToFutureAsyncio:@deferred_f_from_coro_fasync def test_deferred(self):@@ -412,6 +420,9 @@ class TestMaybeDeferredToFutureAsyncio:@pytest.mark.only_not_asyncio+# needs a reactor or an event loop for is_asyncio_available()+# (for maybe_deferred_to_future())[email protected]_reactorclass TestMaybeDeferredToFutureNotAsyncio:def test_deferred(self):d = Deferred()diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex 34fca06c1..4a9e375fb 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, TypeVarimport pytestfrom scrapy.utils.asyncgen import as_async_generator, collect_asyncgen-from scrapy.utils.defer import aiter_errback, deferred_f_from_coro_f+from scrapy.utils.defer import aiter_errbackfrom scrapy.utils.python import (MutableAsyncChain,MutableChain,@@ -20,6 +20,7 @@ from scrapy.utils.python import (to_unicode,without_none_values,)+from tests.utils.decorators import deferred_f_from_coro_fif TYPE_CHECKING:from collections.abc import Iterable, Mappingdiff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.pyindex 6cbdcfccc..658e68016 100644--- a/tests/test_utils_reactor.py+++ b/tests/test_utils_reactor.py@@ -3,20 +3,22 @@ import warningsimport pytest-from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.reactor import (_asyncio_reactor_path,install_reactor,is_asyncio_reactor_installed,set_asyncio_event_loop,)+from tests.utils.decorators import deferred_f_from_coro_fclass TestAsyncio:+ @pytest.mark.requires_reactordef test_is_asyncio_reactor_installed(self, reactor_pytest: str) -> None:# the result should depend only on the pytest --reactor argumentassert is_asyncio_reactor_installed() == (reactor_pytest == "asyncio")+ @pytest.mark.requires_reactordef test_install_asyncio_reactor(self):from twisted.internet import reactor as original_reactordiff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.pyindex 51e496575..e47e9013a 100644--- a/tests/test_utils_signal.py+++ b/tests/test_utils_signal.py@@ -4,7 +4,6 @@ import pytestfrom pydispatch import dispatcherfrom testfixtures import LogCapturefrom twisted.internet import defer-from twisted.internet.defer import inlineCallbacksfrom twisted.python.failure import Failurefrom scrapy.utils.asyncio import call_later@@ -15,6 +14,7 @@ from scrapy.utils.signal import (send_catch_log_deferred,)from scrapy.utils.test import get_from_asyncio_queue+from tests.utils.decorators import inlineCallbacksclass TestSendCatchLog:diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex 50cab9a60..0ad6bd9d1 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -30,6 +30,8 @@ from tests.mockserver.http_resources import (from tests.mockserver.utils import ssl_context_factoryfrom tests.test_core_downloader import TestContextFactoryBase+pytestmark = pytest.mark.requires_reactor+def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):"""Adapted version of twisted.web.client.getPage"""diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.pyindex e2c8ed205..7b11aac9e 100644--- a/tests/test_zz_resources.py+++ b/tests/test_zz_resources.py@@ -30,6 +30,7 @@ def test_stderr_log_handler() -> None:assert c == 0[email protected]_reactor # needs a running event loop for asyncio.all_tasks()@pytest.mark.only_asynciodef test_pending_asyncio_tasks() -> None:"""Test that there are no pending asyncio tasks."""diff --git a/tests/utils/decorators.py b/tests/utils/decorators.pynew file mode 100644index 000000000..dc9d2e113--- /dev/null+++ b/tests/utils/decorators.py@@ -0,0 +1,47 @@+from __future__ import annotations++from functools import wraps+from typing import TYPE_CHECKING, Any, ParamSpec++import pytest+from twisted.internet.defer import Deferred+from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig++from scrapy.utils.defer import deferred_from_coro++if TYPE_CHECKING:+ from collections.abc import Awaitable, Callable, Generator+++_P = ParamSpec("_P")+++def inlineCallbacks(+ f: Callable[_P, Generator[Deferred[Any], Any, None]],+) -> Callable[_P, Deferred[None]]:+ """Like :func:`twisted.internet.defer.inlineCallbacks`, but marks the+ decorated function with ``@pytest.mark.requires_reactor``."""++ @pytest.mark.requires_reactor+ @wraps(f)+ @inlineCallbacks_orig+ def wrapper(+ *args: _P.args, **kwargs: _P.kwargs+ ) -> Generator[Deferred[Any], Any, None]:+ return f(*args, **kwargs)++ return wrapper+++def deferred_f_from_coro_f(+ coro_f: Callable[_P, Awaitable[None]],+) -> Callable[_P, Deferred[None]]:+ """Like :func:`scrapy.utils.defer.deferred_f_from_coro_f`, but marks the+ decorated function with ``@pytest.mark.requires_reactor``."""++ @pytest.mark.requires_reactor+ @wraps(coro_f)+ def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Deferred[None]:+ return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))++ return fdiff --git a/tox.ini b/tox.iniindex 5200b26e0..fc4d1b824 100644--- a/tox.ini+++ b/tox.ini@@ -161,6 +161,10 @@ commands = {[pinned]commands}commands ={[testenv]commands} --reactor=default+[testenv:no-reactor]+commands =+ {[testenv]commands} -p no:twisted --reactor=none+[testenv:default-reactor-pinned]basepython = {[pinned]basepython}deps = {[testenv:pinned]deps}@@ -168,6 +172,13 @@ commands = {[pinned]commands} --reactor=defaultsetenv ={[pinned]setenv}+[testenv:no-reactor-pinned]+basepython = {[pinned]basepython}+deps = {[testenv:pinned]deps}+commands = {[pinned]commands} -p no:twisted --reactor=none+setenv =+ {[pinned]setenv}+[testenv:pypy3]basepython = pypy3commands =