Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Reactorless tests 1: the test env (#6952)

Andrey Rakhmatullin3186ccf5d5ad
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 31ca6b2e9..585cdd307 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -32,6 +32,9 @@ jobs:
- python-version: "3.13"
env:
TOXENV: default-reactor
+ - python-version: "3.13"
+ env:
+ TOXENV: no-reactor
- python-version: pypy3.11
env:
TOXENV: pypy3
@@ -43,6 +46,9 @@ jobs:
- python-version: "3.10.19"
env:
TOXENV: default-reactor-pinned
+ - python-version: "3.10.19"
+ env:
+ TOXENV: no-reactor-pinned
- python-version: pypy3.11
env:
TOXENV: pypy3-pinned
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 5f4b383fd..9b03b594d 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -32,6 +32,9 @@ jobs:
- python-version: "3.13"
env:
TOXENV: default-reactor
+ - python-version: "3.13"
+ env:
+ TOXENV: no-reactor
# pinned deps
- python-version: "3.10.11"
diff --git a/conftest.py b/conftest.py
index 55b5a28c1..b17789b3a 100644
--- a/conftest.py
+++ b/conftest.py
@@ -8,6 +8,7 @@ import pytest
from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import set_asyncio_event_loop_policy
+from scrapy.utils.reactorless import install_reactor_import_hook
from tests.keys import generate_keys
from tests.mockserver.http import MockServer
@@ -25,10 +26,6 @@ collect_ignore = [
# not a test, but looks like a test
"scrapy/utils/testproc.py",
"scrapy/utils/testsite.py",
- "tests/ftpserver.py",
- "tests/mockserver.py",
- "tests/pipelines.py",
- "tests/spiders.py",
# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess
*_py_files("tests/AsyncCrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess
@@ -56,6 +53,17 @@ if not H2_ENABLED:
)
+def pytest_addoption(parser, pluginmanager):
+ if pluginmanager.hasplugin("twisted"):
+ return
+ # add the full choice set so that pytest doesn't complain about invalid choices in some cases
+ parser.addoption(
+ "--reactor",
+ default="none",
+ choices=["asyncio", "default", "none"],
+ )
+
+
@pytest.fixture(scope="session")
def mockserver() -> Generator[MockServer]:
with MockServer() as mockserver:
@@ -72,17 +80,26 @@ def pytest_configure(config):
# Needed on Windows to switch from proactor to selector for Twisted reactor compatibility.
# If we decide to run tests with both, we will need to add a new option and check it here.
set_asyncio_event_loop_policy()
+ elif config.getoption("--reactor") == "none":
+ install_reactor_import_hook()
def pytest_runtest_setup(item):
# Skip tests based on reactor markers
reactor = item.config.getoption("--reactor")
- if item.get_closest_marker("only_asyncio") and reactor != "asyncio":
- pytest.skip("This test is only run with --reactor=asyncio")
+ if item.get_closest_marker("requires_reactor") and reactor == "none":
+ pytest.skip('This test is only run when the --reactor value is not "none"')
- if item.get_closest_marker("only_not_asyncio") and reactor == "asyncio":
- pytest.skip("This test is only run without --reactor=asyncio")
+ if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}:
+ pytest.skip(
+ 'This test is only run when the --reactor value is "asyncio" (default) or "none"'
+ )
+
+ if item.get_closest_marker("only_not_asyncio") and reactor in {"asyncio", "none"}:
+ pytest.skip(
+ 'This test is only run when the --reactor value is not "asyncio" (default) or "none"'
+ )
# Skip tests requiring optional dependencies
optional_deps = [
diff --git a/pyproject.toml b/pyproject.toml
index 7982b43b8..a2ace0ec7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -227,8 +227,9 @@ addopts = [
xfail_strict = true
python_files = ["test_*.py", "test_*/__init__.py"]
markers = [
- "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed",
- "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed",
+ "only_asyncio: marks tests that require the asyncio loop to be used",
+ "only_not_asyncio: marks tests that require the asyncio loop to not be used",
+ "requires_reactor: marks tests that require a reactor",
"requires_uvloop: marks tests as only enabled when uvloop is known to be working",
"requires_botocore: marks tests that need botocore (but not boto3)",
"requires_boto3: marks tests that need botocore and boto3",
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 4e0356350..7d81868a9 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -17,7 +17,7 @@ from unittest import mock
from twisted.trial.unittest import SkipTest
from twisted.web.client import Agent
-from scrapy.crawler import CrawlerRunner
+from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner, CrawlerRunnerBase
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.deprecate import create_deprecated_class
@@ -116,16 +116,24 @@ def get_reactor_settings() -> dict[str, Any]:
``Crawler._apply_settings()`` checks that the installed reactor matches the
settings, so tests that run the crawler in the current process may need to
- pass a correct ``"TWISTED_REACTOR"`` setting value when creating it.
+ pass a correct :setting:`TWISTED_REACTOR` setting value when creating it.
"""
- if not is_reactor_installed():
- raise RuntimeError(
- "get_reactor_settings() called without an installed reactor,"
- " you may need to install a reactor explicitly when running your tests."
- )
settings: dict[str, Any] = {}
- if not is_asyncio_reactor_installed():
- settings["TWISTED_REACTOR"] = None
+ if is_reactor_installed():
+ if not is_asyncio_reactor_installed():
+ settings["TWISTED_REACTOR"] = None
+ else:
+ # We are either running Scrapy tests for the reactorless mode, or
+ # running some 3rd-party library tests for the reactorless mode, or
+ # running some 3rd-party library tests without initializing a reactor
+ # properly. The first two cases are fine, but we cannot distinguish the
+ # last one from them.
+ settings["TWISTED_ENABLED"] = False
+ settings["DOWNLOAD_HANDLERS"] = {
+ "ftp": None,
+ "http": None,
+ "https": None,
+ }
return settings
@@ -144,7 +152,11 @@ def get_crawler(
**get_reactor_settings(),
**(settings_dict or {}),
}
- runner = CrawlerRunner(settings)
+ runner: CrawlerRunnerBase
+ if is_reactor_installed():
+ runner = CrawlerRunner(settings)
+ else:
+ runner = AsyncCrawlerRunner(settings)
crawler = runner.create_crawler(spidercls or DefaultSpider)
crawler._apply_settings()
return crawler
diff --git a/tests/test_addons.py b/tests/test_addons.py
index 457945ea5..fd673b2a5 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -2,13 +2,12 @@ import itertools
from typing import Any
from unittest.mock import patch
-from twisted.internet.defer import inlineCallbacks
-
from scrapy import Spider
-from scrapy.crawler import Crawler, CrawlerRunner
+from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner
from scrapy.exceptions import NotConfigured
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.test import get_crawler, get_reactor_settings
+from tests.utils.decorators import inlineCallbacks
class SimpleAddon:
@@ -109,9 +108,15 @@ class TestAddonManager:
crawler = get_crawler(settings_dict=settings_dict)
assert crawler.settings.getint("KEY") == 15
+ runner_cls = (
+ CrawlerRunner
+ if settings_dict.get("TWISTED_ENABLED", True)
+ else AsyncCrawlerRunner
+ )
+
settings = Settings(settings_dict)
settings.set("KEY", 0, priority="default")
- runner = CrawlerRunner(settings)
+ runner = runner_cls(settings)
crawler = runner.create_crawler(Spider)
crawler._apply_settings()
assert crawler.settings.getint("KEY") == 15
@@ -123,44 +128,39 @@ class TestAddonManager:
}
settings = Settings(settings_dict)
settings.set("KEY", 0, priority="default")
- runner = CrawlerRunner(settings)
+ runner = runner_cls(settings)
crawler = runner.create_crawler(Spider)
assert crawler.settings.getint("KEY") == 20
def test_fallback_workflow(self):
- FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
+ FALLBACK_SETTING = "MY_FALLBACK_SCHEDULER"
class AddonWithFallback:
def update_settings(self, settings):
if not settings.get(FALLBACK_SETTING):
settings.set(
FALLBACK_SETTING,
- settings.getwithbase("DOWNLOAD_HANDLERS")["https"],
+ settings.get("SCHEDULER"),
"addon",
)
- settings["DOWNLOAD_HANDLERS"]["https"] = "AddonHandler"
+ settings["SCHEDULER"] = "AddonScheduler"
settings_dict = {
"ADDONS": {AddonWithFallback: 1},
}
crawler = get_crawler(settings_dict=settings_dict)
+ assert crawler.settings.get("SCHEDULER") == "AddonScheduler"
assert (
- crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"] == "AddonHandler"
- )
- assert (
- crawler.settings.get(FALLBACK_SETTING)
- == "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler"
+ crawler.settings.get(FALLBACK_SETTING) == "scrapy.core.scheduler.Scheduler"
)
settings_dict = {
"ADDONS": {AddonWithFallback: 1},
- "DOWNLOAD_HANDLERS": {"https": "UserHandler"},
+ "SCHEDULER": "UserScheduler",
}
crawler = get_crawler(settings_dict=settings_dict)
- assert (
- crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"] == "AddonHandler"
- )
- assert crawler.settings.get(FALLBACK_SETTING) == "UserHandler"
+ assert crawler.settings.get("SCHEDULER") == "AddonScheduler"
+ assert crawler.settings.get(FALLBACK_SETTING) == "UserScheduler"
def test_logging_message(self):
class LoggedAddon:
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 2243cd371..623ed89c8 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -1,5 +1,3 @@
-from twisted.internet.defer import inlineCallbacks
-
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import (
@@ -9,6 +7,7 @@ from tests.spiders import (
MaxItemsAndRequestsSpider,
SlowSpider,
)
+from tests.utils.decorators import inlineCallbacks
class TestCloseSpider:
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 326be570e..505960685 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -1,7 +1,6 @@
from unittest import TextTestResult
import pytest
-from twisted.internet.defer import inlineCallbacks
from twisted.python import failure
from scrapy import FormRequest
@@ -19,6 +18,7 @@ from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
+from tests.utils.decorators import inlineCallbacks
class DemoItem(Item):
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index a35bfd590..d7940842a 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -18,13 +18,14 @@ from scrapy.core.downloader.contextfactory import (
from scrapy.core.downloader.handlers.http11 import _RequestBodyProducer
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http_resources import PayloadResource
from tests.mockserver.utils import ssl_context_factory
+from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
diff --git a/tests/test_core_scraper.py b/tests/test_core_scraper.py
index 47dee1271..3986656f0 100644
--- a/tests/test_core_scraper.py
+++ b/tests/test_core_scraper.py
@@ -2,9 +2,9 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
+from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
import pytest
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 1d64c2f39..c88526eea 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -9,7 +9,7 @@ from urllib.parse import urlencode, urlparse
import pytest
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks, succeed
+from twisted.internet.defer import succeed
from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
@@ -18,7 +18,7 @@ from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.engine import format_engine_status, get_engine_status
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler, get_reactor_settings
@@ -55,6 +55,7 @@ from tests.spiders import (
StartGoodAndBadOutput,
StartItemSpider,
)
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
if TYPE_CHECKING:
from scrapy.statscollectors import StatsCollector
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 6d0c82825..9235ef885 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -14,7 +14,7 @@ from typing import Any
import pytest
from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
-from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.internet.defer import Deferred
from w3lib import __version__ as w3lib_version
from zope.interface.exceptions import MultipleInvalid
@@ -31,11 +31,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.throttle import AutoThrottle
from scrapy.settings import Settings, default_settings
from scrapy.utils.asyncio import call_later
-from scrapy.utils.defer import (
- deferred_f_from_coro_f,
- deferred_from_coro,
- maybe_deferred_to_future,
-)
+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.log import (
_uninstall_scrapy_root_handler,
configure_logging,
@@ -45,6 +41,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests.mockserver.http import MockServer
from tests.utils import get_script_run_env
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
BASE_SETTINGS: dict[str, Any] = {}
diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py
index 6cda607cc..60d2dfd12 100644
--- a/tests/test_downloader_handler_twisted_ftp.py
+++ b/tests/test_downloader_handler_twisted_ftp.py
@@ -25,6 +25,9 @@ if TYPE_CHECKING:
from twisted.protocols.ftp import FTPFactory
+pytestmark = pytest.mark.requires_reactor
+
+
class TestFTPBase(ABC):
username = "scrapy"
password = "passwd"
diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py
index 939e2c8ad..99f3697f8 100644
--- a/tests/test_downloader_handler_twisted_http10.py
+++ b/tests/test_downloader_handler_twisted_http10.py
@@ -16,6 +16,9 @@ if TYPE_CHECKING:
from tests.mockserver.http import MockServer
+pytestmark = pytest.mark.requires_reactor
+
+
class HTTP10DownloadHandlerMixin:
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py
index eb2735c4d..ffaa2c569 100644
--- a/tests/test_downloader_handler_twisted_http11.py
+++ b/tests/test_downloader_handler_twisted_http11.py
@@ -4,6 +4,8 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any
+import pytest
+
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
from tests.test_downloader_handlers_http_base import (
TestHttp11Base,
@@ -21,6 +23,9 @@ if TYPE_CHECKING:
from scrapy.core.downloader.handlers import DownloadHandlerProtocol
+pytestmark = pytest.mark.requires_reactor
+
+
class HTTP11DownloadHandlerMixin:
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 5ae35a461..22eb56a94 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -27,9 +27,13 @@ if TYPE_CHECKING:
from tests.mockserver.http import MockServer
from tests.mockserver.proxy_echo import ProxyEchoMockServer
-pytestmark = pytest.mark.skipif(
- not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"
-)
+
+pytestmark = [
+ pytest.mark.requires_reactor,
+ pytest.mark.skipif(
+ not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"
+ ),
+]
class H2DownloadHandlerMixin:
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 602b2186f..7844415f5 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -19,9 +19,9 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.responsetypes import responsetypes
from scrapy.utils.boto import is_botocore_available
-from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
+from tests.utils.decorators import deferred_f_from_coro_f
class DummyDH:
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index 7ad787346..4d4b4ecaa 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -26,11 +26,7 @@ from scrapy.exceptions import (
UnsupportedURLSchemeError,
)
from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
-from scrapy.utils.defer import (
- deferred_f_from_coro_f,
- deferred_from_coro,
- maybe_deferred_to_future,
-)
+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@@ -38,6 +34,7 @@ from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver.proxy_echo import ProxyEchoMockServer
from tests.mockserver.simple_https import SimpleMockServer
from tests.spiders import SingleRequestSpider
+from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Generator
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index bc1748dfb..10ac77aa0 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -13,9 +13,10 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
from scrapy.http import Request, Response
from scrapy.spiders import Spider
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
+from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 1e99d5162..9b4b2750b 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -14,12 +14,9 @@ from scrapy.http import Request, Response, TextResponse
from scrapy.http.request import NO_CALLBACK
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
-from scrapy.utils.defer import (
- deferred_f_from_coro_f,
- deferred_from_coro,
- maybe_deferred_to_future,
-)
+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from tests.test_robotstxt_interface import rerp_available
+from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
from scrapy.crawler import Crawler
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index dffe95406..66dafd9b6 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -2,17 +2,16 @@ import time
from typing import Any
import pytest
-from twisted.internet.defer import inlineCallbacks
from scrapy import Request
from scrapy.core.downloader import Downloader, Slot
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import MetaSpider
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
class DownloaderSlotsSettingsTestSpider(MetaSpider):
@@ -85,6 +84,7 @@ class TestCrawl:
assert max(list(error_delta.values())) < tolerance
[email protected]_reactor # needs a reactor or an event loop for Downloader._slot_gc_loop
def test_params():
params = {
"concurrency": 1,
@@ -109,6 +109,7 @@ def test_params():
)
[email protected]_reactor # needs a reactor or an event loop for Downloader._slot_gc_loop
def test_get_slot_deprecated_spider_arg():
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
diff --git a/tests/test_engine.py b/tests/test_engine.py
index d3485e344..d4f9fd794 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -17,7 +17,6 @@ from itemadapter import ItemAdapter
from pydispatch import dispatcher
from testfixtures import LogCapture
from twisted.internet import defer
-from twisted.internet.defer import inlineCallbacks
from scrapy import signals
from scrapy.core.engine import ExecutionEngine, _Slot
@@ -29,7 +28,6 @@ from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Spider
from scrapy.utils.defer import (
_schedule_coro,
- deferred_f_from_coro_f,
deferred_from_coro,
maybe_deferred_to_future,
)
@@ -37,6 +35,7 @@ from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests import get_testdata
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
if TYPE_CHECKING:
from scrapy.core.scheduler import Scheduler
@@ -599,6 +598,7 @@ class TestEngineDownload(TestEngineDownloadAsync):
return await maybe_deferred_to_future(engine.download(request))
[email protected]_reactor # needs a reactor or an event loop for _Slot.heartbeat
def test_request_scheduled_signal(caplog):
class TestScheduler(BaseScheduler):
def __init__(self):
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index c219a0f6f..f9cd437f7 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -7,10 +7,11 @@ from typing import TYPE_CHECKING
from twisted.internet.defer import Deferred
from scrapy import Request, Spider, signals
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.test_scheduler import MemoryScheduler
+from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
import pytest
diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py
index b46ebe2d9..e330adb3f 100644
--- a/tests/test_engine_stop_download_bytes.py
+++ b/tests/test_engine_stop_download_bytes.py
@@ -3,7 +3,6 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy.exceptions import StopDownload
-from scrapy.utils.defer import deferred_f_from_coro_f
from tests.test_engine import (
AttrsItemsSpider,
CrawlerRun,
@@ -12,6 +11,7 @@ from tests.test_engine import (
MySpider,
TestEngineBase,
)
+from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
import pytest
diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py
index 76cfbe8ed..f46f46bd7 100644
--- a/tests/test_engine_stop_download_headers.py
+++ b/tests/test_engine_stop_download_headers.py
@@ -3,7 +3,6 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy.exceptions import StopDownload
-from scrapy.utils.defer import deferred_f_from_coro_f
from tests.test_engine import (
AttrsItemsSpider,
CrawlerRun,
@@ -12,6 +11,7 @@ from tests.test_engine import (
MySpider,
TestEngineBase,
)
+from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
import pytest
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index 8d6c22ad7..4f368d5ec 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -3,6 +3,8 @@ from __future__ import annotations
import datetime
from typing import TYPE_CHECKING, Any
+import pytest
+
from scrapy.extensions.periodic_log import PeriodicLog
from scrapy.utils.test import get_crawler
@@ -86,6 +88,7 @@ class TestPeriodicLog:
assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})
assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})
+ @pytest.mark.requires_reactor # needs a reactor or an event loop for PeriodicLog.task
def test_log_delta(self):
def emulate(settings=None):
spider = MetaSpider()
@@ -143,6 +146,7 @@ class TestPeriodicLog:
and ("downloader/" in k and "bytes" not in k),
)
+ @pytest.mark.requires_reactor # needs a reactor or an event loop for PeriodicLog.task
def test_log_stats(self):
def emulate(settings=None):
spider = MetaSpider()
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 6b4ad450f..15e5aacce 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -1,10 +1,10 @@
import pytest
from twisted.conch.telnet import ITelnetProtocol
from twisted.cred import credentials
-from twisted.internet.defer import inlineCallbacks
from scrapy.extensions.telnet import TelnetConsole
from scrapy.utils.test import get_crawler
+from tests.utils.decorators import inlineCallbacks
class TestTelnetExtension:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index a9d608e52..914d9a950 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -29,7 +29,6 @@ import lxml.etree
import pytest
from packaging.version import Version
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
from w3lib.url import file_uri_to_path, path_to_file_uri
from zope.interface import implementer
from zope.interface.verify import verifyObject
@@ -50,12 +49,13 @@ from scrapy.extensions.feedexport import (
StdoutFeedStorage,
)
from scrapy.settings import Settings
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
from tests.mockserver.http import MockServer
from tests.spiders import ItemSpider
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 46ef75064..944a67920 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -39,9 +39,12 @@ if TYPE_CHECKING:
from scrapy.core.http2.protocol import H2ClientProtocol
-pytestmark = pytest.mark.skipif(
- not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"
-)
+pytestmark = [
+ pytest.mark.requires_reactor,
+ pytest.mark.skipif(
+ not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"
+ ),
+]
def generate_random_string(size: int) -> str:
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 426be13a7..5cd816d76 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -2,7 +2,6 @@ import logging
import pytest
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
from twisted.python.failure import Failure
from scrapy.exceptions import DropItem
@@ -13,6 +12,7 @@ from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import ItemSpider
+from tests.utils.decorators import inlineCallbacks
class CustomItem(Item):
diff --git a/tests/test_logstats.py b/tests/test_logstats.py
index d25bc0c93..c2e6d3456 100644
--- a/tests/test_logstats.py
+++ b/tests/test_logstats.py
@@ -16,6 +16,7 @@ class TestLogStats:
self.stats.set_value("response_received_count", 4802)
self.stats.set_value("item_scraped_count", 3201)
+ @pytest.mark.requires_reactor # needs a reactor or an event loop for LogStats.task
def test_stats_calculations(self):
logstats = LogStats.from_crawler(self.crawler)
diff --git a/tests/test_mail.py b/tests/test_mail.py
index 4e3523208..c845ba320 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -1,12 +1,14 @@
from email.charset import Charset
from io import BytesIO
+import pytest
from twisted.internet import defer
from twisted.internet._sslverify import ClientTLSOptions
from scrapy.mail import MailSender
class TestMailSender:
def test_send(self):
mailsender = MailSender(debug=True)
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 494833fe1..f29bc90e8 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any
import pytest
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
from w3lib.url import add_or_replace_parameter
from scrapy import Spider, signals
@@ -15,6 +14,7 @@ from scrapy.utils.misc import load_object
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import SimpleSpider
+from tests.utils.decorators import inlineCallbacks
if TYPE_CHECKING:
from scrapy.crawler import Crawler
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 6ab5d9a8e..18fa6cdf2 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -19,7 +19,6 @@ from urllib.parse import urlparse
import attr
import pytest
from itemadapter import ItemAdapter
-from twisted.internet.defer import inlineCallbacks
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
@@ -32,10 +31,10 @@ from scrapy.pipelines.files import (
S3FilesStore,
)
from scrapy.settings import Settings
-from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
from .test_pipeline_media import _mocked_download_func
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 05ad43d16..37e87f1a9 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -13,11 +13,12 @@ from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FileException
from scrapy.pipelines.media import MediaPipeline
-from scrapy.utils.defer import _defer_sleep_async, deferred_f_from_coro_f
+from scrapy.utils.defer import _defer_sleep_async
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
+from tests.utils.decorators import deferred_f_from_coro_f
async def _mocked_download_func(request):
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index acf8dd412..5bdca3035 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -10,14 +10,11 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.pipelines import ItemPipelineManager
from scrapy.utils.asyncio import call_later
from scrapy.utils.conf import build_component_list
-from scrapy.utils.defer import (
- deferred_f_from_coro_f,
- deferred_to_future,
- maybe_deferred_to_future,
-)
+from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.mockserver.http import MockServer
+from tests.utils.decorators import deferred_f_from_coro_f
class SimplePipeline:
@@ -251,6 +248,9 @@ class TestPipeline:
class TestCustomPipelineManager:
+ # needs a reactor or an event loop for is_asyncio_available()
+ # (for ItemPipelineManager.process_item())
+ @pytest.mark.requires_reactor
def test_deprecated_process_item_spider_arg(self) -> None:
class CustomPipelineManager(ItemPipelineManager):
def process_item(self, item, spider): # pylint: disable=useless-parent-delegation
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 27a78f802..4071224d6 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -8,12 +8,12 @@ from urllib.parse import urlsplit, urlunsplit
import pytest
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
from scrapy.http import Request
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import SimpleSpider, SingleRequestSpider
+from tests.utils.decorators import inlineCallbacks
class MitmProxy:
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index e2c1668d8..4065a5a2c 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -1,11 +1,11 @@
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
from scrapy import Request, signals
from scrapy.http.response import Response
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import SingleRequestSpider
+from tests.utils.decorators import inlineCallbacks
OVERRIDDEN_URL = "https://example.org"
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 67ad4ba1e..56d7264da 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -1,10 +1,10 @@
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
from scrapy.http import Request
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider
+from tests.utils.decorators import inlineCallbacks
class InjectArgumentsDownloaderMiddleware:
diff --git a/tests/test_request_left.py b/tests/test_request_left.py
index 721bfc76e..a5d5efffa 100644
--- a/tests/test_request_left.py
+++ b/tests/test_request_left.py
@@ -1,9 +1,8 @@
-from twisted.internet.defer import inlineCallbacks
-
from scrapy.signals import request_left_downloader
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
+from tests.utils.decorators import inlineCallbacks
class SignalCatcherSpider(Spider):
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index ef1da9b3a..8730489f7 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -8,7 +8,6 @@ from collections import deque
from typing import Any, NamedTuple
import pytest
-from twisted.internet.defer import inlineCallbacks
from scrapy.core.downloader import Downloader
from scrapy.core.scheduler import BaseScheduler, Scheduler
@@ -20,6 +19,7 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
+from tests.utils.decorators import inlineCallbacks
class MemoryScheduler(BaseScheduler):
@@ -99,6 +99,9 @@ class MockCrawler(Crawler):
self.stats = load_object(self.settings["STATS_CLASS"])(self)
+# needs a reactor or an event loop for is_asyncio_available()
+# (for _schedule_coro())
class SchedulerHandler(ABC):
jobdir = None
@@ -243,6 +246,9 @@ _URLS_WITH_SLOTS = [
class TestMigration:
+ # needs a reactor or an event loop for is_asyncio_available()
+ # (for _schedule_coro())
+ @pytest.mark.requires_reactor
def test_migration(self, tmpdir):
class PrevSchedulerHandler(SchedulerHandler):
jobdir = tmpdir
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index 5dd2de8d8..538464788 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -5,7 +5,6 @@ from urllib.parse import urljoin
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
-from twisted.internet.defer import inlineCallbacks
from scrapy.core.scheduler import BaseScheduler
from scrapy.http import Request
@@ -14,6 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.request import fingerprint
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
+from tests.utils.decorators import inlineCallbacks
PATHS = ["/a", "/b", "/c"]
URLS = [urljoin("https://example.org", p) for p in PATHS]
diff --git a/tests/test_signals.py b/tests/test_signals.py
index 89b7a74dc..048af00ab 100644
--- a/tests/test_signals.py
+++ b/tests/test_signals.py
@@ -1,10 +1,9 @@
import pytest
-from twisted.internet.defer import inlineCallbacks
from scrapy import Request, Spider, signals
-from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.mockserver.http import MockServer
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
class ItemSpider(Spider):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index a1649aa6f..dd1cc15b0 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -12,7 +12,6 @@ from unittest import mock
import pytest
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
from w3lib.url import safe_url_string
from scrapy import signals
@@ -29,9 +28,9 @@ from scrapy.spiders import (
XMLFeedSpider,
)
from scrapy.spiders.init import InitSpider
-from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import get_testdata, tests_datadir
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
class TestSpider:
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index e608d8975..d6767d74c 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -9,10 +9,11 @@ from testfixtures import LogCapture
from scrapy import Spider, signals
from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler
from .utils import twisted_sleep
+from .utils.decorators import deferred_f_from_coro_f
SLEEP_SECONDS = 0.1
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 3ac6d9af7..17d897526 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -15,9 +15,10 @@ from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.asyncio import call_later
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
+from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
from twisted.python.failure import Failure
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 9899a0cd7..c0284c5ed 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -4,7 +4,6 @@ import logging
import pytest
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
@@ -12,6 +11,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider
+from tests.utils.decorators import inlineCallbacks
class _HttpErrorSpider(MockServerSpider):
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 97d9d6470..b35a4c704 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -1,9 +1,9 @@
from testfixtures import LogCapture
from scrapy import Request, Spider
-from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
+from tests.utils.decorators import deferred_f_from_coro_f
class _BaseSpiderMiddleware:
diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py
index 18868bf8c..250b58544 100644
--- a/tests/test_spidermiddleware_process_start.py
+++ b/tests/test_spidermiddleware_process_start.py
@@ -5,11 +5,12 @@ import pytest
from scrapy import Spider, signals
from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler
from tests.test_spider_start import SLEEP_SECONDS
from .utils import twisted_sleep
+from .utils.decorators import deferred_f_from_coro_f
ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"}
diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py
index c2efe47c9..1f2209f60 100644
--- a/tests/test_spidermiddleware_start.py
+++ b/tests/test_spidermiddleware_start.py
@@ -1,9 +1,9 @@
from scrapy.http import Request
from scrapy.spidermiddlewares.start import StartSpiderMiddleware
from scrapy.spiders import Spider
-from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
+from tests.utils.decorators import deferred_f_from_coro_f
class TestMiddleware:
diff --git a/tests/test_stats.py b/tests/test_stats.py
index b5ab06901..187d240a8 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -10,9 +10,9 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.corestats import CoreStats
from scrapy.spiders import Spider
from scrapy.statscollectors import DummyStatsCollector, StatsCollector
-from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
+from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
from scrapy.crawler import Crawler
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index b2d2b04c7..dba1ac5da 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -1,5 +1,5 @@
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
-from scrapy.utils.defer import deferred_f_from_coro_f
+from tests.utils.decorators import deferred_f_from_coro_f
class TestAsyncgenUtils:
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index b6b77c9e3..6706c1c80 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -14,13 +14,14 @@ from scrapy.utils.asyncio import (
_parallel_asyncio,
is_asyncio_available,
)
-from scrapy.utils.defer import deferred_f_from_coro_f
+from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
class TestAsyncio:
+ @pytest.mark.requires_reactor # needs a reactor or an event loop for is_asyncio_available()
def test_is_asyncio_available(self, reactor_pytest: str) -> None:
# the result should depend only on the pytest --reactor argument
assert is_asyncio_available() == (reactor_pytest == "asyncio")
@@ -103,6 +104,7 @@ class TestParallelAsyncio:
assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
[email protected]_reactor # needs a running event loop for AsyncioLoopingCall.start()
@pytest.mark.only_asyncio
class TestAsyncioLoopingCall:
def test_looping_call(self):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index be623c3fb..ad317f3ae 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -6,7 +6,8 @@ from asyncio import Future
from typing import TYPE_CHECKING, Any
import pytest
-from twisted.internet.defer import Deferred, inlineCallbacks, succeed
+from twisted.internet.defer import Deferred, succeed
+from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import (
@@ -19,6 +20,7 @@ from scrapy.utils.defer import (
mustbe_deferred,
parallel_async,
)
+from tests.utils.decorators import inlineCallbacks
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
@@ -107,6 +109,7 @@ class TestIterErrback:
assert isinstance(errors[0].value, ZeroDivisionError)
class TestAiterErrback:
@deferred_f_from_coro_f
async def test_aiter_errback_good(self):
@@ -134,6 +137,7 @@ class TestAiterErrback:
assert isinstance(errors[0].value, ZeroDivisionError)
class TestAsyncDefTestsuite:
@deferred_f_from_coro_f
async def test_deferred_f_from_coro_f(self):
@@ -304,7 +308,7 @@ class TestDeferredFromCoro:
class TestDeferredFFromCoroF:
- @inlineCallbacks
+ @inlineCallbacks_orig
def _assert_result(
self, c_f: Callable[[], Awaitable[int]]
) -> Generator[Deferred[Any], Any, None]:
@@ -342,6 +346,7 @@ class TestDeferredFFromCoroF:
@pytest.mark.only_asyncio
class TestDeferredToFuture:
@deferred_f_from_coro_f
async def test_deferred(self):
@@ -377,6 +382,9 @@ class TestDeferredToFuture:
@pytest.mark.only_asyncio
+# needs a reactor or an event loop for is_asyncio_available()
+# (for maybe_deferred_to_future())
class TestMaybeDeferredToFutureAsyncio:
@deferred_f_from_coro_f
async def test_deferred(self):
@@ -412,6 +420,9 @@ class TestMaybeDeferredToFutureAsyncio:
@pytest.mark.only_not_asyncio
+# needs a reactor or an event loop for is_asyncio_available()
+# (for maybe_deferred_to_future())
class TestMaybeDeferredToFutureNotAsyncio:
def test_deferred(self):
d = Deferred()
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 34fca06c1..4a9e375fb 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, TypeVar
import pytest
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
-from scrapy.utils.defer import aiter_errback, deferred_f_from_coro_f
+from scrapy.utils.defer import aiter_errback
from scrapy.utils.python import (
MutableAsyncChain,
MutableChain,
@@ -20,6 +20,7 @@ from scrapy.utils.python import (
to_unicode,
without_none_values,
)
+from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
from collections.abc import Iterable, Mapping
diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py
index 6cbdcfccc..658e68016 100644
--- a/tests/test_utils_reactor.py
+++ b/tests/test_utils_reactor.py
@@ -3,20 +3,22 @@ import warnings
import pytest
-from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.reactor import (
_asyncio_reactor_path,
install_reactor,
is_asyncio_reactor_installed,
set_asyncio_event_loop,
)
+from tests.utils.decorators import deferred_f_from_coro_f
class TestAsyncio:
+ @pytest.mark.requires_reactor
def test_is_asyncio_reactor_installed(self, reactor_pytest: str) -> None:
# the result should depend only on the pytest --reactor argument
assert is_asyncio_reactor_installed() == (reactor_pytest == "asyncio")
+ @pytest.mark.requires_reactor
def test_install_asyncio_reactor(self):
from twisted.internet import reactor as original_reactor
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index 51e496575..e47e9013a 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -4,7 +4,6 @@ import pytest
from pydispatch import dispatcher
from testfixtures import LogCapture
from twisted.internet import defer
-from twisted.internet.defer import inlineCallbacks
from twisted.python.failure import Failure
from scrapy.utils.asyncio import call_later
@@ -15,6 +14,7 @@ from scrapy.utils.signal import (
send_catch_log_deferred,
)
from scrapy.utils.test import get_from_asyncio_queue
+from tests.utils.decorators import inlineCallbacks
class TestSendCatchLog:
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 50cab9a60..0ad6bd9d1 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -30,6 +30,8 @@ from tests.mockserver.http_resources import (
from tests.mockserver.utils import ssl_context_factory
from tests.test_core_downloader import TestContextFactoryBase
+pytestmark = pytest.mark.requires_reactor
+
def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
"""Adapted version of twisted.web.client.getPage"""
diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.py
index e2c8ed205..7b11aac9e 100644
--- a/tests/test_zz_resources.py
+++ b/tests/test_zz_resources.py
@@ -30,6 +30,7 @@ def test_stderr_log_handler() -> None:
assert c == 0
[email protected]_reactor # needs a running event loop for asyncio.all_tasks()
@pytest.mark.only_asyncio
def test_pending_asyncio_tasks() -> None:
"""Test that there are no pending asyncio tasks."""
diff --git a/tests/utils/decorators.py b/tests/utils/decorators.py
new file mode 100644
index 000000000..dc9d2e113
--- /dev/null
+++ b/tests/utils/decorators.py
@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from functools import wraps
+from typing import TYPE_CHECKING, Any, ParamSpec
+
+import pytest
+from twisted.internet.defer import Deferred
+from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig
+
+from scrapy.utils.defer import deferred_from_coro
+
+if TYPE_CHECKING:
+ from collections.abc import Awaitable, Callable, Generator
+
+
+_P = ParamSpec("_P")
+
+
+def inlineCallbacks(
+ f: Callable[_P, Generator[Deferred[Any], Any, None]],
+) -> Callable[_P, Deferred[None]]:
+ """Like :func:`twisted.internet.defer.inlineCallbacks`, but marks the
+ decorated function with ``@pytest.mark.requires_reactor``."""
+
+ @pytest.mark.requires_reactor
+ @wraps(f)
+ @inlineCallbacks_orig
+ def wrapper(
+ *args: _P.args, **kwargs: _P.kwargs
+ ) -> Generator[Deferred[Any], Any, None]:
+ return f(*args, **kwargs)
+
+ return wrapper
+
+
+def deferred_f_from_coro_f(
+ coro_f: Callable[_P, Awaitable[None]],
+) -> Callable[_P, Deferred[None]]:
+ """Like :func:`scrapy.utils.defer.deferred_f_from_coro_f`, but marks the
+ decorated function with ``@pytest.mark.requires_reactor``."""
+
+ @pytest.mark.requires_reactor
+ @wraps(coro_f)
+ def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Deferred[None]:
+ return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
+
+ return f
diff --git a/tox.ini b/tox.ini
index 5200b26e0..fc4d1b824 100644
--- a/tox.ini
+++ b/tox.ini
@@ -161,6 +161,10 @@ commands = {[pinned]commands}
commands =
{[testenv]commands} --reactor=default
+[testenv:no-reactor]
+commands =
+ {[testenv]commands} -p no:twisted --reactor=none
+
[testenv:default-reactor-pinned]
basepython = {[pinned]basepython}
deps = {[testenv:pinned]deps}
@@ -168,6 +172,13 @@ commands = {[pinned]commands} --reactor=default
setenv =
{[pinned]setenv}
+[testenv:no-reactor-pinned]
+basepython = {[pinned]basepython}
+deps = {[testenv:pinned]deps}
+commands = {[pinned]commands} -p no:twisted --reactor=none
+setenv =
+ {[pinned]setenv}
+
[testenv:pypy3]
basepython = pypy3
commands =