Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Reactorless tests 2: enable pytest-asyncio. (#7233)

Andrey Rakhmatullinc8aa429c9bec
diff --git a/conftest.py b/conftest.py
index b17789b3a..50925a311 100644
--- a/conftest.py
+++ b/conftest.py
@@ -88,7 +88,10 @@ def pytest_runtest_setup(item):
# Skip tests based on reactor markers
reactor = item.config.getoption("--reactor")
- if item.get_closest_marker("requires_reactor") and reactor == "none":
+ if (
+ item.get_closest_marker("requires_reactor")
+ or item.get_closest_marker("requires_http_handler")
+ ) and reactor == "none":
pytest.skip('This test is only run when the --reactor value is not "none"')
if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}:
diff --git a/pyproject.toml b/pyproject.toml
index a2ace0ec7..e2484b02a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -230,13 +230,14 @@ markers = [
"only_asyncio: marks tests that require the asyncio loop to be used",
"only_not_asyncio: marks tests that require the asyncio loop to not be used",
"requires_reactor: marks tests that require a reactor",
+ "requires_http_handler: marks tests that require a HTTP handler",
"requires_uvloop: marks tests as only enabled when uvloop is known to be working",
"requires_botocore: marks tests that need botocore (but not boto3)",
"requires_boto3: marks tests that need botocore and boto3",
"requires_mitmproxy: marks tests that need mitmproxy",
]
filterwarnings = [
- "ignore::DeprecationWarning:twisted.web.static"
+ "ignore::DeprecationWarning:twisted.web.static",
]
[tool.ruff.lint]
diff --git a/tests/test_addons.py b/tests/test_addons.py
index fd673b2a5..675574d48 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -199,7 +199,12 @@ class TestAddonManager:
settings = Settings()
settings.setdict(get_reactor_settings())
settings.set("KEY", "default", priority="default")
- runner = CrawlerRunner(settings)
+ runner_cls = (
+ CrawlerRunner
+ if settings.getbool("TWISTED_ENABLED", True)
+ else AsyncCrawlerRunner
+ )
+ runner = runner_cls(settings)
crawler = runner.create_crawler(MySpider)
assert crawler.settings.get("KEY") == "default"
yield crawler.crawl()
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 623ed89c8..20c311c5a 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -1,3 +1,5 @@
+import pytest
+
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import (
@@ -10,6 +12,7 @@ from tests.spiders import (
from tests.utils.decorators import inlineCallbacks
[email protected]_http_handler
class TestCloseSpider:
@classmethod
def setup_class(cls):
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 505960685..9dface8aa 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -501,6 +501,7 @@ class TestContractsManager:
assert not self.results.failures
assert self.results.errors
+ @pytest.mark.requires_http_handler
@inlineCallbacks
def test_same_url(self):
class TestSameUrlSpider(Spider):
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index d7940842a..c479e66c9 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -38,6 +38,7 @@ class TestSlot:
assert repr(slot) == "Slot(concurrency=8, delay=0.10, randomize_delay=True)"
class TestContextFactoryBase:
context_factory = None
diff --git a/tests/test_core_scraper.py b/tests/test_core_scraper.py
index 3986656f0..249daf943 100644
--- a/tests/test_core_scraper.py
+++ b/tests/test_core_scraper.py
@@ -2,16 +2,17 @@ from __future__ import annotations
from typing import TYPE_CHECKING
+import pytest
+
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
- import pytest
-
from tests.mockserver.http import MockServer
[email protected]_http_handler
@deferred_f_from_coro_f
async def test_scraper_exception(
mockserver: MockServer,
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index c88526eea..61a4d387f 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -61,6 +61,7 @@ if TYPE_CHECKING:
from scrapy.statscollectors import StatsCollector
[email protected]_http_handler # easier than marking many individual tests
class TestCrawl:
mockserver: MockServer
@@ -428,6 +429,7 @@ with multiples lines
assert "NotSupported: Unsupported URL scheme 'foo'" in caplog.text
[email protected]_http_handler
class TestCrawlSpider:
mockserver: MockServer
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 9235ef885..8de38114c 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -646,12 +646,16 @@ class TestCrawlerProcess(TestBaseCrawler):
@pytest.mark.only_asyncio
class TestAsyncCrawlerProcess(TestBaseCrawler):
- def test_crawler_process_accepts_dict(self):
- runner = AsyncCrawlerProcess({"foo": "bar"}, install_root_handler=False)
+ def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:
+ runner = AsyncCrawlerProcess(
+ {"foo": "bar", "TWISTED_ENABLED": reactor_pytest != "none"},
+ install_root_handler=False,
+ )
assert runner.settings["foo"] == "bar"
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
- def test_crawler_process_accepts_None(self):
+ @pytest.mark.requires_reactor # can't pass TWISTED_ENABLED=False
+ def test_crawler_process_accepts_None(self) -> None:
runner = AsyncCrawlerProcess(install_root_handler=False)
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
@@ -672,6 +676,7 @@ class NoRequestsSpider(scrapy.Spider):
yield
class TestCrawlerRunnerHasSpider:
@staticmethod
def _runner():
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 22eb56a94..df19646ff 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -200,7 +200,7 @@ class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
) -> None:
with pytest.raises(NotImplementedError):
await maybe_deferred_to_future(
- super().test_download_with_proxy_https_timeout(proxy_mockserver)
+ super().test_download_with_proxy_https_timeout(proxy_mockserver) # type: ignore[arg-type]
)
@deferred_f_from_coro_f
@@ -209,5 +209,5 @@ class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
) -> None:
with pytest.raises(UnsupportedURLSchemeError):
await maybe_deferred_to_future(
- super().test_download_with_proxy_without_http_scheme(proxy_mockserver)
+ super().test_download_with_proxy_without_http_scheme(proxy_mockserver) # type: ignore[arg-type]
)
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 66dafd9b6..7d2001e47 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -67,6 +67,7 @@ class TestCrawl:
def setup_method(self):
self.runner = CrawlerRunner()
+ @pytest.mark.requires_http_handler
@inlineCallbacks
def test_delay(self):
crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
@@ -128,6 +129,7 @@ def test_get_slot_deprecated_spider_arg():
assert slot1 == slot2
[email protected]_http_handler
@pytest.mark.parametrize(
"priority_queue_class",
[
diff --git a/tests/test_engine.py b/tests/test_engine.py
index d4f9fd794..939ea0f6f 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -373,6 +373,7 @@ class TestEngineBase:
class TestEngine(TestEngineBase):
+ @pytest.mark.requires_http_handler
@deferred_f_from_coro_f
async def test_crawler(self, mockserver: MockServer) -> None:
for spider in (
@@ -390,6 +391,7 @@ class TestEngine(TestEngineBase):
self._assert_signals_caught(run)
self._assert_bytes_received(run)
+ @pytest.mark.requires_http_handler
@deferred_f_from_coro_f
async def test_crawler_dupefilter(self, mockserver: MockServer) -> None:
run = CrawlerRun(DupeFilterSpider)
@@ -397,12 +399,14 @@ class TestEngine(TestEngineBase):
self._assert_scheduled_requests(run, count=8)
self._assert_dropped_requests(run)
+ @pytest.mark.requires_http_handler
@deferred_f_from_coro_f
async def test_crawler_itemerror(self, mockserver: MockServer) -> None:
run = CrawlerRun(ItemZeroDivisionErrorSpider)
await run.run(mockserver)
self._assert_items_error(run)
+ @pytest.mark.requires_http_handler
@deferred_f_from_coro_f
async def test_crawler_change_close_reason_on_idle(
self, mockserver: MockServer
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index f9cd437f7..f12409029 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -4,6 +4,7 @@ from collections import deque
from logging import ERROR
from typing import TYPE_CHECKING
+import pytest
from twisted.internet.defer import Deferred
from scrapy import Request, Spider, signals
@@ -14,8 +15,6 @@ from tests.test_scheduler import MemoryScheduler
from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
- import pytest
-
from scrapy.http import Response
@@ -28,6 +27,7 @@ async def sleep(seconds: float = 0.001) -> None:
class TestMain:
+ @pytest.mark.requires_reactor # TODO
@deferred_f_from_coro_f
async def test_sleep(self):
"""Neither asynchronous sleeps on Spider.start() nor the equivalent on
@@ -332,6 +332,7 @@ class TestRequestSendOrder:
# Examples from the “Start requests” section of the documentation about
# spiders.
+ @pytest.mark.requires_http_handler
@deferred_f_from_coro_f
async def test_lazy(self):
start_nums = [1, 2, 4]
diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py
index e330adb3f..bd29e9ef6 100644
--- a/tests/test_engine_stop_download_bytes.py
+++ b/tests/test_engine_stop_download_bytes.py
@@ -2,6 +2,8 @@ from __future__ import annotations
from typing import TYPE_CHECKING
+import pytest
+
from scrapy.exceptions import StopDownload
from tests.test_engine import (
AttrsItemsSpider,
@@ -14,8 +16,6 @@ from tests.test_engine import (
from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
- import pytest
-
from tests.mockserver.http import MockServer
@@ -26,6 +26,7 @@ class BytesReceivedCrawlerRun(CrawlerRun):
class TestBytesReceivedEngine(TestEngineBase):
+ @pytest.mark.requires_http_handler
@deferred_f_from_coro_f
async def test_crawler(
self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py
index f46f46bd7..7dbaa41d2 100644
--- a/tests/test_engine_stop_download_headers.py
+++ b/tests/test_engine_stop_download_headers.py
@@ -2,6 +2,8 @@ from __future__ import annotations
from typing import TYPE_CHECKING
+import pytest
+
from scrapy.exceptions import StopDownload
from tests.test_engine import (
AttrsItemsSpider,
@@ -14,8 +16,6 @@ from tests.test_engine import (
from tests.utils.decorators import deferred_f_from_coro_f
if TYPE_CHECKING:
- import pytest
-
from tests.mockserver.http import MockServer
@@ -26,6 +26,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun):
class TestHeadersReceivedEngine(TestEngineBase):
+ @pytest.mark.requires_http_handler
@deferred_f_from_coro_f
async def test_crawler(
self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 15e5aacce..fcb027832 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -6,6 +6,8 @@ from scrapy.extensions.telnet import TelnetConsole
from scrapy.utils.test import get_crawler
from tests.utils.decorators import inlineCallbacks
+pytestmark = pytest.mark.requires_reactor
+
class TestTelnetExtension:
def _get_console_and_portal(self, settings=None):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 914d9a950..c65bd9df9 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -163,6 +163,7 @@ class TestFileFeedStorage:
assert storage.path == path
[email protected]_reactor # needs a reactor for BlockingFeedStorage
class TestFTPFeedStorage:
def get_test_spider(self, settings=None):
class TestSpider(scrapy.Spider):
@@ -277,6 +278,7 @@ class TestBlockingFeedStorage:
@pytest.mark.requires_boto3
[email protected]_reactor # needs a reactor for BlockingFeedStorage
class TestS3FeedStorage:
def test_parse_credentials(self):
aws_credentials = {
@@ -506,6 +508,7 @@ class TestS3FeedStorage:
assert "S3 does not support appending to files" in str(log)
[email protected]_reactor # needs a reactor for BlockingFeedStorage
class TestGCSFeedStorage:
def test_parse_settings(self):
try:
@@ -846,6 +849,7 @@ class ExceptionJsonItemExporter(JsonItemExporter):
raise RuntimeError("foo")
[email protected]_http_handler
class TestFeedExport(TestFeedExportBase):
async def run_and_export(
self, spider_cls: type[Spider], settings: dict[str, Any]
@@ -1830,6 +1834,7 @@ class TestFeedExport(TestFeedExportBase):
assert not Storage.file_was_closed
[email protected]_http_handler
class TestFeedPostProcessedExports(TestFeedExportBase):
items = [{"foo": "bar"}]
expected = b"foo\r\nbar\r\n"
@@ -2348,6 +2353,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert result == expected
[email protected]_http_handler
class TestBatchDeliveries(TestFeedExportBase):
_file_mark = "_%(batch_time)s_#%(batch_id)02d_"
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 5cd816d76..0602be0c8 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -253,6 +253,7 @@ class DropSomeItemsPipeline:
self.drop = True
[email protected]_http_handler
class TestShowOrSkipMessages:
@classmethod
def setup_class(cls):
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index f29bc90e8..dddb6d335 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -57,6 +57,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
)
[email protected]_http_handler
class TestFileDownloadCrawl:
pipeline_class = "scrapy.pipelines.files.FilesPipeline"
store_setting_key = "FILES_STORE"
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 18fa6cdf2..7a6e9a942 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -691,6 +691,7 @@ class TestGCSFilesStore:
store.bucket.get_blob.assert_called_with(expected_blob_path)
[email protected]_reactor # needs a reactor for FTPFilesStore
class TestFTPFileStore:
@inlineCallbacks
def test_persist(self):
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 5bdca3035..69a6c890d 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -127,6 +127,7 @@ class ItemSpider(Spider):
return {"field": 42}
[email protected]_http_handler
class TestPipeline:
def _on_item_scraped(self, item):
assert isinstance(item, dict)
@@ -265,6 +266,7 @@ class TestCustomPipelineManager:
):
itemproc.process_item({}, crawler.spider)
+ @pytest.mark.requires_http_handler
@deferred_f_from_coro_f
async def test_integration_recommended(self, mockserver: MockServer) -> None:
class CustomPipelineManager(ItemPipelineManager):
@@ -291,6 +293,7 @@ class TestCustomPipelineManager:
assert len(items) == 1
+ @pytest.mark.requires_http_handler
@deferred_f_from_coro_f
async def test_integration_no_async_subclass(self, mockserver: MockServer) -> None:
class CustomPipelineManager(ItemPipelineManager):
@@ -349,6 +352,7 @@ class TestCustomPipelineManager:
assert len(items) == 1
+ @pytest.mark.requires_http_handler
@deferred_f_from_coro_f
async def test_integration_no_async_not_subclass(
self, mockserver: MockServer
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 4071224d6..daa635b54 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -61,6 +61,7 @@ def _wrong_credentials(proxy_url):
return urlunsplit(bad_auth_proxy)
[email protected]_http_handler
@pytest.mark.requires_mitmproxy
class TestProxyConnect:
@classmethod
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index 4065a5a2c..48234adb3 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -1,3 +1,4 @@
+import pytest
from testfixtures import LogCapture
from scrapy import Request, signals
@@ -62,6 +63,7 @@ class AlternativeCallbacksMiddleware:
return response.replace(request=new_request)
[email protected]_http_handler
class TestCrawl:
@classmethod
def setup_class(cls):
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 56d7264da..c91a670c1 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -1,3 +1,4 @@
+import pytest
from testfixtures import LogCapture
from scrapy.http import Request
@@ -148,6 +149,7 @@ class KeywordArgumentsSpider(MockServerSpider):
self.crawler.stats.inc_value("boolean_checks", 1)
[email protected]_http_handler
class TestCallbackKeywordArguments:
@classmethod
def setup_class(cls):
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 8730489f7..f363b1cd5 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -369,6 +369,7 @@ class TestIntegrationWithDownloaderAwareInMemory:
},
)
+ @pytest.mark.requires_http_handler
@inlineCallbacks
def test_integration_downloader_aware_priority_queue(self):
with MockServer() as mockserver:
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index 538464788..277c00623 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -144,6 +144,7 @@ class TestSimpleScheduler(InterfaceCheckMixin):
assert close_result == "close"
[email protected]_http_handler
class TestMinimalSchedulerCrawl:
scheduler_cls = MinimalScheduler
diff --git a/tests/test_signals.py b/tests/test_signals.py
index 048af00ab..5769f6588 100644
--- a/tests/test_signals.py
+++ b/tests/test_signals.py
@@ -50,6 +50,7 @@ class TestMockServer:
item = await get_from_asyncio_queue(item)
self.items.append(item)
+ @pytest.mark.requires_http_handler
@pytest.mark.only_asyncio
@inlineCallbacks
def test_simple_pipeline(self):
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index d6767d74c..b84b0d8ac 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -162,6 +162,7 @@ class TestMain:
await self._test_start(start, [ITEM_A])
+ @pytest.mark.requires_reactor # needs a reactor for twisted_sleep()
@deferred_f_from_coro_f
async def test_twisted_delayed(self):
async def start(spider):
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index c0284c5ed..d4d0a4fb1 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -191,6 +191,7 @@ class TestHttpErrorMiddlewareHandleAll:
mw.process_spider_input(res402)
[email protected]_http_handler
class TestHttpErrorMiddlewareIntegrational:
@classmethod
def setup_class(cls):
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index b35a4c704..07df2ecc4 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -1,3 +1,4 @@
+import pytest
from testfixtures import LogCapture
from scrapy import Request, Spider
@@ -318,6 +319,7 @@ class NotGeneratorOutputChainSpider(Spider):
# ================================================================================
[email protected]_http_handler
class TestSpiderMiddleware:
mockserver: MockServer
diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py
index 250b58544..e7c9640b8 100644
--- a/tests/test_spidermiddleware_process_start.py
+++ b/tests/test_spidermiddleware_process_start.py
@@ -341,10 +341,12 @@ class TestMain:
[NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware]
)
+ @pytest.mark.requires_reactor
@deferred_f_from_coro_f
async def test_twisted_sleep_single(self):
await self._test_sleep([TwistedSleepSpiderMiddleware])
+ @pytest.mark.requires_reactor
@deferred_f_from_coro_f
async def test_twisted_sleep_multiple(self):
await self._test_sleep(
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index ad317f3ae..ad4cc466d 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -26,6 +26,7 @@ if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestMustbeDeferred:
@inlineCallbacks
@@ -153,6 +154,7 @@ class TestAsyncDefTestsuite:
raise RuntimeError("This is expected to be raised")
class TestParallelAsync:
"""This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.
diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py
index 658e68016..a045c05a4 100644
--- a/tests/test_utils_reactor.py
+++ b/tests/test_utils_reactor.py
@@ -29,6 +29,7 @@ class TestAsyncio:
assert original_reactor == reactor
+ @pytest.mark.requires_reactor
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
async def test_set_asyncio_event_loop(self):
diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.py
index 7b11aac9e..e6ff46f5c 100644
--- a/tests/test_zz_resources.py
+++ b/tests/test_zz_resources.py
@@ -8,6 +8,7 @@ import logging
import pytest
from scrapy.utils.log import LogCounterHandler
+from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed
def test_counter_handler() -> None:
@@ -35,3 +36,14 @@ def test_stderr_log_handler() -> None:
def test_pending_asyncio_tasks() -> None:
"""Test that there are no pending asyncio tasks."""
assert not asyncio.all_tasks()
+
+
+def test_installed_reactor(reactor_pytest: str) -> None:
+ """Test that the correct reactor is installed."""
+ match reactor_pytest:
+ case "asyncio":
+ assert is_asyncio_reactor_installed()
+ case "default":
+ assert not is_asyncio_reactor_installed()
+ case "none":
+ assert not is_reactor_installed()
diff --git a/tests/utils/decorators.py b/tests/utils/decorators.py
index dc9d2e113..633d8d97d 100644
--- a/tests/utils/decorators.py
+++ b/tests/utils/decorators.py
@@ -7,7 +7,8 @@ import pytest
from twisted.internet.defer import Deferred
from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig
-from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.defer import deferred_from_coro, deferred_to_future
+from scrapy.utils.reactor import is_reactor_installed
if TYPE_CHECKING:
from collections.abc import Awaitable, Callable, Generator
@@ -18,28 +19,48 @@ _P = ParamSpec("_P")
def inlineCallbacks(
f: Callable[_P, Generator[Deferred[Any], Any, None]],
-) -> Callable[_P, Deferred[None]]:
- """Like :func:`twisted.internet.defer.inlineCallbacks`, but marks the
- decorated function with ``@pytest.mark.requires_reactor``."""
+) -> Callable[_P, Awaitable[None]]:
+ """Mark a test function written in a :func:`twisted.internet.defer.inlineCallbacks` style.
+
+ This calls :func:`twisted.internet.defer.inlineCallbacks` and then:
+
+ * with ``pytest-twisted`` this returns the resulting Deferred
+ * with ``pytest-asyncio`` this converts the resulting Deferred into a
+ coroutine
+ """
+
+ if not is_reactor_installed():
+
+ @pytest.mark.asyncio
+ @wraps(f)
+ async def wrapper_coro(*args: _P.args, **kwargs: _P.kwargs) -> None:
+ await deferred_to_future(inlineCallbacks_orig(f)(*args, **kwargs))
+
+ return wrapper_coro
- @pytest.mark.requires_reactor
@wraps(f)
@inlineCallbacks_orig
- def wrapper(
+ def wrapper_dfd(
*args: _P.args, **kwargs: _P.kwargs
) -> Generator[Deferred[Any], Any, None]:
return f(*args, **kwargs)
- return wrapper
+ return wrapper_dfd
def deferred_f_from_coro_f(
coro_f: Callable[_P, Awaitable[None]],
-) -> Callable[_P, Deferred[None]]:
- """Like :func:`scrapy.utils.defer.deferred_f_from_coro_f`, but marks the
- decorated function with ``@pytest.mark.requires_reactor``."""
+) -> Callable[_P, Awaitable[None]]:
+ """Mark a test function that returns a coroutine.
+
+ * with ``pytest-twisted`` this converts a coroutine into a
+ :class:`twisted.internet.defer.Deferred`
+ * with ``pytest-asyncio`` this is a no-op
+ """
+
+ if not is_reactor_installed():
+ return pytest.mark.asyncio(coro_f)
- @pytest.mark.requires_reactor
@wraps(coro_f)
def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Deferred[None]:
return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
diff --git a/tox.ini b/tox.ini
index fc4d1b824..83cd06561 100644
--- a/tox.ini
+++ b/tox.ini
@@ -162,6 +162,9 @@ commands =
{[testenv]commands} --reactor=default
[testenv:no-reactor]
+deps =
+ {[testenv]deps}
+ pytest-asyncio
commands =
{[testenv]commands} -p no:twisted --reactor=none
@@ -174,7 +177,9 @@ setenv =
[testenv:no-reactor-pinned]
basepython = {[pinned]basepython}
-deps = {[testenv:pinned]deps}
+deps =
+ {[testenv:pinned]deps}
+ pytest-asyncio
commands = {[pinned]commands} -p no:twisted --reactor=none
setenv =
{[pinned]setenv}