Scrapy, a fast high-level web crawling & scraping framework for Python.
Add more no-reactor tests (#7259)
Andrey Rakhmatullin6fe27ba33e91
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.ymlindex 3adf3de38..2a1c62833 100644--- a/.github/workflows/tests-macos.yml+++ b/.github/workflows/tests-macos.yml@@ -19,6 +19,12 @@ jobs:fail-fast: falsematrix:python-version: ["3.10", "3.11", "3.12", "3.13"]+ env:+ - TOXENV: py+ include:+ - python-version: '3.13'+ env:+ TOXENV: no-reactorsteps:- uses: actions/checkout@v6@@ -29,9 +35,10 @@ jobs:python-version: ${{ matrix.python-version }}- name: Run tests+ env: ${{ matrix.env }}run: |pip install -U tox- tox -e py+ tox- name: Upload coverage reportuses: codecov/codecov-action@v5diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlindex c07cd49f0..a34112698 100644--- a/.github/workflows/tests-ubuntu.yml+++ b/.github/workflows/tests-ubuntu.yml@@ -64,6 +64,9 @@ jobs:- python-version: "3.13"env:TOXENV: extra-deps+ - python-version: "3.13"+ env:+ TOXENV: no-reactor-extra-deps- python-version: pypy3.11env:TOXENV: pypy3-extra-depsdiff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.pyindex 596e44f4c..916937152 100644--- a/scrapy/core/downloader/handlers/_httpx.py+++ b/scrapy/core/downloader/handlers/_httpx.py@@ -72,7 +72,7 @@ class HttpxDownloadHandler(BaseHttpDownloadHandler):_DEFAULT_CONNECT_TIMEOUT = 10def __init__(self, crawler: Crawler):- # we don't run extra-deps tests with the non-asyncio reactor+ # we skip HttpxDownloadHandler tests with the non-asyncio reactorif not is_asyncio_available(): # pragma: no coverraise NotConfigured(f"{type(self).__name__} requires the asyncio support. Make"diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex dc4106963..94920e840 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -8,6 +8,7 @@ For more information see docs/topics/architecture.rstfrom __future__ import annotationsimport asyncio+import contextlibimport loggingimport warningsfrom time import time@@ -197,7 +198,8 @@ class ExecutionEngine:self._start_request_processing_awaitable = asyncio.ensure_future(coro)else:self._start_request_processing_awaitable = Deferred.fromCoroutine(coro)- await maybe_deferred_to_future(self._closewait)+ with contextlib.suppress(asyncio.exceptions.CancelledError):+ await maybe_deferred_to_future(self._closewait)def stop(self) -> Deferred[None]: # pragma: no coverwarnings.warn(diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex b2f08450b..2e07a3b87 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -143,6 +143,10 @@ class Crawler:change them here when the reactor is not used."""self.settings.set("TELNETCONSOLE_ENABLED", False, priority="default")+ for scheme in ("http", "https"):+ self.settings["DOWNLOAD_HANDLERS_BASE"][scheme] = (+ "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler"+ )# Cannot use @deferred_f_from_coro_f because that relies on the reactor# being installed already, which is done within _apply_settings(), insidediff --git a/tests/test_command_check.py b/tests/test_command_check.pyindex dc07ed695..c34e75624 100644--- a/tests/test_command_check.py+++ b/tests/test_command_check.py@@ -46,10 +46,17 @@ class CheckSpider(scrapy.Spider):)def _test_contract(- self, proj_path: Path, contracts: str = "", parse_def: str = "pass"+ self,+ proj_path: Path,+ contracts: str = "",+ parse_def: str = "pass",+ use_reactor: bool = True,) -> None:self._write_contract(proj_path, contracts, parse_def)- ret, out, err = proc("check", cwd=proj_path)+ args = ["check"]+ if not use_reactor:+ args += ["-s", "TWISTED_ENABLED=False"]+ ret, out, err = proc(*args, cwd=proj_path)assert "F" not in outassert "OK" in errassert ret == 0@@ -63,6 +70,15 @@ class CheckSpider(scrapy.Spider):"""self._test_contract(proj_path, contracts, parse_def)+ def test_check_no_reactor(self, proj_path: Path) -> None:+ contracts = """+ @returns requests 1+ """+ parse_def = """+ yield scrapy.Request(url='http://next-url.com')+ """+ self._test_contract(proj_path, contracts, parse_def, use_reactor=False)+def test_check_returns_items_contract(self, proj_path: Path) -> None:contracts = """@returns items 1diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.pyindex 5a223d122..1b5dee961 100644--- a/tests/test_command_crawl.py+++ b/tests/test_command_crawl.py@@ -124,3 +124,20 @@ class MySpider(scrapy.Spider):not in log)assert "Spider closed (finished)" in log++ def test_no_reactor(self, proj_path: Path) -> None:+ spider_code = """+import scrapy++class MySpider(scrapy.Spider):+ name = 'myspider'++ async def start(self):+ self.logger.debug('It works!')+ return+ yield+"""+ log = self.get_log(spider_code, proj_path, args=("-s", "TWISTED_ENABLED=False"))+ assert "[myspider] DEBUG: It works!" in log+ assert "Not using a Twisted reactor" in log+ assert "Spider closed (finished)" in logdiff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.pyindex 4f4f19c3c..57db7dc50 100644--- a/tests/test_command_fetch.py+++ b/tests/test_command_fetch.py@@ -30,3 +30,9 @@ class TestFetchCommand:out = out.replace("\r", "") # required on win32assert "Server: TwistedWeb" in outassert "Content-Type: text/plain" in out++ def test_no_reactor(self, mockserver: MockServer) -> None:+ _, out, _ = proc(+ "fetch", "-s", "TWISTED_ENABLED=False", mockserver.url("/text")+ )+ assert out.strip() == "Works"diff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex 85529e607..d612b705b 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -513,3 +513,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}assert namespace.depth == 2assert namespace.spider == self.spider_nameassert namespace.verbose++ def test_no_reactor(self, proj_path: Path, mockserver: MockServer) -> None:+ _, out, stderr = proc(+ "parse",+ "--spider",+ "asyncdef_asyncio_return",+ "-c",+ "parse",+ mockserver.url("/html"),+ "-s",+ "TWISTED_ENABLED=False",+ cwd=proj_path,+ )+ assert "INFO: Got response 200" in stderr+ assert "{'id': 1}" in out+ assert "{'id': 2}" in outdiff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.pyindex b5408ccfe..9bc65f152 100644--- a/tests/test_command_runspider.py+++ b/tests/test_command_runspider.py@@ -213,6 +213,17 @@ class MySpider(scrapy.Spider):in log)+ def test_no_reactor(self, tmp_path: Path) -> None:+ log = self.get_log(+ tmp_path,+ self.debug_log_spider,+ args=[+ "-s",+ "TWISTED_ENABLED=False",+ ],+ )+ assert "Not using a Twisted reactor" in log+def test_output(self, tmp_path: Path) -> None:spider_code = """import scrapydiff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex 25262d809..ee515ede4 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -125,6 +125,13 @@ class TestShellCommand:assert ret == 0, errassert "RuntimeError: There is no current event loop in thread" not in err+ @pytest.mark.xfail(reason="Not implemented yet", strict=True)+ def test_shell_fetch_no_reactor(self, mockserver: MockServer) -> None:+ url = mockserver.url("/html")+ code = f"fetch('{url}')"+ ret, _, err = proc("shell", "-c", code, "--set", "TWISTED_ENABLED=False")+ assert ret == 0, err+class TestInteractiveShell:def test_fetch(self, mockserver: MockServer) -> None:diff --git a/tests/test_commands.py b/tests/test_commands.pyindex 07bd0b45c..209ffdc08 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -345,14 +345,18 @@ Unknown command: abcclass TestBenchCommand:- def test_run(self) -> None:- _, _, err = proc(+ @pytest.mark.parametrize("use_reactor", [True, False])+ def test_run(self, use_reactor: bool) -> None:+ args: list[str] = ["bench","-s","LOGSTATS_INTERVAL=0.001","-s","CLOSESPIDER_TIMEOUT=0.01",- )+ ]+ if not use_reactor:+ args += ["-s", "TWISTED_ENABLED=False"]+ _, _, err = proc(*args)assert "INFO: Crawled" in errassert "Unhandled Error" not in errassert "log_count/ERROR" not in errdiff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex 35de52a4b..92991fb8f 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -39,7 +39,7 @@ class TestSlot:assert repr(slot) == "Slot(concurrency=8, delay=0.10, randomize_delay=True)"[email protected]_reactor[email protected]_reactor # this test is related to the Twisted HTTP codeclass TestContextFactoryBase:context_factory: ContextFactory | None = Nonediff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 5c053089d..f4b4fbec9 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -666,7 +666,7 @@ class NoRequestsSpider(scrapy.Spider):yield[email protected]_reactor[email protected]_reactor # CrawlerRunner requires a reactorclass TestCrawlerRunnerHasSpider:@staticmethoddef _runner() -> CrawlerRunnerBase:diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pyindex 8d146b496..1c11a1034 100644--- a/tests/test_downloader_handler_httpx.py+++ b/tests/test_downloader_handler_httpx.py@@ -26,6 +26,8 @@ if TYPE_CHECKING:from tests.mockserver.http import MockServer+pytestmark = pytest.mark.only_asyncio+pytest.importorskip("httpx")diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.pyindex 957ad35c9..f39e5ecaf 100644--- a/tests/test_downloader_handler_twisted_ftp.py+++ b/tests/test_downloader_handler_twisted_ftp.py@@ -25,7 +25,7 @@ if TYPE_CHECKING:from twisted.protocols.ftp import FTPFactory-pytestmark = pytest.mark.requires_reactor+pytestmark = pytest.mark.requires_reactor # FTPDownloadHandler requires a reactorclass TestFTPBase(ABC):diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.pyindex 99f3697f8..a1cd0b7a6 100644--- a/tests/test_downloader_handler_twisted_http10.py+++ b/tests/test_downloader_handler_twisted_http10.py@@ -16,7 +16,7 @@ if TYPE_CHECKING:from tests.mockserver.http import MockServer-pytestmark = pytest.mark.requires_reactor+pytestmark = pytest.mark.requires_reactor # HTTP10DownloadHandler requires a reactorclass HTTP10DownloadHandlerMixin:diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.pyindex ffaa2c569..954dcd246 100644--- a/tests/test_downloader_handler_twisted_http11.py+++ b/tests/test_downloader_handler_twisted_http11.py@@ -23,7 +23,7 @@ if TYPE_CHECKING:from scrapy.core.downloader.handlers import DownloadHandlerProtocol-pytestmark = pytest.mark.requires_reactor+pytestmark = pytest.mark.requires_reactor # HTTP11DownloadHandler requires a reactorclass HTTP11DownloadHandlerMixin:diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 1b9dc21e7..b14356453 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -29,7 +29,7 @@ if TYPE_CHECKING:pytestmark = [- pytest.mark.requires_reactor,+ pytest.mark.requires_reactor, # H2DownloadHandler requires a reactorpytest.mark.skipif(not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"),diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 1a3221dc9..a717b18a6 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -85,8 +85,8 @@ class TestCrawl:assert max(list(error_delta.values())) < tolerance[email protected]_reactor # needs a reactor or an event loop for Downloader._slot_gc_loop-def test_params():+@coroutine_test+async def test_params():params = {"concurrency": 1,"delay": 2,@@ -110,8 +110,8 @@ def test_params():)[email protected]_reactor # needs a reactor or an event loop for Downloader._slot_gc_loop-def test_get_slot_deprecated_spider_arg():+@coroutine_test+async def test_get_slot_deprecated_spider_arg():crawler = get_crawler(DefaultSpider)crawler.spider = crawler._create_spider()downloader = Downloader(crawler)diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 131beeead..2cd583721 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -605,8 +605,8 @@ class TestEngineDownload(TestEngineDownloadAsync):return await maybe_deferred_to_future(engine.download(request))[email protected]_reactor # needs a reactor or an event loop for _Slot.heartbeat-def test_request_scheduled_signal(caplog):+@coroutine_test+async def test_request_scheduled_signal(caplog):class TestScheduler(BaseScheduler):def __init__(self):self.enqueued = []diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.pyindex 115deb821..8b5705487 100644--- a/tests/test_engine_loop.py+++ b/tests/test_engine_loop.py@@ -4,30 +4,21 @@ from collections import dequefrom logging import ERRORfrom typing import TYPE_CHECKING-import pytest-from twisted.internet.defer import Deferred-from scrapy import Request, Spider, signals-from scrapy.utils.defer import maybe_deferred_to_future+from scrapy.utils.asyncio import call_laterfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.test_scheduler import MemoryScheduler+from tests.utils import async_sleepfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:- from scrapy.http import Response-+ import pytest-async def sleep(seconds: float = 0.001) -> None:- from twisted.internet import reactor-- deferred: Deferred[None] = Deferred()- reactor.callLater(seconds, deferred.callback, None)- await maybe_deferred_to_future(deferred)+ from scrapy.http import Responseclass TestMain:- @pytest.mark.requires_reactor # TODO@coroutine_testasync def test_sleep(self):"""Neither asynchronous sleeps on Spider.start() nor the equivalent on@@ -40,27 +31,25 @@ class TestMain:name = "test"async def start(self):- from twisted.internet import reactor-yield Request("data:,a")- await sleep(seconds)+ await async_sleep(seconds)self.crawler.engine._slot.scheduler.pause()self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))# During this time, the scheduler reports having requests but# returns None.- await sleep(seconds)+ await async_sleep(seconds)self.crawler.engine._slot.scheduler.unpause()# The scheduler request is processed.- await sleep(seconds)+ await async_sleep(seconds)yield Request("data:,c")- await sleep(seconds)+ await async_sleep(seconds)self.crawler.engine._slot.scheduler.pause()self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d"))@@ -69,7 +58,7 @@ class TestMain:# delayed call below, proving that the start iteration can# finish before a scheduler “sleep” without causing the# scheduler to finish.- reactor.callLater(seconds, self.crawler.engine._slot.scheduler.unpause)+ call_later(seconds, self.crawler.engine._slot.scheduler.unpause)def parse(self, response):passdiff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.pyindex 86517e17b..18782fb70 100644--- a/tests/test_extension_periodic_log.py+++ b/tests/test_extension_periodic_log.py@@ -3,12 +3,11 @@ from __future__ import annotationsimport datetimefrom typing import TYPE_CHECKING, Any-import pytest-from scrapy.extensions.periodic_log import PeriodicLogfrom scrapy.utils.test import get_crawlerfrom .spiders import MetaSpider+from .utils.decorators import coroutine_testif TYPE_CHECKING:from collections.abc import Callable@@ -88,8 +87,8 @@ class TestPeriodicLog:assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})- @pytest.mark.requires_reactor # needs a reactor or an event loop for PeriodicLog.task- def test_log_delta(self):+ @coroutine_test+ async def test_log_delta(self):def emulate(settings: dict[str, Any] | None = None,) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]:@@ -154,8 +153,8 @@ class TestPeriodicLog:),)- @pytest.mark.requires_reactor # needs a reactor or an event loop for PeriodicLog.task- def test_log_stats(self):+ @coroutine_test+ async def test_log_stats(self):def emulate(settings: dict[str, Any] | None = None,) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]:diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.pyindex a956ab184..f1c86ce62 100644--- a/tests/test_extension_telnet.py+++ b/tests/test_extension_telnet.py@@ -6,7 +6,7 @@ from scrapy.extensions.telnet import TelnetConsolefrom scrapy.utils.test import get_crawlerfrom tests.utils.decorators import inline_callbacks_test-pytestmark = pytest.mark.requires_reactor+pytestmark = pytest.mark.requires_reactor # TelnetConsole requires a reactorclass TestTelnetExtension:diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex ff9b0a3f9..e50b2be42 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1151,7 +1151,7 @@ class TestFeedExport(TestFeedExportBase):data = await self.exported_no_data(settings)assert data["csv"] == b""- @pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage+ @pytest.mark.requires_reactor # TODO: needs a reactor for BlockingFeedStorage@coroutine_testasync def test_multiple_feeds_success_logs_blocking_feed_storage(self):settings = {diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.pyindex 3b70e896c..16c4357bb 100644--- a/tests/test_feedexport_batch.py+++ b/tests/test_feedexport_batch.py@@ -381,6 +381,7 @@ class TestBatchDeliveries(TestFeedExportBase):assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12+ @pytest.mark.requires_reactor # TODO: needs a reactor for BlockingFeedStorage@pytest.mark.requires_boto3@inline_callbacks_testdef test_s3_export(self):diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.pyindex b4da9bc68..c5cd5c782 100644--- a/tests/test_feedexport_storages.py+++ b/tests/test_feedexport_storages.py@@ -116,7 +116,7 @@ class TestFileFeedStorage:assert storage.path == path[email protected]_reactor # needs a reactor for BlockingFeedStorage[email protected]_reactor # TODO: needs a reactor for BlockingFeedStorageclass TestFTPFeedStorage:def get_test_spider(self, settings=None):class TestSpider(scrapy.Spider):@@ -231,7 +231,7 @@ class TestBlockingFeedStorage:@pytest.mark.requires_boto3[email protected]_reactor # needs a reactor for BlockingFeedStorage[email protected]_reactor # TODO: needs a reactor for BlockingFeedStorageclass TestS3FeedStorage:def test_parse_credentials(self):aws_credentials = {@@ -461,7 +461,7 @@ class TestS3FeedStorage:assert "S3 does not support appending to files" in str(log)[email protected]_reactor # needs a reactor for BlockingFeedStorage[email protected]_reactor # TODO: needs a reactor for BlockingFeedStorageclass TestGCSFeedStorage:def test_parse_settings(self):try:diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 49b9180b9..65da89472 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -40,7 +40,7 @@ if TYPE_CHECKING:pytestmark = [- pytest.mark.requires_reactor,+ pytest.mark.requires_reactor, # H2ClientProtocol requires a reactorpytest.mark.skipif(not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"),diff --git a/tests/test_logstats.py b/tests/test_logstats.pyindex c2e6d3456..370728e6a 100644--- a/tests/test_logstats.py+++ b/tests/test_logstats.py@@ -5,6 +5,7 @@ import pytestfrom scrapy.extensions.logstats import LogStatsfrom scrapy.utils.test import get_crawlerfrom tests.spiders import SimpleSpider+from tests.utils.decorators import coroutine_testclass TestLogStats:@@ -16,8 +17,8 @@ class TestLogStats:self.stats.set_value("response_received_count", 4802)self.stats.set_value("item_scraped_count", 3201)- @pytest.mark.requires_reactor # needs a reactor or an event loop for LogStats.task- def test_stats_calculations(self):+ @coroutine_test+ async def test_stats_calculations(self):logstats = LogStats.from_crawler(self.crawler)with pytest.raises(AttributeError):diff --git a/tests/test_mail.py b/tests/test_mail.pyindex c845ba320..61136dfbe 100644--- a/tests/test_mail.py+++ b/tests/test_mail.py@@ -8,7 +8,7 @@ from twisted.internet._sslverify import ClientTLSOptionsfrom scrapy.mail import MailSender[email protected]_reactor[email protected]_reactor # MailSender requires a reactorclass TestMailSender:def test_send(self):mailsender = MailSender(debug=True)diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 7f59af7de..201b29f83 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -584,6 +584,7 @@ class TestFilesPipelineCustomSettings:assert fs_store.basedir == str(tmp_path)[email protected]_reactor # TODO: needs a reactor for S3FilesStore@pytest.mark.requires_botocoreclass TestS3FilesStore:@inline_callbacks_test@@ -714,7 +715,7 @@ class TestGCSFilesStore:store.bucket.get_blob.assert_called_with(expected_blob_path)[email protected]_reactor # needs a reactor for FTPFilesStore[email protected]_reactor # TODO: needs a reactor for FTPFilesStoreclass TestFTPFileStore:@inline_callbacks_testdef test_persist(self):@@ -752,13 +753,13 @@ class ItemWithFiles(Item):files = Field()-def _create_item_with_files(*files):+def _create_item_with_files(*files: str) -> ItemWithFiles:item = ItemWithFiles()item["file_urls"] = filesreturn item-def _prepare_request_object(item_url, flags=None):+def _prepare_request_object(item_url: str, flags: list[str] | None = None) -> Request:return Request(item_url,meta={"response": Response(item_url, status=200, body=b"data", flags=flags)},diff --git a/tests/test_pipelines.py b/tests/test_pipelines.pyindex 2d5836ad6..e65b5bc32 100644--- a/tests/test_pipelines.py+++ b/tests/test_pipelines.py@@ -248,10 +248,8 @@ class TestPipeline:class TestCustomPipelineManager:- # needs a reactor or an event loop for is_asyncio_available()- # (for ItemPipelineManager.process_item())- @pytest.mark.requires_reactor- def test_deprecated_process_item_spider_arg(self) -> None:+ @coroutine_test+ async def test_deprecated_process_item_spider_arg(self) -> None:class CustomPipelineManager(ItemPipelineManager):def process_item(self, item: Any, spider: Spider) -> Deferred[Any]: # pylint: disable=useless-parent-delegationreturn super().process_item(item, spider)diff --git a/tests/test_pqueues.py b/tests/test_pqueues.pyindex 2d42ba791..350b3e10d 100644--- a/tests/test_pqueues.py+++ b/tests/test_pqueues.py@@ -1,4 +1,5 @@import tempfile+from unittest.mock import Mockimport pytestimport queuelib@@ -9,7 +10,7 @@ from scrapy.spiders import Spiderfrom scrapy.squeues import FifoMemoryQueuefrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.test import get_crawler-from tests.test_scheduler import MockDownloader, MockEngine+from tests.test_scheduler import MockDownloaderclass TestPriorityQueue:@@ -98,7 +99,7 @@ class TestPriorityQueue:class TestDownloaderAwarePriorityQueue:def setup_method(self):crawler = get_crawler(Spider)- crawler.engine = MockEngine(downloader=MockDownloader())+ crawler.engine = Mock(downloader=MockDownloader())self.queue = DownloaderAwarePriorityQueue.from_crawler(crawler=crawler,downstream_queue_cls=FifoMemoryQueue,diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex 0d2d8be93..7fec479f3 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -1,11 +1,11 @@from __future__ import annotations-import shutil-import tempfileimport warningsfrom abc import ABC, abstractmethodfrom collections import deque-from typing import Any, NamedTuple+from contextlib import AbstractAsyncContextManager, asynccontextmanager+from typing import TYPE_CHECKING, Any, NamedTuple+from unittest.mock import Mockimport pytest@@ -14,12 +14,16 @@ from scrapy.core.scheduler import BaseScheduler, Schedulerfrom scrapy.crawler import Crawlerfrom scrapy.http import Requestfrom scrapy.spiders import Spider-from scrapy.utils.defer import _schedule_coro+from scrapy.utils.defer import ensure_awaitablefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import load_objectfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer-from tests.utils.decorators import inline_callbacks_test+from tests.utils.decorators import coroutine_test, inline_callbacks_test++if TYPE_CHECKING:+ from collections.abc import AsyncGenerator+ from pathlib import Pathclass MemoryScheduler(BaseScheduler):@@ -54,78 +58,62 @@ class MemoryScheduler(BaseScheduler):self.paused = False-class MockEngine(NamedTuple):- downloader: MockDownloader--class MockSlot(NamedTuple):active: list[Any]class MockDownloader:- def __init__(self):- self.slots = {}+ def __init__(self) -> None:+ self.slots: dict[str, MockSlot] = {}- def get_slot_key(self, request):+ def get_slot_key(self, request: Request) -> str:if Downloader.DOWNLOAD_SLOT in request.meta:return request.meta[Downloader.DOWNLOAD_SLOT]return urlparse_cached(request).hostname or ""- def increment(self, slot_key):+ def increment(self, slot_key: str) -> None:slot = self.slots.setdefault(slot_key, MockSlot(active=[]))slot.active.append(1)- def decrement(self, slot_key):- slot = self.slots.get(slot_key)+ def decrement(self, slot_key: str) -> None:+ slot = self.slots[slot_key]slot.active.pop()- def close(self):+ def close(self) -> None:passclass MockCrawler(Crawler):- def __init__(self, priority_queue_cls, jobdir):+ def __init__(self, priority_queue_cls: str, jobdir: Path | None):settings = {"SCHEDULER_DEBUG": False,"SCHEDULER_DISK_QUEUE": "scrapy.squeues.PickleLifoDiskQueue","SCHEDULER_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue","SCHEDULER_PRIORITY_QUEUE": priority_queue_cls,- "JOBDIR": jobdir,+ "JOBDIR": str(jobdir) if jobdir is not None else None,"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",}super().__init__(Spider, settings)- self.engine = MockEngine(downloader=MockDownloader())+ self.engine = Mock(downloader=MockDownloader())self.stats = load_object(self.settings["STATS_CLASS"])(self)-# needs a reactor or an event loop for is_asyncio_available()-# (for _schedule_coro())[email protected]_reactor-class SchedulerHandler(ABC):- jobdir = None-- @property- @abstractmethod- def priority_queue_cls(self) -> str:- raise NotImplementedError-- def create_scheduler(self):- self.mock_crawler = MockCrawler(self.priority_queue_cls, self.jobdir)- self.scheduler = Scheduler.from_crawler(self.mock_crawler)- self.spider = Spider(name="spider")- self.scheduler.open(self.spider)-- def close_scheduler(self):- self.scheduler.close("finished")- _schedule_coro(self.mock_crawler.stop_async())- self.mock_crawler.engine.downloader.close()-- def setup_method(self):- self.create_scheduler()-- def teardown_method(self):- self.close_scheduler()+@asynccontextmanager+async def create_scheduler(+ priority_queue_cls: str, jobdir: Path | None+) -> AsyncGenerator[Scheduler]:+ mock_crawler = MockCrawler(priority_queue_cls, jobdir)+ scheduler = Scheduler.from_crawler(mock_crawler)+ spider = Spider(name="spider")+ await ensure_awaitable(scheduler.open(spider))+ try:+ yield scheduler+ finally:+ await ensure_awaitable(scheduler.close("finished"))+ await mock_crawler.stop_async()+ assert mock_crawler.engine+ mock_crawler.engine.downloader.close()_PRIORITIES = [@@ -140,99 +128,118 @@ _PRIORITIES = [_URLS = {"http://foo.com/a", "http://foo.com/b", "http://foo.com/c"}-class TestSchedulerInMemoryBase(SchedulerHandler):- def test_length(self):- assert not self.scheduler.has_pending_requests()- assert len(self.scheduler) == 0+class TestSchedulerBase(ABC):+ @property+ @abstractmethod+ def priority_queue_cls(self) -> str:+ raise NotImplementedError- for url in _URLS:- self.scheduler.enqueue_request(Request(url))+ @pytest.fixture+ def jobdir(self) -> Path | None:+ return None- assert self.scheduler.has_pending_requests()- assert len(self.scheduler) == len(_URLS)+ def create_scheduler(+ self, jobdir: Path | None+ ) -> AbstractAsyncContextManager[Scheduler]:+ return create_scheduler(self.priority_queue_cls, jobdir)- def test_dequeue(self):- for url in _URLS:- self.scheduler.enqueue_request(Request(url))+ # TODO: unify test methods using "reopen" like in DownloaderAwareSchedulerTestMixin- urls = set()- while self.scheduler.has_pending_requests():- urls.add(self.scheduler.next_request().url)- assert urls == _URLS+class TestSchedulerInMemoryBase(TestSchedulerBase):+ @coroutine_test+ async def test_length(self, jobdir: Path | None) -> None:+ async with self.create_scheduler(jobdir) as scheduler:+ assert not scheduler.has_pending_requests()+ assert len(scheduler) == 0- def test_dequeue_priorities(self):- for url, priority in _PRIORITIES:- self.scheduler.enqueue_request(Request(url, priority=priority))+ for url in _URLS:+ scheduler.enqueue_request(Request(url))- priorities = []- while self.scheduler.has_pending_requests():- priorities.append(self.scheduler.next_request().priority)+ assert scheduler.has_pending_requests()+ assert len(scheduler) == len(_URLS)- assert priorities == sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)+ @coroutine_test+ async def test_dequeue(self, jobdir: Path | None) -> None:+ async with self.create_scheduler(jobdir) as scheduler:+ for url in _URLS:+ scheduler.enqueue_request(Request(url))+ urls = set()+ while scheduler.has_pending_requests():+ request = scheduler.next_request()+ assert request is not None+ urls.add(request.url)-class TestSchedulerOnDiskBase(SchedulerHandler):- def setup_method(self):- self.jobdir = tempfile.mkdtemp()- self.create_scheduler()+ assert urls == _URLS- def teardown_method(self):- self.close_scheduler()+ @coroutine_test+ async def test_dequeue_priorities(self, jobdir: Path | None) -> None:+ async with self.create_scheduler(jobdir) as scheduler:+ for url, priority in _PRIORITIES:+ scheduler.enqueue_request(Request(url, priority=priority))- shutil.rmtree(self.jobdir)- self.jobdir = None+ priorities = []+ while scheduler.has_pending_requests():+ request = scheduler.next_request()+ assert request is not None+ priorities.append(request.priority)- def test_length(self):- assert not self.scheduler.has_pending_requests()- assert len(self.scheduler) == 0+ assert priorities == sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)- for url in _URLS:- self.scheduler.enqueue_request(Request(url))- self.close_scheduler()- self.create_scheduler()+class TestSchedulerOnDiskBase(TestSchedulerBase):+ @pytest.fixture+ def jobdir(self, tmp_path: Path) -> Path | None:+ return tmp_path- assert self.scheduler.has_pending_requests()- assert len(self.scheduler) == len(_URLS)+ @coroutine_test+ async def test_length(self, jobdir: Path | None) -> None:+ async with self.create_scheduler(jobdir) as scheduler:+ assert not scheduler.has_pending_requests()+ assert len(scheduler) == 0+ for url in _URLS:+ scheduler.enqueue_request(Request(url))- def test_dequeue(self):- for url in _URLS:- self.scheduler.enqueue_request(Request(url))+ async with self.create_scheduler(jobdir) as scheduler:+ assert scheduler.has_pending_requests()+ assert len(scheduler) == len(_URLS)- self.close_scheduler()- self.create_scheduler()+ @coroutine_test+ async def test_dequeue(self, jobdir: Path | None) -> None:+ async with self.create_scheduler(jobdir) as scheduler:+ for url in _URLS:+ scheduler.enqueue_request(Request(url))urls = set()- while self.scheduler.has_pending_requests():- urls.add(self.scheduler.next_request().url)-+ async with self.create_scheduler(jobdir) as scheduler:+ while scheduler.has_pending_requests():+ request = scheduler.next_request()+ assert request is not None+ urls.add(request.url)assert urls == _URLS- def test_dequeue_priorities(self):- for url, priority in _PRIORITIES:- self.scheduler.enqueue_request(Request(url, priority=priority))-- self.close_scheduler()- self.create_scheduler()+ @coroutine_test+ async def test_dequeue_priorities(self, jobdir: Path | None) -> None:+ async with self.create_scheduler(jobdir) as scheduler:+ for url, priority in _PRIORITIES:+ scheduler.enqueue_request(Request(url, priority=priority))priorities = []- while self.scheduler.has_pending_requests():- priorities.append(self.scheduler.next_request().priority)-+ async with self.create_scheduler(jobdir) as scheduler:+ while scheduler.has_pending_requests():+ request = scheduler.next_request()+ assert request is not None+ priorities.append(request.priority)assert priorities == sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)class TestSchedulerInMemory(TestSchedulerInMemoryBase):- @property- def priority_queue_cls(self) -> str:- return "scrapy.pqueues.ScrapyPriorityQueue"+ priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue"class TestSchedulerOnDisk(TestSchedulerOnDiskBase):- @property- def priority_queue_cls(self) -> str:- return "scrapy.pqueues.ScrapyPriorityQueue"+ priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue"_URLS_WITH_SLOTS = [@@ -246,39 +253,25 @@ _URLS_WITH_SLOTS = [class TestMigration:- # needs a reactor or an event loop for is_asyncio_available()- # (for _schedule_coro())- @pytest.mark.requires_reactor- def test_migration(self, tmpdir):- class PrevSchedulerHandler(SchedulerHandler):- jobdir = tmpdir-- @property- def priority_queue_cls(self) -> str:- return "scrapy.pqueues.ScrapyPriorityQueue"-- class NextSchedulerHandler(SchedulerHandler):- jobdir = tmpdir-- @property- def priority_queue_cls(self) -> str:- return "scrapy.pqueues.DownloaderAwarePriorityQueue"-- prev_scheduler_handler = PrevSchedulerHandler()- prev_scheduler_handler.create_scheduler()- for url in _URLS:- prev_scheduler_handler.scheduler.enqueue_request(Request(url))- prev_scheduler_handler.close_scheduler()-- next_scheduler_handler = NextSchedulerHandler()+ @coroutine_test+ async def test_migration(self, tmp_path: Path) -> None:+ async with create_scheduler(+ "scrapy.pqueues.ScrapyPriorityQueue", tmp_path+ ) as prev_scheduler:+ for url in _URLS:+ prev_scheduler.enqueue_request(Request(url))+with pytest.raises(ValueError,match="DownloaderAwarePriorityQueue accepts ``slot_startprios`` as a dict",):- next_scheduler_handler.create_scheduler()+ async with create_scheduler(+ "scrapy.pqueues.DownloaderAwarePriorityQueue", tmp_path+ ):+ pass-def _is_scheduling_fair(enqueued_slots, dequeued_slots):+def _is_scheduling_fair(enqueued_slots: list[str], dequeued_slots: list[str]) -> bool:"""We enqueued same number of requests for every slot.Assert correct order, e.g.@@ -303,39 +296,49 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots):return True-class DownloaderAwareSchedulerTestMixin:+class DownloaderAwareSchedulerTestMixin(TestSchedulerBase):reopen = False-- @property- def priority_queue_cls(self) -> str:- return "scrapy.pqueues.DownloaderAwarePriorityQueue"-- def test_logic(self):- for url, slot in _URLS_WITH_SLOTS:- request = Request(url)- request.meta[Downloader.DOWNLOAD_SLOT] = slot- self.scheduler.enqueue_request(request)+ priority_queue_cls = "scrapy.pqueues.DownloaderAwarePriorityQueue"++ @coroutine_test+ async def test_logic(self, jobdir: Path | None) -> None:+ def _setup(scheduler: Scheduler) -> None:+ for url, slot in _URLS_WITH_SLOTS:+ request = Request(url)+ request.meta[Downloader.DOWNLOAD_SLOT] = slot+ scheduler.enqueue_request(request)++ def _assert(scheduler: Scheduler) -> None:+ dequeued_slots: list[str] = []+ requests: list[Request] = []+ assert scheduler.crawler+ assert scheduler.crawler.engine+ downloader = scheduler.crawler.engine.downloader+ assert isinstance(downloader, MockDownloader)+ while scheduler.has_pending_requests():+ request = scheduler.next_request()+ assert request is not None+ slot = downloader.get_slot_key(request)+ dequeued_slots.append(slot)+ downloader.increment(slot)+ requests.append(request)++ for request in requests:+ slot = downloader.get_slot_key(request)+ downloader.decrement(slot)++ assert _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots)+ assert sum(len(s.active) for s in downloader.slots.values()) == 0if self.reopen:- self.close_scheduler()- self.create_scheduler()-- dequeued_slots = []- requests = []- downloader = self.mock_crawler.engine.downloader- while self.scheduler.has_pending_requests():- request = self.scheduler.next_request()- slot = downloader.get_slot_key(request)- dequeued_slots.append(slot)- downloader.increment(slot)- requests.append(request)-- for request in requests:- slot = downloader.get_slot_key(request)- downloader.decrement(slot)-- assert _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots)- assert sum(len(s.active) for s in downloader.slots.values()) == 0+ async with self.create_scheduler(jobdir) as scheduler:+ _setup(scheduler)+ async with self.create_scheduler(jobdir) as scheduler:+ _assert(scheduler)+ else:+ async with self.create_scheduler(jobdir) as scheduler:+ _setup(scheduler)+ _assert(scheduler)class TestSchedulerWithDownloaderAwareInMemory(diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.pyindex 3fbe9c94e..ddece52c4 100644--- a/tests/test_spidermiddleware_process_start.py+++ b/tests/test_spidermiddleware_process_start.py@@ -346,12 +346,12 @@ class TestMain:[NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware])- @pytest.mark.requires_reactor+ @pytest.mark.requires_reactor # needs a reactor for twisted_sleep()@coroutine_testasync def test_twisted_sleep_single(self):await self._test_sleep([TwistedSleepSpiderMiddleware])- @pytest.mark.requires_reactor+ @pytest.mark.requires_reactor # needs a reactor for twisted_sleep()@coroutine_testasync def test_twisted_sleep_multiple(self):await self._test_sleep(diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex 5489fd948..a871a282e 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -21,10 +21,10 @@ if TYPE_CHECKING:class TestAsyncio:- @pytest.mark.requires_reactor # needs a reactor or an event loop for is_asyncio_available()- def test_is_asyncio_available(self, reactor_pytest: str) -> None:+ @coroutine_test+ async def test_is_asyncio_available(self, reactor_pytest: str) -> None:# the result should depend only on the pytest --reactor argument- assert is_asyncio_available() == (reactor_pytest == "asyncio")+ assert is_asyncio_available() == (reactor_pytest != "default")@pytest.mark.only_asyncio@@ -104,10 +104,10 @@ class TestParallelAsyncio:assert max_parallel_count[0] <= self.CONCURRENT_ITEMS[email protected]_reactor # needs a running event loop for AsyncioLoopingCall.start()@pytest.mark.only_asyncioclass TestAsyncioLoopingCall:- def test_looping_call(self):+ @coroutine_test+ async def test_looping_call(self):func = mock.MagicMock()looping_call = AsyncioLoopingCall(func)looping_call.start(1, now=False)@@ -116,21 +116,24 @@ class TestAsyncioLoopingCall:assert not looping_call.runningassert not func.called- def test_looping_call_now(self):+ @coroutine_test+ async def test_looping_call_now(self):func = mock.MagicMock()looping_call = AsyncioLoopingCall(func)looping_call.start(1)looping_call.stop()assert func.called- def test_looping_call_already_running(self):+ @coroutine_test+ async def test_looping_call_already_running(self):looping_call = AsyncioLoopingCall(lambda: None)looping_call.start(1)with pytest.raises(RuntimeError):looping_call.start(1)looping_call.stop()- def test_looping_call_interval(self):+ @coroutine_test+ async def test_looping_call_interval(self):looping_call = AsyncioLoopingCall(lambda: None)with pytest.raises(ValueError, match="Interval must be greater than 0"):looping_call.start(0)@@ -138,7 +141,8 @@ class TestAsyncioLoopingCall:looping_call.start(-1)assert not looping_call.running- def test_looping_call_bad_function(self):+ @coroutine_test+ async def test_looping_call_bad_function(self):looping_call = AsyncioLoopingCall(Deferred)with pytest.raises(TypeError):looping_call.start(0.1)diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 1b062d28d..978c24f5a 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -19,13 +19,13 @@ from scrapy.utils.defer import (mustbe_deferred,parallel_async,)-from tests.utils.decorators import inline_callbacks_test+from tests.utils.decorators import coroutine_test, inline_callbacks_testif TYPE_CHECKING:from collections.abc import AsyncGenerator, Awaitable, Callable, Generator[email protected]_reactor[email protected]_reactor # mustbe_deferred() requires a reactor@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")class TestMustbeDeferred:@inline_callbacks_test@@ -89,9 +89,8 @@ class TestIterErrback:assert isinstance(errors[0].value, ZeroDivisionError)[email protected]_reactorclass TestAiterErrback:- @deferred_f_from_coro_f+ @coroutine_testasync def test_aiter_errback_good(self):async def itergood() -> AsyncGenerator[int, None]:for x in range(10):@@ -102,7 +101,7 @@ class TestAiterErrback:assert out == list(range(10))assert not errors- @deferred_f_from_coro_f+ @coroutine_testasync def test_iter_errback_bad(self):async def iterbad() -> AsyncGenerator[int, None]:for x in range(10):@@ -117,23 +116,18 @@ class TestAiterErrback:assert isinstance(errors[0].value, ZeroDivisionError)[email protected]_reactorclass TestAsyncDefTestsuite:- @deferred_f_from_coro_f- async def test_deferred_f_from_coro_f(self):+ @coroutine_test+ async def test_coroutine_test(self):pass- @deferred_f_from_coro_f- async def test_deferred_f_from_coro_f_generator(self):- yield-@pytest.mark.xfail(reason="Checks that the test is actually executed", strict=True)- @deferred_f_from_coro_f- async def test_deferred_f_from_coro_f_xfail(self):+ @coroutine_test+ async def test_coroutine_test_xfail(self):raise RuntimeError("This is expected to be raised")[email protected]_reactor[email protected]_reactor # parallel_async() requires a reactorclass TestParallelAsync:"""This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.@@ -327,9 +321,8 @@ class TestDeferredFFromCoroF:@pytest.mark.only_asyncio[email protected]_reactorclass TestDeferredToFuture:- @deferred_f_from_coro_f+ @coroutine_testasync def test_deferred(self):d = Deferred()result = deferred_to_future(d)@@ -338,7 +331,7 @@ class TestDeferredToFuture:future_result = await resultassert future_result == 42- @deferred_f_from_coro_f+ @coroutine_testasync def test_wrapped_coroutine(self):async def c_f() -> int:return 42@@ -349,7 +342,7 @@ class TestDeferredToFuture:future_result = await resultassert future_result == 42- @deferred_f_from_coro_f+ @coroutine_testasync def test_wrapped_coroutine_asyncio(self):async def c_f() -> int:await asyncio.sleep(0.01)@@ -363,11 +356,8 @@ class TestDeferredToFuture:@pytest.mark.only_asyncio-# needs a reactor or an event loop for is_asyncio_available()-# (for maybe_deferred_to_future())[email protected]_reactorclass TestMaybeDeferredToFutureAsyncio:- @deferred_f_from_coro_f+ @coroutine_testasync def test_deferred(self):d = Deferred()result = maybe_deferred_to_future(d)@@ -376,7 +366,7 @@ class TestMaybeDeferredToFutureAsyncio:future_result = await resultassert future_result == 42- @deferred_f_from_coro_f+ @coroutine_testasync def test_wrapped_coroutine(self):async def c_f() -> int:return 42@@ -387,7 +377,7 @@ class TestMaybeDeferredToFutureAsyncio:future_result = await resultassert future_result == 42- @deferred_f_from_coro_f+ @coroutine_testasync def test_wrapped_coroutine_asyncio(self):async def c_f() -> int:await asyncio.sleep(0.01)@@ -401,11 +391,9 @@ class TestMaybeDeferredToFutureAsyncio:@pytest.mark.only_not_asyncio-# needs a reactor or an event loop for is_asyncio_available()-# (for maybe_deferred_to_future())[email protected]_reactorclass TestMaybeDeferredToFutureNotAsyncio:- def test_deferred(self):+ @coroutine_test+ async def test_deferred(self):d = Deferred()result = maybe_deferred_to_future(d)assert isinstance(result, Deferred)diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.pyindex 3255d5940..d11fbc582 100644--- a/tests/test_utils_reactor.py+++ b/tests/test_utils_reactor.py@@ -13,12 +13,12 @@ from tests.utils.decorators import coroutine_testclass TestAsyncio:- @pytest.mark.requires_reactor+ @pytest.mark.requires_reactor # needs a reactordef test_is_asyncio_reactor_installed(self, reactor_pytest: str) -> None:# the result should depend only on the pytest --reactor argumentassert is_asyncio_reactor_installed() == (reactor_pytest == "asyncio")- @pytest.mark.requires_reactor+ @pytest.mark.requires_reactor # installs a reactordef test_install_asyncio_reactor(self):from twisted.internet import reactor as original_reactor@@ -29,7 +29,7 @@ class TestAsyncio:assert original_reactor == reactor- @pytest.mark.requires_reactor+ @pytest.mark.requires_reactor # installs a reactor@pytest.mark.only_asyncio@coroutine_testasync def test_set_asyncio_event_loop(self):diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex 7834e3b31..b14d15be4 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -30,6 +30,7 @@ from tests.mockserver.http_resources import (from tests.mockserver.utils import ssl_context_factoryfrom tests.test_core_downloader import TestContextFactoryBase+# these tests are related to the Twisted HTTP codepytestmark = pytest.mark.requires_reactordiff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.pyindex e6ff46f5c..1faa2487a 100644--- a/tests/test_zz_resources.py+++ b/tests/test_zz_resources.py@@ -9,6 +9,7 @@ import pytestfrom scrapy.utils.log import LogCounterHandlerfrom scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed+from tests.utils.decorators import coroutine_testdef test_counter_handler() -> None:@@ -31,11 +32,20 @@ def test_stderr_log_handler() -> None:assert c == 0[email protected]_reactor # needs a running event loop for asyncio.all_tasks()@pytest.mark.only_asyncio-def test_pending_asyncio_tasks() -> None:+@coroutine_test+async def test_pending_asyncio_tasks() -> None:"""Test that there are no pending asyncio tasks."""- assert not asyncio.all_tasks()+ # note that pytest-asyncio uses separate loops per function so this isn't as useful there+ tasks = []+ for t in asyncio.all_tasks():+ coro = t.get_coro()+ if (+ coro is not None+ and getattr(coro, "__name__", None) != "test_pending_asyncio_tasks"+ ):+ tasks.append(t)+ assert not tasksdef test_installed_reactor(reactor_pytest: str) -> None:diff --git a/tests/utils/__init__.py b/tests/utils/__init__.pyindex a81f2d1a2..5b7848d54 100644--- a/tests/utils/__init__.py+++ b/tests/utils/__init__.py@@ -1,17 +1,28 @@+import asyncioimport osfrom pathlib import Pathfrom twisted.internet.defer import Deferred+from scrapy.utils.asyncio import is_asyncio_available+from scrapy.utils.defer import maybe_deferred_to_future-def twisted_sleep(seconds):++def twisted_sleep(seconds: float):from twisted.internet import reactor- d = Deferred()+ d: Deferred[None] = Deferred()reactor.callLater(seconds, d.callback, None)return d+async def async_sleep(seconds: float) -> None:+ if is_asyncio_available():+ await asyncio.sleep(seconds)+ else:+ await maybe_deferred_to_future(twisted_sleep(seconds))++def get_script_run_env() -> dict[str, str]:"""Return a OS environment dict suitable to run scripts shipped with tests."""diff --git a/tests/utils/decorators.py b/tests/utils/decorators.pyindex dd9043ab1..4750158e5 100644--- a/tests/utils/decorators.py+++ b/tests/utils/decorators.py@@ -55,6 +55,11 @@ def coroutine_test(* with ``pytest-twisted`` this converts a coroutine into a:class:`twisted.internet.defer.Deferred`* with ``pytest-asyncio`` this is a no-op++ In addition to handling asynchronous test functions this can also be used+ to mark "synchronous" test functions (they still need to be made+ ``async def``) that call code that needs a reactor or a running event loop,+ so that ``pytest-asyncio`` starts a loop for them too."""if not is_reactor_installed():diff --git a/tox.ini b/tox.iniindex 1bebaf2aa..ca304330c 100644--- a/tox.ini+++ b/tox.ini@@ -11,6 +11,7 @@ minversion = 1.7.0deps =attrscoverage >= 7.10.6+ httpxpexpect >= 4.8.0pyftpdlib >= 2.0.1pygments@@ -107,6 +108,7 @@ deps =Twisted==21.7.0cryptography==37.0.0cssselect==0.9.1+ httpx==0.26.0itemadapter==0.1.0lxml==4.6.4parsel==1.5.0@@ -170,26 +172,31 @@ commands = {[pinned]commands}commands ={[testenv]commands} --reactor=default+[testenv:default-reactor-pinned]+basepython = {[pinned]basepython}+deps = {[testenv:pinned]deps}+commands = {[pinned]commands} --reactor=default+setenv =+ {[pinned]setenv}+[testenv:no-reactor]deps ={[testenv]deps}- httpxpytest-asynciocommands ={[testenv]commands} -p no:twisted --reactor=none-[testenv:default-reactor-pinned]-basepython = {[pinned]basepython}-deps = {[testenv:pinned]deps}-commands = {[pinned]commands} --reactor=default-setenv =- {[pinned]setenv}+[testenv:no-reactor-extra-deps]+deps =+ {[testenv:extra-deps]deps}+ pytest-asyncio+commands =+ {[testenv]commands} -p no:twisted --reactor=none[testenv:no-reactor-pinned]basepython = {[pinned]basepython}deps ={[testenv:pinned]deps}- httpx==0.26.0pytest-asynciocommands = {[pinned]commands} -p no:twisted --reactor=nonesetenv =