Scrapy, a fast high-level web crawling & scraping framework for Python.
Generic improvements related to reactorless tests (#6968)
Andrey Rakhmatullin552f2fb91ebd
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rstindex ad5c71fbf..ca5bc9a5c 100644--- a/docs/topics/asyncio.rst+++ b/docs/topics/asyncio.rst@@ -49,6 +49,7 @@ You can usually fix the issue by moving those offending module-level Twistedimports to the method or function definitions where they are used. For example,if you have something like:+.. skip: next.. code-block:: pythonfrom twisted.internet import reactordiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex db65fb993..4e9519eef 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -2003,6 +2003,7 @@ reactor is installed.In order to use the reactor installed by Scrapy:+.. skip: next.. code-block:: pythonimport scrapydiff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex e7e7440a8..70674c021 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -30,6 +30,7 @@ from scrapy.crawler import (from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions.throttle import AutoThrottlefrom scrapy.settings import Settings, default_settings+from scrapy.utils.asyncio import call_laterfrom scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_corofrom scrapy.utils.log import configure_logging, get_scrapy_root_handlerfrom scrapy.utils.spider import DefaultSpider@@ -932,8 +933,6 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):@inlineCallbacksdef test_shutdown_forced(self):- from twisted.internet import reactor-sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAKargs = self.get_script_args("sleeping.py", "10")p = PopenSpawn(args, timeout=5)@@ -943,7 +942,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):p.expect_exact("shutting down gracefully")# sending the second signal too fast often causes problemsd = Deferred()- reactor.callLater(0.01, d.callback, None)+ call_later(0.01, d.callback, None)yield dp.kill(sig)p.expect_exact("forcing unclean shutdown")diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.pynew file mode 100644index 000000000..a43fe318d--- /dev/null+++ b/tests/test_downloader_handler_twisted_ftp.py@@ -0,0 +1,194 @@+from __future__ import annotations++import os+import sys+from pathlib import Path+from tempfile import mkstemp+from typing import TYPE_CHECKING, Any++import pytest+from pytest_twisted import async_yield_fixture+from twisted.cred import checkers, credentials, portal++from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler+from scrapy.http import HtmlResponse, Request, Response+from scrapy.http.response.text import TextResponse+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.misc import build_from_crawler+from scrapy.utils.python import to_bytes+from scrapy.utils.spider import DefaultSpider+from scrapy.utils.test import get_crawler++if TYPE_CHECKING:+ from collections.abc import AsyncGenerator, Generator+++class TestFTPBase:+ username = "scrapy"+ password = "passwd"+ req_meta: dict[str, Any] = {"ftp_user": username, "ftp_password": password}++ test_files = (+ ("file.txt", b"I have the power!"),+ ("file with spaces.txt", b"Moooooooooo power!"),+ ("html-file-without-extension", b"<!DOCTYPE html>\n<title>.</title>"),+ )++ def _create_files(self, root: Path) -> None:+ userdir = root / self.username+ userdir.mkdir()+ for filename, content in self.test_files:+ (userdir / filename).write_bytes(content)++ def _get_factory(self, root):+ from twisted.protocols.ftp import FTPFactory, FTPRealm++ realm = FTPRealm(anonymousRoot=str(root), userHome=str(root))+ p = portal.Portal(realm)+ users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()+ users_checker.addUser(self.username, self.password)+ p.registerChecker(users_checker, credentials.IUsernamePassword)+ return FTPFactory(portal=p)++ @async_yield_fixture+ async def server_url(self, tmp_path: Path) -> AsyncGenerator[str]:+ from twisted.internet import reactor++ self._create_files(tmp_path)+ factory = self._get_factory(tmp_path)+ port = reactor.listenTCP(0, factory, interface="127.0.0.1")+ portno = port.getHost().port++ yield f"https://127.0.0.1:{portno}/"++ await port.stopListening()++ @staticmethod+ @pytest.fixture+ def dh() -> Generator[FTPDownloadHandler]:+ crawler = get_crawler()+ dh = build_from_crawler(FTPDownloadHandler, crawler)++ yield dh++ # if the test was skipped, there will be no client attribute+ if hasattr(dh, "client"):+ assert dh.client.transport+ dh.client.transport.loseConnection()++ @staticmethod+ async def download_request(dh: FTPDownloadHandler, request: Request) -> Response:+ return await maybe_deferred_to_future(+ dh.download_request(request, DefaultSpider())+ )++ @deferred_f_from_coro_f+ async def test_ftp_download_success(+ self, server_url: str, dh: FTPDownloadHandler+ ) -> None:+ request = Request(url=server_url + "file.txt", meta=self.req_meta)+ r = await self.download_request(dh, request)+ assert r.status == 200+ assert r.body == b"I have the power!"+ assert r.headers == {b"Local Filename": [b""], b"Size": [b"17"]}+ assert r.protocol is None++ @deferred_f_from_coro_f+ async def test_ftp_download_path_with_spaces(+ self, server_url: str, dh: FTPDownloadHandler+ ) -> None:+ request = Request(+ url=server_url + "file with spaces.txt",+ meta=self.req_meta,+ )+ r = await self.download_request(dh, request)+ assert r.status == 200+ assert r.body == b"Moooooooooo power!"+ assert r.headers == {b"Local Filename": [b""], b"Size": [b"18"]}++ @deferred_f_from_coro_f+ async def test_ftp_download_nonexistent(+ self, server_url: str, dh: FTPDownloadHandler+ ) -> None:+ request = Request(url=server_url + "nonexistent.txt", meta=self.req_meta)+ r = await self.download_request(dh, request)+ assert r.status == 404++ @deferred_f_from_coro_f+ async def test_ftp_local_filename(+ self, server_url: str, dh: FTPDownloadHandler+ ) -> None:+ f, local_fname = mkstemp()+ fname_bytes = to_bytes(local_fname)+ local_path = Path(local_fname)+ os.close(f)+ meta = {"ftp_local_filename": fname_bytes}+ meta.update(self.req_meta)+ request = Request(url=server_url + "file.txt", meta=meta)+ r = await self.download_request(dh, request)+ assert r.body == fname_bytes+ assert r.headers == {b"Local Filename": [fname_bytes], b"Size": [b"17"]}+ assert local_path.exists()+ assert local_path.read_bytes() == b"I have the power!"+ local_path.unlink()++ @pytest.mark.parametrize(+ ("filename", "response_class"),+ [+ ("file.txt", TextResponse),+ ("html-file-without-extension", HtmlResponse),+ ],+ )+ @deferred_f_from_coro_f+ async def test_response_class(+ self,+ filename: str,+ response_class: type[Response],+ server_url: str,+ dh: FTPDownloadHandler,+ ) -> None:+ f, local_fname = mkstemp()+ local_fname_path = Path(local_fname)+ os.close(f)+ meta = {}+ meta.update(self.req_meta)+ request = Request(url=server_url + filename, meta=meta)+ r = await self.download_request(dh, request)+ assert type(r) is response_class # pylint: disable=unidiomatic-typecheck+ local_fname_path.unlink()+++class TestFTP(TestFTPBase):+ @deferred_f_from_coro_f+ async def test_invalid_credentials(+ self, server_url: str, dh: FTPDownloadHandler, reactor_pytest: str+ ) -> None:+ if reactor_pytest == "asyncio" and sys.platform == "win32":+ pytest.skip(+ "This test produces DirtyReactorAggregateError on Windows with asyncio"+ )++ from twisted.protocols.ftp import ConnectionLost++ meta = dict(self.req_meta)+ meta.update({"ftp_password": "invalid"})+ request = Request(url=server_url + "file.txt", meta=meta)+ with pytest.raises(ConnectionLost):+ await self.download_request(dh, request)+++class TestAnonymousFTP(TestFTPBase):+ username = "anonymous"+ req_meta = {}++ def _create_files(self, root: Path) -> None:+ for filename, content in self.test_files:+ (root / filename).write_bytes(content)++ def _get_factory(self, tmp_path):+ from twisted.protocols.ftp import FTPFactory, FTPRealm++ realm = FTPRealm(anonymousRoot=str(tmp_path))+ p = portal.Portal(realm)+ p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous)+ return FTPFactory(portal=p, userAnonymous=self.username)diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 9aa53edd9..e37932fbb 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -4,35 +4,25 @@ from __future__ import annotationsimport contextlibimport os-import sysfrom pathlib import Pathfrom tempfile import mkdtemp, mkstemp-from typing import TYPE_CHECKING, Anyfrom unittest import mockimport pytest-from pytest_twisted import async_yield_fixture-from twisted.cred import checkers, credentials, portalfrom w3lib.url import path_to_file_urifrom scrapy.core.downloader.handlers import DownloadHandlersfrom scrapy.core.downloader.handlers.datauri import DataURIDownloadHandlerfrom scrapy.core.downloader.handlers.file import FileDownloadHandler-from scrapy.core.downloader.handlers.ftp import FTPDownloadHandlerfrom scrapy.core.downloader.handlers.s3 import S3DownloadHandlerfrom scrapy.exceptions import NotConfigured-from scrapy.http import HtmlResponse, Request, Response-from scrapy.http.response.text import TextResponse+from scrapy.http import Request, Responsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawler-from scrapy.utils.python import to_bytesfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler-if TYPE_CHECKING:- from collections.abc import AsyncGenerator, Generator-class DummyDH:lazy = False@@ -305,177 +295,6 @@ class TestS3:)-class TestFTPBase:- username = "scrapy"- password = "passwd"- req_meta: dict[str, Any] = {"ftp_user": username, "ftp_password": password}-- test_files = (- ("file.txt", b"I have the power!"),- ("file with spaces.txt", b"Moooooooooo power!"),- ("html-file-without-extension", b"<!DOCTYPE html>\n<title>.</title>"),- )-- def _create_files(self, root: Path) -> None:- userdir = root / self.username- userdir.mkdir()- for filename, content in self.test_files:- (userdir / filename).write_bytes(content)-- def _get_factory(self, root):- from twisted.protocols.ftp import FTPFactory, FTPRealm-- realm = FTPRealm(anonymousRoot=str(root), userHome=str(root))- p = portal.Portal(realm)- users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()- users_checker.addUser(self.username, self.password)- p.registerChecker(users_checker, credentials.IUsernamePassword)- return FTPFactory(portal=p)-- @async_yield_fixture- async def server_url(self, tmp_path: Path) -> AsyncGenerator[str]:- from twisted.internet import reactor-- self._create_files(tmp_path)- factory = self._get_factory(tmp_path)- port = reactor.listenTCP(0, factory, interface="127.0.0.1")- portno = port.getHost().port-- yield f"https://127.0.0.1:{portno}/"-- await port.stopListening()-- @staticmethod- @pytest.fixture- def dh() -> Generator[FTPDownloadHandler]:- crawler = get_crawler()- dh = build_from_crawler(FTPDownloadHandler, crawler)-- yield dh-- # if the test was skipped, there will be no client attribute- if hasattr(dh, "client"):- assert dh.client.transport- dh.client.transport.loseConnection()-- @staticmethod- async def download_request(dh: FTPDownloadHandler, request: Request) -> Response:- return await maybe_deferred_to_future(- dh.download_request(request, DefaultSpider())- )-- @deferred_f_from_coro_f- async def test_ftp_download_success(- self, server_url: str, dh: FTPDownloadHandler- ) -> None:- request = Request(url=server_url + "file.txt", meta=self.req_meta)- r = await self.download_request(dh, request)- assert r.status == 200- assert r.body == b"I have the power!"- assert r.headers == {b"Local Filename": [b""], b"Size": [b"17"]}- assert r.protocol is None-- @deferred_f_from_coro_f- async def test_ftp_download_path_with_spaces(- self, server_url: str, dh: FTPDownloadHandler- ) -> None:- request = Request(- url=server_url + "file with spaces.txt",- meta=self.req_meta,- )- r = await self.download_request(dh, request)- assert r.status == 200- assert r.body == b"Moooooooooo power!"- assert r.headers == {b"Local Filename": [b""], b"Size": [b"18"]}-- @deferred_f_from_coro_f- async def test_ftp_download_nonexistent(- self, server_url: str, dh: FTPDownloadHandler- ) -> None:- request = Request(url=server_url + "nonexistent.txt", meta=self.req_meta)- r = await self.download_request(dh, request)- assert r.status == 404-- @deferred_f_from_coro_f- async def test_ftp_local_filename(- self, server_url: str, dh: FTPDownloadHandler- ) -> None:- f, local_fname = mkstemp()- fname_bytes = to_bytes(local_fname)- local_path = Path(local_fname)- os.close(f)- meta = {"ftp_local_filename": fname_bytes}- meta.update(self.req_meta)- request = Request(url=server_url + "file.txt", meta=meta)- r = await self.download_request(dh, request)- assert r.body == fname_bytes- assert r.headers == {b"Local Filename": [fname_bytes], b"Size": [b"17"]}- assert local_path.exists()- assert local_path.read_bytes() == b"I have the power!"- local_path.unlink()-- @pytest.mark.parametrize(- ("filename", "response_class"),- [- ("file.txt", TextResponse),- ("html-file-without-extension", HtmlResponse),- ],- )- @deferred_f_from_coro_f- async def test_response_class(- self,- filename: str,- response_class: type[Response],- server_url: str,- dh: FTPDownloadHandler,- ) -> None:- f, local_fname = mkstemp()- local_fname_path = Path(local_fname)- os.close(f)- meta = {}- meta.update(self.req_meta)- request = Request(url=server_url + filename, meta=meta)- r = await self.download_request(dh, request)- assert type(r) is response_class # pylint: disable=unidiomatic-typecheck- local_fname_path.unlink()---class TestFTP(TestFTPBase):- @deferred_f_from_coro_f- async def test_invalid_credentials(- self, server_url: str, dh: FTPDownloadHandler, reactor_pytest: str- ) -> None:- if reactor_pytest == "asyncio" and sys.platform == "win32":- pytest.skip(- "This test produces DirtyReactorAggregateError on Windows with asyncio"- )-- from twisted.protocols.ftp import ConnectionLost-- meta = dict(self.req_meta)- meta.update({"ftp_password": "invalid"})- request = Request(url=server_url + "file.txt", meta=meta)- with pytest.raises(ConnectionLost):- await self.download_request(dh, request)---class TestAnonymousFTP(TestFTPBase):- username = "anonymous"- req_meta = {}-- def _create_files(self, root: Path) -> None:- for filename, content in self.test_files:- (root / filename).write_bytes(content)-- def _get_factory(self, tmp_path):- from twisted.protocols.ftp import FTPFactory, FTPRealm-- realm = FTPRealm(anonymousRoot=str(tmp_path))- p = portal.Portal(realm)- p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous)- return FTPFactory(portal=p, userAnonymous=self.username)--class TestDataURI:def setup_method(self):crawler = get_crawler()diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex bbaf9696b..3784d0933 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -17,6 +17,7 @@ from twisted.web.http import _DataLossfrom scrapy.http import Headers, HtmlResponse, Request, Response, TextResponsefrom scrapy.spiders import Spider+from scrapy.utils.asyncio import call_laterfrom scrapy.utils.defer import (deferred_f_from_coro_f,deferred_from_coro,@@ -305,8 +306,6 @@ class TestHttp11Base(TestHttpBase):async def test_download_with_maxsize_very_large_file(self, mockserver: MockServer, download_handler: DownloadHandlerProtocol) -> None:- from twisted.internet import reactor-# TODO: the logger check is specific to scrapy.core.downloader.handlers.http11with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger:request = Request(@@ -326,7 +325,7 @@ class TestHttp11Base(TestHttpBase):# after closing the connection.d: defer.Deferred[mock.Mock] = defer.Deferred()d.addCallback(check)- reactor.callLater(0.1, d.callback, logger)+ call_later(0.1, d.callback, logger)await maybe_deferred_to_future(d)@deferred_f_from_coro_fdiff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.pyindex 2bb5d2ff4..6915e1463 100644--- a/tests/test_engine_loop.py+++ b/tests/test_engine_loop.py@@ -138,7 +138,6 @@ class TestRequestSendOrder:def get_num(self, request_or_response: Request | Response):return int(request_or_response.url.rsplit("&", maxsplit=1)[1])- @deferred_f_from_coro_fasync def _test_request_order(self,start_nums,@@ -218,14 +217,12 @@ class TestRequestSendOrder:returnyield- await maybe_deferred_to_future(- self._test_request_order(- start_nums=nums,- settings={"CONCURRENT_REQUESTS": 1},- response_seconds=response_seconds,- start_fn=start,- parse_fn=parse,- )+ await self._test_request_order(+ start_nums=nums,+ settings={"CONCURRENT_REQUESTS": 1},+ response_seconds=response_seconds,+ start_fn=start,+ parse_fn=parse,)@deferred_f_from_coro_f@@ -260,17 +257,15 @@ class TestRequestSendOrder:returnyield- await maybe_deferred_to_future(- self._test_request_order(- start_nums=nums,- settings={- "CONCURRENT_REQUESTS": 1,- "SCHEDULER_START_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue",- },- response_seconds=response_seconds,- start_fn=start,- parse_fn=parse,- )+ await self._test_request_order(+ start_nums=nums,+ settings={+ "CONCURRENT_REQUESTS": 1,+ "SCHEDULER_START_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue",+ },+ response_seconds=response_seconds,+ start_fn=start,+ parse_fn=parse,)@deferred_f_from_coro_f@@ -320,17 +315,15 @@ class TestRequestSendOrder:returnyield- await maybe_deferred_to_future(- self._test_request_order(- start_nums=nums,- settings={- "CONCURRENT_REQUESTS": 1,- "SCHEDULER_START_MEMORY_QUEUE": None,- },- response_seconds=response_seconds,- start_fn=start,- parse_fn=parse,- )+ await self._test_request_order(+ start_nums=nums,+ settings={+ "CONCURRENT_REQUESTS": 1,+ "SCHEDULER_START_MEMORY_QUEUE": None,+ },+ response_seconds=response_seconds,+ start_fn=start,+ parse_fn=parse,)# Examples from the “Start requests” section of the documentation about@@ -350,14 +343,12 @@ class TestRequestSendOrder:request = self.request(num, response_seconds, download_slots)yield request- await maybe_deferred_to_future(- self._test_request_order(- start_nums=start_nums,- cb_nums=cb_nums,- settings={- "CONCURRENT_REQUESTS": 1,- },- response_seconds=response_seconds,- start_fn=start,- )+ await self._test_request_order(+ start_nums=start_nums,+ cb_nums=cb_nums,+ settings={+ "CONCURRENT_REQUESTS": 1,+ },+ response_seconds=response_seconds,+ start_fn=start,)diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 3247da61b..d100d6b71 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -44,6 +44,11 @@ if TYPE_CHECKING:from scrapy.core.http2.protocol import H2ClientProtocol+pytestmark = pytest.mark.skipif(+ not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"+)++def generate_random_string(size: int) -> str:return "".join(random.choices(string.ascii_uppercase + string.digits, k=size))@@ -187,7 +192,6 @@ async def make_request(client: H2ClientProtocol, request: Request) -> Response:return await maybe_deferred_to_future(make_request_dfd(client, request))[email protected](not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled")class TestHttps2ClientProtocol:scheme = "https"host = "localhost"diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex b309c5d8e..42ae897e8 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -14,6 +14,7 @@ from scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.files import FileExceptionfrom scrapy.pipelines.media import MediaPipelinefrom scrapy.spiders import Spider+from scrapy.utils.asyncio import call_laterfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.signal import disconnect_allfrom scrapy.utils.test import get_crawler@@ -337,10 +338,8 @@ class TestMediaPipeline(TestBaseMediaPipeline):rsp1 = Response("http://url")def rsp1_func():- from twisted.internet import reactor-dfd = Deferred().addCallback(_check_downloading)- reactor.callLater(0.1, dfd.callback, rsp1)+ call_later(0.1, dfd.callback, rsp1)return dfddef rsp2_func():diff --git a/tests/test_pipelines.py b/tests/test_pipelines.pyindex 064033f01..cba6a713c 100644--- a/tests/test_pipelines.py+++ b/tests/test_pipelines.py@@ -4,6 +4,7 @@ import pytestfrom twisted.internet.defer import Deferred, inlineCallbacksfrom scrapy import Request, Spider, signals+from scrapy.utils.asyncio import call_laterfrom scrapy.utils.defer import deferred_to_future, maybe_deferred_to_futurefrom scrapy.utils.test import get_crawler, get_from_asyncio_queuefrom tests.mockserver.http import MockServer@@ -30,9 +31,7 @@ class DeferredPipeline:class AsyncDefPipeline:async def process_item(self, item, spider):d = Deferred()- from twisted.internet import reactor-- reactor.callLater(0, d.callback, None)+ call_later(0, d.callback, None)await maybe_deferred_to_future(d)item["pipeline_passed"] = Truereturn item@@ -41,9 +40,8 @@ class AsyncDefPipeline:class AsyncDefAsyncioPipeline:async def process_item(self, item, spider):d = Deferred()- from twisted.internet import reactor-- reactor.callLater(0, d.callback, None)+ loop = asyncio.get_event_loop()+ loop.call_later(0, d.callback, None)await deferred_to_future(d)await asyncio.sleep(0.2)item["pipeline_passed"] = await get_from_asyncio_queue(True)diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex b0f69c1ff..bc2404d8b 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -14,6 +14,7 @@ from scrapy.exceptions import _InvalidOutputfrom scrapy.http import Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.asyncgen import collect_asyncgen+from scrapy.utils.asyncio import call_laterfrom scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_futurefrom scrapy.utils.test import get_crawler@@ -220,9 +221,7 @@ class ProcessSpiderExceptionAsyncIteratorMiddleware:async def process_spider_exception(self, response, exception, spider):yield {"foo": 1}d = defer.Deferred()- from twisted.internet import reactor-- reactor.callLater(0, d.callback, None)+ call_later(0, d.callback, None)await maybe_deferred_to_future(d)yield {"foo": 2}yield {"foo": 3}diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.pyindex d90d3ed95..6e4bdb49a 100644--- a/tests/test_utils_signal.py+++ b/tests/test_utils_signal.py@@ -7,6 +7,7 @@ from twisted.internet import deferfrom twisted.internet.defer import inlineCallbacksfrom twisted.python.failure import Failure+from scrapy.utils.asyncio import call_laterfrom scrapy.utils.defer import deferred_from_corofrom scrapy.utils.signal import (send_catch_log,@@ -65,12 +66,10 @@ class TestSendCatchLogDeferred(TestSendCatchLog):class TestSendCatchLogDeferred2(TestSendCatchLogDeferred):def ok_handler(self, arg, handlers_called):- from twisted.internet import reactor-handlers_called.add(self.ok_handler)assert arg == "test"d = defer.Deferred()- reactor.callLater(0, d.callback, "OK")+ call_later(0, d.callback, "OK")return d@@ -98,12 +97,10 @@ class TestSendCatchLogAsync(TestSendCatchLog):class TestSendCatchLogAsync2(TestSendCatchLogAsync):def ok_handler(self, arg, handlers_called):- from twisted.internet import reactor-handlers_called.add(self.ok_handler)assert arg == "test"d = defer.Deferred()- reactor.callLater(0, d.callback, "OK")+ call_later(0, d.callback, "OK")return d