Scrapy, a fast high-level web crawling & scraping framework for Python.
Add `AsyncCrawlerProcess` and `Crawler.crawl_async()` (#6817)
Andrey Rakhmatullin05b3b205ce29
diff --git a/conftest.py b/conftest.pyindex 18132b7e6..ed7d14166 100644--- a/conftest.py+++ b/conftest.py@@ -19,6 +19,8 @@ collect_ignore = ["tests/mockserver.py","tests/pipelines.py","tests/spiders.py",+ # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess+ *_py_files("tests/AsyncCrawlerProcess"),# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess*_py_files("tests/AsyncCrawlerRunner"),# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocessdiff --git a/docs/topics/api.rst b/docs/topics/api.rstindex 3e7bc45c5..b11de2914 100644--- a/docs/topics/api.rst+++ b/docs/topics/api.rst@@ -99,13 +99,11 @@ how you :ref:`configure the downloader middlewaresprovided while constructing the crawler, and it is created after thearguments given in the :meth:`crawl` method.- .. method:: crawl(*args, **kwargs)+ .. automethod:: crawl_async- Starts the crawler by instantiating its spider class with the given- ``args`` and ``kwargs`` arguments, while setting the execution engine in- motion. Should be called only once.+ .. automethod:: crawl- Returns a deferred that is fired when the crawl is finished.+ .. automethod:: stop_async.. automethod:: stop@@ -115,6 +113,11 @@ how you :ref:`configure the downloader middlewares.. autoclass:: CrawlerRunner:members:+.. autoclass:: AsyncCrawlerProcess+ :show-inheritance:+ :members:+ :inherited-members:+.. autoclass:: CrawlerProcess:show-inheritance::members:diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rstindex 473ef7bfa..ad5c71fbf 100644--- a/docs/topics/asyncio.rst+++ b/docs/topics/asyncio.rst@@ -20,7 +20,8 @@ To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needsto be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``,which is the default value.-If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to+If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or+:class:`~scrapy.crawler.CrawlerRunner`, you also need toinstall the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`reactor manually. You can do that using:func:`~scrapy.utils.reactor.install_reactor`:@@ -169,4 +170,8 @@ Switching to a non-asyncio reactorIf for some reason your code doesn't work with the asyncio reactor, you can usea different reactor by setting the :setting:`TWISTED_REACTOR` setting to itsimport path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to-``None``, which will use the default reactor for your platform.+``None``, which will use the default reactor for your platform. If you are+using :class:`~scrapy.crawler.AsyncCrawlerRunner` or+:class:`~scrapy.crawler.AsyncCrawlerProcess` you also need to switch to their+Deferred-based counterparts: :class:`~scrapy.crawler.CrawlerRunner` or+:class:`~scrapy.crawler.CrawlerProcess` respectively.diff --git a/docs/topics/practices.rst b/docs/topics/practices.rstindex 18005aaf2..56177ba4e 100644--- a/docs/topics/practices.rst+++ b/docs/topics/practices.rst@@ -21,16 +21,21 @@ Remember that Scrapy is built on top of the Twistedasynchronous networking library, so you need to run it inside the Twisted reactor.The first utility you can use to run your spiders is-:class:`scrapy.crawler.CrawlerProcess`. This class will start a Twisted reactor-for you, configuring the logging and setting shutdown handlers. This class is-the one used by all Scrapy commands.+:class:`scrapy.crawler.AsyncCrawlerProcess` or+:class:`scrapy.crawler.CrawlerProcess`. These classes will start a Twisted+reactor for you, configuring the logging and setting shutdown handlers. These+classes are the ones used by all Scrapy commands. They have similar+functionality, differing in their asynchronous API style:+:class:`~scrapy.crawler.AsyncCrawlerProcess` returns coroutines from its+asynchronous methods while :class:`~scrapy.crawler.CrawlerProcess` returns+:class:`~twisted.internet.defer.Deferred` objects.Here's an example showing how to run a single spider with it... code-block:: pythonimport scrapy- from scrapy.crawler import CrawlerProcess+ from scrapy.crawler import AsyncCrawlerProcessclass MySpider(scrapy.Spider):@@ -38,7 +43,7 @@ Here's an example showing how to run a single spider with it....- process = CrawlerProcess(+ process = AsyncCrawlerProcess(settings={"FEEDS": {"items.json": {"format": "json"},@@ -49,53 +54,57 @@ Here's an example showing how to run a single spider with it.process.crawl(MySpider)process.start() # the script will block here until the crawling is finished-Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess`+You can define :ref:`settings <topics-settings>` within the dictionary passed+to :class:`~scrapy.crawler.AsyncCrawlerProcess`. Make sure to check the+:class:`~scrapy.crawler.AsyncCrawlerProcess`documentation to get acquainted with its usage details.If you are inside a Scrapy project there are some additional helpers you canuse to import those components within the project. You can automatically import-your spiders passing their name to :class:`~scrapy.crawler.CrawlerProcess`, and-use ``get_project_settings`` to get a :class:`~scrapy.settings.Settings`-instance with your project settings.+your spiders passing their name to+:class:`~scrapy.crawler.AsyncCrawlerProcess`, and use+:func:`scrapy.utils.project.get_project_settings` to get a+:class:`~scrapy.settings.Settings` instance with your project settings.What follows is a working example of how to do that, using the `testspiders`_project as example... code-block:: python- from scrapy.crawler import CrawlerProcess+ from scrapy.crawler import AsyncCrawlerProcessfrom scrapy.utils.project import get_project_settings- process = CrawlerProcess(get_project_settings())+ process = AsyncCrawlerProcess(get_project_settings())# 'followall' is the name of one of the spiders of the project.process.crawl("followall", domain="scrapy.org")process.start() # the script will block here until the crawling is finishedThere's another Scrapy utility that provides more control over the crawling-process: :class:`scrapy.crawler.AsyncCrawlerRunner` and+process: :class:`scrapy.crawler.AsyncCrawlerRunner` or:class:`scrapy.crawler.CrawlerRunner`. These classes are thin wrappersthat encapsulate some simple helpers to run multiple crawlers, but they won't-start or interfere with existing reactors in any way. They have similar-functionality, differing in their asynchronous API style:-:class:`~scrapy.crawler.AsyncCrawlerRunner` returns coroutines from its-asynchronous methods while :class:`~scrapy.crawler.CrawlerRunner` returns-:class:`~twisted.internet.defer.Deferred` objects.+start or interfere with existing reactors in any way. Just like+:class:`scrapy.crawler.AsyncCrawlerProcess` and+:class:`scrapy.crawler.CrawlerProcess` they differ in their asynchronous API+style.When using these classes the reactor should be explicitly run after schedulingyour spiders. It's recommended that you use:class:`~scrapy.crawler.AsyncCrawlerRunner` or:class:`~scrapy.crawler.CrawlerRunner` instead of+:class:`~scrapy.crawler.AsyncCrawlerProcess` or:class:`~scrapy.crawler.CrawlerProcess` if your application is already usingTwisted and you want to run Scrapy in the same reactor.If you want to stop the reactor or run any other code right after the spider-finishes you can do that after the :meth:`AsyncCrawlerRunner.crawl()-<scrapy.crawler.AsyncCrawlerRunner.crawl>` coroutine completes (or the Deferred-returned from :meth:`CrawlerRunner.crawl()+finishes you can do that after the task returned from+:meth:`AsyncCrawlerRunner.crawl() <scrapy.crawler.AsyncCrawlerRunner.crawl>`+completes (or the Deferred returned from :meth:`CrawlerRunner.crawl()<scrapy.crawler.CrawlerRunner.crawl>` fires). In the simplest case you can alsouse :func:`twisted.internet.task.react` to start and stop the reactor, though-it may be easier to just use :class:`~scrapy.crawler.CrawlerProcess` instead.+it may be easier to just use :class:`~scrapy.crawler.AsyncCrawlerProcess` or+:class:`~scrapy.crawler.CrawlerProcess` instead.Here's an example of using :class:`~scrapy.crawler.AsyncCrawlerRunner` togetherwith simple reactor management code:@@ -171,7 +180,7 @@ Here is an example that runs multiple spiders simultaneously:.. code-block:: pythonimport scrapy- from scrapy.crawler import CrawlerProcess+ from scrapy.crawler import AsyncCrawlerProcessfrom scrapy.utils.project import get_project_settings@@ -186,7 +195,7 @@ Here is an example that runs multiple spiders simultaneously:settings = get_project_settings()- process = CrawlerProcess(settings)+ process = AsyncCrawlerProcess(settings)process.crawl(MySpider1)process.crawl(MySpider2)process.start() # the script will block here until all crawling jobs are finisheddiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 0f81a0c0a..68c5079cf 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -330,7 +330,8 @@ exception is raised.These settings are:-- :setting:`ASYNCIO_EVENT_LOOP`+- :setting:`ASYNCIO_EVENT_LOOP` (not possible to set per-spider when using+ :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below)- :setting:`DNS_RESOLVER` and settings used by the correspondingcomponent, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE`@@ -338,12 +339,25 @@ These settings are:- :setting:`REACTOR_THREADPOOL_MAXSIZE`-- :setting:`TWISTED_REACTOR`+- :setting:`TWISTED_REACTOR` (ignored when using+ :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below):setting:`ASYNCIO_EVENT_LOOP` and :setting:`TWISTED_REACTOR` are used uponinstalling the reactor. The rest of the settings are applied when startingthe reactor.+There is an additional restriction for :setting:`TWISTED_REACTOR` and+:setting:`ASYNCIO_EVENT_LOOP` when using+:class:`~scrapy.crawler.AsyncCrawlerProcess`: when this class is instantiated,+it installs :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`,+ignoring the value of :setting:`TWISTED_REACTOR` and using the value of+:setting:`ASYNCIO_EVENT_LOOP` that was passed to+:meth:`AsyncCrawlerProcess.__init__()+<scrapy.crawler.AsyncCrawlerProcess.__init__>`. If a different value for+:setting:`TWISTED_REACTOR` or :setting:`ASYNCIO_EVENT_LOOP` is provided later,+e.g. in :ref:`per-spider settings <spider-settings>`, an exception will be+raised.+.. _topics-settings-ref:@@ -1977,9 +1991,11 @@ Import path of a given :mod:`~twisted.internet.reactor`.Scrapy will install this reactor if no other reactor is installed yet, such aswhen the ``scrapy`` CLI program is invoked or when using the+:class:`~scrapy.crawler.AsyncCrawlerProcess` class or the:class:`~scrapy.crawler.CrawlerProcess` class.-If you are using the :class:`~scrapy.crawler.CrawlerRunner` class, you also+If you are using the :class:`~scrapy.crawler.AsyncCrawlerRunner` class or the+:class:`~scrapy.crawler.CrawlerRunner` class, you alsoneed to install the correct reactor manually. You can do that using:func:`~scrapy.utils.reactor.install_reactor`:@@ -1988,12 +2004,12 @@ need to install the correct reactor manually. You can do that usingIf a reactor is already installed,:func:`~scrapy.utils.reactor.install_reactor` has no effect.-:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises-:exc:`Exception` if the installed reactor does not match the+:class:`~scrapy.crawler.AsyncCrawlerRunner` and other similar classes raise an+exception if the installed reactor does not match the:setting:`TWISTED_REACTOR` setting; therefore, having top-level:mod:`~twisted.internet.reactor` imports in project files and imported-third-party libraries will make Scrapy raise :exc:`Exception` when-it checks which reactor is installed.+third-party libraries will make Scrapy raise an exception when it checks which+reactor is installed.In order to use the reactor installed by Scrapy:@@ -2025,7 +2041,7 @@ In order to use the reactor installed by Scrapy:self.crawler.engine.close_spider(self, "timeout")-which raises :exc:`Exception`, becomes:+which raises an exception, becomes:.. code-block:: pythondiff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex b0d9a5452..721c81d81 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -22,6 +22,7 @@ from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequestfrom scrapy.http import Request, Responsefrom scrapy.utils.defer import (deferred_f_from_coro_f,+ deferred_from_coro,maybe_deferred_to_future,)from scrapy.utils.log import failure_to_exc_info, logformatter_adapter@@ -122,8 +123,10 @@ class ExecutionEngine:)return scheduler_cls- @deferred_f_from_coro_f- async def start(self, _start_request_processing=True) -> None:+ def start(self, _start_request_processing=True) -> Deferred[None]:+ return deferred_from_coro(self.start_async(_start_request_processing))++ async def start_async(self, _start_request_processing=True) -> None:if self.running:raise RuntimeError("Engine already running")self.start_time = time()@@ -392,10 +395,15 @@ class ExecutionEngine:finally:self._slot.nextcall.schedule()- @deferred_f_from_coro_f- async def open_spider(+ def open_spider(self, spider: Spider, close_if_idle: bool = True) -> Deferred[None]:+ return deferred_from_coro(+ self.open_spider_async(spider, close_if_idle=close_if_idle)+ )++ async def open_spider_async(self,spider: Spider,+ *,close_if_idle: bool = True,) -> None:if self._slot is not None:diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 1d6532fa9..c22b8603b 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -21,7 +21,8 @@ from scrapy.extension import ExtensionManagerfrom scrapy.interfaces import ISpiderLoaderfrom scrapy.settings import BaseSettings, Settings, overridden_settingsfrom scrapy.signalmanager import SignalManager-from scrapy.utils.defer import deferred_to_future+from scrapy.utils.asyncio import is_asyncio_available+from scrapy.utils.defer import deferred_from_coro, deferred_to_futurefrom scrapy.utils.log import (LogCounterHandler,configure_logging,@@ -33,8 +34,10 @@ from scrapy.utils.log import (from scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.ossignal import install_shutdown_handlers, signal_namesfrom scrapy.utils.reactor import (+ _asyncio_reactor_path,install_reactor,is_asyncio_reactor_installed,+ is_reactor_installed,verify_installed_asyncio_event_loop,verify_installed_reactor,)@@ -142,6 +145,12 @@ class Crawler:# this method.@inlineCallbacksdef crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]:+ """Start the crawler by instantiating its spider class with the given+ *args* and *kwargs* arguments, while setting the execution engine in+ motion. Should be called only once.++ Return a deferred that is fired when the crawl is finished.+ """if self.crawling:raise RuntimeError("Crawling already taking place")if self._started:@@ -163,6 +172,42 @@ class Crawler:yield self.engine.close()raise+ async def crawl_async(self, *args: Any, **kwargs: Any) -> None:+ """Start the crawler by instantiating its spider class with the given+ *args* and *kwargs* arguments, while setting the execution engine in+ motion. Should be called only once.++ .. versionadded:: VERSION++ Complete when the crawl is finished.++ This function requires+ :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be+ installed.+ """+ if not is_asyncio_available():+ raise RuntimeError("Crawler.crawl_async() requires AsyncioSelectorReactor.")+ if self.crawling:+ raise RuntimeError("Crawling already taking place")+ if self._started:+ raise RuntimeError(+ "Cannot run Crawler.crawl_async() more than once on the same instance."+ )+ self.crawling = self._started = True++ try:+ self.spider = self._create_spider(*args, **kwargs)+ self._apply_settings()+ self._update_root_log_handler()+ self.engine = self._create_engine()+ await self.engine.open_spider_async(self.spider)+ await self.engine.start_async()+ except Exception:+ self.crawling = False+ if self.engine is not None:+ await deferred_to_future(self.engine.close())+ raise+def _create_spider(self, *args: Any, **kwargs: Any) -> Spider:return self.spidercls.from_crawler(self, *args, **kwargs)@@ -171,13 +216,26 @@ class Crawler:@inlineCallbacksdef stop(self) -> Generator[Deferred[Any], Any, None]:- """Starts a graceful stop of the crawler and returns a deferred that is+ """Start a graceful stop of the crawler and return a deferred that isfired when the crawler is stopped."""if self.crawling:self.crawling = Falseassert self.engineyield self.engine.stop()+ async def stop_async(self) -> None:+ """Start a graceful stop of the crawler and complete when the crawler is stopped.++ .. versionadded:: VERSION++ This function requires+ :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be+ installed.+ """+ if not is_asyncio_available():+ raise RuntimeError("Crawler.stop_async() requires AsyncioSelectorReactor.")+ await deferred_to_future(self.stop())+@staticmethoddef _get_component(component_class: type[_T], components: Iterable[Any]@@ -318,9 +376,6 @@ class CrawlerRunnerBase:spidercls = self.spider_loader.load(spidercls)return Crawler(spidercls, self.settings)- def _stop(self) -> Deferred[Any]:- return DeferredList([c.stop() for c in list(self.crawlers)])-class CrawlerRunner(CrawlerRunnerBase):"""@@ -397,7 +452,7 @@ class CrawlerRunner(CrawlerRunnerBase):Returns a deferred that is fired when they all have ended."""- return self._stop()+ return DeferredList(c.stop() for c in self.crawlers)@inlineCallbacksdef join(self) -> Generator[Deferred[Any], Any, None]:@@ -429,14 +484,14 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):def __init__(self, settings: dict[str, Any] | Settings | None = None):super().__init__(settings)- self._active: set[asyncio.Future[None]] = set()+ self._active: set[asyncio.Task[None]] = set()def crawl(self,crawler_or_spidercls: type[Spider] | str | Crawler,*args: Any,**kwargs: Any,- ) -> asyncio.Future[None]:+ ) -> asyncio.Task[None]:"""Run a crawler with the provided arguments.@@ -447,7 +502,7 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):instance, this method will try to create one using this parameter asthe spider class given to it.- Returns a :class:`~asyncio.Future` object which completes when the+ Returns a :class:`~asyncio.Task` object which completes when thecrawling is finished.:param crawler_or_spidercls: already created crawler, or a spider class@@ -465,24 +520,27 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):"it must be a spider class (or a Crawler object)")if not is_asyncio_reactor_installed():- raise RuntimeError("AsyncCrawlerRunner requires AsyncioSelectorReactor.")+ raise RuntimeError(+ f"{type(self).__name__} requires AsyncioSelectorReactor."+ )crawler = self.create_crawler(crawler_or_spidercls)return self._crawl(crawler, *args, **kwargs)- def _crawl(- self, crawler: Crawler, *args: Any, **kwargs: Any- ) -> asyncio.Future[None]:+ def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]:+ # At this point the asyncio loop has been installed either by the user+ # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()).+ loop = asyncio.get_event_loop()self.crawlers.add(crawler)- future = deferred_to_future(crawler.crawl(*args, **kwargs))- self._active.add(future)+ task = loop.create_task(crawler.crawl_async(*args, **kwargs))+ self._active.add(task)- def _done(_: asyncio.Future[None]) -> None:+ def _done(_: asyncio.Task[None]) -> None:self.crawlers.discard(crawler)- self._active.discard(future)+ self._active.discard(task)self.bootstrap_failed |= not getattr(crawler, "spider", None)- future.add_done_callback(_done)- return future+ task.add_done_callback(_done)+ return taskasync def stop(self) -> None:"""@@ -490,7 +548,10 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):Completes when they all have ended."""- await deferred_to_future(self._stop())+ if self.crawlers:+ await asyncio.wait(+ [asyncio.create_task(c.stop_async()) for c in self.crawlers]+ )async def join(self) -> None:"""@@ -498,33 +559,10 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):executions."""while self._active:- await asyncio.gather(*self._active)---class CrawlerProcess(CrawlerRunner):- """- A class to run multiple scrapy crawlers in a process simultaneously.-- This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support- for starting a :mod:`~twisted.internet.reactor` and handling shutdown- signals, like the keyboard interrupt command Ctrl-C. It also configures- top-level logging.+ await asyncio.wait(self._active)- This utility should be a better fit than- :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another- :mod:`~twisted.internet.reactor` within your application.-- The CrawlerProcess object must be instantiated with a- :class:`~scrapy.settings.Settings` object.-- :param install_root_handler: whether to install root logging handler- (default: True)-- This class shouldn't be needed (since Scrapy is responsible of using it- accordingly) unless writing scripts that manually handle the crawling- process. See :ref:`run-from-script` for an example.- """+class CrawlerProcessBase(CrawlerRunnerBase):def __init__(self,settings: dict[str, Any] | Settings | None = None,@@ -533,7 +571,6 @@ class CrawlerProcess(CrawlerRunner):super().__init__(settings)configure_logging(self.settings, install_root_handler)log_scrapy_info(self.settings)- self._initialized_reactor: bool = Falsedef _signal_shutdown(self, signum: int, _: Any) -> None:from twisted.internet import reactor@@ -556,6 +593,75 @@ class CrawlerProcess(CrawlerRunner):)reactor.callFromThread(self._stop_reactor)+ def _setup_reactor(self, install_signal_handlers: bool) -> None:+ from twisted.internet import reactor++ resolver_class = load_object(self.settings["DNS_RESOLVER"])+ # We pass self, which is CrawlerProcess, instead of Crawler here,+ # which works because the default resolvers only use crawler.settings.+ resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type]+ resolver.install_on_reactor()+ tp = reactor.getThreadPool()+ tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))+ reactor.addSystemEventTrigger("before", "shutdown", self._stop_dfd)+ if install_signal_handlers:+ reactor.addSystemEventTrigger(+ "after", "startup", install_shutdown_handlers, self._signal_shutdown+ )++ def _stop_dfd(self) -> Deferred[Any]:+ raise NotImplementedError++ @inlineCallbacks+ def _graceful_stop_reactor(self) -> Generator[Deferred[Any], Any, None]:+ try:+ yield self._stop_dfd()+ finally:+ self._stop_reactor()++ def _stop_reactor(self, _: Any = None) -> None:+ from twisted.internet import reactor++ # raised if already stopped or in shutdown stage+ with contextlib.suppress(RuntimeError):+ reactor.stop()+++class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):+ """+ A class to run multiple scrapy crawlers in a process simultaneously.++ This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support+ for starting a :mod:`~twisted.internet.reactor` and handling shutdown+ signals, like the keyboard interrupt command Ctrl-C. It also configures+ top-level logging.++ This utility should be a better fit than+ :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another+ :mod:`~twisted.internet.reactor` within your application.++ The CrawlerProcess object must be instantiated with a+ :class:`~scrapy.settings.Settings` object.++ :param install_root_handler: whether to install root logging handler+ (default: True)++ This class shouldn't be needed (since Scrapy is responsible of using it+ accordingly) unless writing scripts that manually handle the crawling+ process. See :ref:`run-from-script` for an example.++ This class provides Deferred-based APIs. Use :class:`AsyncCrawlerProcess`+ for modern coroutine APIs.+ """++ def __init__(+ self,+ settings: dict[str, Any] | Settings | None = None,+ install_root_handler: bool = True,+ ):+ super().__init__(settings, install_root_handler)+ self._initialized_reactor: bool = False+def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler:if isinstance(spidercls, str):spidercls = self.spider_loader.load(spidercls)@@ -563,6 +669,9 @@ class CrawlerProcess(CrawlerRunner):self._initialized_reactor = Truereturn Crawler(spidercls, self.settings, init_reactor=init_reactor)+ def _stop_dfd(self) -> Deferred[Any]:+ return self.stop()+def start(self, stop_after_crawl: bool = True, install_signal_handlers: bool = True) -> None:@@ -589,30 +698,86 @@ class CrawlerProcess(CrawlerRunner):returnd.addBoth(self._stop_reactor)- resolver_class = load_object(self.settings["DNS_RESOLVER"])- # We pass self, which is CrawlerProcess, instead of Crawler here,- # which works because the default resolvers only use crawler.settings.- resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type]- resolver.install_on_reactor()- tp = reactor.getThreadPool()- tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))- reactor.addSystemEventTrigger("before", "shutdown", self.stop)- if install_signal_handlers:- reactor.addSystemEventTrigger(- "after", "startup", install_shutdown_handlers, self._signal_shutdown- )+ self._setup_reactor(install_signal_handlers)reactor.run(installSignalHandlers=install_signal_handlers) # blocking call- @inlineCallbacks- def _graceful_stop_reactor(self) -> Generator[Deferred[Any], Any, None]:- try:- yield self.stop()- finally:- self._stop_reactor()- def _stop_reactor(self, _: Any = None) -> None:+class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):+ """+ A class to run multiple scrapy crawlers in a process simultaneously.++ This class extends :class:`~scrapy.crawler.AsyncCrawlerRunner` by adding support+ for starting a :mod:`~twisted.internet.reactor` and handling shutdown+ signals, like the keyboard interrupt command Ctrl-C. It also configures+ top-level logging.++ This utility should be a better fit than+ :class:`~scrapy.crawler.AsyncCrawlerRunner` if you aren't running another+ :mod:`~twisted.internet.reactor` within your application.++ The AsyncCrawlerProcess object must be instantiated with a+ :class:`~scrapy.settings.Settings` object.++ :param install_root_handler: whether to install root logging handler+ (default: True)++ This class shouldn't be needed (since Scrapy is responsible of using it+ accordingly) unless writing scripts that manually handle the crawling+ process. See :ref:`run-from-script` for an example.++ This class provides coroutine APIs. It requires+ :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`.+ """++ def __init__(+ self,+ settings: dict[str, Any] | Settings | None = None,+ install_root_handler: bool = True,+ ):+ super().__init__(settings, install_root_handler)+ # We want the asyncio event loop to be installed early, so that it's+ # always the correct one. And as we do that, we can also install the+ # reactor here.+ # The ASYNCIO_EVENT_LOOP setting cannot be overridden by add-ons and+ # spiders when using AsyncCrawlerProcess.+ loop_path = self.settings["ASYNCIO_EVENT_LOOP"]+ if is_reactor_installed():+ # The user could install a reactor before this class is instantiated.+ # We need to make sure the reactor is the correct one and the loop+ # type matches the setting.+ verify_installed_reactor(_asyncio_reactor_path)+ if loop_path:+ verify_installed_asyncio_event_loop(loop_path)+ else:+ install_reactor(_asyncio_reactor_path, loop_path)+ self._initialized_reactor = True++ def _stop_dfd(self) -> Deferred[Any]:+ return deferred_from_coro(self.stop())++ def start(+ self, stop_after_crawl: bool = True, install_signal_handlers: bool = True+ ) -> None:+ """+ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool+ size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache+ based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.++ If ``stop_after_crawl`` is True, the reactor will be stopped after all+ crawlers have finished, using :meth:`join`.++ :param bool stop_after_crawl: stop or not the reactor when all+ crawlers have finished++ :param bool install_signal_handlers: whether to install the OS signal+ handlers from Twisted and Scrapy (default: True)+ """from twisted.internet import reactor- # raised if already stopped or in shutdown stage- with contextlib.suppress(RuntimeError):- reactor.stop()+ if stop_after_crawl:+ loop = asyncio.get_event_loop()+ join_task = loop.create_task(self.join())+ join_task.add_done_callback(self._stop_reactor)++ self._setup_reactor(install_signal_handlers)+ reactor.run(installSignalHandlers=install_signal_handlers) # blocking calldiff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 4649c4daa..2df82c8f2 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -23,7 +23,6 @@ from twisted.python import failurefrom scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarningfrom scrapy.utils.asyncio import is_asyncio_available-from scrapy.utils.reactor import _get_asyncio_event_loopif TYPE_CHECKING:from collections.abc import AsyncIterator, Callable@@ -385,8 +384,7 @@ def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2:# that use asyncio, e.g. "await asyncio.sleep(1)"return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o))# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor- event_loop = _get_asyncio_event_loop()- return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))+ return Deferred.fromFuture(asyncio.ensure_future(o))return o@@ -430,6 +428,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]:Return an :class:`asyncio.Future` object that wraps *d*.+ This function requires+ :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be+ installed.+When :ref:`using the asyncio reactor <install-asyncio>`, you cannot awaiton :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapycallables defined as coroutines <coroutine-support>`, you can only await on@@ -442,8 +444,15 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]:additional_request = scrapy.Request('https://example.org/price')deferred = self.crawler.engine.download(additional_request)additional_response = await deferred_to_future(deferred)++ .. versionchanged:: VERSION+ This function no longer installs an asyncio loop if called before the+ Twisted asyncio reactor is installed. A :exc:`RuntimeError` is raised+ in this case."""- return d.asFuture(_get_asyncio_event_loop())+ if not is_asyncio_available():+ raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.")+ return d.asFuture(asyncio.get_event_loop())def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]:diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex 5e76da37b..2fb1e0ce7 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -10,6 +10,7 @@ from twisted.internet import asyncioreactor, errorfrom twisted.internet.defer import Deferredfrom scrapy.utils.misc import load_object+from scrapy.utils.python import global_object_nameif TYPE_CHECKING:from asyncio import AbstractEventLoop, AbstractEventLoopPolicy@@ -87,6 +88,9 @@ class CallLaterOnce(Generic[_T]):await maybe_deferred_to_future(d)+_asyncio_reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"++def set_asyncio_event_loop_policy() -> None:"""The policy functions from asyncio often behave unexpectedly,so we restrict their use to the absolutely essential case.@@ -161,21 +165,34 @@ def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop:def verify_installed_reactor(reactor_path: str) -> None:- """Raises :exc:`Exception` if the installed+ """Raise :exc:`RuntimeError` if the installed:mod:`~twisted.internet.reactor` does not match the specified import- path."""+ path or if no reactor is installed."""+ if not is_reactor_installed():+ raise RuntimeError(+ "verify_installed_reactor() called without an installed reactor."+ )+from twisted.internet import reactor- reactor_class = load_object(reactor_path)- if not reactor.__class__ == reactor_class:+ expected_reactor_type = load_object(reactor_path)+ reactor_type = type(reactor)+ if not reactor_type == expected_reactor_type:raise RuntimeError(- "The installed reactor "- f"({reactor.__module__}.{reactor.__class__.__name__}) does not "- f"match the requested one ({reactor_path})"+ f"The installed reactor ({global_object_name(reactor_type)}) "+ f"does not match the requested one ({reactor_path})")def verify_installed_asyncio_event_loop(loop_path: str) -> None:+ """Raise :exc:`RuntimeError` if the even loop of the installed+ :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`+ does not match the specified import path or if no reactor is installed."""+ if not is_reactor_installed():+ raise RuntimeError(+ "verify_installed_asyncio_event_loop() called without an installed reactor."+ )+from twisted.internet import reactorloop_class = load_object(loop_path)@@ -185,16 +202,16 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None:f"{reactor._asyncioEventloop.__class__.__module__}"f".{reactor._asyncioEventloop.__class__.__qualname__}")- specified = f"{loop_class.__module__}.{loop_class.__qualname__}"raise RuntimeError("Scrapy found an asyncio Twisted reactor already "f"installed, and its event loop class ({installed}) does ""not match the one specified in the ASYNCIO_EVENT_LOOP "- f"setting ({specified})"+ f"setting ({global_object_name(loop_class)})")def is_reactor_installed() -> bool:+ """Check whether a :mod:`~twisted.internet.reactor` is installed."""return "twisted.internet.reactor" in sys.modulesdiff --git a/tests/AsyncCrawlerProcess/args_settings.py b/tests/AsyncCrawlerProcess/args_settings.pynew file mode 100644index 000000000..5e162e782--- /dev/null+++ b/tests/AsyncCrawlerProcess/args_settings.py@@ -0,0 +1,25 @@+from typing import Any++import scrapy+from scrapy.crawler import AsyncCrawlerProcess, Crawler+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"++ @classmethod+ def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):+ spider = super().from_crawler(crawler, *args, **kwargs)+ spider.settings.set("FOO", kwargs.get("foo"))+ return spider++ async def start(self):+ self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}")+ return+ yield+++process = AsyncCrawlerProcess(settings={})++process.crawl(NoRequestsSpider, foo=42)+process.start()diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop.pynew file mode 100644index 000000000..172e36b7b--- /dev/null+++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop.py@@ -0,0 +1,20 @@+import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"++ async def start(self):+ return+ yield+++process = AsyncCrawlerProcess(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",+ }+)+process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.pynew file mode 100644index 000000000..d76da51a1--- /dev/null+++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py@@ -0,0 +1,23 @@+import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"+ custom_settings = {+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",+ }++ async def start(self):+ return+ yield+++process = AsyncCrawlerProcess(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": None,+ }+)+process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.pynew file mode 100644index 000000000..bd4a99e14--- /dev/null+++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py@@ -0,0 +1,23 @@+import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"+ custom_settings = {+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",+ }++ async def start(self):+ return+ yield+++process = AsyncCrawlerProcess(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",+ }+)+process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.pynew file mode 100644index 000000000..c32aaf37d--- /dev/null+++ b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py@@ -0,0 +1,48 @@+from __future__ import annotations++import asyncio+import sys++from scrapy import Spider+from scrapy.crawler import AsyncCrawlerProcess+from scrapy.utils.defer import deferred_from_coro+++class UppercasePipeline:+ async def _open_spider(self, spider):+ spider.logger.info("async pipeline opened!")+ await asyncio.sleep(0.1)++ def open_spider(self, spider):+ return deferred_from_coro(self._open_spider(spider))++ def process_item(self, item, spider):+ return {"url": item["url"].upper()}+++class UrlSpider(Spider):+ name = "url_spider"+ start_urls = ["data:,"]+ custom_settings = {+ "ITEM_PIPELINES": {UppercasePipeline: 100},+ }++ def parse(self, response):+ yield {"url": response.url}+++if __name__ == "__main__":+ ASYNCIO_EVENT_LOOP: str | None+ try:+ ASYNCIO_EVENT_LOOP = sys.argv[1]+ except IndexError:+ ASYNCIO_EVENT_LOOP = None++ process = AsyncCrawlerProcess(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP,+ }+ )+ process.crawl(UrlSpider)+ process.start()diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.pynew file mode 100644index 000000000..3c47eb826--- /dev/null+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py@@ -0,0 +1,27 @@+import scrapy+from scrapy.crawler import AsyncCrawlerProcess+from scrapy.utils.reactor import is_asyncio_reactor_installed+++class ReactorCheckExtension:+ def __init__(self):+ if not is_asyncio_reactor_installed():+ raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.")+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"++ async def start(self):+ return+ yield+++process = AsyncCrawlerProcess(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "EXTENSIONS": {ReactorCheckExtension: 0},+ }+)+process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.pynew file mode 100644index 000000000..e025e17d1--- /dev/null+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py@@ -0,0 +1,53 @@+import scrapy+from scrapy.crawler import AsyncCrawlerProcess+from scrapy.utils.reactor import (+ install_reactor,+ is_asyncio_reactor_installed,+ is_reactor_installed,+)++if is_reactor_installed():+ raise RuntimeError(+ "Reactor already installed before is_asyncio_reactor_installed()."+ )++try:+ is_asyncio_reactor_installed()+except RuntimeError:+ pass+else:+ raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.")++if is_reactor_installed():+ raise RuntimeError(+ "Reactor already installed after is_asyncio_reactor_installed()."+ )++install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")++if not is_asyncio_reactor_installed():+ raise RuntimeError("Wrong reactor installed after install_reactor().")+++class ReactorCheckExtension:+ def __init__(self):+ if not is_asyncio_reactor_installed():+ raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.")+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"++ async def start(self):+ return+ yield+++process = AsyncCrawlerProcess(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "EXTENSIONS": {ReactorCheckExtension: 0},+ }+)+process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.pynew file mode 100644index 000000000..4257bc0ac--- /dev/null+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py@@ -0,0 +1,29 @@+import asyncio+import sys++from twisted.internet import asyncioreactor++import scrapy+from scrapy.crawler import AsyncCrawlerProcess++if sys.platform == "win32":+ asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())+asyncioreactor.install(asyncio.get_event_loop())+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"++ async def start(self):+ return+ yield+++process = AsyncCrawlerProcess(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",+ }+)+process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.pynew file mode 100644index 000000000..9c6fd090b--- /dev/null+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py@@ -0,0 +1,31 @@+import asyncio+import sys++from twisted.internet import asyncioreactor+from uvloop import Loop++import scrapy+from scrapy.crawler import AsyncCrawlerProcess++if sys.platform == "win32":+ asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())+asyncio.set_event_loop(Loop())+asyncioreactor.install(asyncio.get_event_loop())+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"++ async def start(self):+ return+ yield+++process = AsyncCrawlerProcess(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",+ }+)+process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.pynew file mode 100644index 000000000..5f75d5e17--- /dev/null+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py@@ -0,0 +1,35 @@+import sys++import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class CachingHostnameResolverSpider(scrapy.Spider):+ """+ Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution)+ """++ name = "caching_hostname_resolver_spider"++ async def start(self):+ yield scrapy.Request(self.url)++ def parse(self, response):+ for _ in range(10):+ yield scrapy.Request(+ response.url, dont_filter=True, callback=self.ignore_response+ )++ def ignore_response(self, response):+ self.logger.info(repr(response.ip_address))+++if __name__ == "__main__":+ process = AsyncCrawlerProcess(+ settings={+ "RETRY_ENABLED": False,+ "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",+ }+ )+ process.crawl(CachingHostnameResolverSpider, url=sys.argv[1])+ process.start()diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.pynew file mode 100644index 000000000..c43f0a9c2--- /dev/null+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py@@ -0,0 +1,22 @@+import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class CachingHostnameResolverSpider(scrapy.Spider):+ """+ Finishes without a twisted.internet.error.DNSLookupError exception+ """++ name = "caching_hostname_resolver_spider"+ start_urls = ["http://[::1]"]+++if __name__ == "__main__":+ process = AsyncCrawlerProcess(+ settings={+ "RETRY_ENABLED": False,+ "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",+ }+ )+ process.crawl(CachingHostnameResolverSpider)+ process.start()diff --git a/tests/AsyncCrawlerProcess/default_name_resolver.py b/tests/AsyncCrawlerProcess/default_name_resolver.pynew file mode 100644index 000000000..af56ccd01--- /dev/null+++ b/tests/AsyncCrawlerProcess/default_name_resolver.py@@ -0,0 +1,18 @@+import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class IPv6Spider(scrapy.Spider):+ """+ Raises a twisted.internet.error.DNSLookupError:+ the default name resolver does not handle IPv6 addresses.+ """++ name = "ipv6_spider"+ start_urls = ["http://[::1]"]+++if __name__ == "__main__":+ process = AsyncCrawlerProcess(settings={"RETRY_ENABLED": False})+ process.crawl(IPv6Spider)+ process.start()diff --git a/tests/AsyncCrawlerProcess/multi.py b/tests/AsyncCrawlerProcess/multi.pynew file mode 100644index 000000000..2eede5471--- /dev/null+++ b/tests/AsyncCrawlerProcess/multi.py@@ -0,0 +1,17 @@+import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"++ async def start(self):+ return+ yield+++process = AsyncCrawlerProcess(settings={})++process.crawl(NoRequestsSpider)+process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/reactor_default.py b/tests/AsyncCrawlerProcess/reactor_default.pynew file mode 100644index 000000000..9638652bd--- /dev/null+++ b/tests/AsyncCrawlerProcess/reactor_default.py@@ -0,0 +1,18 @@+from twisted.internet import reactor # noqa: F401,TID253++import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"++ async def start(self):+ return+ yield+++process = AsyncCrawlerProcess(settings={})++d = process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/simple.py b/tests/AsyncCrawlerProcess/simple.pynew file mode 100644index 000000000..d24b4f193--- /dev/null+++ b/tests/AsyncCrawlerProcess/simple.py@@ -0,0 +1,16 @@+import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"++ async def start(self):+ return+ yield+++process = AsyncCrawlerProcess(settings={})++process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/sleeping.py b/tests/AsyncCrawlerProcess/sleeping.pynew file mode 100644index 000000000..88caf5032--- /dev/null+++ b/tests/AsyncCrawlerProcess/sleeping.py@@ -0,0 +1,20 @@+import asyncio+import sys++import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class SleepingSpider(scrapy.Spider):+ name = "sleeping"++ start_urls = ["data:,;"]++ async def parse(self, response):+ await asyncio.sleep(int(sys.argv[1]))+++process = AsyncCrawlerProcess(settings={})++process.crawl(SleepingSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py b/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.pynew file mode 100644index 000000000..dc820ea3a--- /dev/null+++ b/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py@@ -0,0 +1,15 @@+import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class AsyncioReactorSpider(scrapy.Spider):+ name = "asyncio_reactor"+++process = AsyncCrawlerProcess(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ }+)+process.crawl(AsyncioReactorSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.pynew file mode 100644index 000000000..5fd48274a--- /dev/null+++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py@@ -0,0 +1,14 @@+import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class AsyncioReactorSpider(scrapy.Spider):+ name = "asyncio_reactor"+ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ }+++process = AsyncCrawlerProcess()+process.crawl(AsyncioReactorSpider)+process.start()diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.pynew file mode 100644index 000000000..c205c3cd2--- /dev/null+++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py@@ -0,0 +1,22 @@+import scrapy+from scrapy.crawler import AsyncCrawlerProcess+++class AsyncioReactorSpider1(scrapy.Spider):+ name = "asyncio_reactor1"+ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ }+++class AsyncioReactorSpider2(scrapy.Spider):+ name = "asyncio_reactor2"+ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ }+++process = AsyncCrawlerProcess()+process.crawl(AsyncioReactorSpider1)+process.crawl(AsyncioReactorSpider2)+process.start()diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.pynew file mode 100644index 000000000..68239e651--- /dev/null+++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py@@ -0,0 +1,30 @@+from __future__ import annotations++import logging+from typing import TYPE_CHECKING++import scrapy+from scrapy.crawler import AsyncCrawlerProcess++if TYPE_CHECKING:+ from asyncio import Task+++class AsyncioReactorSpider(scrapy.Spider):+ name = "asyncio_reactor"+ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",+ }+++def log_task_exception(task: Task) -> None:+ try:+ task.result()+ except Exception:+ logging.exception("Crawl task failed")+++process = AsyncCrawlerProcess()+task = process.crawl(AsyncioReactorSpider)+task.add_done_callback(log_task_exception)+process.start()diff --git a/tests/AsyncCrawlerRunner/custom_loop_different.py b/tests/AsyncCrawlerRunner/custom_loop_different.pynew file mode 100644index 000000000..89cf0e536--- /dev/null+++ b/tests/AsyncCrawlerRunner/custom_loop_different.py@@ -0,0 +1,31 @@+from twisted.internet.task import react++from scrapy import Spider+from scrapy.crawler import AsyncCrawlerRunner+from scrapy.utils.defer import deferred_f_from_coro_f+from scrapy.utils.log import configure_logging+from scrapy.utils.reactor import install_reactor+++class NoRequestsSpider(Spider):+ name = "no_request"++ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",+ }++ async def start(self):+ return+ yield+++@deferred_f_from_coro_f+async def main(reactor):+ configure_logging()+ runner = AsyncCrawlerRunner()+ await runner.crawl(NoRequestsSpider)+++install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+react(main)diff --git a/tests/AsyncCrawlerRunner/custom_loop_same.py b/tests/AsyncCrawlerRunner/custom_loop_same.pynew file mode 100644index 000000000..43d0dc053--- /dev/null+++ b/tests/AsyncCrawlerRunner/custom_loop_same.py@@ -0,0 +1,31 @@+from twisted.internet.task import react++from scrapy import Spider+from scrapy.crawler import AsyncCrawlerRunner+from scrapy.utils.defer import deferred_f_from_coro_f+from scrapy.utils.log import configure_logging+from scrapy.utils.reactor import install_reactor+++class NoRequestsSpider(Spider):+ name = "no_request"++ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",+ }++ async def start(self):+ return+ yield+++@deferred_f_from_coro_f+async def main(reactor):+ configure_logging()+ runner = AsyncCrawlerRunner()+ await runner.crawl(NoRequestsSpider)+++install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor", "uvloop.Loop")+react(main)diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.pyindex d8c467f40..7c50277b9 100644--- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py@@ -4,12 +4,12 @@ import sysfrom twisted.internet import asyncioreactorfrom twisted.python import log+import scrapy+from scrapy.crawler import CrawlerProcess+if sys.platform == "win32":asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())-asyncioreactor.install(asyncio.get_event_loop())--import scrapy # noqa: E402-from scrapy.crawler import CrawlerProcess # noqa: E402+asyncioreactor.install()class NoRequestsSpider(scrapy.Spider):diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.pyindex e7d3ca9cc..578e0029d 100644--- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py@@ -4,13 +4,13 @@ import sysfrom twisted.internet import asyncioreactorfrom uvloop import Loop+import scrapy+from scrapy.crawler import CrawlerProcess+if sys.platform == "win32":asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())asyncio.set_event_loop(Loop())-asyncioreactor.install(asyncio.get_event_loop())--import scrapy # noqa: E402-from scrapy.crawler import CrawlerProcess # noqa: E402+asyncioreactor.install()class NoRequestsSpider(scrapy.Spider):diff --git a/tests/CrawlerRunner/custom_loop_different.py b/tests/CrawlerRunner/custom_loop_different.pynew file mode 100644index 000000000..86ba1ed47--- /dev/null+++ b/tests/CrawlerRunner/custom_loop_different.py@@ -0,0 +1,29 @@+from twisted.internet.task import react++from scrapy import Spider+from scrapy.crawler import CrawlerRunner+from scrapy.utils.log import configure_logging+from scrapy.utils.reactor import install_reactor+++class NoRequestsSpider(Spider):+ name = "no_request"++ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",+ }++ async def start(self):+ return+ yield+++def main(reactor):+ configure_logging()+ runner = CrawlerRunner()+ return runner.crawl(NoRequestsSpider)+++install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+react(main)diff --git a/tests/CrawlerRunner/custom_loop_same.py b/tests/CrawlerRunner/custom_loop_same.pynew file mode 100644index 000000000..98b8dde87--- /dev/null+++ b/tests/CrawlerRunner/custom_loop_same.py@@ -0,0 +1,29 @@+from twisted.internet.task import react++from scrapy import Spider+from scrapy.crawler import CrawlerRunner+from scrapy.utils.log import configure_logging+from scrapy.utils.reactor import install_reactor+++class NoRequestsSpider(Spider):+ name = "no_request"++ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",+ }++ async def start(self):+ return+ yield+++def main(reactor):+ configure_logging()+ runner = CrawlerRunner()+ return runner.crawl(NoRequestsSpider)+++install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor", "uvloop.Loop")+react(main)diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex a1d3c02fb..ce5963cd7 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -1,3 +1,4 @@+import asyncioimport loggingimport platformimport re@@ -5,6 +6,7 @@ import signalimport subprocessimport sysimport warnings+from abc import ABC, abstractmethodfrom pathlib import Pathfrom typing import Any@@ -18,12 +20,18 @@ from zope.interface.exceptions import MultipleInvalidimport scrapyfrom scrapy import Spider-from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerProcess, CrawlerRunner+from scrapy.crawler import (+ AsyncCrawlerProcess,+ AsyncCrawlerRunner,+ Crawler,+ CrawlerProcess,+ CrawlerRunner,+)from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions.throttle import AutoThrottlefrom scrapy.settings import Settings, default_settingsfrom scrapy.spiderloader import SpiderLoader-from scrapy.utils.defer import deferred_from_coro+from scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_corofrom scrapy.utils.log import configure_logging, get_scrapy_root_handlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler, get_reactor_settings@@ -88,12 +96,39 @@ class TestCrawler(TestBaseCrawler):Crawler(DefaultSpider())@inlineCallbacks- def test_crawler_crawl_twice_unsupported(self):+ def test_crawler_crawl_twice_seq_unsupported(self):crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)yield crawler.crawl()with pytest.raises(RuntimeError, match="more than once on the same instance"):yield crawler.crawl()+ @pytest.mark.only_asyncio+ @deferred_f_from_coro_f+ async def test_crawler_crawl_async_twice_seq_unsupported(self):+ crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)+ await crawler.crawl_async()+ with pytest.raises(RuntimeError, match="more than once on the same instance"):+ await crawler.crawl_async()++ @inlineCallbacks+ def test_crawler_crawl_twice_parallel_unsupported(self):+ crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)+ d1 = crawler.crawl()+ d2 = crawler.crawl()+ yield d1+ with pytest.raises(RuntimeError, match="Crawling already taking place"):+ yield d2++ @pytest.mark.only_asyncio+ @deferred_f_from_coro_f+ async def test_crawler_crawl_async_twice_parallel_unsupported(self):+ crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)+ t1 = asyncio.create_task(crawler.crawl_async())+ t2 = asyncio.create_task(crawler.crawl_async())+ await t1+ with pytest.raises(RuntimeError, match="Crawling already taking place"):+ await t2+def test_get_addon(self):class ParentAddon:pass@@ -590,6 +625,18 @@ class TestCrawlerProcess(TestBaseCrawler):self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")[email protected]_asyncio+class TestAsyncCrawlerProcess(TestBaseCrawler):+ def test_crawler_process_accepts_dict(self):+ runner = AsyncCrawlerProcess({"foo": "bar"})+ assert runner.settings["foo"] == "bar"+ self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")++ def test_crawler_process_accepts_None(self):+ runner = AsyncCrawlerProcess()+ self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")++class ExceptionSpider(scrapy.Spider):name = "exception"@@ -692,8 +739,15 @@ class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):pytest.skip("This test is only for CrawlerRunner")-class ScriptRunnerMixin:- script_dir: Path+class ScriptRunnerMixin(ABC):+ @property+ @abstractmethod+ def script_dir(self) -> Path:+ raise NotImplementedError++ @staticmethod+ def get_script_dir(name: str) -> Path:+ return Path(__file__).parent.resolve() / namedef get_script_args(self, script_name: str, *script_args: str) -> list[str]:script_path = self.script_dir / script_name@@ -711,8 +765,10 @@ class ScriptRunnerMixin:return stderr.decode("utf-8")-class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):- script_dir = Path(__file__).parent.resolve() / "CrawlerProcess"+class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin, unittest.TestCase):+ """Common tests between CrawlerProcess and AsyncCrawlerProcess,+ with the same file names and expectations.+ """def test_simple(self):log = self.run_script("simple.py")@@ -739,48 +795,6 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):"(twisted.internet.asyncioreactor.AsyncioSelectorReactor)") in log- def test_reactor_default_twisted_reactor_select(self):- log = self.run_script("reactor_default_twisted_reactor_select.py")- if platform.system() in ["Windows", "Darwin"]:- # The goal of this test function is to test that, when a reactor is- # installed (the default one here) and a different reactor is- # configured (select here), an error raises.- #- # In Windows the default reactor is the select reactor, so that- # error does not raise.- #- # If that ever becomes the case on more platforms (i.e. if Linux- # also starts using the select reactor by default in a future- # version of Twisted), then we will need to rethink this test.- assert "Spider closed (finished)" in log- else:- assert "Spider closed (finished)" not in log- assert (- "does not match the requested one "- "(twisted.internet.selectreactor.SelectReactor)"- ) in log-- def test_reactor_select(self):- log = self.run_script("reactor_select.py")- assert "Spider closed (finished)" not in log- assert (- "does not match the requested one "- "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"- ) in log-- def test_reactor_select_twisted_reactor_select(self):- log = self.run_script("reactor_select_twisted_reactor_select.py")- assert "Spider closed (finished)" in log- assert "ReactorAlreadyInstalledError" not in log-- def test_reactor_select_subclass_twisted_reactor_select(self):- log = self.run_script("reactor_select_subclass_twisted_reactor_select.py")- assert "Spider closed (finished)" not in log- assert (- "does not match the requested one "- "(twisted.internet.selectreactor.SelectReactor)"- ) in log-def test_asyncio_enabled_no_reactor(self):log = self.run_script("asyncio_enabled_no_reactor.py")assert "Spider closed (finished)" in log@@ -829,19 +843,6 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):assert "TimeoutError" not in logassert "twisted.internet.error.DNSLookupError" not in log- def test_twisted_reactor_select(self):- log = self.run_script("twisted_reactor_select.py")- assert "Spider closed (finished)" in log- assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log-- @pytest.mark.skipif(- platform.system() == "Windows", reason="PollReactor is not supported on Windows"- )- def test_twisted_reactor_poll(self):- log = self.run_script("twisted_reactor_poll.py")- assert "Spider closed (finished)" in log- assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log-def test_twisted_reactor_asyncio(self):log = self.run_script("twisted_reactor_asyncio.py")assert "Spider closed (finished)" in log@@ -866,14 +867,6 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):in log)- def test_twisted_reactor_asyncio_custom_settings_conflict(self):- log = self.run_script("twisted_reactor_custom_settings_conflict.py")- assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log- assert (- "(twisted.internet.selectreactor.SelectReactor) does not match the requested one"- in log- )-@pytest.mark.requires_uvloopdef test_custom_loop_asyncio(self):log = self.run_script("asyncio_custom_loop.py")@@ -960,23 +953,120 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):p.wait()-class TestCrawlerRunnerSubprocess(ScriptRunnerMixin):- script_dir = Path(__file__).parent.resolve() / "CrawlerRunner"+class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):+ @property+ def script_dir(self) -> Path:+ return self.get_script_dir("CrawlerProcess")- def test_simple(self):- log = self.run_script("simple.py")+ def test_reactor_default_twisted_reactor_select(self):+ log = self.run_script("reactor_default_twisted_reactor_select.py")+ if platform.system() in ["Windows", "Darwin"]:+ # The goal of this test function is to test that, when a reactor is+ # installed (the default one here) and a different reactor is+ # configured (select here), an error raises.+ #+ # In Windows the default reactor is the select reactor, so that+ # error does not raise.+ #+ # If that ever becomes the case on more platforms (i.e. if Linux+ # also starts using the select reactor by default in a future+ # version of Twisted), then we will need to rethink this test.+ assert "Spider closed (finished)" in log+ else:+ assert "Spider closed (finished)" not in log+ assert (+ "does not match the requested one "+ "(twisted.internet.selectreactor.SelectReactor)"+ ) in log++ def test_reactor_select(self):+ log = self.run_script("reactor_select.py")+ assert "Spider closed (finished)" not in log+ assert (+ "does not match the requested one "+ "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"+ ) in log++ def test_reactor_select_twisted_reactor_select(self):+ log = self.run_script("reactor_select_twisted_reactor_select.py")+ assert "Spider closed (finished)" in log+ assert "ReactorAlreadyInstalledError" not in log++ def test_reactor_select_subclass_twisted_reactor_select(self):+ log = self.run_script("reactor_select_subclass_twisted_reactor_select.py")+ assert "Spider closed (finished)" not in log+ assert (+ "does not match the requested one "+ "(twisted.internet.selectreactor.SelectReactor)"+ ) in log++ def test_twisted_reactor_select(self):+ log = self.run_script("twisted_reactor_select.py")+ assert "Spider closed (finished)" in log+ assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log++ @pytest.mark.skipif(+ platform.system() == "Windows", reason="PollReactor is not supported on Windows"+ )+ def test_twisted_reactor_poll(self):+ log = self.run_script("twisted_reactor_poll.py")+ assert "Spider closed (finished)" in log+ assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log++ def test_twisted_reactor_asyncio_custom_settings_conflict(self):+ log = self.run_script("twisted_reactor_custom_settings_conflict.py")+ assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log+ assert (+ "(twisted.internet.selectreactor.SelectReactor) does not match the requested one"+ in log+ )+++class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):+ @property+ def script_dir(self) -> Path:+ return self.get_script_dir("AsyncCrawlerProcess")++ def test_twisted_reactor_custom_settings_select(self):+ log = self.run_script("twisted_reactor_custom_settings_select.py")+ assert "Spider closed (finished)" not in log+ assert (+ "(twisted.internet.asyncioreactor.AsyncioSelectorReactor) "+ "does not match the requested one "+ "(twisted.internet.selectreactor.SelectReactor)"+ ) in log++ @pytest.mark.requires_uvloop+ def test_asyncio_enabled_reactor_same_loop(self):+ log = self.run_script("asyncio_custom_loop_custom_settings_same.py")assert "Spider closed (finished)" in logassert ("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"in log)+ assert "Using asyncio event loop: uvloop.Loop" in log- def test_explicit_default_reactor(self):- log = self.run_script("explicit_default_reactor.py")+ @pytest.mark.requires_uvloop+ def test_asyncio_enabled_reactor_different_loop(self):+ log = self.run_script("asyncio_custom_loop_custom_settings_different.py")+ assert "Spider closed (finished)" not in log+ assert (+ "does not match the one specified in the ASYNCIO_EVENT_LOOP "+ "setting (uvloop.Loop)"+ ) in log+++class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):+ """Common tests between CrawlerRunner and AsyncCrawlerRunner,+ with the same file names and expectations.+ """++ def test_simple(self):+ log = self.run_script("simple.py")assert "Spider closed (finished)" in logassert ("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"- not in log+ in log)def test_multi_parallel(self):@@ -1005,6 +1095,39 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin):re.DOTALL,)+ @pytest.mark.requires_uvloop+ def test_custom_loop_same(self):+ log = self.run_script("custom_loop_same.py")+ assert "Spider closed (finished)" in log+ assert (+ "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"+ in log+ )+ assert "Using asyncio event loop: uvloop.Loop" in log++ @pytest.mark.requires_uvloop+ def test_custom_loop_different(self):+ log = self.run_script("custom_loop_different.py")+ assert "Spider closed (finished)" not in log+ assert (+ "does not match the one specified in the ASYNCIO_EVENT_LOOP "+ "setting (uvloop.Loop)"+ ) in log+++class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):+ @property+ def script_dir(self) -> Path:+ return self.get_script_dir("CrawlerRunner")++ def test_explicit_default_reactor(self):+ log = self.run_script("explicit_default_reactor.py")+ assert "Spider closed (finished)" in log+ assert (+ "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"+ not in log+ )+def test_response_ip_address(self):log = self.run_script("ip_address.py")assert "INFO: Spider closed (finished)" in log@@ -1021,48 +1144,16 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin):assert "DEBUG: Using asyncio event loop" in log-class TestAsyncCrawlerRunnerSubprocess(ScriptRunnerMixin):- script_dir = Path(__file__).parent.resolve() / "AsyncCrawlerRunner"-- def test_simple(self):- log = self.run_script("simple.py")- assert "Spider closed (finished)" in log- assert (- "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"- in log- )+class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):+ @property+ def script_dir(self) -> Path:+ return self.get_script_dir("AsyncCrawlerRunner")def test_simple_default_reactor(self):log = self.run_script("simple_default_reactor.py")assert "Spider closed (finished)" not in logassert "RuntimeError: AsyncCrawlerRunner requires AsyncioSelectorReactor" in log- def test_multi_parallel(self):- log = self.run_script("multi_parallel.py")- assert "Spider closed (finished)" in log- assert (- "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"- in log- )- assert re.search(- r"Spider opened.+Spider opened.+Closing spider.+Closing spider",- log,- re.DOTALL,- )-- def test_multi_seq(self):- log = self.run_script("multi_seq.py")- assert "Spider closed (finished)" in log- assert (- "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"- in log- )- assert re.search(- r"Spider opened.+Closing spider.+Spider opened.+Closing spider",- log,- re.DOTALL,- )-@pytest.mark.parametrize(("settings", "items"),diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.pyindex 99f175c60..eb00ab193 100644--- a/tests/test_utils_reactor.py+++ b/tests/test_utils_reactor.py@@ -6,6 +6,7 @@ from twisted.trial.unittest import TestCasefrom scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.reactor import (+ _asyncio_reactor_path,install_reactor,is_asyncio_reactor_installed,set_asyncio_event_loop,@@ -22,7 +23,7 @@ class TestAsyncio(TestCase):from twisted.internet import reactor as original_reactorwith warnings.catch_warnings(record=True) as w:- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+ install_reactor(_asyncio_reactor_path)assert len(w) == 0, [str(warning) for warning in w]from twisted.internet import reactor # pylint: disable=reimported@@ -31,5 +32,5 @@ class TestAsyncio(TestCase):@pytest.mark.only_asyncio@deferred_f_from_coro_fasync def test_set_asyncio_event_loop(self):- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+ install_reactor(_asyncio_reactor_path)assert set_asyncio_event_loop(None) is asyncio.get_running_loop()