Scrapy, a fast high-level web crawling & scraping framework for Python.
Enable AsyncioSelectorReactor by default. (#6713)
Andrey Rakhmatullind0dabbc09706
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlindex f74575ee1..444aa3557 100644--- a/.github/workflows/tests-ubuntu.yml+++ b/.github/workflows/tests-ubuntu.yml@@ -34,25 +34,25 @@ jobs:TOXENV: py- python-version: "3.13"env:- TOXENV: asyncio+ TOXENV: default-reactor- python-version: pypy3.10env:TOXENV: pypy3# pinned deps- - python-version: 3.9.19+ - python-version: "3.9.21"env:TOXENV: pinned- - python-version: 3.9.19+ - python-version: "3.9.21"env:- TOXENV: asyncio-pinned+ TOXENV: default-reactor-pinned- python-version: pypy3.10env:TOXENV: pypy3-pinned- - python-version: 3.9.19+ - python-version: "3.9.21"env:TOXENV: extra-deps-pinned- - python-version: 3.9.19+ - python-version: "3.9.21"env:TOXENV: botocore-pinned@@ -78,7 +78,7 @@ jobs:if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned')run: |sudo apt-get update- sudo apt-get install libxml2-dev libxslt-dev libjpeg-dev+ sudo apt-get install libxml2-dev libxslt-dev- name: Run testsenv: ${{ matrix.env }}diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.ymlindex 21d621240..537a01e29 100644--- a/.github/workflows/tests-windows.yml+++ b/.github/workflows/tests-windows.yml@@ -19,7 +19,7 @@ jobs:include:- python-version: "3.9"env:- TOXENV: windows-pinned+ TOXENV: py- python-version: "3.10"env:TOXENV: py@@ -34,7 +34,19 @@ jobs:TOXENV: py- python-version: "3.13"env:- TOXENV: asyncio+ TOXENV: default-reactor++ # pinned deps+ - python-version: "3.9.13"+ env:+ TOXENV: pinned+ - python-version: "3.9.13"+ env:+ TOXENV: extra-deps-pinned++ - python-version: "3.13"+ env:+ TOXENV: extra-depssteps:- uses: actions/checkout@v4diff --git a/conftest.py b/conftest.pyindex f33ffb1a4..9999e41d2 100644--- a/conftest.py+++ b/conftest.py@@ -51,7 +51,7 @@ def chdir(tmpdir):def pytest_addoption(parser):parser.addoption("--reactor",- default="default",+ default="asyncio",choices=["default", "asyncio"],)@@ -67,17 +67,17 @@ def reactor_pytest(request):@pytest.fixture(autouse=True)def only_asyncio(request, reactor_pytest):- if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio":- pytest.skip("This test is only run with --reactor=asyncio")+ if request.node.get_closest_marker("only_asyncio") and reactor_pytest == "default":+ pytest.skip("This test is only run without --reactor=default")@pytest.fixture(autouse=True)def only_not_asyncio(request, reactor_pytest):if (request.node.get_closest_marker("only_not_asyncio")- and reactor_pytest == "asyncio"+ and reactor_pytest != "default"):- pytest.skip("This test is only run without --reactor=asyncio")+ pytest.skip("This test is only run with --reactor=default")@pytest.fixture(autouse=True)@@ -117,7 +117,7 @@ def requires_boto3(request):def pytest_configure(config):- if config.getoption("--reactor") == "asyncio":+ if config.getoption("--reactor") != "default":install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rstindex 07baea071..35afdc11b 100644--- a/docs/topics/asyncio.rst+++ b/docs/topics/asyncio.rst@@ -16,15 +16,19 @@ asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` andInstalling the asyncio reactor==============================-To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to-``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``.+To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs+to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``,+which is the default value.If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need toinstall the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`reactor manually. You can do that using-:func:`~scrapy.utils.reactor.install_reactor`::+:func:`~scrapy.utils.reactor.install_reactor`:- install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')+.. skip: next+.. code-block:: python++ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor").. _asyncio-preinstalled-reactor:@@ -144,3 +148,14 @@ Using custom asyncio loopsYou can also use custom asyncio event loops with the asyncio reactor. Set the:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired eventloop class to use it instead of the default asyncio event loop.+++.. _disable-asyncio:++Switching to a non-asyncio reactor+==================================++If for some reason your code doesn't work with the asyncio reactor, you can use+a different reactor by setting the :setting:`TWISTED_REACTOR` setting to its+import path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to+``None``, which will use the default reactor for your platform.diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex cc1fe8703..01da53342 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that youcan configure some extra functions like generating thumbnails and filteringthe images based on their size.-The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for+The Images Pipeline requires Pillow_ 8.0.0 or greater. It is used forthumbnailing and normalizing images to JPEG/RGB format... _Pillow: https://github.com/python-pillow/Pillowdiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex b2bb7148f..ca0af569f 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1911,7 +1911,7 @@ TWISTED_REACTOR.. versionadded:: 2.0-Default: ``None``+Default: ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``Import path of a given :mod:`~twisted.internet.reactor`.@@ -1996,17 +1996,19 @@ which raises :exc:`Exception`, becomes:self.crawler.engine.close_spider(self, "timeout")-The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which-means that Scrapy will use the existing reactor if one is already installed, or-install the default reactor defined by Twisted for the current platform. This-is to maintain backward compatibility and avoid possible problems caused by-using a non-default reactor.+If this setting is set ``None``, Scrapy will use the existing reactor if one is+already installed, or install the default reactor defined by Twisted for the+current platform... versionchanged:: 2.7The :command:`startproject` command now sets this setting to``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated``settings.py`` file.+.. versionchanged:: VERSION+ The default value was changed from ``None`` to+ ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``.+For additional information, see :doc:`core/howto/choosing-reactor`.diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 29dc13f0a..63c6908dc 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -68,7 +68,7 @@ class ImagesPipeline(FilesPipeline):self._Image = Imageexcept ImportError:raise NotConfigured(- "ImagesPipeline requires installing Pillow 4.0.0 or later"+ "ImagesPipeline requires installing Pillow 8.0.0 or later")super().__init__(diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex c473b369c..645e50301 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -341,7 +341,7 @@ TELNETCONSOLE_HOST = "127.0.0.1"TELNETCONSOLE_USERNAME = "scrapy"TELNETCONSOLE_PASSWORD = None-TWISTED_REACTOR = None+TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"SPIDER_CONTRACTS = {}SPIDER_CONTRACTS_BASE = {diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmplindex 0bb31ffaa..db7400af8 100644--- a/scrapy/templates/project/module/settings.py.tmpl+++ b/scrapy/templates/project/module/settings.py.tmpl@@ -90,5 +90,4 @@ ROBOTSTXT_OBEY = True#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"# Set settings whose default value is deprecated to a future-proof value-TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"FEED_EXPORT_ENCODING = "utf-8"diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex b865cf48d..24e17ecb6 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -182,11 +182,9 @@ def log_scrapy_info(settings: Settings) -> None:def log_reactor_info() -> None:- from twisted.internet import reactor+ from twisted.internet import asyncioreactor, reactorlogger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)- from twisted.internet import asyncioreactor-if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):logger.debug("Using asyncio event loop: %s.%s",diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex b69f43438..2da526cd8 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -18,6 +18,7 @@ from twisted.trial.unittest import SkipTestfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.deprecate import create_deprecated_class+from scrapy.utils.reactor import is_asyncio_reactor_installedfrom scrapy.utils.spider import DefaultSpiderif TYPE_CHECKING:@@ -109,6 +110,19 @@ def get_ftp_content_and_delete(TestSpider = create_deprecated_class("TestSpider", DefaultSpider)+def get_reactor_settings() -> dict[str, Any]:+ """Return a settings dict that works with the installed reactor.++ ``Crawler._apply_settings()`` checks that the installed reactor matches the+ settings, so tests that run the crawler in the current process may need to+ pass a correct ``"TWISTED_REACTOR"`` setting value when creating it.+ """+ settings: dict[str, Any] = {}+ if not is_asyncio_reactor_installed():+ settings["TWISTED_REACTOR"] = None+ return settings++def get_crawler(spidercls: type[Spider] | None = None,settings_dict: dict[str, Any] | None = None,@@ -120,9 +134,12 @@ def get_crawler("""from scrapy.crawler import CrawlerRunner- # Set by default settings that prevent deprecation warnings.- settings: dict[str, Any] = {}- settings.update(settings_dict or {})+ # When needed, useful settings can be added here, e.g. ones that prevent+ # deprecation warnings.+ settings: dict[str, Any] = {+ **get_reactor_settings(),+ **(settings_dict or {}),+ }runner = CrawlerRunner(settings)crawler = runner.create_crawler(spidercls or DefaultSpider)crawler._apply_settings()diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.pyindex f013eed27..0c380610d 100644--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py@@ -1,14 +1,8 @@-import asyncio-import sys+import scrapy+from scrapy.crawler import CrawlerProcess+from scrapy.utils.reactor import install_reactor-from twisted.internet import asyncioreactor--if sys.platform == "win32":- asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())-asyncioreactor.install(asyncio.get_event_loop())--import scrapy # noqa: E402-from scrapy.crawler import CrawlerProcess # noqa: E402+install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")class NoRequestsSpider(scrapy.Spider):diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.pyindex 078cb72cb..e2933338b 100644--- a/tests/CrawlerProcess/reactor_default.py+++ b/tests/CrawlerProcess/reactor_default.py@@ -1,4 +1,5 @@from twisted.internet import reactor # noqa: F401+from twisted.python import logimport scrapyfrom scrapy.crawler import CrawlerProcess@@ -13,5 +14,6 @@ class NoRequestsSpider(scrapy.Spider):process = CrawlerProcess(settings={})-process.crawl(NoRequestsSpider)+d = process.crawl(NoRequestsSpider)+d.addErrback(log.err)process.start()diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.pyindex 814a2a46d..b61e52625 100644--- a/tests/CrawlerProcess/reactor_select.py+++ b/tests/CrawlerProcess/reactor_select.py@@ -1,4 +1,5 @@from twisted.internet import selectreactor+from twisted.python import logimport scrapyfrom scrapy.crawler import CrawlerProcess@@ -15,5 +16,6 @@ class NoRequestsSpider(scrapy.Spider):process = CrawlerProcess(settings={})-process.crawl(NoRequestsSpider)+d = process.crawl(NoRequestsSpider)+d.addErrback(log.err)process.start()diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.pyindex 2f1bb7713..b1b297777 100644--- a/tests/CrawlerRunner/ip_address.py+++ b/tests/CrawlerRunner/ip_address.py@@ -1,3 +1,9 @@+# ruff: noqa: E402++from scrapy.utils.reactor import install_reactor++install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+from urllib.parse import urlparsefrom twisted.internet import reactordiff --git a/tests/test_addons.py b/tests/test_addons.pyindex 686bf9952..b4294c815 100644--- a/tests/test_addons.py+++ b/tests/test_addons.py@@ -9,7 +9,7 @@ from scrapy import Spiderfrom scrapy.crawler import Crawler, CrawlerRunnerfrom scrapy.exceptions import NotConfiguredfrom scrapy.settings import BaseSettings, Settings-from scrapy.utils.test import get_crawler+from scrapy.utils.test import get_crawler, get_reactor_settingsclass SimpleAddon:@@ -105,6 +105,7 @@ class TestAddonManager(unittest.TestCase):}settings_dict = {"ADDONS": {get_addon_cls(config): 1},+ **get_reactor_settings(),}crawler = get_crawler(settings_dict=settings_dict)assert crawler.settings.getint("KEY") == 15@@ -119,6 +120,7 @@ class TestAddonManager(unittest.TestCase):settings_dict = {"KEY": 20, # priority=project"ADDONS": {get_addon_cls(config): 1},+ **get_reactor_settings(),}settings = Settings(settings_dict)settings.set("KEY", 0, priority="default")@@ -196,6 +198,7 @@ class TestAddonManager(unittest.TestCase):return spidersettings = Settings()+ settings.setdict(get_reactor_settings())settings.set("KEY", "default", priority="default")runner = CrawlerRunner(settings)crawler = runner.create_crawler(MySpider)diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 6f4045fc8..f49deac1f 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -18,7 +18,7 @@ from scrapy.exceptions import StopDownloadfrom scrapy.http import Requestfrom scrapy.http.response import Responsefrom scrapy.utils.python import to_unicode-from scrapy.utils.test import get_crawler+from scrapy.utils.test import get_crawler, get_reactor_settingsfrom tests import NON_EXISTING_RESOLVABLEfrom tests.mockserver import MockServerfrom tests.spiders import (@@ -412,7 +412,7 @@ with multiples lines@defer.inlineCallbacksdef test_crawl_multiple(self):- runner = CrawlerRunner()+ runner = CrawlerRunner(get_reactor_settings())runner.crawl(SimpleSpider,self.mockserver.url("/status?n=200"),diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 0bbcc0843..98352b66e 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -25,7 +25,7 @@ from scrapy.settings import Settings, default_settingsfrom scrapy.spiderloader import SpiderLoaderfrom scrapy.utils.log import configure_logging, get_scrapy_root_handlerfrom scrapy.utils.spider import DefaultSpider-from scrapy.utils.test import get_crawler+from scrapy.utils.test import get_crawler, get_reactor_settingsfrom tests.mockserver import MockServer, get_mockserver_envBASE_SETTINGS: dict[str, Any] = {}@@ -35,6 +35,7 @@ def get_raw_crawler(spidercls=None, settings_dict=None):"""get_crawler alternative that only calls the __init__ method of thecrawler."""settings = Settings()+ settings.setdict(get_reactor_settings())settings.setdict(settings_dict or {})return Crawler(spidercls or DefaultSpider, settings)@@ -48,7 +49,12 @@ class TestBaseCrawler(unittest.TestCase):class TestCrawler(TestBaseCrawler):def test_populate_spidercls_settings(self):spider_settings = {"TEST1": "spider", "TEST2": "spider"}- project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"}+ project_settings = {+ **BASE_SETTINGS,+ "TEST1": "project",+ "TEST3": "project",+ **get_reactor_settings(),+ }class CustomSettingsSpider(DefaultSpider):custom_settings = spider_settings@@ -581,7 +587,7 @@ class NoRequestsSpider(scrapy.Spider):@pytest.mark.usefixtures("reactor_pytest")class TestCrawlerRunnerHasSpider(unittest.TestCase):def _runner(self):- return CrawlerRunner()+ return CrawlerRunner(get_reactor_settings())@inlineCallbacksdef test_crawler_runner_bootstrap_successful(self):@@ -626,13 +632,7 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase):@inlineCallbacksdef test_crawler_runner_asyncio_enabled_true(self):- if self.reactor_pytest == "asyncio":- CrawlerRunner(- settings={- "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",- }- )- else:+ if self.reactor_pytest == "default":runner = CrawlerRunner(settings={"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",@@ -643,6 +643,12 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase):match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",):yield runner.crawl(NoRequestsSpider)+ else:+ CrawlerRunner(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ }+ )class ScriptRunnerMixin:@@ -672,7 +678,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):assert "Spider closed (finished)" in logassert ("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"- not in log+ in log)def test_multi(self):@@ -680,18 +686,17 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):assert "Spider closed (finished)" in logassert ("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"- not in log+ in log)assert "ReactorAlreadyInstalledError" not in logdef test_reactor_default(self):log = self.run_script("reactor_default.py")- assert "Spider closed (finished)" in log+ assert "Spider closed (finished)" not in logassert (- "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"- not in log- )- assert "ReactorAlreadyInstalledError" not in log+ "does not match the requested one "+ "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"+ ) in logdef test_reactor_default_twisted_reactor_select(self):log = self.run_script("reactor_default_twisted_reactor_select.py")@@ -716,8 +721,11 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):def test_reactor_select(self):log = self.run_script("reactor_select.py")- assert "Spider closed (finished)" in log- assert "ReactorAlreadyInstalledError" not in log+ assert "Spider closed (finished)" not in log+ assert (+ "does not match the requested one "+ "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"+ ) in logdef test_reactor_select_twisted_reactor_select(self):log = self.run_script("reactor_select_twisted_reactor_select.py")diff --git a/tests/test_dependencies.py b/tests/test_dependencies.pyindex 162747581..c2df67c66 100644--- a/tests/test_dependencies.py+++ b/tests/test_dependencies.py@@ -33,7 +33,7 @@ class TestScrapyUtils:tox_config_file_path = Path(__file__).parent / ".." / "tox.ini"config_parser = ConfigParser()config_parser.read(tox_config_file_path)- pattern = r"Twisted\[http2\]==([\d.]+)"+ pattern = r"Twisted==([\d.]+)"match = re.search(pattern, config_parser["pinned"]["deps"])pinned_twisted_version_string = match[1]diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 19bd02498..bc18e76e1 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -307,7 +307,7 @@ class TestHttp(unittest.TestCase, ABC):@defer.inlineCallbacksdef test_timeout_download_from_spider_nodata_rcvd(self):- if self.reactor_pytest == "asyncio" and sys.platform == "win32":+ if self.reactor_pytest != "default" and sys.platform == "win32":# https://twistedmatrix.com/trac/ticket/10279raise unittest.SkipTest("This test produces DirtyReactorAggregateError on Windows with asyncio"@@ -322,7 +322,7 @@ class TestHttp(unittest.TestCase, ABC):@defer.inlineCallbacksdef test_timeout_download_from_spider_server_hangs(self):- if self.reactor_pytest == "asyncio" and sys.platform == "win32":+ if self.reactor_pytest != "default" and sys.platform == "win32":# https://twistedmatrix.com/trac/ticket/10279raise unittest.SkipTest("This test produces DirtyReactorAggregateError on Windows with asyncio"@@ -1136,7 +1136,7 @@ class TestFTPBase(unittest.TestCase):class TestFTP(TestFTPBase):def test_invalid_credentials(self):- if self.reactor_pytest == "asyncio" and sys.platform == "win32":+ if self.reactor_pytest != "default" and sys.platform == "win32":raise unittest.SkipTest("This test produces DirtyReactorAggregateError on Windows with asyncio")diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 15b3ad5af..4fca9eefb 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -64,7 +64,7 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_delay(self):- crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider)+ crawler = get_crawler(DownloaderSlotsSettingsTestSpider)yield crawler.crawl(mockserver=self.mockserver)slots = crawler.engine.downloader.slotstimes = crawler.spider.timesdiff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.pyindex ca5ffdc26..85bd42857 100644--- a/tests/test_extension_periodic_log.py+++ b/tests/test_extension_periodic_log.py@@ -1,9 +1,11 @@+from __future__ import annotations+import datetime-import typingimport unittest+from typing import Any, Callable-from scrapy.crawler import Crawlerfrom scrapy.extensions.periodic_log import PeriodicLog+from scrapy.utils.test import get_crawlerfrom .spiders import MetaSpider@@ -59,9 +61,8 @@ class CustomPeriodicLog(PeriodicLog):self.stats._stats = stats_dump_2-def extension(settings=None):- crawler = Crawler(MetaSpider, settings=settings)- crawler._apply_settings()+def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog:+ crawler = get_crawler(MetaSpider, settings)return CustomPeriodicLog.from_crawler(crawler)@@ -94,7 +95,7 @@ class TestPeriodicLog(unittest.TestCase):ext.spider_closed(spider, reason="finished")return ext, a, b- def check(settings: dict, condition: typing.Callable):+ def check(settings: dict[str, Any], condition: Callable) -> None:ext, a, b = emulate(settings)assert list(a["delta"].keys()) == [k for k, v in ext.stats._stats.items() if condition(k, v)@@ -151,7 +152,7 @@ class TestPeriodicLog(unittest.TestCase):ext.spider_closed(spider, reason="finished")return ext, a, b- def check(settings: dict, condition: typing.Callable):+ def check(settings: dict[str, Any], condition: Callable) -> None:ext, a, b = emulate(settings)assert list(a["stats"].keys()) == [k for k, v in ext.stats._stats.items() if condition(k, v)diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex 162dfdaf4..c5f1b6321 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -3,18 +3,22 @@ from __future__ import annotationsimport shutilfrom pathlib import Pathfrom tempfile import mkdtemp+from typing import TYPE_CHECKING, Anyfrom testfixtures import LogCapturefrom twisted.internet import deferfrom twisted.trial.unittest import TestCasefrom w3lib.url import add_or_replace_parameter-from scrapy import signals-from scrapy.crawler import CrawlerRunner+from scrapy import Spider, signalsfrom scrapy.utils.misc import load_object+from scrapy.utils.test import get_crawlerfrom tests.mockserver import MockServerfrom tests.spiders import SimpleSpider+if TYPE_CHECKING:+ from scrapy.crawler import Crawler+class MediaDownloadSpider(SimpleSpider):name = "mediadownload"@@ -80,7 +84,6 @@ class TestFileDownloadCrawl(TestCase):"ITEM_PIPELINES": {self.pipeline_class: 1},self.store_setting_key: str(self.tmpmediastore),}- self.runner = CrawlerRunner(self.settings)self.items = []def tearDown(self):@@ -90,10 +93,12 @@ class TestFileDownloadCrawl(TestCase):def _on_item_scraped(self, item):self.items.append(item)- def _create_crawler(self, spider_class, runner=None, **kwargs):- if runner is None:- runner = self.runner- crawler = runner.create_crawler(spider_class, **kwargs)+ def _create_crawler(+ self, spider_class: type[Spider], settings: dict[str, Any] | None = None+ ) -> Crawler:+ if settings is None:+ settings = self.settings+ crawler = get_crawler(spider_class, settings)crawler.signals.connect(self._on_item_scraped, signals.item_scraped)return crawler@@ -175,10 +180,11 @@ class TestFileDownloadCrawl(TestCase):@defer.inlineCallbacksdef test_download_media_redirected_allowed(self):- settings = dict(self.settings)- settings.update({"MEDIA_ALLOW_REDIRECTS": True})- runner = CrawlerRunner(settings)- crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner)+ settings = {+ **self.settings,+ "MEDIA_ALLOW_REDIRECTS": True,+ }+ crawler = self._create_crawler(RedirectedMediaDownloadSpider, settings)with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/files/images/"),@@ -201,8 +207,7 @@ class TestFileDownloadCrawl(TestCase):**self.settings,"ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1},}- runner = CrawlerRunner(settings)- crawler = self._create_crawler(MediaDownloadSpider, runner=runner)+ crawler = self._create_crawler(MediaDownloadSpider, settings)with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/files/images/"),diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 4e8330c06..aaf72390d 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -27,7 +27,7 @@ from scrapy.spiders import (XMLFeedSpider,)from scrapy.spiders.init import InitSpider-from scrapy.utils.test import get_crawler+from scrapy.utils.test import get_crawler, get_reactor_settingsfrom tests import get_testdata, tests_datadir@@ -108,7 +108,11 @@ class TestSpider(unittest.TestCase):@inlineCallbacksdef test_settings_in_from_crawler(self):spider_settings = {"TEST1": "spider", "TEST2": "spider"}- project_settings = {"TEST1": "project", "TEST3": "project"}+ project_settings = {+ "TEST1": "project",+ "TEST3": "project",+ **get_reactor_settings(),+ }class TestSpider(self.spider_class):name = "test"diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex a65a36219..901e03d59 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -2,6 +2,7 @@ import asyncioimport warningsimport pytest+from twisted.trial.unittest import TestCasefrom scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.reactor import (@@ -12,10 +13,10 @@ from scrapy.utils.reactor import (@pytest.mark.usefixtures("reactor_pytest")-class TestAsyncio:+class TestAsyncio(TestCase):def test_is_asyncio_reactor_installed(self):# the result should depend only on the pytest --reactor argument- assert is_asyncio_reactor_installed() == (self.reactor_pytest == "asyncio")+ assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default")def test_install_asyncio_reactor(self):from twisted.internet import reactor as original_reactordiff --git a/tox.ini b/tox.iniindex 70c841603..eb084f0f5 100644--- a/tox.ini+++ b/tox.ini@@ -26,7 +26,7 @@ deps ={[test-requirements]deps}# mitmproxy does not support PyPy- mitmproxy; implementation_name != 'pypy'+ mitmproxy; implementation_name != "pypy"setenv =COVERAGE_CORE=sysmonpassenv =@@ -96,19 +96,18 @@ commands =[pinned]basepython = python3.9deps =+ Protego==0.1.15+ Twisted==21.7.0cryptography==37.0.0cssselect==0.9.1- h2==3.0itemadapter==0.1.0+ lxml==4.6.0parsel==1.5.0- Protego==0.1.15pyOpenSSL==22.0.0queuelib==1.4.2service_identity==18.1.0- Twisted[http2]==21.7.0w3lib==1.17.0zope.interface==5.1.0- lxml==4.6.0{[test-requirements]deps}# mitmproxy 8.0.0 requires upgrading some of the pinned dependencies@@ -131,60 +130,50 @@ setenv ={[pinned]setenv}commands = {[pinned]commands}-[testenv:windows-pinned]-basepython = {[pinned]basepython}-deps =- {[pinned]deps}- PyDispatcher==2.0.5-install_command = {[pinned]install_command}-setenv =- {[pinned]setenv}-commands = {[pinned]commands}-[testenv:extra-deps]basepython = python3deps ={[testenv]deps}- boto3- google-cloud-storage- robotexclusionrulesparserPillowTwisted[http2]- uvloop; platform_system != "Windows"+ boto3bpython # optional for shell wrapper tests- brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests- brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests- zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests+ brotli; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests+ brotlicffi; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests+ google-cloud-storageipython+ robotexclusionrulesparser+ uvloop; platform_system != "Windows"+ zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests[testenv:extra-deps-pinned]basepython = {[pinned]basepython}deps ={[pinned]deps}+ Pillow==8.0.0boto3==1.20.0+ bpython==0.7.1+ brotli==0.5.2; implementation_name != "pypy"+ brotlicffi==0.8.0; implementation_name == "pypy"+ brotlipygoogle-cloud-storage==1.29.0- Pillow==7.1.0+ ipython==2.0.0robotexclusionrulesparser==1.6.2- brotlipyuvloop==0.14.0; platform_system != "Windows"- bpython==0.7.1- zstandard==0.1; implementation_name != 'pypy'- ipython==2.0.0- brotli==0.5.2; implementation_name != 'pypy'- brotlicffi==0.8.0; implementation_name == 'pypy'+ zstandard==0.1; implementation_name != "pypy"install_command = {[pinned]install_command}setenv ={[pinned]setenv}commands = {[pinned]commands}-[testenv:asyncio]+[testenv:default-reactor]commands =- {[testenv]commands} --reactor=asyncio+ {[testenv]commands} --reactor=default-[testenv:asyncio-pinned]+[testenv:default-reactor-pinned]basepython = {[pinned]basepython}deps = {[testenv:pinned]deps}-commands = {[pinned]commands} --reactor=asyncio+commands = {[pinned]commands} --reactor=defaultinstall_command = {[pinned]install_command}setenv ={[pinned]setenv}@@ -204,21 +193,20 @@ commands = {[testenv:pypy3]commands}[testenv:pypy3-pinned]basepython = pypy3.10deps =+ PyPyDispatcher==2.1.0+ {[test-requirements]deps}+ Protego==0.1.15+ Twisted==21.7.0cryptography==41.0.5cssselect==0.9.1- h2==3.1itemadapter==0.1.0+ lxml==4.6.0parsel==1.5.0- Protego==0.1.15pyOpenSSL==23.3.0queuelib==1.4.2service_identity==18.1.0- Twisted[http2]==21.7.0w3lib==1.17.0zope.interface==5.1.0- lxml==4.6.0- {[test-requirements]deps}- PyPyDispatcher==2.1.0commands =; disabling both coverage and docs testspytest {posargs:--durations=10 scrapy tests}