Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Enable AsyncioSelectorReactor by default. (#6713)

Andrey Rakhmatullind0dabbc09706
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index f74575ee1..444aa3557 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -34,25 +34,25 @@ jobs:
TOXENV: py
- python-version: "3.13"
env:
- TOXENV: asyncio
+ TOXENV: default-reactor
- python-version: pypy3.10
env:
TOXENV: pypy3
# pinned deps
- - python-version: 3.9.19
+ - python-version: "3.9.21"
env:
TOXENV: pinned
- - python-version: 3.9.19
+ - python-version: "3.9.21"
env:
- TOXENV: asyncio-pinned
+ TOXENV: default-reactor-pinned
- python-version: pypy3.10
env:
TOXENV: pypy3-pinned
- - python-version: 3.9.19
+ - python-version: "3.9.21"
env:
TOXENV: extra-deps-pinned
- - python-version: 3.9.19
+ - python-version: "3.9.21"
env:
TOXENV: botocore-pinned
@@ -78,7 +78,7 @@ jobs:
if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned')
run: |
sudo apt-get update
- sudo apt-get install libxml2-dev libxslt-dev libjpeg-dev
+ sudo apt-get install libxml2-dev libxslt-dev
- name: Run tests
env: ${{ matrix.env }}
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 21d621240..537a01e29 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -19,7 +19,7 @@ jobs:
include:
- python-version: "3.9"
env:
- TOXENV: windows-pinned
+ TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
@@ -34,7 +34,19 @@ jobs:
TOXENV: py
- python-version: "3.13"
env:
- TOXENV: asyncio
+ TOXENV: default-reactor
+
+ # pinned deps
+ - python-version: "3.9.13"
+ env:
+ TOXENV: pinned
+ - python-version: "3.9.13"
+ env:
+ TOXENV: extra-deps-pinned
+
+ - python-version: "3.13"
+ env:
+ TOXENV: extra-deps
steps:
- uses: actions/checkout@v4
diff --git a/conftest.py b/conftest.py
index f33ffb1a4..9999e41d2 100644
--- a/conftest.py
+++ b/conftest.py
@@ -51,7 +51,7 @@ def chdir(tmpdir):
def pytest_addoption(parser):
parser.addoption(
"--reactor",
- default="default",
+ default="asyncio",
choices=["default", "asyncio"],
)
@@ -67,17 +67,17 @@ def reactor_pytest(request):
@pytest.fixture(autouse=True)
def only_asyncio(request, reactor_pytest):
- if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio":
- pytest.skip("This test is only run with --reactor=asyncio")
+ if request.node.get_closest_marker("only_asyncio") and reactor_pytest == "default":
+ pytest.skip("This test is only run without --reactor=default")
@pytest.fixture(autouse=True)
def only_not_asyncio(request, reactor_pytest):
if (
request.node.get_closest_marker("only_not_asyncio")
- and reactor_pytest == "asyncio"
+ and reactor_pytest != "default"
):
- pytest.skip("This test is only run without --reactor=asyncio")
+ pytest.skip("This test is only run with --reactor=default")
@pytest.fixture(autouse=True)
@@ -117,7 +117,7 @@ def requires_boto3(request):
def pytest_configure(config):
- if config.getoption("--reactor") == "asyncio":
+ if config.getoption("--reactor") != "default":
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 07baea071..35afdc11b 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -16,15 +16,19 @@ asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
Installing the asyncio reactor
==============================
-To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to
-``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``.
+To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs
+to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``,
+which is the default value.
If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to
install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`
reactor manually. You can do that using
-:func:`~scrapy.utils.reactor.install_reactor`::
+:func:`~scrapy.utils.reactor.install_reactor`:
- install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
+.. skip: next
+.. code-block:: python
+
+ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
.. _asyncio-preinstalled-reactor:
@@ -144,3 +148,14 @@ Using custom asyncio loops
You can also use custom asyncio event loops with the asyncio reactor. Set the
:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event
loop class to use it instead of the default asyncio event loop.
+
+
+.. _disable-asyncio:
+
+Switching to a non-asyncio reactor
+==================================
+
+If for some reason your code doesn't work with the asyncio reactor, you can use
+a different reactor by setting the :setting:`TWISTED_REACTOR` setting to its
+import path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to
+``None``, which will use the default reactor for your platform.
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index cc1fe8703..01da53342 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
can configure some extra functions like generating thumbnails and filtering
the images based on their size.
-The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
+The Images Pipeline requires Pillow_ 8.0.0 or greater. It is used for
thumbnailing and normalizing images to JPEG/RGB format.
.. _Pillow: https://github.com/python-pillow/Pillow
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index b2bb7148f..ca0af569f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1911,7 +1911,7 @@ TWISTED_REACTOR
.. versionadded:: 2.0
-Default: ``None``
+Default: ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``
Import path of a given :mod:`~twisted.internet.reactor`.
@@ -1996,17 +1996,19 @@ which raises :exc:`Exception`, becomes:
self.crawler.engine.close_spider(self, "timeout")
-The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
-means that Scrapy will use the existing reactor if one is already installed, or
-install the default reactor defined by Twisted for the current platform. This
-is to maintain backward compatibility and avoid possible problems caused by
-using a non-default reactor.
+If this setting is set ``None``, Scrapy will use the existing reactor if one is
+already installed, or install the default reactor defined by Twisted for the
+current platform.
.. versionchanged:: 2.7
The :command:`startproject` command now sets this setting to
``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated
``settings.py`` file.
+.. versionchanged:: VERSION
+ The default value was changed from ``None`` to
+ ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``.
+
For additional information, see :doc:`core/howto/choosing-reactor`.
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 29dc13f0a..63c6908dc 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -68,7 +68,7 @@ class ImagesPipeline(FilesPipeline):
self._Image = Image
except ImportError:
raise NotConfigured(
- "ImagesPipeline requires installing Pillow 4.0.0 or later"
+ "ImagesPipeline requires installing Pillow 8.0.0 or later"
)
super().__init__(
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index c473b369c..645e50301 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -341,7 +341,7 @@ TELNETCONSOLE_HOST = "127.0.0.1"
TELNETCONSOLE_USERNAME = "scrapy"
TELNETCONSOLE_PASSWORD = None
-TWISTED_REACTOR = None
+TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
SPIDER_CONTRACTS = {}
SPIDER_CONTRACTS_BASE = {
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 0bb31ffaa..db7400af8 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -90,5 +90,4 @@ ROBOTSTXT_OBEY = True
#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
# Set settings whose default value is deprecated to a future-proof value
-TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
FEED_EXPORT_ENCODING = "utf-8"
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index b865cf48d..24e17ecb6 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -182,11 +182,9 @@ def log_scrapy_info(settings: Settings) -> None:
def log_reactor_info() -> None:
- from twisted.internet import reactor
+ from twisted.internet import asyncioreactor, reactor
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
- from twisted.internet import asyncioreactor
-
if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):
logger.debug(
"Using asyncio event loop: %s.%s",
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index b69f43438..2da526cd8 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -18,6 +18,7 @@ from twisted.trial.unittest import SkipTest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.deprecate import create_deprecated_class
+from scrapy.utils.reactor import is_asyncio_reactor_installed
from scrapy.utils.spider import DefaultSpider
if TYPE_CHECKING:
@@ -109,6 +110,19 @@ def get_ftp_content_and_delete(
TestSpider = create_deprecated_class("TestSpider", DefaultSpider)
+def get_reactor_settings() -> dict[str, Any]:
+ """Return a settings dict that works with the installed reactor.
+
+ ``Crawler._apply_settings()`` checks that the installed reactor matches the
+ settings, so tests that run the crawler in the current process may need to
+ pass a correct ``"TWISTED_REACTOR"`` setting value when creating it.
+ """
+ settings: dict[str, Any] = {}
+ if not is_asyncio_reactor_installed():
+ settings["TWISTED_REACTOR"] = None
+ return settings
+
+
def get_crawler(
spidercls: type[Spider] | None = None,
settings_dict: dict[str, Any] | None = None,
@@ -120,9 +134,12 @@ def get_crawler(
"""
from scrapy.crawler import CrawlerRunner
- # Set by default settings that prevent deprecation warnings.
- settings: dict[str, Any] = {}
- settings.update(settings_dict or {})
+ # When needed, useful settings can be added here, e.g. ones that prevent
+ # deprecation warnings.
+ settings: dict[str, Any] = {
+ **get_reactor_settings(),
+ **(settings_dict or {}),
+ }
runner = CrawlerRunner(settings)
crawler = runner.create_crawler(spidercls or DefaultSpider)
crawler._apply_settings()
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py
index f013eed27..0c380610d 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@@ -1,14 +1,8 @@
-import asyncio
-import sys
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from scrapy.utils.reactor import install_reactor
-from twisted.internet import asyncioreactor
-
-if sys.platform == "win32":
- asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-asyncioreactor.install(asyncio.get_event_loop())
-
-import scrapy # noqa: E402
-from scrapy.crawler import CrawlerProcess # noqa: E402
+install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
class NoRequestsSpider(scrapy.Spider):
diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py
index 078cb72cb..e2933338b 100644
--- a/tests/CrawlerProcess/reactor_default.py
+++ b/tests/CrawlerProcess/reactor_default.py
@@ -1,4 +1,5 @@
from twisted.internet import reactor # noqa: F401
+from twisted.python import log
import scrapy
from scrapy.crawler import CrawlerProcess
@@ -13,5 +14,6 @@ class NoRequestsSpider(scrapy.Spider):
process = CrawlerProcess(settings={})
-process.crawl(NoRequestsSpider)
+d = process.crawl(NoRequestsSpider)
+d.addErrback(log.err)
process.start()
diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py
index 814a2a46d..b61e52625 100644
--- a/tests/CrawlerProcess/reactor_select.py
+++ b/tests/CrawlerProcess/reactor_select.py
@@ -1,4 +1,5 @@
from twisted.internet import selectreactor
+from twisted.python import log
import scrapy
from scrapy.crawler import CrawlerProcess
@@ -15,5 +16,6 @@ class NoRequestsSpider(scrapy.Spider):
process = CrawlerProcess(settings={})
-process.crawl(NoRequestsSpider)
+d = process.crawl(NoRequestsSpider)
+d.addErrback(log.err)
process.start()
diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
index 2f1bb7713..b1b297777 100644
--- a/tests/CrawlerRunner/ip_address.py
+++ b/tests/CrawlerRunner/ip_address.py
@@ -1,3 +1,9 @@
+# ruff: noqa: E402
+
+from scrapy.utils.reactor import install_reactor
+
+install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+
from urllib.parse import urlparse
from twisted.internet import reactor
diff --git a/tests/test_addons.py b/tests/test_addons.py
index 686bf9952..b4294c815 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -9,7 +9,7 @@ from scrapy import Spider
from scrapy.crawler import Crawler, CrawlerRunner
from scrapy.exceptions import NotConfigured
from scrapy.settings import BaseSettings, Settings
-from scrapy.utils.test import get_crawler
+from scrapy.utils.test import get_crawler, get_reactor_settings
class SimpleAddon:
@@ -105,6 +105,7 @@ class TestAddonManager(unittest.TestCase):
}
settings_dict = {
"ADDONS": {get_addon_cls(config): 1},
+ **get_reactor_settings(),
}
crawler = get_crawler(settings_dict=settings_dict)
assert crawler.settings.getint("KEY") == 15
@@ -119,6 +120,7 @@ class TestAddonManager(unittest.TestCase):
settings_dict = {
"KEY": 20, # priority=project
"ADDONS": {get_addon_cls(config): 1},
+ **get_reactor_settings(),
}
settings = Settings(settings_dict)
settings.set("KEY", 0, priority="default")
@@ -196,6 +198,7 @@ class TestAddonManager(unittest.TestCase):
return spider
settings = Settings()
+ settings.setdict(get_reactor_settings())
settings.set("KEY", "default", priority="default")
runner = CrawlerRunner(settings)
crawler = runner.create_crawler(MySpider)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 6f4045fc8..f49deac1f 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -18,7 +18,7 @@ from scrapy.exceptions import StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.python import to_unicode
-from scrapy.utils.test import get_crawler
+from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver import MockServer
from tests.spiders import (
@@ -412,7 +412,7 @@ with multiples lines
@defer.inlineCallbacks
def test_crawl_multiple(self):
- runner = CrawlerRunner()
+ runner = CrawlerRunner(get_reactor_settings())
runner.crawl(
SimpleSpider,
self.mockserver.url("/status?n=200"),
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 0bbcc0843..98352b66e 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -25,7 +25,7 @@ from scrapy.settings import Settings, default_settings
from scrapy.spiderloader import SpiderLoader
from scrapy.utils.log import configure_logging, get_scrapy_root_handler
from scrapy.utils.spider import DefaultSpider
-from scrapy.utils.test import get_crawler
+from scrapy.utils.test import get_crawler, get_reactor_settings
from tests.mockserver import MockServer, get_mockserver_env
BASE_SETTINGS: dict[str, Any] = {}
@@ -35,6 +35,7 @@ def get_raw_crawler(spidercls=None, settings_dict=None):
"""get_crawler alternative that only calls the __init__ method of the
crawler."""
settings = Settings()
+ settings.setdict(get_reactor_settings())
settings.setdict(settings_dict or {})
return Crawler(spidercls or DefaultSpider, settings)
@@ -48,7 +49,12 @@ class TestBaseCrawler(unittest.TestCase):
class TestCrawler(TestBaseCrawler):
def test_populate_spidercls_settings(self):
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
- project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"}
+ project_settings = {
+ **BASE_SETTINGS,
+ "TEST1": "project",
+ "TEST3": "project",
+ **get_reactor_settings(),
+ }
class CustomSettingsSpider(DefaultSpider):
custom_settings = spider_settings
@@ -581,7 +587,7 @@ class NoRequestsSpider(scrapy.Spider):
@pytest.mark.usefixtures("reactor_pytest")
class TestCrawlerRunnerHasSpider(unittest.TestCase):
def _runner(self):
- return CrawlerRunner()
+ return CrawlerRunner(get_reactor_settings())
@inlineCallbacks
def test_crawler_runner_bootstrap_successful(self):
@@ -626,13 +632,7 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase):
@inlineCallbacks
def test_crawler_runner_asyncio_enabled_true(self):
- if self.reactor_pytest == "asyncio":
- CrawlerRunner(
- settings={
- "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
- }
- )
- else:
+ if self.reactor_pytest == "default":
runner = CrawlerRunner(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
@@ -643,6 +643,12 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase):
match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
):
yield runner.crawl(NoRequestsSpider)
+ else:
+ CrawlerRunner(
+ settings={
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ }
+ )
class ScriptRunnerMixin:
@@ -672,7 +678,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
- not in log
+ in log
)
def test_multi(self):
@@ -680,18 +686,17 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
- not in log
+ in log
)
assert "ReactorAlreadyInstalledError" not in log
def test_reactor_default(self):
log = self.run_script("reactor_default.py")
- assert "Spider closed (finished)" in log
+ assert "Spider closed (finished)" not in log
assert (
- "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
- not in log
- )
- assert "ReactorAlreadyInstalledError" not in log
+ "does not match the requested one "
+ "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
+ ) in log
def test_reactor_default_twisted_reactor_select(self):
log = self.run_script("reactor_default_twisted_reactor_select.py")
@@ -716,8 +721,11 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
def test_reactor_select(self):
log = self.run_script("reactor_select.py")
- assert "Spider closed (finished)" in log
- assert "ReactorAlreadyInstalledError" not in log
+ assert "Spider closed (finished)" not in log
+ assert (
+ "does not match the requested one "
+ "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
+ ) in log
def test_reactor_select_twisted_reactor_select(self):
log = self.run_script("reactor_select_twisted_reactor_select.py")
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index 162747581..c2df67c66 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -33,7 +33,7 @@ class TestScrapyUtils:
tox_config_file_path = Path(__file__).parent / ".." / "tox.ini"
config_parser = ConfigParser()
config_parser.read(tox_config_file_path)
- pattern = r"Twisted\[http2\]==([\d.]+)"
+ pattern = r"Twisted==([\d.]+)"
match = re.search(pattern, config_parser["pinned"]["deps"])
pinned_twisted_version_string = match[1]
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 19bd02498..bc18e76e1 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -307,7 +307,7 @@ class TestHttp(unittest.TestCase, ABC):
@defer.inlineCallbacks
def test_timeout_download_from_spider_nodata_rcvd(self):
- if self.reactor_pytest == "asyncio" and sys.platform == "win32":
+ if self.reactor_pytest != "default" and sys.platform == "win32":
# https://twistedmatrix.com/trac/ticket/10279
raise unittest.SkipTest(
"This test produces DirtyReactorAggregateError on Windows with asyncio"
@@ -322,7 +322,7 @@ class TestHttp(unittest.TestCase, ABC):
@defer.inlineCallbacks
def test_timeout_download_from_spider_server_hangs(self):
- if self.reactor_pytest == "asyncio" and sys.platform == "win32":
+ if self.reactor_pytest != "default" and sys.platform == "win32":
# https://twistedmatrix.com/trac/ticket/10279
raise unittest.SkipTest(
"This test produces DirtyReactorAggregateError on Windows with asyncio"
@@ -1136,7 +1136,7 @@ class TestFTPBase(unittest.TestCase):
class TestFTP(TestFTPBase):
def test_invalid_credentials(self):
- if self.reactor_pytest == "asyncio" and sys.platform == "win32":
+ if self.reactor_pytest != "default" and sys.platform == "win32":
raise unittest.SkipTest(
"This test produces DirtyReactorAggregateError on Windows with asyncio"
)
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 15b3ad5af..4fca9eefb 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -64,7 +64,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_delay(self):
- crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider)
+ crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
yield crawler.crawl(mockserver=self.mockserver)
slots = crawler.engine.downloader.slots
times = crawler.spider.times
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index ca5ffdc26..85bd42857 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -1,9 +1,11 @@
+from __future__ import annotations
+
import datetime
-import typing
import unittest
+from typing import Any, Callable
-from scrapy.crawler import Crawler
from scrapy.extensions.periodic_log import PeriodicLog
+from scrapy.utils.test import get_crawler
from .spiders import MetaSpider
@@ -59,9 +61,8 @@ class CustomPeriodicLog(PeriodicLog):
self.stats._stats = stats_dump_2
-def extension(settings=None):
- crawler = Crawler(MetaSpider, settings=settings)
- crawler._apply_settings()
+def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog:
+ crawler = get_crawler(MetaSpider, settings)
return CustomPeriodicLog.from_crawler(crawler)
@@ -94,7 +95,7 @@ class TestPeriodicLog(unittest.TestCase):
ext.spider_closed(spider, reason="finished")
return ext, a, b
- def check(settings: dict, condition: typing.Callable):
+ def check(settings: dict[str, Any], condition: Callable) -> None:
ext, a, b = emulate(settings)
assert list(a["delta"].keys()) == [
k for k, v in ext.stats._stats.items() if condition(k, v)
@@ -151,7 +152,7 @@ class TestPeriodicLog(unittest.TestCase):
ext.spider_closed(spider, reason="finished")
return ext, a, b
- def check(settings: dict, condition: typing.Callable):
+ def check(settings: dict[str, Any], condition: Callable) -> None:
ext, a, b = emulate(settings)
assert list(a["stats"].keys()) == [
k for k, v in ext.stats._stats.items() if condition(k, v)
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 162dfdaf4..c5f1b6321 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -3,18 +3,22 @@ from __future__ import annotations
import shutil
from pathlib import Path
from tempfile import mkdtemp
+from typing import TYPE_CHECKING, Any
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase
from w3lib.url import add_or_replace_parameter
-from scrapy import signals
-from scrapy.crawler import CrawlerRunner
+from scrapy import Spider, signals
from scrapy.utils.misc import load_object
+from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
from tests.spiders import SimpleSpider
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
+
class MediaDownloadSpider(SimpleSpider):
name = "mediadownload"
@@ -80,7 +84,6 @@ class TestFileDownloadCrawl(TestCase):
"ITEM_PIPELINES": {self.pipeline_class: 1},
self.store_setting_key: str(self.tmpmediastore),
}
- self.runner = CrawlerRunner(self.settings)
self.items = []
def tearDown(self):
@@ -90,10 +93,12 @@ class TestFileDownloadCrawl(TestCase):
def _on_item_scraped(self, item):
self.items.append(item)
- def _create_crawler(self, spider_class, runner=None, **kwargs):
- if runner is None:
- runner = self.runner
- crawler = runner.create_crawler(spider_class, **kwargs)
+ def _create_crawler(
+ self, spider_class: type[Spider], settings: dict[str, Any] | None = None
+ ) -> Crawler:
+ if settings is None:
+ settings = self.settings
+ crawler = get_crawler(spider_class, settings)
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
return crawler
@@ -175,10 +180,11 @@ class TestFileDownloadCrawl(TestCase):
@defer.inlineCallbacks
def test_download_media_redirected_allowed(self):
- settings = dict(self.settings)
- settings.update({"MEDIA_ALLOW_REDIRECTS": True})
- runner = CrawlerRunner(settings)
- crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner)
+ settings = {
+ **self.settings,
+ "MEDIA_ALLOW_REDIRECTS": True,
+ }
+ crawler = self._create_crawler(RedirectedMediaDownloadSpider, settings)
with LogCapture() as log:
yield crawler.crawl(
self.mockserver.url("/files/images/"),
@@ -201,8 +207,7 @@ class TestFileDownloadCrawl(TestCase):
**self.settings,
"ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1},
}
- runner = CrawlerRunner(settings)
- crawler = self._create_crawler(MediaDownloadSpider, runner=runner)
+ crawler = self._create_crawler(MediaDownloadSpider, settings)
with LogCapture() as log:
yield crawler.crawl(
self.mockserver.url("/files/images/"),
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 4e8330c06..aaf72390d 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -27,7 +27,7 @@ from scrapy.spiders import (
XMLFeedSpider,
)
from scrapy.spiders.init import InitSpider
-from scrapy.utils.test import get_crawler
+from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import get_testdata, tests_datadir
@@ -108,7 +108,11 @@ class TestSpider(unittest.TestCase):
@inlineCallbacks
def test_settings_in_from_crawler(self):
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
- project_settings = {"TEST1": "project", "TEST3": "project"}
+ project_settings = {
+ "TEST1": "project",
+ "TEST3": "project",
+ **get_reactor_settings(),
+ }
class TestSpider(self.spider_class):
name = "test"
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index a65a36219..901e03d59 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -2,6 +2,7 @@ import asyncio
import warnings
import pytest
+from twisted.trial.unittest import TestCase
from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.reactor import (
@@ -12,10 +13,10 @@ from scrapy.utils.reactor import (
@pytest.mark.usefixtures("reactor_pytest")
-class TestAsyncio:
+class TestAsyncio(TestCase):
def test_is_asyncio_reactor_installed(self):
# the result should depend only on the pytest --reactor argument
- assert is_asyncio_reactor_installed() == (self.reactor_pytest == "asyncio")
+ assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default")
def test_install_asyncio_reactor(self):
from twisted.internet import reactor as original_reactor
diff --git a/tox.ini b/tox.ini
index 70c841603..eb084f0f5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -26,7 +26,7 @@ deps =
{[test-requirements]deps}
# mitmproxy does not support PyPy
- mitmproxy; implementation_name != 'pypy'
+ mitmproxy; implementation_name != "pypy"
setenv =
COVERAGE_CORE=sysmon
passenv =
@@ -96,19 +96,18 @@ commands =
[pinned]
basepython = python3.9
deps =
+ Protego==0.1.15
+ Twisted==21.7.0
cryptography==37.0.0
cssselect==0.9.1
- h2==3.0
itemadapter==0.1.0
+ lxml==4.6.0
parsel==1.5.0
- Protego==0.1.15
pyOpenSSL==22.0.0
queuelib==1.4.2
service_identity==18.1.0
- Twisted[http2]==21.7.0
w3lib==1.17.0
zope.interface==5.1.0
- lxml==4.6.0
{[test-requirements]deps}
# mitmproxy 8.0.0 requires upgrading some of the pinned dependencies
@@ -131,60 +130,50 @@ setenv =
{[pinned]setenv}
commands = {[pinned]commands}
-[testenv:windows-pinned]
-basepython = {[pinned]basepython}
-deps =
- {[pinned]deps}
- PyDispatcher==2.0.5
-install_command = {[pinned]install_command}
-setenv =
- {[pinned]setenv}
-commands = {[pinned]commands}
-
[testenv:extra-deps]
basepython = python3
deps =
{[testenv]deps}
- boto3
- google-cloud-storage
- robotexclusionrulesparser
Pillow
Twisted[http2]
- uvloop; platform_system != "Windows"
+ boto3
bpython # optional for shell wrapper tests
- brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests
- brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests
- zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests
+ brotli; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests
+ brotlicffi; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests
+ google-cloud-storage
ipython
+ robotexclusionrulesparser
+ uvloop; platform_system != "Windows"
+ zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests
[testenv:extra-deps-pinned]
basepython = {[pinned]basepython}
deps =
{[pinned]deps}
+ Pillow==8.0.0
boto3==1.20.0
+ bpython==0.7.1
+ brotli==0.5.2; implementation_name != "pypy"
+ brotlicffi==0.8.0; implementation_name == "pypy"
+ brotlipy
google-cloud-storage==1.29.0
- Pillow==7.1.0
+ ipython==2.0.0
robotexclusionrulesparser==1.6.2
- brotlipy
uvloop==0.14.0; platform_system != "Windows"
- bpython==0.7.1
- zstandard==0.1; implementation_name != 'pypy'
- ipython==2.0.0
- brotli==0.5.2; implementation_name != 'pypy'
- brotlicffi==0.8.0; implementation_name == 'pypy'
+ zstandard==0.1; implementation_name != "pypy"
install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}
commands = {[pinned]commands}
-[testenv:asyncio]
+[testenv:default-reactor]
commands =
- {[testenv]commands} --reactor=asyncio
+ {[testenv]commands} --reactor=default
-[testenv:asyncio-pinned]
+[testenv:default-reactor-pinned]
basepython = {[pinned]basepython}
deps = {[testenv:pinned]deps}
-commands = {[pinned]commands} --reactor=asyncio
+commands = {[pinned]commands} --reactor=default
install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}
@@ -204,21 +193,20 @@ commands = {[testenv:pypy3]commands}
[testenv:pypy3-pinned]
basepython = pypy3.10
deps =
+ PyPyDispatcher==2.1.0
+ {[test-requirements]deps}
+ Protego==0.1.15
+ Twisted==21.7.0
cryptography==41.0.5
cssselect==0.9.1
- h2==3.1
itemadapter==0.1.0
+ lxml==4.6.0
parsel==1.5.0
- Protego==0.1.15
pyOpenSSL==23.3.0
queuelib==1.4.2
service_identity==18.1.0
- Twisted[http2]==21.7.0
w3lib==1.17.0
zope.interface==5.1.0
- lxml==4.6.0
- {[test-requirements]deps}
- PyPyDispatcher==2.1.0
commands =
; disabling both coverage and docs tests
pytest {posargs:--durations=10 scrapy tests}