Scrapy, a fast high-level web crawling & scraping framework for Python.
Test restructuring (#7736)
Andrey Rakhmatullin4ee3676464d9
diff --git a/tests/spiders.py b/tests/spiders.pyindex 612dc11c9..da14fdbe3 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -571,3 +571,19 @@ class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):def headers_received(self, headers, body_length, request, spider):self.meta["headers_received"] = headersraise StopDownload(fail=True)+++class ExceptionSpider(Spider):+ name = "exception"++ @classmethod+ def from_crawler(cls, crawler, *args, **kwargs):+ raise ValueError("Exception in from_crawler method")+++class NoRequestsSpider(Spider):+ name = "no_request"++ async def start(self):+ return+ yielddiff --git a/tests/test_command_check.py b/tests/test_command_check.pyindex 815f87026..794c6d400 100644--- a/tests/test_command_check.py+++ b/tests/test_command_check.py@@ -7,7 +7,7 @@ from unittest import TestCasefrom unittest.mock import MagicMock, Mock, PropertyMock, call, patchfrom scrapy.commands.check import Command, TextTestResult-from tests.test_commands import TestProjectBase+from tests.utils.base_commands import TestProjectBasefrom tests.utils.cmdline import procif TYPE_CHECKING:diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.pyindex dd69d36ba..6293e973e 100644--- a/tests/test_command_crawl.py+++ b/tests/test_command_crawl.py@@ -2,7 +2,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-from tests.test_commands import TestProjectBase+from tests.utils.base_commands import TestProjectBasefrom tests.utils.cmdline import procif TYPE_CHECKING:diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.pyindex 94db14c30..f465d0b30 100644--- a/tests/test_command_genspider.py+++ b/tests/test_command_genspider.py@@ -6,15 +6,8 @@ from pathlib import Pathimport pytest-from tests.test_commands import TestProjectBase-from tests.utils.cmdline import call, proc---def write_recording_editor(editor: Path) -> None:- """Create an executable editor script that writes the path it is asked to- open (its last argument) into the file given as its first argument."""- editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")- editor.chmod(0o755)+from tests.utils.base_commands import TestProjectBase+from tests.utils.cmdline import call, proc, write_recording_editordef find_in_file(filename: Path, regex: str) -> re.Match[str] | None:diff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex c210a06e8..a3581d764 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -8,7 +8,7 @@ import pytestfrom scrapy.commands import parsefrom scrapy.settings import Settings-from tests.test_commands import TestProjectBase+from tests.utils.base_commands import TestProjectBasefrom tests.utils.cmdline import call, procif TYPE_CHECKING:diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.pyindex 9dabbc942..2b410b5c6 100644--- a/tests/test_command_runspider.py+++ b/tests/test_command_runspider.py@@ -8,7 +8,7 @@ from typing import TYPE_CHECKINGimport pytest-from tests.test_crawler import ExceptionSpider, NoRequestsSpider+from tests.spiders import ExceptionSpider, NoRequestsSpiderfrom tests.utils.cmdline import procif TYPE_CHECKING:@@ -65,13 +65,14 @@ class BadSpider(scrapy.Spider):def test_run_fail_spider(self, tmp_path: Path) -> None:ret, _, _ = self.runspider(- tmp_path, "import scrapy\n" + inspect.getsource(ExceptionSpider)+ tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider))assert ret != 0def test_run_good_spider(self, tmp_path: Path) -> None:ret, _, _ = self.runspider(- tmp_path, "import scrapy\n" + inspect.getsource(NoRequestsSpider)+ tmp_path,+ "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider),)assert ret == 0diff --git a/tests/test_commands.py b/tests/test_commands.pyindex bb4d96f00..9e7d4d5a1 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -4,7 +4,6 @@ import argparseimport jsonimport sysfrom io import StringIO-from shutil import copytreefrom typing import TYPE_CHECKINGfrom unittest import mock@@ -16,7 +15,8 @@ from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, viewfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.settings import Settingsfrom scrapy.utils.reactor import _asyncio_reactor_path-from tests.utils.cmdline import call, proc+from tests.utils.base_commands import TestProjectBase+from tests.utils.cmdline import call, proc, write_recording_editorif TYPE_CHECKING:from pathlib import Path@@ -109,29 +109,6 @@ class TestCommandSettings:)-class TestProjectBase:- """A base class for tests that may need a Scrapy project."""-- project_name = "testproject"-- @pytest.fixture(scope="session")- def _proj_path_cached(self, tmp_path_factory: pytest.TempPathFactory) -> Path:- """Create a Scrapy project in a temporary directory and return its path.-- Used as a cache for ``proj_path``.- """- tmp_path = tmp_path_factory.mktemp("proj")- call("startproject", self.project_name, cwd=tmp_path)- return tmp_path / self.project_name-- @pytest.fixture- def proj_path(self, tmp_path: Path, _proj_path_cached: Path) -> Path:- """Copy a pre-generated Scrapy project into a temporary directory and return its path."""- proj_path = tmp_path / self.project_name- copytree(_proj_path_cached, proj_path)- return proj_path--class TestCommandCrawlerProcess(TestProjectBase):"""Test that the command uses the expected kind of *CrawlerProcessand produces expected errors when needed."""@@ -430,9 +407,7 @@ class TestEditCommand(TestProjectBase):spider = proj_path / self.project_name / "spiders" / "example.py"edited = proj_path / "edited.txt"editor = proj_path / "fake-editor.sh"- # Records the file it is asked to open ($2) into the file given as $1.- editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")- editor.chmod(0o755)+ write_recording_editor(editor)monkeypatch.setenv("EDITOR", f"{editor} {edited}")assert call("genspider", "example", "example.com", cwd=proj_path) == 0diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 8c1716345..9c479068a 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -14,14 +14,13 @@ from twisted.internet.ssl import Certificatefrom twisted.python.failure import Failurefrom scrapy import Spider, signals-from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunnerfrom scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownloadfrom scrapy.http import Requestfrom scrapy.http.response import Response-from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.engine import format_engine_status, get_engine_statusfrom scrapy.utils.python import to_unicode-from scrapy.utils.test import get_crawler, get_reactor_settings+from scrapy.utils.test import get_crawlerfrom tests import NON_EXISTING_RESOLVABLEfrom tests.spiders import (AsyncDefAsyncioGenComplexSpider,@@ -430,50 +429,6 @@ with multiples lines)assert not crawler.crawling- @coroutine_test- async def test_crawlerrunner_accepts_crawler(- self, caplog: pytest.LogCaptureFixture, mockserver: MockServer- ) -> None:- crawler = Crawler(SimpleSpider, get_reactor_settings())- runner = CrawlerRunner()- with caplog.at_level(logging.DEBUG):- await maybe_deferred_to_future(- runner.crawl(- crawler,- mockserver.url("/status?n=200"),- mockserver=mockserver,- )- )- assert "Got response 200" in caplog.text-- @coroutine_test- async def test_crawl_multiple(- self, caplog: pytest.LogCaptureFixture, mockserver: MockServer- ) -> None:- settings_dict = get_reactor_settings()- runner_cls = (- CrawlerRunner- if settings_dict.get("TWISTED_REACTOR_ENABLED", True)- else AsyncCrawlerRunner- )- runner = runner_cls(settings_dict)- runner.crawl(- SimpleSpider,- mockserver.url("/status?n=200"),- mockserver=mockserver,- )- runner.crawl(- SimpleSpider,- mockserver.url("/status?n=503"),- mockserver=mockserver,- )-- with caplog.at_level(logging.DEBUG):- await ensure_awaitable(runner.join())-- self._assert_retried(caplog.text)- assert "Got response 200" in caplog.text-@coroutine_testasync def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None:crawler = get_crawler(SimpleSpider)diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 6a1eea027..b4f906e25 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -10,22 +10,14 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom unittest.mock import MagicMockimport pytest-from zope.interface.exceptions import MultipleInvalidimport scrapyfrom scrapy import Spider-from scrapy.crawler import (- AsyncCrawlerProcess,- AsyncCrawlerRunner,- Crawler,- CrawlerProcess,- CrawlerRunner,- CrawlerRunnerBase,-)+from scrapy.crawler import AsyncCrawlerProcess, Crawler, CrawlerProcessfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions.throttle import AutoThrottlefrom scrapy.settings import Settings, default_settings-from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future+from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.log import (_uninstall_scrapy_root_handler,configure_logging,@@ -33,6 +25,8 @@ from scrapy.utils.log import ()from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler, get_reactor_settings+from tests.spiders import NoRequestsSpider+from tests.utils import assert_option_is_defaultfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:@@ -52,14 +46,7 @@ def get_raw_crawler(return Crawler(spidercls or DefaultSpider, settings)-class TestBaseCrawler:- @staticmethod- def assertOptionIsDefault(settings: Settings, key: str) -> None:- assert isinstance(settings, Settings)- assert settings[key] == getattr(default_settings, key)---class TestCrawler(TestBaseCrawler):+class TestCrawler:def test_populate_spidercls_settings(self) -> None:spider_settings: dict[str, Any] = {"TEST1": "spider",@@ -90,11 +77,11 @@ class TestCrawler(TestBaseCrawler):def test_crawler_accepts_dict(self) -> None:crawler = get_crawler(DefaultSpider, {"foo": "bar"})assert crawler.settings["foo"] == "bar"- self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")+ assert_option_is_default(crawler.settings, "RETRY_ENABLED")def test_crawler_accepts_None(self) -> None:crawler = Crawler(DefaultSpider)- self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")+ assert_option_is_default(crawler.settings, "RETRY_ENABLED")def test_crawler_rejects_spider_objects(self) -> None:with pytest.raises(ValueError, match="spidercls argument must be a class"):@@ -586,78 +573,6 @@ class TestCrawlerLogging:assert "debug message" in logged-class SpiderLoaderWithWrongInterface:- def unneeded_method(self) -> None:- pass---class TestCrawlerRunner(TestBaseCrawler):- def test_spider_manager_verify_interface(self) -> None:- settings = Settings(- {- "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,- }- )- with pytest.raises(MultipleInvalid):- CrawlerRunner(settings)-- def test_crawler_runner_accepts_dict(self) -> None:- runner = CrawlerRunner({"foo": "bar"})- assert runner.settings["foo"] == "bar"- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")-- def test_crawler_runner_accepts_None(self) -> None:- runner = CrawlerRunner()- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")---class TestAsyncCrawlerRunner(TestBaseCrawler):- def test_spider_manager_verify_interface(self) -> None:- settings = Settings(- {- "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,- }- )- with pytest.raises(MultipleInvalid):- AsyncCrawlerRunner(settings)-- def test_crawler_runner_accepts_dict(self) -> None:- runner = AsyncCrawlerRunner({"foo": "bar"})- assert runner.settings["foo"] == "bar"- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")-- def test_crawler_runner_accepts_None(self) -> None:- runner = AsyncCrawlerRunner()- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")---class TestCrawlerProcess(TestBaseCrawler):- def test_crawler_process_accepts_dict(self) -> None:- runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)- assert runner.settings["foo"] == "bar"- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")-- def test_crawler_process_accepts_None(self) -> None:- runner = CrawlerProcess(install_root_handler=False)- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")--[email protected]_asyncio-class TestAsyncCrawlerProcess(TestBaseCrawler):- def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:- runner = AsyncCrawlerProcess(- {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},- install_root_handler=False,- )- assert runner.settings["foo"] == "bar"- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")-- @pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False- def test_crawler_process_accepts_None(self) -> None:- runner = AsyncCrawlerProcess(install_root_handler=False)- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")--class TestAsyncCrawlerProcessReactorlessHelpers:"""Unit tests for the reactorless shutdown helpers of AsyncCrawlerProcess.@@ -797,161 +712,6 @@ class TestAsyncCrawlerProcessReactorlessHelpers:)[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])-def test_runner_settings_applied_to_crawler_instance(- runner_cls: type[CrawlerRunnerBase],-) -> None:- runner = runner_cls({"FOO": "runner"})- crawler = Crawler(DefaultSpider)- result = runner.create_crawler(crawler)- assert result is crawler- assert result.settings["FOO"] == "runner"--[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])-def test_spider_custom_settings_override_runner(- runner_cls: type[CrawlerRunnerBase],-) -> None:- class MySpider(DefaultSpider):- custom_settings = {"FOO": "spider"}-- runner = runner_cls({"FOO": "runner"})- crawler = Crawler(MySpider)- runner.create_crawler(crawler)- assert crawler.settings["FOO"] == "spider"---def test_create_crawler_instance_consistent_with_spider_class() -> None:- runner = AsyncCrawlerRunner({"FOO": "runner"})-- crawler_from_class = runner.create_crawler(DefaultSpider)-- pre_built = Crawler(DefaultSpider)- runner.create_crawler(pre_built)-- assert crawler_from_class.settings["FOO"] == "runner"- assert pre_built.settings["FOO"] == "runner"--[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])-def test_create_crawler_rejects_spider_object(- runner_cls: type[CrawlerRunnerBase],-) -> None:- runner = runner_cls()- with pytest.raises(ValueError, match="cannot be a spider object"):- runner.create_crawler(DefaultSpider()) # type: ignore[arg-type]--[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])-def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:- runner = runner_cls()- with pytest.raises(ValueError, match="cannot be a spider object"):- runner.crawl(DefaultSpider()) # type: ignore[arg-type]---class ExceptionSpider(scrapy.Spider):- name = "exception"-- @classmethod- def from_crawler(cls, crawler, *args, **kwargs):- raise ValueError("Exception in from_crawler method")---class NoRequestsSpider(scrapy.Spider):- name = "no_request"-- async def start(self):- return- yield--[email protected]_reactor # CrawlerRunner requires a reactor-class TestCrawlerRunnerHasSpider:- @pytest.fixture- def runner(self) -> CrawlerRunnerBase:- return CrawlerRunner(get_reactor_settings())-- @staticmethod- async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:- await ensure_awaitable(runner.crawl(spider))-- @coroutine_test- async def test_crawler_runner_bootstrap_successful(- self, runner: CrawlerRunnerBase- ) -> None:- await self._crawl(runner, NoRequestsSpider)- assert not runner.bootstrap_failed-- @coroutine_test- async def test_crawler_runner_bootstrap_successful_for_several(- self, runner: CrawlerRunnerBase- ) -> None:- await self._crawl(runner, NoRequestsSpider)- await self._crawl(runner, NoRequestsSpider)- assert not runner.bootstrap_failed-- @coroutine_test- async def test_crawler_runner_bootstrap_failed(- self, runner: CrawlerRunnerBase- ) -> None:- try:- await self._crawl(runner, ExceptionSpider)- except ValueError:- pass- else:- pytest.fail("Exception should be raised from spider")-- assert runner.bootstrap_failed-- @coroutine_test- async def test_crawler_runner_bootstrap_failed_for_several(- self, runner: CrawlerRunnerBase- ) -> None:- try:- await self._crawl(runner, ExceptionSpider)- except ValueError:- pass- else:- pytest.fail("Exception should be raised from spider")-- await self._crawl(runner, NoRequestsSpider)-- assert runner.bootstrap_failed-- @coroutine_test- async def test_crawler_runner_asyncio_enabled_true(- self, reactor_pytest: str- ) -> None:- if reactor_pytest != "asyncio":- runner = CrawlerRunner(- settings={- "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",- }- )- with pytest.raises(- Exception,- match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",- ):- await self._crawl(runner, NoRequestsSpider)- else:- runner = CrawlerRunner(- settings={- "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",- }- )- await self._crawl(runner, NoRequestsSpider)--[email protected]_asyncio-class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):- @pytest.fixture- def runner(self) -> CrawlerRunnerBase:- return AsyncCrawlerRunner(get_reactor_settings())-- def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override]- pytest.skip("This test is only for CrawlerRunner")--@pytest.mark.parametrize(("settings", "items"),[diff --git a/tests/test_crawler_runners.py b/tests/test_crawler_runners.pynew file mode 100644index 000000000..2f3649b43--- /dev/null+++ b/tests/test_crawler_runners.py@@ -0,0 +1,284 @@+from __future__ import annotations++import logging+from typing import TYPE_CHECKING++import pytest+from zope.interface.exceptions import MultipleInvalid++from scrapy.crawler import (+ AsyncCrawlerProcess,+ AsyncCrawlerRunner,+ Crawler,+ CrawlerProcess,+ CrawlerRunner,+ CrawlerRunnerBase,+)+from scrapy.settings import Settings+from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future+from scrapy.utils.spider import DefaultSpider+from scrapy.utils.test import get_reactor_settings+from tests.spiders import ExceptionSpider, NoRequestsSpider, SimpleSpider+from tests.utils import assert_option_is_default+from tests.utils.decorators import coroutine_test++if TYPE_CHECKING:+ from scrapy import Spider+ from tests.mockserver.http import MockServer+++class SpiderLoaderWithWrongInterface:+ def unneeded_method(self) -> None:+ pass+++class TestCrawlerRunner:+ def test_spider_manager_verify_interface(self) -> None:+ settings = Settings(+ {+ "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,+ }+ )+ with pytest.raises(MultipleInvalid):+ CrawlerRunner(settings)++ def test_crawler_runner_accepts_dict(self) -> None:+ runner = CrawlerRunner({"foo": "bar"})+ assert runner.settings["foo"] == "bar"+ assert_option_is_default(runner.settings, "RETRY_ENABLED")++ def test_crawler_runner_accepts_None(self) -> None:+ runner = CrawlerRunner()+ assert_option_is_default(runner.settings, "RETRY_ENABLED")+++class TestAsyncCrawlerRunner:+ def test_spider_manager_verify_interface(self) -> None:+ settings = Settings(+ {+ "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,+ }+ )+ with pytest.raises(MultipleInvalid):+ AsyncCrawlerRunner(settings)++ def test_crawler_runner_accepts_dict(self) -> None:+ runner = AsyncCrawlerRunner({"foo": "bar"})+ assert runner.settings["foo"] == "bar"+ assert_option_is_default(runner.settings, "RETRY_ENABLED")++ def test_crawler_runner_accepts_None(self) -> None:+ runner = AsyncCrawlerRunner()+ assert_option_is_default(runner.settings, "RETRY_ENABLED")+++class TestCrawlerProcess:+ def test_crawler_process_accepts_dict(self) -> None:+ runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)+ assert runner.settings["foo"] == "bar"+ assert_option_is_default(runner.settings, "RETRY_ENABLED")++ def test_crawler_process_accepts_None(self) -> None:+ runner = CrawlerProcess(install_root_handler=False)+ assert_option_is_default(runner.settings, "RETRY_ENABLED")++[email protected]_asyncio+class TestAsyncCrawlerProcess:+ def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:+ runner = AsyncCrawlerProcess(+ {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},+ install_root_handler=False,+ )+ assert runner.settings["foo"] == "bar"+ assert_option_is_default(runner.settings, "RETRY_ENABLED")++ @pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False+ def test_crawler_process_accepts_None(self) -> None:+ runner = AsyncCrawlerProcess(install_root_handler=False)+ assert_option_is_default(runner.settings, "RETRY_ENABLED")++[email protected]_reactor # CrawlerRunner requires a reactor+class TestCrawlerRunnerHasSpider:+ @pytest.fixture+ def runner(self) -> CrawlerRunnerBase:+ return CrawlerRunner(get_reactor_settings())++ @staticmethod+ async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:+ await ensure_awaitable(runner.crawl(spider))++ @coroutine_test+ async def test_crawler_runner_bootstrap_successful(+ self, runner: CrawlerRunnerBase+ ) -> None:+ await self._crawl(runner, NoRequestsSpider)+ assert not runner.bootstrap_failed++ @coroutine_test+ async def test_crawler_runner_bootstrap_successful_for_several(+ self, runner: CrawlerRunnerBase+ ) -> None:+ await self._crawl(runner, NoRequestsSpider)+ await self._crawl(runner, NoRequestsSpider)+ assert not runner.bootstrap_failed++ @coroutine_test+ async def test_crawler_runner_bootstrap_failed(+ self, runner: CrawlerRunnerBase+ ) -> None:+ try:+ await self._crawl(runner, ExceptionSpider)+ except ValueError:+ pass+ else:+ pytest.fail("Exception should be raised from spider")++ assert runner.bootstrap_failed++ @coroutine_test+ async def test_crawler_runner_bootstrap_failed_for_several(+ self, runner: CrawlerRunnerBase+ ) -> None:+ try:+ await self._crawl(runner, ExceptionSpider)+ except ValueError:+ pass+ else:+ pytest.fail("Exception should be raised from spider")++ await self._crawl(runner, NoRequestsSpider)++ assert runner.bootstrap_failed++ @coroutine_test+ async def test_crawler_runner_asyncio_enabled_true(+ self, reactor_pytest: str+ ) -> None:+ if reactor_pytest != "asyncio":+ runner = CrawlerRunner(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ }+ )+ with pytest.raises(+ Exception,+ match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",+ ):+ await self._crawl(runner, NoRequestsSpider)+ else:+ runner = CrawlerRunner(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ }+ )+ await self._crawl(runner, NoRequestsSpider)++[email protected]_asyncio+class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):+ @pytest.fixture+ def runner(self) -> CrawlerRunnerBase:+ return AsyncCrawlerRunner(get_reactor_settings())++ def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override]+ pytest.skip("This test is only for CrawlerRunner")++[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])+def test_runner_settings_applied_to_crawler_instance(+ runner_cls: type[CrawlerRunnerBase],+) -> None:+ runner = runner_cls({"FOO": "runner"})+ crawler = Crawler(DefaultSpider)+ result = runner.create_crawler(crawler)+ assert result is crawler+ assert result.settings["FOO"] == "runner"++[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])+def test_spider_custom_settings_override_runner(+ runner_cls: type[CrawlerRunnerBase],+) -> None:+ class MySpider(DefaultSpider):+ custom_settings = {"FOO": "spider"}++ runner = runner_cls({"FOO": "runner"})+ crawler = Crawler(MySpider)+ runner.create_crawler(crawler)+ assert crawler.settings["FOO"] == "spider"+++def test_create_crawler_instance_consistent_with_spider_class() -> None:+ runner = AsyncCrawlerRunner({"FOO": "runner"})++ crawler_from_class = runner.create_crawler(DefaultSpider)++ pre_built = Crawler(DefaultSpider)+ runner.create_crawler(pre_built)++ assert crawler_from_class.settings["FOO"] == "runner"+ assert pre_built.settings["FOO"] == "runner"++[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])+def test_create_crawler_rejects_spider_object(+ runner_cls: type[CrawlerRunnerBase],+) -> None:+ runner = runner_cls()+ with pytest.raises(ValueError, match="cannot be a spider object"):+ runner.create_crawler(DefaultSpider()) # type: ignore[arg-type]++[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])+def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:+ runner = runner_cls()+ with pytest.raises(ValueError, match="cannot be a spider object"):+ runner.crawl(DefaultSpider()) # type: ignore[arg-type]+++@coroutine_test+async def test_crawlerrunner_accepts_crawler(+ caplog: pytest.LogCaptureFixture, mockserver: MockServer+) -> None:+ crawler = Crawler(SimpleSpider, get_reactor_settings())+ runner = CrawlerRunner()+ with caplog.at_level(logging.DEBUG):+ await maybe_deferred_to_future(+ runner.crawl(+ crawler,+ mockserver.url("/status?n=200"),+ mockserver=mockserver,+ )+ )+ assert "Got response 200" in caplog.text+++@coroutine_test+async def test_crawl_multiple(+ caplog: pytest.LogCaptureFixture, mockserver: MockServer+) -> None:+ settings_dict = get_reactor_settings()+ runner_cls = (+ CrawlerRunner+ if settings_dict.get("TWISTED_REACTOR_ENABLED", True)+ else AsyncCrawlerRunner+ )+ runner = runner_cls(settings_dict)+ runner.crawl(+ SimpleSpider,+ mockserver.url("/status?n=200"),+ mockserver=mockserver,+ )+ runner.crawl(+ SimpleSpider,+ mockserver.url("/status?n=503"),+ mockserver=mockserver,+ )++ with caplog.at_level(logging.DEBUG):+ await ensure_awaitable(runner.join())++ assert "Got response 200" in caplog.text+ assert "Gave up retrying" in caplog.textdiff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex 1da7bbf3e..0824ab531 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -16,11 +16,8 @@ from scrapy.spiders import Spiderfrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler-from tests.test_downloadermiddleware_redirect_base import (- REDIRECT_SCHEME_CASES,- SCHEME_PARAMS,- Base,-)+from tests.test_downloadermiddleware_redirect_base import Base+from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMSclass TestRedirectMiddleware(Base.Test):diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.pyindex 32935769f..bd8bc796c 100644--- a/tests/test_downloadermiddleware_redirect_base.py+++ b/tests/test_downloadermiddleware_redirect_base.py@@ -1,7 +1,5 @@from __future__ import annotations-from itertools import product-import pytestfrom scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware@@ -10,43 +8,6 @@ from scrapy.http import Request, Responsefrom scrapy.utils.misc import set_environfrom scrapy.utils.test import get_crawler-SCHEME_PARAMS = ("url", "location", "target")-HTTP_SCHEMES = ("http", "https")-NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo")-REDIRECT_SCHEME_CASES = (- # http/https → http/https redirects- *(- (- f"{input_scheme}://example.com/a",- f"{output_scheme}://example.com/b",- f"{output_scheme}://example.com/b",- )- for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2)- ),- # http/https → data/file/ftp/s3/foo does not redirect- *(- (- f"{input_scheme}://example.com/a",- f"{output_scheme}://example.com/b",- None,- )- for input_scheme in HTTP_SCHEMES- for output_scheme in NON_HTTP_SCHEMES- ),- # http/https → relative redirects- *(- (- f"{scheme}://example.com/a",- location,- f"{scheme}://example.com/b",- )- for scheme in HTTP_SCHEMES- for location in ("//example.com/b", "/b")- ),- # Note: We do not test data/file/ftp/s3 schemes for the initial URL- # because their download handlers cannot return a status code of 3xx.-)-class Base:class Test:diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.pyindex d849cc8fb..b5d39080c 100644--- a/tests/test_downloadermiddleware_redirect_metarefresh.py+++ b/tests/test_downloadermiddleware_redirect_metarefresh.py@@ -12,12 +12,12 @@ from scrapy.http import HtmlResponse, Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawler-from tests.test_downloadermiddleware_redirect_base import (+from tests.test_downloadermiddleware_redirect_base import Base+from tests.utils.redirect import (HTTP_SCHEMES,NON_HTTP_SCHEMES,REDIRECT_SCHEME_CASES,SCHEME_PARAMS,- Base,)diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex 082fc743e..f82041a62 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -15,8 +15,8 @@ from scrapy.http.request import NO_CALLBACKfrom scrapy.settings import Settingsfrom scrapy.utils.asyncio import call_laterfrom scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future-from tests.test_robotstxt_interface import rerp_availablefrom tests.utils.decorators import coroutine_test+from tests.utils.robotstxt import rerp_availableif TYPE_CHECKING:from scrapy.crawler import Crawlerdiff --git a/tests/test_engine.py b/tests/test_engine.pyindex 2b857d35a..3ab8f6d1b 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -6,8 +6,8 @@ import subprocessimport sysfrom collections import defaultdictfrom dataclasses import dataclass-from typing import TYPE_CHECKING, Any, cast-from unittest.mock import Mock, call+from typing import TYPE_CHECKING, Any+from unittest.mock import Mockfrom urllib.parse import urlparseimport attr@@ -25,7 +25,6 @@ from scrapy.http import Headers, Request, Responsefrom scrapy.item import Field, Itemfrom scrapy.linkextractors import LinkExtractorfrom scrapy.spiders import Spider-from scrapy.statscollectors import MemoryStatsCollectorfrom scrapy.utils.defer import (_schedule_coro,deferred_from_coro,@@ -42,8 +41,6 @@ if TYPE_CHECKING:from twisted.python.failure import Failure- from scrapy.core.scheduler import Scheduler- from scrapy.crawler import Crawlerfrom tests.mockserver.http import MockServer@@ -515,98 +512,6 @@ class TestEngine(TestEngineBase):assert "AssertionError" not in stderr_str, stderr_str-class TestEngineDownloadAsync:- """Test cases for ExecutionEngine.download_async()."""-- @pytest.fixture- def engine(self) -> ExecutionEngine:- crawler = get_crawler(MySpider)- engine = ExecutionEngine(crawler, lambda _: None)- engine.downloader.close()- engine.downloader = Mock()- engine._slot = Mock()- engine._slot.inprogress = set()- return engine-- @staticmethod- async def _download(engine: ExecutionEngine, request: Request) -> Response:- return await engine.download_async(request)-- @coroutine_test- async def test_download_async_success(self, engine):- """Test basic successful async download of a request."""- request = Request("http://example.com")- response = Response("http://example.com", body=b"test body")- engine.spider = Mock()- engine.downloader.fetch.return_value = defer.succeed(response)- engine._slot.add_request = Mock()- engine._slot.remove_request = Mock()-- result = await self._download(engine, request)- assert result == response- engine._slot.add_request.assert_called_once_with(request)- engine._slot.remove_request.assert_called_once_with(request)- engine.downloader.fetch.assert_called_once_with(request)-- @coroutine_test- async def test_download_async_redirect(self, engine):- """Test async download with a redirect request."""- original_request = Request("http://example.com")- redirect_request = Request("http://example.com/redirect")- final_response = Response("http://example.com/redirect", body=b"redirected")-- # First call returns redirect request, second call returns final response- engine.downloader.fetch.side_effect = [- defer.succeed(redirect_request),- defer.succeed(final_response),- ]- engine.spider = Mock()- engine._slot.add_request = Mock()- engine._slot.remove_request = Mock()-- result = await self._download(engine, original_request)- assert result == final_response- assert engine.downloader.fetch.call_count == 2- engine._slot.add_request.assert_has_calls(- [call(original_request), call(redirect_request)]- )- engine._slot.remove_request.assert_has_calls(- [call(original_request), call(redirect_request)]- )-- @coroutine_test- async def test_download_async_no_spider(self, engine):- """Test async download attempt when no spider is available."""- request = Request("http://example.com")- engine.spider = None- with pytest.raises(RuntimeError, match="No open spider to crawl:"):- await self._download(engine, request)-- @coroutine_test- async def test_download_async_failure(self, engine):- """Test async download when the downloader raises an exception."""- request = Request("http://example.com")- error = RuntimeError("Download failed")- engine.spider = Mock()- engine.downloader.fetch.return_value = defer.fail(error)- engine._slot.add_request = Mock()- engine._slot.remove_request = Mock()-- with pytest.raises(RuntimeError, match="Download failed"):- await self._download(engine, request)- engine._slot.add_request.assert_called_once_with(request)- engine._slot.remove_request.assert_called_once_with(request)--[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")-class TestEngineDownload(TestEngineDownloadAsync):- """Test cases for ExecutionEngine.download()."""-- @staticmethod- async def _download(engine: ExecutionEngine, request: Request) -> Response:- return await maybe_deferred_to_future(engine.download(request))--@coroutine_testasync def test_request_scheduled_signal():class TestScheduler(BaseScheduler):@@ -640,132 +545,3 @@ async def test_request_scheduled_signal():f"{scheduler.enqueued!r} != [{keep_request!r}]")crawler.signals.disconnect(signal_handler, signals.request_scheduled)---class TestEngineCloseSpider:- """Tests for exception handling coverage during close_spider_async()."""-- @pytest.fixture- def crawler(self) -> Crawler:- crawler = get_crawler(DefaultSpider)- crawler.spider = crawler._create_spider()- return crawler-- @coroutine_test- async def test_no_slot(self, crawler: Crawler) -> None:- engine = ExecutionEngine(crawler, lambda _: None)- crawler.engine = engine- await engine.open_spider_async()- slot = engine._slot- engine._slot = None- with pytest.raises(RuntimeError, match="Engine slot not assigned"):- await engine.close_spider_async()- # close it correctly- engine._slot = slot- await engine.close_spider_async()-- @coroutine_test- async def test_no_spider(self, crawler: Crawler) -> None:- engine = ExecutionEngine(crawler, lambda _: None)- with pytest.raises(RuntimeError, match="Spider not opened"):- await engine.close_spider_async()- engine.downloader.close() # cleanup-- @coroutine_test- async def test_exception_slot(- self, crawler: Crawler, caplog: pytest.LogCaptureFixture- ) -> None:- engine = ExecutionEngine(crawler, lambda _: None)- crawler.engine = engine- await engine.open_spider_async()- assert engine._slot- del engine._slot.heartbeat- await engine.close_spider_async()- assert "Slot close failure" in caplog.text-- @coroutine_test- async def test_exception_downloader(- self, crawler: Crawler, caplog: pytest.LogCaptureFixture- ) -> None:- engine = ExecutionEngine(crawler, lambda _: None)- crawler.engine = engine- await engine.open_spider_async()- del engine.downloader.slots- await engine.close_spider_async()- assert "Downloader close failure" in caplog.text-- @coroutine_test- async def test_exception_scraper(- self, crawler: Crawler, caplog: pytest.LogCaptureFixture- ) -> None:- engine = ExecutionEngine(crawler, lambda _: None)- crawler.engine = engine- await engine.open_spider_async()- engine.scraper.slot = None- await engine.close_spider_async()- assert "Scraper close failure" in caplog.text-- @coroutine_test- async def test_exception_scheduler(- self, crawler: Crawler, caplog: pytest.LogCaptureFixture- ) -> None:- engine = ExecutionEngine(crawler, lambda _: None)- crawler.engine = engine- await engine.open_spider_async()- assert engine._slot- del cast("Scheduler", engine._slot.scheduler).dqs- await engine.close_spider_async()- assert "Scheduler close failure" in caplog.text-- @coroutine_test- async def test_exception_signal(- self, crawler: Crawler, caplog: pytest.LogCaptureFixture- ) -> None:- engine = ExecutionEngine(crawler, lambda _: None)- crawler.engine = engine- await engine.open_spider_async()- signal_manager = engine.signals- del engine.signals- await engine.close_spider_async()- assert "Error while sending spider_close signal" in caplog.text- # send the spider_closed signal to close various components- await signal_manager.send_catch_log_async(- signal=signals.spider_closed,- spider=engine.spider,- reason="cancelled",- )-- @coroutine_test- async def test_exception_stats(- self, crawler: Crawler, caplog: pytest.LogCaptureFixture- ) -> None:- engine = ExecutionEngine(crawler, lambda _: None)- crawler.engine = engine- await engine.open_spider_async()- assert isinstance(crawler.stats, MemoryStatsCollector)- del crawler.stats.spider_stats- await engine.close_spider_async()- assert "Stats close failure" in caplog.text-- @coroutine_test- async def test_exception_callback(- self, crawler: Crawler, caplog: pytest.LogCaptureFixture- ) -> None:- engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))- crawler.engine = engine- await engine.open_spider_async()- await engine.close_spider_async()- assert "Error running spider_closed_callback" in caplog.text-- @coroutine_test- async def test_exception_async_callback(- self, crawler: Crawler, caplog: pytest.LogCaptureFixture- ) -> None:- async def cb(_):- raise ValueError-- engine = ExecutionEngine(crawler, cb)- crawler.engine = engine- await engine.open_spider_async()- await engine.close_spider_async()- assert "Error running spider_closed_callback" in caplog.textdiff --git a/tests/test_engine_close_spider.py b/tests/test_engine_close_spider.pynew file mode 100644index 000000000..8e9c9bd43--- /dev/null+++ b/tests/test_engine_close_spider.py@@ -0,0 +1,153 @@+from __future__ import annotations++from typing import TYPE_CHECKING, cast++import pytest+from twisted.internet import defer++from scrapy import signals+from scrapy.core.engine import ExecutionEngine+from scrapy.statscollectors import MemoryStatsCollector+from scrapy.utils.spider import DefaultSpider+from scrapy.utils.test import get_crawler+from tests.utils.decorators import coroutine_test++if TYPE_CHECKING:+ from scrapy.core.scheduler import Scheduler+ from scrapy.crawler import Crawler+++def crawler() -> Crawler:+ crawler = get_crawler(DefaultSpider)+ crawler.spider = crawler._create_spider()+ return crawler+++@coroutine_test+async def test_no_slot(crawler: Crawler) -> None:+ engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engine+ await engine.open_spider_async()+ slot = engine._slot+ engine._slot = None+ with pytest.raises(RuntimeError, match="Engine slot not assigned"):+ await engine.close_spider_async()+ # close it correctly+ engine._slot = slot+ await engine.close_spider_async()+++@coroutine_test+async def test_no_spider(crawler: Crawler) -> None:+ engine = ExecutionEngine(crawler, lambda _: None)+ with pytest.raises(RuntimeError, match="Spider not opened"):+ await engine.close_spider_async()+ engine.downloader.close() # cleanup+++@coroutine_test+async def test_exception_slot(+ crawler: Crawler, caplog: pytest.LogCaptureFixture+) -> None:+ engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engine+ await engine.open_spider_async()+ assert engine._slot+ del engine._slot.heartbeat+ await engine.close_spider_async()+ assert "Slot close failure" in caplog.text+++@coroutine_test+async def test_exception_downloader(+ crawler: Crawler, caplog: pytest.LogCaptureFixture+) -> None:+ engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engine+ await engine.open_spider_async()+ del engine.downloader.slots+ await engine.close_spider_async()+ assert "Downloader close failure" in caplog.text+++@coroutine_test+async def test_exception_scraper(+ crawler: Crawler, caplog: pytest.LogCaptureFixture+) -> None:+ engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engine+ await engine.open_spider_async()+ engine.scraper.slot = None+ await engine.close_spider_async()+ assert "Scraper close failure" in caplog.text+++@coroutine_test+async def test_exception_scheduler(+ crawler: Crawler, caplog: pytest.LogCaptureFixture+) -> None:+ engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engine+ await engine.open_spider_async()+ assert engine._slot+ del cast("Scheduler", engine._slot.scheduler).dqs+ await engine.close_spider_async()+ assert "Scheduler close failure" in caplog.text+++@coroutine_test+async def test_exception_signal(+ crawler: Crawler, caplog: pytest.LogCaptureFixture+) -> None:+ engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engine+ await engine.open_spider_async()+ signal_manager = engine.signals+ del engine.signals+ await engine.close_spider_async()+ assert "Error while sending spider_close signal" in caplog.text+ # send the spider_closed signal to close various components+ await signal_manager.send_catch_log_async(+ signal=signals.spider_closed,+ spider=engine.spider,+ reason="cancelled",+ )+++@coroutine_test+async def test_exception_stats(+ crawler: Crawler, caplog: pytest.LogCaptureFixture+) -> None:+ engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engine+ await engine.open_spider_async()+ assert isinstance(crawler.stats, MemoryStatsCollector)+ del crawler.stats.spider_stats+ await engine.close_spider_async()+ assert "Stats close failure" in caplog.text+++@coroutine_test+async def test_exception_callback(+ crawler: Crawler, caplog: pytest.LogCaptureFixture+) -> None:+ engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))+ crawler.engine = engine+ await engine.open_spider_async()+ await engine.close_spider_async()+ assert "Error running spider_closed_callback" in caplog.text+++@coroutine_test+async def test_exception_async_callback(+ crawler: Crawler, caplog: pytest.LogCaptureFixture+) -> None:+ async def cb(_):+ raise ValueError++ engine = ExecutionEngine(crawler, cb)+ crawler.engine = engine+ await engine.open_spider_async()+ await engine.close_spider_async()+ assert "Error running spider_closed_callback" in caplog.textdiff --git a/tests/test_engine_download.py b/tests/test_engine_download.pynew file mode 100644index 000000000..f15bfd5e2--- /dev/null+++ b/tests/test_engine_download.py@@ -0,0 +1,106 @@+from __future__ import annotations++from unittest.mock import Mock, call++import pytest+from twisted.internet import defer++from scrapy.core.engine import ExecutionEngine+from scrapy.http import Request, Response+from scrapy.utils.defer import maybe_deferred_to_future+from scrapy.utils.spider import DefaultSpider+from scrapy.utils.test import get_crawler+from tests.utils.decorators import coroutine_test+++def engine() -> ExecutionEngine:+ crawler = get_crawler(DefaultSpider)+ engine = ExecutionEngine(crawler, lambda _: None)+ engine.downloader.close()+ engine.downloader = Mock()+ engine._slot = Mock()+ engine._slot.inprogress = set()+ return engine+++class TestEngineDownloadAsync:+ """Test cases for ExecutionEngine.download_async()."""++ @staticmethod+ async def _download(engine: ExecutionEngine, request: Request) -> Response:+ return await engine.download_async(request)++ @coroutine_test+ async def test_download_async_success(self, engine):+ """Test basic successful async download of a request."""+ request = Request("http://example.com")+ response = Response("http://example.com", body=b"test body")+ engine.spider = Mock()+ engine.downloader.fetch.return_value = defer.succeed(response)+ engine._slot.add_request = Mock()+ engine._slot.remove_request = Mock()++ result = await self._download(engine, request)+ assert result == response+ engine._slot.add_request.assert_called_once_with(request)+ engine._slot.remove_request.assert_called_once_with(request)+ engine.downloader.fetch.assert_called_once_with(request)++ @coroutine_test+ async def test_download_async_redirect(self, engine):+ """Test async download with a redirect request."""+ original_request = Request("http://example.com")+ redirect_request = Request("http://example.com/redirect")+ final_response = Response("http://example.com/redirect", body=b"redirected")++ # First call returns redirect request, second call returns final response+ engine.downloader.fetch.side_effect = [+ defer.succeed(redirect_request),+ defer.succeed(final_response),+ ]+ engine.spider = Mock()+ engine._slot.add_request = Mock()+ engine._slot.remove_request = Mock()++ result = await self._download(engine, original_request)+ assert result == final_response+ assert engine.downloader.fetch.call_count == 2+ engine._slot.add_request.assert_has_calls(+ [call(original_request), call(redirect_request)]+ )+ engine._slot.remove_request.assert_has_calls(+ [call(original_request), call(redirect_request)]+ )++ @coroutine_test+ async def test_download_async_no_spider(self, engine):+ """Test async download attempt when no spider is available."""+ request = Request("http://example.com")+ engine.spider = None+ with pytest.raises(RuntimeError, match="No open spider to crawl:"):+ await self._download(engine, request)++ @coroutine_test+ async def test_download_async_failure(self, engine):+ """Test async download when the downloader raises an exception."""+ request = Request("http://example.com")+ error = RuntimeError("Download failed")+ engine.spider = Mock()+ engine.downloader.fetch.return_value = defer.fail(error)+ engine._slot.add_request = Mock()+ engine._slot.remove_request = Mock()++ with pytest.raises(RuntimeError, match="Download failed"):+ await self._download(engine, request)+ engine._slot.add_request.assert_called_once_with(request)+ engine._slot.remove_request.assert_called_once_with(request)++[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")+class TestEngineDownload(TestEngineDownloadAsync):+ """Test cases for ExecutionEngine.download()."""++ @staticmethod+ async def _download(engine: ExecutionEngine, request: Request) -> Response:+ return await maybe_deferred_to_future(engine.download(request))diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.pyindex 6cc8c0650..ca52961f4 100644--- a/tests/test_engine_loop.py+++ b/tests/test_engine_loop.py@@ -2,13 +2,13 @@ from __future__ import annotationsfrom collections import dequefrom logging import ERROR-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyfrom scrapy import Request, Spider, signals+from scrapy.core.scheduler import BaseSchedulerfrom scrapy.utils.asyncio import call_laterfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer-from tests.test_scheduler import MemorySchedulerfrom tests.utils import async_sleepfrom tests.utils.decorators import coroutine_test@@ -18,6 +18,38 @@ if TYPE_CHECKING:from scrapy.http import Response+class MemoryScheduler(BaseScheduler):+ paused = False++ def __init__(self, *args: Any, **kwargs: Any):+ super().__init__(*args, **kwargs)+ self.queue: deque[Request] = deque(+ Request(value) if isinstance(value, str) else value+ for value in getattr(self, "queue", [])+ )++ def enqueue_request(self, request: Request) -> bool:+ self.queue.append(request)+ return True++ def has_pending_requests(self) -> bool:+ return self.paused or bool(self.queue)++ def next_request(self) -> Request | None:+ if self.paused:+ return None+ try:+ return self.queue.pop()+ except IndexError:+ return None++ def pause(self) -> None:+ self.paused = True++ def unpause(self) -> None:+ self.paused = False++class TestMain:@coroutine_testasync def test_sleep(self):diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 5ab1a54c1..853243b97 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -13,8 +13,6 @@ from pathlib import Pathfrom string import ascii_letters, digitsfrom typing import IO, TYPE_CHECKING, Anyfrom unittest import mock-from urllib.parse import urljoin-from urllib.request import pathname2urlimport lxml.etreeimport pytest@@ -39,19 +37,12 @@ from scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.spiders import ItemSpiderfrom tests.utils.decorators import coroutine_test, inline_callbacks_test+from tests.utils.feedexport import MyItem, MyItem2, path_to_url, printf_escapeif TYPE_CHECKING:from collections.abc import Awaitable, Callable, Iterable-def path_to_url(path: str | Path) -> str:- return urljoin("file:", pathname2url(str(path)))---def printf_escape(s: str) -> str:- return s.replace("%", "%%")--class FromCrawlerMixin:init_with_crawler = False@@ -114,15 +105,6 @@ class LogOnStoreFileStorage:class TestFeedExportBase(ABC):mockserver: MockServer- class MyItem(scrapy.Item):- foo = scrapy.Field()- egg = scrapy.Field()- baz = scrapy.Field()-- class MyItem2(scrapy.Item):- foo = scrapy.Field()- hello = scrapy.Field()-def _random_temp_filename(self, inter_dir="") -> Path:chars = [random.choice(ascii_letters + digits) for _ in range(15)]filename = "".join(chars)@@ -517,21 +499,21 @@ class TestFeedExport(TestFeedExportBase):async def test_export_items(self):# feed exporters use field names from Itemitems = [- self.MyItem({"foo": "bar1", "egg": "spam1"}),- self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),+ MyItem({"foo": "bar1", "egg": "spam1"}),+ MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),]rows = [{"egg": "spam1", "foo": "bar1", "baz": ""},{"egg": "spam2", "foo": "bar2", "baz": "quux2"},]- header = self.MyItem.fields.keys()+ header = MyItem.fields.keys()await self.assertExported(items, header, rows)@coroutine_testasync def test_pathlib_uri_with_placeholders(self):feed_dir = Path(self.temp_dir, "pathlib_placeholders")feed_dir.mkdir()- items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]+ items = [MyItem({"foo": "bar1", "egg": "spam1"})]class TestSpider(scrapy.Spider):name = "testspider"@@ -561,7 +543,7 @@ class TestFeedExport(TestFeedExportBase):# so the resulting file name can be asserted exactly.feed_dir = Path(self.temp_dir, "pathlib_spaces_unicode")feed_dir.mkdir()- items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]+ items = [MyItem({"foo": "bar1", "egg": "spam1"})]class TestSpider(scrapy.Spider):name = "testspider"@@ -590,7 +572,7 @@ class TestFeedExport(TestFeedExportBase):# and #5794.feed_dir = Path(self.temp_dir, "dir with spaces")feed_dir.mkdir()- items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]+ items = [MyItem({"foo": "bar1", "egg": "spam1"})]class TestSpider(scrapy.Spider):name = "testspider"@@ -627,7 +609,7 @@ class TestFeedExport(TestFeedExportBase):@coroutine_testasync def test_start_finish_exporting_items(self):items = [- self.MyItem({"foo": "bar1", "egg": "spam1"}),+ MyItem({"foo": "bar1", "egg": "spam1"}),]settings = {"FEEDS": {@@ -665,7 +647,7 @@ class TestFeedExport(TestFeedExportBase):@coroutine_testasync def test_start_finish_exporting_items_exception(self):items = [- self.MyItem({"foo": "bar1", "egg": "spam1"}),+ MyItem({"foo": "bar1", "egg": "spam1"}),]settings = {"FEEDS": {@@ -743,15 +725,15 @@ class TestFeedExport(TestFeedExportBase):@coroutine_testasync def test_export_multiple_item_classes(self):items = [- self.MyItem({"foo": "bar1", "egg": "spam1"}),- self.MyItem2({"hello": "world2", "foo": "bar2"}),- self.MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}),+ MyItem({"foo": "bar1", "egg": "spam1"}),+ MyItem2({"hello": "world2", "foo": "bar2"}),+ MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}),{"hello": "world4", "egg": "spam4"},]# by default, Scrapy uses fields of the first Item for CSV and# all fields for JSON Lines- header = self.MyItem.fields.keys()+ header = MyItem.fields.keys()rows_csv = [{"egg": "spam1", "foo": "bar1", "baz": ""},{"egg": "", "foo": "bar2", "baz": ""},@@ -826,8 +808,8 @@ class TestFeedExport(TestFeedExportBase):@coroutine_testasync def test_export_based_on_item_classes(self):items = [- self.MyItem({"foo": "bar1", "egg": "spam1"}),- self.MyItem2({"hello": "world2", "foo": "bar2"}),+ MyItem({"foo": "bar1", "egg": "spam1"}),+ MyItem2({"hello": "world2", "foo": "bar2"}),{"hello": "world3", "egg": "spam3"},]@@ -849,15 +831,15 @@ class TestFeedExport(TestFeedExportBase):"FEEDS": {self._random_temp_filename(): {"format": "csv",- "item_classes": [self.MyItem],+ "item_classes": [MyItem],},self._random_temp_filename(): {"format": "json",- "item_classes": [self.MyItem2],+ "item_classes": [MyItem2],},self._random_temp_filename(): {"format": "jsonlines",- "item_classes": [self.MyItem, self.MyItem2],+ "item_classes": [MyItem, MyItem2],},self._random_temp_filename(): {"format": "xml",@@ -872,13 +854,11 @@ class TestFeedExport(TestFeedExportBase):@coroutine_testasync def test_export_based_on_custom_filters(self):items = [- self.MyItem({"foo": "bar1", "egg": "spam1"}),- self.MyItem2({"hello": "world2", "foo": "bar2"}),+ MyItem({"foo": "bar1", "egg": "spam1"}),+ MyItem2({"hello": "world2", "foo": "bar2"}),{"hello": "world3", "egg": "spam3"},]- MyItem = self.MyItem-class CustomFilter1:def __init__(self, feed_options):pass@@ -918,7 +898,7 @@ class TestFeedExport(TestFeedExportBase):},self._random_temp_filename(): {"format": "jsonlines",- "item_classes": [self.MyItem, self.MyItem2],+ "item_classes": [MyItem, MyItem2],"item_filter": CustomFilter3,},},@@ -957,7 +937,7 @@ class TestFeedExport(TestFeedExportBase):# FEED_EXPORT_FIELDS option allows to order export fields# and to select a subset of fields to export, both for Items and dicts.- for item_cls in [self.MyItem, dict]:+ for item_cls in [MyItem, dict]:items = [item_cls({"foo": "bar1", "egg": "spam1"}),item_cls({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.pyindex 469985288..6e0536110 100644--- a/tests/test_feedexport_batch.py+++ b/tests/test_feedexport_batch.py@@ -24,6 +24,7 @@ from scrapy.utils.test import get_crawlerfrom tests.spiders import ItemSpiderfrom tests.test_feedexport import TestFeedExportBasefrom tests.utils.decorators import coroutine_test, inline_callbacks_test+from tests.utils.feedexport import MyItemif TYPE_CHECKING:from os import PathLike@@ -197,9 +198,9 @@ class TestBatchDeliveries(TestFeedExportBase):async def test_export_items(self):"""Test partial deliveries in all supported formats"""items = [- self.MyItem({"foo": "bar1", "egg": "spam1"}),- self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),- self.MyItem({"foo": "bar3", "baz": "quux3"}),+ MyItem({"foo": "bar1", "egg": "spam1"}),+ MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),+ MyItem({"foo": "bar3", "baz": "quux3"}),]rows = [{"egg": "spam1", "foo": "bar1", "baz": ""},@@ -207,7 +208,7 @@ class TestBatchDeliveries(TestFeedExportBase):{"foo": "bar3", "baz": "quux3", "egg": ""},]settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2}- header = self.MyItem.fields.keys()+ header = MyItem.fields.keys()await self.assertExported(items, header, rows, settings=settings)def test_wrong_path(self):@@ -349,9 +350,9 @@ class TestBatchDeliveries(TestFeedExportBase):So %(batch_id)d replaced with the current id."""items = [- self.MyItem({"foo": "bar1", "egg": "spam1"}),- self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),- self.MyItem({"foo": "bar3", "baz": "quux3"}),+ MyItem({"foo": "bar1", "egg": "spam1"}),+ MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),+ MyItem({"foo": "bar3", "baz": "quux3"}),]settings = {"FEEDS": {@@ -387,9 +388,9 @@ class TestBatchDeliveries(TestFeedExportBase):def test_s3_export(self):bucket = "mybucket"items = [- self.MyItem({"foo": "bar1", "egg": "spam1"}),- self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),- self.MyItem({"foo": "bar3", "baz": "quux3"}),+ MyItem({"foo": "bar1", "egg": "spam1"}),+ MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),+ MyItem({"foo": "bar3", "baz": "quux3"}),]class CustomS3FeedStorage(S3FeedStorage):diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.pyindex 6ebcab152..83d976e94 100644--- a/tests/test_feedexport_postprocess.py+++ b/tests/test_feedexport_postprocess.py@@ -13,8 +13,9 @@ from typing import TYPE_CHECKING, Anyimport pytestfrom scrapy.utils.test import get_crawler-from tests.test_feedexport import TestFeedExportBase, path_to_url, printf_escape+from tests.test_feedexport import TestFeedExportBasefrom tests.utils.decorators import coroutine_test+from tests.utils.feedexport import path_to_url, printf_escapeif TYPE_CHECKING:from scrapy import Spiderdiff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex 5249736f2..da94a4e95 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -7,17 +7,7 @@ from scrapy.robotstxt import (decode_robotstxt,)from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER---def rerp_available() -> bool:- # check if robotexclusionrulesparser is installed- try:- from robotexclusionrulesparser import ( # noqa: PLC0415- RobotExclusionRulesParser, # noqa: F401- )- except ImportError:- return False- return True+from tests.utils.robotstxt import rerp_availableclass BaseRobotParserTest:diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex 39ce87b65..4fb594444 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport warningsfrom abc import ABC, abstractmethod-from collections import dequefrom contextlib import AbstractAsyncContextManager, asynccontextmanagerfrom typing import TYPE_CHECKING, Any, NamedTuple, castfrom unittest.mock import Mock@@ -10,7 +9,7 @@ from unittest.mock import Mockimport pytestfrom scrapy.core.downloader import Downloader-from scrapy.core.scheduler import BaseScheduler, Scheduler+from scrapy.core.scheduler import Schedulerfrom scrapy.crawler import Crawlerfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request@@ -27,38 +26,6 @@ if TYPE_CHECKING:from pathlib import Path-class MemoryScheduler(BaseScheduler):- paused = False-- def __init__(self, *args: Any, **kwargs: Any):- super().__init__(*args, **kwargs)- self.queue: deque[Request] = deque(- Request(value) if isinstance(value, str) else value- for value in getattr(self, "queue", [])- )-- def enqueue_request(self, request: Request) -> bool:- self.queue.append(request)- return True-- def has_pending_requests(self) -> bool:- return self.paused or bool(self.queue)-- def next_request(self) -> Request | None:- if self.paused:- return None- try:- return self.queue.pop()- except IndexError:- return None-- def pause(self) -> None:- self.paused = True-- def unpause(self) -> None:- self.paused = False--class MockSlot(NamedTuple):active: list[Any]diff --git a/tests/utils/__init__.py b/tests/utils/__init__.pyindex 73c0e3776..b0632a7ea 100644--- a/tests/utils/__init__.py+++ b/tests/utils/__init__.py@@ -1,13 +1,19 @@+from __future__ import annotations+import asyncioimport os-from collections.abc import Callablefrom pathlib import Path+from typing import TYPE_CHECKINGfrom twisted.internet.defer import Deferred+from scrapy.settings import Settings, default_settingsfrom scrapy.utils.asyncio import is_asyncio_availablefrom scrapy.utils.defer import maybe_deferred_to_future+if TYPE_CHECKING:+ from collections.abc import Callable+def twisted_sleep(seconds: float):from twisted.internet import reactor@@ -48,3 +54,8 @@ class OneShotLoop:def stop(self) -> None:self.running = False+++def assert_option_is_default(settings: Settings, key: str) -> None:+ assert isinstance(settings, Settings)+ assert settings[key] == getattr(default_settings, key)diff --git a/tests/utils/base_commands.py b/tests/utils/base_commands.pynew file mode 100644index 000000000..594544c83--- /dev/null+++ b/tests/utils/base_commands.py@@ -0,0 +1,34 @@+from __future__ import annotations++from shutil import copytree+from typing import TYPE_CHECKING++import pytest++from tests.utils.cmdline import call++if TYPE_CHECKING:+ from pathlib import Path+++class TestProjectBase:+ """A base class for tests that may need a Scrapy project."""++ project_name = "testproject"++ @pytest.fixture(scope="session")+ def _proj_path_cached(self, tmp_path_factory: pytest.TempPathFactory) -> Path:+ """Create a Scrapy project in a temporary directory and return its path.++ Used as a cache for ``proj_path``.+ """+ tmp_path = tmp_path_factory.mktemp("proj")+ call("startproject", self.project_name, cwd=tmp_path)+ return tmp_path / self.project_name++ @pytest.fixture+ def proj_path(self, tmp_path: Path, _proj_path_cached: Path) -> Path:+ """Copy a pre-generated Scrapy project into a temporary directory and return its path."""+ proj_path = tmp_path / self.project_name+ copytree(_proj_path_cached, proj_path)+ return proj_pathdiff --git a/tests/utils/cmdline.py b/tests/utils/cmdline.pyindex 122e0236d..62dff3d4c 100644--- a/tests/utils/cmdline.py+++ b/tests/utils/cmdline.py@@ -2,12 +2,15 @@ from __future__ import annotationsimport subprocessimport sys-from typing import Any+from typing import TYPE_CHECKING, Anyimport pytestfrom scrapy.utils.test import get_testenv+if TYPE_CHECKING:+ from pathlib import Path+def call(*args: str, **popen_kwargs: Any) -> int:args = (sys.executable, "-m", "scrapy.cmdline", *args)@@ -36,3 +39,10 @@ def proc(*args: str, **popen_kwargs: Any) -> tuple[int, str, str]:pytest.fail("Command took too much time to complete")return p.returncode, p.stdout, p.stderr+++def write_recording_editor(editor: Path) -> None:+ """Create an executable editor script that writes the path it is asked to+ open (its last argument) into the file given as its first argument."""+ editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")+ editor.chmod(0o755)diff --git a/tests/utils/feedexport.py b/tests/utils/feedexport.pynew file mode 100644index 000000000..515082217--- /dev/null+++ b/tests/utils/feedexport.py@@ -0,0 +1,29 @@+from __future__ import annotations++from typing import TYPE_CHECKING+from urllib.parse import urljoin+from urllib.request import pathname2url++from scrapy import Field, Item++if TYPE_CHECKING:+ from pathlib import Path+++def path_to_url(path: str | Path) -> str:+ return urljoin("file:", pathname2url(str(path)))+++def printf_escape(s: str) -> str:+ return s.replace("%", "%%")+++class MyItem(Item):+ foo = Field()+ egg = Field()+ baz = Field()+++class MyItem2(Item):+ foo = Field()+ hello = Field()diff --git a/tests/utils/redirect.py b/tests/utils/redirect.pynew file mode 100644index 000000000..a3a572988--- /dev/null+++ b/tests/utils/redirect.py@@ -0,0 +1,40 @@+from __future__ import annotations++from itertools import product++SCHEME_PARAMS = ("url", "location", "target")+HTTP_SCHEMES = ("http", "https")+NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo")+REDIRECT_SCHEME_CASES = (+ # http/https → http/https redirects+ *(+ (+ f"{input_scheme}://example.com/a",+ f"{output_scheme}://example.com/b",+ f"{output_scheme}://example.com/b",+ )+ for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2)+ ),+ # http/https → data/file/ftp/s3/foo does not redirect+ *(+ (+ f"{input_scheme}://example.com/a",+ f"{output_scheme}://example.com/b",+ None,+ )+ for input_scheme in HTTP_SCHEMES+ for output_scheme in NON_HTTP_SCHEMES+ ),+ # http/https → relative redirects+ *(+ (+ f"{scheme}://example.com/a",+ location,+ f"{scheme}://example.com/b",+ )+ for scheme in HTTP_SCHEMES+ for location in ("//example.com/b", "/b")+ ),+ # Note: We do not test data/file/ftp/s3 schemes for the initial URL+ # because their download handlers cannot return a status code of 3xx.+)diff --git a/tests/utils/robotstxt.py b/tests/utils/robotstxt.pynew file mode 100644index 000000000..c1b2f28c6--- /dev/null+++ b/tests/utils/robotstxt.py@@ -0,0 +1,12 @@+from __future__ import annotations+++def rerp_available() -> bool:+ # check if robotexclusionrulesparser is installed+ try:+ from robotexclusionrulesparser import ( # noqa: PLC0415+ RobotExclusionRulesParser, # noqa: F401+ )+ except ImportError:+ return False+ return True