Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Test restructuring (#7736)

Andrey Rakhmatullin4ee3676464d9
diff --git a/tests/spiders.py b/tests/spiders.py
index 612dc11c9..da14fdbe3 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -571,3 +571,19 @@ class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):
def headers_received(self, headers, body_length, request, spider):
self.meta["headers_received"] = headers
raise StopDownload(fail=True)
+
+
+class ExceptionSpider(Spider):
+ name = "exception"
+
+ @classmethod
+ def from_crawler(cls, crawler, *args, **kwargs):
+ raise ValueError("Exception in from_crawler method")
+
+
+class NoRequestsSpider(Spider):
+ name = "no_request"
+
+ async def start(self):
+ return
+ yield
diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 815f87026..794c6d400 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -7,7 +7,7 @@ from unittest import TestCase
from unittest.mock import MagicMock, Mock, PropertyMock, call, patch
from scrapy.commands.check import Command, TextTestResult
-from tests.test_commands import TestProjectBase
+from tests.utils.base_commands import TestProjectBase
from tests.utils.cmdline import proc
if TYPE_CHECKING:
diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py
index dd69d36ba..6293e973e 100644
--- a/tests/test_command_crawl.py
+++ b/tests/test_command_crawl.py
@@ -2,7 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-from tests.test_commands import TestProjectBase
+from tests.utils.base_commands import TestProjectBase
from tests.utils.cmdline import proc
if TYPE_CHECKING:
diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py
index 94db14c30..f465d0b30 100644
--- a/tests/test_command_genspider.py
+++ b/tests/test_command_genspider.py
@@ -6,15 +6,8 @@ from pathlib import Path
import pytest
-from tests.test_commands import TestProjectBase
-from tests.utils.cmdline import call, proc
-
-
-def write_recording_editor(editor: Path) -> None:
- """Create an executable editor script that writes the path it is asked to
- open (its last argument) into the file given as its first argument."""
- editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
- editor.chmod(0o755)
+from tests.utils.base_commands import TestProjectBase
+from tests.utils.cmdline import call, proc, write_recording_editor
def find_in_file(filename: Path, regex: str) -> re.Match[str] | None:
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index c210a06e8..a3581d764 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -8,7 +8,7 @@ import pytest
from scrapy.commands import parse
from scrapy.settings import Settings
-from tests.test_commands import TestProjectBase
+from tests.utils.base_commands import TestProjectBase
from tests.utils.cmdline import call, proc
if TYPE_CHECKING:
diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py
index 9dabbc942..2b410b5c6 100644
--- a/tests/test_command_runspider.py
+++ b/tests/test_command_runspider.py
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
import pytest
-from tests.test_crawler import ExceptionSpider, NoRequestsSpider
+from tests.spiders import ExceptionSpider, NoRequestsSpider
from tests.utils.cmdline import proc
if TYPE_CHECKING:
@@ -65,13 +65,14 @@ class BadSpider(scrapy.Spider):
def test_run_fail_spider(self, tmp_path: Path) -> None:
ret, _, _ = self.runspider(
- tmp_path, "import scrapy\n" + inspect.getsource(ExceptionSpider)
+ tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider)
)
assert ret != 0
def test_run_good_spider(self, tmp_path: Path) -> None:
ret, _, _ = self.runspider(
- tmp_path, "import scrapy\n" + inspect.getsource(NoRequestsSpider)
+ tmp_path,
+ "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider),
)
assert ret == 0
diff --git a/tests/test_commands.py b/tests/test_commands.py
index bb4d96f00..9e7d4d5a1 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -4,7 +4,6 @@ import argparse
import json
import sys
from io import StringIO
-from shutil import copytree
from typing import TYPE_CHECKING
from unittest import mock
@@ -16,7 +15,8 @@ from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils.reactor import _asyncio_reactor_path
-from tests.utils.cmdline import call, proc
+from tests.utils.base_commands import TestProjectBase
+from tests.utils.cmdline import call, proc, write_recording_editor
if TYPE_CHECKING:
from pathlib import Path
@@ -109,29 +109,6 @@ class TestCommandSettings:
)
-class TestProjectBase:
- """A base class for tests that may need a Scrapy project."""
-
- project_name = "testproject"
-
- @pytest.fixture(scope="session")
- def _proj_path_cached(self, tmp_path_factory: pytest.TempPathFactory) -> Path:
- """Create a Scrapy project in a temporary directory and return its path.
-
- Used as a cache for ``proj_path``.
- """
- tmp_path = tmp_path_factory.mktemp("proj")
- call("startproject", self.project_name, cwd=tmp_path)
- return tmp_path / self.project_name
-
- @pytest.fixture
- def proj_path(self, tmp_path: Path, _proj_path_cached: Path) -> Path:
- """Copy a pre-generated Scrapy project into a temporary directory and return its path."""
- proj_path = tmp_path / self.project_name
- copytree(_proj_path_cached, proj_path)
- return proj_path
-
-
class TestCommandCrawlerProcess(TestProjectBase):
"""Test that the command uses the expected kind of *CrawlerProcess
and produces expected errors when needed."""
@@ -430,9 +407,7 @@ class TestEditCommand(TestProjectBase):
spider = proj_path / self.project_name / "spiders" / "example.py"
edited = proj_path / "edited.txt"
editor = proj_path / "fake-editor.sh"
- # Records the file it is asked to open ($2) into the file given as $1.
- editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
- editor.chmod(0o755)
+ write_recording_editor(editor)
monkeypatch.setenv("EDITOR", f"{editor} {edited}")
assert call("genspider", "example", "example.com", cwd=proj_path) == 0
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 8c1716345..9c479068a 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -14,14 +14,13 @@ from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
from scrapy import Spider, signals
-from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner
from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
-from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.engine import format_engine_status, get_engine_status
from scrapy.utils.python import to_unicode
-from scrapy.utils.test import get_crawler, get_reactor_settings
+from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
from tests.spiders import (
AsyncDefAsyncioGenComplexSpider,
@@ -430,50 +429,6 @@ with multiples lines
)
assert not crawler.crawling
- @coroutine_test
- async def test_crawlerrunner_accepts_crawler(
- self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
- ) -> None:
- crawler = Crawler(SimpleSpider, get_reactor_settings())
- runner = CrawlerRunner()
- with caplog.at_level(logging.DEBUG):
- await maybe_deferred_to_future(
- runner.crawl(
- crawler,
- mockserver.url("/status?n=200"),
- mockserver=mockserver,
- )
- )
- assert "Got response 200" in caplog.text
-
- @coroutine_test
- async def test_crawl_multiple(
- self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
- ) -> None:
- settings_dict = get_reactor_settings()
- runner_cls = (
- CrawlerRunner
- if settings_dict.get("TWISTED_REACTOR_ENABLED", True)
- else AsyncCrawlerRunner
- )
- runner = runner_cls(settings_dict)
- runner.crawl(
- SimpleSpider,
- mockserver.url("/status?n=200"),
- mockserver=mockserver,
- )
- runner.crawl(
- SimpleSpider,
- mockserver.url("/status?n=503"),
- mockserver=mockserver,
- )
-
- with caplog.at_level(logging.DEBUG):
- await ensure_awaitable(runner.join())
-
- self._assert_retried(caplog.text)
- assert "Got response 200" in caplog.text
-
@coroutine_test
async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None:
crawler = get_crawler(SimpleSpider)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 6a1eea027..b4f906e25 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -10,22 +10,14 @@ from typing import TYPE_CHECKING, Any, ClassVar
from unittest.mock import MagicMock
import pytest
-from zope.interface.exceptions import MultipleInvalid
import scrapy
from scrapy import Spider
-from scrapy.crawler import (
- AsyncCrawlerProcess,
- AsyncCrawlerRunner,
- Crawler,
- CrawlerProcess,
- CrawlerRunner,
- CrawlerRunnerBase,
-)
+from scrapy.crawler import AsyncCrawlerProcess, Crawler, CrawlerProcess
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.throttle import AutoThrottle
from scrapy.settings import Settings, default_settings
-from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.log import (
_uninstall_scrapy_root_handler,
configure_logging,
@@ -33,6 +25,8 @@ from scrapy.utils.log import (
)
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_reactor_settings
+from tests.spiders import NoRequestsSpider
+from tests.utils import assert_option_is_default
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@@ -52,14 +46,7 @@ def get_raw_crawler(
return Crawler(spidercls or DefaultSpider, settings)
-class TestBaseCrawler:
- @staticmethod
- def assertOptionIsDefault(settings: Settings, key: str) -> None:
- assert isinstance(settings, Settings)
- assert settings[key] == getattr(default_settings, key)
-
-
-class TestCrawler(TestBaseCrawler):
+class TestCrawler:
def test_populate_spidercls_settings(self) -> None:
spider_settings: dict[str, Any] = {
"TEST1": "spider",
@@ -90,11 +77,11 @@ class TestCrawler(TestBaseCrawler):
def test_crawler_accepts_dict(self) -> None:
crawler = get_crawler(DefaultSpider, {"foo": "bar"})
assert crawler.settings["foo"] == "bar"
- self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
+ assert_option_is_default(crawler.settings, "RETRY_ENABLED")
def test_crawler_accepts_None(self) -> None:
crawler = Crawler(DefaultSpider)
- self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
+ assert_option_is_default(crawler.settings, "RETRY_ENABLED")
def test_crawler_rejects_spider_objects(self) -> None:
with pytest.raises(ValueError, match="spidercls argument must be a class"):
@@ -586,78 +573,6 @@ class TestCrawlerLogging:
assert "debug message" in logged
-class SpiderLoaderWithWrongInterface:
- def unneeded_method(self) -> None:
- pass
-
-
-class TestCrawlerRunner(TestBaseCrawler):
- def test_spider_manager_verify_interface(self) -> None:
- settings = Settings(
- {
- "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
- }
- )
- with pytest.raises(MultipleInvalid):
- CrawlerRunner(settings)
-
- def test_crawler_runner_accepts_dict(self) -> None:
- runner = CrawlerRunner({"foo": "bar"})
- assert runner.settings["foo"] == "bar"
- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
- def test_crawler_runner_accepts_None(self) -> None:
- runner = CrawlerRunner()
- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
-
-class TestAsyncCrawlerRunner(TestBaseCrawler):
- def test_spider_manager_verify_interface(self) -> None:
- settings = Settings(
- {
- "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
- }
- )
- with pytest.raises(MultipleInvalid):
- AsyncCrawlerRunner(settings)
-
- def test_crawler_runner_accepts_dict(self) -> None:
- runner = AsyncCrawlerRunner({"foo": "bar"})
- assert runner.settings["foo"] == "bar"
- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
- def test_crawler_runner_accepts_None(self) -> None:
- runner = AsyncCrawlerRunner()
- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
-
-class TestCrawlerProcess(TestBaseCrawler):
- def test_crawler_process_accepts_dict(self) -> None:
- runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
- assert runner.settings["foo"] == "bar"
- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
- def test_crawler_process_accepts_None(self) -> None:
- runner = CrawlerProcess(install_root_handler=False)
- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
-
-class TestAsyncCrawlerProcess(TestBaseCrawler):
- def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:
- runner = AsyncCrawlerProcess(
- {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},
- install_root_handler=False,
- )
- assert runner.settings["foo"] == "bar"
- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
- @pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False
- def test_crawler_process_accepts_None(self) -> None:
- runner = AsyncCrawlerProcess(install_root_handler=False)
- self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
-
class TestAsyncCrawlerProcessReactorlessHelpers:
"""Unit tests for the reactorless shutdown helpers of AsyncCrawlerProcess.
@@ -797,161 +712,6 @@ class TestAsyncCrawlerProcessReactorlessHelpers:
)
[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
-def test_runner_settings_applied_to_crawler_instance(
- runner_cls: type[CrawlerRunnerBase],
-) -> None:
- runner = runner_cls({"FOO": "runner"})
- crawler = Crawler(DefaultSpider)
- result = runner.create_crawler(crawler)
- assert result is crawler
- assert result.settings["FOO"] == "runner"
-
-
[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
-def test_spider_custom_settings_override_runner(
- runner_cls: type[CrawlerRunnerBase],
-) -> None:
- class MySpider(DefaultSpider):
- custom_settings = {"FOO": "spider"}
-
- runner = runner_cls({"FOO": "runner"})
- crawler = Crawler(MySpider)
- runner.create_crawler(crawler)
- assert crawler.settings["FOO"] == "spider"
-
-
-def test_create_crawler_instance_consistent_with_spider_class() -> None:
- runner = AsyncCrawlerRunner({"FOO": "runner"})
-
- crawler_from_class = runner.create_crawler(DefaultSpider)
-
- pre_built = Crawler(DefaultSpider)
- runner.create_crawler(pre_built)
-
- assert crawler_from_class.settings["FOO"] == "runner"
- assert pre_built.settings["FOO"] == "runner"
-
-
[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
-def test_create_crawler_rejects_spider_object(
- runner_cls: type[CrawlerRunnerBase],
-) -> None:
- runner = runner_cls()
- with pytest.raises(ValueError, match="cannot be a spider object"):
- runner.create_crawler(DefaultSpider()) # type: ignore[arg-type]
-
-
[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
-def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:
- runner = runner_cls()
- with pytest.raises(ValueError, match="cannot be a spider object"):
- runner.crawl(DefaultSpider()) # type: ignore[arg-type]
-
-
-class ExceptionSpider(scrapy.Spider):
- name = "exception"
-
- @classmethod
- def from_crawler(cls, crawler, *args, **kwargs):
- raise ValueError("Exception in from_crawler method")
-
-
-class NoRequestsSpider(scrapy.Spider):
- name = "no_request"
-
- async def start(self):
- return
- yield
-
-
[email protected]_reactor # CrawlerRunner requires a reactor
-class TestCrawlerRunnerHasSpider:
- @pytest.fixture
- def runner(self) -> CrawlerRunnerBase:
- return CrawlerRunner(get_reactor_settings())
-
- @staticmethod
- async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:
- await ensure_awaitable(runner.crawl(spider))
-
- @coroutine_test
- async def test_crawler_runner_bootstrap_successful(
- self, runner: CrawlerRunnerBase
- ) -> None:
- await self._crawl(runner, NoRequestsSpider)
- assert not runner.bootstrap_failed
-
- @coroutine_test
- async def test_crawler_runner_bootstrap_successful_for_several(
- self, runner: CrawlerRunnerBase
- ) -> None:
- await self._crawl(runner, NoRequestsSpider)
- await self._crawl(runner, NoRequestsSpider)
- assert not runner.bootstrap_failed
-
- @coroutine_test
- async def test_crawler_runner_bootstrap_failed(
- self, runner: CrawlerRunnerBase
- ) -> None:
- try:
- await self._crawl(runner, ExceptionSpider)
- except ValueError:
- pass
- else:
- pytest.fail("Exception should be raised from spider")
-
- assert runner.bootstrap_failed
-
- @coroutine_test
- async def test_crawler_runner_bootstrap_failed_for_several(
- self, runner: CrawlerRunnerBase
- ) -> None:
- try:
- await self._crawl(runner, ExceptionSpider)
- except ValueError:
- pass
- else:
- pytest.fail("Exception should be raised from spider")
-
- await self._crawl(runner, NoRequestsSpider)
-
- assert runner.bootstrap_failed
-
- @coroutine_test
- async def test_crawler_runner_asyncio_enabled_true(
- self, reactor_pytest: str
- ) -> None:
- if reactor_pytest != "asyncio":
- runner = CrawlerRunner(
- settings={
- "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
- }
- )
- with pytest.raises(
- Exception,
- match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
- ):
- await self._crawl(runner, NoRequestsSpider)
- else:
- runner = CrawlerRunner(
- settings={
- "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
- }
- )
- await self._crawl(runner, NoRequestsSpider)
-
-
-class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
- @pytest.fixture
- def runner(self) -> CrawlerRunnerBase:
- return AsyncCrawlerRunner(get_reactor_settings())
-
- def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override]
- pytest.skip("This test is only for CrawlerRunner")
-
-
@pytest.mark.parametrize(
("settings", "items"),
[
diff --git a/tests/test_crawler_runners.py b/tests/test_crawler_runners.py
new file mode 100644
index 000000000..2f3649b43
--- /dev/null
+++ b/tests/test_crawler_runners.py
@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+import pytest
+from zope.interface.exceptions import MultipleInvalid
+
+from scrapy.crawler import (
+ AsyncCrawlerProcess,
+ AsyncCrawlerRunner,
+ Crawler,
+ CrawlerProcess,
+ CrawlerRunner,
+ CrawlerRunnerBase,
+)
+from scrapy.settings import Settings
+from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
+from scrapy.utils.spider import DefaultSpider
+from scrapy.utils.test import get_reactor_settings
+from tests.spiders import ExceptionSpider, NoRequestsSpider, SimpleSpider
+from tests.utils import assert_option_is_default
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+ from scrapy import Spider
+ from tests.mockserver.http import MockServer
+
+
+class SpiderLoaderWithWrongInterface:
+ def unneeded_method(self) -> None:
+ pass
+
+
+class TestCrawlerRunner:
+ def test_spider_manager_verify_interface(self) -> None:
+ settings = Settings(
+ {
+ "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
+ }
+ )
+ with pytest.raises(MultipleInvalid):
+ CrawlerRunner(settings)
+
+ def test_crawler_runner_accepts_dict(self) -> None:
+ runner = CrawlerRunner({"foo": "bar"})
+ assert runner.settings["foo"] == "bar"
+ assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+ def test_crawler_runner_accepts_None(self) -> None:
+ runner = CrawlerRunner()
+ assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+
+class TestAsyncCrawlerRunner:
+ def test_spider_manager_verify_interface(self) -> None:
+ settings = Settings(
+ {
+ "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
+ }
+ )
+ with pytest.raises(MultipleInvalid):
+ AsyncCrawlerRunner(settings)
+
+ def test_crawler_runner_accepts_dict(self) -> None:
+ runner = AsyncCrawlerRunner({"foo": "bar"})
+ assert runner.settings["foo"] == "bar"
+ assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+ def test_crawler_runner_accepts_None(self) -> None:
+ runner = AsyncCrawlerRunner()
+ assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+
+class TestCrawlerProcess:
+ def test_crawler_process_accepts_dict(self) -> None:
+ runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
+ assert runner.settings["foo"] == "bar"
+ assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+ def test_crawler_process_accepts_None(self) -> None:
+ runner = CrawlerProcess(install_root_handler=False)
+ assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+
+class TestAsyncCrawlerProcess:
+ def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:
+ runner = AsyncCrawlerProcess(
+ {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},
+ install_root_handler=False,
+ )
+ assert runner.settings["foo"] == "bar"
+ assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+ @pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False
+ def test_crawler_process_accepts_None(self) -> None:
+ runner = AsyncCrawlerProcess(install_root_handler=False)
+ assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+
[email protected]_reactor # CrawlerRunner requires a reactor
+class TestCrawlerRunnerHasSpider:
+ @pytest.fixture
+ def runner(self) -> CrawlerRunnerBase:
+ return CrawlerRunner(get_reactor_settings())
+
+ @staticmethod
+ async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:
+ await ensure_awaitable(runner.crawl(spider))
+
+ @coroutine_test
+ async def test_crawler_runner_bootstrap_successful(
+ self, runner: CrawlerRunnerBase
+ ) -> None:
+ await self._crawl(runner, NoRequestsSpider)
+ assert not runner.bootstrap_failed
+
+ @coroutine_test
+ async def test_crawler_runner_bootstrap_successful_for_several(
+ self, runner: CrawlerRunnerBase
+ ) -> None:
+ await self._crawl(runner, NoRequestsSpider)
+ await self._crawl(runner, NoRequestsSpider)
+ assert not runner.bootstrap_failed
+
+ @coroutine_test
+ async def test_crawler_runner_bootstrap_failed(
+ self, runner: CrawlerRunnerBase
+ ) -> None:
+ try:
+ await self._crawl(runner, ExceptionSpider)
+ except ValueError:
+ pass
+ else:
+ pytest.fail("Exception should be raised from spider")
+
+ assert runner.bootstrap_failed
+
+ @coroutine_test
+ async def test_crawler_runner_bootstrap_failed_for_several(
+ self, runner: CrawlerRunnerBase
+ ) -> None:
+ try:
+ await self._crawl(runner, ExceptionSpider)
+ except ValueError:
+ pass
+ else:
+ pytest.fail("Exception should be raised from spider")
+
+ await self._crawl(runner, NoRequestsSpider)
+
+ assert runner.bootstrap_failed
+
+ @coroutine_test
+ async def test_crawler_runner_asyncio_enabled_true(
+ self, reactor_pytest: str
+ ) -> None:
+ if reactor_pytest != "asyncio":
+ runner = CrawlerRunner(
+ settings={
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ }
+ )
+ with pytest.raises(
+ Exception,
+ match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
+ ):
+ await self._crawl(runner, NoRequestsSpider)
+ else:
+ runner = CrawlerRunner(
+ settings={
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ }
+ )
+ await self._crawl(runner, NoRequestsSpider)
+
+
+class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
+ @pytest.fixture
+ def runner(self) -> CrawlerRunnerBase:
+ return AsyncCrawlerRunner(get_reactor_settings())
+
+ def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override]
+ pytest.skip("This test is only for CrawlerRunner")
+
+
[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
+def test_runner_settings_applied_to_crawler_instance(
+ runner_cls: type[CrawlerRunnerBase],
+) -> None:
+ runner = runner_cls({"FOO": "runner"})
+ crawler = Crawler(DefaultSpider)
+ result = runner.create_crawler(crawler)
+ assert result is crawler
+ assert result.settings["FOO"] == "runner"
+
+
[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
+def test_spider_custom_settings_override_runner(
+ runner_cls: type[CrawlerRunnerBase],
+) -> None:
+ class MySpider(DefaultSpider):
+ custom_settings = {"FOO": "spider"}
+
+ runner = runner_cls({"FOO": "runner"})
+ crawler = Crawler(MySpider)
+ runner.create_crawler(crawler)
+ assert crawler.settings["FOO"] == "spider"
+
+
+def test_create_crawler_instance_consistent_with_spider_class() -> None:
+ runner = AsyncCrawlerRunner({"FOO": "runner"})
+
+ crawler_from_class = runner.create_crawler(DefaultSpider)
+
+ pre_built = Crawler(DefaultSpider)
+ runner.create_crawler(pre_built)
+
+ assert crawler_from_class.settings["FOO"] == "runner"
+ assert pre_built.settings["FOO"] == "runner"
+
+
[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
+def test_create_crawler_rejects_spider_object(
+ runner_cls: type[CrawlerRunnerBase],
+) -> None:
+ runner = runner_cls()
+ with pytest.raises(ValueError, match="cannot be a spider object"):
+ runner.create_crawler(DefaultSpider()) # type: ignore[arg-type]
+
+
[email protected]("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
+def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:
+ runner = runner_cls()
+ with pytest.raises(ValueError, match="cannot be a spider object"):
+ runner.crawl(DefaultSpider()) # type: ignore[arg-type]
+
+
+@coroutine_test
+async def test_crawlerrunner_accepts_crawler(
+ caplog: pytest.LogCaptureFixture, mockserver: MockServer
+) -> None:
+ crawler = Crawler(SimpleSpider, get_reactor_settings())
+ runner = CrawlerRunner()
+ with caplog.at_level(logging.DEBUG):
+ await maybe_deferred_to_future(
+ runner.crawl(
+ crawler,
+ mockserver.url("/status?n=200"),
+ mockserver=mockserver,
+ )
+ )
+ assert "Got response 200" in caplog.text
+
+
+@coroutine_test
+async def test_crawl_multiple(
+ caplog: pytest.LogCaptureFixture, mockserver: MockServer
+) -> None:
+ settings_dict = get_reactor_settings()
+ runner_cls = (
+ CrawlerRunner
+ if settings_dict.get("TWISTED_REACTOR_ENABLED", True)
+ else AsyncCrawlerRunner
+ )
+ runner = runner_cls(settings_dict)
+ runner.crawl(
+ SimpleSpider,
+ mockserver.url("/status?n=200"),
+ mockserver=mockserver,
+ )
+ runner.crawl(
+ SimpleSpider,
+ mockserver.url("/status?n=503"),
+ mockserver=mockserver,
+ )
+
+ with caplog.at_level(logging.DEBUG):
+ await ensure_awaitable(runner.join())
+
+ assert "Got response 200" in caplog.text
+ assert "Gave up retrying" in caplog.text
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 1da7bbf3e..0824ab531 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -16,11 +16,8 @@ from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
-from tests.test_downloadermiddleware_redirect_base import (
- REDIRECT_SCHEME_CASES,
- SCHEME_PARAMS,
- Base,
-)
+from tests.test_downloadermiddleware_redirect_base import Base
+from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS
class TestRedirectMiddleware(Base.Test):
diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py
index 32935769f..bd8bc796c 100644
--- a/tests/test_downloadermiddleware_redirect_base.py
+++ b/tests/test_downloadermiddleware_redirect_base.py
@@ -1,7 +1,5 @@
from __future__ import annotations
-from itertools import product
-
import pytest
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
@@ -10,43 +8,6 @@ from scrapy.http import Request, Response
from scrapy.utils.misc import set_environ
from scrapy.utils.test import get_crawler
-SCHEME_PARAMS = ("url", "location", "target")
-HTTP_SCHEMES = ("http", "https")
-NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo")
-REDIRECT_SCHEME_CASES = (
- # http/https → http/https redirects
- *(
- (
- f"{input_scheme}://example.com/a",
- f"{output_scheme}://example.com/b",
- f"{output_scheme}://example.com/b",
- )
- for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2)
- ),
- # http/https → data/file/ftp/s3/foo does not redirect
- *(
- (
- f"{input_scheme}://example.com/a",
- f"{output_scheme}://example.com/b",
- None,
- )
- for input_scheme in HTTP_SCHEMES
- for output_scheme in NON_HTTP_SCHEMES
- ),
- # http/https → relative redirects
- *(
- (
- f"{scheme}://example.com/a",
- location,
- f"{scheme}://example.com/b",
- )
- for scheme in HTTP_SCHEMES
- for location in ("//example.com/b", "/b")
- ),
- # Note: We do not test data/file/ftp/s3 schemes for the initial URL
- # because their download handlers cannot return a status code of 3xx.
-)
-
class Base:
class Test:
diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py
index d849cc8fb..b5d39080c 100644
--- a/tests/test_downloadermiddleware_redirect_metarefresh.py
+++ b/tests/test_downloadermiddleware_redirect_metarefresh.py
@@ -12,12 +12,12 @@ from scrapy.http import HtmlResponse, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
-from tests.test_downloadermiddleware_redirect_base import (
+from tests.test_downloadermiddleware_redirect_base import Base
+from tests.utils.redirect import (
HTTP_SCHEMES,
NON_HTTP_SCHEMES,
REDIRECT_SCHEME_CASES,
SCHEME_PARAMS,
- Base,
)
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 082fc743e..f82041a62 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -15,8 +15,8 @@ from scrapy.http.request import NO_CALLBACK
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
-from tests.test_robotstxt_interface import rerp_available
from tests.utils.decorators import coroutine_test
+from tests.utils.robotstxt import rerp_available
if TYPE_CHECKING:
from scrapy.crawler import Crawler
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 2b857d35a..3ab8f6d1b 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -6,8 +6,8 @@ import subprocess
import sys
from collections import defaultdict
from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, cast
-from unittest.mock import Mock, call
+from typing import TYPE_CHECKING, Any
+from unittest.mock import Mock
from urllib.parse import urlparse
import attr
@@ -25,7 +25,6 @@ from scrapy.http import Headers, Request, Response
from scrapy.item import Field, Item
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Spider
-from scrapy.statscollectors import MemoryStatsCollector
from scrapy.utils.defer import (
_schedule_coro,
deferred_from_coro,
@@ -42,8 +41,6 @@ if TYPE_CHECKING:
from twisted.python.failure import Failure
- from scrapy.core.scheduler import Scheduler
- from scrapy.crawler import Crawler
from tests.mockserver.http import MockServer
@@ -515,98 +512,6 @@ class TestEngine(TestEngineBase):
assert "AssertionError" not in stderr_str, stderr_str
-class TestEngineDownloadAsync:
- """Test cases for ExecutionEngine.download_async()."""
-
- @pytest.fixture
- def engine(self) -> ExecutionEngine:
- crawler = get_crawler(MySpider)
- engine = ExecutionEngine(crawler, lambda _: None)
- engine.downloader.close()
- engine.downloader = Mock()
- engine._slot = Mock()
- engine._slot.inprogress = set()
- return engine
-
- @staticmethod
- async def _download(engine: ExecutionEngine, request: Request) -> Response:
- return await engine.download_async(request)
-
- @coroutine_test
- async def test_download_async_success(self, engine):
- """Test basic successful async download of a request."""
- request = Request("http://example.com")
- response = Response("http://example.com", body=b"test body")
- engine.spider = Mock()
- engine.downloader.fetch.return_value = defer.succeed(response)
- engine._slot.add_request = Mock()
- engine._slot.remove_request = Mock()
-
- result = await self._download(engine, request)
- assert result == response
- engine._slot.add_request.assert_called_once_with(request)
- engine._slot.remove_request.assert_called_once_with(request)
- engine.downloader.fetch.assert_called_once_with(request)
-
- @coroutine_test
- async def test_download_async_redirect(self, engine):
- """Test async download with a redirect request."""
- original_request = Request("http://example.com")
- redirect_request = Request("http://example.com/redirect")
- final_response = Response("http://example.com/redirect", body=b"redirected")
-
- # First call returns redirect request, second call returns final response
- engine.downloader.fetch.side_effect = [
- defer.succeed(redirect_request),
- defer.succeed(final_response),
- ]
- engine.spider = Mock()
- engine._slot.add_request = Mock()
- engine._slot.remove_request = Mock()
-
- result = await self._download(engine, original_request)
- assert result == final_response
- assert engine.downloader.fetch.call_count == 2
- engine._slot.add_request.assert_has_calls(
- [call(original_request), call(redirect_request)]
- )
- engine._slot.remove_request.assert_has_calls(
- [call(original_request), call(redirect_request)]
- )
-
- @coroutine_test
- async def test_download_async_no_spider(self, engine):
- """Test async download attempt when no spider is available."""
- request = Request("http://example.com")
- engine.spider = None
- with pytest.raises(RuntimeError, match="No open spider to crawl:"):
- await self._download(engine, request)
-
- @coroutine_test
- async def test_download_async_failure(self, engine):
- """Test async download when the downloader raises an exception."""
- request = Request("http://example.com")
- error = RuntimeError("Download failed")
- engine.spider = Mock()
- engine.downloader.fetch.return_value = defer.fail(error)
- engine._slot.add_request = Mock()
- engine._slot.remove_request = Mock()
-
- with pytest.raises(RuntimeError, match="Download failed"):
- await self._download(engine, request)
- engine._slot.add_request.assert_called_once_with(request)
- engine._slot.remove_request.assert_called_once_with(request)
-
-
[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestEngineDownload(TestEngineDownloadAsync):
- """Test cases for ExecutionEngine.download()."""
-
- @staticmethod
- async def _download(engine: ExecutionEngine, request: Request) -> Response:
- return await maybe_deferred_to_future(engine.download(request))
-
-
@coroutine_test
async def test_request_scheduled_signal():
class TestScheduler(BaseScheduler):
@@ -640,132 +545,3 @@ async def test_request_scheduled_signal():
f"{scheduler.enqueued!r} != [{keep_request!r}]"
)
crawler.signals.disconnect(signal_handler, signals.request_scheduled)
-
-
-class TestEngineCloseSpider:
- """Tests for exception handling coverage during close_spider_async()."""
-
- @pytest.fixture
- def crawler(self) -> Crawler:
- crawler = get_crawler(DefaultSpider)
- crawler.spider = crawler._create_spider()
- return crawler
-
- @coroutine_test
- async def test_no_slot(self, crawler: Crawler) -> None:
- engine = ExecutionEngine(crawler, lambda _: None)
- crawler.engine = engine
- await engine.open_spider_async()
- slot = engine._slot
- engine._slot = None
- with pytest.raises(RuntimeError, match="Engine slot not assigned"):
- await engine.close_spider_async()
- # close it correctly
- engine._slot = slot
- await engine.close_spider_async()
-
- @coroutine_test
- async def test_no_spider(self, crawler: Crawler) -> None:
- engine = ExecutionEngine(crawler, lambda _: None)
- with pytest.raises(RuntimeError, match="Spider not opened"):
- await engine.close_spider_async()
- engine.downloader.close() # cleanup
-
- @coroutine_test
- async def test_exception_slot(
- self, crawler: Crawler, caplog: pytest.LogCaptureFixture
- ) -> None:
- engine = ExecutionEngine(crawler, lambda _: None)
- crawler.engine = engine
- await engine.open_spider_async()
- assert engine._slot
- del engine._slot.heartbeat
- await engine.close_spider_async()
- assert "Slot close failure" in caplog.text
-
- @coroutine_test
- async def test_exception_downloader(
- self, crawler: Crawler, caplog: pytest.LogCaptureFixture
- ) -> None:
- engine = ExecutionEngine(crawler, lambda _: None)
- crawler.engine = engine
- await engine.open_spider_async()
- del engine.downloader.slots
- await engine.close_spider_async()
- assert "Downloader close failure" in caplog.text
-
- @coroutine_test
- async def test_exception_scraper(
- self, crawler: Crawler, caplog: pytest.LogCaptureFixture
- ) -> None:
- engine = ExecutionEngine(crawler, lambda _: None)
- crawler.engine = engine
- await engine.open_spider_async()
- engine.scraper.slot = None
- await engine.close_spider_async()
- assert "Scraper close failure" in caplog.text
-
- @coroutine_test
- async def test_exception_scheduler(
- self, crawler: Crawler, caplog: pytest.LogCaptureFixture
- ) -> None:
- engine = ExecutionEngine(crawler, lambda _: None)
- crawler.engine = engine
- await engine.open_spider_async()
- assert engine._slot
- del cast("Scheduler", engine._slot.scheduler).dqs
- await engine.close_spider_async()
- assert "Scheduler close failure" in caplog.text
-
- @coroutine_test
- async def test_exception_signal(
- self, crawler: Crawler, caplog: pytest.LogCaptureFixture
- ) -> None:
- engine = ExecutionEngine(crawler, lambda _: None)
- crawler.engine = engine
- await engine.open_spider_async()
- signal_manager = engine.signals
- del engine.signals
- await engine.close_spider_async()
- assert "Error while sending spider_close signal" in caplog.text
- # send the spider_closed signal to close various components
- await signal_manager.send_catch_log_async(
- signal=signals.spider_closed,
- spider=engine.spider,
- reason="cancelled",
- )
-
- @coroutine_test
- async def test_exception_stats(
- self, crawler: Crawler, caplog: pytest.LogCaptureFixture
- ) -> None:
- engine = ExecutionEngine(crawler, lambda _: None)
- crawler.engine = engine
- await engine.open_spider_async()
- assert isinstance(crawler.stats, MemoryStatsCollector)
- del crawler.stats.spider_stats
- await engine.close_spider_async()
- assert "Stats close failure" in caplog.text
-
- @coroutine_test
- async def test_exception_callback(
- self, crawler: Crawler, caplog: pytest.LogCaptureFixture
- ) -> None:
- engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))
- crawler.engine = engine
- await engine.open_spider_async()
- await engine.close_spider_async()
- assert "Error running spider_closed_callback" in caplog.text
-
- @coroutine_test
- async def test_exception_async_callback(
- self, crawler: Crawler, caplog: pytest.LogCaptureFixture
- ) -> None:
- async def cb(_):
- raise ValueError
-
- engine = ExecutionEngine(crawler, cb)
- crawler.engine = engine
- await engine.open_spider_async()
- await engine.close_spider_async()
- assert "Error running spider_closed_callback" in caplog.text
diff --git a/tests/test_engine_close_spider.py b/tests/test_engine_close_spider.py
new file mode 100644
index 000000000..8e9c9bd43
--- /dev/null
+++ b/tests/test_engine_close_spider.py
@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, cast
+
+import pytest
+from twisted.internet import defer
+
+from scrapy import signals
+from scrapy.core.engine import ExecutionEngine
+from scrapy.statscollectors import MemoryStatsCollector
+from scrapy.utils.spider import DefaultSpider
+from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+ from scrapy.core.scheduler import Scheduler
+ from scrapy.crawler import Crawler
+
+
+def crawler() -> Crawler:
+ crawler = get_crawler(DefaultSpider)
+ crawler.spider = crawler._create_spider()
+ return crawler
+
+
+@coroutine_test
+async def test_no_slot(crawler: Crawler) -> None:
+ engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
+ await engine.open_spider_async()
+ slot = engine._slot
+ engine._slot = None
+ with pytest.raises(RuntimeError, match="Engine slot not assigned"):
+ await engine.close_spider_async()
+ # close it correctly
+ engine._slot = slot
+ await engine.close_spider_async()
+
+
+@coroutine_test
+async def test_no_spider(crawler: Crawler) -> None:
+ engine = ExecutionEngine(crawler, lambda _: None)
+ with pytest.raises(RuntimeError, match="Spider not opened"):
+ await engine.close_spider_async()
+ engine.downloader.close() # cleanup
+
+
+@coroutine_test
+async def test_exception_slot(
+ crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+ engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
+ await engine.open_spider_async()
+ assert engine._slot
+ del engine._slot.heartbeat
+ await engine.close_spider_async()
+ assert "Slot close failure" in caplog.text
+
+
+@coroutine_test
+async def test_exception_downloader(
+ crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+ engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
+ await engine.open_spider_async()
+ del engine.downloader.slots
+ await engine.close_spider_async()
+ assert "Downloader close failure" in caplog.text
+
+
+@coroutine_test
+async def test_exception_scraper(
+ crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+ engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
+ await engine.open_spider_async()
+ engine.scraper.slot = None
+ await engine.close_spider_async()
+ assert "Scraper close failure" in caplog.text
+
+
+@coroutine_test
+async def test_exception_scheduler(
+ crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+ engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
+ await engine.open_spider_async()
+ assert engine._slot
+ del cast("Scheduler", engine._slot.scheduler).dqs
+ await engine.close_spider_async()
+ assert "Scheduler close failure" in caplog.text
+
+
+@coroutine_test
+async def test_exception_signal(
+ crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+ engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
+ await engine.open_spider_async()
+ signal_manager = engine.signals
+ del engine.signals
+ await engine.close_spider_async()
+ assert "Error while sending spider_close signal" in caplog.text
+ # send the spider_closed signal to close various components
+ await signal_manager.send_catch_log_async(
+ signal=signals.spider_closed,
+ spider=engine.spider,
+ reason="cancelled",
+ )
+
+
+@coroutine_test
+async def test_exception_stats(
+ crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+ engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
+ await engine.open_spider_async()
+ assert isinstance(crawler.stats, MemoryStatsCollector)
+ del crawler.stats.spider_stats
+ await engine.close_spider_async()
+ assert "Stats close failure" in caplog.text
+
+
+@coroutine_test
+async def test_exception_callback(
+ crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+ engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))
+ crawler.engine = engine
+ await engine.open_spider_async()
+ await engine.close_spider_async()
+ assert "Error running spider_closed_callback" in caplog.text
+
+
+@coroutine_test
+async def test_exception_async_callback(
+ crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+ async def cb(_):
+ raise ValueError
+
+ engine = ExecutionEngine(crawler, cb)
+ crawler.engine = engine
+ await engine.open_spider_async()
+ await engine.close_spider_async()
+ assert "Error running spider_closed_callback" in caplog.text
diff --git a/tests/test_engine_download.py b/tests/test_engine_download.py
new file mode 100644
index 000000000..f15bfd5e2
--- /dev/null
+++ b/tests/test_engine_download.py
@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from unittest.mock import Mock, call
+
+import pytest
+from twisted.internet import defer
+
+from scrapy.core.engine import ExecutionEngine
+from scrapy.http import Request, Response
+from scrapy.utils.defer import maybe_deferred_to_future
+from scrapy.utils.spider import DefaultSpider
+from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+
+
+def engine() -> ExecutionEngine:
+ crawler = get_crawler(DefaultSpider)
+ engine = ExecutionEngine(crawler, lambda _: None)
+ engine.downloader.close()
+ engine.downloader = Mock()
+ engine._slot = Mock()
+ engine._slot.inprogress = set()
+ return engine
+
+
+class TestEngineDownloadAsync:
+ """Test cases for ExecutionEngine.download_async()."""
+
+ @staticmethod
+ async def _download(engine: ExecutionEngine, request: Request) -> Response:
+ return await engine.download_async(request)
+
+ @coroutine_test
+ async def test_download_async_success(self, engine):
+ """Test basic successful async download of a request."""
+ request = Request("http://example.com")
+ response = Response("http://example.com", body=b"test body")
+ engine.spider = Mock()
+ engine.downloader.fetch.return_value = defer.succeed(response)
+ engine._slot.add_request = Mock()
+ engine._slot.remove_request = Mock()
+
+ result = await self._download(engine, request)
+ assert result == response
+ engine._slot.add_request.assert_called_once_with(request)
+ engine._slot.remove_request.assert_called_once_with(request)
+ engine.downloader.fetch.assert_called_once_with(request)
+
+ @coroutine_test
+ async def test_download_async_redirect(self, engine):
+ """Test async download with a redirect request."""
+ original_request = Request("http://example.com")
+ redirect_request = Request("http://example.com/redirect")
+ final_response = Response("http://example.com/redirect", body=b"redirected")
+
+ # First call returns redirect request, second call returns final response
+ engine.downloader.fetch.side_effect = [
+ defer.succeed(redirect_request),
+ defer.succeed(final_response),
+ ]
+ engine.spider = Mock()
+ engine._slot.add_request = Mock()
+ engine._slot.remove_request = Mock()
+
+ result = await self._download(engine, original_request)
+ assert result == final_response
+ assert engine.downloader.fetch.call_count == 2
+ engine._slot.add_request.assert_has_calls(
+ [call(original_request), call(redirect_request)]
+ )
+ engine._slot.remove_request.assert_has_calls(
+ [call(original_request), call(redirect_request)]
+ )
+
+ @coroutine_test
+ async def test_download_async_no_spider(self, engine):
+ """Test async download attempt when no spider is available."""
+ request = Request("http://example.com")
+ engine.spider = None
+ with pytest.raises(RuntimeError, match="No open spider to crawl:"):
+ await self._download(engine, request)
+
+ @coroutine_test
+ async def test_download_async_failure(self, engine):
+ """Test async download when the downloader raises an exception."""
+ request = Request("http://example.com")
+ error = RuntimeError("Download failed")
+ engine.spider = Mock()
+ engine.downloader.fetch.return_value = defer.fail(error)
+ engine._slot.add_request = Mock()
+ engine._slot.remove_request = Mock()
+
+ with pytest.raises(RuntimeError, match="Download failed"):
+ await self._download(engine, request)
+ engine._slot.add_request.assert_called_once_with(request)
+ engine._slot.remove_request.assert_called_once_with(request)
+
+
[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
+class TestEngineDownload(TestEngineDownloadAsync):
+ """Test cases for ExecutionEngine.download()."""
+
+ @staticmethod
+ async def _download(engine: ExecutionEngine, request: Request) -> Response:
+ return await maybe_deferred_to_future(engine.download(request))
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index 6cc8c0650..ca52961f4 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -2,13 +2,13 @@ from __future__ import annotations
from collections import deque
from logging import ERROR
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider, signals
+from scrapy.core.scheduler import BaseScheduler
from scrapy.utils.asyncio import call_later
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
-from tests.test_scheduler import MemoryScheduler
from tests.utils import async_sleep
from tests.utils.decorators import coroutine_test
@@ -18,6 +18,38 @@ if TYPE_CHECKING:
from scrapy.http import Response
+class MemoryScheduler(BaseScheduler):
+ paused = False
+
+ def __init__(self, *args: Any, **kwargs: Any):
+ super().__init__(*args, **kwargs)
+ self.queue: deque[Request] = deque(
+ Request(value) if isinstance(value, str) else value
+ for value in getattr(self, "queue", [])
+ )
+
+ def enqueue_request(self, request: Request) -> bool:
+ self.queue.append(request)
+ return True
+
+ def has_pending_requests(self) -> bool:
+ return self.paused or bool(self.queue)
+
+ def next_request(self) -> Request | None:
+ if self.paused:
+ return None
+ try:
+ return self.queue.pop()
+ except IndexError:
+ return None
+
+ def pause(self) -> None:
+ self.paused = True
+
+ def unpause(self) -> None:
+ self.paused = False
+
+
class TestMain:
@coroutine_test
async def test_sleep(self):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 5ab1a54c1..853243b97 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -13,8 +13,6 @@ from pathlib import Path
from string import ascii_letters, digits
from typing import IO, TYPE_CHECKING, Any
from unittest import mock
-from urllib.parse import urljoin
-from urllib.request import pathname2url
import lxml.etree
import pytest
@@ -39,19 +37,12 @@ from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import ItemSpider
from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.feedexport import MyItem, MyItem2, path_to_url, printf_escape
if TYPE_CHECKING:
from collections.abc import Awaitable, Callable, Iterable
-def path_to_url(path: str | Path) -> str:
- return urljoin("file:", pathname2url(str(path)))
-
-
-def printf_escape(s: str) -> str:
- return s.replace("%", "%%")
-
-
class FromCrawlerMixin:
init_with_crawler = False
@@ -114,15 +105,6 @@ class LogOnStoreFileStorage:
class TestFeedExportBase(ABC):
mockserver: MockServer
- class MyItem(scrapy.Item):
- foo = scrapy.Field()
- egg = scrapy.Field()
- baz = scrapy.Field()
-
- class MyItem2(scrapy.Item):
- foo = scrapy.Field()
- hello = scrapy.Field()
-
def _random_temp_filename(self, inter_dir="") -> Path:
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
filename = "".join(chars)
@@ -517,21 +499,21 @@ class TestFeedExport(TestFeedExportBase):
async def test_export_items(self):
# feed exporters use field names from Item
items = [
- self.MyItem({"foo": "bar1", "egg": "spam1"}),
- self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
+ MyItem({"foo": "bar1", "egg": "spam1"}),
+ MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
]
rows = [
{"egg": "spam1", "foo": "bar1", "baz": ""},
{"egg": "spam2", "foo": "bar2", "baz": "quux2"},
]
- header = self.MyItem.fields.keys()
+ header = MyItem.fields.keys()
await self.assertExported(items, header, rows)
@coroutine_test
async def test_pathlib_uri_with_placeholders(self):
feed_dir = Path(self.temp_dir, "pathlib_placeholders")
feed_dir.mkdir()
- items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
+ items = [MyItem({"foo": "bar1", "egg": "spam1"})]
class TestSpider(scrapy.Spider):
name = "testspider"
@@ -561,7 +543,7 @@ class TestFeedExport(TestFeedExportBase):
# so the resulting file name can be asserted exactly.
feed_dir = Path(self.temp_dir, "pathlib_spaces_unicode")
feed_dir.mkdir()
- items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
+ items = [MyItem({"foo": "bar1", "egg": "spam1"})]
class TestSpider(scrapy.Spider):
name = "testspider"
@@ -590,7 +572,7 @@ class TestFeedExport(TestFeedExportBase):
# and #5794.
feed_dir = Path(self.temp_dir, "dir with spaces")
feed_dir.mkdir()
- items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
+ items = [MyItem({"foo": "bar1", "egg": "spam1"})]
class TestSpider(scrapy.Spider):
name = "testspider"
@@ -627,7 +609,7 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_start_finish_exporting_items(self):
items = [
- self.MyItem({"foo": "bar1", "egg": "spam1"}),
+ MyItem({"foo": "bar1", "egg": "spam1"}),
]
settings = {
"FEEDS": {
@@ -665,7 +647,7 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_start_finish_exporting_items_exception(self):
items = [
- self.MyItem({"foo": "bar1", "egg": "spam1"}),
+ MyItem({"foo": "bar1", "egg": "spam1"}),
]
settings = {
"FEEDS": {
@@ -743,15 +725,15 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_export_multiple_item_classes(self):
items = [
- self.MyItem({"foo": "bar1", "egg": "spam1"}),
- self.MyItem2({"hello": "world2", "foo": "bar2"}),
- self.MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}),
+ MyItem({"foo": "bar1", "egg": "spam1"}),
+ MyItem2({"hello": "world2", "foo": "bar2"}),
+ MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}),
{"hello": "world4", "egg": "spam4"},
]
# by default, Scrapy uses fields of the first Item for CSV and
# all fields for JSON Lines
- header = self.MyItem.fields.keys()
+ header = MyItem.fields.keys()
rows_csv = [
{"egg": "spam1", "foo": "bar1", "baz": ""},
{"egg": "", "foo": "bar2", "baz": ""},
@@ -826,8 +808,8 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_export_based_on_item_classes(self):
items = [
- self.MyItem({"foo": "bar1", "egg": "spam1"}),
- self.MyItem2({"hello": "world2", "foo": "bar2"}),
+ MyItem({"foo": "bar1", "egg": "spam1"}),
+ MyItem2({"hello": "world2", "foo": "bar2"}),
{"hello": "world3", "egg": "spam3"},
]
@@ -849,15 +831,15 @@ class TestFeedExport(TestFeedExportBase):
"FEEDS": {
self._random_temp_filename(): {
"format": "csv",
- "item_classes": [self.MyItem],
+ "item_classes": [MyItem],
},
self._random_temp_filename(): {
"format": "json",
- "item_classes": [self.MyItem2],
+ "item_classes": [MyItem2],
},
self._random_temp_filename(): {
"format": "jsonlines",
- "item_classes": [self.MyItem, self.MyItem2],
+ "item_classes": [MyItem, MyItem2],
},
self._random_temp_filename(): {
"format": "xml",
@@ -872,13 +854,11 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_export_based_on_custom_filters(self):
items = [
- self.MyItem({"foo": "bar1", "egg": "spam1"}),
- self.MyItem2({"hello": "world2", "foo": "bar2"}),
+ MyItem({"foo": "bar1", "egg": "spam1"}),
+ MyItem2({"hello": "world2", "foo": "bar2"}),
{"hello": "world3", "egg": "spam3"},
]
- MyItem = self.MyItem
-
class CustomFilter1:
def __init__(self, feed_options):
pass
@@ -918,7 +898,7 @@ class TestFeedExport(TestFeedExportBase):
},
self._random_temp_filename(): {
"format": "jsonlines",
- "item_classes": [self.MyItem, self.MyItem2],
+ "item_classes": [MyItem, MyItem2],
"item_filter": CustomFilter3,
},
},
@@ -957,7 +937,7 @@ class TestFeedExport(TestFeedExportBase):
# FEED_EXPORT_FIELDS option allows to order export fields
# and to select a subset of fields to export, both for Items and dicts.
- for item_cls in [self.MyItem, dict]:
+ for item_cls in [MyItem, dict]:
items = [
item_cls({"foo": "bar1", "egg": "spam1"}),
item_cls({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index 469985288..6e0536110 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -24,6 +24,7 @@ from scrapy.utils.test import get_crawler
from tests.spiders import ItemSpider
from tests.test_feedexport import TestFeedExportBase
from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.feedexport import MyItem
if TYPE_CHECKING:
from os import PathLike
@@ -197,9 +198,9 @@ class TestBatchDeliveries(TestFeedExportBase):
async def test_export_items(self):
"""Test partial deliveries in all supported formats"""
items = [
- self.MyItem({"foo": "bar1", "egg": "spam1"}),
- self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
- self.MyItem({"foo": "bar3", "baz": "quux3"}),
+ MyItem({"foo": "bar1", "egg": "spam1"}),
+ MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
+ MyItem({"foo": "bar3", "baz": "quux3"}),
]
rows = [
{"egg": "spam1", "foo": "bar1", "baz": ""},
@@ -207,7 +208,7 @@ class TestBatchDeliveries(TestFeedExportBase):
{"foo": "bar3", "baz": "quux3", "egg": ""},
]
settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2}
- header = self.MyItem.fields.keys()
+ header = MyItem.fields.keys()
await self.assertExported(items, header, rows, settings=settings)
def test_wrong_path(self):
@@ -349,9 +350,9 @@ class TestBatchDeliveries(TestFeedExportBase):
So %(batch_id)d replaced with the current id.
"""
items = [
- self.MyItem({"foo": "bar1", "egg": "spam1"}),
- self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
- self.MyItem({"foo": "bar3", "baz": "quux3"}),
+ MyItem({"foo": "bar1", "egg": "spam1"}),
+ MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
+ MyItem({"foo": "bar3", "baz": "quux3"}),
]
settings = {
"FEEDS": {
@@ -387,9 +388,9 @@ class TestBatchDeliveries(TestFeedExportBase):
def test_s3_export(self):
bucket = "mybucket"
items = [
- self.MyItem({"foo": "bar1", "egg": "spam1"}),
- self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
- self.MyItem({"foo": "bar3", "baz": "quux3"}),
+ MyItem({"foo": "bar1", "egg": "spam1"}),
+ MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
+ MyItem({"foo": "bar3", "baz": "quux3"}),
]
class CustomS3FeedStorage(S3FeedStorage):
diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py
index 6ebcab152..83d976e94 100644
--- a/tests/test_feedexport_postprocess.py
+++ b/tests/test_feedexport_postprocess.py
@@ -13,8 +13,9 @@ from typing import TYPE_CHECKING, Any
import pytest
from scrapy.utils.test import get_crawler
-from tests.test_feedexport import TestFeedExportBase, path_to_url, printf_escape
+from tests.test_feedexport import TestFeedExportBase
from tests.utils.decorators import coroutine_test
+from tests.utils.feedexport import path_to_url, printf_escape
if TYPE_CHECKING:
from scrapy import Spider
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 5249736f2..da94a4e95 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -7,17 +7,7 @@ from scrapy.robotstxt import (
decode_robotstxt,
)
from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
-
-
-def rerp_available() -> bool:
- # check if robotexclusionrulesparser is installed
- try:
- from robotexclusionrulesparser import ( # noqa: PLC0415
- RobotExclusionRulesParser, # noqa: F401
- )
- except ImportError:
- return False
- return True
+from tests.utils.robotstxt import rerp_available
class BaseRobotParserTest:
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 39ce87b65..4fb594444 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -2,7 +2,6 @@ from __future__ import annotations
import warnings
from abc import ABC, abstractmethod
-from collections import deque
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from typing import TYPE_CHECKING, Any, NamedTuple, cast
from unittest.mock import Mock
@@ -10,7 +9,7 @@ from unittest.mock import Mock
import pytest
from scrapy.core.downloader import Downloader
-from scrapy.core.scheduler import BaseScheduler, Scheduler
+from scrapy.core.scheduler import Scheduler
from scrapy.crawler import Crawler
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
@@ -27,38 +26,6 @@ if TYPE_CHECKING:
from pathlib import Path
-class MemoryScheduler(BaseScheduler):
- paused = False
-
- def __init__(self, *args: Any, **kwargs: Any):
- super().__init__(*args, **kwargs)
- self.queue: deque[Request] = deque(
- Request(value) if isinstance(value, str) else value
- for value in getattr(self, "queue", [])
- )
-
- def enqueue_request(self, request: Request) -> bool:
- self.queue.append(request)
- return True
-
- def has_pending_requests(self) -> bool:
- return self.paused or bool(self.queue)
-
- def next_request(self) -> Request | None:
- if self.paused:
- return None
- try:
- return self.queue.pop()
- except IndexError:
- return None
-
- def pause(self) -> None:
- self.paused = True
-
- def unpause(self) -> None:
- self.paused = False
-
-
class MockSlot(NamedTuple):
active: list[Any]
diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py
index 73c0e3776..b0632a7ea 100644
--- a/tests/utils/__init__.py
+++ b/tests/utils/__init__.py
@@ -1,13 +1,19 @@
+from __future__ import annotations
+
import asyncio
import os
-from collections.abc import Callable
from pathlib import Path
+from typing import TYPE_CHECKING
from twisted.internet.defer import Deferred
+from scrapy.settings import Settings, default_settings
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.defer import maybe_deferred_to_future
+if TYPE_CHECKING:
+ from collections.abc import Callable
+
def twisted_sleep(seconds: float):
from twisted.internet import reactor
@@ -48,3 +54,8 @@ class OneShotLoop:
def stop(self) -> None:
self.running = False
+
+
+def assert_option_is_default(settings: Settings, key: str) -> None:
+ assert isinstance(settings, Settings)
+ assert settings[key] == getattr(default_settings, key)
diff --git a/tests/utils/base_commands.py b/tests/utils/base_commands.py
new file mode 100644
index 000000000..594544c83
--- /dev/null
+++ b/tests/utils/base_commands.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from shutil import copytree
+from typing import TYPE_CHECKING
+
+import pytest
+
+from tests.utils.cmdline import call
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+
+class TestProjectBase:
+ """A base class for tests that may need a Scrapy project."""
+
+ project_name = "testproject"
+
+ @pytest.fixture(scope="session")
+ def _proj_path_cached(self, tmp_path_factory: pytest.TempPathFactory) -> Path:
+ """Create a Scrapy project in a temporary directory and return its path.
+
+ Used as a cache for ``proj_path``.
+ """
+ tmp_path = tmp_path_factory.mktemp("proj")
+ call("startproject", self.project_name, cwd=tmp_path)
+ return tmp_path / self.project_name
+
+ @pytest.fixture
+ def proj_path(self, tmp_path: Path, _proj_path_cached: Path) -> Path:
+ """Copy a pre-generated Scrapy project into a temporary directory and return its path."""
+ proj_path = tmp_path / self.project_name
+ copytree(_proj_path_cached, proj_path)
+ return proj_path
diff --git a/tests/utils/cmdline.py b/tests/utils/cmdline.py
index 122e0236d..62dff3d4c 100644
--- a/tests/utils/cmdline.py
+++ b/tests/utils/cmdline.py
@@ -2,12 +2,15 @@ from __future__ import annotations
import subprocess
import sys
-from typing import Any
+from typing import TYPE_CHECKING, Any
import pytest
from scrapy.utils.test import get_testenv
+if TYPE_CHECKING:
+ from pathlib import Path
+
def call(*args: str, **popen_kwargs: Any) -> int:
args = (sys.executable, "-m", "scrapy.cmdline", *args)
@@ -36,3 +39,10 @@ def proc(*args: str, **popen_kwargs: Any) -> tuple[int, str, str]:
pytest.fail("Command took too much time to complete")
return p.returncode, p.stdout, p.stderr
+
+
+def write_recording_editor(editor: Path) -> None:
+ """Create an executable editor script that writes the path it is asked to
+ open (its last argument) into the file given as its first argument."""
+ editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
+ editor.chmod(0o755)
diff --git a/tests/utils/feedexport.py b/tests/utils/feedexport.py
new file mode 100644
index 000000000..515082217
--- /dev/null
+++ b/tests/utils/feedexport.py
@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from urllib.parse import urljoin
+from urllib.request import pathname2url
+
+from scrapy import Field, Item
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+
+def path_to_url(path: str | Path) -> str:
+ return urljoin("file:", pathname2url(str(path)))
+
+
+def printf_escape(s: str) -> str:
+ return s.replace("%", "%%")
+
+
+class MyItem(Item):
+ foo = Field()
+ egg = Field()
+ baz = Field()
+
+
+class MyItem2(Item):
+ foo = Field()
+ hello = Field()
diff --git a/tests/utils/redirect.py b/tests/utils/redirect.py
new file mode 100644
index 000000000..a3a572988
--- /dev/null
+++ b/tests/utils/redirect.py
@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from itertools import product
+
+SCHEME_PARAMS = ("url", "location", "target")
+HTTP_SCHEMES = ("http", "https")
+NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo")
+REDIRECT_SCHEME_CASES = (
+ # http/https → http/https redirects
+ *(
+ (
+ f"{input_scheme}://example.com/a",
+ f"{output_scheme}://example.com/b",
+ f"{output_scheme}://example.com/b",
+ )
+ for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2)
+ ),
+ # http/https → data/file/ftp/s3/foo does not redirect
+ *(
+ (
+ f"{input_scheme}://example.com/a",
+ f"{output_scheme}://example.com/b",
+ None,
+ )
+ for input_scheme in HTTP_SCHEMES
+ for output_scheme in NON_HTTP_SCHEMES
+ ),
+ # http/https → relative redirects
+ *(
+ (
+ f"{scheme}://example.com/a",
+ location,
+ f"{scheme}://example.com/b",
+ )
+ for scheme in HTTP_SCHEMES
+ for location in ("//example.com/b", "/b")
+ ),
+ # Note: We do not test data/file/ftp/s3 schemes for the initial URL
+ # because their download handlers cannot return a status code of 3xx.
+)
diff --git a/tests/utils/robotstxt.py b/tests/utils/robotstxt.py
new file mode 100644
index 000000000..c1b2f28c6
--- /dev/null
+++ b/tests/utils/robotstxt.py
@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+
+def rerp_available() -> bool:
+ # check if robotexclusionrulesparser is installed
+ try:
+ from robotexclusionrulesparser import ( # noqa: PLC0415
+ RobotExclusionRulesParser, # noqa: F401
+ )
+ except ImportError:
+ return False
+ return True