Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Unpack some test classes that are just containers. (#8000)

Andrey Rakhmatullinc7eacdbb30a4
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 81c122b81..99c552b80 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -10,110 +10,104 @@ from tests.spiders import (
from tests.utils.decorators import inline_callbacks_test
-class TestCloseSpider:
- mockserver: MockServer
+@inline_callbacks_test
+def test_closespider_itemcount(mockserver: MockServer):
+ close_on = 5
+ crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})
+ yield crawler.crawl(mockserver=mockserver)
+ assert isinstance(crawler.spider, ItemSpider)
+ reason = crawler.spider.meta["close_reason"]
+ assert reason == "closespider_itemcount"
+ itemcount = crawler.stats.get_value("item_scraped_count")
+ assert itemcount >= close_on
- @classmethod
- def setup_class(cls):
- cls.mockserver = MockServer()
- cls.mockserver.__enter__()
- @classmethod
- def teardown_class(cls):
- cls.mockserver.__exit__(None, None, None)
+@inline_callbacks_test
+def test_closespider_pagecount(mockserver: MockServer):
+ close_on = 5
+ crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})
+ yield crawler.crawl(mockserver=mockserver)
+ assert isinstance(crawler.spider, FollowAllSpider)
+ reason = crawler.spider.meta["close_reason"]
+ assert reason == "closespider_pagecount"
+ pagecount = crawler.stats.get_value("response_received_count")
+ assert pagecount >= close_on
- @inline_callbacks_test
- def test_closespider_itemcount(self):
- close_on = 5
- crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})
- yield crawler.crawl(mockserver=self.mockserver)
- assert isinstance(crawler.spider, ItemSpider)
- reason = crawler.spider.meta["close_reason"]
- assert reason == "closespider_itemcount"
- itemcount = crawler.stats.get_value("item_scraped_count")
- assert itemcount >= close_on
- @inline_callbacks_test
- def test_closespider_pagecount(self):
- close_on = 5
- crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})
- yield crawler.crawl(mockserver=self.mockserver)
- assert isinstance(crawler.spider, FollowAllSpider)
- reason = crawler.spider.meta["close_reason"]
- assert reason == "closespider_pagecount"
- pagecount = crawler.stats.get_value("response_received_count")
- assert pagecount >= close_on
+@inline_callbacks_test
+def test_closespider_pagecount_no_item(mockserver: MockServer):
+ close_on = 5
+ max_items = 5
+ max_requests = close_on + max_items
+ crawler = get_crawler(
+ MaxItemsAndRequestsSpider,
+ {
+ "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on,
+ },
+ )
+ yield crawler.crawl(
+ max_items=max_items, max_requests=max_requests, mockserver=mockserver
+ )
+ assert isinstance(crawler.spider, MaxItemsAndRequestsSpider)
+ reason = crawler.spider.meta["close_reason"]
+ assert reason == "closespider_pagecount_no_item"
+ pagecount = crawler.stats.get_value("response_received_count")
+ itemcount = crawler.stats.get_value("item_scraped_count")
+ assert pagecount <= close_on + itemcount
- @inline_callbacks_test
- def test_closespider_pagecount_no_item(self):
- close_on = 5
- max_items = 5
- max_requests = close_on + max_items
- crawler = get_crawler(
- MaxItemsAndRequestsSpider,
- {
- "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on,
- },
- )
- yield crawler.crawl(
- max_items=max_items, max_requests=max_requests, mockserver=self.mockserver
- )
- assert isinstance(crawler.spider, MaxItemsAndRequestsSpider)
- reason = crawler.spider.meta["close_reason"]
- assert reason == "closespider_pagecount_no_item"
- pagecount = crawler.stats.get_value("response_received_count")
- itemcount = crawler.stats.get_value("item_scraped_count")
- assert pagecount <= close_on + itemcount
- @inline_callbacks_test
- def test_closespider_pagecount_no_item_with_pagecount(self):
- close_on_pagecount_no_item = 5
- close_on_pagecount = 20
- crawler = get_crawler(
- FollowAllSpider,
- {
- "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on_pagecount_no_item,
- "CLOSESPIDER_PAGECOUNT": close_on_pagecount,
- },
- )
- yield crawler.crawl(mockserver=self.mockserver)
- assert isinstance(crawler.spider, FollowAllSpider)
- reason = crawler.spider.meta["close_reason"]
- assert reason == "closespider_pagecount_no_item"
- pagecount = crawler.stats.get_value("response_received_count")
- assert pagecount < close_on_pagecount
+@inline_callbacks_test
+def test_closespider_pagecount_no_item_with_pagecount(mockserver: MockServer):
+ close_on_pagecount_no_item = 5
+ close_on_pagecount = 20
+ crawler = get_crawler(
+ FollowAllSpider,
+ {
+ "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on_pagecount_no_item,
+ "CLOSESPIDER_PAGECOUNT": close_on_pagecount,
+ },
+ )
+ yield crawler.crawl(mockserver=mockserver)
+ assert isinstance(crawler.spider, FollowAllSpider)
+ reason = crawler.spider.meta["close_reason"]
+ assert reason == "closespider_pagecount_no_item"
+ pagecount = crawler.stats.get_value("response_received_count")
+ assert pagecount < close_on_pagecount
- @inline_callbacks_test
- def test_closespider_errorcount(self):
- close_on = 5
- crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})
- yield crawler.crawl(total=1000000, mockserver=self.mockserver)
- assert isinstance(crawler.spider, ErrorSpider)
- reason = crawler.spider.meta["close_reason"]
- assert reason == "closespider_errorcount"
- key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}"
- errorcount = crawler.stats.get_value(key)
- assert crawler.stats.get_value("spider_exceptions/count") >= close_on
- assert errorcount >= close_on
- @inline_callbacks_test
- def test_closespider_timeout(self):
- close_on = 0.1
- crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})
- yield crawler.crawl(total=1000000, mockserver=self.mockserver)
- assert isinstance(crawler.spider, FollowAllSpider)
- reason = crawler.spider.meta["close_reason"]
- assert reason == "closespider_timeout"
- total_seconds = crawler.stats.get_value("elapsed_time_seconds")
- assert total_seconds >= close_on
+@inline_callbacks_test
+def test_closespider_errorcount(mockserver: MockServer):
+ close_on = 5
+ crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})
+ yield crawler.crawl(total=1000000, mockserver=mockserver)
+ assert isinstance(crawler.spider, ErrorSpider)
+ reason = crawler.spider.meta["close_reason"]
+ assert reason == "closespider_errorcount"
+ key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}"
+ errorcount = crawler.stats.get_value(key)
+ assert crawler.stats.get_value("spider_exceptions/count") >= close_on
+ assert errorcount >= close_on
- @inline_callbacks_test
- def test_closespider_timeout_no_item(self):
- timeout = 1
- crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})
- yield crawler.crawl(n=3, mockserver=self.mockserver)
- assert isinstance(crawler.spider, SlowSpider)
- reason = crawler.spider.meta["close_reason"]
- assert reason == "closespider_timeout_no_item"
- total_seconds = crawler.stats.get_value("elapsed_time_seconds")
- assert total_seconds >= timeout
+
+@inline_callbacks_test
+def test_closespider_timeout(mockserver: MockServer):
+ close_on = 0.1
+ crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})
+ yield crawler.crawl(total=1000000, mockserver=mockserver)
+ assert isinstance(crawler.spider, FollowAllSpider)
+ reason = crawler.spider.meta["close_reason"]
+ assert reason == "closespider_timeout"
+ total_seconds = crawler.stats.get_value("elapsed_time_seconds")
+ assert total_seconds >= close_on
+
+
+@inline_callbacks_test
+def test_closespider_timeout_no_item(mockserver: MockServer):
+ timeout = 1
+ crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})
+ yield crawler.crawl(n=3, mockserver=mockserver)
+ assert isinstance(crawler.spider, SlowSpider)
+ reason = crawler.spider.meta["close_reason"]
+ assert reason == "closespider_timeout_no_item"
+ total_seconds = crawler.stats.get_value("elapsed_time_seconds")
+ assert total_seconds >= timeout
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index f6ebe5865..c068bb0f3 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -12,61 +12,60 @@ from subprocess import PIPE, Popen
from scrapy.utils.test import get_testenv
-class TestCmdline:
- def setup_method(self):
- self.env = get_testenv()
- tests_path = Path(__file__).parent.parent
- self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent)
- self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"
+def _execute(*new_args, **kwargs):
+ env = get_testenv()
+ tests_path = Path(__file__).parent.parent
+ env["PYTHONPATH"] += os.pathsep + str(tests_path.parent)
+ env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"
+ encoding = sys.stdout.encoding or "utf-8"
+ args = (sys.executable, "-m", "scrapy.cmdline", *new_args)
+ proc = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs)
+ comm = proc.communicate()[0].strip()
+ return comm.decode(encoding)
- def _execute(self, *new_args, **kwargs):
- encoding = sys.stdout.encoding or "utf-8"
- args = (sys.executable, "-m", "scrapy.cmdline", *new_args)
- proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs)
- comm = proc.communicate()[0].strip()
- return comm.decode(encoding)
- def test_default_settings(self):
- assert self._execute("settings", "--get", "TEST1") == "default"
+def test_default_settings():
+ assert _execute("settings", "--get", "TEST1") == "default"
- def test_override_settings_using_set_arg(self):
- assert (
- self._execute("settings", "--get", "TEST1", "-s", "TEST1=override")
- == "override"
- )
- def test_profiling(self):
- path = Path(tempfile.mkdtemp())
- filename = path / "res.prof"
- try:
- self._execute("version", "--profile", str(filename))
- assert filename.exists()
- out = StringIO()
- stats = pstats.Stats(str(filename), stream=out)
- stats.print_stats()
- out.seek(0)
- stats = out.read()
- assert str(Path("scrapy", "commands", "version.py")) in stats
- assert "tottime" in stats
- finally:
- shutil.rmtree(path)
+def test_override_settings_using_set_arg():
+ assert _execute("settings", "--get", "TEST1", "-s", "TEST1=override") == "override"
- def test_override_dict_settings(self):
- EXT_PATH = "tests.test_cmdline.extensions.DummyExtension"
- EXTENSIONS = {EXT_PATH: 200}
- settingsstr = self._execute(
- "settings",
- "--get",
- "EXTENSIONS",
- "-s",
- "EXTENSIONS=" + json.dumps(EXTENSIONS),
- )
- assert "..." not in settingsstr
- settingsdict = ast.literal_eval(settingsstr)
- assert set(settingsdict.keys()) == set(EXTENSIONS.keys())
- assert settingsdict[EXT_PATH] == 200
- def test_pathlib_path_as_feeds_key(self):
- assert self._execute("settings", "--get", "FEEDS") == json.dumps(
- {"items.csv": {"format": "csv", "fields": ["price", "name"]}}
- )
+def test_profiling():
+ path = Path(tempfile.mkdtemp())
+ filename = path / "res.prof"
+ try:
+ _execute("version", "--profile", str(filename))
+ assert filename.exists()
+ out = StringIO()
+ stats = pstats.Stats(str(filename), stream=out)
+ stats.print_stats()
+ out.seek(0)
+ stats = out.read()
+ assert str(Path("scrapy", "commands", "version.py")) in stats
+ assert "tottime" in stats
+ finally:
+ shutil.rmtree(path)
+
+
+def test_override_dict_settings():
+ EXT_PATH = "tests.test_cmdline.extensions.DummyExtension"
+ EXTENSIONS = {EXT_PATH: 200}
+ settingsstr = _execute(
+ "settings",
+ "--get",
+ "EXTENSIONS",
+ "-s",
+ "EXTENSIONS=" + json.dumps(EXTENSIONS),
+ )
+ assert "..." not in settingsstr
+ settingsdict = ast.literal_eval(settingsstr)
+ assert set(settingsdict.keys()) == set(EXTENSIONS.keys())
+ assert settingsdict[EXT_PATH] == 200
+
+
+def test_pathlib_path_as_feeds_key():
+ assert _execute("settings", "--get", "FEEDS") == json.dumps(
+ {"items.csv": {"format": "csv", "fields": ["price", "name"]}}
+ )
diff --git a/tests/test_command_version.py b/tests/test_command_version.py
index c4ca9d07b..39132b64b 100644
--- a/tests/test_command_version.py
+++ b/tests/test_command_version.py
@@ -2,24 +2,24 @@ import scrapy
from tests.utils.cmdline import proc
-class TestVersionCommand:
- def test_output(self) -> None:
- _, out, _ = proc("version")
- assert out.strip() == f"Scrapy {scrapy.__version__}"
+def test_output() -> None:
+ _, out, _ = proc("version")
+ assert out.strip() == f"Scrapy {scrapy.__version__}"
- def test_verbose_output(self) -> None:
- _, out, _ = proc("version", "-v")
- headers = [line.partition(":")[0].strip() for line in out.strip().splitlines()]
- assert headers == [
- "Scrapy",
- "lxml",
- "libxml2",
- "cssselect",
- "parsel",
- "w3lib",
- "Twisted",
- "Python",
- "pyOpenSSL",
- "cryptography",
- "Platform",
- ]
+
+def test_verbose_output() -> None:
+ _, out, _ = proc("version", "-v")
+ headers = [line.partition(":")[0].strip() for line in out.strip().splitlines()]
+ assert headers == [
+ "Scrapy",
+ "lxml",
+ "libxml2",
+ "cssselect",
+ "parsel",
+ "w3lib",
+ "Twisted",
+ "Python",
+ "pyOpenSSL",
+ "cryptography",
+ "Platform",
+ ]
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index 41c7ce526..353b6d019 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -9,23 +9,22 @@ import pytest
from twisted import version as twisted_version
-class TestScrapyUtils:
- def test_pinned_twisted_version(self):
- """When running tests within a Tox environment with pinned
- dependencies, make sure that the version of Twisted is the pinned
- version.
+def test_pinned_twisted_version() -> None:
+ """When running tests within a Tox environment with pinned
+ dependencies, make sure that the version of Twisted is the pinned
+ version.
- See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011
- """
- if not os.environ.get("_SCRAPY_MIN", None):
- pytest.skip("Not in a min environment")
+ See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011
+ """
+ if not os.environ.get("_SCRAPY_MIN", None):
+ pytest.skip("Not in a min environment")
- tox_config_file_path = Path(__file__).parent / ".." / "tox.ini"
- config_parser = ConfigParser()
- config_parser.read(tox_config_file_path)
- pattern = r"Twisted==([\d.]+)"
- match = re.search(pattern, config_parser["min"]["deps"])
- assert match
- pinned_twisted_version_string = match[1]
+ tox_config_file_path = Path(__file__).parent / ".." / "tox.ini"
+ config_parser = ConfigParser()
+ config_parser.read(tox_config_file_path)
+ pattern = r"Twisted==([\d.]+)"
+ match = re.search(pattern, config_parser["min"]["deps"])
+ assert match
+ pinned_twisted_version_string = match[1]
- assert twisted_version.short() == pinned_twisted_version_string
+ assert twisted_version.short() == pinned_twisted_version_string
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 01014ddd6..cd7a61301 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -24,6 +24,9 @@ class TestDownloaderStats:
self.req = Request("http://scrapytest.org")
self.res = Response("http://scrapytest.org", status=400)
+ def teardown_method(self) -> None:
+ self.crawler.stats.close_spider()
+
def assertStatsEqual(self, key: str, value: object) -> None:
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
@@ -50,9 +53,6 @@ class TestDownloaderStats:
with pytest.raises(NotConfigured):
build_from_crawler(DownloaderStats, crawler)
- def teardown_method(self) -> None:
- self.crawler.stats.close_spider()
-
def test_get_header_size_non_list_value() -> None:
# Deliberately passing a non-list/tuple header value to make sure
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 077a92ecf..4cf4cf2b1 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -54,30 +54,21 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):
self.times[slot].append(time.time())
-class TestCrawl:
- @classmethod
- def setup_class(cls):
- cls.mockserver = MockServer()
- cls.mockserver.__enter__()
-
- @classmethod
- def teardown_class(cls):
- cls.mockserver.__exit__(None, None, None)
-
- @inline_callbacks_test
- def test_delay(self):
- crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
- yield crawler.crawl(mockserver=self.mockserver)
- slots = crawler.engine.downloader.slots
- times = crawler.spider.times
- # Downloading and processing a response add a roughly constant amount
- # of time on top of the configured delay, so the margin is absolute.
- # It stays well below the 1 second that separates the delays being
- # compared, so a slot using the delay of another one still fails.
- tolerance = 0.75
-
- for slot, (first, second) in times.items():
- assert abs((second - first) - slots[slot].delay) < tolerance
+@inline_callbacks_test
+def test_delay(mockserver: MockServer):
+ crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
+ yield crawler.crawl(mockserver=mockserver)
+ assert isinstance(crawler.spider, DownloaderSlotsSettingsTestSpider)
+ slots = crawler.engine.downloader.slots
+ times = crawler.spider.times
+ # Downloading and processing a response add a roughly constant amount
+ # of time on top of the configured delay, so the margin is absolute.
+ # It stays well below the 1 second that separates the delays being
+ # compared, so a slot using the delay of another one still fails.
+ tolerance = 0.75
+
+ for slot, (first, second) in times.items():
+ assert abs((second - first) - slots[slot].delay) < tolerance
@coroutine_test
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index fd7e3958d..ff59599fc 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -76,190 +76,187 @@ def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog:
return build_from_crawler(CustomPeriodicLog, crawler)
-class TestPeriodicLog:
- def test_extension_enabled(self):
- # Expected that settings for this extension loaded successfully
-
- # "PERIODIC_LOG_STATS": True -> set to {"enabled": True}
- # due to TypeError exception from settings.getdict
- assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60})
-
- # "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True}
- # due to JSONDecodeError(ValueError) exception from settings.getdict
- assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60})
-
- # The ame for PERIODIC_LOG_DELTA:
- assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})
- assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})
-
- def test_no_interval(self):
- with pytest.raises(NotConfigured):
- extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 0})
-
- def test_nothing_enabled(self):
- with pytest.raises(NotConfigured):
- extension({"LOGSTATS_INTERVAL": 60})
-
- @coroutine_test
- async def test_log_delta(self):
- def emulate(
- settings: dict[str, Any] | None = None,
- ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]:
- spider = MetaSpider()
- ext = extension(settings)
- ext.spider_opened(spider)
- ext.set_a()
- a = ext.log_delta()
- ext.set_b()
- b = ext.log_delta()
- ext.spider_closed(spider, reason="finished")
- return ext, a, b
-
- def check(
- settings: dict[str, Any], condition: Callable[[str, Any], bool]
- ) -> None:
- ext, a, b = emulate(settings)
- assert list(a["delta"].keys()) == [
- k for k, v in ext.stats._stats.items() if condition(k, v)
- ]
- assert list(b["delta"].keys()) == [
- k for k, v in ext.stats._stats.items() if condition(k, v)
- ]
-
- # Including all
- check({"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float)))
-
- # include:
- check(
- {"PERIODIC_LOG_DELTA": {"include": ["downloader/"]}},
- lambda k, v: isinstance(v, (int, float)) and "downloader/" in k,
- )
-
- # include multiple
- check(
- {"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}},
- lambda k, v: (
- isinstance(v, (int, float))
- and ("downloader/" in k or "scheduler/" in k)
- ),
- )
-
- # exclude
- check(
- {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}},
- lambda k, v: isinstance(v, (int, float)) and "downloader/" not in k,
- )
-
- # exclude multiple
- check(
- {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}},
- lambda k, v: (
- isinstance(v, (int, float))
- and ("downloader/" not in k and "scheduler/" not in k)
- ),
- )
-
- # include exclude combined
- check(
- {"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}},
- lambda k, v: (
- isinstance(v, (int, float))
- and ("downloader/" in k and "bytes" not in k)
- ),
- )
-
- @coroutine_test
- async def test_log_stats(self):
- def emulate(
- settings: dict[str, Any] | None = None,
- ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]:
- spider = MetaSpider()
- ext = extension(settings)
- ext.spider_opened(spider)
- ext.set_a()
- a = ext.log_crawler_stats()
- ext.set_b()
- b = ext.log_crawler_stats()
- ext.spider_closed(spider, reason="finished")
- return ext, a, b
-
- def check(
- settings: dict[str, Any], condition: Callable[[str, Any], bool]
- ) -> None:
- ext, a, b = emulate(settings)
- assert list(a["stats"].keys()) == [
- k for k, v in ext.stats._stats.items() if condition(k, v)
- ]
- assert list(b["stats"].keys()) == [
- k for k, v in ext.stats._stats.items() if condition(k, v)
- ]
-
- # Including all
- check({"PERIODIC_LOG_STATS": True}, lambda k, v: True)
-
- # include:
- check(
- {"PERIODIC_LOG_STATS": {"include": ["downloader/"]}},
- lambda k, v: "downloader/" in k,
- )
-
- # include multiple
- check(
- {"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}},
- lambda k, v: "downloader/" in k or "scheduler/" in k,
- )
-
- # exclude
- check(
- {"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}},
- lambda k, v: "downloader/" not in k,
- )
-
- # exclude multiple
- check(
- {"PERIODIC_LOG_STATS": {"exclude": ["downloader/", "scheduler/"]}},
- lambda k, v: "downloader/" not in k and "scheduler/" not in k,
- )
-
- # include exclude combined
- check(
- {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}},
- lambda k, v: "downloader/" in k and "bytes" not in k,
- )
-
- @coroutine_test
- async def test_log_timing(self, caplog: pytest.LogCaptureFixture) -> None:
- settings = {
- "EXTENSIONS": {"scrapy.extensions.periodic_log.PeriodicLog": 0},
- "PERIODIC_LOG_TIMING_ENABLED": True,
- "LOGSTATS_INTERVAL": 30,
- }
- crawler = get_crawler(MetaSpider, settings)
- with caplog.at_level(logging.INFO, logger="scrapy.extensions.periodic_log"):
- await crawler.crawl_async()
-
- records = [
- r for r in caplog.records if r.name == "scrapy.extensions.periodic_log"
+def test_extension_enabled() -> None:
+ # Expected that settings for this extension loaded successfully
+
+ # "PERIODIC_LOG_STATS": True -> set to {"enabled": True}
+ # due to TypeError exception from settings.getdict
+ assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60})
+
+ # "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True}
+ # due to JSONDecodeError(ValueError) exception from settings.getdict
+ assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60})
+
+ # The ame for PERIODIC_LOG_DELTA:
+ assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})
+ assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})
+
+
+def test_no_interval() -> None:
+ with pytest.raises(NotConfigured):
+ extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 0})
+
+
+def test_nothing_enabled() -> None:
+ with pytest.raises(NotConfigured):
+ extension({"LOGSTATS_INTERVAL": 60})
+
+
+@coroutine_test
+async def test_log_delta() -> None:
+ def emulate(
+ settings: dict[str, Any] | None = None,
+ ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]:
+ spider = MetaSpider()
+ ext = extension(settings)
+ ext.spider_opened(spider)
+ ext.set_a()
+ a = ext.log_delta()
+ ext.set_b()
+ b = ext.log_delta()
+ ext.spider_closed(spider, reason="finished")
+ return ext, a, b
+
+ def check(settings: dict[str, Any], condition: Callable[[str, Any], bool]) -> None:
+ ext, a, b = emulate(settings)
+ assert list(a["delta"].keys()) == [
+ k for k, v in ext.stats._stats.items() if condition(k, v)
]
- assert records, "PeriodicLog logged nothing"
- # Only the timing section is enabled, and it is logged on spider close.
- data = json.loads(records[-1].getMessage())
- assert list(data) == ["time"]
- assert data["time"]["log_interval"] == 30
- assert data["time"]["log_interval_real"] >= 0
- assert data["time"]["elapsed"] >= 0
- assert data["time"]["start_time"] <= data["time"]["utcnow"]
-
- def test_multiplier_deprecated(self) -> None:
- crawler = get_crawler(
- MetaSpider,
- {"PERIODIC_LOG_TIMING_ENABLED": True, "LOGSTATS_INTERVAL": 30},
- )
- crawler._apply_settings()
- ext = build_from_crawler(PeriodicLog, crawler)
- with pytest.warns(ScrapyDeprecationWarning):
- assert ext.multiplier == 2.0
- with pytest.warns(ScrapyDeprecationWarning):
- ext.multiplier = 3.0
- with pytest.warns(ScrapyDeprecationWarning):
- assert ext.multiplier == 3.0
+ assert list(b["delta"].keys()) == [
+ k for k, v in ext.stats._stats.items() if condition(k, v)
+ ]
+
+ # Including all
+ check({"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float)))
+
+ # include:
+ check(
+ {"PERIODIC_LOG_DELTA": {"include": ["downloader/"]}},
+ lambda k, v: isinstance(v, (int, float)) and "downloader/" in k,
+ )
+
+ # include multiple
+ check(
+ {"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}},
+ lambda k, v: (
+ isinstance(v, (int, float)) and ("downloader/" in k or "scheduler/" in k)
+ ),
+ )
+
+ # exclude
+ check(
+ {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}},
+ lambda k, v: isinstance(v, (int, float)) and "downloader/" not in k,
+ )
+
+ # exclude multiple
+ check(
+ {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}},
+ lambda k, v: (
+ isinstance(v, (int, float))
+ and ("downloader/" not in k and "scheduler/" not in k)
+ ),
+ )
+
+ # include exclude combined
+ check(
+ {"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}},
+ lambda k, v: (
+ isinstance(v, (int, float)) and ("downloader/" in k and "bytes" not in k)
+ ),
+ )
+
+
+@coroutine_test
+async def test_log_stats() -> None:
+ def emulate(
+ settings: dict[str, Any] | None = None,
+ ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]:
+ spider = MetaSpider()
+ ext = extension(settings)
+ ext.spider_opened(spider)
+ ext.set_a()
+ a = ext.log_crawler_stats()
+ ext.set_b()
+ b = ext.log_crawler_stats()
+ ext.spider_closed(spider, reason="finished")
+ return ext, a, b
+
+ def check(settings: dict[str, Any], condition: Callable[[str, Any], bool]) -> None:
+ ext, a, b = emulate(settings)
+ assert list(a["stats"].keys()) == [
+ k for k, v in ext.stats._stats.items() if condition(k, v)
+ ]
+ assert list(b["stats"].keys()) == [
+ k for k, v in ext.stats._stats.items() if condition(k, v)
+ ]
+
+ # Including all
+ check({"PERIODIC_LOG_STATS": True}, lambda k, v: True)
+
+ # include:
+ check(
+ {"PERIODIC_LOG_STATS": {"include": ["downloader/"]}},
+ lambda k, v: "downloader/" in k,
+ )
+
+ # include multiple
+ check(
+ {"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}},
+ lambda k, v: "downloader/" in k or "scheduler/" in k,
+ )
+
+ # exclude
+ check(
+ {"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}},
+ lambda k, v: "downloader/" not in k,
+ )
+
+ # exclude multiple
+ check(
+ {"PERIODIC_LOG_STATS": {"exclude": ["downloader/", "scheduler/"]}},
+ lambda k, v: "downloader/" not in k and "scheduler/" not in k,
+ )
+
+ # include exclude combined
+ check(
+ {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}},
+ lambda k, v: "downloader/" in k and "bytes" not in k,
+ )
+
+
+@coroutine_test
+async def test_log_timing(caplog: pytest.LogCaptureFixture) -> None:
+ settings = {
+ "EXTENSIONS": {"scrapy.extensions.periodic_log.PeriodicLog": 0},
+ "PERIODIC_LOG_TIMING_ENABLED": True,
+ "LOGSTATS_INTERVAL": 30,
+ }
+ crawler = get_crawler(MetaSpider, settings)
+ with caplog.at_level(logging.INFO, logger="scrapy.extensions.periodic_log"):
+ await crawler.crawl_async()
+
+ records = [r for r in caplog.records if r.name == "scrapy.extensions.periodic_log"]
+ assert records, "PeriodicLog logged nothing"
+ # Only the timing section is enabled, and it is logged on spider close.
+ data = json.loads(records[-1].getMessage())
+ assert list(data) == ["time"]
+ assert data["time"]["log_interval"] == 30
+ assert data["time"]["log_interval_real"] >= 0
+ assert data["time"]["elapsed"] >= 0
+ assert data["time"]["start_time"] <= data["time"]["utcnow"]
+
+
+def test_multiplier_deprecated() -> None:
+ crawler = get_crawler(
+ MetaSpider,
+ {"PERIODIC_LOG_TIMING_ENABLED": True, "LOGSTATS_INTERVAL": 30},
+ )
+ crawler._apply_settings()
+ ext = build_from_crawler(PeriodicLog, crawler)
+ with pytest.warns(ScrapyDeprecationWarning):
+ assert ext.multiplier == 2.0
+ with pytest.warns(ScrapyDeprecationWarning):
+ ext.multiplier = 3.0
+ with pytest.warns(ScrapyDeprecationWarning):
+ assert ext.multiplier == 3.0
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index e7ed17615..561546711 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -5,158 +5,174 @@ import pytest
from scrapy.http import Headers
-class TestHeaders:
- def test_basics(self):
- h = Headers({"Content-Type": "text/html", "Content-Length": 1234})
- assert h["Content-Type"]
- assert h["Content-Length"]
-
- with pytest.raises(KeyError):
- h["Accept"]
- assert h.get("Accept") is None
- assert h.getlist("Accept") == []
-
- assert h.get("Accept", "*/*") == b"*/*"
- assert h.getlist("Accept", "*/*") == [b"*/*"]
- assert h.getlist("Accept", ["text/html", "images/jpeg"]) == [
- b"text/html",
- b"images/jpeg",
- ]
-
- def test_single_value(self):
- h = Headers()
- h["Content-Type"] = "text/html"
- assert h["Content-Type"] == b"text/html"
- assert h.get("Content-Type") == b"text/html"
- assert h.getlist("Content-Type") == [b"text/html"]
-
- def test_multivalue(self):
- h = Headers()
- h["X-Forwarded-For"] = hlist = ["ip1", "ip2"]
- assert h["X-Forwarded-For"] == b"ip2"
- assert h.get("X-Forwarded-For") == b"ip2"
- assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]
- assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]
-
- def test_multivalue_for_one_header(self):
- h = Headers((("a", "b"), ("a", "c")))
- assert h["a"] == b"c"
- assert h.get("a") == b"c"
- assert h.getlist("a") == [b"b", b"c"]
-
- def test_encode_utf8(self):
- h = Headers({"key": "\xa3"}, encoding="utf-8")
- key, val = dict(h.items()).popitem()
- assert isinstance(key, bytes), key
- assert isinstance(val[0], bytes), val[0]
- assert val[0] == b"\xc2\xa3"
-
- def test_encode_latin1(self):
- h = Headers({"key": "\xa3"}, encoding="latin1")
- _, val = dict(h.items()).popitem()
- assert val[0] == b"\xa3"
-
- def test_encode_multiple(self):
- h = Headers({"key": ["\xa3"]}, encoding="utf-8")
- _, val = dict(h.items()).popitem()
- assert val[0] == b"\xc2\xa3"
-
- def test_delete_and_contains(self):
- h = Headers()
- h["Content-Type"] = "text/html"
- assert "Content-Type" in h
- del h["Content-Type"]
- assert "Content-Type" not in h
-
- def test_setdefault(self):
- h = Headers()
- hlist = ["ip1", "ip2"]
- olist = h.setdefault("X-Forwarded-For", hlist)
- assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]
- assert h.getlist("X-Forwarded-For") is olist
-
- h = Headers()
- olist = h.setdefault("X-Forwarded-For", "ip1")
- assert h.getlist("X-Forwarded-For") == [b"ip1"]
- assert h.getlist("X-Forwarded-For") is olist
-
- def test_iterables(self):
- idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}
-
- h = Headers(idict)
- assert dict(h.items()) == {
- b"Content-Type": [b"text/html"],
- b"X-Forwarded-For": [b"ip1", b"ip2"],
- }
- assert sorted(h.keys()) == [b"Content-Type", b"X-Forwarded-For"]
- assert sorted(h.items()) == [
- (b"Content-Type", [b"text/html"]),
- (b"X-Forwarded-For", [b"ip1", b"ip2"]),
- ]
- assert set(h.values()) == {b"ip2", b"text/html"}
-
- def test_update(self):
- h = Headers()
- h.update({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]})
- assert h.getlist("Content-Type") == [b"text/html"]
- assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]
-
- def test_copy(self):
- h1 = Headers({"header1": ["value1", "value2"]})
- h2 = copy.copy(h1)
- assert h1 == h2
- assert h1.getlist("header1") == h2.getlist("header1")
- assert h1.getlist("header1") is not h2.getlist("header1")
- assert isinstance(h2, Headers)
-
- def test_appendlist(self):
- h1 = Headers({"header1": "value1"})
- h1.appendlist("header1", "value3")
- assert h1.getlist("header1") == [b"value1", b"value3"]
-
- h1 = Headers()
- h1.appendlist("header1", "value1")
- h1.appendlist("header1", "value3")
- assert h1.getlist("header1") == [b"value1", b"value3"]
-
- def test_setlist(self):
- h1 = Headers({"header1": "value1"})
- assert h1.getlist("header1") == [b"value1"]
- h1.setlist("header1", [b"value2", b"value3"])
- assert h1.getlist("header1") == [b"value2", b"value3"]
-
- def test_setlistdefault(self):
- h1 = Headers({"header1": "value1"})
- h1.setlistdefault("header1", ["value2", "value3"])
- h1.setlistdefault("header2", ["value2", "value3"])
- assert h1.getlist("header1") == [b"value1"]
- assert h1.getlist("header2") == [b"value2", b"value3"]
-
- def test_none_value(self):
- h1 = Headers()
- h1["foo"] = "bar"
- h1["foo"] = None
- h1.setdefault("foo", "bar")
- assert h1["foo"] is None
- assert h1.get("foo") is None
- assert h1.getlist("foo") == []
-
- def test_int_value(self):
- h1 = Headers({"hey": 5})
- h1["foo"] = 1
- h1.setdefault("bar", 2)
- h1.setlist("buz", [1, "dos", 3])
- assert h1.getlist("foo") == [b"1"]
- assert h1.getlist("bar") == [b"2"]
- assert h1.getlist("buz") == [b"1", b"dos", b"3"]
- assert h1.getlist("hey") == [b"5"]
-
- def test_invalid_value(self):
- with pytest.raises(TypeError, match="Unsupported value type"):
- Headers({"foo": object()})
- with pytest.raises(TypeError, match="Unsupported value type"):
- Headers()["foo"] = object()
- with pytest.raises(TypeError, match="Unsupported value type"):
- Headers().setdefault("foo", object())
- with pytest.raises(TypeError, match="Unsupported value type"):
- Headers().setlist("foo", [object()]) # type: ignore[list-item]
+def test_basics() -> None:
+ h = Headers({"Content-Type": "text/html", "Content-Length": 1234})
+ assert h["Content-Type"]
+ assert h["Content-Length"]
+
+ with pytest.raises(KeyError):
+ h["Accept"]
+ assert h.get("Accept") is None
+ assert h.getlist("Accept") == []
+
+ assert h.get("Accept", "*/*") == b"*/*"
+ assert h.getlist("Accept", "*/*") == [b"*/*"]
+ assert h.getlist("Accept", ["text/html", "images/jpeg"]) == [
+ b"text/html",
+ b"images/jpeg",
+ ]
+
+
+def test_single_value() -> None:
+ h = Headers()
+ h["Content-Type"] = "text/html"
+ assert h["Content-Type"] == b"text/html"
+ assert h.get("Content-Type") == b"text/html"
+ assert h.getlist("Content-Type") == [b"text/html"]
+
+
+def test_multivalue() -> None:
+ h = Headers()
+ h["X-Forwarded-For"] = hlist = ["ip1", "ip2"]
+ assert h["X-Forwarded-For"] == b"ip2"
+ assert h.get("X-Forwarded-For") == b"ip2"
+ assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]
+ assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]
+
+
+def test_multivalue_for_one_header() -> None:
+ h = Headers((("a", "b"), ("a", "c")))
+ assert h["a"] == b"c"
+ assert h.get("a") == b"c"
+ assert h.getlist("a") == [b"b", b"c"]
+
+
+def test_encode_utf8() -> None:
+ h = Headers({"key": "\xa3"}, encoding="utf-8")
+ key, val = dict(h.items()).popitem()
+ assert isinstance(key, bytes), key
+ assert isinstance(val[0], bytes), val[0]
+ assert val[0] == b"\xc2\xa3"
+
+
+def test_encode_latin1() -> None:
+ h = Headers({"key": "\xa3"}, encoding="latin1")
+ _, val = dict(h.items()).popitem()
+ assert val[0] == b"\xa3"
+
+
+def test_encode_multiple() -> None:
+ h = Headers({"key": ["\xa3"]}, encoding="utf-8")
+ _, val = dict(h.items()).popitem()
+ assert val[0] == b"\xc2\xa3"
+
+
+def test_delete_and_contains() -> None:
+ h = Headers()
+ h["Content-Type"] = "text/html"
+ assert "Content-Type" in h
+ del h["Content-Type"]
+ assert "Content-Type" not in h
+
+
+def test_setdefault() -> None:
+ h = Headers()
+ hlist = ["ip1", "ip2"]
+ olist = h.setdefault("X-Forwarded-For", hlist)
+ assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]
+ assert h.getlist("X-Forwarded-For") is olist
+
+ h = Headers()
+ olist = h.setdefault("X-Forwarded-For", "ip1")
+ assert h.getlist("X-Forwarded-For") == [b"ip1"]
+ assert h.getlist("X-Forwarded-For") is olist
+
+
+def test_iterables() -> None:
+ idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}
+
+ h = Headers(idict)
+ assert dict(h.items()) == {
+ b"Content-Type": [b"text/html"],
+ b"X-Forwarded-For": [b"ip1", b"ip2"],
+ }
+ assert sorted(h.keys()) == [b"Content-Type", b"X-Forwarded-For"]
+ assert sorted(h.items()) == [
+ (b"Content-Type", [b"text/html"]),
+ (b"X-Forwarded-For", [b"ip1", b"ip2"]),
+ ]
+ assert set(h.values()) == {b"ip2", b"text/html"}
+
+
+def test_update() -> None:
+ h = Headers()
+ h.update({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]})
+ assert h.getlist("Content-Type") == [b"text/html"]
+ assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]
+
+
+def test_copy() -> None:
+ h1 = Headers({"header1": ["value1", "value2"]})
+ h2 = copy.copy(h1)
+ assert h1 == h2
+ assert h1.getlist("header1") == h2.getlist("header1")
+ assert h1.getlist("header1") is not h2.getlist("header1")
+ assert isinstance(h2, Headers)
+
+
+def test_appendlist() -> None:
+ h1 = Headers({"header1": "value1"})
+ h1.appendlist("header1", "value3")
+ assert h1.getlist("header1") == [b"value1", b"value3"]
+
+ h1 = Headers()
+ h1.appendlist("header1", "value1")
+ h1.appendlist("header1", "value3")
+ assert h1.getlist("header1") == [b"value1", b"value3"]
+
+
+def test_setlist() -> None:
+ h1 = Headers({"header1": "value1"})
+ assert h1.getlist("header1") == [b"value1"]
+ h1.setlist("header1", [b"value2", b"value3"])
+ assert h1.getlist("header1") == [b"value2", b"value3"]
+
+
+def test_setlistdefault() -> None:
+ h1 = Headers({"header1": "value1"})
+ h1.setlistdefault("header1", ["value2", "value3"])
+ h1.setlistdefault("header2", ["value2", "value3"])
+ assert h1.getlist("header1") == [b"value1"]
+ assert h1.getlist("header2") == [b"value2", b"value3"]
+
+
+def test_none_value() -> None:
+ h1 = Headers()
+ h1["foo"] = "bar"
+ h1["foo"] = None
+ h1.setdefault("foo", "bar")
+ assert h1["foo"] is None
+ assert h1.get("foo") is None
+ assert h1.getlist("foo") == []
+
+
+def test_int_value() -> None:
+ h1 = Headers({"hey": 5})
+ h1["foo"] = 1
+ h1.setdefault("bar", 2)
+ h1.setlist("buz", [1, "dos", 3])
+ assert h1.getlist("foo") == [b"1"]
+ assert h1.getlist("bar") == [b"2"]
+ assert h1.getlist("buz") == [b"1", b"dos", b"3"]
+ assert h1.getlist("hey") == [b"5"]
+
+
+def test_invalid_value() -> None:
+ with pytest.raises(TypeError, match="Unsupported value type"):
+ Headers({"foo": object()})
+ with pytest.raises(TypeError, match="Unsupported value type"):
+ Headers()["foo"] = object()
+ with pytest.raises(TypeError, match="Unsupported value type"):
+ Headers().setdefault("foo", object())
+ with pytest.raises(TypeError, match="Unsupported value type"):
+ Headers().setlist("foo", [object()]) # type: ignore[list-item]
diff --git a/tests/test_link.py b/tests/test_link.py
index 40c53edd3..56467eb28 100644
--- a/tests/test_link.py
+++ b/tests/test_link.py
@@ -5,59 +5,61 @@ import pytest
from scrapy.link import Link
-class TestLink:
- def _assert_same_links(self, link1: Link, link2: Link) -> None:
- assert link1 == link2
- assert hash(link1) == hash(link2)
-
- def _assert_different_links(self, link1: Link, link2: Link) -> None:
- assert link1 != link2
- assert hash(link1) != hash(link2)
-
- def test_eq_and_hash(self) -> None:
- l1 = Link("http://www.example.com")
- l2 = Link("http://www.example.com/other")
- l3 = Link("http://www.example.com")
-
- self._assert_same_links(l1, l1)
- self._assert_different_links(l1, l2)
- self._assert_same_links(l1, l3)
-
- l4 = Link("http://www.example.com", text="test")
- l5 = Link("http://www.example.com", text="test2")
- l6 = Link("http://www.example.com", text="test")
-
- self._assert_same_links(l4, l4)
- self._assert_different_links(l4, l5)
- self._assert_same_links(l4, l6)
-
- l7 = Link(
- "http://www.example.com", text="test", fragment="something", nofollow=False
- )
- l8 = Link(
- "http://www.example.com", text="test", fragment="something", nofollow=False
- )
- l9 = Link(
- "http://www.example.com", text="test", fragment="something", nofollow=True
- )
- l10 = Link(
- "http://www.example.com", text="test", fragment="other", nofollow=False
- )
- self._assert_same_links(l7, l8)
- self._assert_different_links(l7, l9)
- self._assert_different_links(l7, l10)
-
- def test_repr(self) -> None:
- l1 = Link(
- "http://www.example.com", text="test", fragment="something", nofollow=True
- )
- l2 = eval(repr(l1))
- self._assert_same_links(l1, l2)
-
- def test_bytes_url(self) -> None:
- with pytest.raises(TypeError):
- Link(b"http://www.example.com/\xc2\xa3") # type: ignore[arg-type]
-
- def test_eq_non_link(self) -> None:
- url = "http://example.com"
- assert Link(url) != url
+def _assert_same_links(link1: Link, link2: Link) -> None:
+ assert link1 == link2
+ assert hash(link1) == hash(link2)
+
+
+def _assert_different_links(link1: Link, link2: Link) -> None:
+ assert link1 != link2
+ assert hash(link1) != hash(link2)
+
+
+def test_eq_and_hash() -> None:
+ l1 = Link("http://www.example.com")
+ l2 = Link("http://www.example.com/other")
+ l3 = Link("http://www.example.com")
+
+ _assert_same_links(l1, l1)
+ _assert_different_links(l1, l2)
+ _assert_same_links(l1, l3)
+
+ l4 = Link("http://www.example.com", text="test")
+ l5 = Link("http://www.example.com", text="test2")
+ l6 = Link("http://www.example.com", text="test")
+
+ _assert_same_links(l4, l4)
+ _assert_different_links(l4, l5)
+ _assert_same_links(l4, l6)
+
+ l7 = Link(
+ "http://www.example.com", text="test", fragment="something", nofollow=False
+ )
+ l8 = Link(
+ "http://www.example.com", text="test", fragment="something", nofollow=False
+ )
+ l9 = Link(
+ "http://www.example.com", text="test", fragment="something", nofollow=True
+ )
+ l10 = Link("http://www.example.com", text="test", fragment="other", nofollow=False)
+ _assert_same_links(l7, l8)
+ _assert_different_links(l7, l9)
+ _assert_different_links(l7, l10)
+
+
+def test_repr() -> None:
+ l1 = Link(
+ "http://www.example.com", text="test", fragment="something", nofollow=True
+ )
+ l2 = eval(repr(l1))
+ _assert_same_links(l1, l2)
+
+
+def test_bytes_url() -> None:
+ with pytest.raises(TypeError):
+ Link(b"http://www.example.com/\xc2\xa3") # type: ignore[arg-type]
+
+
+def test_eq_non_link() -> None:
+ url = "http://example.com"
+ assert Link(url) != url
diff --git a/tests/test_logstats.py b/tests/test_logstats.py
index 09cafd218..dc3319aa8 100644
--- a/tests/test_logstats.py
+++ b/tests/test_logstats.py
@@ -1,6 +1,7 @@
from __future__ import annotations
from datetime import datetime
+from typing import TYPE_CHECKING
import pytest
@@ -10,69 +11,86 @@ from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
from tests.utils.decorators import coroutine_test
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
-class TestLogStats:
- def setup_method(self) -> None:
- self.crawler = get_crawler(SimpleSpider)
- self.spider = self.crawler._create_spider("spidey")
- self.stats = self.crawler.stats
-
- self.stats.set_value("response_received_count", 4802)
- self.stats.set_value("item_scraped_count", 3201)
-
- @coroutine_test
- async def test_stats_calculations(self) -> None:
- logstats = build_from_crawler(LogStats, self.crawler)
-
- with pytest.raises(AttributeError):
- logstats.pagesprev
- with pytest.raises(AttributeError):
- logstats.itemsprev
-
- logstats.spider_opened(self.spider)
- assert logstats.pagesprev == 4802
- assert logstats.itemsprev == 3201
-
- logstats.calculate_stats()
- assert logstats.items == 3201
- assert logstats.pages == 4802
- assert logstats.irate == 0.0
- assert logstats.prate == 0.0
- assert logstats.pagesprev == 4802
- assert logstats.itemsprev == 3201
-
- # Simulate what happens after a minute
- self.stats.set_value("response_received_count", 5187)
- self.stats.set_value("item_scraped_count", 3492)
- logstats.calculate_stats()
- assert logstats.items == 3492
- assert logstats.pages == 5187
- assert logstats.irate == 291.0
- assert logstats.prate == 385.0
- assert logstats.pagesprev == 5187
- assert logstats.itemsprev == 3492
-
- # Simulate when spider closes after running for 30 mins
- self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))
- self.stats.set_value("finish_time", datetime.fromtimestamp(1655101972))
- logstats.spider_closed(self.spider, "test reason")
- assert self.stats.get_value("responses_per_minute") == 172.9
- assert self.stats.get_value("items_per_minute") == 116.4
-
- def test_stats_calculations_no_time(self) -> None:
- """The stat values should be None since the start and finish time are
- not available.
- """
- logstats = build_from_crawler(LogStats, self.crawler)
- logstats.spider_closed(self.spider, "test reason")
- assert self.stats.get_value("responses_per_minute") is None
- assert self.stats.get_value("items_per_minute") is None
-
- def test_stats_calculation_no_elapsed_time(self) -> None:
- """The stat values should be None since the elapsed time is 0."""
- logstats = build_from_crawler(LogStats, self.crawler)
- self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))
- self.stats.set_value("finish_time", datetime.fromtimestamp(1655100172))
- logstats.spider_closed(self.spider, "test reason")
- assert self.stats.get_value("responses_per_minute") is None
- assert self.stats.get_value("items_per_minute") is None
+
+def crawler() -> Crawler:
+ crawler = get_crawler(SimpleSpider)
+ crawler.spider = crawler._create_spider("spidey")
+ return crawler
+
+
+def stats(crawler: Crawler) -> StatsCollector:
+ stats = crawler.stats
+ stats.set_value("response_received_count", 4802)
+ stats.set_value("item_scraped_count", 3201)
+ return stats
+
+
+@coroutine_test
+async def test_stats_calculations(crawler: Crawler, stats: StatsCollector) -> None:
+ logstats = build_from_crawler(LogStats, crawler)
+
+ with pytest.raises(AttributeError):
+ logstats.pagesprev
+ with pytest.raises(AttributeError):
+ logstats.itemsprev
+
+ assert crawler.spider
+ logstats.spider_opened(crawler.spider)
+ assert logstats.pagesprev == 4802
+ assert logstats.itemsprev == 3201
+
+ logstats.calculate_stats()
+ assert logstats.items == 3201
+ assert logstats.pages == 4802
+ assert logstats.irate == 0.0
+ assert logstats.prate == 0.0
+ assert logstats.pagesprev == 4802
+ assert logstats.itemsprev == 3201
+
+ # Simulate what happens after a minute
+ stats.set_value("response_received_count", 5187)
+ stats.set_value("item_scraped_count", 3492)
+ logstats.calculate_stats()
+ assert logstats.items == 3492
+ assert logstats.pages == 5187
+ assert logstats.irate == 291.0
+ assert logstats.prate == 385.0
+ assert logstats.pagesprev == 5187
+ assert logstats.itemsprev == 3492
+
+ # Simulate when spider closes after running for 30 mins
+ stats.set_value("start_time", datetime.fromtimestamp(1655100172))
+ stats.set_value("finish_time", datetime.fromtimestamp(1655101972))
+ logstats.spider_closed(crawler.spider, "test reason")
+ assert stats.get_value("responses_per_minute") == 172.9
+ assert stats.get_value("items_per_minute") == 116.4
+
+
+def test_stats_calculations_no_time(crawler: Crawler, stats: StatsCollector) -> None:
+ """The stat values should be None since the start and finish time are
+ not available.
+ """
+ logstats = build_from_crawler(LogStats, crawler)
+ assert crawler.spider
+ logstats.spider_closed(crawler.spider, "test reason")
+ assert stats.get_value("responses_per_minute") is None
+ assert stats.get_value("items_per_minute") is None
+
+
+def test_stats_calculation_no_elapsed_time(
+ crawler: Crawler, stats: StatsCollector
+) -> None:
+ """The stat values should be None since the elapsed time is 0."""
+ logstats = build_from_crawler(LogStats, crawler)
+ stats.set_value("start_time", datetime.fromtimestamp(1655100172))
+ stats.set_value("finish_time", datetime.fromtimestamp(1655100172))
+ assert crawler.spider
+ logstats.spider_closed(crawler.spider, "test reason")
+ assert stats.get_value("responses_per_minute") is None
+ assert stats.get_value("items_per_minute") is None
diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py
index c7596e45d..f85be8977 100644
--- a/tests/test_request_dict.py
+++ b/tests/test_request_dict.py
@@ -1,173 +1,192 @@
-from typing import Any
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
import pytest
-from twisted.python.failure import Failure
from scrapy import Request, Spider
from scrapy.http import JsonRequest, Response
from scrapy.utils.request import request_from_dict
+if TYPE_CHECKING:
+ from twisted.python.failure import Failure
+
class CustomRequest(Request):
pass
-class TestRequestSerialization:
- def setup_method(self) -> None:
- self.spider = MethodsSpider()
-
- def test_basic(self):
- r = Request("http://www.example.com")
- self._assert_serializes_ok(r)
-
- def test_all_attributes(self):
- r = Request(
- url="http://www.example.com",
- callback=self.spider.parse_item,
- errback=self.spider.handle_error,
- method="POST",
- body=b"some body",
- headers={"content-encoding": "text/html; charset=latin-1"},
- cookies={"currency": "руб"},
- encoding="latin-1",
- priority=20,
- meta={"a": "b"},
- cb_kwargs={"k": "v"},
- flags=["testFlag"],
- )
- self._assert_serializes_ok(r, spider=self.spider)
-
- def test_latin1_body(self):
- r = Request("http://www.example.com", body=b"\xa3")
- self._assert_serializes_ok(r)
-
- def test_utf8_body(self):
- r = Request("http://www.example.com", body=b"\xc2\xa3")
- self._assert_serializes_ok(r)
-
- def _assert_serializes_ok(
- self, request: Request, spider: Spider | None = None
- ) -> None:
- d = request.to_dict(spider=spider)
- request2 = request_from_dict(d, spider=spider)
- self._assert_same_request(request, request2)
-
- def _assert_same_request(self, r1: Request, r2: Request) -> None:
- assert r1.__class__ == r2.__class__
- assert r1.url == r2.url
- assert r1.callback == r2.callback
- assert r1.errback == r2.errback
- assert r1.method == r2.method
- assert r1.body == r2.body
- assert r1.headers == r2.headers
- assert r1.cookies == r2.cookies
- assert r1.meta == r2.meta
- assert r1.cb_kwargs == r2.cb_kwargs
- assert r1.encoding == r2.encoding
- assert r1._encoding == r2._encoding
- assert r1.priority == r2.priority
- assert r1.dont_filter == r2.dont_filter
- assert r1.flags == r2.flags
- if isinstance(r1, JsonRequest):
- assert isinstance(r2, JsonRequest)
- assert r1.dumps_kwargs == r2.dumps_kwargs
-
- def test_request_class(self):
- r1 = CustomRequest("http://www.example.com")
- self._assert_serializes_ok(r1, spider=self.spider)
- r2 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
- self._assert_serializes_ok(r2, spider=self.spider)
-
- def test_callback_serialization(self):
- r = Request(
- "http://www.example.com",
- callback=self.spider.parse_item,
- errback=self.spider.handle_error,
- )
- self._assert_serializes_ok(r, spider=self.spider)
-
- def test_reference_callback_serialization(self):
- r = Request(
- "http://www.example.com",
- callback=self.spider.parse_item_reference, # type: ignore[arg-type,misc]
- errback=self.spider.handle_error_reference, # type: ignore[arg-type,misc]
- )
- self._assert_serializes_ok(r, spider=self.spider)
- request_dict = r.to_dict(spider=self.spider)
- assert request_dict["callback"] == "parse_item_reference"
- assert request_dict["errback"] == "handle_error_reference"
-
- def test_private_reference_callback_serialization(self):
- r = Request(
- "http://www.example.com",
- callback=self.spider._MethodsSpider__parse_item_reference, # type: ignore[attr-defined]
- errback=self.spider._MethodsSpider__handle_error_reference, # type: ignore[attr-defined]
- )
- self._assert_serializes_ok(r, spider=self.spider)
- request_dict = r.to_dict(spider=self.spider)
- assert request_dict["callback"] == "_MethodsSpider__parse_item_reference"
- assert request_dict["errback"] == "_MethodsSpider__handle_error_reference"
-
- def test_private_callback_serialization(self):
- r = Request(
- "http://www.example.com",
- callback=self.spider._MethodsSpider__parse_item_private, # type: ignore[attr-defined]
- errback=self.spider.handle_error,
- )
- self._assert_serializes_ok(r, spider=self.spider)
-
- def test_mixin_private_callback_serialization(self):
- r = Request(
- "http://www.example.com",
- callback=self.spider._SpiderMixin__mixin_callback, # type: ignore[attr-defined]
- errback=self.spider.handle_error,
- )
- self._assert_serializes_ok(r, spider=self.spider)
-
- def test_delegated_callback_serialization(self):
- r = Request(
- "http://www.example.com",
- callback=self.spider.delegated_callback,
- errback=self.spider.handle_error,
- )
- self._assert_serializes_ok(r, spider=self.spider)
-
- def test_unserializable_callback1(self):
- r = Request("http://www.example.com", callback=lambda x: x) # type: ignore[misc]
- with pytest.raises(
- ValueError, match="is not an instance method in: <MethodsSpider"
- ):
- r.to_dict(spider=self.spider)
-
- def test_unserializable_callback2(self):
- r = Request("http://www.example.com", callback=self.spider.parse_item)
- with pytest.raises(ValueError, match="is not an instance method in: None"):
- r.to_dict(spider=None)
-
- def test_unserializable_callback3(self):
- """Parser method is removed or replaced dynamically."""
-
- class MySpider(Spider):
- name = "my_spider"
-
- def parse(self, response: Response) -> None:
- pass
-
- spider = MySpider()
- r = Request("http://www.example.com", callback=spider.parse)
- spider.parse = None # type: ignore[method-assign,assignment]
- with pytest.raises(ValueError, match="is not an instance method in: <MySpider"):
- r.to_dict(spider=spider)
-
- def test_callback_not_available(self):
- """Callback method is not available in the spider passed to from_dict"""
- spider = SpiderDelegation()
- r = Request("http://www.example.com", callback=spider.delegated_callback)
- d = r.to_dict(spider=spider) # type: ignore[arg-type]
- with pytest.raises(
- ValueError, match="Method 'delegated_callback' not found in: <Spider"
- ):
- request_from_dict(d, spider=Spider("foo"))
+def spider() -> MethodsSpider:
+ return MethodsSpider()
+
+
+def _assert_serializes_ok(request: Request, spider: Spider | None = None) -> None:
+ d = request.to_dict(spider=spider)
+ request2 = request_from_dict(d, spider=spider)
+ _assert_same_request(request, request2)
+
+
+def _assert_same_request(r1: Request, r2: Request) -> None:
+ assert r1.__class__ == r2.__class__
+ assert r1.url == r2.url
+ assert r1.callback == r2.callback
+ assert r1.errback == r2.errback
+ assert r1.method == r2.method
+ assert r1.body == r2.body
+ assert r1.headers == r2.headers
+ assert r1.cookies == r2.cookies
+ assert r1.meta == r2.meta
+ assert r1.cb_kwargs == r2.cb_kwargs
+ assert r1.encoding == r2.encoding
+ assert r1._encoding == r2._encoding
+ assert r1.priority == r2.priority
+ assert r1.dont_filter == r2.dont_filter
+ assert r1.flags == r2.flags
+ if isinstance(r1, JsonRequest):
+ assert isinstance(r2, JsonRequest)
+ assert r1.dumps_kwargs == r2.dumps_kwargs
+
+
+def test_basic() -> None:
+ r = Request("http://www.example.com")
+ _assert_serializes_ok(r)
+
+
+def test_all_attributes(spider: MethodsSpider) -> None:
+ r = Request(
+ url="http://www.example.com",
+ callback=spider.parse_item,
+ errback=spider.handle_error,
+ method="POST",
+ body=b"some body",
+ headers={"content-encoding": "text/html; charset=latin-1"},
+ cookies={"currency": "руб"},
+ encoding="latin-1",
+ priority=20,
+ meta={"a": "b"},
+ cb_kwargs={"k": "v"},
+ flags=["testFlag"],
+ )
+ _assert_serializes_ok(r, spider=spider)
+
+
+def test_latin1_body() -> None:
+ r = Request("http://www.example.com", body=b"\xa3")
+ _assert_serializes_ok(r)
+
+
+def test_utf8_body() -> None:
+ r = Request("http://www.example.com", body=b"\xc2\xa3")
+ _assert_serializes_ok(r)
+
+
+def test_request_class(spider: MethodsSpider) -> None:
+ r1 = CustomRequest("http://www.example.com")
+ _assert_serializes_ok(r1, spider=spider)
+ r2 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
+ _assert_serializes_ok(r2, spider=spider)
+
+
+def test_callback_serialization(spider: MethodsSpider) -> None:
+ r = Request(
+ "http://www.example.com",
+ callback=spider.parse_item,
+ errback=spider.handle_error,
+ )
+ _assert_serializes_ok(r, spider=spider)
+
+
+def test_reference_callback_serialization(spider: MethodsSpider) -> None:
+ r = Request(
+ "http://www.example.com",
+ callback=spider.parse_item_reference, # type: ignore[arg-type,misc]
+ errback=spider.handle_error_reference, # type: ignore[arg-type,misc]
+ )
+ _assert_serializes_ok(r, spider=spider)
+ request_dict = r.to_dict(spider=spider)
+ assert request_dict["callback"] == "parse_item_reference"
+ assert request_dict["errback"] == "handle_error_reference"
+
+
+def test_private_reference_callback_serialization(spider: MethodsSpider) -> None:
+ r = Request(
+ "http://www.example.com",
+ callback=spider._MethodsSpider__parse_item_reference, # type: ignore[attr-defined]
+ errback=spider._MethodsSpider__handle_error_reference, # type: ignore[attr-defined]
+ )
+ _assert_serializes_ok(r, spider=spider)
+ request_dict = r.to_dict(spider=spider)
+ assert request_dict["callback"] == "_MethodsSpider__parse_item_reference"
+ assert request_dict["errback"] == "_MethodsSpider__handle_error_reference"
+
+
+def test_private_callback_serialization(spider: MethodsSpider) -> None:
+ r = Request(
+ "http://www.example.com",
+ callback=spider._MethodsSpider__parse_item_private, # type: ignore[attr-defined]
+ errback=spider.handle_error,
+ )
+ _assert_serializes_ok(r, spider=spider)
+
+
+def test_mixin_private_callback_serialization(spider: MethodsSpider) -> None:
+ r = Request(
+ "http://www.example.com",
+ callback=spider._SpiderMixin__mixin_callback, # type: ignore[attr-defined]
+ errback=spider.handle_error,
+ )
+ _assert_serializes_ok(r, spider=spider)
+
+
+def test_delegated_callback_serialization(spider: MethodsSpider) -> None:
+ r = Request(
+ "http://www.example.com",
+ callback=spider.delegated_callback,
+ errback=spider.handle_error,
+ )
+ _assert_serializes_ok(r, spider=spider)
+
+
+def test_unserializable_callback1(spider: MethodsSpider) -> None:
+ r = Request("http://www.example.com", callback=lambda x: x) # type: ignore[misc]
+ with pytest.raises(
+ ValueError, match="is not an instance method in: <MethodsSpider"
+ ):
+ r.to_dict(spider=spider)
+
+
+def test_unserializable_callback2(spider: MethodsSpider) -> None:
+ r = Request("http://www.example.com", callback=spider.parse_item)
+ with pytest.raises(ValueError, match="is not an instance method in: None"):
+ r.to_dict(spider=None)
+
+
+def test_unserializable_callback3() -> None:
+ """Parser method is removed or replaced dynamically."""
+
+ class MySpider(Spider):
+ name = "my_spider"
+
+ def parse(self, response: Response) -> None:
+ pass
+
+ spider = MySpider()
+ r = Request("http://www.example.com", callback=spider.parse)
+ spider.parse = None # type: ignore[method-assign,assignment]
+ with pytest.raises(ValueError, match="is not an instance method in: <MySpider"):
+ r.to_dict(spider=spider)
+
+
+def test_callback_not_available() -> None:
+ """Callback method is not available in the spider passed to from_dict"""
+ spider = SpiderDelegation()
+ r = Request("http://www.example.com", callback=spider.delegated_callback)
+ d = r.to_dict(spider=spider) # type: ignore[arg-type]
+ with pytest.raises(
+ ValueError, match="Method 'delegated_callback' not found in: <Spider"
+ ):
+ request_from_dict(d, spider=Spider("foo"))
class SpiderMixin:
diff --git a/tests/test_request_left.py b/tests/test_request_left.py
index 46a16ad1e..e95f1d7d9 100644
--- a/tests/test_request_left.py
+++ b/tests/test_request_left.py
@@ -32,31 +32,33 @@ class SignalCatcherSpider(Spider):
self.caught_times += 1
-class TestCatching:
- @inline_callbacks_test
- def test_success(self, mockserver: MockServer):
- crawler = get_crawler(SignalCatcherSpider)
- yield crawler.crawl(mockserver.url("/status?n=200"))
- assert isinstance(crawler.spider, SignalCatcherSpider)
- assert crawler.spider.caught_times == 1
-
- @inline_callbacks_test
- def test_timeout(self, mockserver: MockServer):
- crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})
- yield crawler.crawl(mockserver.url("/delay?n=0.2"))
- assert isinstance(crawler.spider, SignalCatcherSpider)
- assert crawler.spider.caught_times == 1
-
- @inline_callbacks_test
- def test_disconnect(self, mockserver: MockServer):
- crawler = get_crawler(SignalCatcherSpider)
- yield crawler.crawl(mockserver.url("/drop"))
- assert isinstance(crawler.spider, SignalCatcherSpider)
- assert crawler.spider.caught_times == 1
-
- @inline_callbacks_test
- def test_noconnect(self):
- crawler = get_crawler(SignalCatcherSpider)
- yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")
- assert isinstance(crawler.spider, SignalCatcherSpider)
- assert crawler.spider.caught_times == 1
+@inline_callbacks_test
+def test_success(mockserver: MockServer):
+ crawler = get_crawler(SignalCatcherSpider)
+ yield crawler.crawl(mockserver.url("/status?n=200"))
+ assert isinstance(crawler.spider, SignalCatcherSpider)
+ assert crawler.spider.caught_times == 1
+
+
+@inline_callbacks_test
+def test_timeout(mockserver: MockServer):
+ crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})
+ yield crawler.crawl(mockserver.url("/delay?n=0.2"))
+ assert isinstance(crawler.spider, SignalCatcherSpider)
+ assert crawler.spider.caught_times == 1
+
+
+@inline_callbacks_test
+def test_disconnect(mockserver: MockServer):
+ crawler = get_crawler(SignalCatcherSpider)
+ yield crawler.crawl(mockserver.url("/drop"))
+ assert isinstance(crawler.spider, SignalCatcherSpider)
+ assert crawler.spider.caught_times == 1
+
+
+@inline_callbacks_test
+def test_noconnect():
+ crawler = get_crawler(SignalCatcherSpider)
+ yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")
+ assert isinstance(crawler.spider, SignalCatcherSpider)
+ assert crawler.spider.caught_times == 1
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 7c9f717e4..fa37cfe03 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -11,119 +11,125 @@ from scrapy.http import (
from scrapy.responsetypes import responsetypes
-class TestResponseTypes:
- def test_from_filename(self):
- mappings = [
- ("data.bin", Response),
- ("file.txt", TextResponse),
- ("file.xml.gz", Response),
- ("file.xml", XmlResponse),
- ("file.html", HtmlResponse),
- ("file.unknownext", Response),
- ]
- for source, cls in mappings:
- retcls = responsetypes.from_filename(source)
- assert retcls is cls, f"{source} ==> {retcls} != {cls}"
-
- def test_from_content_disposition(self):
- mappings = [
- (b'attachment; filename="data.xml"', XmlResponse),
- (b"attachment; filename=data.xml", XmlResponse),
- ("attachment;filename=data£.tar.gz".encode(), Response),
- ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),
- ("attachment;filename=data高.doc".encode("gbk"), Response),
- ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),
- (
- "attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"),
- XmlResponse,
- ),
- ]
- for source, cls in mappings:
- retcls = responsetypes.from_content_disposition(source)
- assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
-
- def test_from_content_disposition_no_filename(self):
- assert responsetypes.from_content_disposition(b"attachment") is Response
-
- def test_from_content_type(self):
- mappings: list[tuple[str | bytes, type[Response]]] = [
- ("text/html; charset=UTF-8", HtmlResponse),
- ("text/xml; charset=UTF-8", XmlResponse),
- ("application/xhtml+xml; charset=UTF-8", HtmlResponse),
- ("application/vnd.wap.xhtml+xml; charset=utf-8", HtmlResponse),
- ("application/xml; charset=UTF-8", XmlResponse),
- ("application/octet-stream", Response),
- ("application/json; encoding=UTF8;charset=UTF-8", JsonResponse),
- ("application/x-json; encoding=UTF8;charset=UTF-8", JsonResponse),
- ("application/json-amazonui-streaming;charset=UTF-8", JsonResponse),
- (b"application/x-download; filename=\x80dummy.txt", Response),
- ]
- for source, cls in mappings:
- retcls = responsetypes.from_content_type(source)
- assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
-
- def test_from_body(self):
- mappings = [
- (b"\x03\x02\xdf\xdd\x23", Response),
- (b"Some plain text\ndata with tabs\t and null bytes\0", TextResponse),
- (b"<html><head><title>Hello</title></head>", HtmlResponse),
- # https://codersblock.com/blog/the-smallest-valid-html5-page/
- (b"<!DOCTYPE html>\n<title>.</title>", HtmlResponse),
- (b'<?xml version="1.0" encoding="utf-8"', XmlResponse),
- ]
- for source, cls in mappings:
- retcls = responsetypes.from_body(source)
- assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
-
- def test_from_headers(self):
- mappings = [
- ({"Content-Type": ["text/html; charset=utf-8"]}, HtmlResponse),
- (
- {
- "Content-Type": ["text/html; charset=utf-8"],
- "Content-Encoding": ["gzip"],
- },
- Response,
- ),
- (
- {
- "Content-Type": ["application/octet-stream"],
- "Content-Disposition": ["attachment; filename=data.txt"],
- },
- TextResponse,
- ),
- ]
- for source, cls in mappings:
- source = Headers(source)
- retcls = responsetypes.from_headers(source)
- assert retcls is cls, f"{source} ==> {retcls} != {cls}"
-
- def test_from_args(self):
- # TODO: add more tests that check precedence between the different arguments
- mappings: list[tuple[dict[str, Any], type[Response]]] = [
- ({"url": "http://www.example.com/data.csv"}, TextResponse),
- # headers takes precedence over url
- (
- {
- "headers": Headers({"Content-Type": ["text/html; charset=utf-8"]}),
- "url": "http://www.example.com/item/",
- },
- HtmlResponse,
- ),
- (
- {
- "headers": Headers(
- {"Content-Disposition": ['attachment; filename="data.xml.gz"']}
- ),
- "url": "http://www.example.com/page/",
- },
- Response,
- ),
- ]
- for source, cls in mappings:
- retcls = responsetypes.from_args(**source)
- assert retcls is cls, f"{source} ==> {retcls} != {cls}"
-
- def test_custom_mime_types_loaded(self):
- # check that mime.types files shipped with scrapy are loaded
- assert responsetypes.mimetypes.guess_type("x.scrapytest")[0] == "x-scrapy/test"
+def test_from_filename() -> None:
+ mappings = [
+ ("data.bin", Response),
+ ("file.txt", TextResponse),
+ ("file.xml.gz", Response),
+ ("file.xml", XmlResponse),
+ ("file.html", HtmlResponse),
+ ("file.unknownext", Response),
+ ]
+ for source, cls in mappings:
+ retcls = responsetypes.from_filename(source)
+ assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+
+
+def test_from_content_disposition() -> None:
+ mappings = [
+ (b'attachment; filename="data.xml"', XmlResponse),
+ (b"attachment; filename=data.xml", XmlResponse),
+ ("attachment;filename=data£.tar.gz".encode(), Response),
+ ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),
+ ("attachment;filename=data高.doc".encode("gbk"), Response),
+ ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),
+ (
+ "attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"),
+ XmlResponse,
+ ),
+ ]
+ for source, cls in mappings:
+ retcls = responsetypes.from_content_disposition(source)
+ assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
+
+
+def test_from_content_disposition_no_filename() -> None:
+ assert responsetypes.from_content_disposition(b"attachment") is Response
+
+
+def test_from_content_type() -> None:
+ mappings: list[tuple[str | bytes, type[Response]]] = [
+ ("text/html; charset=UTF-8", HtmlResponse),
+ ("text/xml; charset=UTF-8", XmlResponse),
+ ("application/xhtml+xml; charset=UTF-8", HtmlResponse),
+ ("application/vnd.wap.xhtml+xml; charset=utf-8", HtmlResponse),
+ ("application/xml; charset=UTF-8", XmlResponse),
+ ("application/octet-stream", Response),
+ ("application/json; encoding=UTF8;charset=UTF-8", JsonResponse),
+ ("application/x-json; encoding=UTF8;charset=UTF-8", JsonResponse),
+ ("application/json-amazonui-streaming;charset=UTF-8", JsonResponse),
+ (b"application/x-download; filename=\x80dummy.txt", Response),
+ ]
+ for source, cls in mappings:
+ retcls = responsetypes.from_content_type(source)
+ assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
+
+
+def test_from_body() -> None:
+ mappings = [
+ (b"\x03\x02\xdf\xdd\x23", Response),
+ (b"Some plain text\ndata with tabs\t and null bytes\0", TextResponse),
+ (b"<html><head><title>Hello</title></head>", HtmlResponse),
+ # https://codersblock.com/blog/the-smallest-valid-html5-page/
+ (b"<!DOCTYPE html>\n<title>.</title>", HtmlResponse),
+ (b'<?xml version="1.0" encoding="utf-8"', XmlResponse),
+ ]
+ for source, cls in mappings:
+ retcls = responsetypes.from_body(source)
+ assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
+
+
+def test_from_headers() -> None:
+ mappings = [
+ ({"Content-Type": ["text/html; charset=utf-8"]}, HtmlResponse),
+ (
+ {
+ "Content-Type": ["text/html; charset=utf-8"],
+ "Content-Encoding": ["gzip"],
+ },
+ Response,
+ ),
+ (
+ {
+ "Content-Type": ["application/octet-stream"],
+ "Content-Disposition": ["attachment; filename=data.txt"],
+ },
+ TextResponse,
+ ),
+ ]
+ for source, cls in mappings:
+ source = Headers(source)
+ retcls = responsetypes.from_headers(source)
+ assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+
+
+def test_from_args() -> None:
+ # TODO: add more tests that check precedence between the different arguments
+ mappings: list[tuple[dict[str, Any], type[Response]]] = [
+ ({"url": "http://www.example.com/data.csv"}, TextResponse),
+ # headers takes precedence over url
+ (
+ {
+ "headers": Headers({"Content-Type": ["text/html; charset=utf-8"]}),
+ "url": "http://www.example.com/item/",
+ },
+ HtmlResponse,
+ ),
+ (
+ {
+ "headers": Headers(
+ {"Content-Disposition": ['attachment; filename="data.xml.gz"']}
+ ),
+ "url": "http://www.example.com/page/",
+ },
+ Response,
+ ),
+ ]
+ for source, cls in mappings:
+ retcls = responsetypes.from_args(**source)
+ assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+
+
+def test_custom_mime_types_loaded() -> None:
+ # check that mime.types files shipped with scrapy are loaded
+ assert responsetypes.mimetypes.guess_type("x.scrapytest")[0] == "x-scrapy/test"
diff --git a/tests/test_signalmanager.py b/tests/test_signalmanager.py
index ce4d97adb..cf8cd489f 100644
--- a/tests/test_signalmanager.py
+++ b/tests/test_signalmanager.py
@@ -1,21 +1,20 @@
from scrapy.signalmanager import SignalManager
-class TestSignalManager:
- def test_disconnect_all(self):
- signal = object()
- sender = object()
- sm = SignalManager(sender)
+def test_disconnect_all() -> None:
+ signal = object()
+ sender = object()
+ sm = SignalManager(sender)
- calls = []
+ calls = []
- def handler():
- calls.append(1)
+ def handler():
+ calls.append(1)
- sm.connect(handler, signal)
- sm.send_catch_log(signal)
- assert calls == [1]
+ sm.connect(handler, signal)
+ sm.send_catch_log(signal)
+ assert calls == [1]
- sm.disconnect_all(signal)
- sm.send_catch_log(signal)
- assert calls == [1] # handler no longer called after disconnect_all
+ sm.disconnect_all(signal)
+ sm.send_catch_log(signal)
+ assert calls == [1] # handler no longer called after disconnect_all
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index fb72ece6c..482b42d68 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -21,79 +21,83 @@ ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"}
-class TestMain:
- async def _test_spider(
- self, spider: type[Spider], expected_items: list[Any] | None = None
- ) -> None:
- actual_items = []
- expected_items = [] if expected_items is None else expected_items
-
- def track_item(item, response, spider):
- actual_items.append(item)
-
- crawler = get_crawler(spider)
- crawler.signals.connect(track_item, signals.item_scraped)
- await crawler.crawl_async()
- assert crawler.stats.get_value("finish_reason") == "finished"
- assert actual_items == expected_items
-
- @coroutine_test
- async def test_start_urls(self):
- class TestSpider(Spider):
- name = "test"
- start_urls = ["data:,"]
-
- async def parse(self, response):
- yield ITEM_A
-
- await self._test_spider(TestSpider, [ITEM_A])
-
- @coroutine_test
- async def test_start(self):
- class TestSpider(Spider):
- name = "test"
-
- async def start(self):
- yield ITEM_A
-
- await self._test_spider(TestSpider, [ITEM_A])
-
- @coroutine_test
- async def test_start_subclass(self):
- class BaseSpider(Spider):
- async def start(self):
- yield ITEM_A
-
- class TestSpider(BaseSpider):
- name = "test"
-
- await self._test_spider(TestSpider, [ITEM_A])
-
- async def _test_start(
- self,
- start_: Callable[[Any], AsyncIterator[Any]],
- expected_items: list[Any] | None = None,
- ) -> None:
- class TestSpider(Spider):
- name = "test"
- start = start_
-
- await self._test_spider(TestSpider, expected_items)
-
- @pytest.mark.only_asyncio
- @coroutine_test
- async def test_asyncio_delayed(self):
- async def start(spider):
- await sleep(SLEEP_SECONDS)
+async def _test_spider(
+ spider: type[Spider], expected_items: list[Any] | None = None
+) -> None:
+ actual_items = []
+ expected_items = [] if expected_items is None else expected_items
+
+ def track_item(item, response, spider):
+ actual_items.append(item)
+
+ crawler = get_crawler(spider)
+ crawler.signals.connect(track_item, signals.item_scraped)
+ await crawler.crawl_async()
+ assert crawler.stats.get_value("finish_reason") == "finished"
+ assert actual_items == expected_items
+
+
+@coroutine_test
+async def test_start_urls() -> None:
+ class TestSpider(Spider):
+ name = "test"
+ start_urls = ["data:,"]
+
+ async def parse(self, response):
yield ITEM_A
- await self._test_start(start, [ITEM_A])
+ await _test_spider(TestSpider, [ITEM_A])
+
+
+@coroutine_test
+async def test_start() -> None:
+ class TestSpider(Spider):
+ name = "test"
+
+ async def start(self):
+ yield ITEM_A
+
+ await _test_spider(TestSpider, [ITEM_A])
+
- @pytest.mark.requires_reactor # needs a reactor for twisted_sleep()
- @coroutine_test
- async def test_twisted_delayed(self):
- async def start(spider):
- await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS))
+@coroutine_test
+async def test_start_subclass() -> None:
+ class BaseSpider(Spider):
+ async def start(self):
yield ITEM_A
- await self._test_start(start, [ITEM_A])
+ class TestSpider(BaseSpider):
+ name = "test"
+
+ await _test_spider(TestSpider, [ITEM_A])
+
+
+async def _test_start(
+ start_: Callable[[Any], AsyncIterator[Any]],
+ expected_items: list[Any] | None = None,
+) -> None:
+ class TestSpider(Spider):
+ name = "test"
+ start = start_
+
+ await _test_spider(TestSpider, expected_items)
+
+
+@coroutine_test
+async def test_asyncio_delayed() -> None:
+ async def start(spider):
+ await sleep(SLEEP_SECONDS)
+ yield ITEM_A
+
+ await _test_start(start, [ITEM_A])
+
+
[email protected]_reactor # needs a reactor for twisted_sleep()
+@coroutine_test
+async def test_twisted_delayed() -> None:
+ async def start(spider):
+ await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS))
+ yield ITEM_A
+
+ await _test_start(start, [ITEM_A])
diff --git a/tests/test_spidermiddleware_metacopy.py b/tests/test_spidermiddleware_metacopy.py
index 8fe6e983e..dac0d6b70 100644
--- a/tests/test_spidermiddleware_metacopy.py
+++ b/tests/test_spidermiddleware_metacopy.py
@@ -41,82 +41,88 @@ def process(
return list(mw.process_spider_output(response, requests))
-class TestInternalKeysCheck:
- def test_no_warning_for_clean_request(
- self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
- ) -> None:
- req = Request("https://example.com/1", meta={"my_key": "value"})
- with caplog.at_level(WARNING):
- process(mw, [req])
- assert not caplog.records
-
- def test_warns_on_internal_key(
- self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
- ) -> None:
+def test_no_warning_for_clean_request(
+ mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+) -> None:
+ req = Request("https://example.com/1", meta={"my_key": "value"})
+ with caplog.at_level(WARNING):
+ process(mw, [req])
+ assert not caplog.records
+
+
+def test_warns_on_internal_key(
+ mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+) -> None:
+ req = Request("https://example.com/1", meta={"retry_times": 1})
+ with caplog.at_level(WARNING):
+ process(mw, [req])
+ assert len(caplog.records) == 1
+ assert "retry_times" in caplog.text
+ assert "https://example.com/1" in caplog.text
+
+
+def test_warns_once_across_multiple_requests(
+ mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+) -> None:
+ reqs = [
+ Request("https://example.com/1", meta={"retry_times": 1}),
+ Request("https://example.com/2", meta={"retry_times": 2}),
+ ]
+ with caplog.at_level(WARNING):
+ process(mw, reqs)
+ assert len(caplog.records) == 1
+
+
+def test_reports_all_found_keys(
+ mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+) -> None:
+ req = Request(
+ "https://example.com/1",
+ meta={"retry_times": 1, "redirect_times": 2},
+ )
+ with caplog.at_level(WARNING):
+ process(mw, [req])
+ assert "retry_times" in caplog.text
+ assert "redirect_times" in caplog.text
+
+
+def test_includes_source_response_in_message(
+ mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+) -> None:
+ response = make_response("https://source.example.com")
+ req = Request("https://example.com/1", meta={"retry_times": 1})
+ with caplog.at_level(WARNING):
+ process(mw, [req], response=response)
+ assert "https://source.example.com" in caplog.text
+
+
+def test_skip_keys_setting(caplog: pytest.LogCaptureFixture) -> None:
+ with caplog.at_level(WARNING):
+ crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
+ mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)
req = Request("https://example.com/1", meta={"retry_times": 1})
- with caplog.at_level(WARNING):
- process(mw, [req])
- assert len(caplog.records) == 1
- assert "retry_times" in caplog.text
- assert "https://example.com/1" in caplog.text
-
- def test_warns_once_across_multiple_requests(
- self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
- ) -> None:
- reqs = [
- Request("https://example.com/1", meta={"retry_times": 1}),
- Request("https://example.com/2", meta={"retry_times": 2}),
- ]
- with caplog.at_level(WARNING):
- process(mw, reqs)
- assert len(caplog.records) == 1
-
- def test_reports_all_found_keys(
- self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
- ) -> None:
+ process(mw, [req])
+ assert not caplog.records
+
+
+def test_skip_keys_setting_partial(caplog: pytest.LogCaptureFixture) -> None:
+ with caplog.at_level(WARNING):
+ crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
+ mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)
req = Request(
"https://example.com/1",
meta={"retry_times": 1, "redirect_times": 2},
)
- with caplog.at_level(WARNING):
- process(mw, [req])
- assert "retry_times" in caplog.text
- assert "redirect_times" in caplog.text
-
- def test_includes_source_response_in_message(
- self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
- ) -> None:
- response = make_response("https://source.example.com")
- req = Request("https://example.com/1", meta={"retry_times": 1})
- with caplog.at_level(WARNING):
- process(mw, [req], response=response)
- assert "https://source.example.com" in caplog.text
-
- def test_skip_keys_setting(self, caplog: pytest.LogCaptureFixture) -> None:
- with caplog.at_level(WARNING):
- crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
- mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)
- req = Request("https://example.com/1", meta={"retry_times": 1})
- process(mw, [req])
- assert not caplog.records
-
- def test_skip_keys_setting_partial(self, caplog: pytest.LogCaptureFixture) -> None:
- with caplog.at_level(WARNING):
- crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
- mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)
- req = Request(
- "https://example.com/1",
- meta={"retry_times": 1, "redirect_times": 2},
- )
- process(mw, [req])
- assert len(caplog.records) == 1
- assert "retry_times" not in caplog.text
- assert "redirect_times" in caplog.text
-
- def test_no_warning_for_start_requests(
- self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
- ) -> None:
- req = Request("https://example.com/1", meta={"retry_times": 1})
- with caplog.at_level(WARNING):
- list(mw.process_spider_output(None, [req]))
- assert not caplog.records
+ process(mw, [req])
+ assert len(caplog.records) == 1
+ assert "retry_times" not in caplog.text
+ assert "redirect_times" in caplog.text
+
+
+def test_no_warning_for_start_requests(
+ mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+) -> None:
+ req = Request("https://example.com/1", meta={"retry_times": 1})
+ with caplog.at_level(WARNING):
+ list(mw.process_spider_output(None, [req]))
+ assert not caplog.records
diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py
index 46414bb59..bea58aaaf 100644
--- a/tests/test_spidermiddleware_process_start.py
+++ b/tests/test_spidermiddleware_process_start.py
@@ -62,69 +62,73 @@ class ModernWrapSpiderMiddleware:
yield ITEM_C
-class TestMain:
- async def _test(
- self,
- spider_middlewares: list[type],
- spider_cls: type[Spider],
- expected_items: list[Any],
- ) -> None:
- actual_items = []
-
- def track_item(item: Any, response: Response, spider: Spider) -> None:
- actual_items.append(item)
-
- settings = {
- "SPIDER_MIDDLEWARES": {cls: n for n, cls in enumerate(spider_middlewares)},
- }
- crawler = get_crawler(spider_cls, settings_dict=settings)
- crawler.signals.connect(track_item, signals.item_scraped)
- await crawler.crawl_async()
- assert crawler.stats.get_value("finish_reason") == "finished"
- assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"
-
- async def _test_wrap(
- self,
- spider_middleware: type,
- spider_cls: type[Spider],
- expected_items: list[Any] | None = None,
- ) -> None:
- expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]
- await self._test([spider_middleware], spider_cls, expected_items)
-
- @coroutine_test
- async def test_modern_mw_modern_spider(self):
- await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
-
- async def _test_sleep(self, spider_middlewares: list[type]) -> None:
- class TestSpider(Spider):
- name = "test"
-
- async def start(self) -> AsyncIterator[Any]:
- yield ITEM_A
-
- await self._test(spider_middlewares, TestSpider, [ITEM_A])
-
- @pytest.mark.only_asyncio
- @coroutine_test
- async def test_asyncio_sleep_single(self):
- await self._test_sleep([AsyncioSleepSpiderMiddleware])
-
- @pytest.mark.only_asyncio
- @coroutine_test
- async def test_asyncio_sleep_multiple(self):
- await self._test_sleep(
- [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware]
- )
-
- @pytest.mark.requires_reactor # needs a reactor for twisted_sleep()
- @coroutine_test
- async def test_twisted_sleep_single(self):
- await self._test_sleep([TwistedSleepSpiderMiddleware])
-
- @pytest.mark.requires_reactor # needs a reactor for twisted_sleep()
- @coroutine_test
- async def test_twisted_sleep_multiple(self):
- await self._test_sleep(
- [NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware]
- )
+async def _test(
+ spider_middlewares: list[type],
+ spider_cls: type[Spider],
+ expected_items: list[Any],
+) -> None:
+ actual_items = []
+
+ def track_item(item: Any, response: Response, spider: Spider) -> None:
+ actual_items.append(item)
+
+ settings = {
+ "SPIDER_MIDDLEWARES": {cls: n for n, cls in enumerate(spider_middlewares)},
+ }
+ crawler = get_crawler(spider_cls, settings_dict=settings)
+ crawler.signals.connect(track_item, signals.item_scraped)
+ await crawler.crawl_async()
+ assert crawler.stats.get_value("finish_reason") == "finished"
+ assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"
+
+
+async def _test_wrap(
+ spider_middleware: type,
+ spider_cls: type[Spider],
+ expected_items: list[Any] | None = None,
+) -> None:
+ expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]
+ await _test([spider_middleware], spider_cls, expected_items)
+
+
+@coroutine_test
+async def test_modern_mw_modern_spider() -> None:
+ await _test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
+
+
+async def _test_sleep(spider_middlewares: list[type]) -> None:
+ class TestSpider(Spider):
+ name = "test"
+
+ async def start(self) -> AsyncIterator[Any]:
+ yield ITEM_A
+
+ await _test(spider_middlewares, TestSpider, [ITEM_A])
+
+
+@coroutine_test
+async def test_asyncio_sleep_single() -> None:
+ await _test_sleep([AsyncioSleepSpiderMiddleware])
+
+
+@coroutine_test
+async def test_asyncio_sleep_multiple() -> None:
+ await _test_sleep(
+ [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware]
+ )
+
+
[email protected]_reactor # needs a reactor for twisted_sleep()
+@coroutine_test
+async def test_twisted_sleep_single() -> None:
+ await _test_sleep([TwistedSleepSpiderMiddleware])
+
+
[email protected]_reactor # needs a reactor for twisted_sleep()
+@coroutine_test
+async def test_twisted_sleep_multiple() -> None:
+ await _test_sleep(
+ [NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware]
+ )
diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py
index e879f3a32..26c62f015 100644
--- a/tests/test_spidermiddleware_start.py
+++ b/tests/test_spidermiddleware_start.py
@@ -13,30 +13,29 @@ if TYPE_CHECKING:
from collections.abc import AsyncIterator
-class TestMiddleware:
- @coroutine_test
- async def test_async(self):
- crawler = get_crawler(Spider)
- mw = build_from_crawler(StartSpiderMiddleware, crawler)
-
- async def start() -> AsyncIterator[Request]:
- yield Request("data:,1")
- yield Request("data:,2", meta={"is_start_request": True})
- yield Request("data:,2", meta={"is_start_request": False})
- yield Request("data:,2", meta={"is_start_request": "foo"})
-
- result = [
- request.meta["is_start_request"]
- async for request in mw.process_start(start())
- ]
- assert result == [True, True, False, "foo"]
-
- def test_spider_output_not_marked(self):
- # Requests from a non-None response (spider output) are not flagged.
- crawler = get_crawler(Spider)
- mw = build_from_crawler(StartSpiderMiddleware, crawler)
- response = Response("data:,")
- request = Request("data:,1")
- out = list(mw.process_spider_output(response, [request]))
- assert out == [request]
- assert "is_start_request" not in request.meta
+@coroutine_test
+async def test_async() -> None:
+ crawler = get_crawler(Spider)
+ mw = build_from_crawler(StartSpiderMiddleware, crawler)
+
+ async def start() -> AsyncIterator[Request]:
+ yield Request("data:,1")
+ yield Request("data:,2", meta={"is_start_request": True})
+ yield Request("data:,2", meta={"is_start_request": False})
+ yield Request("data:,2", meta={"is_start_request": "foo"})
+
+ result = [
+ request.meta["is_start_request"] async for request in mw.process_start(start())
+ ]
+ assert result == [True, True, False, "foo"]
+
+
+def test_spider_output_not_marked() -> None:
+ # Requests from a non-None response (spider output) are not flagged.
+ crawler = get_crawler(Spider)
+ mw = build_from_crawler(StartSpiderMiddleware, crawler)
+ response = Response("data:,")
+ request = Request("data:,1")
+ out = list(mw.process_spider_output(response, [request]))
+ assert out == [request]
+ assert "is_start_request" not in request.meta