Scrapy, a fast high-level web crawling & scraping framework for Python.
Unpack some test classes that are just containers. (#8000)
Andrey Rakhmatullinc7eacdbb30a4
diff --git a/tests/test_closespider.py b/tests/test_closespider.pyindex 81c122b81..99c552b80 100644--- a/tests/test_closespider.py+++ b/tests/test_closespider.py@@ -10,110 +10,104 @@ from tests.spiders import (from tests.utils.decorators import inline_callbacks_test-class TestCloseSpider:- mockserver: MockServer+@inline_callbacks_test+def test_closespider_itemcount(mockserver: MockServer):+ close_on = 5+ crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})+ yield crawler.crawl(mockserver=mockserver)+ assert isinstance(crawler.spider, ItemSpider)+ reason = crawler.spider.meta["close_reason"]+ assert reason == "closespider_itemcount"+ itemcount = crawler.stats.get_value("item_scraped_count")+ assert itemcount >= close_on- @classmethod- def setup_class(cls):- cls.mockserver = MockServer()- cls.mockserver.__enter__()- @classmethod- def teardown_class(cls):- cls.mockserver.__exit__(None, None, None)+@inline_callbacks_test+def test_closespider_pagecount(mockserver: MockServer):+ close_on = 5+ crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})+ yield crawler.crawl(mockserver=mockserver)+ assert isinstance(crawler.spider, FollowAllSpider)+ reason = crawler.spider.meta["close_reason"]+ assert reason == "closespider_pagecount"+ pagecount = crawler.stats.get_value("response_received_count")+ assert pagecount >= close_on- @inline_callbacks_test- def test_closespider_itemcount(self):- close_on = 5- crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})- yield crawler.crawl(mockserver=self.mockserver)- assert isinstance(crawler.spider, ItemSpider)- reason = crawler.spider.meta["close_reason"]- assert reason == "closespider_itemcount"- itemcount = crawler.stats.get_value("item_scraped_count")- assert itemcount >= close_on- @inline_callbacks_test- def test_closespider_pagecount(self):- close_on = 5- crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})- yield crawler.crawl(mockserver=self.mockserver)- assert isinstance(crawler.spider, FollowAllSpider)- reason = crawler.spider.meta["close_reason"]- assert reason == "closespider_pagecount"- pagecount = crawler.stats.get_value("response_received_count")- assert pagecount >= close_on+@inline_callbacks_test+def test_closespider_pagecount_no_item(mockserver: MockServer):+ close_on = 5+ max_items = 5+ max_requests = close_on + max_items+ crawler = get_crawler(+ MaxItemsAndRequestsSpider,+ {+ "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on,+ },+ )+ yield crawler.crawl(+ max_items=max_items, max_requests=max_requests, mockserver=mockserver+ )+ assert isinstance(crawler.spider, MaxItemsAndRequestsSpider)+ reason = crawler.spider.meta["close_reason"]+ assert reason == "closespider_pagecount_no_item"+ pagecount = crawler.stats.get_value("response_received_count")+ itemcount = crawler.stats.get_value("item_scraped_count")+ assert pagecount <= close_on + itemcount- @inline_callbacks_test- def test_closespider_pagecount_no_item(self):- close_on = 5- max_items = 5- max_requests = close_on + max_items- crawler = get_crawler(- MaxItemsAndRequestsSpider,- {- "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on,- },- )- yield crawler.crawl(- max_items=max_items, max_requests=max_requests, mockserver=self.mockserver- )- assert isinstance(crawler.spider, MaxItemsAndRequestsSpider)- reason = crawler.spider.meta["close_reason"]- assert reason == "closespider_pagecount_no_item"- pagecount = crawler.stats.get_value("response_received_count")- itemcount = crawler.stats.get_value("item_scraped_count")- assert pagecount <= close_on + itemcount- @inline_callbacks_test- def test_closespider_pagecount_no_item_with_pagecount(self):- close_on_pagecount_no_item = 5- close_on_pagecount = 20- crawler = get_crawler(- FollowAllSpider,- {- "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on_pagecount_no_item,- "CLOSESPIDER_PAGECOUNT": close_on_pagecount,- },- )- yield crawler.crawl(mockserver=self.mockserver)- assert isinstance(crawler.spider, FollowAllSpider)- reason = crawler.spider.meta["close_reason"]- assert reason == "closespider_pagecount_no_item"- pagecount = crawler.stats.get_value("response_received_count")- assert pagecount < close_on_pagecount+@inline_callbacks_test+def test_closespider_pagecount_no_item_with_pagecount(mockserver: MockServer):+ close_on_pagecount_no_item = 5+ close_on_pagecount = 20+ crawler = get_crawler(+ FollowAllSpider,+ {+ "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on_pagecount_no_item,+ "CLOSESPIDER_PAGECOUNT": close_on_pagecount,+ },+ )+ yield crawler.crawl(mockserver=mockserver)+ assert isinstance(crawler.spider, FollowAllSpider)+ reason = crawler.spider.meta["close_reason"]+ assert reason == "closespider_pagecount_no_item"+ pagecount = crawler.stats.get_value("response_received_count")+ assert pagecount < close_on_pagecount- @inline_callbacks_test- def test_closespider_errorcount(self):- close_on = 5- crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})- yield crawler.crawl(total=1000000, mockserver=self.mockserver)- assert isinstance(crawler.spider, ErrorSpider)- reason = crawler.spider.meta["close_reason"]- assert reason == "closespider_errorcount"- key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}"- errorcount = crawler.stats.get_value(key)- assert crawler.stats.get_value("spider_exceptions/count") >= close_on- assert errorcount >= close_on- @inline_callbacks_test- def test_closespider_timeout(self):- close_on = 0.1- crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})- yield crawler.crawl(total=1000000, mockserver=self.mockserver)- assert isinstance(crawler.spider, FollowAllSpider)- reason = crawler.spider.meta["close_reason"]- assert reason == "closespider_timeout"- total_seconds = crawler.stats.get_value("elapsed_time_seconds")- assert total_seconds >= close_on+@inline_callbacks_test+def test_closespider_errorcount(mockserver: MockServer):+ close_on = 5+ crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})+ yield crawler.crawl(total=1000000, mockserver=mockserver)+ assert isinstance(crawler.spider, ErrorSpider)+ reason = crawler.spider.meta["close_reason"]+ assert reason == "closespider_errorcount"+ key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}"+ errorcount = crawler.stats.get_value(key)+ assert crawler.stats.get_value("spider_exceptions/count") >= close_on+ assert errorcount >= close_on- @inline_callbacks_test- def test_closespider_timeout_no_item(self):- timeout = 1- crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})- yield crawler.crawl(n=3, mockserver=self.mockserver)- assert isinstance(crawler.spider, SlowSpider)- reason = crawler.spider.meta["close_reason"]- assert reason == "closespider_timeout_no_item"- total_seconds = crawler.stats.get_value("elapsed_time_seconds")- assert total_seconds >= timeout++@inline_callbacks_test+def test_closespider_timeout(mockserver: MockServer):+ close_on = 0.1+ crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})+ yield crawler.crawl(total=1000000, mockserver=mockserver)+ assert isinstance(crawler.spider, FollowAllSpider)+ reason = crawler.spider.meta["close_reason"]+ assert reason == "closespider_timeout"+ total_seconds = crawler.stats.get_value("elapsed_time_seconds")+ assert total_seconds >= close_on+++@inline_callbacks_test+def test_closespider_timeout_no_item(mockserver: MockServer):+ timeout = 1+ crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})+ yield crawler.crawl(n=3, mockserver=mockserver)+ assert isinstance(crawler.spider, SlowSpider)+ reason = crawler.spider.meta["close_reason"]+ assert reason == "closespider_timeout_no_item"+ total_seconds = crawler.stats.get_value("elapsed_time_seconds")+ assert total_seconds >= timeoutdiff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.pyindex f6ebe5865..c068bb0f3 100644--- a/tests/test_cmdline/__init__.py+++ b/tests/test_cmdline/__init__.py@@ -12,61 +12,60 @@ from subprocess import PIPE, Popenfrom scrapy.utils.test import get_testenv-class TestCmdline:- def setup_method(self):- self.env = get_testenv()- tests_path = Path(__file__).parent.parent- self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent)- self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"+def _execute(*new_args, **kwargs):+ env = get_testenv()+ tests_path = Path(__file__).parent.parent+ env["PYTHONPATH"] += os.pathsep + str(tests_path.parent)+ env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"+ encoding = sys.stdout.encoding or "utf-8"+ args = (sys.executable, "-m", "scrapy.cmdline", *new_args)+ proc = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs)+ comm = proc.communicate()[0].strip()+ return comm.decode(encoding)- def _execute(self, *new_args, **kwargs):- encoding = sys.stdout.encoding or "utf-8"- args = (sys.executable, "-m", "scrapy.cmdline", *new_args)- proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs)- comm = proc.communicate()[0].strip()- return comm.decode(encoding)- def test_default_settings(self):- assert self._execute("settings", "--get", "TEST1") == "default"+def test_default_settings():+ assert _execute("settings", "--get", "TEST1") == "default"- def test_override_settings_using_set_arg(self):- assert (- self._execute("settings", "--get", "TEST1", "-s", "TEST1=override")- == "override"- )- def test_profiling(self):- path = Path(tempfile.mkdtemp())- filename = path / "res.prof"- try:- self._execute("version", "--profile", str(filename))- assert filename.exists()- out = StringIO()- stats = pstats.Stats(str(filename), stream=out)- stats.print_stats()- out.seek(0)- stats = out.read()- assert str(Path("scrapy", "commands", "version.py")) in stats- assert "tottime" in stats- finally:- shutil.rmtree(path)+def test_override_settings_using_set_arg():+ assert _execute("settings", "--get", "TEST1", "-s", "TEST1=override") == "override"- def test_override_dict_settings(self):- EXT_PATH = "tests.test_cmdline.extensions.DummyExtension"- EXTENSIONS = {EXT_PATH: 200}- settingsstr = self._execute(- "settings",- "--get",- "EXTENSIONS",- "-s",- "EXTENSIONS=" + json.dumps(EXTENSIONS),- )- assert "..." not in settingsstr- settingsdict = ast.literal_eval(settingsstr)- assert set(settingsdict.keys()) == set(EXTENSIONS.keys())- assert settingsdict[EXT_PATH] == 200- def test_pathlib_path_as_feeds_key(self):- assert self._execute("settings", "--get", "FEEDS") == json.dumps(- {"items.csv": {"format": "csv", "fields": ["price", "name"]}}- )+def test_profiling():+ path = Path(tempfile.mkdtemp())+ filename = path / "res.prof"+ try:+ _execute("version", "--profile", str(filename))+ assert filename.exists()+ out = StringIO()+ stats = pstats.Stats(str(filename), stream=out)+ stats.print_stats()+ out.seek(0)+ stats = out.read()+ assert str(Path("scrapy", "commands", "version.py")) in stats+ assert "tottime" in stats+ finally:+ shutil.rmtree(path)+++def test_override_dict_settings():+ EXT_PATH = "tests.test_cmdline.extensions.DummyExtension"+ EXTENSIONS = {EXT_PATH: 200}+ settingsstr = _execute(+ "settings",+ "--get",+ "EXTENSIONS",+ "-s",+ "EXTENSIONS=" + json.dumps(EXTENSIONS),+ )+ assert "..." not in settingsstr+ settingsdict = ast.literal_eval(settingsstr)+ assert set(settingsdict.keys()) == set(EXTENSIONS.keys())+ assert settingsdict[EXT_PATH] == 200+++def test_pathlib_path_as_feeds_key():+ assert _execute("settings", "--get", "FEEDS") == json.dumps(+ {"items.csv": {"format": "csv", "fields": ["price", "name"]}}+ )diff --git a/tests/test_command_version.py b/tests/test_command_version.pyindex c4ca9d07b..39132b64b 100644--- a/tests/test_command_version.py+++ b/tests/test_command_version.py@@ -2,24 +2,24 @@ import scrapyfrom tests.utils.cmdline import proc-class TestVersionCommand:- def test_output(self) -> None:- _, out, _ = proc("version")- assert out.strip() == f"Scrapy {scrapy.__version__}"+def test_output() -> None:+ _, out, _ = proc("version")+ assert out.strip() == f"Scrapy {scrapy.__version__}"- def test_verbose_output(self) -> None:- _, out, _ = proc("version", "-v")- headers = [line.partition(":")[0].strip() for line in out.strip().splitlines()]- assert headers == [- "Scrapy",- "lxml",- "libxml2",- "cssselect",- "parsel",- "w3lib",- "Twisted",- "Python",- "pyOpenSSL",- "cryptography",- "Platform",- ]++def test_verbose_output() -> None:+ _, out, _ = proc("version", "-v")+ headers = [line.partition(":")[0].strip() for line in out.strip().splitlines()]+ assert headers == [+ "Scrapy",+ "lxml",+ "libxml2",+ "cssselect",+ "parsel",+ "w3lib",+ "Twisted",+ "Python",+ "pyOpenSSL",+ "cryptography",+ "Platform",+ ]diff --git a/tests/test_dependencies.py b/tests/test_dependencies.pyindex 41c7ce526..353b6d019 100644--- a/tests/test_dependencies.py+++ b/tests/test_dependencies.py@@ -9,23 +9,22 @@ import pytestfrom twisted import version as twisted_version-class TestScrapyUtils:- def test_pinned_twisted_version(self):- """When running tests within a Tox environment with pinned- dependencies, make sure that the version of Twisted is the pinned- version.+def test_pinned_twisted_version() -> None:+ """When running tests within a Tox environment with pinned+ dependencies, make sure that the version of Twisted is the pinned+ version.- See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011- """- if not os.environ.get("_SCRAPY_MIN", None):- pytest.skip("Not in a min environment")+ See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011+ """+ if not os.environ.get("_SCRAPY_MIN", None):+ pytest.skip("Not in a min environment")- tox_config_file_path = Path(__file__).parent / ".." / "tox.ini"- config_parser = ConfigParser()- config_parser.read(tox_config_file_path)- pattern = r"Twisted==([\d.]+)"- match = re.search(pattern, config_parser["min"]["deps"])- assert match- pinned_twisted_version_string = match[1]+ tox_config_file_path = Path(__file__).parent / ".." / "tox.ini"+ config_parser = ConfigParser()+ config_parser.read(tox_config_file_path)+ pattern = r"Twisted==([\d.]+)"+ match = re.search(pattern, config_parser["min"]["deps"])+ assert match+ pinned_twisted_version_string = match[1]- assert twisted_version.short() == pinned_twisted_version_string+ assert twisted_version.short() == pinned_twisted_version_stringdiff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.pyindex 01014ddd6..cd7a61301 100644--- a/tests/test_downloadermiddleware_stats.py+++ b/tests/test_downloadermiddleware_stats.py@@ -24,6 +24,9 @@ class TestDownloaderStats:self.req = Request("http://scrapytest.org")self.res = Response("http://scrapytest.org", status=400)+ def teardown_method(self) -> None:+ self.crawler.stats.close_spider()+def assertStatsEqual(self, key: str, value: object) -> None:assert self.crawler.stats.get_value(key) == value, str(self.crawler.stats.get_stats()@@ -50,9 +53,6 @@ class TestDownloaderStats:with pytest.raises(NotConfigured):build_from_crawler(DownloaderStats, crawler)- def teardown_method(self) -> None:- self.crawler.stats.close_spider()-def test_get_header_size_non_list_value() -> None:# Deliberately passing a non-list/tuple header value to make surediff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 077a92ecf..4cf4cf2b1 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -54,30 +54,21 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):self.times[slot].append(time.time())-class TestCrawl:- @classmethod- def setup_class(cls):- cls.mockserver = MockServer()- cls.mockserver.__enter__()-- @classmethod- def teardown_class(cls):- cls.mockserver.__exit__(None, None, None)-- @inline_callbacks_test- def test_delay(self):- crawler = get_crawler(DownloaderSlotsSettingsTestSpider)- yield crawler.crawl(mockserver=self.mockserver)- slots = crawler.engine.downloader.slots- times = crawler.spider.times- # Downloading and processing a response add a roughly constant amount- # of time on top of the configured delay, so the margin is absolute.- # It stays well below the 1 second that separates the delays being- # compared, so a slot using the delay of another one still fails.- tolerance = 0.75-- for slot, (first, second) in times.items():- assert abs((second - first) - slots[slot].delay) < tolerance+@inline_callbacks_test+def test_delay(mockserver: MockServer):+ crawler = get_crawler(DownloaderSlotsSettingsTestSpider)+ yield crawler.crawl(mockserver=mockserver)+ assert isinstance(crawler.spider, DownloaderSlotsSettingsTestSpider)+ slots = crawler.engine.downloader.slots+ times = crawler.spider.times+ # Downloading and processing a response add a roughly constant amount+ # of time on top of the configured delay, so the margin is absolute.+ # It stays well below the 1 second that separates the delays being+ # compared, so a slot using the delay of another one still fails.+ tolerance = 0.75++ for slot, (first, second) in times.items():+ assert abs((second - first) - slots[slot].delay) < tolerance@coroutine_testdiff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.pyindex fd7e3958d..ff59599fc 100644--- a/tests/test_extension_periodic_log.py+++ b/tests/test_extension_periodic_log.py@@ -76,190 +76,187 @@ def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog:return build_from_crawler(CustomPeriodicLog, crawler)-class TestPeriodicLog:- def test_extension_enabled(self):- # Expected that settings for this extension loaded successfully-- # "PERIODIC_LOG_STATS": True -> set to {"enabled": True}- # due to TypeError exception from settings.getdict- assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60})-- # "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True}- # due to JSONDecodeError(ValueError) exception from settings.getdict- assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60})-- # The ame for PERIODIC_LOG_DELTA:- assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})- assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})-- def test_no_interval(self):- with pytest.raises(NotConfigured):- extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 0})-- def test_nothing_enabled(self):- with pytest.raises(NotConfigured):- extension({"LOGSTATS_INTERVAL": 60})-- @coroutine_test- async def test_log_delta(self):- def emulate(- settings: dict[str, Any] | None = None,- ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]:- spider = MetaSpider()- ext = extension(settings)- ext.spider_opened(spider)- ext.set_a()- a = ext.log_delta()- ext.set_b()- b = ext.log_delta()- ext.spider_closed(spider, reason="finished")- return ext, a, b-- def check(- settings: dict[str, Any], condition: Callable[[str, Any], bool]- ) -> None:- ext, a, b = emulate(settings)- assert list(a["delta"].keys()) == [- k for k, v in ext.stats._stats.items() if condition(k, v)- ]- assert list(b["delta"].keys()) == [- k for k, v in ext.stats._stats.items() if condition(k, v)- ]-- # Including all- check({"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float)))-- # include:- check(- {"PERIODIC_LOG_DELTA": {"include": ["downloader/"]}},- lambda k, v: isinstance(v, (int, float)) and "downloader/" in k,- )-- # include multiple- check(- {"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}},- lambda k, v: (- isinstance(v, (int, float))- and ("downloader/" in k or "scheduler/" in k)- ),- )-- # exclude- check(- {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}},- lambda k, v: isinstance(v, (int, float)) and "downloader/" not in k,- )-- # exclude multiple- check(- {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}},- lambda k, v: (- isinstance(v, (int, float))- and ("downloader/" not in k and "scheduler/" not in k)- ),- )-- # include exclude combined- check(- {"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}},- lambda k, v: (- isinstance(v, (int, float))- and ("downloader/" in k and "bytes" not in k)- ),- )-- @coroutine_test- async def test_log_stats(self):- def emulate(- settings: dict[str, Any] | None = None,- ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]:- spider = MetaSpider()- ext = extension(settings)- ext.spider_opened(spider)- ext.set_a()- a = ext.log_crawler_stats()- ext.set_b()- b = ext.log_crawler_stats()- ext.spider_closed(spider, reason="finished")- return ext, a, b-- def check(- settings: dict[str, Any], condition: Callable[[str, Any], bool]- ) -> None:- ext, a, b = emulate(settings)- assert list(a["stats"].keys()) == [- k for k, v in ext.stats._stats.items() if condition(k, v)- ]- assert list(b["stats"].keys()) == [- k for k, v in ext.stats._stats.items() if condition(k, v)- ]-- # Including all- check({"PERIODIC_LOG_STATS": True}, lambda k, v: True)-- # include:- check(- {"PERIODIC_LOG_STATS": {"include": ["downloader/"]}},- lambda k, v: "downloader/" in k,- )-- # include multiple- check(- {"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}},- lambda k, v: "downloader/" in k or "scheduler/" in k,- )-- # exclude- check(- {"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}},- lambda k, v: "downloader/" not in k,- )-- # exclude multiple- check(- {"PERIODIC_LOG_STATS": {"exclude": ["downloader/", "scheduler/"]}},- lambda k, v: "downloader/" not in k and "scheduler/" not in k,- )-- # include exclude combined- check(- {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}},- lambda k, v: "downloader/" in k and "bytes" not in k,- )-- @coroutine_test- async def test_log_timing(self, caplog: pytest.LogCaptureFixture) -> None:- settings = {- "EXTENSIONS": {"scrapy.extensions.periodic_log.PeriodicLog": 0},- "PERIODIC_LOG_TIMING_ENABLED": True,- "LOGSTATS_INTERVAL": 30,- }- crawler = get_crawler(MetaSpider, settings)- with caplog.at_level(logging.INFO, logger="scrapy.extensions.periodic_log"):- await crawler.crawl_async()-- records = [- r for r in caplog.records if r.name == "scrapy.extensions.periodic_log"+def test_extension_enabled() -> None:+ # Expected that settings for this extension loaded successfully++ # "PERIODIC_LOG_STATS": True -> set to {"enabled": True}+ # due to TypeError exception from settings.getdict+ assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60})++ # "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True}+ # due to JSONDecodeError(ValueError) exception from settings.getdict+ assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60})++ # The ame for PERIODIC_LOG_DELTA:+ assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})+ assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})+++def test_no_interval() -> None:+ with pytest.raises(NotConfigured):+ extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 0})+++def test_nothing_enabled() -> None:+ with pytest.raises(NotConfigured):+ extension({"LOGSTATS_INTERVAL": 60})+++@coroutine_test+async def test_log_delta() -> None:+ def emulate(+ settings: dict[str, Any] | None = None,+ ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]:+ spider = MetaSpider()+ ext = extension(settings)+ ext.spider_opened(spider)+ ext.set_a()+ a = ext.log_delta()+ ext.set_b()+ b = ext.log_delta()+ ext.spider_closed(spider, reason="finished")+ return ext, a, b++ def check(settings: dict[str, Any], condition: Callable[[str, Any], bool]) -> None:+ ext, a, b = emulate(settings)+ assert list(a["delta"].keys()) == [+ k for k, v in ext.stats._stats.items() if condition(k, v)]- assert records, "PeriodicLog logged nothing"- # Only the timing section is enabled, and it is logged on spider close.- data = json.loads(records[-1].getMessage())- assert list(data) == ["time"]- assert data["time"]["log_interval"] == 30- assert data["time"]["log_interval_real"] >= 0- assert data["time"]["elapsed"] >= 0- assert data["time"]["start_time"] <= data["time"]["utcnow"]-- def test_multiplier_deprecated(self) -> None:- crawler = get_crawler(- MetaSpider,- {"PERIODIC_LOG_TIMING_ENABLED": True, "LOGSTATS_INTERVAL": 30},- )- crawler._apply_settings()- ext = build_from_crawler(PeriodicLog, crawler)- with pytest.warns(ScrapyDeprecationWarning):- assert ext.multiplier == 2.0- with pytest.warns(ScrapyDeprecationWarning):- ext.multiplier = 3.0- with pytest.warns(ScrapyDeprecationWarning):- assert ext.multiplier == 3.0+ assert list(b["delta"].keys()) == [+ k for k, v in ext.stats._stats.items() if condition(k, v)+ ]++ # Including all+ check({"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float)))++ # include:+ check(+ {"PERIODIC_LOG_DELTA": {"include": ["downloader/"]}},+ lambda k, v: isinstance(v, (int, float)) and "downloader/" in k,+ )++ # include multiple+ check(+ {"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}},+ lambda k, v: (+ isinstance(v, (int, float)) and ("downloader/" in k or "scheduler/" in k)+ ),+ )++ # exclude+ check(+ {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}},+ lambda k, v: isinstance(v, (int, float)) and "downloader/" not in k,+ )++ # exclude multiple+ check(+ {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}},+ lambda k, v: (+ isinstance(v, (int, float))+ and ("downloader/" not in k and "scheduler/" not in k)+ ),+ )++ # include exclude combined+ check(+ {"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}},+ lambda k, v: (+ isinstance(v, (int, float)) and ("downloader/" in k and "bytes" not in k)+ ),+ )+++@coroutine_test+async def test_log_stats() -> None:+ def emulate(+ settings: dict[str, Any] | None = None,+ ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]:+ spider = MetaSpider()+ ext = extension(settings)+ ext.spider_opened(spider)+ ext.set_a()+ a = ext.log_crawler_stats()+ ext.set_b()+ b = ext.log_crawler_stats()+ ext.spider_closed(spider, reason="finished")+ return ext, a, b++ def check(settings: dict[str, Any], condition: Callable[[str, Any], bool]) -> None:+ ext, a, b = emulate(settings)+ assert list(a["stats"].keys()) == [+ k for k, v in ext.stats._stats.items() if condition(k, v)+ ]+ assert list(b["stats"].keys()) == [+ k for k, v in ext.stats._stats.items() if condition(k, v)+ ]++ # Including all+ check({"PERIODIC_LOG_STATS": True}, lambda k, v: True)++ # include:+ check(+ {"PERIODIC_LOG_STATS": {"include": ["downloader/"]}},+ lambda k, v: "downloader/" in k,+ )++ # include multiple+ check(+ {"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}},+ lambda k, v: "downloader/" in k or "scheduler/" in k,+ )++ # exclude+ check(+ {"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}},+ lambda k, v: "downloader/" not in k,+ )++ # exclude multiple+ check(+ {"PERIODIC_LOG_STATS": {"exclude": ["downloader/", "scheduler/"]}},+ lambda k, v: "downloader/" not in k and "scheduler/" not in k,+ )++ # include exclude combined+ check(+ {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}},+ lambda k, v: "downloader/" in k and "bytes" not in k,+ )+++@coroutine_test+async def test_log_timing(caplog: pytest.LogCaptureFixture) -> None:+ settings = {+ "EXTENSIONS": {"scrapy.extensions.periodic_log.PeriodicLog": 0},+ "PERIODIC_LOG_TIMING_ENABLED": True,+ "LOGSTATS_INTERVAL": 30,+ }+ crawler = get_crawler(MetaSpider, settings)+ with caplog.at_level(logging.INFO, logger="scrapy.extensions.periodic_log"):+ await crawler.crawl_async()++ records = [r for r in caplog.records if r.name == "scrapy.extensions.periodic_log"]+ assert records, "PeriodicLog logged nothing"+ # Only the timing section is enabled, and it is logged on spider close.+ data = json.loads(records[-1].getMessage())+ assert list(data) == ["time"]+ assert data["time"]["log_interval"] == 30+ assert data["time"]["log_interval_real"] >= 0+ assert data["time"]["elapsed"] >= 0+ assert data["time"]["start_time"] <= data["time"]["utcnow"]+++def test_multiplier_deprecated() -> None:+ crawler = get_crawler(+ MetaSpider,+ {"PERIODIC_LOG_TIMING_ENABLED": True, "LOGSTATS_INTERVAL": 30},+ )+ crawler._apply_settings()+ ext = build_from_crawler(PeriodicLog, crawler)+ with pytest.warns(ScrapyDeprecationWarning):+ assert ext.multiplier == 2.0+ with pytest.warns(ScrapyDeprecationWarning):+ ext.multiplier = 3.0+ with pytest.warns(ScrapyDeprecationWarning):+ assert ext.multiplier == 3.0diff --git a/tests/test_http_headers.py b/tests/test_http_headers.pyindex e7ed17615..561546711 100644--- a/tests/test_http_headers.py+++ b/tests/test_http_headers.py@@ -5,158 +5,174 @@ import pytestfrom scrapy.http import Headers-class TestHeaders:- def test_basics(self):- h = Headers({"Content-Type": "text/html", "Content-Length": 1234})- assert h["Content-Type"]- assert h["Content-Length"]-- with pytest.raises(KeyError):- h["Accept"]- assert h.get("Accept") is None- assert h.getlist("Accept") == []-- assert h.get("Accept", "*/*") == b"*/*"- assert h.getlist("Accept", "*/*") == [b"*/*"]- assert h.getlist("Accept", ["text/html", "images/jpeg"]) == [- b"text/html",- b"images/jpeg",- ]-- def test_single_value(self):- h = Headers()- h["Content-Type"] = "text/html"- assert h["Content-Type"] == b"text/html"- assert h.get("Content-Type") == b"text/html"- assert h.getlist("Content-Type") == [b"text/html"]-- def test_multivalue(self):- h = Headers()- h["X-Forwarded-For"] = hlist = ["ip1", "ip2"]- assert h["X-Forwarded-For"] == b"ip2"- assert h.get("X-Forwarded-For") == b"ip2"- assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]- assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]-- def test_multivalue_for_one_header(self):- h = Headers((("a", "b"), ("a", "c")))- assert h["a"] == b"c"- assert h.get("a") == b"c"- assert h.getlist("a") == [b"b", b"c"]-- def test_encode_utf8(self):- h = Headers({"key": "\xa3"}, encoding="utf-8")- key, val = dict(h.items()).popitem()- assert isinstance(key, bytes), key- assert isinstance(val[0], bytes), val[0]- assert val[0] == b"\xc2\xa3"-- def test_encode_latin1(self):- h = Headers({"key": "\xa3"}, encoding="latin1")- _, val = dict(h.items()).popitem()- assert val[0] == b"\xa3"-- def test_encode_multiple(self):- h = Headers({"key": ["\xa3"]}, encoding="utf-8")- _, val = dict(h.items()).popitem()- assert val[0] == b"\xc2\xa3"-- def test_delete_and_contains(self):- h = Headers()- h["Content-Type"] = "text/html"- assert "Content-Type" in h- del h["Content-Type"]- assert "Content-Type" not in h-- def test_setdefault(self):- h = Headers()- hlist = ["ip1", "ip2"]- olist = h.setdefault("X-Forwarded-For", hlist)- assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]- assert h.getlist("X-Forwarded-For") is olist-- h = Headers()- olist = h.setdefault("X-Forwarded-For", "ip1")- assert h.getlist("X-Forwarded-For") == [b"ip1"]- assert h.getlist("X-Forwarded-For") is olist-- def test_iterables(self):- idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}-- h = Headers(idict)- assert dict(h.items()) == {- b"Content-Type": [b"text/html"],- b"X-Forwarded-For": [b"ip1", b"ip2"],- }- assert sorted(h.keys()) == [b"Content-Type", b"X-Forwarded-For"]- assert sorted(h.items()) == [- (b"Content-Type", [b"text/html"]),- (b"X-Forwarded-For", [b"ip1", b"ip2"]),- ]- assert set(h.values()) == {b"ip2", b"text/html"}-- def test_update(self):- h = Headers()- h.update({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]})- assert h.getlist("Content-Type") == [b"text/html"]- assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]-- def test_copy(self):- h1 = Headers({"header1": ["value1", "value2"]})- h2 = copy.copy(h1)- assert h1 == h2- assert h1.getlist("header1") == h2.getlist("header1")- assert h1.getlist("header1") is not h2.getlist("header1")- assert isinstance(h2, Headers)-- def test_appendlist(self):- h1 = Headers({"header1": "value1"})- h1.appendlist("header1", "value3")- assert h1.getlist("header1") == [b"value1", b"value3"]-- h1 = Headers()- h1.appendlist("header1", "value1")- h1.appendlist("header1", "value3")- assert h1.getlist("header1") == [b"value1", b"value3"]-- def test_setlist(self):- h1 = Headers({"header1": "value1"})- assert h1.getlist("header1") == [b"value1"]- h1.setlist("header1", [b"value2", b"value3"])- assert h1.getlist("header1") == [b"value2", b"value3"]-- def test_setlistdefault(self):- h1 = Headers({"header1": "value1"})- h1.setlistdefault("header1", ["value2", "value3"])- h1.setlistdefault("header2", ["value2", "value3"])- assert h1.getlist("header1") == [b"value1"]- assert h1.getlist("header2") == [b"value2", b"value3"]-- def test_none_value(self):- h1 = Headers()- h1["foo"] = "bar"- h1["foo"] = None- h1.setdefault("foo", "bar")- assert h1["foo"] is None- assert h1.get("foo") is None- assert h1.getlist("foo") == []-- def test_int_value(self):- h1 = Headers({"hey": 5})- h1["foo"] = 1- h1.setdefault("bar", 2)- h1.setlist("buz", [1, "dos", 3])- assert h1.getlist("foo") == [b"1"]- assert h1.getlist("bar") == [b"2"]- assert h1.getlist("buz") == [b"1", b"dos", b"3"]- assert h1.getlist("hey") == [b"5"]-- def test_invalid_value(self):- with pytest.raises(TypeError, match="Unsupported value type"):- Headers({"foo": object()})- with pytest.raises(TypeError, match="Unsupported value type"):- Headers()["foo"] = object()- with pytest.raises(TypeError, match="Unsupported value type"):- Headers().setdefault("foo", object())- with pytest.raises(TypeError, match="Unsupported value type"):- Headers().setlist("foo", [object()]) # type: ignore[list-item]+def test_basics() -> None:+ h = Headers({"Content-Type": "text/html", "Content-Length": 1234})+ assert h["Content-Type"]+ assert h["Content-Length"]++ with pytest.raises(KeyError):+ h["Accept"]+ assert h.get("Accept") is None+ assert h.getlist("Accept") == []++ assert h.get("Accept", "*/*") == b"*/*"+ assert h.getlist("Accept", "*/*") == [b"*/*"]+ assert h.getlist("Accept", ["text/html", "images/jpeg"]) == [+ b"text/html",+ b"images/jpeg",+ ]+++def test_single_value() -> None:+ h = Headers()+ h["Content-Type"] = "text/html"+ assert h["Content-Type"] == b"text/html"+ assert h.get("Content-Type") == b"text/html"+ assert h.getlist("Content-Type") == [b"text/html"]+++def test_multivalue() -> None:+ h = Headers()+ h["X-Forwarded-For"] = hlist = ["ip1", "ip2"]+ assert h["X-Forwarded-For"] == b"ip2"+ assert h.get("X-Forwarded-For") == b"ip2"+ assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]+ assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]+++def test_multivalue_for_one_header() -> None:+ h = Headers((("a", "b"), ("a", "c")))+ assert h["a"] == b"c"+ assert h.get("a") == b"c"+ assert h.getlist("a") == [b"b", b"c"]+++def test_encode_utf8() -> None:+ h = Headers({"key": "\xa3"}, encoding="utf-8")+ key, val = dict(h.items()).popitem()+ assert isinstance(key, bytes), key+ assert isinstance(val[0], bytes), val[0]+ assert val[0] == b"\xc2\xa3"+++def test_encode_latin1() -> None:+ h = Headers({"key": "\xa3"}, encoding="latin1")+ _, val = dict(h.items()).popitem()+ assert val[0] == b"\xa3"+++def test_encode_multiple() -> None:+ h = Headers({"key": ["\xa3"]}, encoding="utf-8")+ _, val = dict(h.items()).popitem()+ assert val[0] == b"\xc2\xa3"+++def test_delete_and_contains() -> None:+ h = Headers()+ h["Content-Type"] = "text/html"+ assert "Content-Type" in h+ del h["Content-Type"]+ assert "Content-Type" not in h+++def test_setdefault() -> None:+ h = Headers()+ hlist = ["ip1", "ip2"]+ olist = h.setdefault("X-Forwarded-For", hlist)+ assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]+ assert h.getlist("X-Forwarded-For") is olist++ h = Headers()+ olist = h.setdefault("X-Forwarded-For", "ip1")+ assert h.getlist("X-Forwarded-For") == [b"ip1"]+ assert h.getlist("X-Forwarded-For") is olist+++def test_iterables() -> None:+ idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}++ h = Headers(idict)+ assert dict(h.items()) == {+ b"Content-Type": [b"text/html"],+ b"X-Forwarded-For": [b"ip1", b"ip2"],+ }+ assert sorted(h.keys()) == [b"Content-Type", b"X-Forwarded-For"]+ assert sorted(h.items()) == [+ (b"Content-Type", [b"text/html"]),+ (b"X-Forwarded-For", [b"ip1", b"ip2"]),+ ]+ assert set(h.values()) == {b"ip2", b"text/html"}+++def test_update() -> None:+ h = Headers()+ h.update({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]})+ assert h.getlist("Content-Type") == [b"text/html"]+ assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]+++def test_copy() -> None:+ h1 = Headers({"header1": ["value1", "value2"]})+ h2 = copy.copy(h1)+ assert h1 == h2+ assert h1.getlist("header1") == h2.getlist("header1")+ assert h1.getlist("header1") is not h2.getlist("header1")+ assert isinstance(h2, Headers)+++def test_appendlist() -> None:+ h1 = Headers({"header1": "value1"})+ h1.appendlist("header1", "value3")+ assert h1.getlist("header1") == [b"value1", b"value3"]++ h1 = Headers()+ h1.appendlist("header1", "value1")+ h1.appendlist("header1", "value3")+ assert h1.getlist("header1") == [b"value1", b"value3"]+++def test_setlist() -> None:+ h1 = Headers({"header1": "value1"})+ assert h1.getlist("header1") == [b"value1"]+ h1.setlist("header1", [b"value2", b"value3"])+ assert h1.getlist("header1") == [b"value2", b"value3"]+++def test_setlistdefault() -> None:+ h1 = Headers({"header1": "value1"})+ h1.setlistdefault("header1", ["value2", "value3"])+ h1.setlistdefault("header2", ["value2", "value3"])+ assert h1.getlist("header1") == [b"value1"]+ assert h1.getlist("header2") == [b"value2", b"value3"]+++def test_none_value() -> None:+ h1 = Headers()+ h1["foo"] = "bar"+ h1["foo"] = None+ h1.setdefault("foo", "bar")+ assert h1["foo"] is None+ assert h1.get("foo") is None+ assert h1.getlist("foo") == []+++def test_int_value() -> None:+ h1 = Headers({"hey": 5})+ h1["foo"] = 1+ h1.setdefault("bar", 2)+ h1.setlist("buz", [1, "dos", 3])+ assert h1.getlist("foo") == [b"1"]+ assert h1.getlist("bar") == [b"2"]+ assert h1.getlist("buz") == [b"1", b"dos", b"3"]+ assert h1.getlist("hey") == [b"5"]+++def test_invalid_value() -> None:+ with pytest.raises(TypeError, match="Unsupported value type"):+ Headers({"foo": object()})+ with pytest.raises(TypeError, match="Unsupported value type"):+ Headers()["foo"] = object()+ with pytest.raises(TypeError, match="Unsupported value type"):+ Headers().setdefault("foo", object())+ with pytest.raises(TypeError, match="Unsupported value type"):+ Headers().setlist("foo", [object()]) # type: ignore[list-item]diff --git a/tests/test_link.py b/tests/test_link.pyindex 40c53edd3..56467eb28 100644--- a/tests/test_link.py+++ b/tests/test_link.py@@ -5,59 +5,61 @@ import pytestfrom scrapy.link import Link-class TestLink:- def _assert_same_links(self, link1: Link, link2: Link) -> None:- assert link1 == link2- assert hash(link1) == hash(link2)-- def _assert_different_links(self, link1: Link, link2: Link) -> None:- assert link1 != link2- assert hash(link1) != hash(link2)-- def test_eq_and_hash(self) -> None:- l1 = Link("http://www.example.com")- l2 = Link("http://www.example.com/other")- l3 = Link("http://www.example.com")-- self._assert_same_links(l1, l1)- self._assert_different_links(l1, l2)- self._assert_same_links(l1, l3)-- l4 = Link("http://www.example.com", text="test")- l5 = Link("http://www.example.com", text="test2")- l6 = Link("http://www.example.com", text="test")-- self._assert_same_links(l4, l4)- self._assert_different_links(l4, l5)- self._assert_same_links(l4, l6)-- l7 = Link(- "http://www.example.com", text="test", fragment="something", nofollow=False- )- l8 = Link(- "http://www.example.com", text="test", fragment="something", nofollow=False- )- l9 = Link(- "http://www.example.com", text="test", fragment="something", nofollow=True- )- l10 = Link(- "http://www.example.com", text="test", fragment="other", nofollow=False- )- self._assert_same_links(l7, l8)- self._assert_different_links(l7, l9)- self._assert_different_links(l7, l10)-- def test_repr(self) -> None:- l1 = Link(- "http://www.example.com", text="test", fragment="something", nofollow=True- )- l2 = eval(repr(l1))- self._assert_same_links(l1, l2)-- def test_bytes_url(self) -> None:- with pytest.raises(TypeError):- Link(b"http://www.example.com/\xc2\xa3") # type: ignore[arg-type]-- def test_eq_non_link(self) -> None:- url = "http://example.com"- assert Link(url) != url+def _assert_same_links(link1: Link, link2: Link) -> None:+ assert link1 == link2+ assert hash(link1) == hash(link2)+++def _assert_different_links(link1: Link, link2: Link) -> None:+ assert link1 != link2+ assert hash(link1) != hash(link2)+++def test_eq_and_hash() -> None:+ l1 = Link("http://www.example.com")+ l2 = Link("http://www.example.com/other")+ l3 = Link("http://www.example.com")++ _assert_same_links(l1, l1)+ _assert_different_links(l1, l2)+ _assert_same_links(l1, l3)++ l4 = Link("http://www.example.com", text="test")+ l5 = Link("http://www.example.com", text="test2")+ l6 = Link("http://www.example.com", text="test")++ _assert_same_links(l4, l4)+ _assert_different_links(l4, l5)+ _assert_same_links(l4, l6)++ l7 = Link(+ "http://www.example.com", text="test", fragment="something", nofollow=False+ )+ l8 = Link(+ "http://www.example.com", text="test", fragment="something", nofollow=False+ )+ l9 = Link(+ "http://www.example.com", text="test", fragment="something", nofollow=True+ )+ l10 = Link("http://www.example.com", text="test", fragment="other", nofollow=False)+ _assert_same_links(l7, l8)+ _assert_different_links(l7, l9)+ _assert_different_links(l7, l10)+++def test_repr() -> None:+ l1 = Link(+ "http://www.example.com", text="test", fragment="something", nofollow=True+ )+ l2 = eval(repr(l1))+ _assert_same_links(l1, l2)+++def test_bytes_url() -> None:+ with pytest.raises(TypeError):+ Link(b"http://www.example.com/\xc2\xa3") # type: ignore[arg-type]+++def test_eq_non_link() -> None:+ url = "http://example.com"+ assert Link(url) != urldiff --git a/tests/test_logstats.py b/tests/test_logstats.pyindex 09cafd218..dc3319aa8 100644--- a/tests/test_logstats.py+++ b/tests/test_logstats.py@@ -1,6 +1,7 @@from __future__ import annotationsfrom datetime import datetime+from typing import TYPE_CHECKINGimport pytest@@ -10,69 +11,86 @@ from scrapy.utils.test import get_crawlerfrom tests.spiders import SimpleSpiderfrom tests.utils.decorators import coroutine_test+if TYPE_CHECKING:+ from scrapy.crawler import Crawler+ from scrapy.statscollectors import StatsCollector-class TestLogStats:- def setup_method(self) -> None:- self.crawler = get_crawler(SimpleSpider)- self.spider = self.crawler._create_spider("spidey")- self.stats = self.crawler.stats-- self.stats.set_value("response_received_count", 4802)- self.stats.set_value("item_scraped_count", 3201)-- @coroutine_test- async def test_stats_calculations(self) -> None:- logstats = build_from_crawler(LogStats, self.crawler)-- with pytest.raises(AttributeError):- logstats.pagesprev- with pytest.raises(AttributeError):- logstats.itemsprev-- logstats.spider_opened(self.spider)- assert logstats.pagesprev == 4802- assert logstats.itemsprev == 3201-- logstats.calculate_stats()- assert logstats.items == 3201- assert logstats.pages == 4802- assert logstats.irate == 0.0- assert logstats.prate == 0.0- assert logstats.pagesprev == 4802- assert logstats.itemsprev == 3201-- # Simulate what happens after a minute- self.stats.set_value("response_received_count", 5187)- self.stats.set_value("item_scraped_count", 3492)- logstats.calculate_stats()- assert logstats.items == 3492- assert logstats.pages == 5187- assert logstats.irate == 291.0- assert logstats.prate == 385.0- assert logstats.pagesprev == 5187- assert logstats.itemsprev == 3492-- # Simulate when spider closes after running for 30 mins- self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))- self.stats.set_value("finish_time", datetime.fromtimestamp(1655101972))- logstats.spider_closed(self.spider, "test reason")- assert self.stats.get_value("responses_per_minute") == 172.9- assert self.stats.get_value("items_per_minute") == 116.4-- def test_stats_calculations_no_time(self) -> None:- """The stat values should be None since the start and finish time are- not available.- """- logstats = build_from_crawler(LogStats, self.crawler)- logstats.spider_closed(self.spider, "test reason")- assert self.stats.get_value("responses_per_minute") is None- assert self.stats.get_value("items_per_minute") is None-- def test_stats_calculation_no_elapsed_time(self) -> None:- """The stat values should be None since the elapsed time is 0."""- logstats = build_from_crawler(LogStats, self.crawler)- self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))- self.stats.set_value("finish_time", datetime.fromtimestamp(1655100172))- logstats.spider_closed(self.spider, "test reason")- assert self.stats.get_value("responses_per_minute") is None- assert self.stats.get_value("items_per_minute") is None++def crawler() -> Crawler:+ crawler = get_crawler(SimpleSpider)+ crawler.spider = crawler._create_spider("spidey")+ return crawler+++def stats(crawler: Crawler) -> StatsCollector:+ stats = crawler.stats+ stats.set_value("response_received_count", 4802)+ stats.set_value("item_scraped_count", 3201)+ return stats+++@coroutine_test+async def test_stats_calculations(crawler: Crawler, stats: StatsCollector) -> None:+ logstats = build_from_crawler(LogStats, crawler)++ with pytest.raises(AttributeError):+ logstats.pagesprev+ with pytest.raises(AttributeError):+ logstats.itemsprev++ assert crawler.spider+ logstats.spider_opened(crawler.spider)+ assert logstats.pagesprev == 4802+ assert logstats.itemsprev == 3201++ logstats.calculate_stats()+ assert logstats.items == 3201+ assert logstats.pages == 4802+ assert logstats.irate == 0.0+ assert logstats.prate == 0.0+ assert logstats.pagesprev == 4802+ assert logstats.itemsprev == 3201++ # Simulate what happens after a minute+ stats.set_value("response_received_count", 5187)+ stats.set_value("item_scraped_count", 3492)+ logstats.calculate_stats()+ assert logstats.items == 3492+ assert logstats.pages == 5187+ assert logstats.irate == 291.0+ assert logstats.prate == 385.0+ assert logstats.pagesprev == 5187+ assert logstats.itemsprev == 3492++ # Simulate when spider closes after running for 30 mins+ stats.set_value("start_time", datetime.fromtimestamp(1655100172))+ stats.set_value("finish_time", datetime.fromtimestamp(1655101972))+ logstats.spider_closed(crawler.spider, "test reason")+ assert stats.get_value("responses_per_minute") == 172.9+ assert stats.get_value("items_per_minute") == 116.4+++def test_stats_calculations_no_time(crawler: Crawler, stats: StatsCollector) -> None:+ """The stat values should be None since the start and finish time are+ not available.+ """+ logstats = build_from_crawler(LogStats, crawler)+ assert crawler.spider+ logstats.spider_closed(crawler.spider, "test reason")+ assert stats.get_value("responses_per_minute") is None+ assert stats.get_value("items_per_minute") is None+++def test_stats_calculation_no_elapsed_time(+ crawler: Crawler, stats: StatsCollector+) -> None:+ """The stat values should be None since the elapsed time is 0."""+ logstats = build_from_crawler(LogStats, crawler)+ stats.set_value("start_time", datetime.fromtimestamp(1655100172))+ stats.set_value("finish_time", datetime.fromtimestamp(1655100172))+ assert crawler.spider+ logstats.spider_closed(crawler.spider, "test reason")+ assert stats.get_value("responses_per_minute") is None+ assert stats.get_value("items_per_minute") is Nonediff --git a/tests/test_request_dict.py b/tests/test_request_dict.pyindex c7596e45d..f85be8977 100644--- a/tests/test_request_dict.py+++ b/tests/test_request_dict.py@@ -1,173 +1,192 @@-from typing import Any+from __future__ import annotations++from typing import TYPE_CHECKING, Anyimport pytest-from twisted.python.failure import Failurefrom scrapy import Request, Spiderfrom scrapy.http import JsonRequest, Responsefrom scrapy.utils.request import request_from_dict+if TYPE_CHECKING:+ from twisted.python.failure import Failure+class CustomRequest(Request):pass-class TestRequestSerialization:- def setup_method(self) -> None:- self.spider = MethodsSpider()-- def test_basic(self):- r = Request("http://www.example.com")- self._assert_serializes_ok(r)-- def test_all_attributes(self):- r = Request(- url="http://www.example.com",- callback=self.spider.parse_item,- errback=self.spider.handle_error,- method="POST",- body=b"some body",- headers={"content-encoding": "text/html; charset=latin-1"},- cookies={"currency": "руб"},- encoding="latin-1",- priority=20,- meta={"a": "b"},- cb_kwargs={"k": "v"},- flags=["testFlag"],- )- self._assert_serializes_ok(r, spider=self.spider)-- def test_latin1_body(self):- r = Request("http://www.example.com", body=b"\xa3")- self._assert_serializes_ok(r)-- def test_utf8_body(self):- r = Request("http://www.example.com", body=b"\xc2\xa3")- self._assert_serializes_ok(r)-- def _assert_serializes_ok(- self, request: Request, spider: Spider | None = None- ) -> None:- d = request.to_dict(spider=spider)- request2 = request_from_dict(d, spider=spider)- self._assert_same_request(request, request2)-- def _assert_same_request(self, r1: Request, r2: Request) -> None:- assert r1.__class__ == r2.__class__- assert r1.url == r2.url- assert r1.callback == r2.callback- assert r1.errback == r2.errback- assert r1.method == r2.method- assert r1.body == r2.body- assert r1.headers == r2.headers- assert r1.cookies == r2.cookies- assert r1.meta == r2.meta- assert r1.cb_kwargs == r2.cb_kwargs- assert r1.encoding == r2.encoding- assert r1._encoding == r2._encoding- assert r1.priority == r2.priority- assert r1.dont_filter == r2.dont_filter- assert r1.flags == r2.flags- if isinstance(r1, JsonRequest):- assert isinstance(r2, JsonRequest)- assert r1.dumps_kwargs == r2.dumps_kwargs-- def test_request_class(self):- r1 = CustomRequest("http://www.example.com")- self._assert_serializes_ok(r1, spider=self.spider)- r2 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})- self._assert_serializes_ok(r2, spider=self.spider)-- def test_callback_serialization(self):- r = Request(- "http://www.example.com",- callback=self.spider.parse_item,- errback=self.spider.handle_error,- )- self._assert_serializes_ok(r, spider=self.spider)-- def test_reference_callback_serialization(self):- r = Request(- "http://www.example.com",- callback=self.spider.parse_item_reference, # type: ignore[arg-type,misc]- errback=self.spider.handle_error_reference, # type: ignore[arg-type,misc]- )- self._assert_serializes_ok(r, spider=self.spider)- request_dict = r.to_dict(spider=self.spider)- assert request_dict["callback"] == "parse_item_reference"- assert request_dict["errback"] == "handle_error_reference"-- def test_private_reference_callback_serialization(self):- r = Request(- "http://www.example.com",- callback=self.spider._MethodsSpider__parse_item_reference, # type: ignore[attr-defined]- errback=self.spider._MethodsSpider__handle_error_reference, # type: ignore[attr-defined]- )- self._assert_serializes_ok(r, spider=self.spider)- request_dict = r.to_dict(spider=self.spider)- assert request_dict["callback"] == "_MethodsSpider__parse_item_reference"- assert request_dict["errback"] == "_MethodsSpider__handle_error_reference"-- def test_private_callback_serialization(self):- r = Request(- "http://www.example.com",- callback=self.spider._MethodsSpider__parse_item_private, # type: ignore[attr-defined]- errback=self.spider.handle_error,- )- self._assert_serializes_ok(r, spider=self.spider)-- def test_mixin_private_callback_serialization(self):- r = Request(- "http://www.example.com",- callback=self.spider._SpiderMixin__mixin_callback, # type: ignore[attr-defined]- errback=self.spider.handle_error,- )- self._assert_serializes_ok(r, spider=self.spider)-- def test_delegated_callback_serialization(self):- r = Request(- "http://www.example.com",- callback=self.spider.delegated_callback,- errback=self.spider.handle_error,- )- self._assert_serializes_ok(r, spider=self.spider)-- def test_unserializable_callback1(self):- r = Request("http://www.example.com", callback=lambda x: x) # type: ignore[misc]- with pytest.raises(- ValueError, match="is not an instance method in: <MethodsSpider"- ):- r.to_dict(spider=self.spider)-- def test_unserializable_callback2(self):- r = Request("http://www.example.com", callback=self.spider.parse_item)- with pytest.raises(ValueError, match="is not an instance method in: None"):- r.to_dict(spider=None)-- def test_unserializable_callback3(self):- """Parser method is removed or replaced dynamically."""-- class MySpider(Spider):- name = "my_spider"-- def parse(self, response: Response) -> None:- pass-- spider = MySpider()- r = Request("http://www.example.com", callback=spider.parse)- spider.parse = None # type: ignore[method-assign,assignment]- with pytest.raises(ValueError, match="is not an instance method in: <MySpider"):- r.to_dict(spider=spider)-- def test_callback_not_available(self):- """Callback method is not available in the spider passed to from_dict"""- spider = SpiderDelegation()- r = Request("http://www.example.com", callback=spider.delegated_callback)- d = r.to_dict(spider=spider) # type: ignore[arg-type]- with pytest.raises(- ValueError, match="Method 'delegated_callback' not found in: <Spider"- ):- request_from_dict(d, spider=Spider("foo"))+def spider() -> MethodsSpider:+ return MethodsSpider()+++def _assert_serializes_ok(request: Request, spider: Spider | None = None) -> None:+ d = request.to_dict(spider=spider)+ request2 = request_from_dict(d, spider=spider)+ _assert_same_request(request, request2)+++def _assert_same_request(r1: Request, r2: Request) -> None:+ assert r1.__class__ == r2.__class__+ assert r1.url == r2.url+ assert r1.callback == r2.callback+ assert r1.errback == r2.errback+ assert r1.method == r2.method+ assert r1.body == r2.body+ assert r1.headers == r2.headers+ assert r1.cookies == r2.cookies+ assert r1.meta == r2.meta+ assert r1.cb_kwargs == r2.cb_kwargs+ assert r1.encoding == r2.encoding+ assert r1._encoding == r2._encoding+ assert r1.priority == r2.priority+ assert r1.dont_filter == r2.dont_filter+ assert r1.flags == r2.flags+ if isinstance(r1, JsonRequest):+ assert isinstance(r2, JsonRequest)+ assert r1.dumps_kwargs == r2.dumps_kwargs+++def test_basic() -> None:+ r = Request("http://www.example.com")+ _assert_serializes_ok(r)+++def test_all_attributes(spider: MethodsSpider) -> None:+ r = Request(+ url="http://www.example.com",+ callback=spider.parse_item,+ errback=spider.handle_error,+ method="POST",+ body=b"some body",+ headers={"content-encoding": "text/html; charset=latin-1"},+ cookies={"currency": "руб"},+ encoding="latin-1",+ priority=20,+ meta={"a": "b"},+ cb_kwargs={"k": "v"},+ flags=["testFlag"],+ )+ _assert_serializes_ok(r, spider=spider)+++def test_latin1_body() -> None:+ r = Request("http://www.example.com", body=b"\xa3")+ _assert_serializes_ok(r)+++def test_utf8_body() -> None:+ r = Request("http://www.example.com", body=b"\xc2\xa3")+ _assert_serializes_ok(r)+++def test_request_class(spider: MethodsSpider) -> None:+ r1 = CustomRequest("http://www.example.com")+ _assert_serializes_ok(r1, spider=spider)+ r2 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})+ _assert_serializes_ok(r2, spider=spider)+++def test_callback_serialization(spider: MethodsSpider) -> None:+ r = Request(+ "http://www.example.com",+ callback=spider.parse_item,+ errback=spider.handle_error,+ )+ _assert_serializes_ok(r, spider=spider)+++def test_reference_callback_serialization(spider: MethodsSpider) -> None:+ r = Request(+ "http://www.example.com",+ callback=spider.parse_item_reference, # type: ignore[arg-type,misc]+ errback=spider.handle_error_reference, # type: ignore[arg-type,misc]+ )+ _assert_serializes_ok(r, spider=spider)+ request_dict = r.to_dict(spider=spider)+ assert request_dict["callback"] == "parse_item_reference"+ assert request_dict["errback"] == "handle_error_reference"+++def test_private_reference_callback_serialization(spider: MethodsSpider) -> None:+ r = Request(+ "http://www.example.com",+ callback=spider._MethodsSpider__parse_item_reference, # type: ignore[attr-defined]+ errback=spider._MethodsSpider__handle_error_reference, # type: ignore[attr-defined]+ )+ _assert_serializes_ok(r, spider=spider)+ request_dict = r.to_dict(spider=spider)+ assert request_dict["callback"] == "_MethodsSpider__parse_item_reference"+ assert request_dict["errback"] == "_MethodsSpider__handle_error_reference"+++def test_private_callback_serialization(spider: MethodsSpider) -> None:+ r = Request(+ "http://www.example.com",+ callback=spider._MethodsSpider__parse_item_private, # type: ignore[attr-defined]+ errback=spider.handle_error,+ )+ _assert_serializes_ok(r, spider=spider)+++def test_mixin_private_callback_serialization(spider: MethodsSpider) -> None:+ r = Request(+ "http://www.example.com",+ callback=spider._SpiderMixin__mixin_callback, # type: ignore[attr-defined]+ errback=spider.handle_error,+ )+ _assert_serializes_ok(r, spider=spider)+++def test_delegated_callback_serialization(spider: MethodsSpider) -> None:+ r = Request(+ "http://www.example.com",+ callback=spider.delegated_callback,+ errback=spider.handle_error,+ )+ _assert_serializes_ok(r, spider=spider)+++def test_unserializable_callback1(spider: MethodsSpider) -> None:+ r = Request("http://www.example.com", callback=lambda x: x) # type: ignore[misc]+ with pytest.raises(+ ValueError, match="is not an instance method in: <MethodsSpider"+ ):+ r.to_dict(spider=spider)+++def test_unserializable_callback2(spider: MethodsSpider) -> None:+ r = Request("http://www.example.com", callback=spider.parse_item)+ with pytest.raises(ValueError, match="is not an instance method in: None"):+ r.to_dict(spider=None)+++def test_unserializable_callback3() -> None:+ """Parser method is removed or replaced dynamically."""++ class MySpider(Spider):+ name = "my_spider"++ def parse(self, response: Response) -> None:+ pass++ spider = MySpider()+ r = Request("http://www.example.com", callback=spider.parse)+ spider.parse = None # type: ignore[method-assign,assignment]+ with pytest.raises(ValueError, match="is not an instance method in: <MySpider"):+ r.to_dict(spider=spider)+++def test_callback_not_available() -> None:+ """Callback method is not available in the spider passed to from_dict"""+ spider = SpiderDelegation()+ r = Request("http://www.example.com", callback=spider.delegated_callback)+ d = r.to_dict(spider=spider) # type: ignore[arg-type]+ with pytest.raises(+ ValueError, match="Method 'delegated_callback' not found in: <Spider"+ ):+ request_from_dict(d, spider=Spider("foo"))class SpiderMixin:diff --git a/tests/test_request_left.py b/tests/test_request_left.pyindex 46a16ad1e..e95f1d7d9 100644--- a/tests/test_request_left.py+++ b/tests/test_request_left.py@@ -32,31 +32,33 @@ class SignalCatcherSpider(Spider):self.caught_times += 1-class TestCatching:- @inline_callbacks_test- def test_success(self, mockserver: MockServer):- crawler = get_crawler(SignalCatcherSpider)- yield crawler.crawl(mockserver.url("/status?n=200"))- assert isinstance(crawler.spider, SignalCatcherSpider)- assert crawler.spider.caught_times == 1-- @inline_callbacks_test- def test_timeout(self, mockserver: MockServer):- crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})- yield crawler.crawl(mockserver.url("/delay?n=0.2"))- assert isinstance(crawler.spider, SignalCatcherSpider)- assert crawler.spider.caught_times == 1-- @inline_callbacks_test- def test_disconnect(self, mockserver: MockServer):- crawler = get_crawler(SignalCatcherSpider)- yield crawler.crawl(mockserver.url("/drop"))- assert isinstance(crawler.spider, SignalCatcherSpider)- assert crawler.spider.caught_times == 1-- @inline_callbacks_test- def test_noconnect(self):- crawler = get_crawler(SignalCatcherSpider)- yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")- assert isinstance(crawler.spider, SignalCatcherSpider)- assert crawler.spider.caught_times == 1+@inline_callbacks_test+def test_success(mockserver: MockServer):+ crawler = get_crawler(SignalCatcherSpider)+ yield crawler.crawl(mockserver.url("/status?n=200"))+ assert isinstance(crawler.spider, SignalCatcherSpider)+ assert crawler.spider.caught_times == 1+++@inline_callbacks_test+def test_timeout(mockserver: MockServer):+ crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})+ yield crawler.crawl(mockserver.url("/delay?n=0.2"))+ assert isinstance(crawler.spider, SignalCatcherSpider)+ assert crawler.spider.caught_times == 1+++@inline_callbacks_test+def test_disconnect(mockserver: MockServer):+ crawler = get_crawler(SignalCatcherSpider)+ yield crawler.crawl(mockserver.url("/drop"))+ assert isinstance(crawler.spider, SignalCatcherSpider)+ assert crawler.spider.caught_times == 1+++@inline_callbacks_test+def test_noconnect():+ crawler = get_crawler(SignalCatcherSpider)+ yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")+ assert isinstance(crawler.spider, SignalCatcherSpider)+ assert crawler.spider.caught_times == 1diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.pyindex 7c9f717e4..fa37cfe03 100644--- a/tests/test_responsetypes.py+++ b/tests/test_responsetypes.py@@ -11,119 +11,125 @@ from scrapy.http import (from scrapy.responsetypes import responsetypes-class TestResponseTypes:- def test_from_filename(self):- mappings = [- ("data.bin", Response),- ("file.txt", TextResponse),- ("file.xml.gz", Response),- ("file.xml", XmlResponse),- ("file.html", HtmlResponse),- ("file.unknownext", Response),- ]- for source, cls in mappings:- retcls = responsetypes.from_filename(source)- assert retcls is cls, f"{source} ==> {retcls} != {cls}"-- def test_from_content_disposition(self):- mappings = [- (b'attachment; filename="data.xml"', XmlResponse),- (b"attachment; filename=data.xml", XmlResponse),- ("attachment;filename=data£.tar.gz".encode(), Response),- ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),- ("attachment;filename=data高.doc".encode("gbk"), Response),- ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),- (- "attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"),- XmlResponse,- ),- ]- for source, cls in mappings:- retcls = responsetypes.from_content_disposition(source)- assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"-- def test_from_content_disposition_no_filename(self):- assert responsetypes.from_content_disposition(b"attachment") is Response-- def test_from_content_type(self):- mappings: list[tuple[str | bytes, type[Response]]] = [- ("text/html; charset=UTF-8", HtmlResponse),- ("text/xml; charset=UTF-8", XmlResponse),- ("application/xhtml+xml; charset=UTF-8", HtmlResponse),- ("application/vnd.wap.xhtml+xml; charset=utf-8", HtmlResponse),- ("application/xml; charset=UTF-8", XmlResponse),- ("application/octet-stream", Response),- ("application/json; encoding=UTF8;charset=UTF-8", JsonResponse),- ("application/x-json; encoding=UTF8;charset=UTF-8", JsonResponse),- ("application/json-amazonui-streaming;charset=UTF-8", JsonResponse),- (b"application/x-download; filename=\x80dummy.txt", Response),- ]- for source, cls in mappings:- retcls = responsetypes.from_content_type(source)- assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"-- def test_from_body(self):- mappings = [- (b"\x03\x02\xdf\xdd\x23", Response),- (b"Some plain text\ndata with tabs\t and null bytes\0", TextResponse),- (b"<html><head><title>Hello</title></head>", HtmlResponse),- # https://codersblock.com/blog/the-smallest-valid-html5-page/- (b"<!DOCTYPE html>\n<title>.</title>", HtmlResponse),- (b'<?xml version="1.0" encoding="utf-8"', XmlResponse),- ]- for source, cls in mappings:- retcls = responsetypes.from_body(source)- assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"-- def test_from_headers(self):- mappings = [- ({"Content-Type": ["text/html; charset=utf-8"]}, HtmlResponse),- (- {- "Content-Type": ["text/html; charset=utf-8"],- "Content-Encoding": ["gzip"],- },- Response,- ),- (- {- "Content-Type": ["application/octet-stream"],- "Content-Disposition": ["attachment; filename=data.txt"],- },- TextResponse,- ),- ]- for source, cls in mappings:- source = Headers(source)- retcls = responsetypes.from_headers(source)- assert retcls is cls, f"{source} ==> {retcls} != {cls}"-- def test_from_args(self):- # TODO: add more tests that check precedence between the different arguments- mappings: list[tuple[dict[str, Any], type[Response]]] = [- ({"url": "http://www.example.com/data.csv"}, TextResponse),- # headers takes precedence over url- (- {- "headers": Headers({"Content-Type": ["text/html; charset=utf-8"]}),- "url": "http://www.example.com/item/",- },- HtmlResponse,- ),- (- {- "headers": Headers(- {"Content-Disposition": ['attachment; filename="data.xml.gz"']}- ),- "url": "http://www.example.com/page/",- },- Response,- ),- ]- for source, cls in mappings:- retcls = responsetypes.from_args(**source)- assert retcls is cls, f"{source} ==> {retcls} != {cls}"-- def test_custom_mime_types_loaded(self):- # check that mime.types files shipped with scrapy are loaded- assert responsetypes.mimetypes.guess_type("x.scrapytest")[0] == "x-scrapy/test"+def test_from_filename() -> None:+ mappings = [+ ("data.bin", Response),+ ("file.txt", TextResponse),+ ("file.xml.gz", Response),+ ("file.xml", XmlResponse),+ ("file.html", HtmlResponse),+ ("file.unknownext", Response),+ ]+ for source, cls in mappings:+ retcls = responsetypes.from_filename(source)+ assert retcls is cls, f"{source} ==> {retcls} != {cls}"+++def test_from_content_disposition() -> None:+ mappings = [+ (b'attachment; filename="data.xml"', XmlResponse),+ (b"attachment; filename=data.xml", XmlResponse),+ ("attachment;filename=data£.tar.gz".encode(), Response),+ ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),+ ("attachment;filename=data高.doc".encode("gbk"), Response),+ ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),+ (+ "attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"),+ XmlResponse,+ ),+ ]+ for source, cls in mappings:+ retcls = responsetypes.from_content_disposition(source)+ assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"+++def test_from_content_disposition_no_filename() -> None:+ assert responsetypes.from_content_disposition(b"attachment") is Response+++def test_from_content_type() -> None:+ mappings: list[tuple[str | bytes, type[Response]]] = [+ ("text/html; charset=UTF-8", HtmlResponse),+ ("text/xml; charset=UTF-8", XmlResponse),+ ("application/xhtml+xml; charset=UTF-8", HtmlResponse),+ ("application/vnd.wap.xhtml+xml; charset=utf-8", HtmlResponse),+ ("application/xml; charset=UTF-8", XmlResponse),+ ("application/octet-stream", Response),+ ("application/json; encoding=UTF8;charset=UTF-8", JsonResponse),+ ("application/x-json; encoding=UTF8;charset=UTF-8", JsonResponse),+ ("application/json-amazonui-streaming;charset=UTF-8", JsonResponse),+ (b"application/x-download; filename=\x80dummy.txt", Response),+ ]+ for source, cls in mappings:+ retcls = responsetypes.from_content_type(source)+ assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"+++def test_from_body() -> None:+ mappings = [+ (b"\x03\x02\xdf\xdd\x23", Response),+ (b"Some plain text\ndata with tabs\t and null bytes\0", TextResponse),+ (b"<html><head><title>Hello</title></head>", HtmlResponse),+ # https://codersblock.com/blog/the-smallest-valid-html5-page/+ (b"<!DOCTYPE html>\n<title>.</title>", HtmlResponse),+ (b'<?xml version="1.0" encoding="utf-8"', XmlResponse),+ ]+ for source, cls in mappings:+ retcls = responsetypes.from_body(source)+ assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"+++def test_from_headers() -> None:+ mappings = [+ ({"Content-Type": ["text/html; charset=utf-8"]}, HtmlResponse),+ (+ {+ "Content-Type": ["text/html; charset=utf-8"],+ "Content-Encoding": ["gzip"],+ },+ Response,+ ),+ (+ {+ "Content-Type": ["application/octet-stream"],+ "Content-Disposition": ["attachment; filename=data.txt"],+ },+ TextResponse,+ ),+ ]+ for source, cls in mappings:+ source = Headers(source)+ retcls = responsetypes.from_headers(source)+ assert retcls is cls, f"{source} ==> {retcls} != {cls}"+++def test_from_args() -> None:+ # TODO: add more tests that check precedence between the different arguments+ mappings: list[tuple[dict[str, Any], type[Response]]] = [+ ({"url": "http://www.example.com/data.csv"}, TextResponse),+ # headers takes precedence over url+ (+ {+ "headers": Headers({"Content-Type": ["text/html; charset=utf-8"]}),+ "url": "http://www.example.com/item/",+ },+ HtmlResponse,+ ),+ (+ {+ "headers": Headers(+ {"Content-Disposition": ['attachment; filename="data.xml.gz"']}+ ),+ "url": "http://www.example.com/page/",+ },+ Response,+ ),+ ]+ for source, cls in mappings:+ retcls = responsetypes.from_args(**source)+ assert retcls is cls, f"{source} ==> {retcls} != {cls}"+++def test_custom_mime_types_loaded() -> None:+ # check that mime.types files shipped with scrapy are loaded+ assert responsetypes.mimetypes.guess_type("x.scrapytest")[0] == "x-scrapy/test"diff --git a/tests/test_signalmanager.py b/tests/test_signalmanager.pyindex ce4d97adb..cf8cd489f 100644--- a/tests/test_signalmanager.py+++ b/tests/test_signalmanager.py@@ -1,21 +1,20 @@from scrapy.signalmanager import SignalManager-class TestSignalManager:- def test_disconnect_all(self):- signal = object()- sender = object()- sm = SignalManager(sender)+def test_disconnect_all() -> None:+ signal = object()+ sender = object()+ sm = SignalManager(sender)- calls = []+ calls = []- def handler():- calls.append(1)+ def handler():+ calls.append(1)- sm.connect(handler, signal)- sm.send_catch_log(signal)- assert calls == [1]+ sm.connect(handler, signal)+ sm.send_catch_log(signal)+ assert calls == [1]- sm.disconnect_all(signal)- sm.send_catch_log(signal)- assert calls == [1] # handler no longer called after disconnect_all+ sm.disconnect_all(signal)+ sm.send_catch_log(signal)+ assert calls == [1] # handler no longer called after disconnect_alldiff --git a/tests/test_spider_start.py b/tests/test_spider_start.pyindex fb72ece6c..482b42d68 100644--- a/tests/test_spider_start.py+++ b/tests/test_spider_start.py@@ -21,79 +21,83 @@ ITEM_A = {"id": "a"}ITEM_B = {"id": "b"}-class TestMain:- async def _test_spider(- self, spider: type[Spider], expected_items: list[Any] | None = None- ) -> None:- actual_items = []- expected_items = [] if expected_items is None else expected_items-- def track_item(item, response, spider):- actual_items.append(item)-- crawler = get_crawler(spider)- crawler.signals.connect(track_item, signals.item_scraped)- await crawler.crawl_async()- assert crawler.stats.get_value("finish_reason") == "finished"- assert actual_items == expected_items-- @coroutine_test- async def test_start_urls(self):- class TestSpider(Spider):- name = "test"- start_urls = ["data:,"]-- async def parse(self, response):- yield ITEM_A-- await self._test_spider(TestSpider, [ITEM_A])-- @coroutine_test- async def test_start(self):- class TestSpider(Spider):- name = "test"-- async def start(self):- yield ITEM_A-- await self._test_spider(TestSpider, [ITEM_A])-- @coroutine_test- async def test_start_subclass(self):- class BaseSpider(Spider):- async def start(self):- yield ITEM_A-- class TestSpider(BaseSpider):- name = "test"-- await self._test_spider(TestSpider, [ITEM_A])-- async def _test_start(- self,- start_: Callable[[Any], AsyncIterator[Any]],- expected_items: list[Any] | None = None,- ) -> None:- class TestSpider(Spider):- name = "test"- start = start_-- await self._test_spider(TestSpider, expected_items)-- @pytest.mark.only_asyncio- @coroutine_test- async def test_asyncio_delayed(self):- async def start(spider):- await sleep(SLEEP_SECONDS)+async def _test_spider(+ spider: type[Spider], expected_items: list[Any] | None = None+) -> None:+ actual_items = []+ expected_items = [] if expected_items is None else expected_items++ def track_item(item, response, spider):+ actual_items.append(item)++ crawler = get_crawler(spider)+ crawler.signals.connect(track_item, signals.item_scraped)+ await crawler.crawl_async()+ assert crawler.stats.get_value("finish_reason") == "finished"+ assert actual_items == expected_items+++@coroutine_test+async def test_start_urls() -> None:+ class TestSpider(Spider):+ name = "test"+ start_urls = ["data:,"]++ async def parse(self, response):yield ITEM_A- await self._test_start(start, [ITEM_A])+ await _test_spider(TestSpider, [ITEM_A])+++@coroutine_test+async def test_start() -> None:+ class TestSpider(Spider):+ name = "test"++ async def start(self):+ yield ITEM_A++ await _test_spider(TestSpider, [ITEM_A])+- @pytest.mark.requires_reactor # needs a reactor for twisted_sleep()- @coroutine_test- async def test_twisted_delayed(self):- async def start(spider):- await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS))+@coroutine_test+async def test_start_subclass() -> None:+ class BaseSpider(Spider):+ async def start(self):yield ITEM_A- await self._test_start(start, [ITEM_A])+ class TestSpider(BaseSpider):+ name = "test"++ await _test_spider(TestSpider, [ITEM_A])+++async def _test_start(+ start_: Callable[[Any], AsyncIterator[Any]],+ expected_items: list[Any] | None = None,+) -> None:+ class TestSpider(Spider):+ name = "test"+ start = start_++ await _test_spider(TestSpider, expected_items)++[email protected]_asyncio+@coroutine_test+async def test_asyncio_delayed() -> None:+ async def start(spider):+ await sleep(SLEEP_SECONDS)+ yield ITEM_A++ await _test_start(start, [ITEM_A])++[email protected]_reactor # needs a reactor for twisted_sleep()+@coroutine_test+async def test_twisted_delayed() -> None:+ async def start(spider):+ await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS))+ yield ITEM_A++ await _test_start(start, [ITEM_A])diff --git a/tests/test_spidermiddleware_metacopy.py b/tests/test_spidermiddleware_metacopy.pyindex 8fe6e983e..dac0d6b70 100644--- a/tests/test_spidermiddleware_metacopy.py+++ b/tests/test_spidermiddleware_metacopy.py@@ -41,82 +41,88 @@ def process(return list(mw.process_spider_output(response, requests))-class TestInternalKeysCheck:- def test_no_warning_for_clean_request(- self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture- ) -> None:- req = Request("https://example.com/1", meta={"my_key": "value"})- with caplog.at_level(WARNING):- process(mw, [req])- assert not caplog.records-- def test_warns_on_internal_key(- self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture- ) -> None:+def test_no_warning_for_clean_request(+ mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+) -> None:+ req = Request("https://example.com/1", meta={"my_key": "value"})+ with caplog.at_level(WARNING):+ process(mw, [req])+ assert not caplog.records+++def test_warns_on_internal_key(+ mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+) -> None:+ req = Request("https://example.com/1", meta={"retry_times": 1})+ with caplog.at_level(WARNING):+ process(mw, [req])+ assert len(caplog.records) == 1+ assert "retry_times" in caplog.text+ assert "https://example.com/1" in caplog.text+++def test_warns_once_across_multiple_requests(+ mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+) -> None:+ reqs = [+ Request("https://example.com/1", meta={"retry_times": 1}),+ Request("https://example.com/2", meta={"retry_times": 2}),+ ]+ with caplog.at_level(WARNING):+ process(mw, reqs)+ assert len(caplog.records) == 1+++def test_reports_all_found_keys(+ mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+) -> None:+ req = Request(+ "https://example.com/1",+ meta={"retry_times": 1, "redirect_times": 2},+ )+ with caplog.at_level(WARNING):+ process(mw, [req])+ assert "retry_times" in caplog.text+ assert "redirect_times" in caplog.text+++def test_includes_source_response_in_message(+ mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+) -> None:+ response = make_response("https://source.example.com")+ req = Request("https://example.com/1", meta={"retry_times": 1})+ with caplog.at_level(WARNING):+ process(mw, [req], response=response)+ assert "https://source.example.com" in caplog.text+++def test_skip_keys_setting(caplog: pytest.LogCaptureFixture) -> None:+ with caplog.at_level(WARNING):+ crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})+ mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)req = Request("https://example.com/1", meta={"retry_times": 1})- with caplog.at_level(WARNING):- process(mw, [req])- assert len(caplog.records) == 1- assert "retry_times" in caplog.text- assert "https://example.com/1" in caplog.text-- def test_warns_once_across_multiple_requests(- self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture- ) -> None:- reqs = [- Request("https://example.com/1", meta={"retry_times": 1}),- Request("https://example.com/2", meta={"retry_times": 2}),- ]- with caplog.at_level(WARNING):- process(mw, reqs)- assert len(caplog.records) == 1-- def test_reports_all_found_keys(- self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture- ) -> None:+ process(mw, [req])+ assert not caplog.records+++def test_skip_keys_setting_partial(caplog: pytest.LogCaptureFixture) -> None:+ with caplog.at_level(WARNING):+ crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})+ mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)req = Request("https://example.com/1",meta={"retry_times": 1, "redirect_times": 2},)- with caplog.at_level(WARNING):- process(mw, [req])- assert "retry_times" in caplog.text- assert "redirect_times" in caplog.text-- def test_includes_source_response_in_message(- self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture- ) -> None:- response = make_response("https://source.example.com")- req = Request("https://example.com/1", meta={"retry_times": 1})- with caplog.at_level(WARNING):- process(mw, [req], response=response)- assert "https://source.example.com" in caplog.text-- def test_skip_keys_setting(self, caplog: pytest.LogCaptureFixture) -> None:- with caplog.at_level(WARNING):- crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})- mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)- req = Request("https://example.com/1", meta={"retry_times": 1})- process(mw, [req])- assert not caplog.records-- def test_skip_keys_setting_partial(self, caplog: pytest.LogCaptureFixture) -> None:- with caplog.at_level(WARNING):- crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})- mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)- req = Request(- "https://example.com/1",- meta={"retry_times": 1, "redirect_times": 2},- )- process(mw, [req])- assert len(caplog.records) == 1- assert "retry_times" not in caplog.text- assert "redirect_times" in caplog.text-- def test_no_warning_for_start_requests(- self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture- ) -> None:- req = Request("https://example.com/1", meta={"retry_times": 1})- with caplog.at_level(WARNING):- list(mw.process_spider_output(None, [req]))- assert not caplog.records+ process(mw, [req])+ assert len(caplog.records) == 1+ assert "retry_times" not in caplog.text+ assert "redirect_times" in caplog.text+++def test_no_warning_for_start_requests(+ mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture+) -> None:+ req = Request("https://example.com/1", meta={"retry_times": 1})+ with caplog.at_level(WARNING):+ list(mw.process_spider_output(None, [req]))+ assert not caplog.recordsdiff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.pyindex 46414bb59..bea58aaaf 100644--- a/tests/test_spidermiddleware_process_start.py+++ b/tests/test_spidermiddleware_process_start.py@@ -62,69 +62,73 @@ class ModernWrapSpiderMiddleware:yield ITEM_C-class TestMain:- async def _test(- self,- spider_middlewares: list[type],- spider_cls: type[Spider],- expected_items: list[Any],- ) -> None:- actual_items = []-- def track_item(item: Any, response: Response, spider: Spider) -> None:- actual_items.append(item)-- settings = {- "SPIDER_MIDDLEWARES": {cls: n for n, cls in enumerate(spider_middlewares)},- }- crawler = get_crawler(spider_cls, settings_dict=settings)- crawler.signals.connect(track_item, signals.item_scraped)- await crawler.crawl_async()- assert crawler.stats.get_value("finish_reason") == "finished"- assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"-- async def _test_wrap(- self,- spider_middleware: type,- spider_cls: type[Spider],- expected_items: list[Any] | None = None,- ) -> None:- expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]- await self._test([spider_middleware], spider_cls, expected_items)-- @coroutine_test- async def test_modern_mw_modern_spider(self):- await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)-- async def _test_sleep(self, spider_middlewares: list[type]) -> None:- class TestSpider(Spider):- name = "test"-- async def start(self) -> AsyncIterator[Any]:- yield ITEM_A-- await self._test(spider_middlewares, TestSpider, [ITEM_A])-- @pytest.mark.only_asyncio- @coroutine_test- async def test_asyncio_sleep_single(self):- await self._test_sleep([AsyncioSleepSpiderMiddleware])-- @pytest.mark.only_asyncio- @coroutine_test- async def test_asyncio_sleep_multiple(self):- await self._test_sleep(- [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware]- )-- @pytest.mark.requires_reactor # needs a reactor for twisted_sleep()- @coroutine_test- async def test_twisted_sleep_single(self):- await self._test_sleep([TwistedSleepSpiderMiddleware])-- @pytest.mark.requires_reactor # needs a reactor for twisted_sleep()- @coroutine_test- async def test_twisted_sleep_multiple(self):- await self._test_sleep(- [NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware]- )+async def _test(+ spider_middlewares: list[type],+ spider_cls: type[Spider],+ expected_items: list[Any],+) -> None:+ actual_items = []++ def track_item(item: Any, response: Response, spider: Spider) -> None:+ actual_items.append(item)++ settings = {+ "SPIDER_MIDDLEWARES": {cls: n for n, cls in enumerate(spider_middlewares)},+ }+ crawler = get_crawler(spider_cls, settings_dict=settings)+ crawler.signals.connect(track_item, signals.item_scraped)+ await crawler.crawl_async()+ assert crawler.stats.get_value("finish_reason") == "finished"+ assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"+++async def _test_wrap(+ spider_middleware: type,+ spider_cls: type[Spider],+ expected_items: list[Any] | None = None,+) -> None:+ expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]+ await _test([spider_middleware], spider_cls, expected_items)+++@coroutine_test+async def test_modern_mw_modern_spider() -> None:+ await _test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)+++async def _test_sleep(spider_middlewares: list[type]) -> None:+ class TestSpider(Spider):+ name = "test"++ async def start(self) -> AsyncIterator[Any]:+ yield ITEM_A++ await _test(spider_middlewares, TestSpider, [ITEM_A])++[email protected]_asyncio+@coroutine_test+async def test_asyncio_sleep_single() -> None:+ await _test_sleep([AsyncioSleepSpiderMiddleware])++[email protected]_asyncio+@coroutine_test+async def test_asyncio_sleep_multiple() -> None:+ await _test_sleep(+ [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware]+ )++[email protected]_reactor # needs a reactor for twisted_sleep()+@coroutine_test+async def test_twisted_sleep_single() -> None:+ await _test_sleep([TwistedSleepSpiderMiddleware])++[email protected]_reactor # needs a reactor for twisted_sleep()+@coroutine_test+async def test_twisted_sleep_multiple() -> None:+ await _test_sleep(+ [NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware]+ )diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.pyindex e879f3a32..26c62f015 100644--- a/tests/test_spidermiddleware_start.py+++ b/tests/test_spidermiddleware_start.py@@ -13,30 +13,29 @@ if TYPE_CHECKING:from collections.abc import AsyncIterator-class TestMiddleware:- @coroutine_test- async def test_async(self):- crawler = get_crawler(Spider)- mw = build_from_crawler(StartSpiderMiddleware, crawler)-- async def start() -> AsyncIterator[Request]:- yield Request("data:,1")- yield Request("data:,2", meta={"is_start_request": True})- yield Request("data:,2", meta={"is_start_request": False})- yield Request("data:,2", meta={"is_start_request": "foo"})-- result = [- request.meta["is_start_request"]- async for request in mw.process_start(start())- ]- assert result == [True, True, False, "foo"]-- def test_spider_output_not_marked(self):- # Requests from a non-None response (spider output) are not flagged.- crawler = get_crawler(Spider)- mw = build_from_crawler(StartSpiderMiddleware, crawler)- response = Response("data:,")- request = Request("data:,1")- out = list(mw.process_spider_output(response, [request]))- assert out == [request]- assert "is_start_request" not in request.meta+@coroutine_test+async def test_async() -> None:+ crawler = get_crawler(Spider)+ mw = build_from_crawler(StartSpiderMiddleware, crawler)++ async def start() -> AsyncIterator[Request]:+ yield Request("data:,1")+ yield Request("data:,2", meta={"is_start_request": True})+ yield Request("data:,2", meta={"is_start_request": False})+ yield Request("data:,2", meta={"is_start_request": "foo"})++ result = [+ request.meta["is_start_request"] async for request in mw.process_start(start())+ ]+ assert result == [True, True, False, "foo"]+++def test_spider_output_not_marked() -> None:+ # Requests from a non-None response (spider output) are not flagged.+ crawler = get_crawler(Spider)+ mw = build_from_crawler(StartSpiderMiddleware, crawler)+ response = Response("data:,")+ request = Request("data:,1")+ out = list(mw.process_spider_output(response, [request]))+ assert out == [request]+ assert "is_start_request" not in request.meta