Scrapy, a fast high-level web crawling & scraping framework for Python.
Add type hints to test_link.py and test_downloadermiddleware_stats.py (#7785)
SpiliosDmk13be37e4b107
diff --git a/pyproject.toml b/pyproject.tomlindex 4220b12ed..5ed07a9ce 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -136,7 +136,6 @@ module = ["tests.test_downloadermiddleware_redirect_metarefresh","tests.test_downloadermiddleware_retry","tests.test_downloadermiddleware_robotstxt",- "tests.test_downloadermiddleware_stats","tests.test_downloaderslotssettings","tests.test_dupefilters","tests.test_engine_loop",@@ -154,11 +153,9 @@ module = ["tests.test_http_response","tests.test_http_response_text","tests.test_item",- "tests.test_link","tests.test_linkextractors","tests.test_loader","tests.test_logformatter",- "tests.test_logstats","tests.test_mail","tests.test_pipeline_crawl","tests.test_pipeline_files",diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.pyindex 5609360a7..b8fda25b4 100644--- a/tests/test_downloadermiddleware_stats.py+++ b/tests/test_downloadermiddleware_stats.py@@ -1,3 +1,5 @@+from __future__ import annotations+import pytestfrom scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size@@ -12,8 +14,9 @@ class MyException(Exception):class TestDownloaderStats:- def setup_method(self):+ def setup_method(self) -> None:self.crawler = get_crawler(Spider)+ assert self.crawler.stats is not Noneself.mw = DownloaderStats(self.crawler.stats)self.crawler.stats.open_spider()@@ -21,20 +24,21 @@ class TestDownloaderStats:self.req = Request("http://scrapytest.org")self.res = Response("http://scrapytest.org", status=400)- def assertStatsEqual(self, key, value):+ def assertStatsEqual(self, key: str, value: object) -> None:+ assert self.crawler.stats is not Noneassert self.crawler.stats.get_value(key) == value, str(self.crawler.stats.get_stats())- def test_process_request(self):+ def test_process_request(self) -> None:self.mw.process_request(self.req)self.assertStatsEqual("downloader/request_count", 1)- def test_process_response(self):+ def test_process_response(self) -> None:self.mw.process_response(self.req, self.res)self.assertStatsEqual("downloader/response_count", 1)- def test_process_exception(self):+ def test_process_exception(self) -> None:self.mw.process_exception(self.req, MyException())self.assertStatsEqual("downloader/exception_count", 1)self.assertStatsEqual(@@ -42,14 +46,17 @@ class TestDownloaderStats:1,)- def test_from_crawler_not_configured(self):+ def test_from_crawler_not_configured(self) -> None:crawler = get_crawler(Spider, {"DOWNLOADER_STATS": False})with pytest.raises(NotConfigured):DownloaderStats.from_crawler(crawler)- def teardown_method(self):+ def teardown_method(self) -> None:+ assert self.crawler.stats is not Noneself.crawler.stats.close_spider()-def test_get_header_size_non_list_value():- assert get_header_size({"Content-Type": "text/html"}) == 0+def test_get_header_size_non_list_value() -> None:+ # Deliberately passing a non-list/tuple header value to make sure+ # get_header_size() degrades gracefully instead of raising.+ assert get_header_size({"Content-Type": "text/html"}) == 0 # type: ignore[dict-item]diff --git a/tests/test_link.py b/tests/test_link.pyindex 0eeffe12b..40c53edd3 100644--- a/tests/test_link.py+++ b/tests/test_link.py@@ -1,18 +1,20 @@+from __future__ import annotations+import pytestfrom scrapy.link import Linkclass TestLink:- def _assert_same_links(self, link1, link2):+ def _assert_same_links(self, link1: Link, link2: Link) -> None:assert link1 == link2assert hash(link1) == hash(link2)- def _assert_different_links(self, link1, link2):+ def _assert_different_links(self, link1: Link, link2: Link) -> None:assert link1 != link2assert hash(link1) != hash(link2)- def test_eq_and_hash(self):+ def test_eq_and_hash(self) -> None:l1 = Link("http://www.example.com")l2 = Link("http://www.example.com/other")l3 = Link("http://www.example.com")@@ -45,17 +47,17 @@ class TestLink:self._assert_different_links(l7, l9)self._assert_different_links(l7, l10)- def test_repr(self):+ def test_repr(self) -> None:l1 = Link("http://www.example.com", text="test", fragment="something", nofollow=True)l2 = eval(repr(l1))self._assert_same_links(l1, l2)- def test_bytes_url(self):+ def test_bytes_url(self) -> None:with pytest.raises(TypeError):- Link(b"http://www.example.com/\xc2\xa3")+ Link(b"http://www.example.com/\xc2\xa3") # type: ignore[arg-type]- def test_eq_non_link(self):+ def test_eq_non_link(self) -> None:url = "http://example.com"assert Link(url) != urldiff --git a/tests/test_logstats.py b/tests/test_logstats.pyindex 370728e6a..213681ad1 100644--- a/tests/test_logstats.py+++ b/tests/test_logstats.py@@ -1,3 +1,5 @@+from __future__ import annotations+from datetime import datetimeimport pytest@@ -9,16 +11,17 @@ from tests.utils.decorators import coroutine_testclass TestLogStats:- def setup_method(self):+ def setup_method(self) -> None:self.crawler = get_crawler(SimpleSpider)self.spider = self.crawler._create_spider("spidey")+ assert self.crawler.stats is not Noneself.stats = self.crawler.statsself.stats.set_value("response_received_count", 4802)self.stats.set_value("item_scraped_count", 3201)@coroutine_test- async def test_stats_calculations(self):+ async def test_stats_calculations(self) -> None:logstats = LogStats.from_crawler(self.crawler)with pytest.raises(AttributeError):@@ -56,7 +59,7 @@ class TestLogStats:assert self.stats.get_value("responses_per_minute") == 172.9assert self.stats.get_value("items_per_minute") == 116.4- def test_stats_calculations_no_time(self):+ def test_stats_calculations_no_time(self) -> None:"""The stat values should be None since the start and finish time arenot available."""@@ -65,7 +68,7 @@ class TestLogStats:assert self.stats.get_value("responses_per_minute") is Noneassert self.stats.get_value("items_per_minute") is None- def test_stats_calculation_no_elapsed_time(self):+ def test_stats_calculation_no_elapsed_time(self) -> None:"""The stat values should be None since the elapsed time is 0."""logstats = LogStats.from_crawler(self.crawler)self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))