Scrapy, a fast high-level web crawling & scraping framework for Python.
More granular untyped defs checking in tests. (#7712)
Andrey Rakhmatullin61f99f2df1d3
diff --git a/pyproject.toml b/pyproject.tomlindex ed2e99bea..ac9d37e78 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -94,7 +94,82 @@ untyped_calls_exclude = [[[tool.mypy.overrides]]module = "tests.*"allow_untyped_defs = true-allow_incomplete_defs = true # 48 errors+allow_incomplete_defs = true # 59 errors++# TODO+[[tool.mypy.overrides]]+module = [+ "tests.mockserver.*",+ "tests.spiders",+ "tests.test_closespider",+ "tests.test_cmdline",+ "tests.test_contracts",+ "tests.test_core_downloader",+ "tests.test_downloader_handler_twisted_ftp",+ "tests.test_downloadermiddleware_cookies",+ "tests.test_downloadermiddleware_httpauth",+ "tests.test_downloadermiddleware_httpcache",+ "tests.test_downloadermiddleware_httpcompression",+ "tests.test_downloadermiddleware_httpproxy",+ "tests.test_downloadermiddleware_offsite",+ "tests.test_downloadermiddleware_redirect",+ "tests.test_downloadermiddleware_redirect_base",+ "tests.test_downloadermiddleware_redirect_metarefresh",+ "tests.test_downloadermiddleware_retry",+ "tests.test_downloadermiddleware_robotstxt",+ "tests.test_downloadermiddleware_stats",+ "tests.test_downloaderslotssettings",+ "tests.test_dupefilters",+ "tests.test_engine_loop",+ "tests.test_exporters",+ "tests.test_extension_statsmailer",+ "tests.test_extension_throttle",+ "tests.test_feedexport",+ "tests.test_feedexport_postprocess",+ "tests.test_feedexport_storages",+ "tests.test_feedexport_uri_params",+ "tests.test_http2_client_protocol",+ "tests.test_http_headers",+ "tests.test_http_request",+ "tests.test_http_request_form",+ "tests.test_http_response",+ "tests.test_http_response_text",+ "tests.test_item",+ "tests.test_link",+ "tests.test_linkextractors",+ "tests.test_loader",+ "tests.test_logformatter",+ "tests.test_logstats",+ "tests.test_mail",+ "tests.test_pipeline_crawl",+ "tests.test_pipeline_files",+ "tests.test_pipeline_images",+ "tests.test_pipeline_media",+ "tests.test_pipelines",+ "tests.test_pqueues",+ "tests.test_request_attribute_binding",+ "tests.test_request_cb_kwargs",+ "tests.test_request_dict",+ "tests.test_request_left",+ "tests.test_robotstxt_interface",+ "tests.test_scheduler_base",+ "tests.test_settings",+ "tests.test_spider",+ "tests.test_spider_crawl",+ "tests.test_spidermiddleware_output_chain",+ "tests.test_spidermiddleware_process_start",+ "tests.test_spider_sitemap",+ "tests.test_squeues",+ "tests.test_squeues_request",+ "tests.test_stats",+ "tests.test_utils_datatypes",+ "tests.test_utils_decorators",+ "tests.test_utils_defer",+ "tests.test_utils_deprecate",+ "tests.test_utils_misc.test_return_with_argument_inside_generator",+ "tests.test_utils_python",+ "tests.test_utils_request",+]check_untyped_defs = false# Interface classes are hard to supportdiff --git a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.pyindex e000a23db..2f4f2d051 100644--- a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py+++ b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py@@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coroclass UppercasePipeline:- async def _open_spider(self, spider):+ async def _open_spider(self, spider: Spider) -> None:spider.logger.info("async pipeline opened!")await asyncio.sleep(0.1)diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.pyindex a6520ca22..7e3988d92 100644--- a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py@@ -10,6 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):"""name = "caching_hostname_resolver_spider"+ url: strasync def start(self):yield scrapy.Request(self.url)diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.pyindex eb2e1105c..6e4db1990 100644--- a/tests/CrawlerProcess/asyncio_deferred_signal.py+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py@@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coroclass UppercasePipeline:- async def _open_spider(self, spider):+ async def _open_spider(self, spider: Spider) -> None:spider.logger.info("async pipeline opened!")await asyncio.sleep(0.1)diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.pyindex 7f687e7d9..79ce9c707 100644--- a/tests/CrawlerProcess/caching_hostname_resolver.py+++ b/tests/CrawlerProcess/caching_hostname_resolver.py@@ -10,6 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):"""name = "caching_hostname_resolver_spider"+ url: strasync def start(self):yield scrapy.Request(self.url)diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.pyindex a577b1909..9f7d14606 100644--- a/tests/CrawlerProcess/sleeping.py+++ b/tests/CrawlerProcess/sleeping.py@@ -15,7 +15,7 @@ class SleepingSpider(scrapy.Spider):async def parse(self, response):from twisted.internet import reactor- d = Deferred()+ d: Deferred[None] = Deferred()reactor.callLater(int(sys.argv[1]), d.callback, None)await maybe_deferred_to_future(d)diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.pyindex fae206610..bab4f65db 100644--- a/tests/CrawlerRunner/ip_address.py+++ b/tests/CrawlerRunner/ip_address.py@@ -36,6 +36,7 @@ def createResolver(servers: list[tuple[str, int]]) -> ResolverBase:class LocalhostSpider(Spider):name = "localhost_spider"+ url: strasync def start(self):yield Request(self.url)diff --git a/tests/test_addons.py b/tests/test_addons.pyindex 14ebddda8..a041af6d6 100644--- a/tests/test_addons.py+++ b/tests/test_addons.py@@ -107,11 +107,11 @@ class TestAddonManager:addonlist = []for i in range(3):addon = get_addon_cls({"KEY1": i})- addon.number = i+ addon.number = i # type: ignore[attr-defined]addonlist.append(addon)# Test for every possible orderingfor ordered_addons in itertools.permutations(addonlist):- expected_order = [a.number for a in ordered_addons]+ expected_order = [a.number for a in ordered_addons] # type: ignore[attr-defined]settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}crawler = get_crawler(settings_dict=settings)manager = crawler.addons@@ -177,7 +177,7 @@ class TestAddonManager:)settings["SCHEDULER"] = "AddonScheduler"- settings_dict = {+ settings_dict: dict[str, Any] = {"ADDONS": {AddonWithFallback: 1},}crawler = get_crawler(settings_dict=settings_dict)diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.pyindex f17543575..50b04de0b 100644--- a/tests/test_cmdline_crawl_with_pipeline/__init__.py+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py@@ -1,21 +1,24 @@+from __future__ import annotations+import sysfrom pathlib import Pathfrom subprocess import PIPE, Popen-class TestCmdlineCrawlPipeline:- def _execute(self, spname):- args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname)- cwd = Path(__file__).resolve().parent- proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)- _, stderr = proc.communicate()- return proc.returncode, stderr+def _execute(spname: str) -> int:+ args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname)+ cwd = Path(__file__).resolve().parent+ proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)+ proc.communicate()+ return proc.returncode+++def test_open_spider_normally_in_pipeline():+ returncode = _execute("normal")+ assert returncode == 0- def test_open_spider_normally_in_pipeline(self):- returncode, _ = self._execute("normal")- assert returncode == 0- def test_exception_at_open_spider_in_pipeline(self):- returncode, _ = self._execute("exception")- # An exception in pipeline's open_spider should result in a non-zero exit code- assert returncode == 1+def test_exception_at_open_spider_in_pipeline():+ returncode = _execute("exception")+ # An exception in pipeline's open_spider should result in a non-zero exit code+ assert returncode == 1diff --git a/tests/test_commands.py b/tests/test_commands.pyindex 9f81e8602..bb4d96f00 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -89,6 +89,7 @@ class TestCommandSettings:args=["-s", f"FEEDS={feeds_json}", "spider.py"])self.command.process_options(args, opts)+ assert self.command.settings is not Noneassert isinstance(self.command.settings["FEEDS"], scrapy.settings.BaseSettings)assert dict(self.command.settings["FEEDS"]) == json.loads(feeds_json)diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex be6c80429..8c1716345 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -356,9 +356,10 @@ with multiples lines@coroutine_testasync def test_engine_status(self, mockserver: MockServer) -> None:- est = []+ est: list[list[tuple[str, Any]]] = []def cb(response):+ assert crawler.engineest.append(get_engine_status(crawler.engine))crawler = get_crawler(SingleRequestSpider)@@ -373,9 +374,10 @@ with multiples lines@coroutine_testasync def test_format_engine_status(self, mockserver: MockServer) -> None:- est = []+ est: list[str] = []def cb(response):+ assert crawler.engineest.append(format_engine_status(crawler.engine))crawler = get_crawler(SingleRequestSpider)@@ -386,8 +388,8 @@ with multiples linesassert len(est) == 1, estest = est[0].split("\n")[2:-2] # remove header & footer# convert to dict- est = [x.split(":") for x in est]- est = [x for sublist in est for x in sublist] # flatten+ est_split = [x.split(":") for x in est]+ est = [x for sublist in est_split for x in sublist] # flattenest = [x.lstrip().rstrip() for x in est]it = iter(est)s = dict(zip(it, it, strict=True))diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex adac32df1..6a1eea027 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -523,6 +523,7 @@ class TestCrawlerLogging:}async def start(self):+ assert crawler.statsinfo_count_start = crawler.stats.get_value("log_count/INFO")logging.debug("debug message") # noqa: LOG015logging.info("info message") # noqa: LOG015diff --git a/tests/test_dependencies.py b/tests/test_dependencies.pyindex 578e84359..41c7ce526 100644--- a/tests/test_dependencies.py+++ b/tests/test_dependencies.py@@ -1,3 +1,5 @@+from __future__ import annotations+import osimport refrom configparser import ConfigParser@@ -23,6 +25,7 @@ class TestScrapyUtils:config_parser.read(tox_config_file_path)pattern = r"Twisted==([\d.]+)"match = re.search(pattern, config_parser["min"]["deps"])+ assert matchpinned_twisted_version_string = match[1]assert twisted_version.short() == pinned_twisted_version_stringdiff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 275199a76..4af5ca0e8 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -16,7 +16,7 @@ from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandlerfrom scrapy.core.downloader.handlers.file import FileDownloadHandlerfrom scrapy.core.downloader.handlers.s3 import S3DownloadHandlerfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning-from scrapy.http import Request+from scrapy.http import Request, TextResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.misc import build_from_crawler@@ -354,6 +354,7 @@ class TestDataURI:response = await self.download_request(request)assert response.text == "A brief note"assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck+ assert isinstance(response, TextResponse)assert response.encoding == "US-ASCII"@coroutine_test@@ -362,6 +363,7 @@ class TestDataURI:response = await self.download_request(request)assert response.text == "\u038e\u03a3\u038e"assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck+ assert isinstance(response, TextResponse)assert response.encoding == "iso-8859-7"@coroutine_test@@ -370,6 +372,7 @@ class TestDataURI:response = await self.download_request(request)assert response.text == "\u038e\u03a3\u038e"assert response.body == b"\xbe\xd3\xbe"+ assert isinstance(response, TextResponse)assert response.encoding == "iso-8859-7"@coroutine_test@@ -382,6 +385,7 @@ class TestDataURI:response = await self.download_request(request)assert response.text == "\u038e\u03a3\u038e"assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck+ assert isinstance(response, TextResponse)assert response.encoding == "utf-8"@coroutine_testdiff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex d321c9047..92eb18d33 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -233,7 +233,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):return resultdef process_request(self, request):- d = Deferred()+ d: Deferred[Response] = Deferred()d.addCallback(self.cb)d.callback(resp)return d@@ -298,6 +298,8 @@ class TestDownloadDeprecated(TestManagerBase):return succeed(resp)async with self.get_mwman() as mwman:+ assert mwman.crawler+ assert mwman.crawler.spiderwith pytest.warns(ScrapyDeprecationWarning,match=r"DownloaderMiddlewareManager.download\(\) is deprecated, use download_async\(\) instead",diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.pyindex e697de9b7..8c89c3ffb 100644--- a/tests/test_downloadermiddleware_defaultheaders.py+++ b/tests/test_downloadermiddleware_defaultheaders.py@@ -1,3 +1,5 @@+from __future__ import annotations+from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddlewarefrom scrapy.http import Requestfrom scrapy.spiders import Spider@@ -5,28 +7,29 @@ from scrapy.utils.python import to_bytesfrom scrapy.utils.test import get_crawler-class TestDefaultHeadersMiddleware:- def get_defaults_mw(self):- crawler = get_crawler(Spider)- defaults = {- to_bytes(k): [to_bytes(v)]- for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()- }- return defaults, DefaultHeadersMiddleware.from_crawler(crawler)-- def test_process_request(self):- defaults, mw = self.get_defaults_mw()- req = Request("http://www.scrapytest.org")- mw.process_request(req)- assert req.headers == defaults-- def test_update_headers(self):- defaults, mw = self.get_defaults_mw()- headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}- bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}- req = Request("http://www.scrapytest.org", headers=headers)- assert req.headers == bytes_headers-- mw.process_request(req)- defaults.update(bytes_headers)- assert req.headers == defaults+def get_defaults_mw() -> tuple[dict[bytes, list[bytes]], DefaultHeadersMiddleware]:+ crawler = get_crawler(Spider)+ defaults = {+ to_bytes(k): [to_bytes(v)]+ for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()+ }+ return defaults, DefaultHeadersMiddleware.from_crawler(crawler)+++def test_process_request():+ defaults, mw = get_defaults_mw()+ req = Request("http://www.scrapytest.org")+ mw.process_request(req)+ assert req.headers == defaults+++def test_update_headers():+ defaults, mw = get_defaults_mw()+ headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}+ bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}+ req = Request("http://www.scrapytest.org", headers=headers)+ assert req.headers == bytes_headers++ mw.process_request(req)+ defaults.update(bytes_headers)+ assert req.headers == defaultsdiff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.pyindex e6b17960e..9b64cf349 100644--- a/tests/test_downloadermiddleware_downloadtimeout.py+++ b/tests/test_downloadermiddleware_downloadtimeout.py@@ -1,43 +1,51 @@+from __future__ import annotations++from typing import Any+from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddlewarefrom scrapy.http import Requestfrom scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler-class TestDownloadTimeoutMiddleware:- def get_request_spider_mw(self, settings=None):- crawler = get_crawler(Spider, settings)- spider = crawler._create_spider("foo")- request = Request("http://scrapytest.org/")- return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)-- def test_default_download_timeout(self):- req, spider, mw = self.get_request_spider_mw()- mw.spider_opened(spider)- assert mw.process_request(req) is None- assert req.meta.get("download_timeout") == 180-- def test_string_download_timeout(self):- req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})- mw.spider_opened(spider)- assert mw.process_request(req) is None- assert req.meta.get("download_timeout") == 20.1-- def test_setting_has_download_timeout(self):- req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})- mw.spider_opened(spider)- assert mw.process_request(req) is None- assert req.meta.get("download_timeout") == 2-- def test_request_has_download_timeout(self):- req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})- mw.spider_opened(spider)- req.meta["download_timeout"] = 1- assert mw.process_request(req) is None- assert req.meta.get("download_timeout") == 1-- def test_zero_download_timeout(self):- req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0})- mw.spider_opened(spider)- assert mw.process_request(req) is None- assert req.meta.get("download_timeout") is None+def get_request_spider_mw(settings: dict[str, Any] | None = None):+ crawler = get_crawler(Spider, settings)+ spider = crawler._create_spider("foo")+ request = Request("http://scrapytest.org/")+ return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)+++def test_default_download_timeout():+ req, spider, mw = get_request_spider_mw()+ mw.spider_opened(spider)+ assert mw.process_request(req) is None+ assert req.meta.get("download_timeout") == 180+++def test_string_download_timeout():+ req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})+ mw.spider_opened(spider)+ assert mw.process_request(req) is None+ assert req.meta.get("download_timeout") == 20.1+++def test_setting_has_download_timeout():+ req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})+ mw.spider_opened(spider)+ assert mw.process_request(req) is None+ assert req.meta.get("download_timeout") == 2+++def test_request_has_download_timeout():+ req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})+ mw.spider_opened(spider)+ req.meta["download_timeout"] = 1+ assert mw.process_request(req) is None+ assert req.meta.get("download_timeout") == 1+++def test_zero_download_timeout():+ req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0})+ mw.spider_opened(spider)+ assert mw.process_request(req) is None+ assert req.meta.get("download_timeout") is Nonediff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.pyindex 539183cd6..0c925a0d1 100644--- a/tests/test_downloadermiddleware_useragent.py+++ b/tests/test_downloadermiddleware_useragent.py@@ -1,33 +1,37 @@+from __future__ import annotations+from scrapy.downloadermiddlewares.useragent import UserAgentMiddlewarefrom scrapy.http import Requestfrom scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler-class TestUserAgentMiddleware:- def get_spider_and_mw(self, default_useragent):- crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})- spider = crawler._create_spider("foo")- return spider, UserAgentMiddleware.from_crawler(crawler)-- def test_default_agent(self):- _, mw = self.get_spider_and_mw("default_useragent")- req = Request("http://scrapytest.org/")- assert mw.process_request(req) is None- assert req.headers["User-Agent"] == b"default_useragent"-- def test_header_agent(self):- spider, mw = self.get_spider_and_mw("default_useragent")- mw.spider_opened(spider)- req = Request(- "http://scrapytest.org/", headers={"User-Agent": "header_useragent"}- )- assert mw.process_request(req) is None- assert req.headers["User-Agent"] == b"header_useragent"-- def test_no_agent(self):- spider, mw = self.get_spider_and_mw(None)- mw.spider_opened(spider)- req = Request("http://scrapytest.org/")- assert mw.process_request(req) is None- assert "User-Agent" not in req.headers+def get_spider_and_mw(+ default_useragent: str | None,+) -> tuple[Spider, UserAgentMiddleware]:+ crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})+ spider = crawler._create_spider("foo")+ return spider, UserAgentMiddleware.from_crawler(crawler)+++def test_default_agent():+ _, mw = get_spider_and_mw("default_useragent")+ req = Request("http://scrapytest.org/")+ assert mw.process_request(req) is None+ assert req.headers["User-Agent"] == b"default_useragent"+++def test_header_agent():+ spider, mw = get_spider_and_mw("default_useragent")+ mw.spider_opened(spider)+ req = Request("http://scrapytest.org/", headers={"User-Agent": "header_useragent"})+ assert mw.process_request(req) is None+ assert req.headers["User-Agent"] == b"header_useragent"+++def test_no_agent():+ spider, mw = get_spider_and_mw(None)+ mw.spider_opened(spider)+ req = Request("http://scrapytest.org/")+ assert mw.process_request(req) is None+ assert "User-Agent" not in req.headersdiff --git a/tests/test_engine.py b/tests/test_engine.pyindex e51eb4664..2b857d35a 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -38,6 +38,8 @@ from tests import get_testdatafrom tests.utils.decorators import coroutine_test, inline_callbacks_testif TYPE_CHECKING:+ from collections.abc import AsyncIterator+from twisted.python.failure import Failurefrom scrapy.core.scheduler import Scheduler@@ -608,8 +610,8 @@ class TestEngineDownload(TestEngineDownloadAsync):@coroutine_testasync def test_request_scheduled_signal():class TestScheduler(BaseScheduler):- def __init__(self):- self.enqueued = []+ def __init__(self) -> None:+ self.enqueued: list[Request] = []def enqueue_request(self, request: Request) -> bool:self.enqueued.append(request)@@ -621,9 +623,9 @@ async def test_request_scheduled_signal():crawler = get_crawler(MySpider)engine = ExecutionEngine(crawler, lambda _: None)- scheduler = TestScheduler()+ scheduler = TestScheduler() # type: ignore[abstract]- async def start():+ async def start() -> AsyncIterator[Any]:returnyielddiff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.pyindex 18782fb70..0fe2caca4 100644--- a/tests/test_extension_periodic_log.py+++ b/tests/test_extension_periodic_log.py@@ -102,7 +102,9 @@ class TestPeriodicLog:ext.spider_closed(spider, reason="finished")return ext, a, b- def check(settings: dict[str, Any], condition: Callable) -> None:+ def check(+ settings: dict[str, Any], condition: Callable[[str, Any], bool]+ ) -> None:ext, a, b = emulate(settings)assert list(a["delta"].keys()) == [k for k, v in ext.stats._stats.items() if condition(k, v)@@ -168,7 +170,9 @@ class TestPeriodicLog:ext.spider_closed(spider, reason="finished")return ext, a, b- def check(settings: dict[str, Any], condition: Callable) -> None:+ def check(+ settings: dict[str, Any], condition: Callable[[str, Any], bool]+ ) -> None:ext, a, b = emulate(settings)assert list(a["stats"].keys()) == [k for k, v in ext.stats._stats.items() if condition(k, v)diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.pyindex 0a926479b..469985288 100644--- a/tests/test_feedexport_batch.py+++ b/tests/test_feedexport_batch.py@@ -378,6 +378,7 @@ class TestBatchDeliveries(TestFeedExportBase):}crawler = get_crawler(ItemSpider, settings)yield crawler.crawl(total=2, mockserver=self.mockserver)+ assert crawler.statsassert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12@@ -392,7 +393,9 @@ class TestBatchDeliveries(TestFeedExportBase):]class CustomS3FeedStorage(S3FeedStorage):- stubs = []+ from botocore.stub import Stubber # noqa: PLC0415++ stubs: list[Stubber] = []def open(self, *args, **kwargs):from botocore import __version__ as botocore_version # noqa: PLC0415@@ -449,6 +452,7 @@ class TestBatchDeliveries(TestFeedExportBase):assert len(CustomS3FeedStorage.stubs) == len(items)for stub in CustomS3FeedStorage.stubs:stub.assert_no_pending_responses()+ assert crawler.statsassert ("feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats())diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.pyindex ce9296764..5d4d5a2bd 100644--- a/tests/test_http_cookies.py+++ b/tests/test_http_cookies.py@@ -1,3 +1,5 @@+from __future__ import annotations+from http.cookiejar import DefaultCookiePolicyfrom scrapy.http import Request, Response@@ -46,7 +48,7 @@ class TestCookieJar:def test_set_policy(self):policy = DefaultCookiePolicy()self.jar.set_policy(policy)- assert self.jar.jar._policy is policy+ assert self.jar.jar._policy is policy # type: ignore[attr-defined]def test_check_expired_frequency(self):jar = CookieJar(check_expired_frequency=1)diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.pyindex 42ab29267..7c9f717e4 100644--- a/tests/test_responsetypes.py+++ b/tests/test_responsetypes.py@@ -1,3 +1,5 @@+from typing import Any+from scrapy.http import (Headers,HtmlResponse,@@ -38,13 +40,13 @@ class TestResponseTypes:]for source, cls in mappings:retcls = responsetypes.from_content_disposition(source)- assert retcls is cls, f"{source} ==> {retcls} != {cls}"+ assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"def test_from_content_disposition_no_filename(self):assert responsetypes.from_content_disposition(b"attachment") is Responsedef test_from_content_type(self):- mappings = [+ mappings: list[tuple[str | bytes, type[Response]]] = [("text/html; charset=UTF-8", HtmlResponse),("text/xml; charset=UTF-8", XmlResponse),("application/xhtml+xml; charset=UTF-8", HtmlResponse),@@ -58,7 +60,7 @@ class TestResponseTypes:]for source, cls in mappings:retcls = responsetypes.from_content_type(source)- assert retcls is cls, f"{source} ==> {retcls} != {cls}"+ assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"def test_from_body(self):mappings = [@@ -71,7 +73,7 @@ class TestResponseTypes:]for source, cls in mappings:retcls = responsetypes.from_body(source)- assert retcls is cls, f"{source} ==> {retcls} != {cls}"+ assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"def test_from_headers(self):mappings = [@@ -98,7 +100,7 @@ class TestResponseTypes:def test_from_args(self):# TODO: add more tests that check precedence between the different arguments- mappings = [+ mappings: list[tuple[dict[str, Any], type[Response]]] = [({"url": "http://www.example.com/data.csv"}, TextResponse),# headers takes precedence over url(diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex 8637de41e..39ce87b65 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -379,13 +379,14 @@ class TestIntegrationWithDownloaderAwareInMemory:url = mockserver.url("/status?n=200", is_secure=False)start_urls = [url] * 6yield self.crawler.crawl(start_urls)+ assert self.crawler.statsassert self.crawler.stats.get_value("downloader/response_count") == len(start_urls)class TestIncompatibility:- def _incompatible(self):+ def _incompatible(self) -> None:settings = {"SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue","CONCURRENT_REQUESTS_PER_IP": 1,diff --git a/tests/test_signals.py b/tests/test_signals.pyindex f5f13b992..fc9520f49 100644--- a/tests/test_signals.py+++ b/tests/test_signals.py@@ -1,13 +1,20 @@+from __future__ import annotations++from typing import TYPE_CHECKING+import pytestfrom scrapy import Request, Spider, signalsfrom scrapy.utils.test import get_crawler, get_from_asyncio_queue-from tests.mockserver.http import MockServer-from tests.utils.decorators import coroutine_test, inline_callbacks_test+from tests.utils.decorators import coroutine_test++if TYPE_CHECKING:+ from tests.mockserver.http import MockServerclass ItemSpider(Spider):name = "itemspider"+ mockserver: MockServerasync def start(self):for index in range(10):@@ -34,15 +41,6 @@ class TestMain:class TestMockServer:- @classmethod- def setup_class(cls):- cls.mockserver = MockServer()- cls.mockserver.__enter__()-- @classmethod- def teardown_class(cls):- cls.mockserver.__exit__(None, None, None)-def setup_method(self):self.items = []@@ -51,11 +49,11 @@ class TestMockServer:self.items.append(item)@pytest.mark.only_asyncio- @inline_callbacks_test- def test_simple_pipeline(self):+ @coroutine_test+ async def test_simple_pipeline(self, mockserver: MockServer) -> None:crawler = get_crawler(ItemSpider)crawler.signals.connect(self._on_item_scraped, signals.item_scraped)- yield crawler.crawl(mockserver=self.mockserver)+ await crawler.crawl_async(mockserver=mockserver)assert len(self.items) == 10for index in range(10):assert {"index": index} in self.itemsdiff --git a/tests/test_spider_start.py b/tests/test_spider_start.pyindex 9257e0232..7cef2fefa 100644--- a/tests/test_spider_start.py+++ b/tests/test_spider_start.py@@ -1,7 +1,7 @@from __future__ import annotationsfrom asyncio import sleep-from typing import Any+from typing import TYPE_CHECKING, Anyimport pytest@@ -12,6 +12,9 @@ from scrapy.utils.test import get_crawlerfrom .utils import twisted_sleepfrom .utils.decorators import coroutine_test+if TYPE_CHECKING:+ from collections.abc import AsyncIterator, Callable+SLEEP_SECONDS = 0.1ITEM_A = {"id": "a"}@@ -67,7 +70,11 @@ class TestMain:await self._test_spider(TestSpider, [ITEM_A])- async def _test_start(self, start_, expected_items=None):+ async def _test_start(+ self,+ start_: Callable[[Any], AsyncIterator[Any]],+ expected_items: list[Any] | None = None,+ ) -> None:class TestSpider(Spider):name = "test"start = start_diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex e5891474b..7824d5c20 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -186,7 +186,7 @@ class ProcessSpiderExceptionSimpleIterableMiddleware:class ProcessSpiderExceptionAsyncIteratorMiddleware:async def process_spider_exception(self, response, exception):yield {"foo": 1}- d = defer.Deferred()+ d: defer.Deferred[None] = defer.Deferred()call_later(0, d.callback, None)await maybe_deferred_to_future(d)yield {"foo": 2}diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex 6a054ded5..2ba083fd6 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -22,6 +22,7 @@ class _HttpErrorSpider(MockServerSpider):def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)+ assert self.mockserverself.start_urls = [self.mockserver.url("/status?n=200"),self.mockserver.url("/status?n=404"),@@ -36,7 +37,7 @@ class _HttpErrorSpider(MockServerSpider):for url in self.start_urls:yield Request(url, self.parse, errback=self.on_error)- def parse(self, response):+ def parse(self, response: Response) -> None:self.parsed.add(response.url[-3:])def on_error(self, failure):diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 32e83f990..7591c0e7b 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -1,7 +1,7 @@from __future__ import annotationsimport warnings-from typing import Any+from typing import TYPE_CHECKING, Anyfrom urllib.parse import urlparseimport pytest@@ -35,6 +35,9 @@ from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerfrom tests.utils.decorators import coroutine_test+if TYPE_CHECKING:+ from collections.abc import AsyncIterator+class TestRefererMiddleware:req_meta: dict[str, Any] = {}@@ -1031,7 +1034,7 @@ async def test_response_policy_only_supports_policy_names():crawler = get_crawler(settings_dict={"REFERRER_POLICY": "no-referrer"})mw = build_from_crawler(RefererMiddleware, crawler)- async def input_result():+ async def input_result() -> AsyncIterator[Any]:yield Request("https://example.com/")response = Response(@@ -1079,7 +1082,7 @@ async def test_referer_policies_setting():)mw = build_from_crawler(RefererMiddleware, crawler)- async def input_result():+ async def input_result() -> AsyncIterator[Any]:yield Request("https://example.com/")# "no-referrer-when-downgrade": None,diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.pyindex def3a3df3..e879f3a32 100644--- a/tests/test_spidermiddleware_start.py+++ b/tests/test_spidermiddleware_start.py@@ -1,3 +1,7 @@+from __future__ import annotations++from typing import TYPE_CHECKING+from scrapy.http import Request, Responsefrom scrapy.spidermiddlewares.start import StartSpiderMiddlewarefrom scrapy.spiders import Spider@@ -5,6 +9,9 @@ from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerfrom tests.utils.decorators import coroutine_test+if TYPE_CHECKING:+ from collections.abc import AsyncIterator+class TestMiddleware:@coroutine_test@@ -12,7 +19,7 @@ class TestMiddleware:crawler = get_crawler(Spider)mw = build_from_crawler(StartSpiderMiddleware, crawler)- async def start():+ async def start() -> AsyncIterator[Request]:yield Request("data:,1")yield Request("data:,2", meta={"is_start_request": True})yield Request("data:,2", meta={"is_start_request": False})diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex a198d1c09..7528dc51a 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -72,7 +72,7 @@ class TestParallelAsyncio:for length in [20, 50, 100]:parallel_count = [0]max_parallel_count = [0]- results = []+ results: list[int] = []ait = self.get_async_iterable(length)await _parallel_asyncio(ait,@@ -91,7 +91,7 @@ class TestParallelAsyncio:for length in [20, 50, 100]:parallel_count = [0]max_parallel_count = [0]- results = []+ results: list[int] = []ait = self.get_async_iterable_with_delays(length)await _parallel_asyncio(ait,diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex b6a9d8b06..518cc3518 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -1,3 +1,7 @@+from __future__ import annotations++from typing import Any+import pytestfrom scrapy.exceptions import UsageError@@ -106,7 +110,7 @@ class TestFeedExportConfig:)def test_feed_complete_default_values_from_settings_empty(self):- feed = {}+ feed: dict[str, Any] = {}settings = Settings({"FEED_EXPORT_ENCODING": "custom encoding",diff --git a/tests/test_utils_console.py b/tests/test_utils_console.pyindex dc1d96f66..05b272832 100644--- a/tests/test_utils_console.py+++ b/tests/test_utils_console.py@@ -1,3 +1,5 @@+from __future__ import annotations+import pytestfrom scrapy.utils.console import get_shell_embed_func@@ -38,4 +40,5 @@ def test_get_shell_embed_func_bpython():def test_get_shell_embed_func_ipython():# default shell should be 'ipython'shell = get_shell_embed_func()+ assert shell is not Noneassert shell.__name__ == "_embed_ipython_shell"diff --git a/tests/test_utils_display.py b/tests/test_utils_display.pyindex 8b02116b8..9f9e24957 100644--- a/tests/test_utils_display.py+++ b/tests/test_utils_display.py@@ -1,9 +1,16 @@+from __future__ import annotations+import builtinsfrom io import StringIO+from typing import TYPE_CHECKINGfrom unittest import mockfrom scrapy.utils.display import pformat, pprint+if TYPE_CHECKING:+ from collections.abc import Mapping, Sequence+ from types import ModuleType+value = {"a": 1}colorized_strings = {(@@ -77,14 +84,19 @@ def test_pformat_no_pygments(isatty):real_import = builtins.__import__- def mock_import(name, globals_, locals_, fromlist, level):+ def mock_import(+ name: str,+ globals_: Mapping[str, object] | None = None,+ locals_: Mapping[str, object] | None = None,+ fromlist: Sequence[str] | None = (),+ level: int = 0,+ ) -> ModuleType:if "pygments" in name:raise ImportErrorreturn real_import(name, globals_, locals_, fromlist, level)- builtins.__import__ = mock_import- assert pformat(value) == plain_string- builtins.__import__ = real_import+ with mock.patch("builtins.__import__", mock_import):+ assert pformat(value) == plain_stringdef test_pprint():diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.pyindex 1276c1392..a59541edb 100644--- a/tests/test_utils_iterators.py+++ b/tests/test_utils_iterators.py@@ -238,7 +238,7 @@ class TestXmliterBase(ABC):next(my_iter)def test_xmliter_objtype_exception(self):- i = self.xmliter(42, "product")+ i = self.xmliter(42, "product") # type: ignore[arg-type]with pytest.raises(TypeError):next(i)@@ -353,7 +353,7 @@ class TestLxmlXmliter(TestXmliterBase):assert node.xpath("f:name/text()").getall() == ["African Coffee Table"]def test_xmliter_objtype_exception(self):- i = self.xmliter(42, "product")+ i = self.xmliter(42, "product") # type: ignore[arg-type]with pytest.raises(TypeError):next(i)diff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex ee552df64..fe57492b6 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -38,7 +38,7 @@ class TestFailureToExcInfo:assert exc_info == failure_to_exc_info(failure)def test_non_failure(self):- assert failure_to_exc_info("test") is None+ assert failure_to_exc_info("test") is None # type: ignore[arg-type]class TestTopLevelFormatter:diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.pyindex c4c861404..ab6965ed5 100644--- a/tests/test_utils_misc/__init__.py+++ b/tests/test_utils_misc/__init__.py@@ -1,3 +1,5 @@+from __future__ import annotations+import osimport sysfrom pathlib import Path@@ -37,7 +39,7 @@ class TestUtilsMisc:with pytest.raises(NameError):load_object("scrapy.utils.misc.load_object999")with pytest.raises(TypeError):- load_object({})+ load_object({}) # type: ignore[arg-type]def test_walk_modules(self):mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")@@ -114,7 +116,7 @@ class TestUtilsMisc:args = (True, 100.0)kwargs = {"key": "val"}- def _test_with_crawler(mock, crawler):+ def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:build_from_crawler(mock, crawler, *args, **kwargs)if hasattr(mock, "from_crawler"):mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)diff --git a/tests/test_utils_response.py b/tests/test_utils_response.pyindex 4544cd29e..146cdb802 100644--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@ -38,7 +38,7 @@ def test_open_in_browser():resp = Response(url, body=body)with pytest.raises(TypeError):- open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg+ open_in_browser(resp, _openfunc=browser_open) # type: ignore[arg-type]def test_get_meta_refresh():@@ -320,4 +320,4 @@ def test_open_in_browser_text_response_uses_txt_extension():def test_open_in_browser_raises_for_unsupported_response_type():response = Response("http://www.example.com", body=b"binary")with pytest.raises(TypeError):- open_in_browser(response, _openfunc=lambda _: True)+ open_in_browser(response, _openfunc=lambda _: True) # type: ignore[arg-type]diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.pyindex 9ac0fd0c0..dc615eca6 100644--- a/tests/test_utils_signal.py+++ b/tests/test_utils_signal.py@@ -1,4 +1,7 @@+from __future__ import annotations+import asyncio+from typing import TYPE_CHECKINGimport pytestfrom pydispatch import dispatcher@@ -16,6 +19,9 @@ from scrapy.utils.signal import (from scrapy.utils.test import get_from_asyncio_queuefrom tests.utils.decorators import inline_callbacks_test+if TYPE_CHECKING:+ from collections.abc import Callable+class TestSendCatchLog:# whether the function being tested returns exceptions or failures@@ -24,7 +30,7 @@ class TestSendCatchLog:@inline_callbacks_testdef test_send_catch_log(self):test_signal = object()- handlers_called = set()+ handlers_called: set[Callable[..., None]] = set()dispatcher.connect(self.error_handler, signal=test_signal)dispatcher.connect(self.ok_handler, signal=test_signal)@@ -74,7 +80,7 @@ class TestSendCatchLogDeferred2(TestSendCatchLogDeferred):def ok_handler(self, arg, handlers_called):handlers_called.add(self.ok_handler)assert arg == "test"- d = defer.Deferred()+ d: defer.Deferred[str] = defer.Deferred()call_later(0, d.callback, "OK")return d@@ -108,7 +114,7 @@ class TestSendCatchLogAsync2(TestSendCatchLogAsync):def ok_handler(self, arg, handlers_called):handlers_called.add(self.ok_handler)assert arg == "test"- d = defer.Deferred()+ d: defer.Deferred[str] = defer.Deferred()call_later(0, d.callback, "OK")return ddiff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.pyindex 05971f423..71f0f76ad 100644--- a/tests/test_utils_spider.py+++ b/tests/test_utils_spider.py@@ -1,3 +1,5 @@+from __future__ import annotations+from scrapy import Spiderfrom scrapy.http import Requestfrom scrapy.item import Item@@ -17,7 +19,7 @@ def test_iterate_spider_output():r = Request("http://scrapytest.org")o = object()- assert list(iterate_spider_output(i)) == [i]+ assert list(iterate_spider_output(i)) == [i] # type: ignore[call-overload]assert list(iterate_spider_output(r)) == [r]assert list(iterate_spider_output(o)) == [o]assert list(iterate_spider_output([r, i, o])) == [r, i, o]diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.pyindex 7a7a264d3..2334c76e9 100644--- a/tests/test_utils_trackref.py+++ b/tests/test_utils_trackref.py@@ -79,9 +79,9 @@ def test_get_oldest():stale entries in `live_refs`, affecting results unless explicitly cleared."""- def _delete_o1():+ def _delete_o1() -> None:"""Delete `o1` and ensure it is actually collected on PyPy."""- nonlocal o1+ nonlocal o1 # type: ignore[misc]del o1if _IS_PYPY:@@ -89,7 +89,7 @@ def test_get_oldest():# still exist until the GC runs, so we force a collection cycle.garbage_collect()- def _do_asserts():+ def _do_asserts() -> None:assert trackref.get_oldest("Foo") is o1assert trackref.get_oldest("Bar") is o2# Ensure the newer Foo is not incorrectly considered the oldest