Scrapy, a fast high-level web crawling & scraping framework for Python.
Type the last tests.* modules under check_untyped_defs = false (#8017)
Adriane9830e6cb4c2
diff --git a/pyproject.toml b/pyproject.tomlindex d75e361f6..513f0f27c 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -116,26 +116,6 @@ module = "tests.*"allow_untyped_defs = trueallow_incomplete_defs = true # 59 errors-# TODO-[[tool.mypy.overrides]]-module = [- "tests.test_cmdline",- "tests.test_downloaderslotssettings",- "tests.test_dupefilters",- "tests.test_extension_statsmailer",- "tests.test_extension_throttle",- "tests.test_linkextractors",- "tests.test_logformatter",- "tests.test_mail",- "tests.test_pqueues",- "tests.test_scheduler_base",- "tests.test_settings",- "tests.test_squeues",- "tests.test_squeues_request",- "tests.test_stats",-]-check_untyped_defs = false-# Interface classes are hard to support[[tool.mypy.overrides]]module = "twisted.internet.interfaces"diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 75f9753e7..0a636e2d7 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -309,8 +309,8 @@ class LxmlLinkExtractor:allow_domains: str | Iterable[str] = (),deny_domains: str | Iterable[str] = (),restrict_xpaths: str | Iterable[str] = (),- tags: str | Iterable[str] = ("a", "area"),- attrs: str | Iterable[str] = ("href",),+ tags: str | Iterable[str] | None = ("a", "area"),+ attrs: str | Iterable[str] | None = ("href",),canonicalize: bool = False,unique: bool = True,process_value: Callable[[Any], Any] | None = None,diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.pyindex c068bb0f3..26c822a93 100644--- a/tests/test_cmdline/__init__.py+++ b/tests/test_cmdline/__init__.py@@ -8,18 +8,19 @@ import tempfilefrom io import StringIOfrom pathlib import Pathfrom subprocess import PIPE, Popen+from typing import Anyfrom scrapy.utils.test import get_testenv-def _execute(*new_args, **kwargs):+def _execute(*new_args: str, **kwargs: Any) -> str:env = get_testenv()tests_path = Path(__file__).parent.parentenv["PYTHONPATH"] += os.pathsep + str(tests_path.parent)env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"encoding = sys.stdout.encoding or "utf-8"args = (sys.executable, "-m", "scrapy.cmdline", *new_args)- proc = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs)+ proc: Popen[bytes] = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs)comm = proc.communicate()[0].strip()return comm.decode(encoding)@@ -42,9 +43,9 @@ def test_profiling():stats = pstats.Stats(str(filename), stream=out)stats.print_stats()out.seek(0)- stats = out.read()- assert str(Path("scrapy", "commands", "version.py")) in stats- assert "tottime" in stats+ stats_text = out.read()+ assert str(Path("scrapy", "commands", "version.py")) in stats_text+ assert "tottime" in stats_textfinally:shutil.rmtree(path)diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex d4b0ea436..c6140295c 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -37,6 +37,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):self.times: dict[str, list[float]] = {}async def start(self):+ assert self.mockserverslots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None]for slot in slots:url = self.mockserver.url(f"/?downloader_slot={slot}")@@ -44,6 +45,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):yield Request(url, callback=self.parse, meta={"download_slot": slot})def parse(self, response):+ assert self.mockserverslot = response.meta.get("download_slot", self.default_slot)self.times[slot].append(time.time())url = self.mockserver.url(f"/?downloader_slot={slot}&req=2")@@ -84,7 +86,7 @@ def test_delay(mockserver: MockServer):@coroutine_testasync def test_params():- params = {+ params: dict[str, Any] = {"concurrency": 1,"delay": 2,"randomize_delay": False,diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex 4b984406a..1dc90d1e3 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -6,7 +6,7 @@ import shutilimport sysimport tempfilefrom pathlib import Path-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, castimport pytest@@ -39,6 +39,8 @@ def _get_dupefilter(class FromCrawlerRFPDupeFilter(RFPDupeFilter):+ method: str+@classmethoddef from_crawler(cls, crawler):df = super().from_crawler(crawler)@@ -58,8 +60,9 @@ class TestRFPDupeFilter:}crawler = get_crawler(settings_dict=settings)scheduler = build_from_crawler(Scheduler, crawler)- assert scheduler.df.debug- assert scheduler.df.method == "from_crawler"+ dupefilter = cast("FromCrawlerRFPDupeFilter", scheduler.df)+ assert dupefilter.debug+ assert dupefilter.method == "from_crawler"def test_df_direct_scheduler(self):settings = {@@ -67,7 +70,7 @@ class TestRFPDupeFilter:}crawler = get_crawler(settings_dict=settings)scheduler = build_from_crawler(Scheduler, crawler)- assert scheduler.df.method == "n/a"+ assert cast("DirectDupeFilter", scheduler.df).method == "n/a"def test_filter(self):dupefilter = _get_dupefilter()@@ -237,4 +240,4 @@ class TestBaseDupeFilter:ScrapyDeprecationWarning,match=r"Calling BaseDupeFilter\.log\(\) is deprecated.",):- dupefilter.log(None, None)+ dupefilter.log(None, None) # type: ignore[arg-type]diff --git a/tests/test_extension_statsmailer.py b/tests/test_extension_statsmailer.pyindex a0e0618f4..8fd861e43 100644--- a/tests/test_extension_statsmailer.py+++ b/tests/test_extension_statsmailer.py@@ -1,9 +1,10 @@import warnings+from typing import Anyfrom unittest.mock import MagicMockimport pytest-from scrapy import signals+from scrapy import Spider, signalsfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.signalmanager import SignalManagerfrom scrapy.statscollectors import StatsCollector@@ -28,11 +29,11 @@ with warnings.catch_warnings():@pytest.fixturedef dummy_stats():class DummyStats(StatsCollector):- def __init__(self):+ def __init__(self) -> None:# pylint: disable=super-init-not-calledself._stats = {"global_item_scraped_count": 42}- def get_stats(self):+ def get_stats(self, spider: Spider | None = None) -> dict[str, Any]:return {"item_scraped_count": 10, **self._stats}return DummyStats()@@ -54,7 +55,7 @@ def test_from_crawler_with_recipients_initializes_extension(dummy_stats, monkeypcrawler.signals = SignalManager(crawler)mailer = MagicMock(spec=MailSender)- monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer)+ monkeypatch.setattr(MailSender, "from_crawler", lambda _: mailer)ext = build_from_crawler(statsmailer.StatsMailer, crawler)@@ -70,7 +71,7 @@ def test_from_crawler_connects_spider_closed_signal(dummy_stats, monkeypatch):crawler.signals = SignalManager(crawler)mailer = MagicMock(spec=MailSender)- monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer)+ monkeypatch.setattr(MailSender, "from_crawler", lambda _: mailer)build_from_crawler(statsmailer.StatsMailer, crawler)diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.pyindex 078b78fac..69f1eb303 100644--- a/tests/test_extension_throttle.py+++ b/tests/test_extension_throttle.py@@ -1,9 +1,11 @@from logging import INFO+from typing import Anyfrom unittest.mock import Mockimport pytest-from scrapy import Request+from scrapy import Request, Spider+from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfiguredfrom scrapy.extensions.throttle import AutoThrottlefrom scrapy.http.response import Response@@ -19,17 +21,20 @@ from scrapy.utils.test import get_crawler as _get_crawlerUNSET = object()-def get_crawler(settings=None, spidercls=None):+def get_crawler(+ settings: dict[str, Any] | None = None, spidercls: type[Spider] | None = None+) -> Crawler:settings = settings or {}settings["AUTOTHROTTLE_ENABLED"] = Truereturn _get_crawler(settings_dict=settings, spidercls=spidercls)-def _mock_downloader(crawler):+def _mock_downloader(crawler: Crawler) -> Mock:"""Give *crawler* a mock engine, whose downloader AutoThrottle reads."""crawler.engine = Mock()- crawler.engine.downloader.slots = {}- return crawler.engine.downloader+ downloader: Mock = crawler.engine.downloader+ downloader.slots = {}+ return downloader@pytest.mark.parametrize(@@ -155,7 +160,7 @@ def test_startdelay_definition(min_setting, start_setting, expected):),],)-def test_skipped(meta, slot):+def test_skipped(meta, slot, monkeypatch):crawler = get_crawler()at = build_from_crawler(AutoThrottle, crawler)downloader = _mock_downloader(crawler)@@ -165,9 +170,10 @@ def test_skipped(meta, slot):if slot is not None:downloader.slots[slot] = object()- at._adjust_delay = None # Raise exception if called.+ # Fail instead of adjusting the delay.+ monkeypatch.setattr(at, "_adjust_delay", Mock(side_effect=AssertionError))- at._response_downloaded(None, request, spider)+ at._response_downloaded(Response("https://example.com"), request, spider)@pytest.mark.parametrize(diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex d144532c1..57bb7b0b2 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -2,8 +2,11 @@ from __future__ import annotationsimport pickleimport re+from typing import Any-from scrapy.http import HtmlResponse, XmlResponse+from w3lib.url import canonicalize_url++from scrapy.http import HtmlResponse, TextResponse, XmlResponsefrom scrapy.link import Linkfrom scrapy.linkextractors import lxmlhtmlfrom scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor, LxmlParserLinkExtractor@@ -499,7 +502,7 @@ class TestLxmlLinkExtractor:"""response = HtmlResponse("http://example.com/index.html", body=html)- def urls(**kwargs):+ def urls(**kwargs: Any) -> list[str]:lx = LxmlLinkExtractor(**kwargs)return [link.url for link in lx.extract_links(response)]@@ -570,7 +573,9 @@ class TestLxmlLinkExtractor:</html>"""- response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)+ response: TextResponse = HtmlResponse(+ "http://example.com/index.xhtml", body=xhtml+ )lx = LxmlLinkExtractor()assert lx.extract_links(response) == [@@ -782,7 +787,6 @@ class TestLxmlLinkExtractor:]def test_canonicalize_once_per_link(self, monkeypatch):- canonicalize_url = lxmlhtml.canonicalize_urlcalls = []def counting_canonicalize_url(url, *args, **kwargs):diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex 2e17de4a5..dc70e6413 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -1,7 +1,7 @@from __future__ import annotationsimport logging-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Any, castimport pytestfrom twisted.python.failure import Failure@@ -63,7 +63,7 @@ class TestLogFormatter:)def test_dropped(self):- item = {}+ item: dict[str, Any] = {}exception = Exception("\u2018")response = Response("http://www.example.com")logkws = self.formatter.dropped(item, exception, response, self.spider)@@ -73,7 +73,7 @@ class TestLogFormatter:assert lines == ["Dropped: \u2018", "{}"]def test_dropitem_default_log_level(self):- item = {}+ item: dict[str, Any] = {}exception = DropItem("Test drop")response = Response("http://www.example.com")spider = Spider.from_crawler(get_crawler(Spider), "foo")@@ -116,7 +116,7 @@ class TestLogFormatter:logging.log(logkws["level"], "message") # noqa: LOG015def test_dropitem_custom_log_level(self):- item = {}+ item: dict[str, Any] = {}response = Response("http://www.example.com")exception = DropItem("Test drop", log_level="INFO")@@ -187,7 +187,7 @@ class LogFormatterSubclass(LogFormatter):def crawled(self, request, response, spider):kwargs = super().crawled(request, response, spider)CRAWLEDMSG = "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"- log_args = kwargs["args"]+ log_args = cast("dict[str, Any]", kwargs["args"])log_args["flags"] = str(request.flags)return {"level": kwargs["level"],@@ -259,7 +259,7 @@ class DropSomeItemsPipeline:class TestShowOrSkipMessages:def setup_method(self):- self.base_settings = {+ self.base_settings: dict[str, Any] = {"LOG_LEVEL": "DEBUG","ITEM_PIPELINES": {DropSomeItemsPipeline: 300,diff --git a/tests/test_mail.py b/tests/test_mail.pyindex e6d65cd10..a43dca7bd 100644--- a/tests/test_mail.py+++ b/tests/test_mail.py@@ -157,7 +157,7 @@ class TestMailSender:mailsender = MailSender(debug=False, smtphost="smtp.testhost.com")factory = mailsender._create_sender_factory(- to_addrs=["[email protected]"], msg="test", d=defer.Deferred()+ to_addrs=["[email protected]"], msg=BytesIO(b"test"), d=defer.Deferred())context = factory.buildProtocol("[email protected]").contextdiff --git a/tests/test_pqueues.py b/tests/test_pqueues.pyindex 25cfb3a12..f07928510 100644--- a/tests/test_pqueues.py+++ b/tests/test_pqueues.py@@ -1,4 +1,5 @@import tempfile+from typing import Anyfrom unittest.mock import Mockimport pytest@@ -14,6 +15,18 @@ from scrapy.utils.test import get_crawlerfrom tests.utils.downloader import MockDownloader+def _pop(queue: ScrapyPriorityQueue | DownloaderAwarePriorityQueue) -> Request:+ request = queue.pop()+ assert request is not None+ return request+++def _peek(queue: ScrapyPriorityQueue | DownloaderAwarePriorityQueue) -> Request:+ request = queue.peek()+ assert request is not None+ return request++class TestPriorityQueue:def setup_method(self):self.crawler = get_crawler(Spider)@@ -29,7 +42,7 @@ class TestPriorityQueue:req1 = Request("https://example.org/1", priority=1)queue.push(req1)assert len(queue) == 1- dequeued = queue.pop()+ dequeued = _pop(queue)assert len(queue) == 0assert dequeued.url == req1.urlassert dequeued.priority == req1.priority@@ -66,14 +79,14 @@ class TestPriorityQueue:queue.push(req2)queue.push(req3)assert len(queue) == 3- assert queue.peek().url == req1.url- assert queue.pop().url == req1.url+ assert _peek(queue).url == req1.url+ assert _pop(queue).url == req1.urlassert len(queue) == 2- assert queue.peek().url == req2.url- assert queue.pop().url == req2.url+ assert _peek(queue).url == req2.url+ assert _pop(queue).url == req2.urlassert len(queue) == 1- assert queue.peek().url == req3.url- assert queue.pop().url == req3.url+ assert _peek(queue).url == req3.url+ assert _pop(queue).url == req3.urlassert not queue.close()def test_init_prios_with_start_queue(self):@@ -98,7 +111,7 @@ class TestPriorityQueue:start_queue_cls=PickleFifoDiskQueue,)assert len(queue2) == 1- assert queue2.pop().url == req.url+ assert _pop(queue2).url == req.urlqueue2.close()def test_queue_push_pop_priorities(self):@@ -115,7 +128,7 @@ class TestPriorityQueue:queue.push(req2)queue.push(req3)assert len(queue) == 3- dequeued = queue.pop()+ dequeued = _pop(queue)assert len(queue) == 2assert dequeued.url == req3.urlassert dequeued.priority == req3.priority@@ -125,7 +138,8 @@ class TestPriorityQueue:class TestDownloaderAwarePriorityQueue:def setup_method(self):crawler = get_crawler(Spider)- crawler.engine = Mock(downloader=MockDownloader())+ self.downloader = MockDownloader()+ crawler.engine = Mock(downloader=self.downloader)self.queue = build_from_crawler(DownloaderAwarePriorityQueue,crawler,@@ -146,11 +160,11 @@ class TestDownloaderAwarePriorityQueue:self.queue.push(req2)self.queue.push(req3)assert len(self.queue) == 3- assert self.queue.pop().url == req1.url+ assert _pop(self.queue).url == req1.urlassert len(self.queue) == 2- assert self.queue.pop().url == req2.url+ assert _pop(self.queue).url == req2.urlassert len(self.queue) == 1- assert self.queue.pop().url == req3.url+ assert _pop(self.queue).url == req3.urlassert len(self.queue) == 0assert self.queue.pop() is None@@ -175,14 +189,14 @@ class TestDownloaderAwarePriorityQueue:self.queue.push(req2)self.queue.push(req3)assert len(self.queue) == 3- assert self.queue.peek().url == req1.url- assert self.queue.pop().url == req1.url+ assert _peek(self.queue).url == req1.url+ assert _pop(self.queue).url == req1.urlassert len(self.queue) == 2- assert self.queue.peek().url == req2.url- assert self.queue.pop().url == req2.url+ assert _peek(self.queue).url == req2.url+ assert _pop(self.queue).url == req2.urlassert len(self.queue) == 1- assert self.queue.peek().url == req3.url- assert self.queue.pop().url == req3.url+ assert _peek(self.queue).url == req3.url+ assert _pop(self.queue).url == req3.urlassert self.queue.peek() is Nonedef test_tie_breaking_rotates_slots(self):@@ -201,10 +215,10 @@ class TestDownloaderAwarePriorityQueue:self.queue.push(request)slots = [- self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],- self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],- self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],- self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],+ _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],]assert slots == ["slot-a", "slot-b", "slot-a", "slot-b"]@@ -225,17 +239,15 @@ class TestDownloaderAwarePriorityQueue:self.queue.push(request)slots = [- self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],- self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],- self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],- self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],+ _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],]assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"]def test_pop_prefers_slot_with_fewer_active_downloads(self):- downloader = self.queue._downloader_interface.downloader-req_a = Request("https://example.org/a")req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"req_b = Request("https://example.org/b")@@ -246,10 +258,10 @@ class TestDownloaderAwarePriorityQueue:for req in (req_a, req_b, req_c):self.queue.push(req)- downloader.increment("slot-a")- downloader.increment("slot-c")+ self.downloader.increment("slot-a")+ self.downloader.increment("slot-c")- popped = self.queue.pop()+ popped = _pop(self.queue)assert popped.url == req_b.urldef test_contains(self):@@ -277,7 +289,7 @@ def test_slot_directory_removed_when_slot_drains(tmp_path):queue.push(request)assert slot_dir.is_dir()- assert queue.pop().url == request.url+ assert _pop(queue).url == request.urlassert not slot_dir.exists()queue.push(request)@@ -300,11 +312,11 @@ def test_slot_directory_removed_when_slot_drains(tmp_path):],)def test_pop_order(input_, output):- def make_url(index):+ def make_url(index: int) -> str:return f"https://toscrape.com/{index}"- def make_request(index, data):- meta = {}+ def make_request(index: int, data: dict[str, Any]) -> Request:+ meta: dict[str, Any] = {}if data.get("start", False):meta["is_start_request"] = Truereturn Request(@@ -332,7 +344,7 @@ def test_pop_order(input_, output):queue.push(request)actual_output_urls = []- while request := queue.pop():- actual_output_urls.append(request.url)+ while popped := queue.pop():+ actual_output_urls.append(popped.url)assert actual_output_urls == expected_output_urlsdiff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.pyindex 2c2d26d53..91c2481ba 100644--- a/tests/test_scheduler_base.py+++ b/tests/test_scheduler_base.py@@ -1,7 +1,7 @@from __future__ import annotationsimport logging-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyfrom urllib.parse import urljoinimport pytest@@ -59,13 +59,15 @@ class PathsSpider(Spider):def __init__(self, mockserver, *args, **kwargs):super().__init__(*args, **kwargs)- self.start_urls = map(mockserver.url, PATHS)+ self.start_urls = [mockserver.url(path) for path in PATHS]def parse(self, response):return {"path": urlparse_cached(response).path}class InterfaceCheckMixin:+ scheduler: Any+def test_scheduler_class(self):assert isinstance(self.scheduler, BaseScheduler)assert issubclass(self.scheduler.__class__, BaseScheduler)@@ -73,7 +75,7 @@ class InterfaceCheckMixin:class TestBaseScheduler(InterfaceCheckMixin):def setup_method(self):- self.scheduler = BaseScheduler()+ self.scheduler = BaseScheduler() # type: ignore[abstract]def test_methods(self):assert self.scheduler.open(Spider("foo")) is Nonediff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex 97c095ae0..2ad65f9bb 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -160,7 +160,7 @@ class TestBaseSettings:lowercase_var = "anothervalue"self.settings.attributes = {}- self.settings.setmodule(ModuleMock(), 10)+ self.settings.setmodule(ModuleMock(), 10) # type: ignore[arg-type]assert "UPPERCASE_VAR" in self.settings.attributesassert "MIXEDcase_VAR" not in self.settings.attributesassert "lowercase_var" not in self.settings.attributes@@ -219,7 +219,7 @@ class TestBaseSettings:)def test_update_kwargs(self):settings = BaseSettings({"key": 0})- settings.update(key=1) # pylint: disable=unexpected-keyword-arg+ settings.update(key=1) # type: ignore[call-arg] # pylint: disable=unexpected-keyword-argdef test_update_iterable(self):settings = BaseSettings({"key": 0}, priority=0)@@ -337,13 +337,13 @@ class TestBaseSettings:def test_getwithbase(self):s = BaseSettings({- "TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"),- "TEST": BaseSettings({1: 10, 3: 30}, "default"),- "HASNOBASE": BaseSettings({3: 3000}, "default"),+ "TEST_BASE": BaseSettings({"1": 1, "2": 2}, "project"),+ "TEST": BaseSettings({"1": 10, "3": 30}, "default"),+ "HASNOBASE": BaseSettings({"3": 3000}, "default"),})- s["TEST"].set(2, 200, "cmdline")- assert set(s.getwithbase("TEST")) == {1, 2, 3}+ s["TEST"].set("2", 200, "cmdline")+ assert set(s.getwithbase("TEST")) == {"1", "2", "3"}assert set(s.getwithbase("HASNOBASE")) == set(s["HASNOBASE"])assert s.getwithbase("NONEXISTENT") == {}@@ -527,7 +527,7 @@ class TestBaseSettings:):settings = BaseSettings()settings["FOO"] = BaseSettings(- {Component1: 1, "tests.test_settings.Component1": 2}+ {Component1: 1, "tests.test_settings.Component1": 2} # type: ignore[arg-type])with caplog.at_level(logging.WARNING):value = settings.get_component_priority_dict_with_base("FOO")@@ -592,14 +592,14 @@ class TestBaseSettings:with pytest.raises(ValueError, match="Base setting key must be a string, got 123"):- settings.getwithbase(123)+ settings.getwithbase(123) # type: ignore[arg-type]def test_get_component_priority_dict_with_base_invalid_setting_name(self):settings = BaseSettings()with pytest.raises(ValueError, match="Base setting key must be a string, got 123"):- settings.get_component_priority_dict_with_base(123)+ settings.get_component_priority_dict_with_base(123) # type: ignore[arg-type]class TestSettings:diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex 6bbc44829..067343b67 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -18,6 +18,12 @@ from scrapy.squeues import (_serializable_queue,)+# The disk queue classes are built at run time, so they are untyped.+_MarshalFifoQueue: Any = _MarshalFifoSerializationDiskQueue+_MarshalLifoQueue: Any = _MarshalLifoSerializationDiskQueue+_PickleFifoQueue: Any = _PickleFifoSerializationDiskQueue+_PickleLifoQueue: Any = _PickleLifoSerializationDiskQueue+class MyItem(Item):name = Field()@@ -71,6 +77,9 @@ def nonserializable_object_test(self):class FifoDiskQueueTestMixin:+ def queue(self) -> Any:+ raise NotImplementedError+def test_serialize(self):q = self.queue()q.push("a")@@ -87,8 +96,8 @@ class FifoDiskQueueTestMixin:class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):chunksize = 100000- def queue(self):- return _MarshalFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize)+ def queue(self) -> Any:+ return _MarshalFifoQueue(self.qpath, chunksize=self.chunksize)class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):@@ -110,8 +119,8 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):chunksize = 100000- def queue(self):- return _PickleFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize)+ def queue(self) -> Any:+ return _PickleFifoQueue(self.qpath, chunksize=self.chunksize)def test_serialize_item(self):q = self.queue()@@ -180,6 +189,9 @@ class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):class LifoDiskQueueTestMixin:+ def queue(self) -> Any:+ raise NotImplementedError+def test_serialize(self):q = self.queue()q.push("a")@@ -194,13 +206,13 @@ class LifoDiskQueueTestMixin:class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):- def queue(self):- return _MarshalLifoSerializationDiskQueue(self.qpath)+ def queue(self) -> Any:+ return _MarshalLifoQueue(self.qpath)class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):- def queue(self):- return _PickleLifoSerializationDiskQueue(self.qpath)+ def queue(self) -> Any:+ return _PickleLifoQueue(self.qpath)def test_serialize_item(self):q = self.queue()diff --git a/tests/test_stats.py b/tests/test_stats.pyindex 0f031fa02..6a762b3ea 100644--- a/tests/test_stats.py+++ b/tests/test_stats.py@@ -179,13 +179,13 @@ class TestStatsCollector:@coroutine_testasync def test_deprecated_spider_arg_custom_collector_subclass(self) -> None:class CustomStatsCollector(StatsCollector):- def open_spider(self, spider): # pylint: disable=signature-differs+ def open_spider(self, spider): # type: ignore[override] # pylint: disable=signature-differssuper().open_spider(spider)def inc_value(self, key, count=1, start=0, spider=None): # pylint: disable=useless-parent-delegationsuper().inc_value(key, count, start, spider)- def close_spider(self, spider, reason): # pylint: disable=signature-differs+ def close_spider(self, spider, reason): # type: ignore[override] # pylint: disable=signature-differssuper().close_spider(spider, reason)crawler = get_crawler(SimpleSpider, {"STATS_CLASS": CustomStatsCollector})