Scrapy, a fast high-level web crawling & scraping framework for Python.
Bump mypy and stubs.
Andrey Rakhmatullinf7af7b282d6d
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.pyindex 545dcaac9..43348f632 100644--- a/scrapy/downloadermiddlewares/cookies.py+++ b/scrapy/downloadermiddlewares/cookies.py@@ -131,25 +131,26 @@ class CookiesMiddleware:decoded = {}flags = set()for key in ("name", "value", "path", "domain"):- if cookie.get(key) is None:+ value = cookie.get(key)+ if value is None:if key in ("name", "value"):msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"logger.warning(msg)return Nonecontinue- # https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168- if isinstance(cookie[key], (bool, float, int, str)): # type: ignore[literal-required]- decoded[key] = str(cookie[key]) # type: ignore[literal-required]+ if isinstance(value, (bool, float, int, str)):+ decoded[key] = str(value)else:+ assert isinstance(value, bytes)try:- decoded[key] = cookie[key].decode("utf8") # type: ignore[literal-required]+ decoded[key] = value.decode("utf8")except UnicodeDecodeError:logger.warning("Non UTF-8 encoded cookie found in request %s: %s",request,cookie,)- decoded[key] = cookie[key].decode("latin1", errors="replace") # type: ignore[literal-required]+ decoded[key] = value.decode("latin1", errors="replace")for flag in ("secure",):value = cookie.get(flag, _UNSET)if value is _UNSET or not value:diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.pyindex 89c83d20d..ee28d86ba 100644--- a/scrapy/extensions/telnet.py+++ b/scrapy/extensions/telnet.py@@ -75,7 +75,7 @@ class TelnetConsole(protocol.ServerFactory):def stop_listening(self) -> None:self.port.stopListening()- def protocol(self) -> telnet.TelnetTransport: # type: ignore[override]+ def protocol(self) -> telnet.TelnetTransport:# these import twisted.internet.reactorfrom twisted.conch import manhole, telnetfrom twisted.conch.insults import insultsdiff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex a96a215f4..3d6cf4816 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -44,10 +44,10 @@ if TYPE_CHECKING:class VerboseCookie(TypedDict):- name: str- value: str- domain: NotRequired[str]- path: NotRequired[str]+ name: str | bytes+ value: str | bytes | bool | float | int+ domain: NotRequired[str | bytes]+ path: NotRequired[str | bytes]secure: NotRequired[bool]diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.pyindex b60fe929e..e0d9f4595 100644--- a/scrapy/utils/sitemap.py+++ b/scrapy/utils/sitemap.py@@ -26,13 +26,15 @@ class Sitemap:)self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320rt = self._root.tag- self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt+ assert isinstance(rt, str)+ self.type = rt.split("}", 1)[1] if "}" in rt else rtdef __iter__(self) -> Iterator[dict[str, Any]]:for elem in self._root.getchildren():d: dict[str, Any] = {}for el in elem.getchildren():tag = el.tag+ assert isinstance(tag, str)name = tag.split("}", 1)[1] if "}" in tag else tagif name == "link":diff --git a/tox.ini b/tox.iniindex 24b674085..39ab1ccd4 100644--- a/tox.ini+++ b/tox.ini@@ -43,12 +43,12 @@ install_command =[testenv:typing]basepython = python3deps =- mypy==1.12.0+ mypy==1.14.0typing-extensions==4.12.2- types-lxml==2024.9.16+ types-lxml==2024.12.13types-Pygments==2.18.0.20240506- botocore-stubs==1.35.39- boto3-stubs[s3]==1.35.39+ botocore-stubs==1.35.90+ boto3-stubs[s3]==1.35.90attrs >= 18.2.0Pillow >= 10.3.0pyOpenSSL >= 24.2.1