Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Bump mypy and stubs.

Andrey Rakhmatullinf7af7b282d6d
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 545dcaac9..43348f632 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -131,25 +131,26 @@ class CookiesMiddleware:
decoded = {}
flags = set()
for key in ("name", "value", "path", "domain"):
- if cookie.get(key) is None:
+ value = cookie.get(key)
+ if value is None:
if key in ("name", "value"):
msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
logger.warning(msg)
return None
continue
- # https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168
- if isinstance(cookie[key], (bool, float, int, str)): # type: ignore[literal-required]
- decoded[key] = str(cookie[key]) # type: ignore[literal-required]
+ if isinstance(value, (bool, float, int, str)):
+ decoded[key] = str(value)
else:
+ assert isinstance(value, bytes)
try:
- decoded[key] = cookie[key].decode("utf8") # type: ignore[literal-required]
+ decoded[key] = value.decode("utf8")
except UnicodeDecodeError:
logger.warning(
"Non UTF-8 encoded cookie found in request %s: %s",
request,
cookie,
)
- decoded[key] = cookie[key].decode("latin1", errors="replace") # type: ignore[literal-required]
+ decoded[key] = value.decode("latin1", errors="replace")
for flag in ("secure",):
value = cookie.get(flag, _UNSET)
if value is _UNSET or not value:
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 89c83d20d..ee28d86ba 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -75,7 +75,7 @@ class TelnetConsole(protocol.ServerFactory):
def stop_listening(self) -> None:
self.port.stopListening()
- def protocol(self) -> telnet.TelnetTransport: # type: ignore[override]
+ def protocol(self) -> telnet.TelnetTransport:
# these import twisted.internet.reactor
from twisted.conch import manhole, telnet
from twisted.conch.insults import insults
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index a96a215f4..3d6cf4816 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -44,10 +44,10 @@ if TYPE_CHECKING:
class VerboseCookie(TypedDict):
- name: str
- value: str
- domain: NotRequired[str]
- path: NotRequired[str]
+ name: str | bytes
+ value: str | bytes | bool | float | int
+ domain: NotRequired[str | bytes]
+ path: NotRequired[str | bytes]
secure: NotRequired[bool]
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index b60fe929e..e0d9f4595 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -26,13 +26,15 @@ class Sitemap:
)
self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320
rt = self._root.tag
- self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt
+ assert isinstance(rt, str)
+ self.type = rt.split("}", 1)[1] if "}" in rt else rt
def __iter__(self) -> Iterator[dict[str, Any]]:
for elem in self._root.getchildren():
d: dict[str, Any] = {}
for el in elem.getchildren():
tag = el.tag
+ assert isinstance(tag, str)
name = tag.split("}", 1)[1] if "}" in tag else tag
if name == "link":
diff --git a/tox.ini b/tox.ini
index 24b674085..39ab1ccd4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -43,12 +43,12 @@ install_command =
[testenv:typing]
basepython = python3
deps =
- mypy==1.12.0
+ mypy==1.14.0
typing-extensions==4.12.2
- types-lxml==2024.9.16
+ types-lxml==2024.12.13
types-Pygments==2.18.0.20240506
- botocore-stubs==1.35.39
- boto3-stubs[s3]==1.35.39
+ botocore-stubs==1.35.90
+ boto3-stubs[s3]==1.35.90
attrs >= 18.2.0
Pillow >= 10.3.0
pyOpenSSL >= 24.2.1