Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix most of the new typing issues in utils.

Andrey Rakhmatullin232aab53b385
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 21cd5e78f..8fee5a7ed 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -10,6 +10,7 @@ from typing import (
AsyncGenerator,
AsyncIterable,
Callable,
+ cast,
Coroutine,
Generator,
Iterable,
@@ -38,8 +39,8 @@ def defer_fail(_failure: Failure) -> Deferred:
"""
from twisted.internet import reactor
- d = Deferred()
- reactor.callLater(0.1, d.errback, _failure)
+ d: Deferred = Deferred()
+ reactor.callLater(0.1, d.errback, _failure) # type: ignore[attr-defined]
return d
@@ -52,8 +53,8 @@ def defer_succeed(result) -> Deferred:
"""
from twisted.internet import reactor
- d = Deferred()
- reactor.callLater(0.1, d.callback, result)
+ d: Deferred = Deferred()
+ reactor.callLater(0.1, d.callback, result) # type: ignore[attr-defined]
return d
@@ -182,7 +183,9 @@ class _AsyncCooperatorAdapter(Iterator):
def _call_anext(self) -> None:
# This starts waiting for the next result from aiterator.
# If aiterator is exhausted, _errback will be called.
- self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
+ self.anext_deferred = cast(
+ Deferred, deferred_from_coro(self.aiterator.__anext__())
+ )
self.anext_deferred.addCallbacks(self._callback, self._errback)
def __next__(self) -> Deferred:
@@ -190,7 +193,7 @@ class _AsyncCooperatorAdapter(Iterator):
# It also calls __anext__() if needed.
if self.finished:
raise StopIteration
- d = Deferred()
+ d: Deferred = Deferred()
self.waiting_deferreds.append(d)
if not self.anext_deferred:
self._call_anext()
@@ -209,7 +212,7 @@ def parallel_async(
def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
"""Return a Deferred built by chaining the given callbacks"""
- d = Deferred()
+ d: Deferred = Deferred()
for x in callbacks:
d.addCallback(x, *a, **kw)
d.callback(input)
@@ -220,7 +223,7 @@ def process_chain_both(
callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw
) -> Deferred:
"""Return a Deferred built by chaining the given callbacks and errbacks"""
- d = Deferred()
+ d: Deferred = Deferred()
for cb, eb in zip(callbacks, errbacks):
d.addCallbacks(
callback=cb,
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index a91a49170..40a32309a 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -6,7 +6,7 @@ import os
import re
import tempfile
import webbrowser
-from typing import Any, Callable, Iterable, Optional, Tuple, Union
+from typing import Any, Callable, Iterable, Tuple, Union
from weakref import WeakKeyDictionary
from twisted.web import http
@@ -38,12 +38,13 @@ _metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[floa
def get_meta_refresh(
response: "scrapy.http.response.text.TextResponse",
- ignore_tags: Optional[Iterable[str]] = ("script", "noscript"),
+ ignore_tags: Iterable[str] = ("script", "noscript"),
) -> Union[Tuple[None, None], Tuple[float, str]]:
"""Parse the http-equiv refresh parameter from the given response"""
if response not in _metaref_cache:
text = response.text[0:4096]
- _metaref_cache[response] = html.get_meta_refresh(
+ # a w3lib typing bug here
+ _metaref_cache[response] = html.get_meta_refresh( # type: ignore[assignment]
text, response.url, response.encoding, ignore_tags=ignore_tags
)
return _metaref_cache[response]
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index a47756c4b..119be1dfb 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -50,6 +50,6 @@ def test_site():
if __name__ == "__main__":
from twisted.internet import reactor
- port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
+ port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") # type: ignore[attr-defined]
print(f"http://localhost:{port.getHost().port}/")
- reactor.run()
+ reactor.run() # type: ignore[attr-defined]