Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix most of the new typing issues in utils.
Andrey Rakhmatullin232aab53b385
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 21cd5e78f..8fee5a7ed 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -10,6 +10,7 @@ from typing import (AsyncGenerator,AsyncIterable,Callable,+ cast,Coroutine,Generator,Iterable,@@ -38,8 +39,8 @@ def defer_fail(_failure: Failure) -> Deferred:"""from twisted.internet import reactor- d = Deferred()- reactor.callLater(0.1, d.errback, _failure)+ d: Deferred = Deferred()+ reactor.callLater(0.1, d.errback, _failure) # type: ignore[attr-defined]return d@@ -52,8 +53,8 @@ def defer_succeed(result) -> Deferred:"""from twisted.internet import reactor- d = Deferred()- reactor.callLater(0.1, d.callback, result)+ d: Deferred = Deferred()+ reactor.callLater(0.1, d.callback, result) # type: ignore[attr-defined]return d@@ -182,7 +183,9 @@ class _AsyncCooperatorAdapter(Iterator):def _call_anext(self) -> None:# This starts waiting for the next result from aiterator.# If aiterator is exhausted, _errback will be called.- self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())+ self.anext_deferred = cast(+ Deferred, deferred_from_coro(self.aiterator.__anext__())+ )self.anext_deferred.addCallbacks(self._callback, self._errback)def __next__(self) -> Deferred:@@ -190,7 +193,7 @@ class _AsyncCooperatorAdapter(Iterator):# It also calls __anext__() if needed.if self.finished:raise StopIteration- d = Deferred()+ d: Deferred = Deferred()self.waiting_deferreds.append(d)if not self.anext_deferred:self._call_anext()@@ -209,7 +212,7 @@ def parallel_async(def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:"""Return a Deferred built by chaining the given callbacks"""- d = Deferred()+ d: Deferred = Deferred()for x in callbacks:d.addCallback(x, *a, **kw)d.callback(input)@@ -220,7 +223,7 @@ def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> Deferred:"""Return a Deferred built by chaining the given callbacks and errbacks"""- d = Deferred()+ d: Deferred = Deferred()for cb, eb in zip(callbacks, errbacks):d.addCallbacks(callback=cb,diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex a91a49170..40a32309a 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -6,7 +6,7 @@ import osimport reimport tempfileimport webbrowser-from typing import Any, Callable, Iterable, Optional, Tuple, Union+from typing import Any, Callable, Iterable, Tuple, Unionfrom weakref import WeakKeyDictionaryfrom twisted.web import http@@ -38,12 +38,13 @@ _metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[floadef get_meta_refresh(response: "scrapy.http.response.text.TextResponse",- ignore_tags: Optional[Iterable[str]] = ("script", "noscript"),+ ignore_tags: Iterable[str] = ("script", "noscript"),) -> Union[Tuple[None, None], Tuple[float, str]]:"""Parse the http-equiv refresh parameter from the given response"""if response not in _metaref_cache:text = response.text[0:4096]- _metaref_cache[response] = html.get_meta_refresh(+ # a w3lib typing bug here+ _metaref_cache[response] = html.get_meta_refresh( # type: ignore[assignment]text, response.url, response.encoding, ignore_tags=ignore_tags)return _metaref_cache[response]diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.pyindex a47756c4b..119be1dfb 100644--- a/scrapy/utils/testsite.py+++ b/scrapy/utils/testsite.py@@ -50,6 +50,6 @@ def test_site():if __name__ == "__main__":from twisted.internet import reactor- port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")+ port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") # type: ignore[attr-defined]print(f"http://localhost:{port.getHost().port}/")- reactor.run()+ reactor.run() # type: ignore[attr-defined]