Scrapy, a fast high-level web crawling & scraping framework for Python.
Ignore typing for twisted.internet.reactor globally.
Andrey Rakhmatullinc0efb271a232
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex 20cd50c5a..25ac0307b 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -28,7 +28,7 @@ class H2DownloadHandler:from twisted.internet import reactor- self._pool = H2ConnectionPool(reactor, settings) # type: ignore[arg-type]+ self._pool = H2ConnectionPool(reactor, settings)self._context_factory = load_context_factory_from_settings(settings, crawler)@classmethod@@ -82,7 +82,7 @@ class ScrapyH2Agent:"Tunneling via CONNECT method using HTTP/2.0 is not yet supported")return self._ProxyAgent(- reactor=reactor, # type: ignore[arg-type]+ reactor=reactor,context_factory=self._context_factory,proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")),connect_timeout=timeout,@@ -91,7 +91,7 @@ class ScrapyH2Agent:)return self._Agent(- reactor=reactor, # type: ignore[arg-type]+ reactor=reactor,context_factory=self._context_factory,connect_timeout=timeout,bind_address=bind_address,@@ -108,7 +108,7 @@ class ScrapyH2Agent:d = agent.request(request, spider)d.addCallback(self._cb_latency, request, start_time)- timeout_cl = reactor.callLater(timeout, d.cancel) # type: ignore[attr-defined]+ timeout_cl = reactor.callLater(timeout, d.cancel)d.addBoth(self._cb_timeout, request, timeout, timeout_cl)return ddiff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.pyindex 750d3c093..32bc2e38c 100644--- a/scrapy/utils/benchserver.py+++ b/scrapy/utils/benchserver.py@@ -37,11 +37,11 @@ if __name__ == "__main__":root = Root()factory = Site(root)- httpPort = reactor.listenTCP(8998, Site(root)) # type: ignore[attr-defined]+ httpPort = reactor.listenTCP(8998, Site(root))def _print_listening():httpHost = httpPort.getHost()print(f"Bench server at http://{httpHost.host}:{httpHost.port}")- reactor.callWhenRunning(_print_listening) # type: ignore[attr-defined]- reactor.run() # type: ignore[attr-defined]+ reactor.callWhenRunning(_print_listening)+ reactor.run()diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 8fee5a7ed..7f2211877 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -40,7 +40,7 @@ def defer_fail(_failure: Failure) -> Deferred:from twisted.internet import reactord: Deferred = Deferred()- reactor.callLater(0.1, d.errback, _failure) # type: ignore[attr-defined]+ reactor.callLater(0.1, d.errback, _failure)return d@@ -54,7 +54,7 @@ def defer_succeed(result) -> Deferred:from twisted.internet import reactord: Deferred = Deferred()- reactor.callLater(0.1, d.callback, result) # type: ignore[attr-defined]+ reactor.callLater(0.1, d.callback, result)return ddiff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.pyindex 119be1dfb..a47756c4b 100644--- a/scrapy/utils/testsite.py+++ b/scrapy/utils/testsite.py@@ -50,6 +50,6 @@ def test_site():if __name__ == "__main__":from twisted.internet import reactor- port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") # type: ignore[attr-defined]+ port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")print(f"http://localhost:{port.getHost().port}/")- reactor.run() # type: ignore[attr-defined]+ reactor.run()diff --git a/setup.cfg b/setup.cfgindex af9c87945..db79c5821 100644--- a/setup.cfg+++ b/setup.cfg@@ -11,6 +11,9 @@ ignore_missing_imports = true[mypy-twisted.internet.interfaces]follow_imports = skip+[mypy-twisted.internet.reactor]+follow_imports = skip+# FIXME: remove the following sections once the issues are solved[mypy-scrapy.downloadermiddlewares.httpproxy]diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.pyindex b9a4485a9..26db16dd6 100644--- a/tests/CrawlerRunner/ip_address.py+++ b/tests/CrawlerRunner/ip_address.py@@ -41,10 +41,10 @@ if __name__ == "__main__":url = f"http://not.a.real.domain:{port}/echo"servers = [(mock_dns_server.host, mock_dns_server.port)]- reactor.installResolver(createResolver(servers=servers)) # type: ignore[attr-defined]+ reactor.installResolver(createResolver(servers=servers))configure_logging()runner = CrawlerRunner()d = runner.crawl(LocalhostSpider, url=url)- d.addBoth(lambda _: reactor.stop()) # type: ignore[attr-defined]- reactor.run() # type: ignore[attr-defined]+ d.addBoth(lambda _: reactor.stop())+ reactor.run()diff --git a/tests/mockserver.py b/tests/mockserver.pyindex 185897373..eb1c03db7 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -375,9 +375,9 @@ if __name__ == "__main__":if args.type == "http":root = Root()factory = Site(root)- httpPort = reactor.listenTCP(0, factory) # type: ignore[attr-defined]+ httpPort = reactor.listenTCP(0, factory)contextFactory = ssl_context_factory()- httpsPort = reactor.listenSSL(0, factory, contextFactory) # type: ignore[attr-defined]+ httpsPort = reactor.listenSSL(0, factory, contextFactory)def print_listening():httpHost = httpPort.getHost()@@ -391,11 +391,11 @@ if __name__ == "__main__":clients = [MockDNSResolver()]factory = DNSServerFactory(clients=clients)protocol = dns.DNSDatagramProtocol(controller=factory)- listener = reactor.listenUDP(0, protocol) # type: ignore[attr-defined]+ listener = reactor.listenUDP(0, protocol)def print_listening():host = listener.getHost()print(f"{host.host}:{host.port}")- reactor.callWhenRunning(print_listening) # type: ignore[attr-defined]- reactor.run() # type: ignore[attr-defined]+ reactor.callWhenRunning(print_listening)+ reactor.run()diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 5fb87424b..7ddb420ba 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -557,4 +557,4 @@ class EngineTest(unittest.TestCase):if __name__ == "__main__":if len(sys.argv) > 1 and sys.argv[1] == "runserver":start_test_site(debug=True)- reactor.run() # type: ignore[attr-defined]+ reactor.run()