Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve Twisted-related type hints. (#8440)

Andrey Rakhmatullin3bf256e88e67
diff --git a/pyproject.toml b/pyproject.toml
index 8dc92a026..7c3460a82 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -129,10 +129,6 @@ follow_imports = "skip"
module = "scrapy.interfaces"
ignore_errors = true
-[[tool.mypy.overrides]]
-module = "twisted.internet.reactor"
-follow_imports = "skip"
-
# just for twisted.version
[[tool.mypy.overrides]]
module = "twisted"
diff --git a/scrapy/core/_http2/agent.py b/scrapy/core/_http2/agent.py
index 6a54d585f..c27d44083 100644
--- a/scrapy/core/_http2/agent.py
+++ b/scrapy/core/_http2/agent.py
@@ -13,9 +13,10 @@ from scrapy.core.downloader.contextfactory import _AcceptableProtocolsContextFac
if TYPE_CHECKING:
from twisted.internet.base import ReactorBase
- from twisted.internet.endpoints import HostnameEndpoint
+ from twisted.internet.interfaces import IStreamClientEndpoint
from twisted.python.failure import Failure
+ from scrapy.core.downloader.contextfactory import _PolicyForHTTPS
from scrapy.crawler import Crawler
from scrapy.http import Request, Response
from scrapy.spiders import Spider
@@ -43,7 +44,7 @@ class H2ConnectionPool:
)
def get_connection(
- self, key: ConnectionKey, uri: URI, endpoint: HostnameEndpoint
+ self, key: ConnectionKey, uri: URI, endpoint: IStreamClientEndpoint
) -> Deferred[H2ClientProtocol]:
if key in self._pending_requests:
# Received a request while connecting to remote
@@ -63,7 +64,7 @@ class H2ConnectionPool:
return self._new_connection(key, uri, endpoint)
def _new_connection(
- self, key: ConnectionKey, uri: URI, endpoint: HostnameEndpoint
+ self, key: ConnectionKey, uri: URI, endpoint: IStreamClientEndpoint
) -> Deferred[H2ClientProtocol]:
self._enforce_limit()
self._pending_requests[key] = deque()
@@ -76,7 +77,7 @@ class H2ConnectionPool:
conn_lost_deferred,
tls_verbose_logging=self._tls_verbose_logging,
)
- conn_d = endpoint.connect(factory)
+ conn_d: Deferred[H2ClientProtocol] = endpoint.connect(factory)
d = self._pending_request(key)
conn_d.addCallback(self.put_connection, key, conn_lost_deferred)
conn_d.addErrback(self._connection_failed, key)
@@ -160,7 +161,7 @@ class H2Agent:
self,
reactor: ReactorBase,
pool: H2ConnectionPool,
- context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008
+ context_factory: _PolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008
connect_timeout: float | None = None,
bind_address: tuple[str, int] | None = None,
) -> None:
@@ -173,8 +174,8 @@ class H2Agent:
self._reactor, self._context_factory, connect_timeout, bind_address
)
- def get_endpoint(self, uri: URI) -> HostnameEndpoint:
- return self.endpoint_factory.endpointForURI(uri) # type: ignore[no-any-return]
+ def get_endpoint(self, uri: URI) -> IStreamClientEndpoint:
+ return self.endpoint_factory.endpointForURI(uri)
def get_key(self, uri: URI) -> ConnectionKey:
"""
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 66f7ffef2..d91f7cf6e 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import warnings
-from typing import TYPE_CHECKING, Any, Self, cast
+from typing import TYPE_CHECKING, Any, Protocol, Self, cast
from formerly import deprecated_class
from OpenSSL import SSL
@@ -33,11 +33,20 @@ from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
from twisted.internet._sslverify import ClientTLSOptions
+ from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
+class _PolicyForHTTPS(Protocol):
+ """:class:`~twisted.web.iweb.IPolicyForHTTPS` for type checkers."""
+
+ def creatorForNetloc(
+ self, hostname: bytes, port: int
+ ) -> IOpenSSLClientConnectionCreator: ...
+
+
@implementer(IPolicyForHTTPS)
class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
"""Non-peer-certificate verifying HTTPS context factory.
@@ -245,8 +254,10 @@ class _AcceptableProtocolsContextFactory:
# When we drop DOWNLOADER_CLIENTCONTEXTFACTORY it looks like we can replace
# all of this with _ScrapyClientContextFactory.acceptableProtocols.
- def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]):
- self._wrapped_context_factory: Any = context_factory
+ def __init__(
+ self, context_factory: _PolicyForHTTPS, acceptable_protocols: list[bytes]
+ ):
+ self._wrapped_context_factory: _PolicyForHTTPS = context_factory
self._acceptable_protocols: list[bytes] = acceptable_protocols
def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
@@ -284,7 +295,7 @@ class _ScrapyCertificateOptions(CertificateOptions):
return ctx
-def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS:
+def _load_context_factory_from_settings(crawler: Crawler) -> _PolicyForHTTPS:
"""Create an instance of :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`.
Also passes values of other relevant settings to the factory class.
@@ -310,14 +321,7 @@ def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS:
context_factory_cls = load_object(
crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
)
- return cast(
- "IPolicyForHTTPS",
- build_from_crawler(
- context_factory_cls,
- crawler,
- method=tls_method,
- ),
- )
+ return build_from_crawler(context_factory_cls, crawler, method=tls_method)
def load_context_factory_from_settings(
@@ -328,4 +332,4 @@ def load_context_factory_from_settings(
ScrapyDeprecationWarning,
stacklevel=2,
)
- return _load_context_factory_from_settings(crawler)
+ return cast("IPolicyForHTTPS", _load_context_factory_from_settings(crawler))
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 591957521..98a85e6ca 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -40,7 +40,7 @@ from twisted.web.client import (
from twisted.web.client import Response as TxResponse
from twisted.web.http import PotentialDataLoss, _DataLoss
from twisted.web.http_headers import Headers as TxHeaders
-from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponse
+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse
from zope.interface import implementer
from scrapy import Request, signals
@@ -81,6 +81,7 @@ if TYPE_CHECKING:
from twisted.internet.interfaces import IAddress, IConsumer, IProtocol
from twisted.web._newclient import Request as TxRequest
+ from scrapy.core.downloader.contextfactory import _PolicyForHTTPS
from scrapy.crawler import Crawler
@@ -289,7 +290,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
)
self._pool.maxPersistentPerHost = self._pool_size_per_host
- self._contextFactory: IPolicyForHTTPS = _load_context_factory_from_settings(
+ self._contextFactory: _PolicyForHTTPS = _load_context_factory_from_settings(
crawler
)
self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
@@ -614,7 +615,7 @@ class _BindAddressAgent(Agent):
def __init__(
self,
reactor: ReactorBase,
- contextFactory: IPolicyForHTTPS,
+ contextFactory: _PolicyForHTTPS,
connectTimeout: float | None = None,
bindAddress: tuple[str, int] | None = None,
pool: HTTPConnectionPool | None = None,
@@ -655,14 +656,14 @@ class _TunnelingAgent(_BindAddressAgent):
*,
reactor: ReactorBase,
proxyConf: tuple[str, int, tuple[tuple[str, str], ...], bool],
- contextFactory: IPolicyForHTTPS,
+ contextFactory: _PolicyForHTTPS,
connectTimeout: float | None = None,
bindAddress: tuple[str, int] | None = None,
pool: HTTPConnectionPool | None = None,
):
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
self._proxyConf: tuple[str, int, tuple[tuple[str, str], ...], bool] = proxyConf
- self._contextFactory: IPolicyForHTTPS = contextFactory
+ self._contextFactory: _PolicyForHTTPS = contextFactory
def _getEndpoint(self, uri: URI) -> IStreamClientEndpoint:
proxyHost, proxyPort, proxyHeaders, proxyTLS = self._proxyConf
@@ -690,19 +691,15 @@ class _TunnelingAgent(_BindAddressAgent):
if proxyTLS:
# Set up TLS with the proxy itself, so that the CONNECT request and
# the tunneled traffic are sent over it (TLS-in-TLS).
- proxyCreator = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc]
- proxyHost.encode("ascii"), # type: ignore[arg-type]
- proxyPort,
+ proxyCreator = self._contextFactory.creatorForNetloc(
+ proxyHost.encode("ascii"), proxyPort
)
endpoint = wrapClientTLS(proxyCreator, endpoint)
endpoint = _TunnelEndpoint(
endpoint, uri.host, uri.port, proxyHost, proxyPort, proxyHeaders
)
# Set proper Server Name Indication extension for the destination.
- destCreator = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc]
- uri.host,
- uri.port,
- )
+ destCreator = self._contextFactory.creatorForNetloc(uri.host, uri.port)
return wrapClientTLS(destCreator, endpoint)
def _requestWithEndpoint(
@@ -736,7 +733,7 @@ class _ScrapyProxyAgent(_BindAddressAgent):
self,
reactor: ReactorBase,
proxyURI: bytes,
- contextFactory: IPolicyForHTTPS,
+ contextFactory: _PolicyForHTTPS,
connectTimeout: float | None = None,
bindAddress: tuple[str, int] | None = None,
pool: HTTPConnectionPool | None = None,
@@ -777,7 +774,7 @@ class _ScrapyAgent:
def __init__(
self,
*,
- contextFactory: IPolicyForHTTPS,
+ contextFactory: _PolicyForHTTPS,
connectTimeout: float = 10,
bindAddress: str | tuple[str, int] | None = None,
pool: HTTPConnectionPool | None = None,
@@ -787,7 +784,7 @@ class _ScrapyAgent:
crawler: Crawler,
tls_verbose_logging: bool = False,
):
- self._contextFactory: IPolicyForHTTPS = contextFactory
+ self._contextFactory: _PolicyForHTTPS = contextFactory
self._connectTimeout: float = connectTimeout
self._bindAddress: str | tuple[str, int] | None = bindAddress
self._pool: HTTPConnectionPool | None = pool
@@ -1165,11 +1162,8 @@ class _ResponseReader(Protocol):
# Twisted ends a response body in one of three ways: ResponseDone,
# PotentialDataLoss, or a ResponseFailed wrapping _DataLoss. See
# twisted.web._newclient.HTTPClientParser.connectionLost().
- assert reason.check(ResponseFailed)
- assert any(
- r.check(_DataLoss)
- for r in reason.value.reasons # type: ignore[union-attr]
- )
+ assert isinstance(reason.value, ResponseFailed)
+ assert any(r.check(_DataLoss) for r in reason.value.reasons)
if not self._fail_on_dataloss:
self._finish_response(flags=["dataloss"])
return
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 2dc5ea848..4729cfc32 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -26,8 +26,8 @@ from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
from twisted.internet.base import DelayedCall
from twisted.internet.defer import Deferred
- from twisted.web.iweb import IPolicyForHTTPS
+ from scrapy.core.downloader.contextfactory import _PolicyForHTTPS
from scrapy.crawler import Crawler
from scrapy.http import Request, Response
from scrapy.spiders import Spider
@@ -81,7 +81,7 @@ class H2DownloadHandler(BaseHttpDownloadHandler):
class _ScrapyH2Agent:
def __init__(
self,
- context_factory: IPolicyForHTTPS,
+ context_factory: _PolicyForHTTPS,
pool: H2ConnectionPool,
connect_timeout: int = 10,
bind_address: str | tuple[str, int] | None = None,
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 3085a4be5..5c9777468 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -23,6 +23,10 @@ from scrapy.utils.reactor import listen_tcp
from scrapy.utils.trackref import print_live_refs
if TYPE_CHECKING:
+ from collections.abc import Callable
+
+ from twisted.cred.credentials import UsernamePassword
+ from twisted.internet.defer import Deferred
from twisted.internet.tcp import Port
from scrapy.crawler import Crawler
@@ -110,7 +114,12 @@ class TelnetConsole(protocol.ServerFactory):
class Portal:
"""An implementation of IPortal"""
- def login(self_, credentials, mind, *interfaces): # type: ignore[no-untyped-def] # pylint: disable=no-self-argument
+ def login( # pylint: disable=no-self-argument
+ self_,
+ credentials: UsernamePassword,
+ mind: object,
+ *interfaces: object,
+ ) -> Deferred[tuple[object, object, Callable[[], None]]]:
if not (
credentials.username == self.username.encode()
and credentials.checkPassword(self.password.encode())
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index 9dfabcdb8..9c1a99287 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -337,7 +337,9 @@ class TestContextFactory(TestContextFactoryBase):
@pytest.fixture
def factory(self) -> _ScrapyClientContextFactory:
crawler = get_crawler()
- return _load_context_factory_from_settings(crawler)
+ factory = _load_context_factory_from_settings(crawler)
+ assert isinstance(factory, _ScrapyClientContextFactory)
+ return factory
@staticmethod
def _get_dummy_protocol() -> TLSMemoryBIOProtocol:
@@ -435,9 +437,7 @@ class TestContextFactory(TestContextFactoryBase):
crawler = get_crawler(
settings_dict={"DOWNLOAD_VERIFY_CERTIFICATES": verify_certs}
)
- factory: _ScrapyClientContextFactory = _load_context_factory_from_settings(
- crawler
- )
+ factory = _load_context_factory_from_settings(crawler)
creator = factory.creatorForNetloc(b"website.tld", 443)
conn = creator.clientConnectionForTLS(self._get_dummy_protocol())
assert conn.get_context()._keylog_callback is not None
@@ -523,6 +523,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
async def test_setting_default(self, server_url: str) -> None:
crawler = get_crawler()
client_context_factory = _load_context_factory_from_settings(crawler)
+ assert isinstance(client_context_factory, _ScrapyClientContextFactory)
assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD
await self._assert_factory_works(server_url, client_context_factory)
@@ -559,6 +560,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated",
):
client_context_factory = _load_context_factory_from_settings(crawler)
+ assert isinstance(client_context_factory, _ScrapyClientContextFactory)
assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD
await self._assert_factory_works(server_url, client_context_factory)
diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py
index c545fa584..c600481f8 100644
--- a/tests/test_downloader_handler_twisted_http11.py
+++ b/tests/test_downloader_handler_twisted_http11.py
@@ -75,7 +75,6 @@ if TYPE_CHECKING:
from twisted.internet.base import ReactorBase
from twisted.internet.interfaces import IProtocol, IStreamClientEndpoint
- from twisted.web.iweb import IPolicyForHTTPS
from scrapy.core.downloader.handlers import DownloadHandlerProtocol
from tests.utils.bases.download_handlers_http import HeaderNameCase
@@ -478,7 +477,7 @@ def _proxy_tunnel_endpoint(
agent = _TunnelingAgent(
reactor=reactor,
proxyConf=(proxy_host, proxy_port, (), False),
- contextFactory=cast("IPolicyForHTTPS", BrowserLikePolicyForHTTPS()),
+ contextFactory=BrowserLikePolicyForHTTPS(),
connectTimeout=10,
)
return agent._getEndpoint(URI.fromBytes(b"https://example.com"))
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 383976295..ceee8d865 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -15,7 +15,6 @@ from pytest_twisted import async_yield_fixture
from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS
-from twisted.internet.task import deferLater
from twisted.web.client import URI, ResponseFailed
from twisted.web.http import H2_ENABLED
from twisted.web.http import Request as TxRequest
@@ -29,6 +28,7 @@ from scrapy.exceptions import (
)
from scrapy.http import JsonRequest, Request, Response
from scrapy.spiders import Spider
+from scrapy.utils.asyncio import sleep
from scrapy.utils.defer import (
deferred_f_from_coro_f,
deferred_from_coro,
@@ -780,10 +780,9 @@ class TestHttps2ClientProtocol:
self, client: H2ClientProtocol
) -> None:
"""A connection with no streams is closed when it times out."""
- from twisted.internet import reactor
-
client.setTimeout(0.1) # type: ignore[no-untyped-call]
- await maybe_deferred_to_future(deferLater(reactor, 0.5))
+ await sleep(0.5)
+ assert client.transport is not None
assert client.transport.disconnecting
@deferred_f_from_coro_f