Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve Twisted-related type hints. (#8440)
Andrey Rakhmatullin3bf256e88e67
diff --git a/pyproject.toml b/pyproject.tomlindex 8dc92a026..7c3460a82 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -129,10 +129,6 @@ follow_imports = "skip"module = "scrapy.interfaces"ignore_errors = true-[[tool.mypy.overrides]]-module = "twisted.internet.reactor"-follow_imports = "skip"-# just for twisted.version[[tool.mypy.overrides]]module = "twisted"diff --git a/scrapy/core/_http2/agent.py b/scrapy/core/_http2/agent.pyindex 6a54d585f..c27d44083 100644--- a/scrapy/core/_http2/agent.py+++ b/scrapy/core/_http2/agent.py@@ -13,9 +13,10 @@ from scrapy.core.downloader.contextfactory import _AcceptableProtocolsContextFacif TYPE_CHECKING:from twisted.internet.base import ReactorBase- from twisted.internet.endpoints import HostnameEndpoint+ from twisted.internet.interfaces import IStreamClientEndpointfrom twisted.python.failure import Failure+ from scrapy.core.downloader.contextfactory import _PolicyForHTTPSfrom scrapy.crawler import Crawlerfrom scrapy.http import Request, Responsefrom scrapy.spiders import Spider@@ -43,7 +44,7 @@ class H2ConnectionPool:)def get_connection(- self, key: ConnectionKey, uri: URI, endpoint: HostnameEndpoint+ self, key: ConnectionKey, uri: URI, endpoint: IStreamClientEndpoint) -> Deferred[H2ClientProtocol]:if key in self._pending_requests:# Received a request while connecting to remote@@ -63,7 +64,7 @@ class H2ConnectionPool:return self._new_connection(key, uri, endpoint)def _new_connection(- self, key: ConnectionKey, uri: URI, endpoint: HostnameEndpoint+ self, key: ConnectionKey, uri: URI, endpoint: IStreamClientEndpoint) -> Deferred[H2ClientProtocol]:self._enforce_limit()self._pending_requests[key] = deque()@@ -76,7 +77,7 @@ class H2ConnectionPool:conn_lost_deferred,tls_verbose_logging=self._tls_verbose_logging,)- conn_d = endpoint.connect(factory)+ conn_d: Deferred[H2ClientProtocol] = endpoint.connect(factory)d = self._pending_request(key)conn_d.addCallback(self.put_connection, key, conn_lost_deferred)conn_d.addErrback(self._connection_failed, key)@@ -160,7 +161,7 @@ class H2Agent:self,reactor: ReactorBase,pool: H2ConnectionPool,- context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008+ context_factory: _PolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008connect_timeout: float | None = None,bind_address: tuple[str, int] | None = None,) -> None:@@ -173,8 +174,8 @@ class H2Agent:self._reactor, self._context_factory, connect_timeout, bind_address)- def get_endpoint(self, uri: URI) -> HostnameEndpoint:- return self.endpoint_factory.endpointForURI(uri) # type: ignore[no-any-return]+ def get_endpoint(self, uri: URI) -> IStreamClientEndpoint:+ return self.endpoint_factory.endpointForURI(uri)def get_key(self, uri: URI) -> ConnectionKey:"""diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex 66f7ffef2..d91f7cf6e 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -1,7 +1,7 @@from __future__ import annotationsimport warnings-from typing import TYPE_CHECKING, Any, Self, cast+from typing import TYPE_CHECKING, Any, Protocol, Self, castfrom formerly import deprecated_classfrom OpenSSL import SSL@@ -33,11 +33,20 @@ from scrapy.utils.misc import build_from_crawler, load_objectif TYPE_CHECKING:from twisted.internet._sslverify import ClientTLSOptions+ from twisted.internet.interfaces import IOpenSSLClientConnectionCreatorfrom scrapy.crawler import Crawlerfrom scrapy.settings import BaseSettings+class _PolicyForHTTPS(Protocol):+ """:class:`~twisted.web.iweb.IPolicyForHTTPS` for type checkers."""++ def creatorForNetloc(+ self, hostname: bytes, port: int+ ) -> IOpenSSLClientConnectionCreator: ...++@implementer(IPolicyForHTTPS)class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):"""Non-peer-certificate verifying HTTPS context factory.@@ -245,8 +254,10 @@ class _AcceptableProtocolsContextFactory:# When we drop DOWNLOADER_CLIENTCONTEXTFACTORY it looks like we can replace# all of this with _ScrapyClientContextFactory.acceptableProtocols.- def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]):- self._wrapped_context_factory: Any = context_factory+ def __init__(+ self, context_factory: _PolicyForHTTPS, acceptable_protocols: list[bytes]+ ):+ self._wrapped_context_factory: _PolicyForHTTPS = context_factoryself._acceptable_protocols: list[bytes] = acceptable_protocolsdef creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:@@ -284,7 +295,7 @@ class _ScrapyCertificateOptions(CertificateOptions):return ctx-def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS:+def _load_context_factory_from_settings(crawler: Crawler) -> _PolicyForHTTPS:"""Create an instance of :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`.Also passes values of other relevant settings to the factory class.@@ -310,14 +321,7 @@ def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS:context_factory_cls = load_object(crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"])- return cast(- "IPolicyForHTTPS",- build_from_crawler(- context_factory_cls,- crawler,- method=tls_method,- ),- )+ return build_from_crawler(context_factory_cls, crawler, method=tls_method)def load_context_factory_from_settings(@@ -328,4 +332,4 @@ def load_context_factory_from_settings(ScrapyDeprecationWarning,stacklevel=2,)- return _load_context_factory_from_settings(crawler)+ return cast("IPolicyForHTTPS", _load_context_factory_from_settings(crawler))diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 591957521..98a85e6ca 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -40,7 +40,7 @@ from twisted.web.client import (from twisted.web.client import Response as TxResponsefrom twisted.web.http import PotentialDataLoss, _DataLossfrom twisted.web.http_headers import Headers as TxHeaders-from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponse+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponsefrom zope.interface import implementerfrom scrapy import Request, signals@@ -81,6 +81,7 @@ if TYPE_CHECKING:from twisted.internet.interfaces import IAddress, IConsumer, IProtocolfrom twisted.web._newclient import Request as TxRequest+ from scrapy.core.downloader.contextfactory import _PolicyForHTTPSfrom scrapy.crawler import Crawler@@ -289,7 +290,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):)self._pool.maxPersistentPerHost = self._pool_size_per_host- self._contextFactory: IPolicyForHTTPS = _load_context_factory_from_settings(+ self._contextFactory: _PolicyForHTTPS = _load_context_factory_from_settings(crawler)self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")@@ -614,7 +615,7 @@ class _BindAddressAgent(Agent):def __init__(self,reactor: ReactorBase,- contextFactory: IPolicyForHTTPS,+ contextFactory: _PolicyForHTTPS,connectTimeout: float | None = None,bindAddress: tuple[str, int] | None = None,pool: HTTPConnectionPool | None = None,@@ -655,14 +656,14 @@ class _TunnelingAgent(_BindAddressAgent):*,reactor: ReactorBase,proxyConf: tuple[str, int, tuple[tuple[str, str], ...], bool],- contextFactory: IPolicyForHTTPS,+ contextFactory: _PolicyForHTTPS,connectTimeout: float | None = None,bindAddress: tuple[str, int] | None = None,pool: HTTPConnectionPool | None = None,):super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)self._proxyConf: tuple[str, int, tuple[tuple[str, str], ...], bool] = proxyConf- self._contextFactory: IPolicyForHTTPS = contextFactory+ self._contextFactory: _PolicyForHTTPS = contextFactorydef _getEndpoint(self, uri: URI) -> IStreamClientEndpoint:proxyHost, proxyPort, proxyHeaders, proxyTLS = self._proxyConf@@ -690,19 +691,15 @@ class _TunnelingAgent(_BindAddressAgent):if proxyTLS:# Set up TLS with the proxy itself, so that the CONNECT request and# the tunneled traffic are sent over it (TLS-in-TLS).- proxyCreator = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc]- proxyHost.encode("ascii"), # type: ignore[arg-type]- proxyPort,+ proxyCreator = self._contextFactory.creatorForNetloc(+ proxyHost.encode("ascii"), proxyPort)endpoint = wrapClientTLS(proxyCreator, endpoint)endpoint = _TunnelEndpoint(endpoint, uri.host, uri.port, proxyHost, proxyPort, proxyHeaders)# Set proper Server Name Indication extension for the destination.- destCreator = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc]- uri.host,- uri.port,- )+ destCreator = self._contextFactory.creatorForNetloc(uri.host, uri.port)return wrapClientTLS(destCreator, endpoint)def _requestWithEndpoint(@@ -736,7 +733,7 @@ class _ScrapyProxyAgent(_BindAddressAgent):self,reactor: ReactorBase,proxyURI: bytes,- contextFactory: IPolicyForHTTPS,+ contextFactory: _PolicyForHTTPS,connectTimeout: float | None = None,bindAddress: tuple[str, int] | None = None,pool: HTTPConnectionPool | None = None,@@ -777,7 +774,7 @@ class _ScrapyAgent:def __init__(self,*,- contextFactory: IPolicyForHTTPS,+ contextFactory: _PolicyForHTTPS,connectTimeout: float = 10,bindAddress: str | tuple[str, int] | None = None,pool: HTTPConnectionPool | None = None,@@ -787,7 +784,7 @@ class _ScrapyAgent:crawler: Crawler,tls_verbose_logging: bool = False,):- self._contextFactory: IPolicyForHTTPS = contextFactory+ self._contextFactory: _PolicyForHTTPS = contextFactoryself._connectTimeout: float = connectTimeoutself._bindAddress: str | tuple[str, int] | None = bindAddressself._pool: HTTPConnectionPool | None = pool@@ -1165,11 +1162,8 @@ class _ResponseReader(Protocol):# Twisted ends a response body in one of three ways: ResponseDone,# PotentialDataLoss, or a ResponseFailed wrapping _DataLoss. See# twisted.web._newclient.HTTPClientParser.connectionLost().- assert reason.check(ResponseFailed)- assert any(- r.check(_DataLoss)- for r in reason.value.reasons # type: ignore[union-attr]- )+ assert isinstance(reason.value, ResponseFailed)+ assert any(r.check(_DataLoss) for r in reason.value.reasons)if not self._fail_on_dataloss:self._finish_response(flags=["dataloss"])returndiff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex 2dc5ea848..4729cfc32 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -26,8 +26,8 @@ from scrapy.utils.httpobj import urlparse_cachedif TYPE_CHECKING:from twisted.internet.base import DelayedCallfrom twisted.internet.defer import Deferred- from twisted.web.iweb import IPolicyForHTTPS+ from scrapy.core.downloader.contextfactory import _PolicyForHTTPSfrom scrapy.crawler import Crawlerfrom scrapy.http import Request, Responsefrom scrapy.spiders import Spider@@ -81,7 +81,7 @@ class H2DownloadHandler(BaseHttpDownloadHandler):class _ScrapyH2Agent:def __init__(self,- context_factory: IPolicyForHTTPS,+ context_factory: _PolicyForHTTPS,pool: H2ConnectionPool,connect_timeout: int = 10,bind_address: str | tuple[str, int] | None = None,diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.pyindex 3085a4be5..5c9777468 100644--- a/scrapy/extensions/telnet.py+++ b/scrapy/extensions/telnet.py@@ -23,6 +23,10 @@ from scrapy.utils.reactor import listen_tcpfrom scrapy.utils.trackref import print_live_refsif TYPE_CHECKING:+ from collections.abc import Callable++ from twisted.cred.credentials import UsernamePassword+ from twisted.internet.defer import Deferredfrom twisted.internet.tcp import Portfrom scrapy.crawler import Crawler@@ -110,7 +114,12 @@ class TelnetConsole(protocol.ServerFactory):class Portal:"""An implementation of IPortal"""- def login(self_, credentials, mind, *interfaces): # type: ignore[no-untyped-def] # pylint: disable=no-self-argument+ def login( # pylint: disable=no-self-argument+ self_,+ credentials: UsernamePassword,+ mind: object,+ *interfaces: object,+ ) -> Deferred[tuple[object, object, Callable[[], None]]]:if not (credentials.username == self.username.encode()and credentials.checkPassword(self.password.encode())diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex 9dfabcdb8..9c1a99287 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -337,7 +337,9 @@ class TestContextFactory(TestContextFactoryBase):@pytest.fixturedef factory(self) -> _ScrapyClientContextFactory:crawler = get_crawler()- return _load_context_factory_from_settings(crawler)+ factory = _load_context_factory_from_settings(crawler)+ assert isinstance(factory, _ScrapyClientContextFactory)+ return factory@staticmethoddef _get_dummy_protocol() -> TLSMemoryBIOProtocol:@@ -435,9 +437,7 @@ class TestContextFactory(TestContextFactoryBase):crawler = get_crawler(settings_dict={"DOWNLOAD_VERIFY_CERTIFICATES": verify_certs})- factory: _ScrapyClientContextFactory = _load_context_factory_from_settings(- crawler- )+ factory = _load_context_factory_from_settings(crawler)creator = factory.creatorForNetloc(b"website.tld", 443)conn = creator.clientConnectionForTLS(self._get_dummy_protocol())assert conn.get_context()._keylog_callback is not None@@ -523,6 +523,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):async def test_setting_default(self, server_url: str) -> None:crawler = get_crawler()client_context_factory = _load_context_factory_from_settings(crawler)+ assert isinstance(client_context_factory, _ScrapyClientContextFactory)assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHODawait self._assert_factory_works(server_url, client_context_factory)@@ -559,6 +560,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated",):client_context_factory = _load_context_factory_from_settings(crawler)+ assert isinstance(client_context_factory, _ScrapyClientContextFactory)assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHODawait self._assert_factory_works(server_url, client_context_factory)diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.pyindex c545fa584..c600481f8 100644--- a/tests/test_downloader_handler_twisted_http11.py+++ b/tests/test_downloader_handler_twisted_http11.py@@ -75,7 +75,6 @@ if TYPE_CHECKING:from twisted.internet.base import ReactorBasefrom twisted.internet.interfaces import IProtocol, IStreamClientEndpoint- from twisted.web.iweb import IPolicyForHTTPSfrom scrapy.core.downloader.handlers import DownloadHandlerProtocolfrom tests.utils.bases.download_handlers_http import HeaderNameCase@@ -478,7 +477,7 @@ def _proxy_tunnel_endpoint(agent = _TunnelingAgent(reactor=reactor,proxyConf=(proxy_host, proxy_port, (), False),- contextFactory=cast("IPolicyForHTTPS", BrowserLikePolicyForHTTPS()),+ contextFactory=BrowserLikePolicyForHTTPS(),connectTimeout=10,)return agent._getEndpoint(URI.fromBytes(b"https://example.com"))diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 383976295..ceee8d865 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -15,7 +15,6 @@ from pytest_twisted import async_yield_fixturefrom twisted.internet.defer import Deferred, DeferredList, inlineCallbacksfrom twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpointfrom twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS-from twisted.internet.task import deferLaterfrom twisted.web.client import URI, ResponseFailedfrom twisted.web.http import H2_ENABLEDfrom twisted.web.http import Request as TxRequest@@ -29,6 +28,7 @@ from scrapy.exceptions import ()from scrapy.http import JsonRequest, Request, Responsefrom scrapy.spiders import Spider+from scrapy.utils.asyncio import sleepfrom scrapy.utils.defer import (deferred_f_from_coro_f,deferred_from_coro,@@ -780,10 +780,9 @@ class TestHttps2ClientProtocol:self, client: H2ClientProtocol) -> None:"""A connection with no streams is closed when it times out."""- from twisted.internet import reactor-client.setTimeout(0.1) # type: ignore[no-untyped-call]- await maybe_deferred_to_future(deferLater(reactor, 0.5))+ await sleep(0.5)+ assert client.transport is not Noneassert client.transport.disconnecting@deferred_f_from_coro_f