Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

More typing for scrapy/core/downloader/handlers/http11.py.

Andrey Rakhmatullin045387e07fab
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 2e7ea559a..40ae1921a 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -8,31 +8,33 @@ import re
from contextlib import suppress
from io import BytesIO
from time import time
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
from urllib.parse import urldefrag, urlunparse
-from twisted.internet import defer, protocol, ssl
+from twisted.internet import ssl
+from twisted.internet.base import ReactorBase
+from twisted.internet.defer import CancelledError, Deferred, succeed
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import TimeoutError
+from twisted.internet.interfaces import IConsumer
+from twisted.internet.protocol import Factory, Protocol, connectionDone
from twisted.python.failure import Failure
-from twisted.web.client import (
- URI,
- Agent,
- HTTPConnectionPool,
- ResponseDone,
- ResponseFailed,
-)
+from twisted.web.client import URI, Agent, HTTPConnectionPool
+from twisted.web.client import Response as TxResponse
+from twisted.web.client import ResponseDone, ResponseFailed
from twisted.web.http import PotentialDataLoss, _DataLoss
from twisted.web.http_headers import Headers as TxHeaders
-from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS
from zope.interface import implementer
from scrapy import Request, Spider, signals
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
+from scrapy.crawler import Crawler
from scrapy.exceptions import StopDownload
from scrapy.http import Headers, Response
from scrapy.responsetypes import responsetypes
+from scrapy.settings import BaseSettings
from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
@@ -46,28 +48,30 @@ logger = logging.getLogger(__name__)
class HTTP11DownloadHandler:
lazy = False
- def __init__(self, settings, crawler):
+ def __init__(self, settings: BaseSettings, crawler: Crawler):
self._crawler = crawler
from twisted.internet import reactor
- self._pool = HTTPConnectionPool(reactor, persistent=True)
+ self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True)
self._pool.maxPersistentPerHost = settings.getint(
"CONCURRENT_REQUESTS_PER_DOMAIN"
)
self._pool._factory.noisy = False
- self._contextFactory = load_context_factory_from_settings(settings, crawler)
- self._default_maxsize = settings.getint("DOWNLOAD_MAXSIZE")
- self._default_warnsize = settings.getint("DOWNLOAD_WARNSIZE")
- self._fail_on_dataloss = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS")
- self._disconnect_timeout = 1
+ self._contextFactory: IPolicyForHTTPS = load_context_factory_from_settings(
+ settings, crawler
+ )
+ self._default_maxsize: int = settings.getint("DOWNLOAD_MAXSIZE")
+ self._default_warnsize: int = settings.getint("DOWNLOAD_WARNSIZE")
+ self._fail_on_dataloss: bool = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS")
+ self._disconnect_timeout: int = 1
@classmethod
- def from_crawler(cls, crawler) -> Self:
+ def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings, crawler)
- def download_request(self, request: Request, spider: Spider) -> Response:
+ def download_request(self, request: Request, spider: Spider) -> Deferred:
"""Return a deferred for the HTTP download"""
agent = ScrapyAgent(
contextFactory=self._contextFactory,
@@ -79,10 +83,10 @@ class HTTP11DownloadHandler:
)
return agent.download_request(request)
- def close(self):
+ def close(self) -> Deferred:
from twisted.internet import reactor
- d = self._pool.closeCachedConnections()
+ d: Deferred = self._pool.closeCachedConnections()
# closeCachedConnections will hang on network or server issues, so
# we'll manually timeout the deferred.
#
@@ -93,7 +97,7 @@ class HTTP11DownloadHandler:
# issue a callback after `_disconnect_timeout` seconds.
delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])
- def cancel_delayed_call(result):
+ def cancel_delayed_call(result: Any) -> Any:
if delayed_call.active():
delayed_call.cancel()
return result
@@ -123,39 +127,41 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def __init__(
self,
- reactor,
- host,
- port,
- proxyConf,
- contextFactory,
- timeout=30,
- bindAddress=None,
+ reactor: ReactorBase,
+ host: str,
+ port: int,
+ proxyConf: Tuple[str, int, Optional[bytes]],
+ contextFactory: IPolicyForHTTPS,
+ timeout: float = 30,
+ bindAddress: Optional[Tuple[str, int]] = None,
):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
- self._tunnelReadyDeferred = defer.Deferred()
- self._tunneledHost = host
- self._tunneledPort = port
- self._contextFactory = contextFactory
- self._connectBuffer = bytearray()
+ self._tunnelReadyDeferred: Deferred = Deferred()
+ self._tunneledHost: str = host
+ self._tunneledPort: int = port
+ self._contextFactory: IPolicyForHTTPS = contextFactory
+ self._connectBuffer: bytearray = bytearray()
- def requestTunnel(self, protocol):
+ def requestTunnel(self, protocol: Protocol) -> Protocol:
"""Asks the proxy to open a tunnel."""
+ assert protocol.transport
tunnelReq = tunnel_request_data(
self._tunneledHost, self._tunneledPort, self._proxyAuthHeader
)
protocol.transport.write(tunnelReq)
self._protocolDataReceived = protocol.dataReceived
- protocol.dataReceived = self.processProxyResponse
+ protocol.dataReceived = self.processProxyResponse # type: ignore[method-assign]
self._protocol = protocol
return protocol
- def processProxyResponse(self, rcvd_bytes):
+ def processProxyResponse(self, data: bytes) -> None:
"""Processes the response from the proxy. If the tunnel is successfully
created, notifies the client that we are ready to send requests. If not
raises a TunnelError.
"""
- self._connectBuffer += rcvd_bytes
+ assert self._protocol.transport
+ self._connectBuffer += data
# make sure that enough (all) bytes are consumed
# and that we've got all HTTP headers (ending with a blank line)
# from the proxy so that we don't send those bytes to the TLS layer
@@ -163,23 +169,24 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
# see https://github.com/scrapy/scrapy/issues/2491
if b"\r\n\r\n" not in self._connectBuffer:
return
- self._protocol.dataReceived = self._protocolDataReceived
+ self._protocol.dataReceived = self._protocolDataReceived # type: ignore[method-assign]
respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
if respm and int(respm.group("status")) == 200:
# set proper Server Name Indication extension
- sslOptions = self._contextFactory.creatorForNetloc(
+ sslOptions = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc]
self._tunneledHost, self._tunneledPort
)
self._protocol.transport.startTLS(sslOptions, self._protocolFactory)
self._tunnelReadyDeferred.callback(self._protocol)
else:
+ extra: Any
if respm:
extra = {
"status": int(respm.group("status")),
"reason": respm.group("reason").strip(),
}
else:
- extra = rcvd_bytes[: self._truncatedLength]
+ extra = data[: self._truncatedLength]
self._tunnelReadyDeferred.errback(
TunnelError(
"Could not open CONNECT tunnel with proxy "
@@ -187,11 +194,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
)
)
- def connectFailed(self, reason):
+ def connectFailed(self, reason: Failure) -> None:
"""Propagates the errback to the appropriate deferred."""
self._tunnelReadyDeferred.errback(reason)
- def connect(self, protocolFactory):
+ def connect(self, protocolFactory: Factory) -> Deferred:
self._protocolFactory = protocolFactory
connectDeferred = super().connect(protocolFactory)
connectDeferred.addCallback(self.requestTunnel)
@@ -199,7 +206,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
return self._tunnelReadyDeferred
-def tunnel_request_data(host, port, proxy_auth_header=None):
+def tunnel_request_data(
+ host: str, port: int, proxy_auth_header: Optional[bytes] = None
+) -> bytes:
r"""
Return binary content of a CONNECT request.
@@ -230,18 +239,20 @@ class TunnelingAgent(Agent):
def __init__(
self,
- reactor,
- proxyConf,
- contextFactory=None,
- connectTimeout=None,
- bindAddress=None,
- pool=None,
+ reactor: ReactorBase,
+ proxyConf: Tuple[str, int, Optional[bytes]],
+ contextFactory: Optional[IPolicyForHTTPS] = None,
+ connectTimeout: Optional[float] = None,
+ bindAddress: Optional[bytes] = None,
+ pool: Optional[HTTPConnectionPool] = None,
):
+ # TODO make this arg required instead
+ assert contextFactory is not None
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
- self._proxyConf = proxyConf
- self._contextFactory = contextFactory
+ self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf
+ self._contextFactory: IPolicyForHTTPS = contextFactory
- def _getEndpoint(self, uri):
+ def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint:
return TunnelingTCP4ClientEndpoint(
reactor=self._reactor,
host=uri.host,
@@ -253,8 +264,15 @@ class TunnelingAgent(Agent):
)
def _requestWithEndpoint(
- self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath
- ):
+ self,
+ key: Any,
+ endpoint: TCP4ClientEndpoint,
+ method: bytes,
+ parsedURI: bytes,
+ headers: Optional[TxHeaders],
+ bodyProducer: Optional[IBodyProducer],
+ requestPath: bytes,
+ ) -> Deferred:
# proxy host and port are required for HTTP pool `key`
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
@@ -272,7 +290,12 @@ class TunnelingAgent(Agent):
class ScrapyProxyAgent(Agent):
def __init__(
- self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None
+ self,
+ reactor: ReactorBase,
+ proxyURI: bytes,
+ connectTimeout: Optional[float] = None,
+ bindAddress: Optional[bytes] = None,
+ pool: Optional[HTTPConnectionPool] = None,
):
super().__init__(
reactor=reactor,
@@ -280,9 +303,15 @@ class ScrapyProxyAgent(Agent):
bindAddress=bindAddress,
pool=pool,
)
- self._proxyURI = URI.fromBytes(proxyURI)
+ self._proxyURI: URI = URI.fromBytes(proxyURI)
- def request(self, method, uri, headers=None, bodyProducer=None):
+ def request(
+ self,
+ method: bytes,
+ uri: bytes,
+ headers: Optional[TxHeaders] = None,
+ bodyProducer: Optional[IBodyProducer] = None,
+ ) -> Deferred:
"""
Issue a new request via the configured proxy.
"""
@@ -306,26 +335,29 @@ class ScrapyAgent:
def __init__(
self,
- contextFactory=None,
- connectTimeout=10,
- bindAddress=None,
- pool=None,
- maxsize=0,
- warnsize=0,
- fail_on_dataloss=True,
- crawler=None,
+ contextFactory: Optional[IPolicyForHTTPS] = None,
+ connectTimeout: float = 10,
+ bindAddress: Optional[bytes] = None,
+ pool: Optional[HTTPConnectionPool] = None,
+ maxsize: int = 0,
+ warnsize: int = 0,
+ fail_on_dataloss: bool = True,
+ crawler: Optional[Crawler] = None,
):
- self._contextFactory = contextFactory
- self._connectTimeout = connectTimeout
- self._bindAddress = bindAddress
- self._pool = pool
- self._maxsize = maxsize
- self._warnsize = warnsize
- self._fail_on_dataloss = fail_on_dataloss
- self._txresponse = None
- self._crawler = crawler
-
- def _get_agent(self, request, timeout):
+ # TODO make these args required instead
+ assert contextFactory is not None
+ assert crawler is not None
+ self._contextFactory: IPolicyForHTTPS = contextFactory
+ self._connectTimeout: float = connectTimeout
+ self._bindAddress: Optional[bytes] = bindAddress
+ self._pool: Optional[HTTPConnectionPool] = pool
+ self._maxsize: int = maxsize
+ self._warnsize: int = warnsize
+ self._fail_on_dataloss: bool = fail_on_dataloss
+ self._txresponse: Optional[TxResponse] = None
+ self._crawler: Crawler = crawler
+
+ def _get_agent(self, request: Request, timeout: float) -> Agent:
from twisted.internet import reactor
bindaddress = request.meta.get("bindaddress") or self._bindAddress
@@ -333,10 +365,10 @@ class ScrapyAgent:
if proxy:
proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
scheme = _parse(request.url)[0]
- proxyHost = to_unicode(proxyHost)
+ proxyHost_str = to_unicode(proxyHost)
if scheme == b"https":
proxyAuth = request.headers.get(b"Proxy-Authorization", None)
- proxyConf = (proxyHost, proxyPort, proxyAuth)
+ proxyConf = (proxyHost_str, proxyPort, proxyAuth)
return self._TunnelingAgent(
reactor=reactor,
proxyConf=proxyConf,
@@ -346,7 +378,9 @@ class ScrapyAgent:
pool=self._pool,
)
proxyScheme = proxyScheme or b"http"
- proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, "", "", ""))
+ proxyURI = urlunparse(
+ (proxyScheme, proxyNetloc, proxyParams, b"", b"", b"")
+ )
return self._ProxyAgent(
reactor=reactor,
proxyURI=to_bytes(proxyURI, encoding="ascii"),
@@ -363,7 +397,7 @@ class ScrapyAgent:
pool=self._pool,
)
- def download_request(self, request):
+ def download_request(self, request: Request) -> Deferred:
from twisted.internet import reactor
timeout = request.meta.get("download_timeout") or self._connectTimeout
@@ -380,7 +414,7 @@ class ScrapyAgent:
else:
bodyproducer = None
start_time = time()
- d = agent.request(
+ d: Deferred = agent.request(
method, to_bytes(url, encoding="ascii"), headers, bodyproducer
)
# set download latency
@@ -393,7 +427,9 @@ class ScrapyAgent:
d.addBoth(self._cb_timeout, request, url, timeout)
return d
- def _cb_timeout(self, result, request, url, timeout):
+ def _cb_timeout(
+ self, result: Any, request: Request, url: str, timeout: float
+ ) -> Any:
if self._timeout_cl.active():
self._timeout_cl.cancel()
return result
@@ -404,19 +440,21 @@ class ScrapyAgent:
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
- def _cb_latency(self, result, request, start_time):
+ def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any:
request.meta["download_latency"] = time() - start_time
return result
@staticmethod
- def _headers_from_twisted_response(response):
+ def _headers_from_twisted_response(response: TxResponse) -> Headers:
headers = Headers()
if response.length != UNKNOWN_LENGTH:
headers[b"Content-Length"] = str(response.length).encode()
headers.update(response.headers.getAllRawHeaders())
return headers
- def _cb_bodyready(self, txresponse, request):
+ def _cb_bodyready(
+ self, txresponse: TxResponse, request: Request
+ ) -> Union[Dict[str, Any], Deferred]:
headers_received_result = self._crawler.signals.send_catch_log(
signal=signals.headers_received,
headers=self._headers_from_twisted_response(txresponse),
@@ -472,7 +510,7 @@ class ScrapyAgent:
logger.warning(warning_msg, warning_args)
txresponse._transport.loseConnection()
- raise defer.CancelledError(warning_msg % warning_args)
+ raise CancelledError(warning_msg % warning_args)
if warnsize and expected_size > warnsize:
logger.warning(
@@ -481,11 +519,11 @@ class ScrapyAgent:
{"size": expected_size, "warnsize": warnsize, "request": request},
)
- def _cancel(_):
+ def _cancel(_: Any) -> None:
# Abort connection immediately.
txresponse._transport._producer.abortConnection()
- d = defer.Deferred(_cancel)
+ d: Deferred = Deferred(_cancel)
txresponse.deliverBody(
_ResponseReader(
finished=d,
@@ -503,7 +541,9 @@ class ScrapyAgent:
return d
- def _cb_bodydone(self, result, request, url):
+ def _cb_bodydone(
+ self, result: Dict[str, Any], request: Request, url: str
+ ) -> Union[Response, Failure]:
headers = self._headers_from_twisted_response(result["txresponse"])
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
try:
@@ -523,53 +563,57 @@ class ScrapyAgent:
)
if result.get("failure"):
result["failure"].value.response = response
- return result["failure"]
+ return cast(Failure, result["failure"])
return response
@implementer(IBodyProducer)
class _RequestBodyProducer:
- def __init__(self, body):
+ def __init__(self, body: bytes):
self.body = body
self.length = len(body)
- def startProducing(self, consumer):
+ def startProducing(self, consumer: IConsumer) -> Deferred:
consumer.write(self.body)
- return defer.succeed(None)
+ return succeed(None)
- def pauseProducing(self):
+ def pauseProducing(self) -> None:
pass
- def stopProducing(self):
+ def stopProducing(self) -> None:
pass
-class _ResponseReader(protocol.Protocol):
+class _ResponseReader(Protocol):
def __init__(
self,
- finished,
- txresponse,
- request,
- maxsize,
- warnsize,
- fail_on_dataloss,
- crawler,
+ finished: Deferred,
+ txresponse: TxResponse,
+ request: Request,
+ maxsize: int,
+ warnsize: int,
+ fail_on_dataloss: bool,
+ crawler: Crawler,
):
- self._finished = finished
- self._txresponse = txresponse
- self._request = request
- self._bodybuf = BytesIO()
- self._maxsize = maxsize
- self._warnsize = warnsize
- self._fail_on_dataloss = fail_on_dataloss
- self._fail_on_dataloss_warned = False
- self._reached_warnsize = False
- self._bytes_received = 0
- self._certificate = None
- self._ip_address = None
- self._crawler = crawler
+ self._finished: Deferred = finished
+ self._txresponse: TxResponse = txresponse
+ self._request: Request = request
+ self._bodybuf: BytesIO = BytesIO()
+ self._maxsize: int = maxsize
+ self._warnsize: int = warnsize
+ self._fail_on_dataloss: bool = fail_on_dataloss
+ self._fail_on_dataloss_warned: bool = False
+ self._reached_warnsize: bool = False
+ self._bytes_received: int = 0
+ self._certificate: Optional[ssl.Certificate] = None
+ self._ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] = (
+ None
+ )
+ self._crawler: Crawler = crawler
- def _finish_response(self, flags=None, failure=None):
+ def _finish_response(
+ self, flags: Optional[List[str]] = None, failure: Optional[Failure] = None
+ ) -> None:
self._finished.callback(
{
"txresponse": self._txresponse,
@@ -581,7 +625,8 @@ class _ResponseReader(protocol.Protocol):
}
)
- def connectionMade(self):
+ def connectionMade(self) -> None:
+ assert self.transport
if self._certificate is None:
with suppress(AttributeError):
self._certificate = ssl.Certificate(
@@ -593,11 +638,12 @@ class _ResponseReader(protocol.Protocol):
self.transport._producer.getPeer().host
)
- def dataReceived(self, bodyBytes):
+ def dataReceived(self, bodyBytes: bytes) -> None:
# This maybe called several times after cancel was called with buffered data.
if self._finished.called:
return
+ assert self.transport
self._bodybuf.write(bodyBytes)
self._bytes_received += len(bodyBytes)
@@ -644,7 +690,7 @@ class _ResponseReader(protocol.Protocol):
{"warnsize": self._warnsize, "request": self._request},
)
- def connectionLost(self, reason):
+ def connectionLost(self, reason: Failure = connectionDone) -> None:
if self._finished.called:
return
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index c88dd2cdc..9a0811a50 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -2,11 +2,12 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, Optional, Type
+from twisted.internet.defer import Deferred
+
from scrapy import Request, Spider
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
-from scrapy.http import Response
from scrapy.settings import BaseSettings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
@@ -74,7 +75,7 @@ class S3DownloadHandler:
def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self:
return cls(crawler.settings, crawler=crawler, **kwargs)
- def download_request(self, request: Request, spider: Spider) -> Response:
+ def download_request(self, request: Request, spider: Spider) -> Deferred:
p = urlparse_cached(request)
scheme = "https" if request.meta.get("is_secure") else "http"
bucket = p.hostname