Scrapy, a fast high-level web crawling & scraping framework for Python.
Add a CONNECTION_KEEPALIVE_TIMEOUT setting (#8342)
Adrian0ecfbb777006
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex dfb59cfe6..8f9d5e313 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -622,6 +622,21 @@ See also: :ref:`topics-autothrottle` and itsIt is possible to change this setting per domain by using:setting:`DOWNLOAD_SLOTS`.+.. setting:: CONNECTION_KEEPALIVE_TIMEOUT++CONNECTION_KEEPALIVE_TIMEOUT+----------------------------++.. versionadded:: VERSION++Default: ``240.0``++How long, in seconds, a connection is kept open for reuse after the response it+was serving is complete. Use ``0`` to close connections as soon as they become+idle.++Servers usually close idle connections earlier than this on their own.+.. setting:: DEFAULT_DROPITEM_LOG_LEVELDEFAULT_DROPITEM_LOG_LEVELdiff --git a/scrapy/core/_http2/agent.py b/scrapy/core/_http2/agent.pyindex 063a1aa91..4467985d5 100644--- a/scrapy/core/_http2/agent.py+++ b/scrapy/core/_http2/agent.py@@ -50,7 +50,7 @@ class H2ConnectionPool:# Check if we already have a connection to the remoteconn = self._connections.get(key, None)- if conn:+ if conn and not conn.closing:# Return this connection instance wrapped inside a deferredreturn defer.succeed(conn)@@ -63,7 +63,6 @@ class H2ConnectionPool:self._pending_requests[key] = deque()conn_lost_deferred: Deferred[None] = Deferred()- conn_lost_deferred.addCallback(self._remove_connection, key)factory = H2ClientFactory(uri,@@ -73,7 +72,7 @@ class H2ConnectionPool:)conn_d = endpoint.connect(factory)d = self._pending_request(key)- conn_d.addCallback(self.put_connection, key)+ conn_d.addCallback(self.put_connection, key, conn_lost_deferred)conn_d.addErrback(self._connection_failed, key)return d@@ -91,9 +90,13 @@ class H2ConnectionPool:return ddef put_connection(- self, conn: H2ClientProtocol, key: ConnectionKey+ self,+ conn: H2ClientProtocol,+ key: ConnectionKey,+ conn_lost_deferred: Deferred[None],) -> H2ClientProtocol:self._connections[key] = conn+ conn_lost_deferred.addCallback(self._remove_connection, key, conn)# Now as we have established a proper HTTP/2 connection# we fire all the deferred's with the connection instance@@ -112,8 +115,12 @@ class H2ConnectionPool:d = pending_requests.popleft()d.errback(failure)- def _remove_connection(self, _: None, key: ConnectionKey) -> None:- self._connections.pop(key)+ def _remove_connection(+ self, _: None, key: ConnectionKey, conn: H2ClientProtocol+ ) -> None:+ # a newer connection may have taken over the key already+ if self._connections.get(key) is conn:+ del self._connections[key]def close_connections(self) -> None:"""Close all the HTTP/2 connections and remove them from pool."""diff --git a/scrapy/core/_http2/protocol.py b/scrapy/core/_http2/protocol.pyindex 1a7e85478..5037938ba 100644--- a/scrapy/core/_http2/protocol.py+++ b/scrapy/core/_http2/protocol.py@@ -84,8 +84,6 @@ class MethodNotAllowed405(H2Error):@implementer(IHandshakeListener)class H2ClientProtocol(Protocol, TimeoutMixin):- IDLE_TIMEOUT = 240-def __init__(self,uri: URI,@@ -107,6 +105,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin):self._crawler: Crawler = crawlerself._conn_lost_deferred: Deferred[None] = conn_lost_deferredself._tls_verbose_logging: bool = tls_verbose_logging+ self._idle_timeout: float = crawler.settings.getfloat(+ "CONNECTION_KEEPALIVE_TIMEOUT"+ )+ self.closing: bool = Falseconfig = H2Configuration(client_side=True)self.conn = H2Connection(config=config)@@ -193,6 +195,8 @@ class H2ClientProtocol(Protocol, TimeoutMixin):stream = self.streams.pop(stream_id)self.metadata["active_streams"] -= 1self._send_pending_requests()+ if not self._idle_timeout and not self.metadata["active_streams"]:+ self._close_idle()return streamdef _new_stream(self, request: Request, spider: Spider) -> Stream:@@ -252,7 +256,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):sending some data now: we should open with the connection preamble."""# Initialize the timeout- self.setTimeout(self.IDLE_TIMEOUT) # type: ignore[no-untyped-call]+ self.setTimeout(self._idle_timeout or None) # type: ignore[no-untyped-call]assert self.transport is not None # typingdestination = self.transport.getPeer()@@ -268,10 +272,16 @@ class H2ClientProtocol(Protocol, TimeoutMixin):def _lose_connection_with_error(self, errors: list[BaseException]) -> None:"""Helper function to lose the connection with the error sent as areason"""+ self.closing = Trueself._conn_lost_errors.extend(errors)assert self.transport is not None # typingself.transport.loseConnection()+ def _close_idle(self) -> None:+ self.conn.close_connection(error_code=ErrorCodes.NO_ERROR)+ self._write_to_transport()+ self._lose_connection_with_error([])+def handshakeCompleted(self) -> None:"""Close the connection if it's not made via the expected protocol@@ -346,7 +356,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):self._lose_connection_with_error([DownloadTimeoutError(- f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s"+ f"Connection was IDLE for more than {self._idle_timeout}s")])diff --git a/scrapy/core/downloader/handlers/_aiohttp.py b/scrapy/core/downloader/handlers/_aiohttp.pyindex 741532320..bc4d7e2c9 100644--- a/scrapy/core/downloader/handlers/_aiohttp.py+++ b/scrapy/core/downloader/handlers/_aiohttp.py@@ -62,6 +62,8 @@ class AiohttpDownloadHandler(BaseStreamingDownloadHandler[_ClientResponse]):limit=self._pool_size_total,# hard limit on simultaneous connections per hostlimit_per_host=self._pool_size_per_host,+ keepalive_timeout=self._keepalive_timeout or None,+ force_close=not self._keepalive_timeout,)self._session: aiohttp.ClientSession = aiohttp.ClientSession(connector=connector,diff --git a/scrapy/core/downloader/handlers/_base_http.py b/scrapy/core/downloader/handlers/_base_http.pyindex 83d130462..eee164aa1 100644--- a/scrapy/core/downloader/handlers/_base_http.py+++ b/scrapy/core/downloader/handlers/_base_http.py@@ -23,3 +23,6 @@ class BaseHttpDownloadHandler(BaseDownloadHandler, ABC):"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING")self._fail_on_dataloss_warned: bool = False+ self._keepalive_timeout: float = crawler.settings.getfloat(+ "CONNECTION_KEEPALIVE_TIMEOUT"+ )diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.pyindex 429b32cc5..e5f2a2ea6 100644--- a/scrapy/core/downloader/handlers/_httpx.py+++ b/scrapy/core/downloader/handlers/_httpx.py@@ -95,8 +95,13 @@ class HttpxDownloadHandler(_Base):# hard limit on simultaneous connections (None for no limit, which# is what a CONCURRENT_REQUESTS of 0 means)max_connections=self._pool_size_total or None,- # total number of idle connections in the pool (extra ones are closed)- max_keepalive_connections=self._pool_size_total or None,+ # total number of idle connections in the pool (extra ones are+ # closed); keeping none of them is how keepalive gets disabled,+ # since expiry alone is subject to the resolution of the clock+ max_keepalive_connections=(+ (self._pool_size_total or None) if self._keepalive_timeout else 0+ ),+ keepalive_expiry=self._keepalive_timeout,)self._default_client: httpx.AsyncClient = self._make_client()diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex edd84b3fe..10a40e29f 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -99,7 +99,10 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):from twisted.internet import reactor- self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True)+ self._pool: HTTPConnectionPool = HTTPConnectionPool(+ reactor, persistent=bool(self._keepalive_timeout)+ )+ self._pool.cachedConnectionTimeout = self._keepalive_timeout # type: ignore[assignment]self._pool.maxPersistentPerHost = crawler.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN")diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 6237fb288..9f55b3717 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -46,6 +46,7 @@ __all__ = ["CONCURRENT_ITEMS","CONCURRENT_REQUESTS","CONCURRENT_REQUESTS_PER_DOMAIN",+ "CONNECTION_KEEPALIVE_TIMEOUT","COOKIES_DEBUG","COOKIES_ENABLED","CRAWLSPIDER_FOLLOW_LINKS",@@ -278,6 +279,8 @@ CONCURRENT_ITEMS = 100CONCURRENT_REQUESTS = 16CONCURRENT_REQUESTS_PER_DOMAIN = 8+CONNECTION_KEEPALIVE_TIMEOUT = 240.0+COOKIES_ENABLED = TrueCOOKIES_DEBUG = Falsediff --git a/tests/mockserver/http.py b/tests/mockserver/http.pyindex 2753be720..e0a6f5981 100644--- a/tests/mockserver/http.py+++ b/tests/mockserver/http.py@@ -19,6 +19,7 @@ from .http_resources import (ChunkedResource,ClientIPResource,Compress,+ ConnectionId,ContentLengthHeaderResource,Delay,Drop,@@ -68,6 +69,7 @@ class Root(BaseResource):put_child(self, b"static", File(str(Path(tests_datadir, "test_site/"))))put_child(self, b"redirect-to", RedirectTo())put_child(self, b"text", Data(b"Works", "text/plain"))+ put_child(self, b"connection-id", ConnectionId())put_child(self,b"html",diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.pyindex d159a04af..b01712577 100644--- a/tests/mockserver/http_resources.py+++ b/tests/mockserver/http_resources.py@@ -1,6 +1,7 @@from __future__ import annotationsimport gzip+import itertoolsimport jsonimport randomfrom typing import TYPE_CHECKING, Any, ClassVar, ParamSpec, TypeVar, cast@@ -483,6 +484,25 @@ class SetCookie(BaseResource):return b""+_connection_ids = itertools.count()+++class ConnectionId(LeafResource):+ """Return the identifier of the connection serving the request.++ Two responses carrying the same identifier were served over the same+ connection, which lets tests tell connection reuse from reconnection.+ """++ def render_GET(self, request: Request) -> bytes:+ # under HTTP/2 the channel of a request is its stream, so the+ # connection has to be reached through it+ channel: Any = getattr(request.channel, "_conn", request.channel)+ if not hasattr(channel, "_mockserver_connection_id"):+ channel._mockserver_connection_id = next(_connection_ids)+ return str(channel._mockserver_connection_id).encode()++def _h2_connection(request: Request) -> H2Connection:"""Return the HTTP/2 connection that *request* was received on.diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 036daad67..7f0b04b23 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -4,8 +4,10 @@ from __future__ import annotationsimport sysfrom typing import TYPE_CHECKING, Any+from unittest.mock import Mockimport pytest+from twisted.internet.defer import Deferredfrom twisted.web.http import H2_ENABLEDfrom scrapy import Spider@@ -17,6 +19,7 @@ from scrapy.exceptions import ()from scrapy.http import Requestfrom scrapy.utils.misc import build_from_crawler+from scrapy.utils.test import get_crawlerfrom tests.utils.bases.download_handlers_http import (TestHttpProxyBase,TestHttpsBase,@@ -73,6 +76,19 @@ def test_not_configured_without_reactor() -> None:build_from_crawler(H2DownloadHandler, crawler)+def test_pool_keeps_newer_connection() -> None:+ from scrapy.core._http2.agent import H2ConnectionPool # noqa: PLC0415++ pool = H2ConnectionPool(get_crawler())+ key = (b"https", b"example.com", 443, None)+ old, new = Mock(), Mock()+ old_lost: Deferred[None] = Deferred()+ pool.put_connection(old, key, old_lost)+ pool.put_connection(new, key, Deferred())+ old_lost.callback(None)+ assert pool._connections[key] is new++class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):http2 = Truediff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 3e40292c3..383976295 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -766,11 +766,9 @@ class TestHttps2ClientProtocol:yield make_request_dfd(client, request)for err in exc_info.value.reasons:- from scrapy.core._http2.protocol import H2ClientProtocol # noqa: PLC0415-if isinstance(err, DownloadTimeoutError):assert (- f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s"+ f"Connection was IDLE for more than {client._idle_timeout}s"in str(err))breakdiff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.pyindex d0b439128..8739436a0 100644--- a/tests/utils/bases/download_handlers_http.py+++ b/tests/utils/bases/download_handlers_http.py@@ -121,6 +121,26 @@ class TestHttpBase(ABC):finally:await dh.close()+ async def _connection_ids(+ self, url: str, settings_dict: dict[str, Any]+ ) -> set[str]:+ """Request *url* twice in a row and return the identifiers of the+ connections that the server used to serve them."""+ async with self.get_dh(settings_dict) as dh:+ responses = [await dh.download_request(Request(url)) for _ in range(2)]+ return {response.text for response in responses}++ @coroutine_test+ async def test_keepalive(self, mockserver: MockServer) -> None:+ url = mockserver.url("/connection-id", is_secure=self.is_secure)+ assert len(await self._connection_ids(url, {})) == 1++ @coroutine_test+ async def test_no_keepalive(self, mockserver: MockServer) -> None:+ url = mockserver.url("/connection-id", is_secure=self.is_secure)+ settings_dict = {"CONNECTION_KEEPALIVE_TIMEOUT": 0}+ assert len(await self._connection_ids(url, settings_dict)) == 2+@coroutine_testasync def test_unsupported_scheme(self) -> None:request = Request("unsupp://unsupported.scheme")