Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add a CONNECTION_KEEPALIVE_TIMEOUT setting (#8342)

Adrian0ecfbb777006
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index dfb59cfe6..8f9d5e313 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -622,6 +622,21 @@ See also: :ref:`topics-autothrottle` and its
It is possible to change this setting per domain by using
:setting:`DOWNLOAD_SLOTS`.
+.. setting:: CONNECTION_KEEPALIVE_TIMEOUT
+
+CONNECTION_KEEPALIVE_TIMEOUT
+----------------------------
+
+.. versionadded:: VERSION
+
+Default: ``240.0``
+
+How long, in seconds, a connection is kept open for reuse after the response it
+was serving is complete. Use ``0`` to close connections as soon as they become
+idle.
+
+Servers usually close idle connections earlier than this on their own.
+
.. setting:: DEFAULT_DROPITEM_LOG_LEVEL
DEFAULT_DROPITEM_LOG_LEVEL
diff --git a/scrapy/core/_http2/agent.py b/scrapy/core/_http2/agent.py
index 063a1aa91..4467985d5 100644
--- a/scrapy/core/_http2/agent.py
+++ b/scrapy/core/_http2/agent.py
@@ -50,7 +50,7 @@ class H2ConnectionPool:
# Check if we already have a connection to the remote
conn = self._connections.get(key, None)
- if conn:
+ if conn and not conn.closing:
# Return this connection instance wrapped inside a deferred
return defer.succeed(conn)
@@ -63,7 +63,6 @@ class H2ConnectionPool:
self._pending_requests[key] = deque()
conn_lost_deferred: Deferred[None] = Deferred()
- conn_lost_deferred.addCallback(self._remove_connection, key)
factory = H2ClientFactory(
uri,
@@ -73,7 +72,7 @@ class H2ConnectionPool:
)
conn_d = endpoint.connect(factory)
d = self._pending_request(key)
- conn_d.addCallback(self.put_connection, key)
+ conn_d.addCallback(self.put_connection, key, conn_lost_deferred)
conn_d.addErrback(self._connection_failed, key)
return d
@@ -91,9 +90,13 @@ class H2ConnectionPool:
return d
def put_connection(
- self, conn: H2ClientProtocol, key: ConnectionKey
+ self,
+ conn: H2ClientProtocol,
+ key: ConnectionKey,
+ conn_lost_deferred: Deferred[None],
) -> H2ClientProtocol:
self._connections[key] = conn
+ conn_lost_deferred.addCallback(self._remove_connection, key, conn)
# Now as we have established a proper HTTP/2 connection
# we fire all the deferred's with the connection instance
@@ -112,8 +115,12 @@ class H2ConnectionPool:
d = pending_requests.popleft()
d.errback(failure)
- def _remove_connection(self, _: None, key: ConnectionKey) -> None:
- self._connections.pop(key)
+ def _remove_connection(
+ self, _: None, key: ConnectionKey, conn: H2ClientProtocol
+ ) -> None:
+ # a newer connection may have taken over the key already
+ if self._connections.get(key) is conn:
+ del self._connections[key]
def close_connections(self) -> None:
"""Close all the HTTP/2 connections and remove them from pool."""
diff --git a/scrapy/core/_http2/protocol.py b/scrapy/core/_http2/protocol.py
index 1a7e85478..5037938ba 100644
--- a/scrapy/core/_http2/protocol.py
+++ b/scrapy/core/_http2/protocol.py
@@ -84,8 +84,6 @@ class MethodNotAllowed405(H2Error):
@implementer(IHandshakeListener)
class H2ClientProtocol(Protocol, TimeoutMixin):
- IDLE_TIMEOUT = 240
-
def __init__(
self,
uri: URI,
@@ -107,6 +105,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
self._crawler: Crawler = crawler
self._conn_lost_deferred: Deferred[None] = conn_lost_deferred
self._tls_verbose_logging: bool = tls_verbose_logging
+ self._idle_timeout: float = crawler.settings.getfloat(
+ "CONNECTION_KEEPALIVE_TIMEOUT"
+ )
+ self.closing: bool = False
config = H2Configuration(client_side=True)
self.conn = H2Connection(config=config)
@@ -193,6 +195,8 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
stream = self.streams.pop(stream_id)
self.metadata["active_streams"] -= 1
self._send_pending_requests()
+ if not self._idle_timeout and not self.metadata["active_streams"]:
+ self._close_idle()
return stream
def _new_stream(self, request: Request, spider: Spider) -> Stream:
@@ -252,7 +256,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
sending some data now: we should open with the connection preamble.
"""
# Initialize the timeout
- self.setTimeout(self.IDLE_TIMEOUT) # type: ignore[no-untyped-call]
+ self.setTimeout(self._idle_timeout or None) # type: ignore[no-untyped-call]
assert self.transport is not None # typing
destination = self.transport.getPeer()
@@ -268,10 +272,16 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
def _lose_connection_with_error(self, errors: list[BaseException]) -> None:
"""Helper function to lose the connection with the error sent as a
reason"""
+ self.closing = True
self._conn_lost_errors.extend(errors)
assert self.transport is not None # typing
self.transport.loseConnection()
+ def _close_idle(self) -> None:
+ self.conn.close_connection(error_code=ErrorCodes.NO_ERROR)
+ self._write_to_transport()
+ self._lose_connection_with_error([])
+
def handshakeCompleted(self) -> None:
"""
Close the connection if it's not made via the expected protocol
@@ -346,7 +356,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
self._lose_connection_with_error(
[
DownloadTimeoutError(
- f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s"
+ f"Connection was IDLE for more than {self._idle_timeout}s"
)
]
)
diff --git a/scrapy/core/downloader/handlers/_aiohttp.py b/scrapy/core/downloader/handlers/_aiohttp.py
index 741532320..bc4d7e2c9 100644
--- a/scrapy/core/downloader/handlers/_aiohttp.py
+++ b/scrapy/core/downloader/handlers/_aiohttp.py
@@ -62,6 +62,8 @@ class AiohttpDownloadHandler(BaseStreamingDownloadHandler[_ClientResponse]):
limit=self._pool_size_total,
# hard limit on simultaneous connections per host
limit_per_host=self._pool_size_per_host,
+ keepalive_timeout=self._keepalive_timeout or None,
+ force_close=not self._keepalive_timeout,
)
self._session: aiohttp.ClientSession = aiohttp.ClientSession(
connector=connector,
diff --git a/scrapy/core/downloader/handlers/_base_http.py b/scrapy/core/downloader/handlers/_base_http.py
index 83d130462..eee164aa1 100644
--- a/scrapy/core/downloader/handlers/_base_http.py
+++ b/scrapy/core/downloader/handlers/_base_http.py
@@ -23,3 +23,6 @@ class BaseHttpDownloadHandler(BaseDownloadHandler, ABC):
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
)
self._fail_on_dataloss_warned: bool = False
+ self._keepalive_timeout: float = crawler.settings.getfloat(
+ "CONNECTION_KEEPALIVE_TIMEOUT"
+ )
diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py
index 429b32cc5..e5f2a2ea6 100644
--- a/scrapy/core/downloader/handlers/_httpx.py
+++ b/scrapy/core/downloader/handlers/_httpx.py
@@ -95,8 +95,13 @@ class HttpxDownloadHandler(_Base):
# hard limit on simultaneous connections (None for no limit, which
# is what a CONCURRENT_REQUESTS of 0 means)
max_connections=self._pool_size_total or None,
- # total number of idle connections in the pool (extra ones are closed)
- max_keepalive_connections=self._pool_size_total or None,
+ # total number of idle connections in the pool (extra ones are
+ # closed); keeping none of them is how keepalive gets disabled,
+ # since expiry alone is subject to the resolution of the clock
+ max_keepalive_connections=(
+ (self._pool_size_total or None) if self._keepalive_timeout else 0
+ ),
+ keepalive_expiry=self._keepalive_timeout,
)
self._default_client: httpx.AsyncClient = self._make_client()
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index edd84b3fe..10a40e29f 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -99,7 +99,10 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
from twisted.internet import reactor
- self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True)
+ self._pool: HTTPConnectionPool = HTTPConnectionPool(
+ reactor, persistent=bool(self._keepalive_timeout)
+ )
+ self._pool.cachedConnectionTimeout = self._keepalive_timeout # type: ignore[assignment]
self._pool.maxPersistentPerHost = crawler.settings.getint(
"CONCURRENT_REQUESTS_PER_DOMAIN"
)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 6237fb288..9f55b3717 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -46,6 +46,7 @@ __all__ = [
"CONCURRENT_ITEMS",
"CONCURRENT_REQUESTS",
"CONCURRENT_REQUESTS_PER_DOMAIN",
+ "CONNECTION_KEEPALIVE_TIMEOUT",
"COOKIES_DEBUG",
"COOKIES_ENABLED",
"CRAWLSPIDER_FOLLOW_LINKS",
@@ -278,6 +279,8 @@ CONCURRENT_ITEMS = 100
CONCURRENT_REQUESTS = 16
CONCURRENT_REQUESTS_PER_DOMAIN = 8
+CONNECTION_KEEPALIVE_TIMEOUT = 240.0
+
COOKIES_ENABLED = True
COOKIES_DEBUG = False
diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py
index 2753be720..e0a6f5981 100644
--- a/tests/mockserver/http.py
+++ b/tests/mockserver/http.py
@@ -19,6 +19,7 @@ from .http_resources import (
ChunkedResource,
ClientIPResource,
Compress,
+ ConnectionId,
ContentLengthHeaderResource,
Delay,
Drop,
@@ -68,6 +69,7 @@ class Root(BaseResource):
put_child(self, b"static", File(str(Path(tests_datadir, "test_site/"))))
put_child(self, b"redirect-to", RedirectTo())
put_child(self, b"text", Data(b"Works", "text/plain"))
+ put_child(self, b"connection-id", ConnectionId())
put_child(
self,
b"html",
diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py
index d159a04af..b01712577 100644
--- a/tests/mockserver/http_resources.py
+++ b/tests/mockserver/http_resources.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import gzip
+import itertools
import json
import random
from typing import TYPE_CHECKING, Any, ClassVar, ParamSpec, TypeVar, cast
@@ -483,6 +484,25 @@ class SetCookie(BaseResource):
return b""
+_connection_ids = itertools.count()
+
+
+class ConnectionId(LeafResource):
+ """Return the identifier of the connection serving the request.
+
+ Two responses carrying the same identifier were served over the same
+ connection, which lets tests tell connection reuse from reconnection.
+ """
+
+ def render_GET(self, request: Request) -> bytes:
+ # under HTTP/2 the channel of a request is its stream, so the
+ # connection has to be reached through it
+ channel: Any = getattr(request.channel, "_conn", request.channel)
+ if not hasattr(channel, "_mockserver_connection_id"):
+ channel._mockserver_connection_id = next(_connection_ids)
+ return str(channel._mockserver_connection_id).encode()
+
+
def _h2_connection(request: Request) -> H2Connection:
"""Return the HTTP/2 connection that *request* was received on.
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 036daad67..7f0b04b23 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -4,8 +4,10 @@ from __future__ import annotations
import sys
from typing import TYPE_CHECKING, Any
+from unittest.mock import Mock
import pytest
+from twisted.internet.defer import Deferred
from twisted.web.http import H2_ENABLED
from scrapy import Spider
@@ -17,6 +19,7 @@ from scrapy.exceptions import (
)
from scrapy.http import Request
from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.test import get_crawler
from tests.utils.bases.download_handlers_http import (
TestHttpProxyBase,
TestHttpsBase,
@@ -73,6 +76,19 @@ def test_not_configured_without_reactor() -> None:
build_from_crawler(H2DownloadHandler, crawler)
+def test_pool_keeps_newer_connection() -> None:
+ from scrapy.core._http2.agent import H2ConnectionPool # noqa: PLC0415
+
+ pool = H2ConnectionPool(get_crawler())
+ key = (b"https", b"example.com", 443, None)
+ old, new = Mock(), Mock()
+ old_lost: Deferred[None] = Deferred()
+ pool.put_connection(old, key, old_lost)
+ pool.put_connection(new, key, Deferred())
+ old_lost.callback(None)
+ assert pool._connections[key] is new
+
+
class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
http2 = True
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 3e40292c3..383976295 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -766,11 +766,9 @@ class TestHttps2ClientProtocol:
yield make_request_dfd(client, request)
for err in exc_info.value.reasons:
- from scrapy.core._http2.protocol import H2ClientProtocol # noqa: PLC0415
-
if isinstance(err, DownloadTimeoutError):
assert (
- f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s"
+ f"Connection was IDLE for more than {client._idle_timeout}s"
in str(err)
)
break
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index d0b439128..8739436a0 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -121,6 +121,26 @@ class TestHttpBase(ABC):
finally:
await dh.close()
+ async def _connection_ids(
+ self, url: str, settings_dict: dict[str, Any]
+ ) -> set[str]:
+ """Request *url* twice in a row and return the identifiers of the
+ connections that the server used to serve them."""
+ async with self.get_dh(settings_dict) as dh:
+ responses = [await dh.download_request(Request(url)) for _ in range(2)]
+ return {response.text for response in responses}
+
+ @coroutine_test
+ async def test_keepalive(self, mockserver: MockServer) -> None:
+ url = mockserver.url("/connection-id", is_secure=self.is_secure)
+ assert len(await self._connection_ids(url, {})) == 1
+
+ @coroutine_test
+ async def test_no_keepalive(self, mockserver: MockServer) -> None:
+ url = mockserver.url("/connection-id", is_secure=self.is_secure)
+ settings_dict = {"CONNECTION_KEEPALIVE_TIMEOUT": 0}
+ assert len(await self._connection_ids(url, settings_dict)) == 2
+
@coroutine_test
async def test_unsupported_scheme(self) -> None:
request = Request("unsupp://unsupported.scheme")