Scrapy, a fast high-level web crawling & scraping framework for Python.
feat: Idle Timeout for H2Connection (240s)
Aditya1dd27a92fa53
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex 5d859d475..bf41a2805 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -6,15 +6,18 @@ from typing import Dict, List, Optionalfrom h2.config import H2Configurationfrom h2.connection import H2Connection+from h2.errors import ErrorCodesfrom h2.events import (Event, DataReceived, ResponseReceived, SettingsAcknowledged,StreamEnded, StreamReset, WindowUpdated)from h2.exceptions import ProtocolErrorfrom twisted.internet.defer import Deferred+from twisted.internet.error import TimeoutErrorfrom twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactoryfrom twisted.internet.protocol import connectionDone, Factory, Protocolfrom twisted.internet.ssl import Certificate+from twisted.protocols.policies import TimeoutMixinfrom twisted.python.failure import Failurefrom twisted.web.client import URIfrom zope.interface import implementer@@ -37,7 +40,9 @@ class InvalidNegotiatedProtocol(ProtocolError):@implementer(IHandshakeListener)-class H2ClientProtocol(Protocol):+class H2ClientProtocol(Protocol, TimeoutMixin):+ IDLE_TIMEOUT = 240+def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None:"""Arguments:@@ -71,6 +76,10 @@ class H2ClientProtocol(Protocol):# We use simple FIFO policy to handle pending requestsself._active_streams = 0+ # Flag to keep track if settings were acknowledged by the remote+ # This ensures that we have established a HTTP/2 connection+ self._settings_acknowledged = False+# Save an instance of errors raised which lead to losing the connection# We pass these instances to the streams ResponseFailed() failureself._conn_lost_errors: List[BaseException] = []@@ -84,12 +93,12 @@ class H2ClientProtocol(Protocol):}@property- def is_connected(self) -> bool:+ def h2_connected(self) -> bool:"""Boolean to keep track of the connection status.This is used while initiating pending streams to make surethat we initiate stream only during active HTTP/2 Connection"""- return bool(self.transport.connected)+ return bool(self.transport.connected) and self._settings_acknowledged@propertydef allowed_max_concurrent_streams(self) -> int:@@ -111,7 +120,7 @@ class H2ClientProtocol(Protocol):while (self._pending_request_stream_pooland self._active_streams < self.allowed_max_concurrent_streams- and self.is_connected+ and self.h2_connected):self._active_streams += 1stream = self._pending_request_stream_pool.popleft()@@ -140,6 +149,9 @@ class H2ClientProtocol(Protocol):""" Write data to the underlying transport connectionfrom the HTTP2 connection instance if any"""+ # Reset the idle timeout as connection is still actively sending data+ self.resetTimeout()+data = self.conn.data_to_send()self.transport.write(data)@@ -153,21 +165,23 @@ class H2ClientProtocol(Protocol):# Add the stream to the request poolself._pending_request_stream_pool.append(stream)- # If we are connection and receive a request- # There is a good chance that the connection was IDLE- # Hence, we need to initiate pending requests- if self.is_connected:- self._send_pending_requests()+ # If we receive a request when connection is idle+ # We need to initiate pending requests+ self._send_pending_requests()return ddef connectionMade(self) -> None:"""Called by Twisted when the connection is established. We can startsending some data now: we should open with the connection preamble."""+ # Initialize the timeout+ self.setTimeout(self.IDLE_TIMEOUT)+destination = self.transport.getPeer()logger.debug('Connection made to {}'.format(destination))self.metadata['ip_address'] = ipaddress.ip_address(destination.host)+ # Initiate H2 Connectionself.conn.initiate_connection()self._write_to_transport()@@ -185,6 +199,9 @@ class H2ClientProtocol(Protocol):self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)])def dataReceived(self, data: bytes) -> None:+ # Reset the idle timeout as connection is still actively receiving data+ self.resetTimeout()+try:events = self.conn.receive_data(data)self._handle_events(events)@@ -196,10 +213,33 @@ class H2ClientProtocol(Protocol):finally:self._write_to_transport()+ def timeoutConnection(self):+ """Called when the connection times out.+ We lose the connection with TimeoutError"""++ # Check whether there are open streams. If there are, we're going to+ # want to use the error code PROTOCOL_ERROR. If there aren't, use+ # NO_ERROR.+ if (+ self.conn.open_outbound_streams > 0+ or self.conn.open_inbound_streams > 0+ or self._active_streams > 0+ ):+ error_code = ErrorCodes.PROTOCOL_ERROR+ else:+ error_code = ErrorCodes.NO_ERROR+ self.conn.close_connection(error_code=error_code)+ self._write_to_transport()++ self._lose_connection_with_error([TimeoutError("Hello")])+def connectionLost(self, reason: Failure = connectionDone) -> None:"""Called by Twisted when the transport connection is lost.No need to write anything to transport here."""+ # Cancel the timeout if not done yet+ self.setTimeout(None)+# Notify the connection pool instance such that no new requests are# sent over current connectionif not reason.check(connectionDone):@@ -250,6 +290,8 @@ class H2ClientProtocol(Protocol):self.streams[event.stream_id].receive_headers(event.headers)def settings_acknowledged(self, event: SettingsAcknowledged) -> None:+ self._settings_acknowledged = True+# Send off all the pending requests as now we have# established a proper HTTP/2 connectionself._send_pending_requests()diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.pyindex 16d54b2fc..f60691945 100644--- a/scrapy/core/http2/stream.py+++ b/scrapy/core/http2/stream.py@@ -134,7 +134,8 @@ class Stream:def _cancel(_):# Close this stream as gracefully as possible- # Check if the stream has started+ # If the associated request is initiated we reset this stream+ # else we directly call close() methodif self.request_sent:self.reset_stream(StreamCloseReason.CANCELLED)else:diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 7fcea58c5..2833801e7 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -9,7 +9,7 @@ from urllib.parse import urlencodefrom h2.exceptions import InvalidBodyLengthErrorfrom twisted.internet import reactor-from twisted.internet.defer import CancelledError, DeferredList, inlineCallbacks+from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacksfrom twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpointfrom twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificatefrom twisted.python.failure import Failure@@ -174,6 +174,7 @@ class Https2ClientProtocolTestCase(TestCase):# Start server for testingself.hostname = u'localhost'context_factory = ssl_context_factory(self.key_file, self.certificate_file)+server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname)self.server = yield server_endpoint.listen(self.site)self.port_number = self.server.getHost().port@@ -186,17 +187,20 @@ class Https2ClientProtocolTestCase(TestCase):acceptableProtocols=[b'h2'])uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8'))- h2_client_factory = H2ClientFactory(uri, Settings())++ self.conn_closed_deferred = Deferred()+ h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)self.client = yield client_endpoint.connect(h2_client_factory)@inlineCallbacksdef tearDown(self):- if self.client.is_connected:+ if self.client.connected:yield self.client.transport.loseConnection()yield self.client.transport.abortConnection()yield self.server.stopListening()shutil.rmtree(self.temp_directory)+ self.conn_closed_deferred = Nonedef get_url(self, path):"""