Scrapy, a fast high-level web crawling & scraping framework for Python.
chore: use deque for pending request pool
Aditya089dbc75e78a
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex 915284c33..dbc048ffa 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -1,4 +1,6 @@+import itertoolsimport logging+from collections import dequefrom h2.config import H2Configurationfrom h2.connection import H2Connection@@ -35,7 +37,7 @@ class H2ClientProtocol(Protocol):# ID of the next request stream# Following the convention made by hyper-h2 each client ID# will be odd.- self.next_stream_id = 1+ self.stream_id_count = itertools.count(start=1, step=2)# Streams are stored in a dictionary keyed off their stream IDsself.streams = {}@@ -45,7 +47,7 @@ class H2ClientProtocol(Protocol):# we keep all requests in a pool and send them as the connection# is madeself.is_connection_made = False- self._pending_request_stream_pool = []+ self._pending_request_stream_pool = deque()def _stream_close_cb(self, stream_id: int):"""Called when stream is closed completely@@ -55,8 +57,7 @@ class H2ClientProtocol(Protocol):def _new_stream(self, request: Request):"""Instantiates a new Stream object"""- stream_id = self.next_stream_id- self.next_stream_id += 2+ stream_id = next(self.stream_id_count)stream = Stream(stream_id=stream_id,@@ -72,11 +73,10 @@ class H2ClientProtocol(Protocol):def _send_pending_requests(self):# TODO: handle MAX_CONCURRENT_STREAMS# Initiate all pending requests- for stream in self._pending_request_stream_pool:+ while len(self._pending_request_stream_pool):+ stream = self._pending_request_stream_pool.popleft()stream.initiate_request()- self._pending_request_stream_pool.clear()-def _write_to_transport(self):""" Write data to the underlying transport connectionfrom the HTTP2 connection instance if any@@ -108,6 +108,8 @@ class H2ClientProtocol(Protocol):self._write_to_transport()self.is_connection_made = True+ # Send off all the pending requests+ # as now we have established a proper HTTP/2 connectionself._send_pending_requests()def dataReceived(self, data):diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.pyindex a0b75850d..c2e1adce5 100644--- a/scrapy/core/http2/stream.py+++ b/scrapy/core/http2/stream.py@@ -132,7 +132,7 @@ class Stream:bytes_to_send_size = min(window_size, self.remaining_content_length)# We now need to send a number of data frames.- while bytes_to_send_size > 0:+ while bytes_to_send_size:chunk_size = min(bytes_to_send_size, max_frame_size)data_chunk_start_id = self.content_length - self.remaining_content_length@@ -163,7 +163,7 @@ class Stream:Arguments:delta -- Window change delta"""- if self.remaining_content_length > 0 and not self.stream_closed_local:+ if self.stream_closed_local is False:self.send_data()def receive_data(self, data: bytes, flow_controlled_length: int):