Scrapy, a fast high-level web crawling & scraping framework for Python.
feat(http2): support for POST requests
Adityad09ccf8d3b29
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex 847e74f97..6d926b100 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -1,4 +1,5 @@import logging+from typing import Dict, Listfrom h2.config import H2Configurationfrom h2.connection import H2Connection@@ -7,45 +8,19 @@ from h2.events import (StreamEnded, StreamReset, TrailersReceived, WindowUpdated)from twisted.internet.protocol import connectionDone, Protocol-from zope.interface import implementer, Interfacefrom scrapy.core.http2.stream import Streamfrom scrapy.http import RequestLOGGER = logging.getLogger(__name__)+LOGGER.debug = print-class IH2EventsHandler(Interface):- def connection_terminated(event: ConnectionTerminated):- pass-- def data_received(event: DataReceived):- pass-- def response_received(event: ResponseReceived):- pass-- def remote_settings_changed(event: RemoteSettingsChanged):- pass-- def stream_ended(event: StreamEnded):- pass-- def stream_reset(event: StreamReset):- pass-- def trailers_received(event: TrailersReceived):- pass-- def window_updated(event: WindowUpdated):- pass---@implementer(IH2EventsHandler)class H2ClientProtocol(Protocol):- # TODO: Check for user-agent while testing- # TODO: Add support for cookies- # TODO: Handle priority updates+ # TODO:+ # 1. Check for user-agent while testing+ # 2. Add support for cookies+ # 3. Handle priority updatesdef __init__(self):config = H2Configuration(client_side=True, header_encoding='utf-8')@@ -57,14 +32,14 @@ class H2ClientProtocol(Protocol):self.next_stream_id = 1# Streams are stored in a dictionary keyed off their stream IDs- self.streams = {}+ self.streams: Dict[int, Stream] = {}# Boolean to keep track the connection is made# If requests are received before connection is made# we keep all requests in a pool and send them as the connection# is madeself.is_connection_made = False- self._pending_request_stream_pool = []+ self._pending_request_stream_pool: List[Stream] = []def _new_stream(self, request: Request):"""Instantiates a new Stream object@@ -100,29 +75,27 @@ class H2ClientProtocol(Protocol):"""Called by Twisted when the connection is established. We can startsending some data now: we should open with the connection preamble."""- LOGGER.info("Connection made to {}".format(self.transport))+ LOGGER.debug("Connection made to {}".format(self.transport))self.conn.initiate_connection()self._write_to_transport()self.is_connection_made = True- # Initiate all pending requests- for stream in self._pending_request_stream_pool:- assert isinstance(stream, Stream)- stream.initiate_request()-- self._pending_request_stream_pool.clear()-def dataReceived(self, data):events = self.conn.receive_data(data)self._handle_events(events)self._write_to_transport()def connectionLost(self, reason=connectionDone):-"""Called by Twisted when the transport connection is lost."""- for stream_id in self.streams.keys():+ LOGGER.debug(f"connectionLost {reason}")+ stream_ids = list(self.streams.keys())++ for stream in self._pending_request_stream_pool:+ stream_ids.remove(stream.stream_id)++ for stream_id in stream_ids:# TODO: Close each Stream instance in a clean mannerself.conn.end_stream(stream_id)@@ -135,6 +108,7 @@ class H2ClientProtocol(Protocol):triggered by sending data"""for event in events:+ LOGGER.debug(event)if isinstance(event, ConnectionTerminated):self.connection_terminated(event)elif isinstance(event, DataReceived):@@ -153,38 +127,62 @@ class H2ClientProtocol(Protocol):self.remote_settings_changed(event)def send_headers(self, stream_id, headers):- """ Send the headers for a given stream to the resource+ """Send the headers for a given stream to the resourceInitiates a new connection hence.+ This function is wrapper for :func:`~h2.connection.H2Connection.send_headers`Arguments:stream_id {int} -- Valid stream idheaders {List[Tuple[str, str]]} -- Headers of the request"""- if stream_id in self.streams:- self.conn.send_headers(stream_id, headers, end_stream=True)- self._write_to_transport()- else:- pass+ LOGGER.debug(f'Send Headers: stream_id={stream_id} headers={headers}')+ self.conn.send_headers(stream_id, headers, end_stream=False)++ def send_data(self, stream_id, data):+ """Send the data for a given stream to the resource.+ Requires request headers to be sent at least once before this+ function is called.+ This function is wrapper for :func:`~h2.connection.H2Connection.send_data`++ Arguments:+ stream_id {int} -- Valid stream id+ data {bytes} -- The data to send on the stream.+ """+ LOGGER.debug(f"Send Data: stream_id={stream_id} data={data}")+ self.conn.send_data(stream_id, data, end_stream=False)++ def end_stream(self, stream_id):+ """End the given stream.+ This function is wrapper for :func:`~h2.connection.H2Connection.end_stream`+ Arguments:+ stream_id {int} - Valid stream id+ """+ LOGGER.debug(f"End Stream: stream_id={stream_id}")+ self.conn.end_stream(stream_id)++ # Event handler functions starts heredef connection_terminated(self, event: ConnectionTerminated):passdef data_received(self, event: DataReceived):stream_id = event.stream_id- # TODO: Stream do not exist in self.streams dictself.streams[stream_id].receive_data(event.data)def response_received(self, event: ResponseReceived):stream_id = event.stream_id- # TODO: Stream do not exist in self.streams dictself.streams[stream_id].receive_headers(event.headers)def remote_settings_changed(self, event: RemoteSettingsChanged):- pass+ # TODO: handle MAX_CONCURRENT_STREAMS+ # Initiate all pending requests+ for stream in self._pending_request_stream_pool:+ stream.initiate_request()++ self._pending_request_stream_pool.clear()def stream_ended(self, event: StreamEnded):stream_id = event.stream_id- # TODO: Stream do not exist in self.streams dictself.streams[stream_id].end_stream()def stream_reset(self, event: StreamReset):@@ -194,4 +192,6 @@ class H2ClientProtocol(Protocol):passdef window_updated(self, event: WindowUpdated):- pass+ stream_id = event.stream_id+ if stream_id != 0:+ self.streams[stream_id].window_updated()diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.pyindex d2a9f02fa..ab5a0fc88 100644--- a/scrapy/core/http2/stream.py+++ b/scrapy/core/http2/stream.py@@ -23,13 +23,23 @@ class Stream:stream_id {int} -- For one HTTP/2 connection each stream isuniquely identified by a single integerrequest {Request} -- HTTP request- connection {H2ClientProtocol} -- HTTP/2 connection this stream belongs to+ connection {H2Connection} -- HTTP/2 connection this stream belongs to."""-self.stream_id = stream_idself._request = request- self._conn = connection+ self._client_protocol = connection++ self._request_body = self._request.body+ self.content_length = 0 if self._request_body is None else len(self._request_body)++ # Each time we send a data frame, we will decrease value by the amount send.+ self.remaining_content_length = self.content_length+ # Flag to keep track whether we have ended this stream+ self.stream_ended = True++ # Data received frame by frame from the server is appended+ # and passed to the response Deferred when completely received.self._response_data = b""# Headers received after sending the request@@ -59,10 +69,67 @@ class Stream:# TODO: Check if scheme can be "http" for HTTP/2 ?(":scheme", "https"),- (":path", url.path)+ (":path", url.path),+ # ("Content-Length", str(self.content_length))++ # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers+ # are sent for request having body]- self._conn.send_headers(self.stream_id, http2_request_headers)+ self._client_protocol.send_headers(self.stream_id, http2_request_headers)+ self.send_data()++ def send_data(self):+ """Called immediately after the headers are sent. Here we send all the+ data as part of the request.++ If the content length is 0 initially then we end the stream immediately and+ wait for response data.+ """++ # TODO:+ # 1. Add test for sending very large data+ # 2. Add test for small data+ # 3. Both (1) and (2) should be tested for+ # 3.1 Large number of request+ # 3.2 Small number of requests++ # Firstly, check what the flow control window is for current stream.+ window_size = self._client_protocol.conn.local_flow_control_window(stream_id=self.stream_id)++ # Next, check what the maximum frame size is.+ max_frame_size = self._client_protocol.conn.max_outbound_frame_size++ # We will send no more than the window size or the remaining file size+ # of data in this call, whichever is smaller.+ bytes_to_send = min(window_size, self.remaining_content_length)++ # We now need to send a number of data frames.+ while bytes_to_send > 0:+ chunk_size = min(bytes_to_send, max_frame_size)++ data_chunk_start = self.content_length - self.remaining_content_length+ data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size]++ self._client_protocol.send_data(self.stream_id, data_chunk, end_stream=False)++ bytes_to_send = max(0, bytes_to_send - chunk_size)+ self.remaining_content_length = max(0, self.remaining_content_length - chunk_size)++ # End the stream if no more data has to be send+ if self.remaining_content_length == 0:+ self._client_protocol.end_stream(self.stream_id)+ else:+ # TODO: Continue from here :)+ pass++ def window_updated(self):+ """Flow control window size was changed.+ Send data that earlier could not be sent as we were+ blocked behind the flow control.+ """+ if self.remaining_content_length > 0 and not self.stream_ended:+ self.send_data()def receive_data(self, data: bytes):self._response_data += data@@ -72,7 +139,7 @@ class Stream:self._response_headers[name] = valuedef end_stream(self):- """Stream is ended by the resource hence no further+ """Stream is ended by the server hence no furtherdata or headers should be expected on this stream.We will call the response deferred callback passing