Scrapy, a fast high-level web crawling & scraping framework for Python.
feat: handle response for different reasons
Adityac74ef660c7b2
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex d6134183a..188c14c15 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -11,7 +11,7 @@ from h2.events import ()from twisted.internet.protocol import connectionDone, Protocol-from scrapy.core.http2.stream import Stream+from scrapy.core.http2.stream import Stream, StreamCloseReasonfrom scrapy.http import RequestLOGGER = logging.getLogger(__name__)@@ -71,7 +71,7 @@ class H2ClientProtocol(Protocol):stream_id=stream_id,request=request,connection=self.conn,- metadata=self._metadata,+ conn_metadata=self._metadata,write_to_transport=self._write_to_transport,cb_close=self._stream_close_cb)@@ -133,7 +133,7 @@ class H2ClientProtocol(Protocol):"""# Pop all streams which were pending and were not yet startedfor stream_id in list(self.streams):- self.streams[stream_id].close()+ self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST)self.conn.close_connection()@@ -180,19 +180,18 @@ class H2ClientProtocol(Protocol):def stream_ended(self, event: StreamEnded):stream_id = event.stream_id- self.streams[stream_id].close()+ self.streams[stream_id].close(StreamCloseReason.ENDED)def stream_reset(self, event: StreamReset):# TODO: event.stream_id was abruptly closed# Q. What should be the response? (Failure/Partial/???)- self.streams[event.stream_id].close(event)+ self.streams[event.stream_id].close(StreamCloseReason.RESET)def window_updated(self, event: WindowUpdated):stream_id = event.stream_idif stream_id != 0:- self.streams[stream_id].receive_window_update(event.delta)+ self.streams[stream_id].receive_window_update()else:# Send leftover data for all the streamsfor stream in self.streams.values():- if stream.request_sent:- stream.send_data()+ stream.receive_window_update()diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.pyindex e8b4471d6..07a4428c8 100644--- a/scrapy/core/http2/stream.py+++ b/scrapy/core/http2/stream.py@@ -1,11 +1,15 @@import logging+from enum import IntFlag, auto+from io import BytesIOfrom typing import Dictfrom urllib.parse import urlparsefrom h2.connection import H2Connection-from h2.events import StreamEnded+from h2.errors import ErrorCodesfrom h2.exceptions import StreamClosedError-from twisted.internet.defer import Deferred+from twisted.internet.defer import Deferred, CancelledError+from twisted.python.failure import Failure+from twisted.web.client import ResponseFailedfrom scrapy.http import Requestfrom scrapy.http.headers import Headers@@ -14,6 +18,20 @@ from scrapy.responsetypes import responsetypesLOGGER = logging.getLogger(__name__)+class StreamCloseReason(IntFlag):+ # Received a StreamEnded event+ ENDED = auto()++ # Received a StreamReset event -- ended abruptly+ RESET = auto()++ # Transport connection was lost+ CONNECTION_LOST = auto()++ # Expected response body size is more than allowed limit+ MAXSIZE_EXCEEDED = auto()++class Stream:"""Represents a single HTTP/2 Stream.@@ -26,13 +44,16 @@ class Stream:"""def __init__(- self,- stream_id: int,- request: Request,- connection: H2Connection,- metadata: Dict,- write_to_transport,- cb_close+ self,+ stream_id: int,+ request: Request,+ connection: H2Connection,+ conn_metadata: Dict,+ write_to_transport,+ cb_close,+ download_maxsize=0,+ download_warnsize=0,+ fail_on_data_loss=True):"""Arguments:@@ -40,7 +61,7 @@ class Stream:uniquely identified by a single integerrequest {Request} -- HTTP requestconnection {H2Connection} -- HTTP/2 connection this stream belongs to.- metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection+ conn_metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connectionwrite_to_transport {callable} -- Method used to write & send data to the serverThis method should be used whenever some frame is to be sent to the server.cb_close {callable} -- Method called when this stream is closed@@ -49,16 +70,24 @@ class Stream:self.stream_id = stream_idself._request = requestself._conn = connection- self._metadata = metadata+ self._conn_metadata = conn_metadataself._write_to_transport = write_to_transportself._cb_close = cb_close- self._request_body = self._request.body- self.content_length = 0 if self._request_body is None else len(self._request_body)+ self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)+ self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)+ self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss)++ self.request_start_time = None++ self.content_length = 0 if self._request.body is None else len(self._request.body)# Flag to keep track whether this stream has initiated the requestself.request_sent = False+ # Flag to track whether we have logged about exceeding download warnsize+ self._reached_warnsize = False+# Each time we send a data frame, we will decrease value by the amount send.self.remaining_content_length = self.content_length@@ -68,17 +97,17 @@ class Stream:# Flag to keep track whether the server has closed the streamself.stream_closed_server = False- # The amount of data received that counts against the flow control- # window- self._response_flow_controlled_size = 0-# Private variable used to build the response# this response is then converted to appropriate Response class# passed to the response deferred callbackself._response = {# Data received frame by frame from the server is appended# and passed to the response Deferred when completely received.- 'body': b'',+ 'body': BytesIO(),++ # The amount of data received that counts against the flow control+ # window+ 'flow_controlled_size': 0,# Headers received after sending the request'headers': Headers({})@@ -137,16 +166,8 @@ class Stream:Warning: Only call this method when stream not closed from client sideand has initiated request already by sending HEADER frame. If not then- stream will be closed from client side with 499 response.-- TODO: Q. Should we instead raise ProtocolError here with a proper message?+ stream will raise ProtocolError (raise by h2 state machine)."""- if self.stream_closed_local or self.stream_closed_server:- raise StreamClosedError(self.stream_id)- elif not self.request_sent:- self.close()- return-# TODO:# 1. Add test for sending very large data# 2. Add test for small data@@ -154,6 +175,9 @@ class Stream:# 3.1 Large number of request# 3.2 Small number of requests+ if self.stream_closed_local:+ raise StreamClosedError(self.stream_id)+# Firstly, check what the flow control window is for current stream.window_size = self._conn.local_flow_control_window(stream_id=self.stream_id)@@ -170,7 +194,7 @@ class Stream:chunk_size = min(bytes_to_send_size, max_frame_size)data_chunk_start_id = self.content_length - self.remaining_content_length- data_chunk = self._request_body[data_chunk_start_id:data_chunk_start_id + chunk_size]+ data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size]self._conn.send_data(self.stream_id, data_chunk, end_stream=False)@@ -183,7 +207,7 @@ class Stream:self,self.content_length - self.remaining_content_length, self.content_length,data_frames_sent,- self._metadata['ip_address'])+ self._conn_metadata['ip_address']))# End the stream if no more data needs to be send@@ -197,24 +221,40 @@ class Stream:# Q. What about the rest of the data?# Ans: Remaining Data frames will be sent when we get a WindowUpdate frame- def receive_window_update(self, delta):+ def receive_window_update(self):"""Flow control window size was changed.Send data that earlier could not be sent as we wereblocked behind the flow control.-- Arguments:- delta -- Window change delta"""- if self.remaining_content_length > 0 and not self.stream_closed_server:+ if self.remaining_content_length and not self.stream_closed_server and self.request_sent:self.send_data()def receive_data(self, data: bytes, flow_controlled_length: int):- self._response['body'] += data- self._response_flow_controlled_size += flow_controlled_length+ self._response['body'].write(data)+ self._response['flow_controlled_size'] += flow_controlled_length++ if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize:+ # Clear buffer earlier to avoid keeping data in memory for a long time+ self._response['body'].truncate(0)+ self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)+ return++ if self._download_warnsize \+ and self._response['flow_controlled_size'] > self._download_warnsize \+ and not self._reached_warnsize:+ self._reached_warnsize = True+ warning_msg = ('Received more ({bytes}) bytes than download ',+ 'warn size ({warnsize}) in request {request}')+ warning_args = {+ 'bytes': self._response['flow_controlled_size'],+ 'warnsize': self._download_warnsize,+ 'request': self._request+ }+ LOGGER.warning(warning_msg, warning_args)# Acknowledge the data receivedself._conn.acknowledge_received_data(- self._response_flow_controlled_size,+ self._response['flow_controlled_size'],self.stream_id)@@ -222,35 +262,83 @@ class Stream:for name, value in headers:self._response['headers'][name] = value- def close(self, event=None):- """Based on the event sent we will handle each case.+ # Check if we exceed the allowed max data size which can be received+ expected_size = int(self._response['headers'].get(b'Content-Length', -1))+ if self._download_maxsize and expected_size > self._download_maxsize:+ self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)+ return++ if self._download_warnsize and expected_size > self._download_warnsize:+ warning_msg = ("Expected response size ({size}) larger than ",+ "download warn size ({warnsize}) in request {request}.")+ warning_args = {+ 'size': expected_size, 'warnsize': self._download_warnsize,+ 'request': self._request+ }+ LOGGER.warning(warning_msg, warning_args)++ def reset_stream(self, reason=StreamCloseReason.RESET):+ """Close this stream by sending a RST_FRAME to the remote peer"""+ # TODO: Q. REFUSED_STREAM or CANCEL ?+ if self.stream_closed_local:+ raise StreamClosedError(self.stream_id)++ self.stream_closed_local = True+ self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)+ self._write_to_transport()+ self.close(reason)++ def _is_data_lost(self) -> bool:+ assert self.stream_closed_server- event: StreamEnded- Stream is ended by the server hence no further- data or headers should be expected on this stream.- We will call the response deferred callback passing- the response object+ expected_size = self._response['flow_controlled_size']+ received_body_size = int(self._response['headers'][b'Content-Length'])- event: StreamReset- Stream reset via RST_FRAME by the upstream hence forcefully close- this stream and send TODO: ?+ return expected_size != received_body_size- event: None- No event is launched -- Hence we will simply close this stream+ def close(self, reason: StreamCloseReason):+ """Based on the reason sent we will handle each case."""# TODO: In case of abruptly stream close# Q1. Do we need to send the request again?# Q2. What response should we send now?- assert self.stream_closed_server is False+ if self.stream_closed_server:+ raise StreamClosedError(self.stream_id)+self.stream_closed_server = True- if not isinstance(event, StreamEnded):- # TODO+ flags = None+ if b'Content-Length' not in self._response['headers']:+ # Missing Content-Length - PotentialDataLoss+ flags = ['partial']+ elif self._is_data_lost():+ if self._fail_on_dataloss:+ self._deferred_response.errback(ResponseFailed([Failure()]))+ self._cb_close(self.stream_id)+ return+ else:+ flags = ['dataloss']++ if reason is StreamCloseReason.ENDED:+ self._fire_response_deferred(flags)++ elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST):# Stream was abruptly ended here- # Partial - Content-Length header not provided- pass+ self._deferred_response.errback(ResponseFailed([Failure()]))++ elif reason is StreamCloseReason.MAXSIZE_EXCEEDED:+ expected_size = int(self._response['headers'].get(b'Content-Length', -1))+ error_msg = ("Cancelling download of {url}: expected response "+ "size ({size}) larger than download max size ({maxsize}).")+ error_args = {+ 'url': self._request.url,+ 'size': expected_size,+ 'maxsize': self._download_maxsize+ }++ LOGGER.error(error_msg, error_args)+ self._deferred_response.errback(CancelledError(error_msg.format(**error_args)))- self._fire_response_deferred()self._cb_close(self.stream_id)def _fire_response_deferred(self, flags=None):@@ -258,7 +346,6 @@ class Stream:and fires the response deferred callback with thegenerated response instance"""# TODO:- # 1. Set flags, certificate, ip_address in response# 2. Should we fire this in case of# 2.1 StreamReset in between when data is received partially# 2.2 Forcefully closed the stream@@ -270,7 +357,7 @@ class Stream:body=self._response['body'])- # If there is :status in headers then+ # If there is no :status in headers then# HTTP Status Code: 499 - Client Closed Requeststatus = self._response['headers'].get(':status', '499')@@ -278,11 +365,11 @@ class Stream:url=self._request.url,status=status,headers=self._response['headers'],- body=self._response['body'],+ body=self._response['body'].getvalue(),request=self._request,flags=flags,- certificate=self._metadata['certificate'],- ip_address=self._metadata['ip_address']+ certificate=self._conn_metadata['certificate'],+ ip_address=self._conn_metadata['ip_address'])self._deferred_response.callback(response)