Scrapy, a fast high-level web crawling & scraping framework for Python.
refactor: move TypedDict types to types.py
Aditya23906b6bee95
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex 7fb935f10..0b3e5d304 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -2,6 +2,7 @@ import ipaddressimport itertoolsimport loggingfrom collections import deque+from typing import Union, Dictfrom h2.config import H2Configurationfrom h2.connection import H2Connection@@ -9,20 +10,18 @@ from h2.events import (DataReceived, ResponseReceived, SettingsAcknowledged,StreamEnded, StreamReset, WindowUpdated)+from h2.exceptions import ProtocolErrorfrom twisted.internet.protocol import connectionDone, Protocolfrom twisted.internet.ssl import Certificatefrom scrapy.core.http2.stream import Stream, StreamCloseReason+from scrapy.core.http2.types import H2ConnectionMetadataDictfrom scrapy.http import Request-LOGGER = logging.getLogger(__name__)+logger = logging.getLogger(__name__)class H2ClientProtocol(Protocol):- # TODO:- # 1. Check for user-agent while testing- # 2. Handle case when received events have StreamID = 0 (applied to H2Connection)-def __init__(self):config = H2Configuration(client_side=True, header_encoding='utf-8')self.conn = H2Connection(config=config)@@ -37,7 +36,7 @@ class H2ClientProtocol(Protocol):self.stream_id_count = itertools.count(start=1, step=2)# Streams are stored in a dictionary keyed off their stream IDs- self.streams = {}+ self.streams: Dict[int, Stream] = {}# Boolean to keep track the connection is made# If requests are received before connection is made@@ -46,9 +45,11 @@ class H2ClientProtocol(Protocol):self.is_connection_made = Falseself._pending_request_stream_pool = deque()- # Some meta data of this connection- # initialized when connection is successfully made- self._metadata = {+ # Save an instance of ProtocolError raised by hyper-h2+ # We pass this instance to the streams ResponseFailed() failure+ self._protocol_error: Union[None, ProtocolError] = None++ self._metadata: H2ConnectionMetadataDict = {'certificate': None,'ip_address': None}@@ -68,7 +69,6 @@ class H2ClientProtocol(Protocol):request=request,connection=self.conn,conn_metadata=self._metadata,- write_to_transport=self._write_to_transport,cb_close=self._stream_close_cb)@@ -89,10 +89,10 @@ class H2ClientProtocol(Protocol):data = self.conn.data_to_send()self.transport.write(data)- LOGGER.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address']))+ logger.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address']))- def request(self, _request: Request):- stream = self._new_stream(_request)+ def request(self, request: Request):+ stream = self._new_stream(request)d = stream.get_response()# If connection is not yet established then add the@@ -109,7 +109,7 @@ class H2ClientProtocol(Protocol):sending some data now: we should open with the connection preamble."""self.destination = self.transport.getPeer()- LOGGER.info('Connection made to {}'.format(self.destination))+ logger.info('Connection made to {}'.format(self.destination))self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate())self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host)@@ -119,21 +119,37 @@ class H2ClientProtocol(Protocol):self.is_connection_made = Truedef dataReceived(self, data):- events = self.conn.receive_data(data)- self._handle_events(events)- self._write_to_transport()+ try:+ events = self.conn.receive_data(data)+ self._handle_events(events)+ except ProtocolError as e:+ # TODO: In case of InvalidBodyLengthError -- terminate only one stream++ # Save this error as ultimately the connection will be dropped+ # internally by hyper-h2. Saved error will be passed to all the streams+ # closed with the connection.+ self._protocol_error = e++ # We lose the transport connection here+ self.transport.loseConnection()+ finally:+ self._write_to_transport()def connectionLost(self, reason=connectionDone):"""Called by Twisted when the transport connection is lost.No need to write anything to transport here."""# Pop all streams which were pending and were not yet started- for stream_id in list(self.streams):- self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST, reason)+ # NOTE: Stream.close() pops the element from the streams dictionary+ # which raises `RuntimeError: dictionary changed size during iteration`+ # Hence, we copy the streams into a list.+ for stream in list(self.streams.values()):+ stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error)self.conn.close_connection()- LOGGER.warning("Connection lost with reason " + str(reason))+ if not reason.check(connectionDone):+ logger.warning("Connection lost with reason " + str(reason))def _handle_events(self, events):"""Private method which acts as a bridge between the events@@ -144,7 +160,7 @@ class H2ClientProtocol(Protocol):triggered by sending data"""for event in events:- LOGGER.debug(event)+ logger.debug(event)if isinstance(event, DataReceived):self.data_received(event)elif isinstance(event, ResponseReceived):@@ -158,16 +174,14 @@ class H2ClientProtocol(Protocol):elif isinstance(event, SettingsAcknowledged):self.settings_acknowledged(event)else:- LOGGER.info("Received unhandled event {}".format(event))+ logger.info("Received unhandled event {}".format(event))# Event handler functions starts heredef data_received(self, event: DataReceived):- stream_id = event.stream_id- self.streams[stream_id].receive_data(event.data, event.flow_controlled_length)+ self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length)def response_received(self, event: ResponseReceived):- stream_id = event.stream_id- self.streams[stream_id].receive_headers(event.headers)+ self.streams[event.stream_id].receive_headers(event.headers)def settings_acknowledged(self, event: SettingsAcknowledged):# Send off all the pending requests@@ -175,8 +189,7 @@ class H2ClientProtocol(Protocol):self._send_pending_requests()def stream_ended(self, event: StreamEnded):- stream_id = event.stream_id- self.streams[stream_id].close(StreamCloseReason.ENDED)+ self.streams[event.stream_id].close(StreamCloseReason.ENDED)def stream_reset(self, event: StreamReset):self.streams[event.stream_id].close(StreamCloseReason.RESET)diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.pyindex a26a33918..da0181d52 100644--- a/scrapy/core/http2/stream.py+++ b/scrapy/core/http2/stream.py@@ -1,7 +1,7 @@import logging-from enum import IntFlag, auto+from enum import Enumfrom io import BytesIO-from typing import Dict+from typing import Callable, Listfrom urllib.parse import urlparsefrom h2.connection import H2Connection@@ -10,45 +10,31 @@ from h2.exceptions import StreamClosedErrorfrom twisted.internet.defer import Deferred, CancelledErrorfrom twisted.python.failure import Failurefrom twisted.web.client import ResponseFailed-# for python < 3.8 -- typing.TypedDict is undefined-from typing_extensions import TypedDict+from scrapy.core.http2.types import H2ConnectionMetadataDict, H2ResponseDictfrom scrapy.http import Requestfrom scrapy.http.headers import Headersfrom scrapy.responsetypes import responsetypes+logger = logging.getLogger(__name__)-class _ResponseTypedDict(TypedDict):- # Data received frame by frame from the server is appended- # and passed to the response Deferred when completely received.- body: BytesIO- # The amount of data received that counts against the flow control- # window- flow_controlled_size: int-- # Headers received after sending the request- headers: Headers---LOGGER = logging.getLogger(__name__)---class StreamCloseReason(IntFlag):+class StreamCloseReason(Enum):# Received a StreamEnded event- ENDED = auto()+ ENDED = 1# Received a StreamReset event -- ended abruptly- RESET = auto()+ RESET = 2# Transport connection was lost- CONNECTION_LOST = auto()+ CONNECTION_LOST = 3# Expected response body size is more than allowed limit- MAXSIZE_EXCEEDED = auto()+ MAXSIZE_EXCEEDED = 4- # When the response deferred is cancelled- CANCELLED = auto()+ # When the response deferred is cancelled by the client+ # (happens when client called response_deferred.cancel())+ CANCELLED = 5class Stream:@@ -63,34 +49,30 @@ class Stream:"""def __init__(- self,- stream_id: int,- request: Request,- connection: H2Connection,- conn_metadata: Dict,- write_to_transport,- cb_close,- download_maxsize=0,- download_warnsize=0,- fail_on_data_loss=True+ self,+ stream_id: int,+ request: Request,+ connection: H2Connection,+ conn_metadata: H2ConnectionMetadataDict,+ cb_close: Callable[[int], None],+ download_maxsize: int = 0,+ download_warnsize: int = 0,+ fail_on_data_loss: bool = True):"""Arguments:- stream_id {int} -- For one HTTP/2 connection each stream is+ stream_id -- For one HTTP/2 connection each stream isuniquely identified by a single integer- request {Request} -- HTTP request- connection {H2Connection} -- HTTP/2 connection this stream belongs to.- conn_metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection- write_to_transport {callable} -- Method used to write & send data to the server- This method should be used whenever some frame is to be sent to the server.- cb_close {callable} -- Method called when this stream is closed+ request -- HTTP request+ connection -- HTTP/2 connection this stream belongs to.+ conn_metadata -- Reference to dictionary having metadata of HTTP/2 connection+ cb_close -- Method called when this stream is closedto notify the TCP connection instance."""self.stream_id = stream_idself._request = requestself._conn = connectionself._conn_metadata = conn_metadata- self._write_to_transport = write_to_transportself._cb_close = cb_closeself._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)@@ -119,7 +101,7 @@ class Stream:# Private variable used to build the response# this response is then converted to appropriate Response class# passed to the response deferred callback- self._response: _ResponseTypedDict = {+ self._response: H2ResponseDict = {'body': BytesIO(),'flow_controlled_size': 0,'headers': Headers({})@@ -136,6 +118,25 @@ class Stream:__repr__ = __str__+ @property+ def _log_warnsize(self) -> bool:+ """Checks if we have received data which exceeds the download warnsize+ and whether we have not already logged about it.++ Returns:+ True if both the above conditions hold true+ False if any of the conditions is false+ """+ content_length_header = int(self._response['headers'].get(b'Content-Length', -1))+ return (+ self._download_warnsize+ and (+ self._response['flow_controlled_size'] > self._download_warnsize+ or content_length_header > self._download_warnsize+ )+ and not self._reached_warnsize+ )+def get_response(self):"""Simply return a Deferred which fires when responsefrom the asynchronous request is available@@ -166,10 +167,7 @@ class Stream:def initiate_request(self):headers = self._get_request_headers()self._conn.send_headers(self.stream_id, headers, end_stream=False)- self._write_to_transport()-self.request_sent = True-self.send_data()def send_data(self):@@ -211,20 +209,19 @@ class Stream:self.remaining_content_length = self.remaining_content_length - chunk_sizeself.remaining_content_length = max(0, self.remaining_content_length)- LOGGER.debug("{} sending {}/{} data bytes ({} frames) to {}".format(- self,- self.content_length - self.remaining_content_length, self.content_length,- data_frames_sent,- self._conn_metadata['ip_address'])+ logger.debug(+ "{stream} sending {received}/{expected} data bytes ({frames} frames) to {ip_address}".format(+ stream=self,+ received=self.content_length - self.remaining_content_length,+ expected=self.content_length,+ frames=data_frames_sent,+ ip_address=self._conn_metadata['ip_address']))# End the stream if no more data needs to be sendif self.remaining_content_length == 0:self._conn.end_stream(self.stream_id)- # Write data to transport -- Empty the outstanding data- self._write_to_transport()-# Q. What about the rest of the data?# Ans: Remaining Data frames will be sent when we get a WindowUpdate frame@@ -240,24 +237,21 @@ class Stream:self._response['body'].write(data)self._response['flow_controlled_size'] += flow_controlled_length+ # We check maxsize here in case the Content-Length header was not receivedif self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize:- # Clear buffer earlier to avoid keeping data in memory for a long time- self._response['body'].truncate(0)self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)return- if self._download_warnsize \- and self._response['flow_controlled_size'] > self._download_warnsize \- and not self._reached_warnsize:+ if self._log_warnsize:self._reached_warnsize = True- warning_msg = ('Received more ({bytes}) bytes than download ',- 'warn size ({warnsize}) in request {request}')+ warning_msg = 'Received more ({bytes}) bytes than download ' \+ + 'warn size ({warnsize}) in request {request}'warning_args = {'bytes': self._response['flow_controlled_size'],'warnsize': self._download_warnsize,'request': self._request}- LOGGER.warning(warning_msg, warning_args)+ logger.warning(warning_msg.format(**warning_args))# Acknowledge the data receivedself._conn.acknowledge_received_data(@@ -275,23 +269,27 @@ class Stream:self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)return- if self._download_warnsize and expected_size > self._download_warnsize:- warning_msg = ("Expected response size ({size}) larger than ",- "download warn size ({warnsize}) in request {request}.")+ if self._log_warnsize:+ self._reached_warnsize = True+ warning_msg = 'Expected response size ({size}) larger than ' \+ + 'download warn size ({warnsize}) in request {request}'warning_args = {- 'size': expected_size, 'warnsize': self._download_warnsize,+ 'size': expected_size,+ 'warnsize': self._download_warnsize,'request': self._request}- LOGGER.warning(warning_msg, warning_args)+ logger.warning(warning_msg.format(**warning_args))def reset_stream(self, reason=StreamCloseReason.RESET):"""Close this stream by sending a RST_FRAME to the remote peer"""if self.stream_closed_local:raise StreamClosedError(self.stream_id)+ # Clear buffer earlier to avoid keeping data in memory for a long time+ self._response['body'].truncate(0)+self.stream_closed_local = Trueself._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)- self._write_to_transport()self.close(reason)def _is_data_lost(self) -> bool:@@ -302,8 +300,11 @@ class Stream:return expected_size != received_body_size- def close(self, reason: StreamCloseReason, failure=None):+ def close(self, reason: StreamCloseReason, error: Exception = None):"""Based on the reason sent we will handle each case.++ Arguments:+ reason -- One if StreamCloseReason"""if self.stream_closed_server:raise StreamClosedError(self.stream_id)@@ -311,35 +312,16 @@ class Stream:self._cb_close(self.stream_id)self.stream_closed_server = True- # Do nothing if the response deferred was cancelledflags = Noneif b'Content-Length' not in self._response['headers']:- # Missing Content-Length - PotentialDataLoss+ # Missing Content-Length - {twisted.web.http.PotentialDataLoss}flags = ['partial']- elif self._is_data_lost():- if self._fail_on_dataloss:- self._deferred_response.errback(ResponseFailed([Failure()]))- return- else:- flags = ['dataloss']-- if reason is StreamCloseReason.ENDED:- self._fire_response_deferred(flags)-- # Stream was abruptly ended here- elif reason is StreamCloseReason.CANCELLED:- # Client has cancelled the request. Remove all the data- # received and fire the response deferred with no flags set- self._response['body'].truncate(0)- self._response['headers'].clear()- self._fire_response_deferred()- elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST):- if failure is None:- self._deferred_response.errback(ResponseFailed([Failure()]))- else:- self._deferred_response.errback(failure)- elif reason is StreamCloseReason.MAXSIZE_EXCEEDED:+ # NOTE: Order of handling the events is important here+ # As we immediately cancel the request when maxsize is exceeded while+ # receiving DATA_FRAME's when we have received the headers (not+ # having Content-Length)+ if reason is StreamCloseReason.MAXSIZE_EXCEEDED:expected_size = int(self._response['headers'].get(b'Content-Length', -1))error_msg = ("Cancelling download of {url}: expected response ""size ({size}) larger than download max size ({maxsize}).")@@ -349,14 +331,34 @@ class Stream:'maxsize': self._download_maxsize}- LOGGER.error(error_msg, error_args)+ logger.error(error_msg, error_args)self._deferred_response.errback(CancelledError(error_msg.format(**error_args)))- def _fire_response_deferred(self, flags=None):+ elif reason is StreamCloseReason.ENDED:+ self._fire_response_deferred(flags)++ # Stream was abruptly ended here+ elif reason is StreamCloseReason.CANCELLED:+ # Client has cancelled the request. Remove all the data+ # received and fire the response deferred with no flags set++ # NOTE: The data is already flushed in Stream.reset_stream() called+ # immediately when the stream needs to be cancelled++ # There maybe no :status in headers, we make+ # HTTP Status Code: 499 - Client Closed Request+ self._response['headers'][':status'] = '499'+ self._fire_response_deferred()++ elif reason in (StreamCloseReason.RESET, StreamCloseReason.CONNECTION_LOST):+ self._deferred_response.errback(ResponseFailed([+ error if error else Failure()+ ]))++ def _fire_response_deferred(self, flags: List[str] = None):"""Builds response from the self._response dictand fires the response deferred callback with thegenerated response instance"""- # TODO: Update Client Side Status Codes herebody = self._response['body'].getvalue()response_cls = responsetypes.from_args(@@ -365,11 +367,7 @@ class Stream:body=body)- # If there is no :status in headers- # (happens when client called response_deferred.cancel())- # HTTP Status Code: 499 - Client Closed Request- status = self._response['headers'].get(':status', '499')-+ status = self._response['headers'][':status']response = response_cls(url=self._request.url,status=status,diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.pynew file mode 100644index 000000000..f28bf9472--- /dev/null+++ b/scrapy/core/http2/types.py@@ -0,0 +1,30 @@+from io import BytesIO+from ipaddress import IPv4Address, IPv6Address+from typing import Union++from twisted.internet.ssl import Certificate+# for python < 3.8 -- typing.TypedDict is undefined+from typing_extensions import TypedDict++from scrapy.http.headers import Headers+++class H2ConnectionMetadataDict(TypedDict):+ """Some meta data of this connection+ initialized when connection is successfully made+ """+ certificate: Union[None, Certificate]+ ip_address: Union[None, IPv4Address, IPv6Address]+++class H2ResponseDict(TypedDict):+ # Data received frame by frame from the server is appended+ # and passed to the response Deferred when completely received.+ body: BytesIO++ # The amount of data received that counts against the flow control+ # window+ flow_controlled_size: int++ # Headers received after sending the request+ headers: Headersdiff --git a/setup.py b/setup.pyindex 575c74e7f..8e50733e6 100644--- a/setup.py+++ b/setup.py@@ -67,7 +67,7 @@ setup(],python_requires='>=3.5.2',install_requires=[- 'Twisted[http2]>=17.9.0'+ 'Twisted[http2]>=17.9.0','cryptography>=2.0','cssselect>=0.9.1','lxml>=3.5.0',