Scrapy, a fast high-level web crawling & scraping framework for Python.
feat: TypedDict for Stream._response
Aditya69f6d038c0bc
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex 455d8777e..9b8ec6c77 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -9,9 +9,8 @@ from h2.events import (DataReceived, ResponseReceived, SettingsAcknowledged,StreamEnded, StreamReset, WindowUpdated)--from twisted.internet.ssl import Certificatefrom twisted.internet.protocol import connectionDone, Protocol+from twisted.internet.ssl import Certificatefrom scrapy.core.http2.stream import Stream, StreamCloseReasonfrom scrapy.http import Request@@ -22,12 +21,7 @@ LOGGER = logging.getLogger(__name__)class H2ClientProtocol(Protocol):# TODO:# 1. Check for user-agent while testing- # 2. Add support for cookies- # 3. Handle priority updates (Not required)- # 4. Handle case when received events have StreamID = 0 (applied to H2Connection)- # 1 & 2:- # - Automatically handled by the Request middleware- # - request.headers will have 'Set-Cookie' value+ # 2. Handle case when received events have StreamID = 0 (applied to H2Connection)def __init__(self):config = H2Configuration(client_side=True, header_encoding='utf-8')@@ -185,8 +179,6 @@ class H2ClientProtocol(Protocol):self.streams[stream_id].close(StreamCloseReason.ENDED)def stream_reset(self, event: StreamReset):- # TODO: event.stream_id was abruptly closed- # Q. What should be the response? (Failure/Partial/???)self.streams[event.stream_id].close(StreamCloseReason.RESET)def window_updated(self, event: WindowUpdated):diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.pyindex 112ce5bcd..023f4f4eb 100644--- a/scrapy/core/http2/stream.py+++ b/scrapy/core/http2/stream.py@@ -10,11 +10,20 @@ from h2.exceptions import StreamClosedErrorfrom twisted.internet.defer import Deferred, CancelledErrorfrom twisted.python.failure import Failurefrom twisted.web.client import ResponseFailed+# for python < 3.8 -- typing.TypedDict is undefined+from typing_extensions import TypedDictfrom scrapy.http import Requestfrom scrapy.http.headers import Headersfrom scrapy.responsetypes import responsetypes++class _ResponseTypedDict(TypedDict):+ body: BytesIO+ flow_controlled_size: int+ headers: Headers++LOGGER = logging.getLogger(__name__)@@ -100,7 +109,7 @@ class Stream:# Private variable used to build the response# this response is then converted to appropriate Response class# passed to the response deferred callback- self._response = {+ self._response: _ResponseTypedDict = {# Data received frame by frame from the server is appended# and passed to the response Deferred when completely received.'body': BytesIO(),diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.pydeleted file mode 100644index c7782a518..000000000--- a/scrapy/core/http2/test_protocol.py+++ /dev/null@@ -1,67 +0,0 @@-# This is simple script to test--import json--from twisted.internet import reactor-from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint-from twisted.internet.ssl import optionsForClientTLS--from scrapy.core.http2.protocol import H2ClientProtocol-from scrapy.http import Request, Response, JsonRequest--try:- with open('data.json', 'r') as f:- JSON_DATA = json.load(f)-except:- JSON_DATA = {- "data": "To test for really large amount of data -- Add data.json with lots of data.",- "why": "To test whether correct data is sent :)"- }--# Use nghttp2 for testing whether basic setup works - for small response-HTTPBIN_AUTHORITY = u'nghttp2.org'-HTTPBIN_REQUEST_URLS = 1 * [- Request(url='https://nghttp2.org/httpbin/get', method='GET'),- Request(url='https://nghttp2.org/httpbin/post', method='POST'),- JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA),-]--# Use POKE_API for testing large responses-POKE_API_AUTHORITY = u'pokeapi.co'-POKE_API_REQUESTS = 15 * [- Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'),- Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'),- Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'),- Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'), # should give 404-]--AUTHORITY = POKE_API_AUTHORITY-REQUEST_URLS = POKE_API_REQUESTS--options = optionsForClientTLS(- hostname=AUTHORITY,- acceptableProtocols=[b'h2'],-)--protocol = H2ClientProtocol()--count_responses = 1---def print_response(response):- global count_responses- assert isinstance(response, Response)- print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body)))- count_responses = count_responses + 1---for request in REQUEST_URLS:- d = protocol.request(request)- d.addCallback(print_response)--connectProtocol(- SSL4ClientEndpoint(reactor, AUTHORITY, 443, options),- protocol-)--reactor.run()diff --git a/setup.py b/setup.pyindex dafa5684a..d1470df5e 100644--- a/setup.py+++ b/setup.py@@ -65,7 +65,7 @@ setup('Topic :: Software Development :: Libraries :: Application Frameworks','Topic :: Software Development :: Libraries :: Python Modules',],- python_requires='>=3.5',+ python_requires='>=3.5.2',install_requires=['Twisted>=17.9.0','Twisted[http2]>=17.9.0'@@ -80,6 +80,8 @@ setup('w3lib>=1.17.0','zope.interface>=4.1.3','protego>=0.1.15',+ 'itemadapter>=0.1.0',+ 'typing_extensions>=3.7'],extras_require=extras_require,)