Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

feat: TypedDict for Stream._response

Aditya69f6d038c0bc
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 455d8777e..9b8ec6c77 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -9,9 +9,8 @@ from h2.events import (
DataReceived, ResponseReceived, SettingsAcknowledged,
StreamEnded, StreamReset, WindowUpdated
)
-
-from twisted.internet.ssl import Certificate
from twisted.internet.protocol import connectionDone, Protocol
+from twisted.internet.ssl import Certificate
from scrapy.core.http2.stream import Stream, StreamCloseReason
from scrapy.http import Request
@@ -22,12 +21,7 @@ LOGGER = logging.getLogger(__name__)
class H2ClientProtocol(Protocol):
# TODO:
# 1. Check for user-agent while testing
- # 2. Add support for cookies
- # 3. Handle priority updates (Not required)
- # 4. Handle case when received events have StreamID = 0 (applied to H2Connection)
- # 1 & 2:
- # - Automatically handled by the Request middleware
- # - request.headers will have 'Set-Cookie' value
+ # 2. Handle case when received events have StreamID = 0 (applied to H2Connection)
def __init__(self):
config = H2Configuration(client_side=True, header_encoding='utf-8')
@@ -185,8 +179,6 @@ class H2ClientProtocol(Protocol):
self.streams[stream_id].close(StreamCloseReason.ENDED)
def stream_reset(self, event: StreamReset):
- # TODO: event.stream_id was abruptly closed
- # Q. What should be the response? (Failure/Partial/???)
self.streams[event.stream_id].close(StreamCloseReason.RESET)
def window_updated(self, event: WindowUpdated):
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 112ce5bcd..023f4f4eb 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -10,11 +10,20 @@ from h2.exceptions import StreamClosedError
from twisted.internet.defer import Deferred, CancelledError
from twisted.python.failure import Failure
from twisted.web.client import ResponseFailed
+# for python < 3.8 -- typing.TypedDict is undefined
+from typing_extensions import TypedDict
from scrapy.http import Request
from scrapy.http.headers import Headers
from scrapy.responsetypes import responsetypes
+
+class _ResponseTypedDict(TypedDict):
+ body: BytesIO
+ flow_controlled_size: int
+ headers: Headers
+
+
LOGGER = logging.getLogger(__name__)
@@ -100,7 +109,7 @@ class Stream:
# Private variable used to build the response
# this response is then converted to appropriate Response class
# passed to the response deferred callback
- self._response = {
+ self._response: _ResponseTypedDict = {
# Data received frame by frame from the server is appended
# and passed to the response Deferred when completely received.
'body': BytesIO(),
diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.py
deleted file mode 100644
index c7782a518..000000000
--- a/scrapy/core/http2/test_protocol.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# This is simple script to test
-
-import json
-
-from twisted.internet import reactor
-from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
-from twisted.internet.ssl import optionsForClientTLS
-
-from scrapy.core.http2.protocol import H2ClientProtocol
-from scrapy.http import Request, Response, JsonRequest
-
-try:
- with open('data.json', 'r') as f:
- JSON_DATA = json.load(f)
-except:
- JSON_DATA = {
- "data": "To test for really large amount of data -- Add data.json with lots of data.",
- "why": "To test whether correct data is sent :)"
- }
-
-# Use nghttp2 for testing whether basic setup works - for small response
-HTTPBIN_AUTHORITY = u'nghttp2.org'
-HTTPBIN_REQUEST_URLS = 1 * [
- Request(url='https://nghttp2.org/httpbin/get', method='GET'),
- Request(url='https://nghttp2.org/httpbin/post', method='POST'),
- JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA),
-]
-
-# Use POKE_API for testing large responses
-POKE_API_AUTHORITY = u'pokeapi.co'
-POKE_API_REQUESTS = 15 * [
- Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'),
- Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'),
- Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'),
- Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'), # should give 404
-]
-
-AUTHORITY = POKE_API_AUTHORITY
-REQUEST_URLS = POKE_API_REQUESTS
-
-options = optionsForClientTLS(
- hostname=AUTHORITY,
- acceptableProtocols=[b'h2'],
-)
-
-protocol = H2ClientProtocol()
-
-count_responses = 1
-
-
-def print_response(response):
- global count_responses
- assert isinstance(response, Response)
- print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body)))
- count_responses = count_responses + 1
-
-
-for request in REQUEST_URLS:
- d = protocol.request(request)
- d.addCallback(print_response)
-
-connectProtocol(
- SSL4ClientEndpoint(reactor, AUTHORITY, 443, options),
- protocol
-)
-
-reactor.run()
diff --git a/setup.py b/setup.py
index dafa5684a..d1470df5e 100644
--- a/setup.py
+++ b/setup.py
@@ -65,7 +65,7 @@ setup(
'Topic :: Software Development :: Libraries :: Application Frameworks',
'Topic :: Software Development :: Libraries :: Python Modules',
],
- python_requires='>=3.5',
+ python_requires='>=3.5.2',
install_requires=[
'Twisted>=17.9.0',
'Twisted[http2]>=17.9.0'
@@ -80,6 +80,8 @@ setup(
'w3lib>=1.17.0',
'zope.interface>=4.1.3',
'protego>=0.1.15',
+ 'itemadapter>=0.1.0',
+ 'typing_extensions>=3.7'
],
extras_require=extras_require,
)