Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

test: GET request for HTTP2Client using mockserver

Adityaa97ac0adf86b
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 188c14c15..455d8777e 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -9,6 +9,8 @@ from h2.events import (
DataReceived, ResponseReceived, SettingsAcknowledged,
StreamEnded, StreamReset, WindowUpdated
)
+
+from twisted.internet.ssl import Certificate
from twisted.internet.protocol import connectionDone, Protocol
from scrapy.core.http2.stream import Stream, StreamCloseReason
@@ -115,7 +117,7 @@ class H2ClientProtocol(Protocol):
self.destination = self.transport.getPeer()
LOGGER.info('Connection made to {}'.format(self.destination))
- self._metadata['certificate'] = self.transport.getPeerCertificate()
+ self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host)
self.conn.initiate_connection()
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 07a4428c8..112ce5bcd 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -117,7 +117,7 @@ class Stream:
self._deferred_response = Deferred()
def __str__(self):
- return "Stream(id={})".format(self.stream_id)
+ return "Stream(id={})".format(repr(self.stream_id))
__repr__ = __str__
@@ -299,9 +299,6 @@ class Stream:
def close(self, reason: StreamCloseReason):
"""Based on the reason sent we will handle each case.
"""
- # TODO: In case of abruptly stream close
- # Q1. Do we need to send the request again?
- # Q2. What response should we send now?
if self.stream_closed_server:
raise StreamClosedError(self.stream_id)
@@ -346,10 +343,7 @@ class Stream:
and fires the response deferred callback with the
generated response instance"""
# TODO:
- # 2. Should we fire this in case of
- # 2.1 StreamReset in between when data is received partially
- # 2.2 Forcefully closed the stream
- # 3. Update Client Side Status Codes here
+ # 1. Update Client Side Status Codes here
response_cls = responsetypes.from_args(
headers=self._response['headers'],
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 7830f7028..a67575d3c 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -1,71 +1,50 @@
-import os
-import shutil
+from urllib.parse import urlparse
-from twisted.internet import defer, reactor
+from twisted.internet import reactor
from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
-from twisted.internet.ssl import optionsForClientTLS
-from twisted.protocols.policies import WrappingFactory
-from twisted.python.filepath import FilePath
+from twisted.internet.ssl import CertificateOptions
from twisted.trial import unittest
-from twisted.web import static, server
from scrapy.core.http2.protocol import H2ClientProtocol
-from scrapy.http import Request
-from tests.mockserver import ssl_context_factory
+from scrapy.http import Request, Response
+from tests.mockserver import MockServer
class Http2ClientProtocolTestCase(unittest.TestCase):
scheme = 'https'
- # only used for HTTPS tests
- file_key = 'keys/localhost.key'
- file_certificate = 'keys/localhost.crt'
-
def setUp(self):
# Start server for testing
- self.path_temp = self.mktemp()
- os.mkdir(self.path_temp)
- FilePath(self.path_temp).child('file').setContent(b"0123456789")
- r = static.File(self.path_temp)
+ self.mockserver = MockServer()
+ self.mockserver.__enter__()
- self.site = server.Site(r, timeout=None)
- self.wrapper = WrappingFactory(self.site)
- self.host = 'localhost'
- if self.scheme is 'https':
- self.port = reactor.listenSSL(
- 0, self.wrapper,
- ssl_context_factory(self.file_key, self.file_certificate),
- interface=self.host
- )
+ if self.scheme == 'https':
+ self.url = urlparse(self.mockserver.https_address)
else:
- self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
-
- self.port_number = self.port.getHost().port
-
- # Connect to the server using the custom HTTP2ClientProtocol
- options = optionsForClientTLS(
- hostname=self.host,
- acceptableProtocols=[b'h2']
- )
+ self.url = urlparse(self.mockserver.http_address)
self.protocol = H2ClientProtocol()
- connectProtocol(
- endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options),
- protocol=self.protocol
- )
+ # Connect to the server using the custom HTTP2ClientProtocol
+ options = CertificateOptions(acceptableProtocols=[b'h2'])
+ endpoint = SSL4ClientEndpoint(reactor, self.url.hostname, self.url.port, options)
+ connectProtocol(endpoint, self.protocol)
def getURL(self, path):
- return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path)
+ return "{}://{}:{}/{}".format(self.url.scheme, self.url.hostname, self.url.port, path)
- @defer.inlineCallbacks
def tearDown(self):
- yield self.port.stopListening()
- shutil.rmtree(self.path_temp)
+ self.mockserver.__exit__(None, None, None)
def test_download(self):
- request = Request(self.getURL('file'))
+ request = Request(self.getURL(''))
+
+ def assert_response(response: Response):
+ self.assertEqual(response.body, b'Scrapy mock HTTP server\n')
+ self.assertEqual(response.status, 200)
+ self.assertEqual(response.request, request)
+ self.assertEqual(response.url, request.url)
+
d = self.protocol.request(request)
- d.addCallback(lambda response: response.body)
- d.addCallback(self.assertEqual, b"0123456789")
+ d.addCallback(assert_response)
return d