Scrapy, a fast high-level web crawling & scraping framework for Python.
test: GET request for HTTP2Client using mockserver
Adityaa97ac0adf86b
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex 188c14c15..455d8777e 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -9,6 +9,8 @@ from h2.events import (DataReceived, ResponseReceived, SettingsAcknowledged,StreamEnded, StreamReset, WindowUpdated)++from twisted.internet.ssl import Certificatefrom twisted.internet.protocol import connectionDone, Protocolfrom scrapy.core.http2.stream import Stream, StreamCloseReason@@ -115,7 +117,7 @@ class H2ClientProtocol(Protocol):self.destination = self.transport.getPeer()LOGGER.info('Connection made to {}'.format(self.destination))- self._metadata['certificate'] = self.transport.getPeerCertificate()+ self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate())self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host)self.conn.initiate_connection()diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.pyindex 07a4428c8..112ce5bcd 100644--- a/scrapy/core/http2/stream.py+++ b/scrapy/core/http2/stream.py@@ -117,7 +117,7 @@ class Stream:self._deferred_response = Deferred()def __str__(self):- return "Stream(id={})".format(self.stream_id)+ return "Stream(id={})".format(repr(self.stream_id))__repr__ = __str__@@ -299,9 +299,6 @@ class Stream:def close(self, reason: StreamCloseReason):"""Based on the reason sent we will handle each case."""- # TODO: In case of abruptly stream close- # Q1. Do we need to send the request again?- # Q2. What response should we send now?if self.stream_closed_server:raise StreamClosedError(self.stream_id)@@ -346,10 +343,7 @@ class Stream:and fires the response deferred callback with thegenerated response instance"""# TODO:- # 2. Should we fire this in case of- # 2.1 StreamReset in between when data is received partially- # 2.2 Forcefully closed the stream- # 3. Update Client Side Status Codes here+ # 1. Update Client Side Status Codes hereresponse_cls = responsetypes.from_args(headers=self._response['headers'],diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 7830f7028..a67575d3c 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -1,71 +1,50 @@-import os-import shutil+from urllib.parse import urlparse-from twisted.internet import defer, reactor+from twisted.internet import reactorfrom twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint-from twisted.internet.ssl import optionsForClientTLS-from twisted.protocols.policies import WrappingFactory-from twisted.python.filepath import FilePath+from twisted.internet.ssl import CertificateOptionsfrom twisted.trial import unittest-from twisted.web import static, serverfrom scrapy.core.http2.protocol import H2ClientProtocol-from scrapy.http import Request-from tests.mockserver import ssl_context_factory+from scrapy.http import Request, Response+from tests.mockserver import MockServerclass Http2ClientProtocolTestCase(unittest.TestCase):scheme = 'https'- # only used for HTTPS tests- file_key = 'keys/localhost.key'- file_certificate = 'keys/localhost.crt'-def setUp(self):# Start server for testing- self.path_temp = self.mktemp()- os.mkdir(self.path_temp)- FilePath(self.path_temp).child('file').setContent(b"0123456789")- r = static.File(self.path_temp)+ self.mockserver = MockServer()+ self.mockserver.__enter__()- self.site = server.Site(r, timeout=None)- self.wrapper = WrappingFactory(self.site)- self.host = 'localhost'- if self.scheme is 'https':- self.port = reactor.listenSSL(- 0, self.wrapper,- ssl_context_factory(self.file_key, self.file_certificate),- interface=self.host- )+ if self.scheme == 'https':+ self.url = urlparse(self.mockserver.https_address)else:- self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)-- self.port_number = self.port.getHost().port-- # Connect to the server using the custom HTTP2ClientProtocol- options = optionsForClientTLS(- hostname=self.host,- acceptableProtocols=[b'h2']- )+ self.url = urlparse(self.mockserver.http_address)self.protocol = H2ClientProtocol()- connectProtocol(- endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options),- protocol=self.protocol- )+ # Connect to the server using the custom HTTP2ClientProtocol+ options = CertificateOptions(acceptableProtocols=[b'h2'])+ endpoint = SSL4ClientEndpoint(reactor, self.url.hostname, self.url.port, options)+ connectProtocol(endpoint, self.protocol)def getURL(self, path):- return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path)+ return "{}://{}:{}/{}".format(self.url.scheme, self.url.hostname, self.url.port, path)- @defer.inlineCallbacksdef tearDown(self):- yield self.port.stopListening()- shutil.rmtree(self.path_temp)+ self.mockserver.__exit__(None, None, None)def test_download(self):- request = Request(self.getURL('file'))+ request = Request(self.getURL(''))++ def assert_response(response: Response):+ self.assertEqual(response.body, b'Scrapy mock HTTP server\n')+ self.assertEqual(response.status, 200)+ self.assertEqual(response.request, request)+ self.assertEqual(response.url, request.url)+d = self.protocol.request(request)- d.addCallback(lambda response: response.body)- d.addCallback(self.assertEqual, b"0123456789")+ d.addCallback(assert_response)return d