Scrapy, a fast high-level web crawling & scraping framework for Python.
test: mockserver with h2 protocol for tests
Aditya700df3eeb7c5
diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.pynew file mode 100644index 000000000..c7782a518--- /dev/null+++ b/scrapy/core/http2/test_protocol.py@@ -0,0 +1,67 @@+# This is simple script to test++import json++from twisted.internet import reactor+from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint+from twisted.internet.ssl import optionsForClientTLS++from scrapy.core.http2.protocol import H2ClientProtocol+from scrapy.http import Request, Response, JsonRequest++try:+ with open('data.json', 'r') as f:+ JSON_DATA = json.load(f)+except:+ JSON_DATA = {+ "data": "To test for really large amount of data -- Add data.json with lots of data.",+ "why": "To test whether correct data is sent :)"+ }++# Use nghttp2 for testing whether basic setup works - for small response+HTTPBIN_AUTHORITY = u'nghttp2.org'+HTTPBIN_REQUEST_URLS = 1 * [+ Request(url='https://nghttp2.org/httpbin/get', method='GET'),+ Request(url='https://nghttp2.org/httpbin/post', method='POST'),+ JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA),+]++# Use POKE_API for testing large responses+POKE_API_AUTHORITY = u'pokeapi.co'+POKE_API_REQUESTS = 15 * [+ Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'),+ Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'),+ Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'),+ Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'), # should give 404+]++AUTHORITY = POKE_API_AUTHORITY+REQUEST_URLS = POKE_API_REQUESTS++options = optionsForClientTLS(+ hostname=AUTHORITY,+ acceptableProtocols=[b'h2'],+)++protocol = H2ClientProtocol()++count_responses = 1+++def print_response(response):+ global count_responses+ assert isinstance(response, Response)+ print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body)))+ count_responses = count_responses + 1+++for request in REQUEST_URLS:+ d = protocol.request(request)+ d.addCallback(print_response)++connectProtocol(+ SSL4ClientEndpoint(reactor, AUTHORITY, 443, options),+ protocol+)++reactor.run()diff --git a/setup.py b/setup.pyindex 1b3c6771a..dafa5684a 100644--- a/setup.py+++ b/setup.py@@ -1,8 +1,8 @@from os.path import dirname, join+from pkg_resources import parse_versionfrom setuptools import setup, find_packages, __version__ as setuptools_version-with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f:version = f.read().decode('ascii').strip()@@ -25,12 +25,11 @@ if has_environment_marker_platform_impl_support():'PyPyDispatcher>=2.1.0',]-setup(name='Scrapy',version=version,url='https://scrapy.org',- project_urls = {+ project_urls={'Documentation': 'https://docs.scrapy.org/','Source': 'https://github.com/scrapy/scrapy','Tracker': 'https://github.com/scrapy/scrapy/issues',@@ -69,6 +68,7 @@ setup(python_requires='>=3.5',install_requires=['Twisted>=17.9.0',+ 'Twisted[http2]>=17.9.0''cryptography>=2.0','cssselect>=0.9.1','lxml>=3.5.0',diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pynew file mode 100644index 000000000..7830f7028--- /dev/null+++ b/tests/test_http2_client_protocol.py@@ -0,0 +1,71 @@+import os+import shutil++from twisted.internet import defer, reactor+from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint+from twisted.internet.ssl import optionsForClientTLS+from twisted.protocols.policies import WrappingFactory+from twisted.python.filepath import FilePath+from twisted.trial import unittest+from twisted.web import static, server++from scrapy.core.http2.protocol import H2ClientProtocol+from scrapy.http import Request+from tests.mockserver import ssl_context_factory+++class Http2ClientProtocolTestCase(unittest.TestCase):+ scheme = 'https'++ # only used for HTTPS tests+ file_key = 'keys/localhost.key'+ file_certificate = 'keys/localhost.crt'++ def setUp(self):+ # Start server for testing+ self.path_temp = self.mktemp()+ os.mkdir(self.path_temp)+ FilePath(self.path_temp).child('file').setContent(b"0123456789")+ r = static.File(self.path_temp)++ self.site = server.Site(r, timeout=None)+ self.wrapper = WrappingFactory(self.site)+ self.host = 'localhost'+ if self.scheme is 'https':+ self.port = reactor.listenSSL(+ 0, self.wrapper,+ ssl_context_factory(self.file_key, self.file_certificate),+ interface=self.host+ )+ else:+ self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)++ self.port_number = self.port.getHost().port++ # Connect to the server using the custom HTTP2ClientProtocol+ options = optionsForClientTLS(+ hostname=self.host,+ acceptableProtocols=[b'h2']+ )++ self.protocol = H2ClientProtocol()++ connectProtocol(+ endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options),+ protocol=self.protocol+ )++ def getURL(self, path):+ return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path)++ @defer.inlineCallbacks+ def tearDown(self):+ yield self.port.stopListening()+ shutil.rmtree(self.path_temp)++ def test_download(self):+ request = Request(self.getURL('file'))+ d = self.protocol.request(request)+ d.addCallback(lambda response: response.body)+ d.addCallback(self.assertEqual, b"0123456789")+ return d