Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

test: mockserver with h2 protocol for tests

Aditya700df3eeb7c5
diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.py
new file mode 100644
index 000000000..c7782a518
--- /dev/null
+++ b/scrapy/core/http2/test_protocol.py
@@ -0,0 +1,67 @@
+# This is simple script to test
+
+import json
+
+from twisted.internet import reactor
+from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
+from twisted.internet.ssl import optionsForClientTLS
+
+from scrapy.core.http2.protocol import H2ClientProtocol
+from scrapy.http import Request, Response, JsonRequest
+
+try:
+ with open('data.json', 'r') as f:
+ JSON_DATA = json.load(f)
+except:
+ JSON_DATA = {
+ "data": "To test for really large amount of data -- Add data.json with lots of data.",
+ "why": "To test whether correct data is sent :)"
+ }
+
+# Use nghttp2 for testing whether basic setup works - for small response
+HTTPBIN_AUTHORITY = u'nghttp2.org'
+HTTPBIN_REQUEST_URLS = 1 * [
+ Request(url='https://nghttp2.org/httpbin/get', method='GET'),
+ Request(url='https://nghttp2.org/httpbin/post', method='POST'),
+ JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA),
+]
+
+# Use POKE_API for testing large responses
+POKE_API_AUTHORITY = u'pokeapi.co'
+POKE_API_REQUESTS = 15 * [
+ Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'),
+ Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'),
+ Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'),
+ Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'), # should give 404
+]
+
+AUTHORITY = POKE_API_AUTHORITY
+REQUEST_URLS = POKE_API_REQUESTS
+
+options = optionsForClientTLS(
+ hostname=AUTHORITY,
+ acceptableProtocols=[b'h2'],
+)
+
+protocol = H2ClientProtocol()
+
+count_responses = 1
+
+
+def print_response(response):
+ global count_responses
+ assert isinstance(response, Response)
+ print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body)))
+ count_responses = count_responses + 1
+
+
+for request in REQUEST_URLS:
+ d = protocol.request(request)
+ d.addCallback(print_response)
+
+connectProtocol(
+ SSL4ClientEndpoint(reactor, AUTHORITY, 443, options),
+ protocol
+)
+
+reactor.run()
diff --git a/setup.py b/setup.py
index 1b3c6771a..dafa5684a 100644
--- a/setup.py
+++ b/setup.py
@@ -1,8 +1,8 @@
from os.path import dirname, join
+
from pkg_resources import parse_version
from setuptools import setup, find_packages, __version__ as setuptools_version
-
with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f:
version = f.read().decode('ascii').strip()
@@ -25,12 +25,11 @@ if has_environment_marker_platform_impl_support():
'PyPyDispatcher>=2.1.0',
]
-
setup(
name='Scrapy',
version=version,
url='https://scrapy.org',
- project_urls = {
+ project_urls={
'Documentation': 'https://docs.scrapy.org/',
'Source': 'https://github.com/scrapy/scrapy',
'Tracker': 'https://github.com/scrapy/scrapy/issues',
@@ -69,6 +68,7 @@ setup(
python_requires='>=3.5',
install_requires=[
'Twisted>=17.9.0',
+ 'Twisted[http2]>=17.9.0'
'cryptography>=2.0',
'cssselect>=0.9.1',
'lxml>=3.5.0',
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
new file mode 100644
index 000000000..7830f7028
--- /dev/null
+++ b/tests/test_http2_client_protocol.py
@@ -0,0 +1,71 @@
+import os
+import shutil
+
+from twisted.internet import defer, reactor
+from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
+from twisted.internet.ssl import optionsForClientTLS
+from twisted.protocols.policies import WrappingFactory
+from twisted.python.filepath import FilePath
+from twisted.trial import unittest
+from twisted.web import static, server
+
+from scrapy.core.http2.protocol import H2ClientProtocol
+from scrapy.http import Request
+from tests.mockserver import ssl_context_factory
+
+
+class Http2ClientProtocolTestCase(unittest.TestCase):
+ scheme = 'https'
+
+ # only used for HTTPS tests
+ file_key = 'keys/localhost.key'
+ file_certificate = 'keys/localhost.crt'
+
+ def setUp(self):
+ # Start server for testing
+ self.path_temp = self.mktemp()
+ os.mkdir(self.path_temp)
+ FilePath(self.path_temp).child('file').setContent(b"0123456789")
+ r = static.File(self.path_temp)
+
+ self.site = server.Site(r, timeout=None)
+ self.wrapper = WrappingFactory(self.site)
+ self.host = 'localhost'
+ if self.scheme is 'https':
+ self.port = reactor.listenSSL(
+ 0, self.wrapper,
+ ssl_context_factory(self.file_key, self.file_certificate),
+ interface=self.host
+ )
+ else:
+ self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
+
+ self.port_number = self.port.getHost().port
+
+ # Connect to the server using the custom HTTP2ClientProtocol
+ options = optionsForClientTLS(
+ hostname=self.host,
+ acceptableProtocols=[b'h2']
+ )
+
+ self.protocol = H2ClientProtocol()
+
+ connectProtocol(
+ endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options),
+ protocol=self.protocol
+ )
+
+ def getURL(self, path):
+ return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path)
+
+ @defer.inlineCallbacks
+ def tearDown(self):
+ yield self.port.stopListening()
+ shutil.rmtree(self.path_temp)
+
+ def test_download(self):
+ request = Request(self.getURL('file'))
+ d = self.protocol.request(request)
+ d.addCallback(lambda response: response.body)
+ d.addCallback(self.assertEqual, b"0123456789")
+ return d