Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

refactor: move all http2 tests in separate files

Adityaaf73f141b23a
diff --git a/tests/test_download_handlers_http2.py b/tests/test_download_handlers_http2.py
new file mode 100644
index 000000000..583dc1d17
--- /dev/null
+++ b/tests/test_download_handlers_http2.py
@@ -0,0 +1,158 @@
+from unittest import mock
+
+from twisted.internet import defer, error, reactor
+from twisted.trial import unittest
+from twisted.web import server
+
+from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+from scrapy.http import Request
+from scrapy.spiders import Spider
+from scrapy.utils.misc import create_instance
+from scrapy.utils.test import get_crawler
+from tests.mockserver import ssl_context_factory
+from tests.test_downloader_handlers import (
+ Https11TestCase, Https11CustomCiphers,
+ Http11MockServerTestCase, Http11ProxyTestCase,
+ UriResource
+)
+
+
+class Https2TestCase(Https11TestCase):
+ scheme = 'https'
+ download_handler_cls = H2DownloadHandler
+ HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
+
+ @defer.inlineCallbacks
+ def test_download_with_maxsize_very_large_file(self):
+ with mock.patch('scrapy.core.http2.stream.logger') as logger:
+ request = Request(self.getURL('largechunkedfile'))
+
+ def check(logger):
+ logger.error.assert_called_once_with(mock.ANY)
+
+ d = self.download_request(request, Spider('foo', download_maxsize=1500))
+ yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+
+ # As the error message is logged in the dataReceived callback, we
+ # have to give a bit of time to the reactor to process the queue
+ # after closing the connection.
+ d = defer.Deferred()
+ d.addCallback(check)
+ reactor.callLater(.1, d.callback, logger)
+ yield d
+
+ def test_download_broken_content_cause_data_loss(self, url='broken'):
+ raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+ def test_download_broken_chunked_content_cause_data_loss(self):
+ raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+ def test_download_broken_content_allow_data_loss(self, url='broken'):
+ raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+ def test_download_broken_chunked_content_allow_data_loss(self):
+ raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+ def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
+ raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+ def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
+ raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+
+class Https2WrongHostnameTestCase(Https2TestCase):
+ tls_log_message = (
+ 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The '
+ 'Internet/O=Scrapy/CN=www.example.com/[email protected]", '
+ 'subject "/C=XW/ST=XW/L=The '
+ 'Internet/O=Scrapy/CN=www.example.com/[email protected]"'
+ )
+
+ # above tests use a server certificate for "localhost",
+ # client connection to "localhost" too.
+ # here we test that even if the server certificate is for another domain,
+ # "www.example.com" in this case,
+ # the tests still pass
+ keyfile = 'keys/example-com.key.pem'
+ certfile = 'keys/example-com.cert.pem'
+
+
+class Https2InvalidDNSId(Https2TestCase):
+ """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
+
+ def setUp(self):
+ super(Https2InvalidDNSId, self).setUp()
+ self.host = '127.0.0.1'
+
+
+class Https2InvalidDNSPattern(Https2TestCase):
+ """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
+
+ keyfile = 'keys/localhost.ip.key'
+ certfile = 'keys/localhost.ip.crt'
+
+ def setUp(self):
+ try:
+ from service_identity.exceptions import CertificateError # noqa: F401
+ except ImportError:
+ raise unittest.SkipTest("cryptography lib is too old")
+ self.tls_log_message = (
+ 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
+ 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
+ )
+ super(Https2InvalidDNSPattern, self).setUp()
+
+
+class Https2CustomCiphers(Https11CustomCiphers):
+ scheme = 'https'
+ download_handler_cls = H2DownloadHandler
+
+
+class Http2MockServerTestCase(Http11MockServerTestCase):
+ """HTTP 2.0 test case with MockServer"""
+
+
+class Https2ProxyTestCase(Http11ProxyTestCase):
+ # only used for HTTPS tests
+ keyfile = 'keys/localhost.key'
+ certfile = 'keys/localhost.crt'
+
+ scheme = 'https'
+ host = u'127.0.0.1'
+
+ download_handler_cls = H2DownloadHandler
+ expected_http_proxy_request_body = b'/'
+
+ def setUp(self):
+ site = server.Site(UriResource(), timeout=None)
+ self.port = reactor.listenSSL(
+ 0, site,
+ ssl_context_factory(self.keyfile, self.certfile),
+ interface=self.host
+ )
+ self.portno = self.port.getHost().port
+ self.download_handler = create_instance(self.download_handler_cls, None, get_crawler())
+ self.download_request = self.download_handler.download_request
+
+ def getURL(self, path):
+ return f"{self.scheme}://{self.host}:{self.portno}/{path}"
+
+ def test_download_with_proxy_https_noconnect(self):
+ def _test(response):
+ self.assertEqual(response.status, 200)
+ self.assertEqual(response.url, request.url)
+ self.assertEqual(response.body, b'/')
+
+ http_proxy = '%s?noconnect' % self.getURL('')
+ request = Request('https://example.com', meta={'proxy': http_proxy})
+ with self.assertWarnsRegex(
+ Warning,
+ r'Using HTTPS proxies in the noconnect mode is not supported by the '
+ r'downloader handler.'
+ ):
+ return self.download_request(request, Spider('foo')).addCallback(_test)
+
+ @defer.inlineCallbacks
+ def test_download_with_proxy_https_timeout(self):
+ with self.assertRaises(NotImplementedError):
+ yield super(Https2ProxyTestCase, self).test_download_with_proxy_https_timeout()
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 1cedf6b10..8b2b2c32c 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -23,7 +23,6 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
-from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Headers, Request
@@ -315,8 +314,8 @@ class HttpTestCase(unittest.TestCase):
host = self.host + ':' + str(self.portno)
def _test(response):
- self.assertEqual(response.body, to_bytes(host))
- self.assertEqual(request.headers.get('Host'), to_bytes(host))
+ self.assertEqual(response.body, host.encode())
+ self.assertEqual(request.headers.get('Host'), host.encode())
request = Request(self.getURL('host'), headers={'Host': host})
return self.download_request(request, Spider('foo')).addCallback(_test)
@@ -522,49 +521,6 @@ class Https11TestCase(Http11TestCase):
yield download_handler.close()
-class Https2TestCase(Https11TestCase):
- scheme = 'https'
- download_handler_cls = H2DownloadHandler
- HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
-
- @defer.inlineCallbacks
- def test_download_with_maxsize_very_large_file(self):
- with mock.patch('scrapy.core.http2.stream.logger') as logger:
- request = Request(self.getURL('largechunkedfile'))
-
- def check(logger):
- logger.error.assert_called_once_with(mock.ANY)
-
- d = self.download_request(request, Spider('foo', download_maxsize=1500))
- yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
-
- # As the error message is logged in the dataReceived callback, we
- # have to give a bit of time to the reactor to process the queue
- # after closing the connection.
- d = defer.Deferred()
- d.addCallback(check)
- reactor.callLater(.1, d.callback, logger)
- yield d
-
- def test_download_broken_content_cause_data_loss(self, url='broken'):
- raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
-
- def test_download_broken_chunked_content_cause_data_loss(self):
- raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
-
- def test_download_broken_content_allow_data_loss(self, url='broken'):
- raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
-
- def test_download_broken_chunked_content_allow_data_loss(self):
- raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
-
- def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
- raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
-
- def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
- raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
-
-
class Https11WrongHostnameTestCase(Http11TestCase):
scheme = 'https'
@@ -577,23 +533,6 @@ class Https11WrongHostnameTestCase(Http11TestCase):
certfile = 'keys/example-com.cert.pem'
-class Https2WrongHostnameTestCase(Https2TestCase):
- tls_log_message = (
- 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The '
- 'Internet/O=Scrapy/CN=www.example.com/[email protected]", '
- 'subject "/C=XW/ST=XW/L=The '
- 'Internet/O=Scrapy/CN=www.example.com/[email protected]"'
- )
-
- # above tests use a server certificate for "localhost",
- # client connection to "localhost" too.
- # here we test that even if the server certificate is for another domain,
- # "www.example.com" in this case,
- # the tests still pass
- keyfile = 'keys/example-com.key.pem'
- certfile = 'keys/example-com.cert.pem'
-
-
class Https11InvalidDNSId(Https11TestCase):
"""Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
@@ -602,14 +541,6 @@ class Https11InvalidDNSId(Https11TestCase):
self.host = '127.0.0.1'
-class Https2InvalidDNSId(Https2TestCase):
- """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
-
- def setUp(self):
- super(Https2InvalidDNSId, self).setUp()
- self.host = '127.0.0.1'
-
-
class Https11InvalidDNSPattern(Https11TestCase):
"""Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
@@ -628,24 +559,6 @@ class Https11InvalidDNSPattern(Https11TestCase):
super(Https11InvalidDNSPattern, self).setUp()
-class Https2InvalidDNSPattern(Https2TestCase):
- """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
-
- keyfile = 'keys/localhost.ip.key'
- certfile = 'keys/localhost.ip.crt'
-
- def setUp(self):
- try:
- from service_identity.exceptions import CertificateError # noqa: F401
- except ImportError:
- raise unittest.SkipTest("cryptography lib is too old")
- self.tls_log_message = (
- 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
- 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
- )
- super(Https2InvalidDNSPattern, self).setUp()
-
-
class Https11CustomCiphers(unittest.TestCase):
scheme = 'https'
download_handler_cls = HTTP11DownloadHandler
@@ -686,11 +599,6 @@ class Https11CustomCiphers(unittest.TestCase):
return d
-class Https2CustomCiphers(Https11CustomCiphers):
- scheme = 'https'
- download_handler_cls = H2DownloadHandler
-
-
class Http11MockServerTestCase(unittest.TestCase):
"""HTTP 1.1 test case with MockServer"""
@@ -742,10 +650,6 @@ class Http11MockServerTestCase(unittest.TestCase):
self.assertTrue(reason, 'finished')
-class Http2MockServerTestCase(Http11MockServerTestCase):
- """HTTP 2.0 test case with MockServer"""
-
-
class UriResource(resource.Resource):
"""Return the full uri that was requested"""
@@ -837,47 +741,6 @@ class Http11ProxyTestCase(HttpProxyTestCase):
self.assertIn(domain, timeout.osError)
-class Https2ProxyTestCase(Http11ProxyTestCase):
- # only used for HTTPS tests
- keyfile = 'keys/localhost.key'
- certfile = 'keys/localhost.crt'
-
- scheme = 'https'
- host = u'127.0.0.1'
-
- download_handler_cls = H2DownloadHandler
- expected_http_proxy_request_body = b'/'
-
- def setUp(self):
- site = server.Site(UriResource(), timeout=None)
- self.port = reactor.listenSSL(
- 0, site,
- ssl_context_factory(self.keyfile, self.certfile),
- interface=self.host
- )
- self.portno = self.port.getHost().port
- self.download_handler = create_instance(self.download_handler_cls, None, get_crawler())
- self.download_request = self.download_handler.download_request
-
- def getURL(self, path):
- return f"{self.scheme}://{self.host}:{self.portno}/{path}"
-
- def test_download_with_proxy_https_noconnect(self):
- def _test(response):
- self.assertEqual(response.status, 200)
- self.assertEqual(response.url, request.url)
- self.assertEqual(response.body, b'/')
-
- http_proxy = '%s?noconnect' % self.getURL('')
- request = Request('https://example.com', meta={'proxy': http_proxy})
- with self.assertWarnsRegex(
- Warning,
- r'Using HTTPS proxies in the noconnect mode is not supported by the '
- r'downloader handler.'
- ):
- return self.download_request(request, Spider('foo')).addCallback(_test)
-
-
class HttpDownloadHandlerMock:
def __init__(self, *args, **kwargs):