Scrapy, a fast high-level web crawling & scraping framework for Python.
test: H2DownloadHandler
Adityae8342996f627
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 51deb20f4..f6add82dc 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -23,8 +23,8 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandlerfrom scrapy.core.downloader.handlers.http import HTTPDownloadHandlerfrom scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandlerfrom scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler+from scrapy.core.downloader.handlers.http2 import H2DownloadHandlerfrom scrapy.core.downloader.handlers.s3 import S3DownloadHandler-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Headers, Requestfrom scrapy.http.response.text import TextResponse@@ -33,7 +33,6 @@ from scrapy.spiders import Spiderfrom scrapy.utils.misc import create_instancefrom scrapy.utils.python import to_bytesfrom scrapy.utils.test import get_crawler, skip_if_no_boto-from tests.mockserver import MockServer, ssl_context_factory, Echofrom tests.spiders import SingleRequestSpider@@ -131,6 +130,7 @@ class ContentLengthHeaderResource(resource.Resource):A testing resource which renders itself as the value of the Content-Lengthheader from the request."""+def render(self, request):return request.requestHeaders.getRawHeaders(b"content-length")[0]@@ -142,6 +142,7 @@ class ChunkedResource(resource.Resource):request.write(b"chunked ")request.write(b"content\n")request.finish()+reactor.callLater(0, response)return server.NOT_DONE_YET@@ -155,6 +156,7 @@ class BrokenChunkedResource(resource.Resource):# Disable terminating chunk on finish.request.chunked = FalsecloseConnection(request)+reactor.callLater(0, response)return server.NOT_DONE_YET@@ -186,6 +188,7 @@ class EmptyContentTypeHeaderResource(resource.Resource):A testing resource which renders itself as the value of request bodywithout content-type header in response."""+def render(self, request):request.setHeader("content-type", "")return request.content.read()@@ -197,12 +200,12 @@ class LargeChunkedFileResource(resource.Resource):for i in range(1024):request.write(b"x" * 1024)request.finish()+reactor.callLater(0, response)return server.NOT_DONE_YETclass HttpTestCase(unittest.TestCase):-scheme = 'http'download_handler_cls = HTTPDownloadHandler@@ -230,10 +233,12 @@ class HttpTestCase(unittest.TestCase):r.putChild(b"echo", Echo())self.site = server.Site(r, timeout=None)self.wrapper = WrappingFactory(self.site)- self.host = 'localhost'+ self.host = u'localhost'if self.scheme == 'https':+ # Using WrappingFactory do not enable HTTP/2 failing all the+ # tests with H2DownloadHandlerself.port = reactor.listenSSL(- 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile),+ 0, self.site, ssl_context_factory(self.keyfile, self.certfile),interface=self.host)else:self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)@@ -330,6 +335,7 @@ class HttpTestCase(unittest.TestCase):https://github.com/kennethreitz/requests/issues/405https://bugs.python.org/issue14721"""+def _test(response):self.assertEqual(response.body, b'0')@@ -514,6 +520,30 @@ class Https11TestCase(Http11TestCase):yield download_handler.close()+class Https2TestCase(Https11TestCase):+ scheme = 'https'+ download_handler_cls = H2DownloadHandler+ HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"++ def test_download_broken_content_cause_data_loss(self, url='broken'):+ raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)++ def test_download_broken_chunked_content_cause_data_loss(self):+ raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)++ def test_download_broken_content_allow_data_loss(self, url='broken'):+ raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)++ def test_download_broken_chunked_content_allow_data_loss(self):+ raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)++ def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):+ raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)++ def test_download_broken_chunked_content_allow_data_loss_via_setting(self):+ raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)++class Https11WrongHostnameTestCase(Http11TestCase):scheme = 'https'@@ -526,6 +556,23 @@ class Https11WrongHostnameTestCase(Http11TestCase):certfile = 'keys/example-com.cert.pem'+class Https2WrongHostnameTestCase(Https2TestCase):+ tls_log_message = (+ 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The '+ 'Internet/O=Scrapy/CN=www.example.com/[email protected]", '+ 'subject "/C=XW/ST=XW/L=The '+ 'Internet/O=Scrapy/CN=www.example.com/[email protected]"'+ )++ # above tests use a server certificate for "localhost",+ # client connection to "localhost" too.+ # here we test that even if the server certificate is for another domain,+ # "www.example.com" in this case,+ # the tests still pass+ keyfile = 'keys/example-com.key.pem'+ certfile = 'keys/example-com.cert.pem'++class Https11InvalidDNSId(Https11TestCase):"""Connect to HTTPS hosts with IP while certificate uses domain names IDs."""@@ -534,6 +581,14 @@ class Https11InvalidDNSId(Https11TestCase):self.host = '127.0.0.1'+class Https2InvalidDNSId(Https2TestCase):+ """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""++ def setUp(self):+ super(Https2InvalidDNSId, self).setUp()+ self.host = '127.0.0.1'++class Https11InvalidDNSPattern(Https11TestCase):"""Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""@@ -552,6 +607,24 @@ class Https11InvalidDNSPattern(Https11TestCase):super(Https11InvalidDNSPattern, self).setUp()+class Https2InvalidDNSPattern(Https2TestCase):+ """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""++ keyfile = 'keys/localhost.ip.key'+ certfile = 'keys/localhost.ip.crt'++ def setUp(self):+ try:+ from service_identity.exceptions import CertificateError # noqa: F401+ except ImportError:+ raise unittest.SkipTest("cryptography lib is too old")+ self.tls_log_message = (+ 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '+ 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'+ )+ super(Https2InvalidDNSPattern, self).setUp()++class Https11CustomCiphers(unittest.TestCase):scheme = 'https'download_handler_cls = HTTP11DownloadHandler@@ -565,10 +638,9 @@ class Https11CustomCiphers(unittest.TestCase):FilePath(self.tmpname).child("file").setContent(b"0123456789")r = static.File(self.tmpname)self.site = server.Site(r, timeout=None)- self.wrapper = WrappingFactory(self.site)self.host = 'localhost'self.port = reactor.listenSSL(- 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),+ 0, self.site, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),interface=self.host)self.portno = self.port.getHost().portcrawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})@@ -593,6 +665,11 @@ class Https11CustomCiphers(unittest.TestCase):return d+class Https2CustomCiphers(Https11CustomCiphers):+ scheme = 'https'+ download_handler_cls = H2DownloadHandler++class Http11MockServerTestCase(unittest.TestCase):"""HTTP 1.1 test case with MockServer"""@@ -644,6 +721,10 @@ class Http11MockServerTestCase(unittest.TestCase):self.assertTrue(reason, 'finished')+class Http2MockServerTestCase(Http11MockServerTestCase):+ """HTTP 2.0 test case with MockServer"""++class UriResource(resource.Resource):"""Return the full uri that was requested"""@@ -734,6 +815,11 @@ class Http11ProxyTestCase(HttpProxyTestCase):self.assertIn(domain, timeout.osError)+# TODO:+class Http2ProxyTestCase(Http11ProxyTestCase):+ download_handler_cls = H2DownloadHandler++class HttpDownloadHandlerMock:def __init__(self, *args, **kwargs):@@ -931,7 +1017,6 @@ class S3TestCase(unittest.TestCase):class BaseFTPTestCase(unittest.TestCase):-username = "scrapy"password = "passwd"req_meta = {"ftp_user": username, "ftp_password": password}@@ -969,6 +1054,7 @@ class BaseFTPTestCase(unittest.TestCase):def _clean(data):self.download_handler.client.transport.loseConnection()return data+deferred.addCallback(_clean)if callback:deferred.addCallback(callback)@@ -985,6 +1071,7 @@ class BaseFTPTestCase(unittest.TestCase):self.assertEqual(r.status, 200)self.assertEqual(r.body, b'I have the power!')self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']})+return self._add_test_callbacks(d, _test)def test_ftp_download_path_with_spaces(self):@@ -998,6 +1085,7 @@ class BaseFTPTestCase(unittest.TestCase):self.assertEqual(r.status, 200)self.assertEqual(r.body, b'Moooooooooo power!')self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']})+return self._add_test_callbacks(d, _test)def test_ftp_download_notexist(self):@@ -1007,6 +1095,7 @@ class BaseFTPTestCase(unittest.TestCase):def _test(r):self.assertEqual(r.status, 404)+return self._add_test_callbacks(d, _test)def test_ftp_local_filename(self):@@ -1027,6 +1116,7 @@ class BaseFTPTestCase(unittest.TestCase):with open(local_fname, "rb") as f:self.assertEqual(f.read(), b"I have the power!")os.remove(local_fname)+return self._add_test_callbacks(d, _test)@@ -1043,11 +1133,11 @@ class FTPTestCase(BaseFTPTestCase):def _test(r):self.assertEqual(r.type, ConnectionLost)+return self._add_test_callbacks(d, errback=_test)class AnonymousFTPTestCase(BaseFTPTestCase):-username = "anonymous"req_meta = {}