Scrapy, a fast high-level web crawling & scraping framework for Python.
Make Twisted[http2] installation optional (#5113)
Adrián Chavesc5b1ee810167
diff --git a/conftest.py b/conftest.pyindex e4dd80de0..4931c5a79 100644--- a/conftest.py+++ b/conftest.py@@ -1,6 +1,7 @@from pathlib import Pathimport pytest+from twisted.web.http import H2_ENABLEDfrom scrapy.utils.reactor import install_reactor@@ -25,6 +26,14 @@ for line in open('tests/ignores.txt'):if file_path and file_path[0] != '#':collect_ignore.append(file_path)+if not H2_ENABLED:+ collect_ignore.extend(+ (+ 'scrapy/core/downloader/handlers/http2.py',+ *_py_files("scrapy/core/http2"),+ )+ )+@pytest.fixture()def chdir(tmpdir):diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 2506497e2..0b290598f 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -680,12 +680,16 @@ handler (without replacement), place this in your ``settings.py``::.. _http2:-The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update-:setting:`DOWNLOAD_HANDLERS` as follows::+The default HTTPS handler uses HTTP/1.1. To use HTTP/2:- DOWNLOAD_HANDLERS = {- 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',- }+#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to+ enable HTTP/2 support in Twisted.++#. Update :setting:`DOWNLOAD_HANDLERS` as follows::++ DOWNLOAD_HANDLERS = {+ 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',+ }.. warning::diff --git a/setup.py b/setup.pyindex b1bb64575..ed2b6e347 100644--- a/setup.py+++ b/setup.py@@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support():install_requires = [- 'Twisted[http2]>=17.9.0',+ 'Twisted>=17.9.0','cryptography>=2.0','cssselect>=0.9.1','itemloaders>=1.0.1',@@ -31,7 +31,6 @@ install_requires = ['zope.interface>=4.1.3','protego>=0.1.15','itemadapter>=0.1.0',- 'h2>=3.0,<4.0','setuptools',]extras_require = {}diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.pyindex 439778014..53bb4fe92 100644--- a/tests/test_downloader_handlers_http2.py+++ b/tests/test_downloader_handlers_http2.py@@ -1,5 +1,5 @@import json-from unittest import mock+from unittest import mock, skipIffrom pytest import markfrom testfixtures import LogCapture@@ -7,8 +7,8 @@ from twisted.internet import defer, error, reactorfrom twisted.trial import unittestfrom twisted.web import serverfrom twisted.web.error import SchemeNotSupported+from twisted.web.http import H2_ENABLED-from scrapy.core.downloader.handlers.http2 import H2DownloadHandlerfrom scrapy.http import Requestfrom scrapy.spiders import Spiderfrom scrapy.utils.misc import create_instance@@ -21,11 +21,17 @@ from tests.test_downloader_handlers import ()+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")class Https2TestCase(Https11TestCase):+scheme = 'https'- download_handler_cls = H2DownloadHandlerHTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"+ @classmethod+ def setUpClass(cls):+ from scrapy.core.downloader.handlers.http2 import H2DownloadHandler+ cls.download_handler_cls = H2DownloadHandler+def test_protocol(self):request = Request(self.getURL("host"), method="GET")d = self.download_request(request, Spider("foo"))@@ -187,9 +193,14 @@ class Https2InvalidDNSPattern(Https2TestCase):super(Https2InvalidDNSPattern, self).setUp()+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")class Https2CustomCiphers(Https11CustomCiphers):scheme = 'https'- download_handler_cls = H2DownloadHandler++ @classmethod+ def setUpClass(cls):+ from scrapy.core.downloader.handlers.http2 import H2DownloadHandler+ cls.download_handler_cls = H2DownloadHandlerclass Http2MockServerTestCase(Http11MockServerTestCase):@@ -201,6 +212,7 @@ class Http2MockServerTestCase(Http11MockServerTestCase):}+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")class Https2ProxyTestCase(Http11ProxyTestCase):# only used for HTTPS testskeyfile = 'keys/localhost.key'@@ -209,9 +221,13 @@ class Https2ProxyTestCase(Http11ProxyTestCase):scheme = 'https'host = u'127.0.0.1'- download_handler_cls = H2DownloadHandlerexpected_http_proxy_request_body = b'/'+ @classmethod+ def setUpClass(cls):+ from scrapy.core.downloader.handlers.http2 import H2DownloadHandler+ cls.download_handler_cls = H2DownloadHandler+def setUp(self):site = server.Site(UriResource(), timeout=None)self.port = reactor.listenSSL(diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 8b2f6a11d..677ede92b 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -5,10 +5,9 @@ import reimport shutilimport stringfrom ipaddress import IPv4Address-from unittest import mock+from unittest import mock, skipIffrom urllib.parse import urlencode-from h2.exceptions import InvalidBodyLengthErrorfrom twisted.internet import reactorfrom twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacksfrom twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint@@ -17,12 +16,10 @@ from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certiffrom twisted.python.failure import Failurefrom twisted.trial.unittest import TestCasefrom twisted.web.client import ResponseFailed, URI-from twisted.web.http import Request as TxRequest+from twisted.web.http import H2_ENABLED, Request as TxRequestfrom twisted.web.server import Site, NOT_DONE_YETfrom twisted.web.static import File-from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol-from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostnamefrom scrapy.http import Request, Response, JsonRequestfrom scrapy.settings import Settingsfrom scrapy.spiders import Spider@@ -173,6 +170,7 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:return PrivateCertificate.loadPEM(pem)+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")class Https2ClientProtocolTestCase(TestCase):scheme = 'https'key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key')@@ -220,6 +218,7 @@ class Https2ClientProtocolTestCase(TestCase):uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8'))self.conn_closed_deferred = Deferred()+ from scrapy.core.http2.protocol import H2ClientFactoryh2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)self.client = yield client_endpoint.connect(h2_client_factory)@@ -426,6 +425,7 @@ class Https2ClientProtocolTestCase(TestCase):def assert_failure(failure: Failure):self.assertTrue(len(failure.value.reasons) > 0)+ from h2.exceptions import InvalidBodyLengthErrorself.assertTrue(any(isinstance(error, InvalidBodyLengthError)for error in failure.value.reasons@@ -511,6 +511,7 @@ class Https2ClientProtocolTestCase(TestCase):def assert_inactive_stream(failure):self.assertIsNotNone(failure.check(ResponseFailed))+ from scrapy.core.http2.stream import InactiveStreamClosedself.assertTrue(any(isinstance(e, InactiveStreamClosed)for e in failure.value.reasons@@ -596,6 +597,7 @@ class Https2ClientProtocolTestCase(TestCase):request = Request(url)def assert_invalid_hostname(failure: Failure):+ from scrapy.core.http2.stream import InvalidHostnameself.assertIsNotNone(failure.check(InvalidHostname))error_msg = str(failure.value)self.assertIn('localhost', error_msg)@@ -633,6 +635,7 @@ class Https2ClientProtocolTestCase(TestCase):def assert_timeout_error(failure: Failure):for err in failure.value.reasons:+ from scrapy.core.http2.protocol import H2ClientProtocolif isinstance(err, TimeoutError):self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err))breakdiff --git a/tox.ini b/tox.iniindex 5b0606f8f..8167aff96 100644--- a/tox.ini+++ b/tox.ini@@ -50,6 +50,8 @@ commands =basepython = python3deps ={[testenv]deps}+ # Twisted[http2] is required to import some files+ Twisted[http2]>=17.9.0pytest-flake8commands =py.test --flake8 {posargs:docs scrapy tests}@@ -57,12 +59,7 @@ commands =[testenv:pylint]basepython = python3deps =- {[testenv]deps}- # Optional dependencies- boto- reppy- robotexclusionrulesparser- # Test dependencies+ {[testenv:extra-deps]deps}pylintcommands =pylint conftest.py docs extras scrapy setup.py tests@@ -119,9 +116,11 @@ setenv =[testenv:extra-deps]deps ={[testenv]deps}+ botoreppyrobotexclusionrulesparserPillow>=4.0.0+ Twisted[http2]>=17.9.0[testenv:asyncio]commands =