Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Make Twisted[http2] installation optional (#5113)

Adrián Chavesc5b1ee810167
diff --git a/conftest.py b/conftest.py
index e4dd80de0..4931c5a79 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,6 +1,7 @@
from pathlib import Path
import pytest
+from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import install_reactor
@@ -25,6 +26,14 @@ for line in open('tests/ignores.txt'):
if file_path and file_path[0] != '#':
collect_ignore.append(file_path)
+if not H2_ENABLED:
+ collect_ignore.extend(
+ (
+ 'scrapy/core/downloader/handlers/http2.py',
+ *_py_files("scrapy/core/http2"),
+ )
+ )
+
@pytest.fixture()
def chdir(tmpdir):
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 2506497e2..0b290598f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -680,12 +680,16 @@ handler (without replacement), place this in your ``settings.py``::
.. _http2:
-The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
-:setting:`DOWNLOAD_HANDLERS` as follows::
+The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
- DOWNLOAD_HANDLERS = {
- 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
- }
+#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to
+ enable HTTP/2 support in Twisted.
+
+#. Update :setting:`DOWNLOAD_HANDLERS` as follows::
+
+ DOWNLOAD_HANDLERS = {
+ 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
+ }
.. warning::
diff --git a/setup.py b/setup.py
index b1bb64575..ed2b6e347 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support():
install_requires = [
- 'Twisted[http2]>=17.9.0',
+ 'Twisted>=17.9.0',
'cryptography>=2.0',
'cssselect>=0.9.1',
'itemloaders>=1.0.1',
@@ -31,7 +31,6 @@ install_requires = [
'zope.interface>=4.1.3',
'protego>=0.1.15',
'itemadapter>=0.1.0',
- 'h2>=3.0,<4.0',
'setuptools',
]
extras_require = {}
diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index 439778014..53bb4fe92 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -1,5 +1,5 @@
import json
-from unittest import mock
+from unittest import mock, skipIf
from pytest import mark
from testfixtures import LogCapture
@@ -7,8 +7,8 @@ from twisted.internet import defer, error, reactor
from twisted.trial import unittest
from twisted.web import server
from twisted.web.error import SchemeNotSupported
+from twisted.web.http import H2_ENABLED
-from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import create_instance
@@ -21,11 +21,17 @@ from tests.test_downloader_handlers import (
)
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2TestCase(Https11TestCase):
+
scheme = 'https'
- download_handler_cls = H2DownloadHandler
HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
+ @classmethod
+ def setUpClass(cls):
+ from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+ cls.download_handler_cls = H2DownloadHandler
+
def test_protocol(self):
request = Request(self.getURL("host"), method="GET")
d = self.download_request(request, Spider("foo"))
@@ -187,9 +193,14 @@ class Https2InvalidDNSPattern(Https2TestCase):
super(Https2InvalidDNSPattern, self).setUp()
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2CustomCiphers(Https11CustomCiphers):
scheme = 'https'
- download_handler_cls = H2DownloadHandler
+
+ @classmethod
+ def setUpClass(cls):
+ from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+ cls.download_handler_cls = H2DownloadHandler
class Http2MockServerTestCase(Http11MockServerTestCase):
@@ -201,6 +212,7 @@ class Http2MockServerTestCase(Http11MockServerTestCase):
}
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2ProxyTestCase(Http11ProxyTestCase):
# only used for HTTPS tests
keyfile = 'keys/localhost.key'
@@ -209,9 +221,13 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
scheme = 'https'
host = u'127.0.0.1'
- download_handler_cls = H2DownloadHandler
expected_http_proxy_request_body = b'/'
+ @classmethod
+ def setUpClass(cls):
+ from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+ cls.download_handler_cls = H2DownloadHandler
+
def setUp(self):
site = server.Site(UriResource(), timeout=None)
self.port = reactor.listenSSL(
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 8b2f6a11d..677ede92b 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -5,10 +5,9 @@ import re
import shutil
import string
from ipaddress import IPv4Address
-from unittest import mock
+from unittest import mock, skipIf
from urllib.parse import urlencode
-from h2.exceptions import InvalidBodyLengthError
from twisted.internet import reactor
from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks
from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
@@ -17,12 +16,10 @@ from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certif
from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
from twisted.web.client import ResponseFailed, URI
-from twisted.web.http import Request as TxRequest
+from twisted.web.http import H2_ENABLED, Request as TxRequest
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.static import File
-from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
-from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
from scrapy.http import Request, Response, JsonRequest
from scrapy.settings import Settings
from scrapy.spiders import Spider
@@ -173,6 +170,7 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
return PrivateCertificate.loadPEM(pem)
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2ClientProtocolTestCase(TestCase):
scheme = 'https'
key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key')
@@ -220,6 +218,7 @@ class Https2ClientProtocolTestCase(TestCase):
uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8'))
self.conn_closed_deferred = Deferred()
+ from scrapy.core.http2.protocol import H2ClientFactory
h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)
client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
self.client = yield client_endpoint.connect(h2_client_factory)
@@ -426,6 +425,7 @@ class Https2ClientProtocolTestCase(TestCase):
def assert_failure(failure: Failure):
self.assertTrue(len(failure.value.reasons) > 0)
+ from h2.exceptions import InvalidBodyLengthError
self.assertTrue(any(
isinstance(error, InvalidBodyLengthError)
for error in failure.value.reasons
@@ -511,6 +511,7 @@ class Https2ClientProtocolTestCase(TestCase):
def assert_inactive_stream(failure):
self.assertIsNotNone(failure.check(ResponseFailed))
+ from scrapy.core.http2.stream import InactiveStreamClosed
self.assertTrue(any(
isinstance(e, InactiveStreamClosed)
for e in failure.value.reasons
@@ -596,6 +597,7 @@ class Https2ClientProtocolTestCase(TestCase):
request = Request(url)
def assert_invalid_hostname(failure: Failure):
+ from scrapy.core.http2.stream import InvalidHostname
self.assertIsNotNone(failure.check(InvalidHostname))
error_msg = str(failure.value)
self.assertIn('localhost', error_msg)
@@ -633,6 +635,7 @@ class Https2ClientProtocolTestCase(TestCase):
def assert_timeout_error(failure: Failure):
for err in failure.value.reasons:
+ from scrapy.core.http2.protocol import H2ClientProtocol
if isinstance(err, TimeoutError):
self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err))
break
diff --git a/tox.ini b/tox.ini
index 5b0606f8f..8167aff96 100644
--- a/tox.ini
+++ b/tox.ini
@@ -50,6 +50,8 @@ commands =
basepython = python3
deps =
{[testenv]deps}
+ # Twisted[http2] is required to import some files
+ Twisted[http2]>=17.9.0
pytest-flake8
commands =
py.test --flake8 {posargs:docs scrapy tests}
@@ -57,12 +59,7 @@ commands =
[testenv:pylint]
basepython = python3
deps =
- {[testenv]deps}
- # Optional dependencies
- boto
- reppy
- robotexclusionrulesparser
- # Test dependencies
+ {[testenv:extra-deps]deps}
pylint
commands =
pylint conftest.py docs extras scrapy setup.py tests
@@ -119,9 +116,11 @@ setenv =
[testenv:extra-deps]
deps =
{[testenv]deps}
+ boto
reppy
robotexclusionrulesparser
Pillow>=4.0.0
+ Twisted[http2]>=17.9.0
[testenv:asyncio]
commands =