Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Stop marking the Twisted-based HTTP/2 download handler as experimental, and privatize its API (#7986)

Adrian8bb06bf00b4a
diff --git a/conftest.py b/conftest.py
index 5a535c168..7a5f5b85a 100644
--- a/conftest.py
+++ b/conftest.py
@@ -47,7 +47,7 @@ if not H2_ENABLED:
collect_ignore.extend(
(
"scrapy/core/downloader/handlers/http2.py",
- *_py_files("scrapy/core/http2"),
+ *_py_files("scrapy/core/_http2"),
)
)
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index 433a6d139..d95e7b52b 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -187,12 +187,6 @@ If you want to use this handler you need to replace the default one for the
Features and limitations
^^^^^^^^^^^^^^^^^^^^^^^^
-.. warning::
-
- This handler is experimental, and not yet recommended for production
- environments. Future Scrapy versions may introduce related changes without
- a deprecation period or warning.
-
=========================== ================================================
HTTP proxies No (not implemented)
SOCKS proxies No (not supported by the library)
diff --git a/scrapy/core/http2/__init__.py b/scrapy/core/_http2/__init__.py
similarity index 100%
rename from scrapy/core/http2/__init__.py
rename to scrapy/core/_http2/__init__.py
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/_http2/agent.py
similarity index 98%
rename from scrapy/core/http2/agent.py
rename to scrapy/core/_http2/agent.py
index 042557208..38be86b22 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/_http2/agent.py
@@ -14,8 +14,8 @@ from twisted.web.client import (
)
from twisted.web.error import SchemeNotSupported
+from scrapy.core._http2.protocol import H2ClientFactory, H2ClientProtocol
from scrapy.core.downloader.contextfactory import _AcceptableProtocolsContextFactory
-from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
if TYPE_CHECKING:
from twisted.internet.base import ReactorBase
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/_http2/protocol.py
similarity index 99%
rename from scrapy/core/http2/protocol.py
rename to scrapy/core/_http2/protocol.py
index 2d59aba31..aff985d94 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/_http2/protocol.py
@@ -31,7 +31,7 @@ from twisted.internet.ssl import Certificate
from twisted.protocols.policies import TimeoutMixin
from zope.interface import implementer
-from scrapy.core.http2.stream import Stream, StreamCloseReason
+from scrapy.core._http2.stream import Stream, StreamCloseReason
from scrapy.exceptions import DownloadTimeoutError
from scrapy.http import Request, Response
from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/_http2/stream.py
similarity index 99%
rename from scrapy/core/http2/stream.py
rename to scrapy/core/_http2/stream.py
index 4fc300d90..4a07d198b 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/_http2/stream.py
@@ -27,7 +27,7 @@ from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
from collections.abc import Sequence
- from scrapy.core.http2.protocol import H2ClientProtocol
+ from scrapy.core._http2.protocol import H2ClientProtocol
from scrapy.crawler import Crawler
from scrapy.http import Request, Response
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 9b3d4fbd4..935a2ae4c 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -4,9 +4,9 @@ from time import monotonic
from typing import TYPE_CHECKING
from urllib.parse import urldefrag
+from scrapy.core._http2.agent import H2Agent, H2ConnectionPool
from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
-from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
from scrapy.exceptions import (
DownloadTimeoutError,
NotConfigured,
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 2c3954b5e..95e1ff633 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -143,7 +143,7 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
response = await download_handler.download_request(request)
assert response.text == actual_content_length
assert (
- "scrapy.core.http2.stream",
+ "scrapy.core._http2.stream",
logging.WARNING,
f"Ignoring bad Content-Length header "
f"{bad_content_length!r} of request {request}, sending "
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 431b7a458..161015231 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -36,7 +36,7 @@ from tests.mockserver.utils import ssl_context_factory
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, Coroutine, Generator
- from scrapy.core.http2.protocol import H2ClientProtocol
+ from scrapy.core._http2.protocol import H2ClientProtocol
pytestmark = [
@@ -242,7 +242,7 @@ class TestHttps2ClientProtocol:
) -> AsyncGenerator[H2ClientProtocol]:
from twisted.internet import reactor
- from scrapy.core.http2.protocol import H2ClientFactory # noqa: PLC0415
+ from scrapy.core._http2.protocol import H2ClientFactory # noqa: PLC0415
client_options = optionsForClientTLS(
hostname=self.host,
@@ -460,7 +460,7 @@ class TestHttps2ClientProtocol:
def test_invalid_negotiated_protocol(
self, server_port: int, client: H2ClientProtocol
) -> Generator[Deferred[Any], Any, None]:
- with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", new=b"not-h2"):
+ with mock.patch("scrapy.core._http2.protocol.PROTOCOL_NAME", new=b"not-h2"):
request = Request(url=self.get_url(server_port, "/status?n=200"))
with pytest.raises(ResponseFailed):
yield make_request_dfd(client, request)
@@ -527,7 +527,7 @@ class TestHttps2ClientProtocol:
expected_body: bytes,
caplog: pytest.LogCaptureFixture,
) -> None:
- with caplog.at_level("WARNING", "scrapy.core.http2.stream"):
+ with caplog.at_level("WARNING", "scrapy.core._http2.stream"):
response = await make_request(client, request)
assert response.status == 200
assert response.body == expected_body
@@ -605,7 +605,7 @@ class TestHttps2ClientProtocol:
def assert_inactive_stream(failure):
assert failure.check(ResponseFailed) is not None
- from scrapy.core.http2.stream import InactiveStreamClosed # noqa: PLC0415
+ from scrapy.core._http2.stream import InactiveStreamClosed # noqa: PLC0415
assert any(
isinstance(e, InactiveStreamClosed) for e in failure.value.reasons
@@ -692,7 +692,7 @@ class TestHttps2ClientProtocol:
@staticmethod
async def _check_invalid_netloc(client: H2ClientProtocol, url: str) -> None:
- from scrapy.core.http2.stream import InvalidHostname # noqa: PLC0415
+ from scrapy.core._http2.stream import InvalidHostname # noqa: PLC0415
request = Request(url)
with pytest.raises(InvalidHostname) as exc_info:
@@ -737,7 +737,7 @@ class TestHttps2ClientProtocol:
yield make_request_dfd(client, request)
for err in exc_info.value.reasons:
- from scrapy.core.http2.protocol import H2ClientProtocol # noqa: PLC0415
+ from scrapy.core._http2.protocol import H2ClientProtocol # noqa: PLC0415
if isinstance(err, DownloadTimeoutError):
assert (