Scrapy, a fast high-level web crawling & scraping framework for Python.
Stop marking the Twisted-based HTTP/2 download handler as experimental, and privatize its API (#7986)
Adrian8bb06bf00b4a
diff --git a/conftest.py b/conftest.pyindex 5a535c168..7a5f5b85a 100644--- a/conftest.py+++ b/conftest.py@@ -47,7 +47,7 @@ if not H2_ENABLED:collect_ignore.extend(("scrapy/core/downloader/handlers/http2.py",- *_py_files("scrapy/core/http2"),+ *_py_files("scrapy/core/_http2"),))diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rstindex 433a6d139..d95e7b52b 100644--- a/docs/topics/download-handlers.rst+++ b/docs/topics/download-handlers.rst@@ -187,12 +187,6 @@ If you want to use this handler you need to replace the default one for theFeatures and limitations^^^^^^^^^^^^^^^^^^^^^^^^-.. warning::-- This handler is experimental, and not yet recommended for production- environments. Future Scrapy versions may introduce related changes without- a deprecation period or warning.-=========================== ================================================HTTP proxies No (not implemented)SOCKS proxies No (not supported by the library)diff --git a/scrapy/core/http2/__init__.py b/scrapy/core/_http2/__init__.pysimilarity index 100%rename from scrapy/core/http2/__init__.pyrename to scrapy/core/_http2/__init__.pydiff --git a/scrapy/core/http2/agent.py b/scrapy/core/_http2/agent.pysimilarity index 98%rename from scrapy/core/http2/agent.pyrename to scrapy/core/_http2/agent.pyindex 042557208..38be86b22 100644--- a/scrapy/core/http2/agent.py+++ b/scrapy/core/_http2/agent.py@@ -14,8 +14,8 @@ from twisted.web.client import ()from twisted.web.error import SchemeNotSupported+from scrapy.core._http2.protocol import H2ClientFactory, H2ClientProtocolfrom scrapy.core.downloader.contextfactory import _AcceptableProtocolsContextFactory-from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocolif TYPE_CHECKING:from twisted.internet.base import ReactorBasediff --git a/scrapy/core/http2/protocol.py b/scrapy/core/_http2/protocol.pysimilarity index 99%rename from scrapy/core/http2/protocol.pyrename to scrapy/core/_http2/protocol.pyindex 2d59aba31..aff985d94 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/_http2/protocol.py@@ -31,7 +31,7 @@ from twisted.internet.ssl import Certificatefrom twisted.protocols.policies import TimeoutMixinfrom zope.interface import implementer-from scrapy.core.http2.stream import Stream, StreamCloseReason+from scrapy.core._http2.stream import Stream, StreamCloseReasonfrom scrapy.exceptions import DownloadTimeoutErrorfrom scrapy.http import Request, Responsefrom scrapy.utils.deprecate import warn_on_deprecated_spider_attributediff --git a/scrapy/core/http2/stream.py b/scrapy/core/_http2/stream.pysimilarity index 99%rename from scrapy/core/http2/stream.pyrename to scrapy/core/_http2/stream.pyindex 4fc300d90..4a07d198b 100644--- a/scrapy/core/http2/stream.py+++ b/scrapy/core/_http2/stream.py@@ -27,7 +27,7 @@ from scrapy.utils.httpobj import urlparse_cachedif TYPE_CHECKING:from collections.abc import Sequence- from scrapy.core.http2.protocol import H2ClientProtocol+ from scrapy.core._http2.protocol import H2ClientProtocolfrom scrapy.crawler import Crawlerfrom scrapy.http import Request, Responsediff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex 9b3d4fbd4..935a2ae4c 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -4,9 +4,9 @@ from time import monotonicfrom typing import TYPE_CHECKINGfrom urllib.parse import urldefrag+from scrapy.core._http2.agent import H2Agent, H2ConnectionPoolfrom scrapy.core.downloader.contextfactory import _load_context_factory_from_settingsfrom scrapy.core.downloader.handlers.base import BaseDownloadHandler-from scrapy.core.http2.agent import H2Agent, H2ConnectionPoolfrom scrapy.exceptions import (DownloadTimeoutError,NotConfigured,diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 2c3954b5e..95e1ff633 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -143,7 +143,7 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):response = await download_handler.download_request(request)assert response.text == actual_content_lengthassert (- "scrapy.core.http2.stream",+ "scrapy.core._http2.stream",logging.WARNING,f"Ignoring bad Content-Length header "f"{bad_content_length!r} of request {request}, sending "diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 431b7a458..161015231 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -36,7 +36,7 @@ from tests.mockserver.utils import ssl_context_factoryif TYPE_CHECKING:from collections.abc import AsyncGenerator, Callable, Coroutine, Generator- from scrapy.core.http2.protocol import H2ClientProtocol+ from scrapy.core._http2.protocol import H2ClientProtocolpytestmark = [@@ -242,7 +242,7 @@ class TestHttps2ClientProtocol:) -> AsyncGenerator[H2ClientProtocol]:from twisted.internet import reactor- from scrapy.core.http2.protocol import H2ClientFactory # noqa: PLC0415+ from scrapy.core._http2.protocol import H2ClientFactory # noqa: PLC0415client_options = optionsForClientTLS(hostname=self.host,@@ -460,7 +460,7 @@ class TestHttps2ClientProtocol:def test_invalid_negotiated_protocol(self, server_port: int, client: H2ClientProtocol) -> Generator[Deferred[Any], Any, None]:- with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", new=b"not-h2"):+ with mock.patch("scrapy.core._http2.protocol.PROTOCOL_NAME", new=b"not-h2"):request = Request(url=self.get_url(server_port, "/status?n=200"))with pytest.raises(ResponseFailed):yield make_request_dfd(client, request)@@ -527,7 +527,7 @@ class TestHttps2ClientProtocol:expected_body: bytes,caplog: pytest.LogCaptureFixture,) -> None:- with caplog.at_level("WARNING", "scrapy.core.http2.stream"):+ with caplog.at_level("WARNING", "scrapy.core._http2.stream"):response = await make_request(client, request)assert response.status == 200assert response.body == expected_body@@ -605,7 +605,7 @@ class TestHttps2ClientProtocol:def assert_inactive_stream(failure):assert failure.check(ResponseFailed) is not None- from scrapy.core.http2.stream import InactiveStreamClosed # noqa: PLC0415+ from scrapy.core._http2.stream import InactiveStreamClosed # noqa: PLC0415assert any(isinstance(e, InactiveStreamClosed) for e in failure.value.reasons@@ -692,7 +692,7 @@ class TestHttps2ClientProtocol:@staticmethodasync def _check_invalid_netloc(client: H2ClientProtocol, url: str) -> None:- from scrapy.core.http2.stream import InvalidHostname # noqa: PLC0415+ from scrapy.core._http2.stream import InvalidHostname # noqa: PLC0415request = Request(url)with pytest.raises(InvalidHostname) as exc_info:@@ -737,7 +737,7 @@ class TestHttps2ClientProtocol:yield make_request_dfd(client, request)for err in exc_info.value.reasons:- from scrapy.core.http2.protocol import H2ClientProtocol # noqa: PLC0415+ from scrapy.core._http2.protocol import H2ClientProtocol # noqa: PLC0415if isinstance(err, DownloadTimeoutError):assert (