Scrapy, a fast high-level web crawling & scraping framework for Python.
Make more of the internal handler helpers private. (#7510)
Andrey Rakhmatullin2d007bc45084
diff --git a/docs/news.rst b/docs/news.rstindex 7f3952eaf..5eef10d84 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -9,10 +9,24 @@ Scrapy VERSION (unreleased)Backward-incompatible changes~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-- ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` has been made- private as it's an implementation detail of- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`.- (:issue:`7496`)+- The following classes and functions, intended for internal use by+ :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`+ and :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`, have+ been made private:++ - ``scrapy.core.downloader.handlers.http11.ScrapyAgent``++ - ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent``++ - ``scrapy.core.downloader.handlers.http11.TunnelingAgent``++ - ``scrapy.core.downloader.handlers.http11.TunnelingTCP4ClientEndpoint``++ - ``scrapy.core.downloader.handlers.http11.tunnel_request_data()``++ - ``scrapy.core.downloader.handlers.http2.ScrapyH2Agent``++ (:issue:`7496`, #TBD).. _release-2.15.2:diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex e6451cdd7..fbe24e52e 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -111,7 +111,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):"download_warnsize", "DOWNLOAD_WARNSIZE")- agent = ScrapyAgent(+ agent = _ScrapyAgent(contextFactory=self._contextFactory,bindAddress=self._bind_address,pool=self._pool,@@ -161,7 +161,7 @@ class TunnelError(Exception):"""An HTTP CONNECT tunnel could not be established by the proxy."""-class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):+class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):"""An endpoint that tunnels through proxies to allow HTTPS downloads. Toaccomplish that, this endpoint sends an HTTP CONNECT to the proxy.The HTTP CONNECT is always sent when using this endpoint, I think this could@@ -197,7 +197,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):def requestTunnel(self, protocol: Protocol) -> Protocol:"""Asks the proxy to open a tunnel."""assert protocol.transport- tunnelReq = tunnel_request_data(+ tunnelReq = _tunnel_request_data(self._tunneledHost, self._tunneledPort, self._proxyAuthHeader)protocol.transport.write(tunnelReq)@@ -221,7 +221,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):if b"\r\n\r\n" not in self._connectBuffer:returnself._protocol.dataReceived = self._protocolDataReceived # type: ignore[method-assign]- respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)+ respm = _TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)if respm and int(respm.group("status")) == 200:# set proper Server Name Indication extensionsslOptions = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc]@@ -258,18 +258,18 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):return self._tunnelReadyDeferred-def tunnel_request_data(+def _tunnel_request_data(host: str, port: int, proxy_auth_header: bytes | None = None) -> bytes:r"""Return binary content of a CONNECT request.>>> from scrapy.utils.python import to_unicode as s- >>> s(tunnel_request_data("example.com", 8080))+ >>> s(_tunnel_request_data("example.com", 8080))'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'- >>> s(tunnel_request_data("example.com", 8080, b"123"))+ >>> s(_tunnel_request_data("example.com", 8080, b"123"))'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n'- >>> s(tunnel_request_data(b"example.com", "8090"))+ >>> s(_tunnel_request_data(b"example.com", "8090"))'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'"""host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))@@ -281,7 +281,7 @@ def tunnel_request_data(return tunnel_req-class TunnelingAgent(Agent):+class _TunnelingAgent(Agent):"""An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPSdownloads. It may look strange that we have chosen to subclass Agent and notProxyAgent but consider that after the tunnel is opened the proxy is@@ -303,8 +303,8 @@ class TunnelingAgent(Agent):self._proxyConf: tuple[str, int, bytes | None] = proxyConfself._contextFactory: IPolicyForHTTPS = contextFactory- def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint:- return TunnelingTCP4ClientEndpoint(+ def _getEndpoint(self, uri: URI) -> _TunnelingTCP4ClientEndpoint:+ return _TunnelingTCP4ClientEndpoint(reactor=self._reactor,host=uri.host,port=uri.port,@@ -381,10 +381,7 @@ class _ScrapyProxyAgent(Agent):)-class ScrapyAgent:- _Agent = Agent- _TunnelingAgent = TunnelingAgent-+class _ScrapyAgent:def __init__(self,*,@@ -430,7 +427,7 @@ class ScrapyAgent:assert proxy_host is not NoneproxyAuth = request.headers.get(b"Proxy-Authorization", None)proxyConf = (proxy_host, proxy_port, proxyAuth)- return self._TunnelingAgent(+ return _TunnelingAgent(reactor=reactor,proxyConf=proxyConf,contextFactory=self._contextFactory,@@ -447,7 +444,7 @@ class ScrapyAgent:pool=self._pool,)- return self._Agent( # type: ignore[no-untyped-call]+ return Agent(reactor=reactor,contextFactory=self._contextFactory,connectTimeout=timeout,@@ -465,7 +462,7 @@ class ScrapyAgent:url = urldefrag(request.url)[0]method = to_bytes(request.method)headers = TxHeaders(request.headers)- if isinstance(agent, self._TunnelingAgent):+ if isinstance(agent, _TunnelingAgent):headers.removeHeader(b"Proxy-Authorization")bodyproducer = _RequestBodyProducer(request.body) if request.body else Nonestart_time = monotonic()diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex b28bcfd89..f60c58d1b 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -49,7 +49,7 @@ class H2DownloadHandler(BaseDownloadHandler):raise UnsupportedURLSchemeError(f"{type(self).__name__} doesn't support plain HTTP.")- agent = ScrapyH2Agent(+ agent = _ScrapyH2Agent(context_factory=self._context_factory,pool=self._pool,bind_address=self._bind_address,@@ -65,9 +65,7 @@ class H2DownloadHandler(BaseDownloadHandler):self._pool.close_connections()-class ScrapyH2Agent:- _Agent = H2Agent-+class _ScrapyH2Agent:def __init__(self,context_factory: IPolicyForHTTPS,@@ -89,7 +87,7 @@ class ScrapyH2Agent:raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.")bind_address = request.meta.get("bindaddress") or self._bind_addressbind_address = normalize_bind_address(bind_address)- return self._Agent(+ return H2Agent(reactor=reactor,context_factory=self._context_factory,connect_timeout=timeout,