Scrapy, a fast high-level web crawling & scraping framework for Python.
Cleanup/clarifications of proxy support and improvements for proxy tests (#7496)
Andrey Rakhmatullin7fc84d372aee
diff --git a/conftest.py b/conftest.pyindex 4f8d32e1b..fcbf59426 100644--- a/conftest.py+++ b/conftest.py@@ -11,6 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policyfrom scrapy.utils.reactorless import install_reactor_import_hookfrom tests.keys import generate_keysfrom tests.mockserver.http import MockServer+from tests.mockserver.mitm_proxy import MitmProxyif TYPE_CHECKING:from collections.abc import Generator@@ -72,6 +73,32 @@ def mockserver() -> Generator[MockServer]:yield mockserver[email protected] # function scope because it modifies os.environ+def mitm_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:+ proxy = MitmProxy()+ url = proxy.start()+ monkeypatch.setenv("http_proxy", url)+ monkeypatch.setenv("https_proxy", url)++ try:+ yield proxy+ finally:+ proxy.stop()++[email protected] # function scope because it modifies os.environ+def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:+ proxy = MitmProxy()+ url = proxy.start().replace("http://", "https://")+ monkeypatch.setenv("http_proxy", url)+ monkeypatch.setenv("https_proxy", url)++ try:+ yield proxy+ finally:+ proxy.stop()++@pytest.fixture(scope="session")def reactor_pytest(request) -> str:return request.config.getoption("--reactor")diff --git a/docs/news.rst b/docs/news.rstindex 676e0cbbf..7f3952eaf 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -3,6 +3,17 @@Release notes=============+Scrapy VERSION (unreleased)+---------------------------++Backward-incompatible changes+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~++- ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` has been made+ private as it's an implementation detail of+ :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`.+ (:issue:`7496`)+.. _release-2.15.2:Scrapy 2.15.2 (2026-04-28)diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rstindex 96cf46c35..1e95864c6 100644--- a/docs/topics/download-handlers.rst+++ b/docs/topics/download-handlers.rst@@ -206,6 +206,8 @@ If you want to use this handler you need to replace the default one for theKnown limitations of the HTTP/2 implementation in this handler include:+ - No support for proxies.+- No support for HTTP/2 Cleartext (h2c), since no major browser supportsHTTP/2 unencrypted (refer `http2 faq`_).diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 75e48be41..873c32c13 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -745,8 +745,19 @@ HttpProxyMiddlewareHandling of this meta key needs to be implemented inside the :ref:`downloadhandler <topics-download-handlers>`, so it's not guaranteed to be supportedby all 3rd-party handlers. It's currently unsupported by+ :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` and:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.+.. note::++ Usually a proxy URL uses the ``http://`` scheme. More rarely, it uses the+ ``https://`` one. While both kinds of proxy URLs can be used with both HTTP+ and HTTPS destination URLs, the specifics of the network exchange are+ different for all 4 cases and it's possible that HTTPS proxies are fully or+ partially unsupported by a given download handler. Currently,+ :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`+ supports HTTPS proxies only for HTTP destinations.+HttpProxyMiddleware settings~~~~~~~~~~~~~~~~~~~~~~~~~~~~diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 975078803..1127c533f 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -338,17 +338,19 @@ class TunnelingAgent(Agent):)-class ScrapyProxyAgent(Agent):+class _ScrapyProxyAgent(Agent):def __init__(self,reactor: ReactorBase,proxyURI: bytes,+ contextFactory: IPolicyForHTTPS,connectTimeout: float | None = None,bindAddress: tuple[str, int] | None = None,pool: HTTPConnectionPool | None = None,):super().__init__( # type: ignore[no-untyped-call]reactor=reactor,+ contextFactory=contextFactory,connectTimeout=connectTimeout,bindAddress=bindAddress,pool=pool,@@ -380,7 +382,6 @@ class ScrapyProxyAgent(Agent):class ScrapyAgent:_Agent = Agent- _ProxyAgent = ScrapyProxyAgent_TunnelingAgent = TunnelingAgentdef __init__(@@ -421,6 +422,10 @@ class ScrapyAgent:if not proxy_port:proxy_port = 443 if proxy_parsed.scheme == "https" else 80if urlparse_cached(request).scheme == "https":+ if proxy_parsed.scheme == "https": # pragma: no cover+ raise NotImplementedError(+ "HTTPS proxies for HTTPS destinations are not supported"+ )assert proxy_host is not NoneproxyAuth = request.headers.get(b"Proxy-Authorization", None)proxyConf = (proxy_host, proxy_port, proxyAuth)@@ -432,9 +437,10 @@ class ScrapyAgent:bindAddress=bindaddress,pool=self._pool,)- return self._ProxyAgent(+ return _ScrapyProxyAgent(reactor=reactor,proxyURI=to_bytes(proxy, encoding="ascii"),+ contextFactory=self._contextFactory,connectTimeout=timeout,bindAddress=bindaddress,pool=self._pool,diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex a4f786363..b28bcfd89 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -4,19 +4,20 @@ from time import monotonicfrom typing import TYPE_CHECKINGfrom urllib.parse import urldefrag-from twisted.web.client import URI-from scrapy.core.downloader.contextfactory import _load_context_factory_from_settingsfrom scrapy.core.downloader.handlers.base import BaseDownloadHandler-from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent-from scrapy.exceptions import DownloadTimeoutError, NotConfigured+from scrapy.core.http2.agent import H2Agent, H2ConnectionPool+from scrapy.exceptions import (+ DownloadTimeoutError,+ NotConfigured,+ UnsupportedURLSchemeError,+)from scrapy.utils._download_handlers import (normalize_bind_address,wrap_twisted_exceptions,)from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.httpobj import urlparse_cached-from scrapy.utils.python import to_bytesif TYPE_CHECKING:from twisted.internet.base import DelayedCall@@ -44,6 +45,10 @@ class H2DownloadHandler(BaseDownloadHandler):self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")async def download_request(self, request: Request) -> Response:+ if urlparse_cached(request).scheme == "http": # pragma: no cover+ raise UnsupportedURLSchemeError(+ f"{type(self).__name__} doesn't support plain HTTP."+ )agent = ScrapyH2Agent(context_factory=self._context_factory,pool=self._pool,@@ -62,7 +67,6 @@ class H2DownloadHandler(BaseDownloadHandler):class ScrapyH2Agent:_Agent = H2Agent- _ProxyAgent = ScrapyProxyH2Agentdef __init__(self,@@ -81,24 +85,10 @@ class ScrapyH2Agent:def _get_agent(self, request: Request, timeout: float | None) -> H2Agent:from twisted.internet import reactor+ if request.meta.get("proxy"): # pragma: no cover+ raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.")bind_address = request.meta.get("bindaddress") or self._bind_addressbind_address = normalize_bind_address(bind_address)- proxy = request.meta.get("proxy")- if proxy:- if urlparse_cached(request).scheme == "https":- # ToDo- raise NotImplementedError(- "Tunneling via CONNECT method using HTTP/2.0 is not yet supported"- )- return self._ProxyAgent(- reactor=reactor,- context_factory=self._context_factory,- proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")),- connect_timeout=timeout,- bind_address=bind_address,- pool=self._pool,- )-return self._Agent(reactor=reactor,context_factory=self._context_factory,diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.pyindex 822dffc4b..7137a0f2b 100644--- a/scrapy/core/http2/agent.py+++ b/scrapy/core/http2/agent.py@@ -165,30 +165,3 @@ class H2Agent:lambda conn: conn.request(request, spider))return d2---class ScrapyProxyH2Agent(H2Agent):- def __init__(- self,- reactor: ReactorBase,- proxy_uri: URI,- pool: H2ConnectionPool,- context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008- connect_timeout: float | None = None,- bind_address: tuple[str, int] | None = None,- ) -> None:- super().__init__(- reactor=reactor,- pool=pool,- context_factory=context_factory,- connect_timeout=connect_timeout,- bind_address=bind_address,- )- self._proxy_uri = proxy_uri-- def get_endpoint(self, uri: URI) -> HostnameEndpoint:- return self.endpoint_factory.endpointForURI(self._proxy_uri) # type: ignore[no-any-return]-- def get_key(self, uri: URI) -> ConnectionKeyT:- """We use the proxy uri instead of uri obtained from request url"""- return b"http-proxy", self._proxy_uri.host, self._proxy_uri.portdiff --git a/tests/mockserver/mitm_proxy.py b/tests/mockserver/mitm_proxy.pynew file mode 100644index 000000000..1ec5a79b5--- /dev/null+++ b/tests/mockserver/mitm_proxy.py@@ -0,0 +1,64 @@+from __future__ import annotations++import re+import sys+from pathlib import Path+from subprocess import PIPE, Popen+from urllib.parse import urlsplit, urlunsplit+++class MitmProxy:+ auth_user = "scrapy"+ auth_pass = "scrapy"++ def start(self) -> str:+ script = """+import sys+from mitmproxy.tools.main import mitmdump+sys.argv[0] = "mitmdump"+sys.exit(mitmdump())+ """+ cert_path = Path(__file__).parent.parent.resolve() / "keys"+ args = [+ "--listen-host",+ "127.0.0.1",+ "--listen-port",+ "0",+ "--proxyauth",+ f"{self.auth_user}:{self.auth_pass}",+ "--set",+ f"confdir={cert_path}",+ "--ssl-insecure",+ "-s",+ str(Path(__file__).with_name("mitm_proxy_addon.py")),+ ]+ self.proc: Popen[str] = Popen(+ [+ sys.executable,+ "-u",+ "-c",+ script,+ *args,+ ],+ stdout=PIPE,+ text=True,+ )+ assert self.proc.stdout is not None+ line = ""+ for line in self.proc.stdout:+ m = re.search(r"listening at (?:http://)?([^:]+:\d+)", line)+ if m:+ host_port = m.group(1)+ return f"http://{self.auth_user}:{self.auth_pass}@{host_port}"+ self.stop()+ raise RuntimeError(f"Failed to parse mitmdump output: {line}")++ def stop(self) -> None:+ self.proc.kill()+ self.proc.communicate()+++def wrong_credentials(proxy_url: str) -> str:+ bad_auth_proxy = list(urlsplit(proxy_url))+ bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@")+ return urlunsplit(bad_auth_proxy)diff --git a/tests/mockserver/mitm_proxy_addon.py b/tests/mockserver/mitm_proxy_addon.pynew file mode 100644index 000000000..04ba2e4a1--- /dev/null+++ b/tests/mockserver/mitm_proxy_addon.py@@ -0,0 +1,5 @@+def response(flow) -> None:+ # add custom headers to be able to check that the request went through the proxy+ flow.response.headers["X-Via-Mitmproxy"] = "1"+ if flow.client_conn.tls_established:+ flow.response.headers["X-Via-Mitmproxy-TLS"] = "1"diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pyindex a0e3fccb3..d4dfe8396 100644--- a/tests/test_downloader_handler_httpx.py+++ b/tests/test_downloader_handler_httpx.py@@ -17,6 +17,7 @@ from tests.test_downloader_handlers_http_base import (TestHttpsInvalidDNSPatternBase,TestHttpsWrongHostnameBase,TestHttpWithCrawlerBase,+ TestMitmProxyBase,TestSimpleHttpsBase,)from tests.utils.decorators import coroutine_test@@ -41,6 +42,15 @@ class HttpxDownloadHandlerMixin:return HttpxDownloadHandler+ @property+ def settings_dict(self) -> dict[str, Any] | None:+ return {+ "DOWNLOAD_HANDLERS": {+ "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",+ "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",+ }+ }+class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):handler_supports_bindaddress_meta = False@@ -108,15 +118,8 @@ class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBapass-class TestHttpWithCrawler(TestHttpWithCrawlerBase):- @property- def settings_dict(self) -> dict[str, Any] | None:- return {- "DOWNLOAD_HANDLERS": {- "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",- "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",- }- }+class TestHttpWithCrawler(HttpxDownloadHandlerMixin, TestHttpWithCrawlerBase):+ passclass TestHttpsWithCrawler(TestHttpWithCrawler):@@ -136,3 +139,9 @@ class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):@pytest.mark.skip(reason="Proxy support is not implemented yet")class TestHttpsProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):is_secure = True++[email protected](reason="Proxy support is not implemented yet")[email protected]_mitmproxy+class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):+ passdiff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.pyindex 01fb26d85..89cf58c96 100644--- a/tests/test_downloader_handler_twisted_http11.py+++ b/tests/test_downloader_handler_twisted_http11.py@@ -19,6 +19,7 @@ from tests.test_downloader_handlers_http_base import (TestHttpsInvalidDNSPatternBase,TestHttpsWrongHostnameBase,TestHttpWithCrawlerBase,+ TestMitmProxyBase,TestSimpleHttpsBase,)@@ -34,6 +35,15 @@ class HTTP11DownloadHandlerMixin:def download_handler_cls(self) -> type[DownloadHandlerProtocol]:return HTTP11DownloadHandler+ @property+ def settings_dict(self) -> dict[str, Any] | None:+ return {+ "DOWNLOAD_HANDLERS": {+ "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",+ "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",+ }+ }+def test_not_configured_without_reactor() -> None:crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False})@@ -71,15 +81,8 @@ class TestHttpsCustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBpass-class TestHttpWithCrawler(TestHttpWithCrawlerBase):- @property- def settings_dict(self) -> dict[str, Any] | None:- return {- "DOWNLOAD_HANDLERS": {- "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",- "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",- }- }+class TestHttpWithCrawler(HTTP11DownloadHandlerMixin, TestHttpWithCrawlerBase):+ passclass TestHttpsWithCrawler(TestHttpWithCrawler):@@ -88,3 +91,15 @@ class TestHttpsWithCrawler(TestHttpWithCrawler):class TestHttpProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):pass+++class TestHttpsProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):+ is_secure = True+ # not implemented+ handler_supports_tls_in_tls = False++[email protected]_mitmproxy+class TestMitmProxy(HTTP11DownloadHandlerMixin, TestMitmProxyBase):+ # not implemented+ handler_supports_tls_in_tls = Falsediff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 3273a264e..60b276166 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -10,13 +10,8 @@ from twisted.web.http import H2_ENABLEDfrom scrapy import Spiderfrom scrapy.crawler import Crawler-from scrapy.exceptions import (- DownloadFailedError,- NotConfigured,- UnsupportedURLSchemeError,-)+from scrapy.exceptions import DownloadFailedError, NotConfiguredfrom scrapy.http import Request-from scrapy.utils.defer import maybe_deferred_to_futurefrom tests.test_downloader_handlers_http_base import (TestHttpProxyBase,TestHttpsBase,@@ -25,13 +20,13 @@ from tests.test_downloader_handlers_http_base import (TestHttpsInvalidDNSPatternBase,TestHttpsWrongHostnameBase,TestHttpWithCrawlerBase,+ TestMitmProxyBase,)from tests.utils.decorators import coroutine_testif TYPE_CHECKING:from scrapy.core.downloader.handlers import DownloadHandlerProtocolfrom tests.mockserver.http import MockServer- from tests.mockserver.proxy_echo import ProxyEchoMockServerpytestmark = [@@ -52,6 +47,15 @@ class H2DownloadHandlerMixin:return H2DownloadHandler+ @property+ def settings_dict(self) -> dict[str, Any] | None:+ return {+ "DOWNLOAD_HANDLERS": {+ "http": None,+ "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",+ }+ }+def test_not_configured_without_reactor() -> None:from scrapy.core.downloader.handlers.http2 import H2DownloadHandler # noqa: PLC0415@@ -167,16 +171,7 @@ class TestHttp2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase)pass-class TestHttp2WithCrawler(TestHttpWithCrawlerBase):- @property- def settings_dict(self) -> dict[str, Any] | None:- return {- "DOWNLOAD_HANDLERS": {- "http": None,- "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",- }- }-+class TestHttp2WithCrawler(H2DownloadHandlerMixin, TestHttpWithCrawlerBase):is_secure = Truedef test_bytes_received_stop_download_callback(self) -> None: # type: ignore[override]@@ -192,24 +187,12 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase):pytest.skip("headers_received support is not implemented")[email protected](reason="Proxy support is not implemented yet")class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):is_secure = True- expected_http_proxy_request_body = b"/"- @coroutine_test- async def test_download_with_proxy_https_timeout(- self, proxy_mockserver: ProxyEchoMockServer- ) -> None:- with pytest.raises(NotImplementedError):- await maybe_deferred_to_future(- super().test_download_with_proxy_https_timeout(proxy_mockserver) # type: ignore[arg-type]- )- @coroutine_test- async def test_download_with_proxy_without_http_scheme(- self, proxy_mockserver: ProxyEchoMockServer- ) -> None:- with pytest.raises(UnsupportedURLSchemeError):- await maybe_deferred_to_future(- super().test_download_with_proxy_without_http_scheme(proxy_mockserver) # type: ignore[arg-type]- )[email protected](reason="Proxy support is not implemented yet")[email protected]_mitmproxy+class TestMitmProxy(H2DownloadHandlerMixin, TestMitmProxyBase):+ passdiff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex 2a781dace..138ddd944 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -4,6 +4,8 @@ from __future__ import annotationsimport gzipimport json+import logging+import osimport platformimport reimport sys@@ -36,6 +38,7 @@ from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests import NON_EXISTING_RESOLVABLE+from tests.mockserver.mitm_proxy import MitmProxy, wrong_credentialsfrom tests.mockserver.proxy_echo import ProxyEchoMockServerfrom tests.mockserver.simple_https import SimpleMockServerfrom tests.spiders import (@@ -43,6 +46,7 @@ from tests.spiders import (BytesReceivedErrbackSpider,HeadersReceivedCallbackSpider,HeadersReceivedErrbackSpider,+ SimpleSpider,SingleRequestSpider,)from tests.utils.decorators import coroutine_test@@ -1072,6 +1076,8 @@ class TestHttpWithCrawlerBase(ABC):class TestHttpProxyBase(ABC):is_secure = Falseexpected_http_proxy_request_body = b"http://example.com"+ # whether the handler supports HTTPS proxies with HTTPS destinations+ handler_supports_tls_in_tls: bool = True@property@abstractmethod@@ -1124,6 +1130,8 @@ class TestHttpProxyBase(ABC):) -> None:if NON_EXISTING_RESOLVABLE:pytest.skip("Non-existing hosts are resolvable")+ if self.is_secure and not self.handler_supports_tls_in_tls:+ pytest.skip("HTTPS proxies for HTTPS destinations are not supported")http_proxy = proxy_mockserver.url("", is_secure=self.is_secure)domain = "https://no-such-domain.nosuch"request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2})@@ -1143,3 +1151,118 @@ class TestHttpProxyBase(ABC):assert response.status == 200assert response.url == request.urlassert response.body == self.expected_http_proxy_request_body+++class TestMitmProxyBase(ABC):+ # whether the handler supports HTTPS proxies with HTTPS destinations+ handler_supports_tls_in_tls: bool = True++ @property+ @abstractmethod+ def settings_dict(self) -> dict[str, Any] | None:+ raise NotImplementedError++ @pytest.mark.parametrize(+ "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]+ )+ @coroutine_test+ async def test_http_proxy(+ self,+ caplog: pytest.LogCaptureFixture,+ mockserver: MockServer,+ mitm_proxy_server: MitmProxy,+ https_dest: bool,+ ) -> None:+ """HTTP proxy, HTTP or HTTPS destination."""+ crawler = get_crawler(SingleRequestSpider, self.settings_dict)+ with caplog.at_level(logging.DEBUG):+ await crawler.crawl_async(+ seed=mockserver.url("/status?n=200", is_secure=https_dest)+ )+ assert isinstance(crawler.spider, SingleRequestSpider)+ self._assert_got_response_code(200, caplog.text)+ self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)++ @pytest.mark.parametrize(+ "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]+ )+ @coroutine_test+ async def test_https_proxy(+ self,+ caplog: pytest.LogCaptureFixture,+ mockserver: MockServer,+ mitm_proxy_server_https: MitmProxy,+ https_dest: bool,+ ) -> None:+ """HTTPS proxy, HTTP or HTTPS destination."""+ if https_dest and not self.handler_supports_tls_in_tls:+ pytest.skip("HTTPS proxies for HTTPS destinations are not supported")+ crawler = get_crawler(SingleRequestSpider, self.settings_dict)+ with caplog.at_level(logging.DEBUG):+ await crawler.crawl_async(+ seed=mockserver.url("/status?n=200", is_secure=https_dest)+ )+ assert isinstance(crawler.spider, SingleRequestSpider)+ self._assert_got_response_code(200, caplog.text)+ self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)++ @pytest.mark.parametrize(+ "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]+ )+ @coroutine_test+ async def test_http_proxy_auth_error(+ self,+ caplog: pytest.LogCaptureFixture,+ monkeypatch: pytest.MonkeyPatch,+ mockserver: MockServer,+ mitm_proxy_server: MitmProxy,+ https_dest: bool,+ ) -> None:+ """HTTP proxy, HTTP or HTTPS destination, wrong proxy creds."""+ envvar = "https_proxy" if https_dest else "http_proxy"+ monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar]))+ crawler = get_crawler(SimpleSpider, self.settings_dict)+ with caplog.at_level(logging.DEBUG):+ await crawler.crawl_async(+ mockserver.url("/status?n=200", is_secure=https_dest)+ )+ # The proxy returns a 407 error code but it does not reach the client;+ # it just sees an exception.+ self._assert_got_auth_exception(caplog.text)++ @pytest.mark.parametrize(+ "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]+ )+ @coroutine_test+ async def test_dont_leak_proxy_authorization_header(+ self,+ caplog: pytest.LogCaptureFixture,+ mockserver: MockServer,+ mitm_proxy_server: MitmProxy,+ https_dest: bool,+ ) -> None:+ """HTTP proxy, HTTP or HTTPS destination. Check that the auth header+ is not sent to the destination."""+ request = Request(mockserver.url("/echo", is_secure=https_dest))+ crawler = get_crawler(SingleRequestSpider, self.settings_dict)+ with caplog.at_level(logging.DEBUG):+ await crawler.crawl_async(seed=request)+ assert isinstance(crawler.spider, SingleRequestSpider)+ self._assert_got_response_code(200, caplog.text)+ self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)+ echo = json.loads(crawler.spider.meta["responses"][0].text)+ assert "Proxy-Authorization" not in echo["headers"]++ @staticmethod+ def _assert_headers(headers: Headers, https_dest: bool) -> None:+ assert b"X-Via-Mitmproxy" in headers+ if https_dest:+ assert b"X-Via-Mitmproxy-TLS" in headers++ @staticmethod+ def _assert_got_response_code(code: int, log: str) -> None:+ assert str(log).count(f"Crawled ({code})") == 1++ @staticmethod+ def _assert_got_auth_exception(log: str) -> None:+ assert "Proxy Authentication Required" in log or "407" in logdiff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pydeleted file mode 100644index c89b51ad3..000000000--- a/tests/test_proxy_connect.py+++ /dev/null@@ -1,125 +0,0 @@-import json-import os-import re-import sys-from pathlib import Path-from subprocess import PIPE, Popen-from urllib.parse import urlsplit, urlunsplit--import pytest-from testfixtures import LogCapture--from scrapy.http import Request-from scrapy.utils.test import get_crawler-from tests.mockserver.http import MockServer-from tests.spiders import SimpleSpider, SingleRequestSpider-from tests.utils.decorators import inline_callbacks_test---class MitmProxy:- auth_user = "scrapy"- auth_pass = "scrapy"-- def start(self) -> str:- script = """-import sys-from mitmproxy.tools.main import mitmdump-sys.argv[0] = "mitmdump"-sys.exit(mitmdump())- """- cert_path = Path(__file__).parent.resolve() / "keys"- args = [- "--listen-host",- "127.0.0.1",- "--listen-port",- "0",- "--proxyauth",- f"{self.auth_user}:{self.auth_pass}",- "--set",- f"confdir={cert_path}",- "--ssl-insecure",- ]- self.proc = Popen(- [- sys.executable,- "-u",- "-c",- script,- *args,- ],- stdout=PIPE,- text=True,- )- assert self.proc.stdout is not None- line = self.proc.stdout.readline()- m = re.search(r"listening at (?:http://)?([^:]+:\d+)", line)- if not m:- raise RuntimeError(f"Failed to parse mitmdump output: {line}")- host_port = m.group(1)- return f"http://{self.auth_user}:{self.auth_pass}@{host_port}"-- def stop(self) -> None:- self.proc.kill()- self.proc.communicate()---def _wrong_credentials(proxy_url: str) -> str:- bad_auth_proxy = list(urlsplit(proxy_url))- bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@")- return urlunsplit(bad_auth_proxy)--[email protected]_mitmproxy-class TestProxyConnect:- @classmethod- def setup_class(cls):- cls.mockserver = MockServer()- cls.mockserver.__enter__()-- @classmethod- def teardown_class(cls):- cls.mockserver.__exit__(None, None, None)-- def setup_method(self):- self._oldenv = os.environ.copy()- self._proxy = MitmProxy()- proxy_url = self._proxy.start()- os.environ["https_proxy"] = proxy_url- os.environ["http_proxy"] = proxy_url-- def teardown_method(self):- self._proxy.stop()- os.environ = self._oldenv-- @inline_callbacks_test- def test_https_connect_tunnel(self):- crawler = get_crawler(SimpleSpider)- with LogCapture() as log:- yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))- self._assert_got_response_code(200, log)-- @inline_callbacks_test- def test_https_tunnel_auth_error(self):- os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"])- crawler = get_crawler(SimpleSpider)- with LogCapture() as log:- yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))- # The proxy returns a 407 error code but it does not reach the client;- # he just sees a TunnelError.- self._assert_got_tunnel_error(log)-- @inline_callbacks_test- def test_https_tunnel_without_leak_proxy_authorization_header(self):- request = Request(self.mockserver.url("/echo", is_secure=True))- crawler = get_crawler(SingleRequestSpider)- with LogCapture() as log:- yield crawler.crawl(seed=request)- self._assert_got_response_code(200, log)- echo = json.loads(crawler.spider.meta["responses"][0].text)- assert "Proxy-Authorization" not in echo["headers"]-- def _assert_got_response_code(self, code, log):- assert str(log).count(f"Crawled ({code})") == 1-- def _assert_got_tunnel_error(self, log):- assert "TunnelError" in str(log)