Scrapy, a fast high-level web crawling & scraping framework for Python.
Add support for HTTP/2 and for SOCKS proxies to HttpxDownloadHandler, improve handler docs (#7575)
Andrey Rakhmatullin4e956bd2de5e
diff --git a/conftest.py b/conftest.pyindex fcbf59426..1674086ec 100644--- a/conftest.py+++ b/conftest.py@@ -99,6 +99,19 @@ def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmPrproxy.stop()[email protected] # function scope because it modifies os.environ+def socks5_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:+ proxy = MitmProxy(mode="socks5")+ url = proxy.start()+ monkeypatch.setenv("http_proxy", url)+ monkeypatch.setenv("https_proxy", url)++ try:+ yield proxy+ finally:+ proxy.stop()++@pytest.fixture(scope="session")def reactor_pytest(request) -> str:return request.config.getoption("--reactor")diff --git a/docs/faq.rst b/docs/faq.rstindex 87adbfa4b..df90122f5 100644--- a/docs/faq.rst+++ b/docs/faq.rst@@ -82,10 +82,18 @@ to steal from us!Does Scrapy work with HTTP proxies?------------------------------------Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP-Proxy downloader middleware. See+Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader+middleware. See:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`.+Does Scrapy work with SOCKS proxies?+------------------------------------++Yes, when using+:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See+:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the+handler documentation.+How can I scrape an item with attributes in different pages?------------------------------------------------------------@@ -360,7 +368,10 @@ method for this purpose. For example:Does Scrapy support IPv6 addresses?------------------------------------Yes, by setting :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.+Yes, but when using+:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or+:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to+set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.Note that by doing so, you lose the ability to set a specific timeout for DNS requests(the value of the :setting:`DNS_TIMEOUT` setting is ignored).diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rstindex 84711e5af..888bfaf08 100644--- a/docs/topics/download-handlers.rst+++ b/docs/topics/download-handlers.rst@@ -130,44 +130,39 @@ these exceptions... _download-handlers-ref:-Built-in download handlers reference-====================================--DataURIDownloadHandler-------------------------.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler--| Supported scheme: ``data``.-| Lazy: no.--This handler supports RFC 2397 ``data:content/type;base64,`` data URIs.--FileDownloadHandler----------------------.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler--| Supported scheme: ``file``.-| Lazy: no.--This handler supports ``file:///path`` local file URIs. It doesn't-support remote files.--FTPDownloadHandler---------------------.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler--| Supported scheme: ``ftp``.-| Lazy: no.--This handler supports ``ftp://host/path`` FTP URIs.--It's implemented using :mod:`twisted.protocols.ftp`.--.. note::- This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.+Built-in HTTP download handlers reference+=========================================++Scrapy ships several handlers for HTTP and HTTPS requests. While all of them+support basic features, they may differ in support of specific Scrapy features+and settings and HTTP protocol features. See the documentation of specific+handlers and specific settings for more information. Additionally, as the+underlying HTTP client implementations differ between handlers, the behavior of+specific websites may be different when doing the same Scrapy requests but+using different handlers.++Here is a comparison of some features of the built-in HTTP handlers, see the+individual handler docs for more differences:++================== ================= ===================== ====================+Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler+================== ================= ===================== ====================+Requires asyncio No No Yes+Requires a reactor Yes Yes No+HTTP/1.1 No Yes Yes+HTTP/2 Yes No Yes+TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl``+HTTP proxies No Yes Yes+SOCKS proxies No No Yes+================== ================= ===================== ====================++You can find additional HTTP download handlers in the+scrapy-download-handlers-incubator_ package. This package is made by the Scrapy+developers and contains experimental handlers that may be included in some+later Scrapy version but can already be used. Please refer to the documentation+of this package for more information.++.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator.. _twisted-http2-handler:@@ -177,7 +172,9 @@ H2DownloadHandler.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler| Supported scheme: ``https``.-| Lazy: yes.+| :ref:`Lazy <lazy-download-handlers>`: yes.+| :ref:`Requires asyncio support <using-asyncio>`: no.+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocolfor them.@@ -196,63 +193,96 @@ If you want to use this handler you need to replace the default one for the"https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",}+Features and limitations+^^^^^^^^^^^^^^^^^^^^^^^^+.. warning::This handler is experimental, and not yet recommended for productionenvironments. Future Scrapy versions may introduce related changes withouta deprecation period or warning.-.. note::+=========================== ================================================+HTTP proxies No (not implemented)+SOCKS proxies No (not supported by the library)+HTTP/2 Yes+``response.certificate`` :class:`twisted.internet.ssl.Certificate` object+Per-request ``bindaddress`` Yes+TLS implementation ``pyOpenSSL``/``cryptography``+=========================== ================================================++Other limitations:- Known limitations of the HTTP/2 implementation in this handler include:+- No support for HTTP/1.1.- - No support for proxies.+- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`+ to ``scrapy.resolver.CachingHostnameResolver``.- - No support for HTTP/2 Cleartext (h2c), since no major browser supports- HTTP/2 unencrypted (refer `http2 faq`_).+- No support for the :signal:`bytes_received` and :signal:`headers_received`+ signals.- - No setting to specify a maximum `frame size`_ larger than the default- value, 16384. Connections to servers that send a larger frame will- fail.+Known limitations of the HTTP/2 support:- - No support for `server pushes`_, which are ignored.+- No support for HTTP/2 Cleartext (h2c), since no major browser supports+ HTTP/2 unencrypted (refer `http2 faq`_).- - No support for the :signal:`bytes_received` and- :signal:`headers_received` signals.+- No setting to specify a maximum `frame size`_ larger than the default+ value, 16384. Connections to servers that send a larger frame will fail.++- No support for `server pushes`_, which are ignored... _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2-.. note::- This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.-HTTP11DownloadHandler---------------------.. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler| Supported schemes: ``http``, ``https``.-| Lazy: no.+| :ref:`Lazy <lazy-download-handlers>`: no.+| :ref:`Requires asyncio support <using-asyncio>`: no.+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.This handler supports ``http://host/path`` and ``https://host/path`` URLs anduses the HTTP/1.1 protocol for them.It's implemented using :mod:`twisted.web.client`.-.. note::- This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.+Features and limitations+^^^^^^^^^^^^^^^^^^^^^^^^++=========================== ================================================+HTTP proxies Yes+SOCKS proxies No (not supported by the library)+HTTP/2 No (implemented as a separate handler)+``response.certificate`` :class:`twisted.internet.ssl.Certificate` object+Per-request ``bindaddress`` Yes+TLS implementation ``pyOpenSSL``/``cryptography``+=========================== ================================================++Other limitations:++- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`+ to ``scrapy.resolver.CachingHostnameResolver``.++- HTTPS proxies to HTTPS destinations are not supported.HttpxDownloadHandler--------------------+.. versionadded:: 2.15.0+.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler| Supported schemes: ``http``, ``https``.-| Lazy: no.+| :ref:`Lazy <lazy-download-handlers>`: no.+| :ref:`Requires asyncio support <using-asyncio>`: yes.+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.This handler supports ``http://host/path`` and ``https://host/path`` URLs and-uses the HTTP/1.1 protocol for them.+uses the HTTP/1.1 or HTTP/2 protocol for them.It's implemented using the ``httpx`` library and needs it to be installed.@@ -266,28 +296,83 @@ If you want to use this handler you need to replace the default ones for the"https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",}+Features and limitations+^^^^^^^^^^^^^^^^^^^^^^^^+.. warning::This handler is experimental, and not yet recommended for productionenvironments. Future Scrapy versions may introduce related changes withouta deprecation period or warning or even remove it altogether.-.. note::+=========================== =======================================+HTTP proxies Yes+SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``)+HTTP/2 Yes (requires ``httpx[http2]``)+``response.certificate`` DER bytes+Per-request ``bindaddress`` No (not supported by the library)+TLS implementation Standard library ``ssl``+=========================== =======================================++Other limitations:++- The handler creates a separate connection pool for each proxy URL (due to+ limitations of ``httpx``) which may lead to higher resource usage when+ using proxy rotation.++.. setting:: HTTPX_HTTP2_ENABLED++HTTPX_HTTP2_ENABLED+^^^^^^^^^^^^^^^^^^^++Default: ``False``++Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra+needs to be installed if you want to enable this setting.++.. versionadded:: VERSION++Built-in non-HTTP download handlers reference+=============================================++DataURIDownloadHandler+----------------------++.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler++| Supported scheme: ``data``.+| :ref:`Lazy <lazy-download-handlers>`: no.+| :ref:`Requires asyncio support <using-asyncio>`: no.+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.++This handler supports RFC 2397 ``data:content/type;base64,`` data URIs.++FileDownloadHandler+-------------------- As this handler is based on a different HTTP client implementation compared- to :class:`~.HTTP11DownloadHandler`, it's expected that its behavior on- some websites may be different. Additionally, these are the Scrapy features- that are explicitly not supported when using it:+.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler- - Per-request bind address support (the :reqmeta:`bindaddress` meta key).- The global :setting:`DOWNLOAD_BIND_ADDRESS` setting is supported but the- port number, if specified, will be ignored.+| Supported scheme: ``file``.+| :ref:`Lazy <lazy-download-handlers>`: no.+| :ref:`Requires asyncio support <using-asyncio>`: no.+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.- - Settings specific to the Twisted networking or HTTP implementation, like- :setting:`DNS_RESOLVER`.+This handler supports ``file:///path`` local file URIs. It doesn't+support remote files.- - Using :ref:`non-asyncio reactors <disable-asyncio>` (``httpx`` requires- ``asyncio``).+FTPDownloadHandler+------------------++.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler++| Supported scheme: ``ftp``.+| :ref:`Lazy <lazy-download-handlers>`: no.+| :ref:`Requires asyncio support <using-asyncio>`: no.+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.++This handler supports ``ftp://host/path`` FTP URIs.++It's implemented using :mod:`twisted.protocols.ftp`.S3DownloadHandler-----------------@@ -295,7 +380,9 @@ S3DownloadHandler.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler| Supported scheme: ``s3``.-| Lazy: yes.+| :ref:`Lazy <lazy-download-handlers>`: yes.+| :ref:`Requires asyncio support <using-asyncio>`: no.+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.This handler supports ``s3://bucket/path`` S3 URIs.diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 91ee38b85..8cb29deff 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -745,8 +745,7 @@ HttpProxyMiddlewareHandling of this meta key needs to be implemented inside the :ref:`downloadhandler <topics-download-handlers>`, so it's not guaranteed to be supportedby all 3rd-party handlers. It's currently unsupported by- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` and- :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.+ :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`... note::@@ -758,6 +757,13 @@ HttpProxyMiddleware:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`supports HTTPS proxies only for HTTP destinations.+.. note::++ If the download handler supports it, you can use a SOCKS proxy URL (e.g.+ ``socks5://username:password@some_proxy_server:port``).+ :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`+ supports SOCKS proxies while other built-in handlers don't.+HttpProxyMiddleware settings~~~~~~~~~~~~~~~~~~~~~~~~~~~~diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.pyindex 43e1cd965..c960fc152 100644--- a/scrapy/core/downloader/handlers/_httpx.py+++ b/scrapy/core/downloader/handlers/_httpx.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport ipaddressimport sslfrom contextlib import asynccontextmanager+from socket import gaierrorfrom typing import TYPE_CHECKING, ClassVarfrom scrapy.exceptions import (@@ -17,6 +18,7 @@ from scrapy.exceptions import ()from scrapy.http import Headersfrom scrapy.utils._download_handlers import NullCookieJar+from scrapy.utils.python import _iter_exc_causesfrom scrapy.utils.ssl import (_log_sslobj_debug_info,_make_insecure_ssl_ctx,@@ -34,10 +36,35 @@ if TYPE_CHECKING:from scrapy.crawler import Crawler+HAS_SOCKS = HAS_HTTP2 = False+try:import httpxexcept ImportError:httpx = None # type: ignore[assignment]+else:+ # a small hack to avoid importing these optional extras unconditionally++ DOWNLOAD_FAILED_EXCEPTIONS: tuple[type[BaseException], ...] = (+ httpx.RequestError,+ httpx.InvalidURL,+ )++ try:+ import h2.exceptions++ HAS_HTTP2 = True+ DOWNLOAD_FAILED_EXCEPTIONS += (h2.exceptions.InvalidBodyLengthError,)+ except ImportError: # pragma: no cover+ pass++ try:+ import socksio.exceptions++ HAS_SOCKS = True+ DOWNLOAD_FAILED_EXCEPTIONS += (socksio.exceptions.ProtocolError,)+ except ImportError: # pragma: no cover+ passif TYPE_CHECKING:@@ -54,6 +81,11 @@ class HttpxDownloadHandler(_Base):self._verify_certificates: bool = crawler.settings.getbool("DOWNLOAD_VERIFY_CERTIFICATES")+ self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED")+ if self._enable_h2 and not HAS_HTTP2: # pragma: no cover+ raise NotConfigured(+ f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed."+ )self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)self._bind_host: str | None = self._get_bind_address_host()self._limits: httpx.Limits = httpx.Limits(@@ -90,6 +122,7 @@ class HttpxDownloadHandler(_Base):transport=httpx.AsyncHTTPTransport(verify=self._ssl_context,local_address=self._bind_host,+ http2=self._enable_h2,limits=self._limits,trust_env=False,proxy=proxy,@@ -113,7 +146,13 @@ class HttpxDownloadHandler(_Base):async def _make_request(self, request: Request, timeout: float) -> AsyncIterator[httpx.Response]:- client = self._get_client(self._extract_proxy_url_with_creds(request))+ proxy = self._extract_proxy_url_with_creds(request)+ if proxy and proxy.startswith("socks") and not HAS_SOCKS: # pragma: no cover+ raise ValueError(+ f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed."+ )+ client = self._get_client(proxy)+try:async with client.stream(request.method,@@ -130,18 +169,12 @@ class HttpxDownloadHandler(_Base):except httpx.UnsupportedProtocol as e:raise UnsupportedURLSchemeError(str(e)) from eexcept httpx.ConnectError as e:- error_message = str(e)- if (- "Name or service not known" in error_message- or "getaddrinfo failed" in error_message- or "nodename nor servname" in error_message- or "Temporary failure in name resolution" in error_message- ):- raise CannotResolveHostError(error_message) from e+ if any(isinstance(c, gaierror) for c in _iter_exc_causes(e)):+ raise CannotResolveHostError(str(e)) from eraise DownloadConnectionRefusedError(str(e)) from eexcept httpx.ProxyError as e:raise DownloadConnectionRefusedError(str(e)) from e- except (httpx.NetworkError, httpx.RemoteProtocolError) as e:+ except DOWNLOAD_FAILED_EXCEPTIONS as e:raise DownloadFailedError(str(e)) from e@staticmethoddiff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 909a5fd5c..7d5612026 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -109,6 +109,7 @@ __all__ = ["HTTPCACHE_STORAGE","HTTPPROXY_AUTH_ENCODING","HTTPPROXY_ENABLED",+ "HTTPX_HTTP2_ENABLED","IMAGES_STORE_GCS_ACL","IMAGES_STORE_S3_ACL","ITEM_PIPELINES",@@ -382,6 +383,8 @@ HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"HTTPPROXY_ENABLED = TrueHTTPPROXY_AUTH_ENCODING = "latin-1"+HTTPX_HTTP2_ENABLED = False+IMAGES_STORE_GCS_ACL = ""IMAGES_STORE_S3_ACL = "private"diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 2d40e8555..00cf818b7 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -359,3 +359,13 @@ def _looks_like_import_path(value: str) -> bool:if any(part == "" for part in parts):return Falsereturn all(part.isidentifier() for part in parts)+++def _iter_exc_causes(exc: BaseException) -> Iterable[BaseException]:+ """Iterate over the exception causes/contexts."""+ seen: set[int] = set()+ cur: BaseException | None = exc+ while cur is not None and id(cur) not in seen:+ seen.add(id(cur))+ yield cur+ cur = cur.__cause__ or cur.__context__diff --git a/tests/mockserver/mitm_proxy.py b/tests/mockserver/mitm_proxy.pyindex 1ec5a79b5..56620f84e 100644--- a/tests/mockserver/mitm_proxy.py+++ b/tests/mockserver/mitm_proxy.py@@ -11,6 +11,9 @@ class MitmProxy:auth_user = "scrapy"auth_pass = "scrapy"+ def __init__(self, mode: str | None = None) -> None:+ self.mode = mode+def start(self) -> str:script = """import sys@@ -32,6 +35,8 @@ sys.exit(mitmdump())"-s",str(Path(__file__).with_name("mitm_proxy_addon.py")),]+ if self.mode:+ args += ["--mode", self.mode]self.proc: Popen[str] = Popen([sys.executable,@@ -44,12 +49,13 @@ sys.exit(mitmdump())text=True,)assert self.proc.stdout is not None+ scheme = "socks5" if self.mode == "socks5" else "http"line = ""for line in self.proc.stdout:- m = re.search(r"listening at (?:http://)?([^:]+:\d+)", line)+ m = re.search(r"listening at (?:\w+://)?([^:]+:\d+)", line)if m:host_port = m.group(1)- return f"http://{self.auth_user}:{self.auth_pass}@{host_port}"+ return f"{scheme}://{self.auth_user}:{self.auth_pass}@{host_port}"self.stop()raise RuntimeError(f"Failed to parse mitmdump output: {line}")diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pyindex e7a2ac0ba..fdacad963 100644--- a/tests/test_downloader_handler_httpx.py+++ b/tests/test_downloader_handler_httpx.py@@ -3,11 +3,17 @@from __future__ import annotationsimport sys-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, ClassVarimport pytestfrom scrapy import Request+from scrapy.core.downloader.handlers._httpx import (+ HAS_HTTP2,+ HAS_SOCKS,+ HttpxDownloadHandler,+)+from scrapy.exceptions import DownloadFailedErrorfrom tests.test_downloader_handlers_http_base import (TestHttpBase,TestHttpProxyBase,@@ -37,10 +43,6 @@ pytest.importorskip("httpx")class HttpxDownloadHandlerMixin:@propertydef download_handler_cls(self) -> type[DownloadHandlerProtocol]:- from scrapy.core.downloader.handlers._httpx import ( # noqa: PLC0415- HttpxDownloadHandler,- )-return HttpxDownloadHandler@property@@ -83,6 +85,30 @@ class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):pass[email protected](not HAS_HTTP2, reason="No HTTP/2 support in HttpxDownloadHandler")+class TestHttp2(TestHttps):+ http2 = True+ handler_supports_http2_dataloss = False++ default_handler_settings: ClassVar[dict[str, Any]] = {+ "HTTPX_HTTP2_ENABLED": True,+ }++ @coroutine_test+ async def test_protocol(self, mockserver: MockServer) -> None:+ request = Request(mockserver.url("/host", is_secure=self.is_secure))+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert response.protocol == "HTTP/2"++ @coroutine_test+ async def test_data_loss_handling(self, mockserver: MockServer) -> None:+ request = Request(mockserver.url("/broken", is_secure=self.is_secure))+ async with self.get_dh() as download_handler:+ with pytest.raises(DownloadFailedError):+ await download_handler.download_request(request)++class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):pass@@ -127,7 +153,7 @@ class TestHttpsProxy(TestHttpProxy):@pytest.mark.requires_mitmproxyclass TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):- pass+ handler_supports_socks = HAS_SOCKS@pytest.mark.requires_internetdiff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex e67129abf..0e1ff07c9 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -39,7 +39,7 @@ from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests import NON_EXISTING_RESOLVABLE-from tests.mockserver.mitm_proxy import MitmProxy, wrong_credentials+from tests.mockserver.mitm_proxy import wrong_credentialsfrom tests.mockserver.proxy_echo import ProxyEchoMockServerfrom tests.mockserver.simple_https import SimpleMockServerfrom tests.spiders import (@@ -73,6 +73,7 @@ class TestHttpBase(ABC):handler_supports_http2_dataloss: bool = True# default headers added by the underlying library that cannot be suppressedalways_present_req_headers: ClassVar[frozenset[str]] = frozenset()+ default_handler_settings: ClassVar[dict[str, Any]] = {}@property@abstractmethod@@ -83,6 +84,10 @@ class TestHttpBase(ABC):async def get_dh(self, settings_dict: dict[str, Any] | None = None) -> AsyncGenerator[DownloadHandlerProtocol]:+ settings_dict = {+ **self.default_handler_settings,+ **(settings_dict or {}),+ }crawler = get_crawler(DefaultSpider, settings_dict)crawler.spider = crawler._create_spider()dh = build_from_crawler(self.download_handler_cls, crawler)@@ -339,9 +344,7 @@ class TestHttpBase(ABC):@coroutine_testasync def test_timeout_download_from_spider_server_hangs(- self,- mockserver: MockServer,- reactor_pytest: str,+ self, mockserver: MockServer, reactor_pytest: str) -> None:if reactor_pytest == "asyncio" and sys.platform == "win32":# https://twistedmatrix.com/trac/ticket/10279@@ -1317,6 +1320,7 @@ class TestHttpProxyBase(ABC):class TestMitmProxyBase(ABC):# whether the handler supports HTTPS proxies with HTTPS destinationshandler_supports_tls_in_tls: bool = True+ handler_supports_socks: bool = False@property@abstractmethod@@ -1326,13 +1330,10 @@ class TestMitmProxyBase(ABC):@pytest.mark.parametrize("https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"])+ @pytest.mark.usefixtures("mitm_proxy_server")@coroutine_testasync def test_http_proxy(- self,- caplog: pytest.LogCaptureFixture,- mockserver: MockServer,- mitm_proxy_server: MitmProxy,- https_dest: bool,+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool) -> None:"""HTTP proxy, HTTP or HTTPS destination."""crawler = get_crawler(SingleRequestSpider, self.settings_dict)@@ -1347,13 +1348,10 @@ class TestMitmProxyBase(ABC):@pytest.mark.parametrize("https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"])+ @pytest.mark.usefixtures("mitm_proxy_server_https")@coroutine_testasync def test_https_proxy(- self,- caplog: pytest.LogCaptureFixture,- mockserver: MockServer,- mitm_proxy_server_https: MitmProxy,- https_dest: bool,+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool) -> None:"""HTTPS proxy, HTTP or HTTPS destination."""if https_dest and not self.handler_supports_tls_in_tls:@@ -1370,13 +1368,13 @@ class TestMitmProxyBase(ABC):@pytest.mark.parametrize("https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"])+ @pytest.mark.usefixtures("mitm_proxy_server")@coroutine_testasync def test_http_proxy_auth_error(self,caplog: pytest.LogCaptureFixture,monkeypatch: pytest.MonkeyPatch,mockserver: MockServer,- mitm_proxy_server: MitmProxy,https_dest: bool,) -> None:"""HTTP proxy, HTTP or HTTPS destination, wrong proxy creds."""@@ -1394,13 +1392,10 @@ class TestMitmProxyBase(ABC):@pytest.mark.parametrize("https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"])+ @pytest.mark.usefixtures("mitm_proxy_server")@coroutine_testasync def test_dont_leak_proxy_authorization_header(- self,- caplog: pytest.LogCaptureFixture,- mockserver: MockServer,- mitm_proxy_server: MitmProxy,- https_dest: bool,+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool) -> None:"""HTTP proxy, HTTP or HTTPS destination. Check that the auth headeris not sent to the destination."""@@ -1414,6 +1409,49 @@ class TestMitmProxyBase(ABC):echo = json.loads(crawler.spider.meta["responses"][0].text)assert "Proxy-Authorization" not in echo["headers"]+ @pytest.mark.parametrize(+ "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]+ )+ @pytest.mark.usefixtures("socks5_proxy_server")+ @coroutine_test+ async def test_download_with_socks_proxy(+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool+ ) -> None:+ """SOCKS5 proxy, HTTP or HTTPS destination."""+ if not self.handler_supports_socks:+ pytest.skip("SOCKS proxies are not supported")+ crawler = get_crawler(SingleRequestSpider, self.settings_dict)+ with caplog.at_level(logging.DEBUG):+ await crawler.crawl_async(+ seed=mockserver.url("/status?n=200", is_secure=https_dest)+ )+ assert isinstance(crawler.spider, SingleRequestSpider)+ self._assert_got_response_code(200, caplog.text)+ self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)++ @pytest.mark.parametrize(+ "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]+ )+ @pytest.mark.usefixtures("socks5_proxy_server")+ @coroutine_test+ async def test_socks_proxy_auth_error(+ self,+ caplog: pytest.LogCaptureFixture,+ monkeypatch: pytest.MonkeyPatch,+ mockserver: MockServer,+ https_dest: bool,+ ) -> None:+ if not self.handler_supports_socks:+ pytest.skip("SOCKS proxies are not supported")+ envvar = "https_proxy" if https_dest else "http_proxy"+ monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar]))+ crawler = get_crawler(SimpleSpider, self.settings_dict)+ with caplog.at_level(logging.DEBUG):+ await crawler.crawl_async(+ mockserver.url("/status?n=200", is_secure=https_dest)+ )+ assert "DownloadConnectionRefusedError" in caplog.text+@staticmethoddef _assert_headers(headers: Headers, https_dest: bool) -> None:assert b"X-Via-Mitmproxy" in headersdiff --git a/tox.ini b/tox.iniindex cc916caea..602394761 100644--- a/tox.ini+++ b/tox.ini@@ -60,6 +60,7 @@ deps =ipython==8.39.0pyOpenSSL==26.2.0pytest==9.0.3+ socksio==1.0.0types-Pygments==2.20.0.20260508types-defusedxml==0.7.0.20260504types-lxml==2026.2.16@@ -148,7 +149,7 @@ deps =brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware testsbrotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware testsgoogle-cloud-storage- httpx+ httpx[http2,socks]ipythonrobotexclusionrulesparseruvloop; platform_system != "Windows" and implementation_name != "pypy"@@ -304,5 +305,6 @@ deps ={[testenv]deps}# mitmproxy does not support PyPymitmproxy; implementation_name != "pypy"+ httpx[http2,socks]commands =pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_mitmproxy