Scrapy, a fast high-level web crawling & scraping framework for Python.
Add HttpxDownloader. (#7239)
Andrey Rakhmatullinfc30c47f38e8
diff --git a/conftest.py b/conftest.pyindex 50925a311..26e0434ba 100644--- a/conftest.py+++ b/conftest.py@@ -52,6 +52,11 @@ if not H2_ENABLED:))+try:+ import httpx # noqa: F401+except ImportError:+ collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")+def pytest_addoption(parser, pluginmanager):if pluginmanager.hasplugin("twisted"):diff --git a/docs/news.rst b/docs/news.rstindex bab2e5ff9..ff8e6fb08 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -3,6 +3,17 @@Release notes=============+Scrapy VERSION (unreleased)+---------------------------++New features+~~~~~~~~~~~~++- Added+ :meth:`Headers.to_tuple_list() <scrapy.http.headers.Headers.to_tuple_list>`+ that returns headers as a list of ``(key, value)`` tuples.+ (:issue:`7239`)+.. _release-2.14.1:Scrapy 2.14.1 (2026-01-12)@@ -2997,7 +3008,7 @@ Bug fixesthat does not match the asyncio event loop actually installed(:issue:`5529`).-- Fixed :meth:`Headers.getlist <scrapy.http.headers.Headers.getlist>`+- Fixed :meth:`Headers.getlist() <scrapy.http.headers.Headers.getlist>`returning only the last header (:issue:`5515`, :issue:`5526`).- Fixed :class:`LinkExtractordiff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.pynew file mode 100644index 000000000..1f5cf007e--- /dev/null+++ b/scrapy/core/downloader/handlers/_httpx.py@@ -0,0 +1,269 @@+"""``httpx``-based HTTP(S) download handler. Currently not recommended for production use."""++from __future__ import annotations++import ipaddress+import logging+import ssl+from http.cookiejar import Cookie, CookieJar+from io import BytesIO+from typing import TYPE_CHECKING, Any, NoReturn, TypedDict++import httpx++from scrapy import Request, signals+from scrapy.exceptions import (+ CannotResolveHostError,+ DownloadCancelledError,+ DownloadConnectionRefusedError,+ DownloadFailedError,+ DownloadTimeoutError,+ NotConfigured,+ ResponseDataLossError,+ UnsupportedURLSchemeError,+)+from scrapy.http import Headers, Response+from scrapy.utils._download_handlers import (+ BaseHttpDownloadHandler,+ check_stop_download,+ get_dataloss_msg,+ get_maxsize_msg,+ get_warnsize_msg,+ make_response,+)+from scrapy.utils.asyncio import is_asyncio_available+from scrapy.utils.ssl import _log_sslobj_debug_info, _make_ssl_context++if TYPE_CHECKING:+ from contextlib import AbstractAsyncContextManager+ from http.client import HTTPResponse+ from ipaddress import IPv4Address, IPv6Address+ from urllib.request import Request as ULRequest++ from httpcore import AsyncNetworkStream++ from scrapy.crawler import Crawler+++logger = logging.getLogger(__name__)+++class _BaseResponseArgs(TypedDict):+ status: int+ url: str+ headers: Headers+ ip_address: IPv4Address | IPv6Address+ protocol: str+++# workaround for (and from) https://github.com/encode/httpx/issues/2992+class _NullCookieJar(CookieJar): # pragma: no cover+ """A CookieJar that rejects all cookies."""++ def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None:+ pass++ def set_cookie(self, cookie: Cookie) -> None:+ pass+++class HttpxDownloadHandler(BaseHttpDownloadHandler):+ _DEFAULT_CONNECT_TIMEOUT = 10++ def __init__(self, crawler: Crawler):+ # we don't run extra-deps tests with the non-asyncio reactor+ if not is_asyncio_available(): # pragma: no cover+ raise NotConfigured(+ f"{type(self).__name__} requires the asyncio support. Make"+ f" sure that you have either enabled the asyncio Twisted"+ f" reactor in the TWISTED_REACTOR setting or disabled the"+ f" TWISTED_ENABLED setting. See the asyncio documentation"+ f" of Scrapy for more information."+ )+ super().__init__(crawler)+ logger.warning(+ "HttpxDownloadHandler is experimental and is not recommented for production use."+ )+ self._tls_verbose_logging: bool = self.crawler.settings.getbool(+ "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"+ )+ self._client = httpx.AsyncClient(+ verify=_make_ssl_context(crawler.settings), cookies=_NullCookieJar()+ )++ async def download_request(self, request: Request) -> Response:+ self._warn_unsupported_meta(request.meta)++ timeout: float = request.meta.get(+ "download_timeout", self._DEFAULT_CONNECT_TIMEOUT+ )++ try:+ async with self._get_httpx_response(request, timeout) as httpx_response:+ return await self._read_response(httpx_response, request)+ except httpx.TimeoutException as e:+ raise DownloadTimeoutError(+ f"Getting {request.url} took longer than {timeout} seconds."+ ) from e+ except httpx.UnsupportedProtocol as e:+ raise UnsupportedURLSchemeError(str(e)) from e+ except httpx.ConnectError as e:+ if "Name or service not known" in str(e) or "getaddrinfo failed" in str(e):+ raise CannotResolveHostError(str(e)) from e+ raise DownloadConnectionRefusedError(str(e)) from e+ except httpx.NetworkError as e:+ raise DownloadFailedError(str(e)) from e+ except httpx.RemoteProtocolError as e:+ raise DownloadFailedError(str(e)) from e++ def _warn_unsupported_meta(self, meta: dict[str, Any]) -> None:+ if meta.get("bindaddress"):+ # configurable only per-client:+ # https://github.com/encode/httpx/issues/755#issuecomment-2746121794+ logger.error(+ f"The 'bindaddress' request meta key is not supported by"+ f" {type(self).__name__} and will be ignored."+ )+ if meta.get("proxy"):+ # configurable only per-client:+ # https://github.com/encode/httpx/issues/486+ logger.error(+ f"The 'proxy' request meta key is not supported by"+ f" {type(self).__name__} and will be ignored."+ )++ def _get_httpx_response(+ self, request: Request, timeout: float+ ) -> AbstractAsyncContextManager[httpx.Response]:+ return self._client.stream(+ request.method,+ request.url,+ content=request.body,+ headers=request.headers.to_tuple_list(),+ timeout=timeout,+ )++ async def _read_response(+ self, httpx_response: httpx.Response, request: Request+ ) -> Response:+ maxsize: int = request.meta.get("download_maxsize", self._default_maxsize)+ warnsize: int = request.meta.get("download_warnsize", self._default_warnsize)++ content_length = httpx_response.headers.get("Content-Length")+ expected_size = int(content_length) if content_length is not None else None+ if maxsize and expected_size and expected_size > maxsize:+ self._cancel_maxsize(expected_size, maxsize, request, expected=True)++ reached_warnsize = False+ if warnsize and expected_size and expected_size > warnsize:+ reached_warnsize = True+ logger.warning(+ get_warnsize_msg(expected_size, warnsize, request, expected=True)+ )++ headers = Headers(httpx_response.headers.multi_items())+ network_stream: AsyncNetworkStream = httpx_response.extensions["network_stream"]++ make_response_base_args: _BaseResponseArgs = {+ "status": httpx_response.status_code,+ "url": request.url,+ "headers": headers,+ "ip_address": self._get_server_ip(network_stream),+ "protocol": httpx_response.http_version,+ }++ self._log_tls_info(network_stream)++ if stop_download := check_stop_download(+ signals.headers_received,+ self.crawler,+ request,+ headers=headers,+ body_length=expected_size,+ ):+ return make_response(+ **make_response_base_args,+ stop_download=stop_download,+ )++ response_body = BytesIO()+ bytes_received = 0+ try:+ async for chunk in httpx_response.aiter_raw():+ response_body.write(chunk)+ bytes_received += len(chunk)++ if stop_download := check_stop_download(+ signals.bytes_received, self.crawler, request, data=chunk+ ):+ return make_response(+ **make_response_base_args,+ body=response_body.getvalue(),+ stop_download=stop_download,+ )++ if maxsize and bytes_received > maxsize:+ response_body.truncate(0)+ self._cancel_maxsize(+ bytes_received, maxsize, request, expected=False+ )++ if warnsize and bytes_received > warnsize and not reached_warnsize:+ reached_warnsize = True+ logger.warning(+ get_warnsize_msg(+ bytes_received, warnsize, request, expected=False+ )+ )+ except httpx.RemoteProtocolError as e:+ # special handling of the dataloss case+ if (+ "peer closed connection without sending complete message body"+ not in str(e)+ ):+ raise+ fail_on_dataloss: bool = request.meta.get(+ "download_fail_on_dataloss", self._fail_on_dataloss+ )+ if not fail_on_dataloss:+ return make_response(+ **make_response_base_args,+ body=response_body.getvalue(),+ flags=["dataloss"],+ )+ self._log_dataloss_warning(request.url)+ raise ResponseDataLossError(str(e)) from e++ return make_response(+ **make_response_base_args,+ body=response_body.getvalue(),+ )++ @staticmethod+ def _get_server_ip(network_stream: AsyncNetworkStream) -> IPv4Address | IPv6Address:+ extra_server_addr = network_stream.get_extra_info("server_addr")+ return ipaddress.ip_address(extra_server_addr[0])++ def _log_tls_info(self, network_stream: AsyncNetworkStream) -> None:+ if not self._tls_verbose_logging:+ return+ extra_ssl_object = network_stream.get_extra_info("ssl_object")+ if isinstance(extra_ssl_object, ssl.SSLObject):+ _log_sslobj_debug_info(extra_ssl_object)++ def _log_dataloss_warning(self, url: str) -> None:+ if self._fail_on_dataloss_warned:+ return+ logger.warning(get_dataloss_msg(url))+ self._fail_on_dataloss_warned = True++ @staticmethod+ def _cancel_maxsize(+ size: int, limit: int, request: Request, *, expected: bool+ ) -> NoReturn:+ warning_msg = get_maxsize_msg(size, limit, request, expected=expected)+ logger.warning(warning_msg)+ raise DownloadCancelledError(warning_msg)++ async def close(self):+ await self._client.aclose()diff --git a/scrapy/http/headers.py b/scrapy/http/headers.pyindex f3f60bffb..34d4ec6f2 100644--- a/scrapy/http/headers.py+++ b/scrapy/http/headers.py@@ -124,6 +124,17 @@ class Headers(CaselessDict):for key, value in self.items())+ def to_tuple_list(self) -> list[tuple[str, str]]:+ """Return headers as a list of ``(key, value)`` tuples.++ Multiple values are represented as multiple tuples with the same key.+ """+ return [+ (key.decode(self.encoding), value.decode(self.encoding))+ for key, values in self.items()+ for value in values+ ]+def __copy__(self) -> Self:return self.__class__(self)diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex 6d3bb1db9..07a9e5ff7 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -72,6 +72,12 @@ DEFAULT_LOGGING = {"hpack": {"level": "ERROR",},+ "httpcore": {+ "level": "ERROR",+ },+ "httpx": {+ "level": "WARNING",+ },"scrapy": {"level": "DEBUG",},diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.pyindex 7d46cbd4f..0062da52b 100644--- a/scrapy/utils/ssl.py+++ b/scrapy/utils/ssl.py@@ -1,5 +1,7 @@from __future__ import annotations+import logging+import sslfrom typing import TYPE_CHECKING, Anyimport OpenSSL._util as pyOpenSSLutil@@ -11,6 +13,54 @@ from scrapy.utils.python import to_unicodeif TYPE_CHECKING:from OpenSSL.crypto import X509Name+ from scrapy.settings import BaseSettings++logger = logging.getLogger(__name__)+++# stdlib ssl module utils++# possible documented values for DOWNLOADER_CLIENT_TLS_METHOD+_STDLIB_PROTOCOL_MAP = {+ "TLS": ssl.PROTOCOL_TLS_CLIENT,+ "TLSv1.0": ssl.PROTOCOL_TLSv1,+ "TLSv1.1": ssl.PROTOCOL_TLSv1_1,+ "TLSv1.2": ssl.PROTOCOL_TLSv1_2,+}+++def _make_ssl_context(settings: BaseSettings) -> ssl.SSLContext:+ """Create an :class:`ssl.SSLContext` instance according to the settings.++ It's intended to be used in an HTTPS download handler.+ """++ method_setting: str = settings["DOWNLOADER_CLIENT_TLS_METHOD"]+ if method_setting not in _STDLIB_PROTOCOL_MAP:+ raise ValueError(f"Unsupported TLS method: {method_setting}")+ ciphers_setting: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]++ ctx = ssl.SSLContext(_STDLIB_PROTOCOL_MAP[method_setting])+ ctx.check_hostname = False+ ctx.verify_mode = ssl.CERT_NONE+ if ciphers_setting:+ ctx.set_ciphers(ciphers_setting)+ return ctx+++def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None:+ cipher = sslobj.cipher()+ logger.debug(+ f"SSL connection to {sslobj.server_hostname}"+ f" using protocol {sslobj.version()},"+ f" cipher {cipher[0] if cipher else None}"+ )+ # The peer certificate is unavailable on SSLObject unless peer+ # certificate verification is enabled, which we don't want.+++# pyOpenSSL utils+def ffi_buf_to_string(buf: Any) -> str:return to_unicode(pyOpenSSLutil.ffi.string(buf))@@ -22,7 +72,6 @@ def x509name_to_string(x509name: X509Name) -> str:pyOpenSSLutil.lib.X509_NAME_oneline(x509name._name, result_buffer, len(result_buffer))-return ffi_buf_to_string(result_buffer)diff --git a/tests/spiders.py b/tests/spiders.pyindex 43cdcaef3..de2e64242 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -22,9 +22,10 @@ from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_reqclass MockServerSpider(Spider):- def __init__(self, mockserver=None, *args, **kwargs):+ def __init__(self, *args, mockserver=None, is_secure=False, **kwargs):super().__init__(*args, **kwargs)self.mockserver = mockserver+ self.is_secure = is_secureclass MetaSpider(MockServerSpider):@@ -516,7 +517,7 @@ class BytesReceivedCallbackSpider(MetaSpider):async def start(self):body = b"a" * self.full_response_length- url = self.mockserver.url("/alpayload")+ url = self.mockserver.url("/alpayload", is_secure=self.is_secure)yield Request(url, method="POST", body=body, errback=self.errback)def parse(self, response):@@ -544,7 +545,10 @@ class HeadersReceivedCallbackSpider(MetaSpider):return spiderasync def start(self):- yield Request(self.mockserver.url("/status"), errback=self.errback)+ yield Request(+ self.mockserver.url("/status", is_secure=self.is_secure),+ errback=self.errback,+ )def parse(self, response):self.meta["response"] = responsediff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pynew file mode 100644index 000000000..ad6c6d3c7--- /dev/null+++ b/tests/test_downloader_handler_httpx.py@@ -0,0 +1,127 @@+"""Tests for scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler."""++from __future__ import annotations++from typing import TYPE_CHECKING, Any++import pytest++from scrapy import Request+from tests.test_downloader_handlers_http_base import (+ TestHttp11Base,+ TestHttpProxyBase,+ TestHttps11Base,+ TestHttpsCustomCiphersBase,+ TestHttpsInvalidDNSIdBase,+ TestHttpsInvalidDNSPatternBase,+ TestHttpsWrongHostnameBase,+ TestHttpWithCrawlerBase,+ TestSimpleHttpsBase,+)+from tests.utils.decorators import coroutine_test++if TYPE_CHECKING:+ from scrapy.core.downloader.handlers import DownloadHandlerProtocol+ from tests.mockserver.http import MockServer+++pytest.importorskip("httpx")+++class HttpxDownloadHandlerMixin:+ @property+ def download_handler_cls(self) -> type[DownloadHandlerProtocol]:+ # the import will fail if httpx is not installed+ from scrapy.core.downloader.handlers._httpx import ( # noqa: PLC0415+ HttpxDownloadHandler,+ )++ return HttpxDownloadHandler+++class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base):+ @coroutine_test+ async def test_unsupported_bindaddress(+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer+ ) -> None:+ meta = {"bindaddress": "127.0.0.2"}+ request = Request(mockserver.url("/text"), meta=meta)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert response.body == b"Works"+ assert (+ "The 'bindaddress' request meta key is not supported by HttpxDownloadHandler"+ in caplog.text+ )++ @coroutine_test+ async def test_unsupported_proxy(+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer+ ) -> None:+ meta = {"proxy": "127.0.0.2"}+ request = Request(mockserver.url("/text"), meta=meta)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert response.body == b"Works"+ assert (+ "The 'proxy' request meta key is not supported by HttpxDownloadHandler"+ in caplog.text+ )+++class TestHttps11(HttpxDownloadHandlerMixin, TestHttps11Base):+ tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher"+++class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):+ pass+++class Https11WrongHostnameTestCase(+ HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase+):+ pass+++class Https11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):+ pass+++class Https11InvalidDNSPattern(+ HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase+):+ pass+++class Https11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):+ pass+++class TestHttp11WithCrawler(TestHttpWithCrawlerBase):+ @property+ def settings_dict(self) -> dict[str, Any] | None:+ return {+ "DOWNLOAD_HANDLERS": {+ "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",+ "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",+ }+ }+++class TestHttps11WithCrawler(TestHttp11WithCrawler):+ is_secure = True++ @pytest.mark.skip(reason="response.certificate is not implemented")+ @coroutine_test+ async def test_response_ssl_certificate(self, mockserver: MockServer) -> None:+ pass++[email protected](reason="Proxy support is not implemented yet")+class TestHttp11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):+ pass++[email protected](reason="Proxy support is not implemented yet")+class TestHttps11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):+ is_secure = Truediff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex df19646ff..1b9dc21e7 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -189,6 +189,18 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase):is_secure = True+ def test_bytes_received_stop_download_callback(self) -> None: # type: ignore[override]+ pytest.skip("bytes_received support is not implemented")++ def test_bytes_received_stop_download_errback(self) -> None: # type: ignore[override]+ pytest.skip("bytes_received support is not implemented")++ def test_headers_received_stop_download_callback(self) -> None: # type: ignore[override]+ pytest.skip("headers_received support is not implemented")++ def test_headers_received_stop_download_errback(self) -> None: # type: ignore[override]+ pytest.skip("headers_received support is not implemented")+class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):is_secure = Truediff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex bc7807b4d..8e9fa0ae1 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -16,6 +16,7 @@ from urllib.parse import urlparseimport pytestfrom twisted.internet.ssl import Certificate+from twisted.python.failure import Failurefrom scrapy.exceptions import (CannotResolveHostError,@@ -24,6 +25,7 @@ from scrapy.exceptions import (DownloadFailedError,DownloadTimeoutError,ResponseDataLossError,+ StopDownload,UnsupportedURLSchemeError,)from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse@@ -34,7 +36,13 @@ from scrapy.utils.test import get_crawlerfrom tests import NON_EXISTING_RESOLVABLEfrom tests.mockserver.proxy_echo import ProxyEchoMockServerfrom tests.mockserver.simple_https import SimpleMockServer-from tests.spiders import SingleRequestSpider+from tests.spiders import (+ BytesReceivedCallbackSpider,+ BytesReceivedErrbackSpider,+ HeadersReceivedCallbackSpider,+ HeadersReceivedErrbackSpider,+ SingleRequestSpider,+)from tests.utils.decorators import coroutine_testif TYPE_CHECKING:@@ -832,6 +840,74 @@ class TestHttpWithCrawlerBase(ABC):assert isinstance(ip_address, IPv4Address)assert str(ip_address) == gethostbyname(expected_netloc)+ @coroutine_test+ async def test_bytes_received_stop_download_callback(+ self, mockserver: MockServer+ ) -> None:+ # copy of TestCrawl.test_bytes_received_stop_download_callback()+ crawler = get_crawler(BytesReceivedCallbackSpider, self.settings_dict)+ await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure)+ assert isinstance(crawler.spider, BytesReceivedCallbackSpider)+ assert crawler.spider.meta.get("failure") is None+ assert isinstance(crawler.spider.meta["response"], Response)+ assert crawler.spider.meta["response"].body == crawler.spider.meta.get(+ "bytes_received"+ )+ assert (+ len(crawler.spider.meta["response"].body)+ < crawler.spider.full_response_length+ )++ @coroutine_test+ async def test_bytes_received_stop_download_errback(+ self, mockserver: MockServer+ ) -> None:+ # copy of TestCrawl.test_bytes_received_stop_download_errback()+ crawler = get_crawler(BytesReceivedErrbackSpider, self.settings_dict)+ await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure)+ assert isinstance(crawler.spider, BytesReceivedErrbackSpider)+ assert crawler.spider.meta.get("response") is None+ assert isinstance(crawler.spider.meta["failure"], Failure)+ assert isinstance(crawler.spider.meta["failure"].value, StopDownload)+ assert isinstance(crawler.spider.meta["failure"].value.response, Response)+ assert crawler.spider.meta[+ "failure"+ ].value.response.body == crawler.spider.meta.get("bytes_received")+ assert (+ len(crawler.spider.meta["failure"].value.response.body)+ < crawler.spider.full_response_length+ )++ @coroutine_test+ async def test_headers_received_stop_download_callback(+ self, mockserver: MockServer+ ) -> None:+ # copy of TestCrawl.test_headers_received_stop_download_callback()+ crawler = get_crawler(HeadersReceivedCallbackSpider, self.settings_dict)+ await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure)+ assert isinstance(crawler.spider, HeadersReceivedCallbackSpider)+ assert crawler.spider.meta.get("failure") is None+ assert isinstance(crawler.spider.meta["response"], Response)+ assert crawler.spider.meta["response"].headers == crawler.spider.meta.get(+ "headers_received"+ )++ @coroutine_test+ async def test_headers_received_stop_download_errback(+ self, mockserver: MockServer+ ) -> None:+ # copy of TestCrawl.test_headers_received_stop_download_errback()+ crawler = get_crawler(HeadersReceivedErrbackSpider, self.settings_dict)+ await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure)+ assert isinstance(crawler.spider, HeadersReceivedErrbackSpider)+ assert crawler.spider.meta.get("response") is None+ assert isinstance(crawler.spider.meta["failure"], Failure)+ assert isinstance(crawler.spider.meta["failure"].value, StopDownload)+ assert isinstance(crawler.spider.meta["failure"].value.response, Response)+ assert crawler.spider.meta[+ "failure"+ ].value.response.headers == crawler.spider.meta.get("headers_received")+class TestHttpProxyBase(ABC):is_secure = Falsediff --git a/tox.ini b/tox.iniindex 83cd06561..3be564502 100644--- a/tox.ini+++ b/tox.ini@@ -48,6 +48,7 @@ deps =types-Pygments==2.19.0.20250809botocore-stubs==1.40.59boto3-stubs[s3]==1.40.59+ httpx==0.28.1itemadapter==0.12.2Protego==0.5.0w3lib==2.3.1@@ -134,6 +135,7 @@ deps =brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware testsbrotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware testsgoogle-cloud-storage+ httpxipythonrobotexclusionrulesparseruvloop; platform_system != "Windows" and implementation_name != "pypy"@@ -149,6 +151,7 @@ deps =brotli==1.2.0; implementation_name != "pypy"brotlicffi==1.2.0.0; implementation_name == "pypy"google-cloud-storage==1.29.0+ httpx==0.26.0ipython==7.1.0robotexclusionrulesparser==1.6.2uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"