Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add HttpxDownloader. (#7239)

Andrey Rakhmatullinfc30c47f38e8
diff --git a/conftest.py b/conftest.py
index 50925a311..26e0434ba 100644
--- a/conftest.py
+++ b/conftest.py
@@ -52,6 +52,11 @@ if not H2_ENABLED:
)
)
+try:
+ import httpx # noqa: F401
+except ImportError:
+ collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
+
def pytest_addoption(parser, pluginmanager):
if pluginmanager.hasplugin("twisted"):
diff --git a/docs/news.rst b/docs/news.rst
index bab2e5ff9..ff8e6fb08 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,17 @@
Release notes
=============
+Scrapy VERSION (unreleased)
+---------------------------
+
+New features
+~~~~~~~~~~~~
+
+- Added
+ :meth:`Headers.to_tuple_list() <scrapy.http.headers.Headers.to_tuple_list>`
+ that returns headers as a list of ``(key, value)`` tuples.
+ (:issue:`7239`)
+
.. _release-2.14.1:
Scrapy 2.14.1 (2026-01-12)
@@ -2997,7 +3008,7 @@ Bug fixes
that does not match the asyncio event loop actually installed
(:issue:`5529`).
-- Fixed :meth:`Headers.getlist <scrapy.http.headers.Headers.getlist>`
+- Fixed :meth:`Headers.getlist() <scrapy.http.headers.Headers.getlist>`
returning only the last header (:issue:`5515`, :issue:`5526`).
- Fixed :class:`LinkExtractor
diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py
new file mode 100644
index 000000000..1f5cf007e
--- /dev/null
+++ b/scrapy/core/downloader/handlers/_httpx.py
@@ -0,0 +1,269 @@
+"""``httpx``-based HTTP(S) download handler. Currently not recommended for production use."""
+
+from __future__ import annotations
+
+import ipaddress
+import logging
+import ssl
+from http.cookiejar import Cookie, CookieJar
+from io import BytesIO
+from typing import TYPE_CHECKING, Any, NoReturn, TypedDict
+
+import httpx
+
+from scrapy import Request, signals
+from scrapy.exceptions import (
+ CannotResolveHostError,
+ DownloadCancelledError,
+ DownloadConnectionRefusedError,
+ DownloadFailedError,
+ DownloadTimeoutError,
+ NotConfigured,
+ ResponseDataLossError,
+ UnsupportedURLSchemeError,
+)
+from scrapy.http import Headers, Response
+from scrapy.utils._download_handlers import (
+ BaseHttpDownloadHandler,
+ check_stop_download,
+ get_dataloss_msg,
+ get_maxsize_msg,
+ get_warnsize_msg,
+ make_response,
+)
+from scrapy.utils.asyncio import is_asyncio_available
+from scrapy.utils.ssl import _log_sslobj_debug_info, _make_ssl_context
+
+if TYPE_CHECKING:
+ from contextlib import AbstractAsyncContextManager
+ from http.client import HTTPResponse
+ from ipaddress import IPv4Address, IPv6Address
+ from urllib.request import Request as ULRequest
+
+ from httpcore import AsyncNetworkStream
+
+ from scrapy.crawler import Crawler
+
+
+logger = logging.getLogger(__name__)
+
+
+class _BaseResponseArgs(TypedDict):
+ status: int
+ url: str
+ headers: Headers
+ ip_address: IPv4Address | IPv6Address
+ protocol: str
+
+
+# workaround for (and from) https://github.com/encode/httpx/issues/2992
+class _NullCookieJar(CookieJar): # pragma: no cover
+ """A CookieJar that rejects all cookies."""
+
+ def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None:
+ pass
+
+ def set_cookie(self, cookie: Cookie) -> None:
+ pass
+
+
+class HttpxDownloadHandler(BaseHttpDownloadHandler):
+ _DEFAULT_CONNECT_TIMEOUT = 10
+
+ def __init__(self, crawler: Crawler):
+ # we don't run extra-deps tests with the non-asyncio reactor
+ if not is_asyncio_available(): # pragma: no cover
+ raise NotConfigured(
+ f"{type(self).__name__} requires the asyncio support. Make"
+ f" sure that you have either enabled the asyncio Twisted"
+ f" reactor in the TWISTED_REACTOR setting or disabled the"
+ f" TWISTED_ENABLED setting. See the asyncio documentation"
+ f" of Scrapy for more information."
+ )
+ super().__init__(crawler)
+ logger.warning(
+ "HttpxDownloadHandler is experimental and is not recommented for production use."
+ )
+ self._tls_verbose_logging: bool = self.crawler.settings.getbool(
+ "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
+ )
+ self._client = httpx.AsyncClient(
+ verify=_make_ssl_context(crawler.settings), cookies=_NullCookieJar()
+ )
+
+ async def download_request(self, request: Request) -> Response:
+ self._warn_unsupported_meta(request.meta)
+
+ timeout: float = request.meta.get(
+ "download_timeout", self._DEFAULT_CONNECT_TIMEOUT
+ )
+
+ try:
+ async with self._get_httpx_response(request, timeout) as httpx_response:
+ return await self._read_response(httpx_response, request)
+ except httpx.TimeoutException as e:
+ raise DownloadTimeoutError(
+ f"Getting {request.url} took longer than {timeout} seconds."
+ ) from e
+ except httpx.UnsupportedProtocol as e:
+ raise UnsupportedURLSchemeError(str(e)) from e
+ except httpx.ConnectError as e:
+ if "Name or service not known" in str(e) or "getaddrinfo failed" in str(e):
+ raise CannotResolveHostError(str(e)) from e
+ raise DownloadConnectionRefusedError(str(e)) from e
+ except httpx.NetworkError as e:
+ raise DownloadFailedError(str(e)) from e
+ except httpx.RemoteProtocolError as e:
+ raise DownloadFailedError(str(e)) from e
+
+ def _warn_unsupported_meta(self, meta: dict[str, Any]) -> None:
+ if meta.get("bindaddress"):
+ # configurable only per-client:
+ # https://github.com/encode/httpx/issues/755#issuecomment-2746121794
+ logger.error(
+ f"The 'bindaddress' request meta key is not supported by"
+ f" {type(self).__name__} and will be ignored."
+ )
+ if meta.get("proxy"):
+ # configurable only per-client:
+ # https://github.com/encode/httpx/issues/486
+ logger.error(
+ f"The 'proxy' request meta key is not supported by"
+ f" {type(self).__name__} and will be ignored."
+ )
+
+ def _get_httpx_response(
+ self, request: Request, timeout: float
+ ) -> AbstractAsyncContextManager[httpx.Response]:
+ return self._client.stream(
+ request.method,
+ request.url,
+ content=request.body,
+ headers=request.headers.to_tuple_list(),
+ timeout=timeout,
+ )
+
+ async def _read_response(
+ self, httpx_response: httpx.Response, request: Request
+ ) -> Response:
+ maxsize: int = request.meta.get("download_maxsize", self._default_maxsize)
+ warnsize: int = request.meta.get("download_warnsize", self._default_warnsize)
+
+ content_length = httpx_response.headers.get("Content-Length")
+ expected_size = int(content_length) if content_length is not None else None
+ if maxsize and expected_size and expected_size > maxsize:
+ self._cancel_maxsize(expected_size, maxsize, request, expected=True)
+
+ reached_warnsize = False
+ if warnsize and expected_size and expected_size > warnsize:
+ reached_warnsize = True
+ logger.warning(
+ get_warnsize_msg(expected_size, warnsize, request, expected=True)
+ )
+
+ headers = Headers(httpx_response.headers.multi_items())
+ network_stream: AsyncNetworkStream = httpx_response.extensions["network_stream"]
+
+ make_response_base_args: _BaseResponseArgs = {
+ "status": httpx_response.status_code,
+ "url": request.url,
+ "headers": headers,
+ "ip_address": self._get_server_ip(network_stream),
+ "protocol": httpx_response.http_version,
+ }
+
+ self._log_tls_info(network_stream)
+
+ if stop_download := check_stop_download(
+ signals.headers_received,
+ self.crawler,
+ request,
+ headers=headers,
+ body_length=expected_size,
+ ):
+ return make_response(
+ **make_response_base_args,
+ stop_download=stop_download,
+ )
+
+ response_body = BytesIO()
+ bytes_received = 0
+ try:
+ async for chunk in httpx_response.aiter_raw():
+ response_body.write(chunk)
+ bytes_received += len(chunk)
+
+ if stop_download := check_stop_download(
+ signals.bytes_received, self.crawler, request, data=chunk
+ ):
+ return make_response(
+ **make_response_base_args,
+ body=response_body.getvalue(),
+ stop_download=stop_download,
+ )
+
+ if maxsize and bytes_received > maxsize:
+ response_body.truncate(0)
+ self._cancel_maxsize(
+ bytes_received, maxsize, request, expected=False
+ )
+
+ if warnsize and bytes_received > warnsize and not reached_warnsize:
+ reached_warnsize = True
+ logger.warning(
+ get_warnsize_msg(
+ bytes_received, warnsize, request, expected=False
+ )
+ )
+ except httpx.RemoteProtocolError as e:
+ # special handling of the dataloss case
+ if (
+ "peer closed connection without sending complete message body"
+ not in str(e)
+ ):
+ raise
+ fail_on_dataloss: bool = request.meta.get(
+ "download_fail_on_dataloss", self._fail_on_dataloss
+ )
+ if not fail_on_dataloss:
+ return make_response(
+ **make_response_base_args,
+ body=response_body.getvalue(),
+ flags=["dataloss"],
+ )
+ self._log_dataloss_warning(request.url)
+ raise ResponseDataLossError(str(e)) from e
+
+ return make_response(
+ **make_response_base_args,
+ body=response_body.getvalue(),
+ )
+
+ @staticmethod
+ def _get_server_ip(network_stream: AsyncNetworkStream) -> IPv4Address | IPv6Address:
+ extra_server_addr = network_stream.get_extra_info("server_addr")
+ return ipaddress.ip_address(extra_server_addr[0])
+
+ def _log_tls_info(self, network_stream: AsyncNetworkStream) -> None:
+ if not self._tls_verbose_logging:
+ return
+ extra_ssl_object = network_stream.get_extra_info("ssl_object")
+ if isinstance(extra_ssl_object, ssl.SSLObject):
+ _log_sslobj_debug_info(extra_ssl_object)
+
+ def _log_dataloss_warning(self, url: str) -> None:
+ if self._fail_on_dataloss_warned:
+ return
+ logger.warning(get_dataloss_msg(url))
+ self._fail_on_dataloss_warned = True
+
+ @staticmethod
+ def _cancel_maxsize(
+ size: int, limit: int, request: Request, *, expected: bool
+ ) -> NoReturn:
+ warning_msg = get_maxsize_msg(size, limit, request, expected=expected)
+ logger.warning(warning_msg)
+ raise DownloadCancelledError(warning_msg)
+
+ async def close(self):
+ await self._client.aclose()
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index f3f60bffb..34d4ec6f2 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -124,6 +124,17 @@ class Headers(CaselessDict):
for key, value in self.items()
)
+ def to_tuple_list(self) -> list[tuple[str, str]]:
+ """Return headers as a list of ``(key, value)`` tuples.
+
+ Multiple values are represented as multiple tuples with the same key.
+ """
+ return [
+ (key.decode(self.encoding), value.decode(self.encoding))
+ for key, values in self.items()
+ for value in values
+ ]
+
def __copy__(self) -> Self:
return self.__class__(self)
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 6d3bb1db9..07a9e5ff7 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -72,6 +72,12 @@ DEFAULT_LOGGING = {
"hpack": {
"level": "ERROR",
},
+ "httpcore": {
+ "level": "ERROR",
+ },
+ "httpx": {
+ "level": "WARNING",
+ },
"scrapy": {
"level": "DEBUG",
},
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index 7d46cbd4f..0062da52b 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+import logging
+import ssl
from typing import TYPE_CHECKING, Any
import OpenSSL._util as pyOpenSSLutil
@@ -11,6 +13,54 @@ from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
from OpenSSL.crypto import X509Name
+ from scrapy.settings import BaseSettings
+
+logger = logging.getLogger(__name__)
+
+
+# stdlib ssl module utils
+
+# possible documented values for DOWNLOADER_CLIENT_TLS_METHOD
+_STDLIB_PROTOCOL_MAP = {
+ "TLS": ssl.PROTOCOL_TLS_CLIENT,
+ "TLSv1.0": ssl.PROTOCOL_TLSv1,
+ "TLSv1.1": ssl.PROTOCOL_TLSv1_1,
+ "TLSv1.2": ssl.PROTOCOL_TLSv1_2,
+}
+
+
+def _make_ssl_context(settings: BaseSettings) -> ssl.SSLContext:
+ """Create an :class:`ssl.SSLContext` instance according to the settings.
+
+ It's intended to be used in an HTTPS download handler.
+ """
+
+ method_setting: str = settings["DOWNLOADER_CLIENT_TLS_METHOD"]
+ if method_setting not in _STDLIB_PROTOCOL_MAP:
+ raise ValueError(f"Unsupported TLS method: {method_setting}")
+ ciphers_setting: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
+
+ ctx = ssl.SSLContext(_STDLIB_PROTOCOL_MAP[method_setting])
+ ctx.check_hostname = False
+ ctx.verify_mode = ssl.CERT_NONE
+ if ciphers_setting:
+ ctx.set_ciphers(ciphers_setting)
+ return ctx
+
+
+def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None:
+ cipher = sslobj.cipher()
+ logger.debug(
+ f"SSL connection to {sslobj.server_hostname}"
+ f" using protocol {sslobj.version()},"
+ f" cipher {cipher[0] if cipher else None}"
+ )
+ # The peer certificate is unavailable on SSLObject unless peer
+ # certificate verification is enabled, which we don't want.
+
+
+# pyOpenSSL utils
+
def ffi_buf_to_string(buf: Any) -> str:
return to_unicode(pyOpenSSLutil.ffi.string(buf))
@@ -22,7 +72,6 @@ def x509name_to_string(x509name: X509Name) -> str:
pyOpenSSLutil.lib.X509_NAME_oneline(
x509name._name, result_buffer, len(result_buffer)
)
-
return ffi_buf_to_string(result_buffer)
diff --git a/tests/spiders.py b/tests/spiders.py
index 43cdcaef3..de2e64242 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -22,9 +22,10 @@ from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req
class MockServerSpider(Spider):
- def __init__(self, mockserver=None, *args, **kwargs):
+ def __init__(self, *args, mockserver=None, is_secure=False, **kwargs):
super().__init__(*args, **kwargs)
self.mockserver = mockserver
+ self.is_secure = is_secure
class MetaSpider(MockServerSpider):
@@ -516,7 +517,7 @@ class BytesReceivedCallbackSpider(MetaSpider):
async def start(self):
body = b"a" * self.full_response_length
- url = self.mockserver.url("/alpayload")
+ url = self.mockserver.url("/alpayload", is_secure=self.is_secure)
yield Request(url, method="POST", body=body, errback=self.errback)
def parse(self, response):
@@ -544,7 +545,10 @@ class HeadersReceivedCallbackSpider(MetaSpider):
return spider
async def start(self):
- yield Request(self.mockserver.url("/status"), errback=self.errback)
+ yield Request(
+ self.mockserver.url("/status", is_secure=self.is_secure),
+ errback=self.errback,
+ )
def parse(self, response):
self.meta["response"] = response
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
new file mode 100644
index 000000000..ad6c6d3c7
--- /dev/null
+++ b/tests/test_downloader_handler_httpx.py
@@ -0,0 +1,127 @@
+"""Tests for scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from scrapy import Request
+from tests.test_downloader_handlers_http_base import (
+ TestHttp11Base,
+ TestHttpProxyBase,
+ TestHttps11Base,
+ TestHttpsCustomCiphersBase,
+ TestHttpsInvalidDNSIdBase,
+ TestHttpsInvalidDNSPatternBase,
+ TestHttpsWrongHostnameBase,
+ TestHttpWithCrawlerBase,
+ TestSimpleHttpsBase,
+)
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+ from scrapy.core.downloader.handlers import DownloadHandlerProtocol
+ from tests.mockserver.http import MockServer
+
+
+pytest.importorskip("httpx")
+
+
+class HttpxDownloadHandlerMixin:
+ @property
+ def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
+ # the import will fail if httpx is not installed
+ from scrapy.core.downloader.handlers._httpx import ( # noqa: PLC0415
+ HttpxDownloadHandler,
+ )
+
+ return HttpxDownloadHandler
+
+
+class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base):
+ @coroutine_test
+ async def test_unsupported_bindaddress(
+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+ ) -> None:
+ meta = {"bindaddress": "127.0.0.2"}
+ request = Request(mockserver.url("/text"), meta=meta)
+ async with self.get_dh() as download_handler:
+ response = await download_handler.download_request(request)
+ assert response.body == b"Works"
+ assert (
+ "The 'bindaddress' request meta key is not supported by HttpxDownloadHandler"
+ in caplog.text
+ )
+
+ @coroutine_test
+ async def test_unsupported_proxy(
+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+ ) -> None:
+ meta = {"proxy": "127.0.0.2"}
+ request = Request(mockserver.url("/text"), meta=meta)
+ async with self.get_dh() as download_handler:
+ response = await download_handler.download_request(request)
+ assert response.body == b"Works"
+ assert (
+ "The 'proxy' request meta key is not supported by HttpxDownloadHandler"
+ in caplog.text
+ )
+
+
+class TestHttps11(HttpxDownloadHandlerMixin, TestHttps11Base):
+ tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher"
+
+
+class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):
+ pass
+
+
+class Https11WrongHostnameTestCase(
+ HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase
+):
+ pass
+
+
+class Https11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
+ pass
+
+
+class Https11InvalidDNSPattern(
+ HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
+):
+ pass
+
+
+class Https11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):
+ pass
+
+
+class TestHttp11WithCrawler(TestHttpWithCrawlerBase):
+ @property
+ def settings_dict(self) -> dict[str, Any] | None:
+ return {
+ "DOWNLOAD_HANDLERS": {
+ "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
+ "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
+ }
+ }
+
+
+class TestHttps11WithCrawler(TestHttp11WithCrawler):
+ is_secure = True
+
+ @pytest.mark.skip(reason="response.certificate is not implemented")
+ @coroutine_test
+ async def test_response_ssl_certificate(self, mockserver: MockServer) -> None:
+ pass
+
+
[email protected](reason="Proxy support is not implemented yet")
+class TestHttp11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
+ pass
+
+
[email protected](reason="Proxy support is not implemented yet")
+class TestHttps11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
+ is_secure = True
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index df19646ff..1b9dc21e7 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -189,6 +189,18 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase):
is_secure = True
+ def test_bytes_received_stop_download_callback(self) -> None: # type: ignore[override]
+ pytest.skip("bytes_received support is not implemented")
+
+ def test_bytes_received_stop_download_errback(self) -> None: # type: ignore[override]
+ pytest.skip("bytes_received support is not implemented")
+
+ def test_headers_received_stop_download_callback(self) -> None: # type: ignore[override]
+ pytest.skip("headers_received support is not implemented")
+
+ def test_headers_received_stop_download_errback(self) -> None: # type: ignore[override]
+ pytest.skip("headers_received support is not implemented")
+
class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
is_secure = True
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index bc7807b4d..8e9fa0ae1 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -16,6 +16,7 @@ from urllib.parse import urlparse
import pytest
from twisted.internet.ssl import Certificate
+from twisted.python.failure import Failure
from scrapy.exceptions import (
CannotResolveHostError,
@@ -24,6 +25,7 @@ from scrapy.exceptions import (
DownloadFailedError,
DownloadTimeoutError,
ResponseDataLossError,
+ StopDownload,
UnsupportedURLSchemeError,
)
from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
@@ -34,7 +36,13 @@ from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver.proxy_echo import ProxyEchoMockServer
from tests.mockserver.simple_https import SimpleMockServer
-from tests.spiders import SingleRequestSpider
+from tests.spiders import (
+ BytesReceivedCallbackSpider,
+ BytesReceivedErrbackSpider,
+ HeadersReceivedCallbackSpider,
+ HeadersReceivedErrbackSpider,
+ SingleRequestSpider,
+)
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@@ -832,6 +840,74 @@ class TestHttpWithCrawlerBase(ABC):
assert isinstance(ip_address, IPv4Address)
assert str(ip_address) == gethostbyname(expected_netloc)
+ @coroutine_test
+ async def test_bytes_received_stop_download_callback(
+ self, mockserver: MockServer
+ ) -> None:
+ # copy of TestCrawl.test_bytes_received_stop_download_callback()
+ crawler = get_crawler(BytesReceivedCallbackSpider, self.settings_dict)
+ await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure)
+ assert isinstance(crawler.spider, BytesReceivedCallbackSpider)
+ assert crawler.spider.meta.get("failure") is None
+ assert isinstance(crawler.spider.meta["response"], Response)
+ assert crawler.spider.meta["response"].body == crawler.spider.meta.get(
+ "bytes_received"
+ )
+ assert (
+ len(crawler.spider.meta["response"].body)
+ < crawler.spider.full_response_length
+ )
+
+ @coroutine_test
+ async def test_bytes_received_stop_download_errback(
+ self, mockserver: MockServer
+ ) -> None:
+ # copy of TestCrawl.test_bytes_received_stop_download_errback()
+ crawler = get_crawler(BytesReceivedErrbackSpider, self.settings_dict)
+ await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure)
+ assert isinstance(crawler.spider, BytesReceivedErrbackSpider)
+ assert crawler.spider.meta.get("response") is None
+ assert isinstance(crawler.spider.meta["failure"], Failure)
+ assert isinstance(crawler.spider.meta["failure"].value, StopDownload)
+ assert isinstance(crawler.spider.meta["failure"].value.response, Response)
+ assert crawler.spider.meta[
+ "failure"
+ ].value.response.body == crawler.spider.meta.get("bytes_received")
+ assert (
+ len(crawler.spider.meta["failure"].value.response.body)
+ < crawler.spider.full_response_length
+ )
+
+ @coroutine_test
+ async def test_headers_received_stop_download_callback(
+ self, mockserver: MockServer
+ ) -> None:
+ # copy of TestCrawl.test_headers_received_stop_download_callback()
+ crawler = get_crawler(HeadersReceivedCallbackSpider, self.settings_dict)
+ await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure)
+ assert isinstance(crawler.spider, HeadersReceivedCallbackSpider)
+ assert crawler.spider.meta.get("failure") is None
+ assert isinstance(crawler.spider.meta["response"], Response)
+ assert crawler.spider.meta["response"].headers == crawler.spider.meta.get(
+ "headers_received"
+ )
+
+ @coroutine_test
+ async def test_headers_received_stop_download_errback(
+ self, mockserver: MockServer
+ ) -> None:
+ # copy of TestCrawl.test_headers_received_stop_download_errback()
+ crawler = get_crawler(HeadersReceivedErrbackSpider, self.settings_dict)
+ await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure)
+ assert isinstance(crawler.spider, HeadersReceivedErrbackSpider)
+ assert crawler.spider.meta.get("response") is None
+ assert isinstance(crawler.spider.meta["failure"], Failure)
+ assert isinstance(crawler.spider.meta["failure"].value, StopDownload)
+ assert isinstance(crawler.spider.meta["failure"].value.response, Response)
+ assert crawler.spider.meta[
+ "failure"
+ ].value.response.headers == crawler.spider.meta.get("headers_received")
+
class TestHttpProxyBase(ABC):
is_secure = False
diff --git a/tox.ini b/tox.ini
index 83cd06561..3be564502 100644
--- a/tox.ini
+++ b/tox.ini
@@ -48,6 +48,7 @@ deps =
types-Pygments==2.19.0.20250809
botocore-stubs==1.40.59
boto3-stubs[s3]==1.40.59
+ httpx==0.28.1
itemadapter==0.12.2
Protego==0.5.0
w3lib==2.3.1
@@ -134,6 +135,7 @@ deps =
brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests
brotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests
google-cloud-storage
+ httpx
ipython
robotexclusionrulesparser
uvloop; platform_system != "Windows" and implementation_name != "pypy"
@@ -149,6 +151,7 @@ deps =
brotli==1.2.0; implementation_name != "pypy"
brotlicffi==1.2.0.0; implementation_name == "pypy"
google-cloud-storage==1.29.0
+ httpx==0.26.0
ipython==7.1.0
robotexclusionrulesparser==1.6.2
uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"