Scrapy, a fast high-level web crawling & scraping framework for Python.
Use httpx2 as a preferred alternative to httpx. (#7762)
Andrey Rakhmatullin56dee203e97a
diff --git a/conftest.py b/conftest.pyindex 1bfb002cf..b8a9dc19e 100644--- a/conftest.py+++ b/conftest.py@@ -50,7 +50,7 @@ if not H2_ENABLED:))-if not find_spec("httpx"):+if find_spec("httpx2") is None and find_spec("httpx") is None:collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rstindex 83e40656b..63c217e93 100644--- a/docs/topics/asyncio.rst+++ b/docs/topics/asyncio.rst@@ -153,8 +153,8 @@ Using Scrapy without a Twisted reactor.. note:: As the Twisted download handlers cannot be used without a reactor,the default download handler in this mode is:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. You- will need to additionally install the ``httpx`` library to use it, unless- you switch to some different handler.+ will need to additionally install the :ref:`httpx <extras>` extra to use+ it, unless you switch to some different handler.It's possible to use Scrapy without installing a Twisted reactor at all, bysetting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In thisdiff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rstindex 33cbf185e..34ab4f105 100644--- a/docs/topics/download-handlers.rst+++ b/docs/topics/download-handlers.rst@@ -291,9 +291,9 @@ HttpxDownloadHandlerThis handler supports ``http://host/path`` and ``https://host/path`` URLs anduses the HTTP/1.1 or HTTP/2 protocol for them.-It's implemented using the httpx_ library.+It's implemented using the httpx2_ library.-.. _httpx: https://www.python-httpx.org/+.. _httpx2: https://httpx2.pydantic.dev/If you want to use this handler you need to replace the default ones for the``http`` and ``https`` schemes:diff --git a/pyproject.toml b/pyproject.tomlindex 81d697c12..4220b12ed 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -67,7 +67,7 @@ brotli = ["brotlicffi>=1.2.0.0; implementation_name == 'pypy'",]gcs = ["google-cloud-storage>=1.29.0"]-httpx = ["httpx[http2,socks]>=0.27.1"]+httpx = ["httpx2[http2,socks]>=2.0.0"]images = ["Pillow>=8.3.2"]ipython = ["ipython>=7.1.0"]ptpython = ["ptpython>=2.0.1"]diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.pyindex 9b54b44d8..d5a4e9fcd 100644--- a/scrapy/core/downloader/handlers/_httpx.py+++ b/scrapy/core/downloader/handlers/_httpx.py@@ -30,7 +30,7 @@ from ._base_streaming import BaseStreamingDownloadHandler, _BaseResponseArgsif TYPE_CHECKING:from collections.abc import AsyncIterator- from httpcore import AsyncNetworkStream+ from httpcore2 import AsyncNetworkStreamfrom scrapy import Requestfrom scrapy.crawler import Crawler@@ -39,8 +39,11 @@ if TYPE_CHECKING:HAS_SOCKS = HAS_HTTP2 = Falsetry:- import httpx-except ImportError:+ try:+ import httpx2 as httpx+ except ImportError: # pragma: no cover+ import httpx # type: ignore[import-not-found,no-redef]+except ImportError: # pragma: no coverhttpx = None # type: ignore[assignment]else:# a small hack to avoid importing these optional extras unconditionally@@ -84,7 +87,7 @@ class HttpxDownloadHandler(_Base):self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED")if self._enable_h2 and not HAS_HTTP2: # pragma: no coverraise NotConfigured(- f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed."+ f"HTTP/2 support in {type(self).__name__} requires the 'httpx2[http2]' extra to be installed.")self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)self._bind_host: str | None = self._get_bind_address_host()@@ -96,7 +99,7 @@ class HttpxDownloadHandler(_Base):)self._default_client: httpx.AsyncClient = self._make_client()- # httpx doesn't support per-request proxies: https://github.com/encode/httpx/discussions/3183,+ # httpx2 doesn't support per-request proxies: https://github.com/pydantic/httpx2/issues/818,# so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed.self._proxy_clients: dict[str, httpx.AsyncClient] = {}@@ -104,7 +107,7 @@ class HttpxDownloadHandler(_Base):def _check_deps_installed() -> None:if httpx is None: # pragma: no coverraise NotConfigured(- "HttpxDownloadHandler requires the httpx library to be installed."+ "HttpxDownloadHandler requires the httpx2 library to be installed.")def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient:@@ -128,7 +131,7 @@ class HttpxDownloadHandler(_Base):proxy=proxy,),)- # https://github.com/encode/httpx/discussions/1566+ # https://github.com/pydantic/httpx2/issues/368for header_name in ("accept", "accept-encoding", "user-agent"):client.headers.pop(header_name, None)return client@@ -149,7 +152,7 @@ class HttpxDownloadHandler(_Base):proxy = self._extract_proxy_url_with_creds(request)if proxy and proxy.startswith("socks") and not HAS_SOCKS: # pragma: no coverraise ValueError(- f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed."+ f"SOCKS proxy support in {type(self).__name__} requires the 'httpx2[socks]' extra to be installed.")client = self._get_client(proxy)headers = self._request_headers(request).to_tuple_list()@@ -175,7 +178,7 @@ class HttpxDownloadHandler(_Base):raise DownloadConnectionRefusedError(str(e)) from eexcept httpx.ProxyError as e:raise DownloadConnectionRefusedError(str(e)) from e- except DOWNLOAD_FAILED_EXCEPTIONS as e:+ except DOWNLOAD_FAILED_EXCEPTIONS as e: # pylint: disable=catching-non-exceptionraise DownloadFailedError(str(e)) from e@staticmethod@@ -218,7 +221,7 @@ class HttpxDownloadHandler(_Base):def _log_tls_info(self, response: httpx.Response, request: Request) -> None:network_stream: AsyncNetworkStream = response.extensions["network_stream"]extra_ssl_object = network_stream.get_extra_info("ssl_object")- if isinstance(extra_ssl_object, ssl.SSLObject):+ if isinstance(extra_ssl_object, ssl.SSLObject): # pragma: no branch_log_sslobj_debug_info(extra_ssl_object)async def close(self) -> None:diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pyindex fdacad963..fd50248a7 100644--- a/tests/test_downloader_handler_httpx.py+++ b/tests/test_downloader_handler_httpx.py@@ -3,6 +3,7 @@from __future__ import annotationsimport sys+from importlib.util import find_specfrom typing import TYPE_CHECKING, Any, ClassVarimport pytest@@ -37,7 +38,8 @@ if TYPE_CHECKING:pytestmark = pytest.mark.only_asyncio-pytest.importorskip("httpx")+if find_spec("httpx2") is None and find_spec("httpx") is None:+ pytest.skip("Neither httpx2 nor httpx are installed", allow_module_level=True)class HttpxDownloadHandlerMixin:diff --git a/tox.ini b/tox.iniindex b2f3b0db1..d35fe91db 100644--- a/tox.ini+++ b/tox.ini@@ -36,7 +36,7 @@ minversion = 1.7.0deps =attrscoverage >= 7.10.6- httpx+ httpx2pexpect >= 4.8.0pyftpdlib >= 2.0.1pygments@@ -75,7 +75,7 @@ deps =boto3-stubs[s3]==1.43.41botocore-stubs==1.43.14h2==4.3.0- httpx==0.28.1+ httpx2==2.7.0itemadapter==0.13.1ptpython==3.0.32# newer ones require newer Python@@ -136,7 +136,7 @@ deps =Twisted==21.7.0cryptography==37.0.0cssselect==0.9.1- httpx==0.27.1+ httpx2==2.0.0itemadapter==0.1.0lxml==4.6.4parsel==1.5.0@@ -171,7 +171,7 @@ deps =brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware testsbrotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware testsgoogle-cloud-storage- httpx[http2,socks]+ httpx2[http2,socks]ipythonptpython # optional for shell wrapper testsrobotexclusionrulesparser@@ -189,7 +189,7 @@ deps =brotli==1.2.0; implementation_name != "pypy"brotlicffi==1.2.0.0; implementation_name == "pypy"google-cloud-storage==1.29.0- httpx[http2,socks]==0.27.1+ httpx2[http2,socks]==2.0.0ipython==7.1.0ptpython==2.0.1robotexclusionrulesparser==1.6.2