Scrapy, a fast high-level web crawling & scraping framework for Python.
Support IDNA-2003-only domains by monkey-patching Twisted IDNA helpers (#7846)
Laerte Pereirad12db698454f
diff --git a/scrapy/core/downloader/_idna_patch.py b/scrapy/core/downloader/_idna_patch.pynew file mode 100644index 000000000..99db90c29--- /dev/null+++ b/scrapy/core/downloader/_idna_patch.py@@ -0,0 +1,77 @@+"""Fallbacks for hostnames that the ``idna`` package rejects but that resolve fine.++Twisted encodes hostnames with ``idna``, which enforces IDNA 2008 and rejects+emoji domains even in punycode (``xn--``) form+(https://github.com/scrapy/scrapy/issues/4330) and domains with underscores+(https://github.com/scrapy/scrapy/issues/3321). The standard library codec+implements IDNA 2003 and handles both, so :func:`_install_twisted_idna_fallbacks`+patches Twisted to fall back to it. Ideally Twisted would do this itself.+"""++from __future__ import annotations++import re++from twisted.internet import _idna++_original_idna_bytes = _idna._idnaBytes+_original_idna_text = _idna._idnaText++# Letters, digits and hyphens, plus the underscore that the idna package+# rejects. The stdlib codec passes any other ASCII through as well, e.g. the+# ":" that Twisted parses out of a bracketless IPv6 netloc, and Twisted must+# keep treating those as invalid hostnames instead of resolving them.+_HOSTNAME = re.compile(rb"[a-zA-Z0-9._\-]+\Z")+++def _safe_hostname_bytes(hostname: str) -> bytes:+ """Punycode *hostname* with IDNA 2003, which accepts emoji labels and passes+ already-encoded (``xn--``) ones through.++ Raises UnicodeError if it cannot be encoded into a hostname at all.+ """+ try:+ encoded = hostname.encode("idna")+ except UnicodeError:+ # The stdlib codec also rejects some ASCII hostnames, e.g. overlong+ # labels; non-ASCII ones raise UnicodeError again here.+ encoded = hostname.encode("ascii")+ if not _HOSTNAME.match(encoded):+ raise UnicodeError(f"invalid hostname: {hostname}")+ return encoded+++def _patched_idna_bytes(text: str) -> bytes:+ try:+ return _original_idna_bytes(text)+ except UnicodeError: # parent class of idna.IDNAError+ return _safe_hostname_bytes(text)+++def _patched_idna_text(octets: bytes) -> str:+ try:+ return _original_idna_text(octets)+ except UnicodeError:+ if not _HOSTNAME.match(octets):+ raise+ return octets.decode("idna")+++def _install_twisted_idna_fallbacks() -> None:+ """Patch ``_idnaBytes()``/``_idnaText()`` wherever Twisted uses them for+ client connections.++ Each module binds them at import time, so every reference needs patching,+ not just ``twisted.internet._idna``. Names that are missing or already+ patched are left alone.+ """+ from twisted.internet import _resolver, _sslverify, endpoints # noqa: PLC0415++ patches = (+ ("_idnaBytes", _original_idna_bytes, _patched_idna_bytes),+ ("_idnaText", _original_idna_text, _patched_idna_text),+ )+ for module in (_idna, _resolver, _sslverify, endpoints):+ for name, original, patched in patches:+ if getattr(module, name, None) is original:+ setattr(module, name, patched)diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex c288ae792..da2174f6f 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -38,6 +38,7 @@ from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IRefrom zope.interface import implementerfrom scrapy import Request, signals+from scrapy.core.downloader._idna_patch import _install_twisted_idna_fallbacksfrom scrapy.core.downloader.contextfactory import _load_context_factory_from_settingsfrom scrapy.exceptions import (DownloadCancelledError,@@ -94,6 +95,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):def __init__(self, crawler: Crawler):if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"):raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")+ _install_twisted_idna_fallbacks()super().__init__(crawler)self._crawler = crawlerdiff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex 055520f2b..654183741 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKINGfrom urllib.parse import urldefragfrom scrapy.core._http2.agent import H2Agent, H2ConnectionPool+from scrapy.core.downloader._idna_patch import _install_twisted_idna_fallbacksfrom scrapy.core.downloader.contextfactory import _load_context_factory_from_settingsfrom scrapy.core.downloader.handlers.base import BaseDownloadHandlerfrom scrapy.exceptions import (@@ -35,6 +36,7 @@ class H2DownloadHandler(BaseDownloadHandler):def __init__(self, crawler: Crawler):if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"):raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")+ _install_twisted_idna_fallbacks()super().__init__(crawler)self._crawler = crawlerdiff --git a/tests/__init__.py b/tests/__init__.pyindex cd52ade58..1e5ae1be7 100644--- a/tests/__init__.py+++ b/tests/__init__.py@@ -27,6 +27,11 @@ except socket.gaierror:NON_EXISTING_RESOLVABLE = False+# Hostnames rejected by the idna package but resolvable in practice: the+# punycode form of an emoji domain (i❤.ws) and a domain with an underscore.+IDNA_REJECTED_HOSTNAMES = ["xn--i-7iq.ws", "foo_bar.example"]++def get_testdata(*paths: str) -> bytes:"""Return test data"""return Path(tests_datadir, *paths).read_bytes()diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex e613fd597..7e46f6c6e 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -6,16 +6,27 @@ from typing import TYPE_CHECKING, castimport OpenSSL.SSLimport pytestfrom pytest_twisted import async_yield_fixture+from twisted.internet.endpoints import HostnameEndpointfrom twisted.internet.protocol import Factoryfrom twisted.internet.protocol import Protocol as TxProtocolfrom twisted.internet.ssl import AcceptableCiphers, optionsForClientTLSfrom twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocolfrom twisted.web import server, static-from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody+from twisted.web.client import (+ URI,+ Agent,+ BrowserLikePolicyForHTTPS,+ _StandardEndpointFactory,+ readBody,+)from twisted.web.client import Response as TxResponsefrom scrapy import Request, Spiderfrom scrapy.core.downloader import Downloader, Slot, tls+from scrapy.core.downloader._idna_patch import (+ _install_twisted_idna_fallbacks,+ _safe_hostname_bytes,+)from scrapy.core.downloader.contextfactory import (_get_creator_context,_load_context_factory_from_settings,@@ -29,6 +40,7 @@ from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytesfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler+from tests import IDNA_REJECTED_HOSTNAMESfrom tests.mockserver.http_resources import PayloadResource, put_childfrom tests.mockserver.utils import ssl_context_factoryfrom tests.utils.cmdline import proc@@ -45,6 +57,13 @@ if TYPE_CHECKING:from tests.mockserver.http import MockServer[email protected](autouse=True, scope="module")+def _twisted_idna_fallbacks() -> None:+ """The download handlers install these when built; the tests below exercise+ the patched Twisted helpers directly, so they need them installed too."""+ _install_twisted_idna_fallbacks()++class TestSlot:def test_repr(self):slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)@@ -172,6 +191,31 @@ class TestContextFactory(TestContextFactoryBase):)factory.creatorForNetloc(b"website.tld", 443)+ @pytest.mark.parametrize("hostname", [h.encode() for h in IDNA_REJECTED_HOSTNAMES])+ def test_idna_rejected_hostname(+ self, factory: _ScrapyClientContextFactory, hostname: bytes+ ) -> None:+ """Hostnames rejected by the idna package, such as punycode emoji+ domains or domains with underscores, should work."""+ creator = factory.creatorForNetloc(hostname, 443)+ assert creator._hostnameBytes == hostname+ assert creator._hostnameASCII == hostname.decode("ascii")+ assert creator._hostnameIsDnsName is True+ conn = creator.clientConnectionForTLS(self._get_dummy_protocol())+ assert conn is not None++ @pytest.mark.parametrize("hostname", [h.encode() for h in IDNA_REJECTED_HOSTNAMES])+ def test_idna_rejected_hostname_verify_certificates(self, hostname: bytes) -> None:+ """Same as test_idna_rejected_hostname() but with certificate+ verification enabled, which uses plain optionsForClientTLS()."""+ crawler = get_crawler(settings_dict={"DOWNLOAD_VERIFY_CERTIFICATES": True})+ factory = cast(+ "_ScrapyClientContextFactory", _load_context_factory_from_settings(crawler)+ )+ creator = factory.creatorForNetloc(hostname, 443)+ assert creator._hostnameBytes == hostname+ assert creator._hostnameASCII == hostname.decode("ascii")+@pytest.mark.parametrize("verify_certs", [False, True])def test_keylog(self,@@ -348,6 +392,88 @@ def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:downloader.close()+class TestSafeHostnameBytes:+ """Tests for the workarounds for hostnames rejected by the idna+ package."""++ @pytest.mark.parametrize(+ ("hostname", "expected"),+ [+ ("example.com", b"example.com"),+ ("xn--i-7iq.ws", b"xn--i-7iq.ws"), # i❤.ws+ ("i❤.ws", b"xn--i-7iq.ws"),+ ("ü.example", b"xn--tda.example"),+ ("foo_bar.example", b"foo_bar.example"),+ # overlong ASCII label, rejected by the stdlib codec+ ("a" * 64 + ".com", b"a" * 64 + b".com"),+ ],+ )+ def test_valid(self, hostname: str, expected: bytes) -> None:+ assert _safe_hostname_bytes(hostname) == expected++ @pytest.mark.parametrize(+ "hostname",+ [+ "❤" * 200 + ".ws",+ # not hostnames at all: Twisted parses ":" out of a bracketless+ # IPv6 netloc on old versions+ ":",+ "[::1]",+ ],+ )+ def test_invalid(self, hostname: str) -> None:+ with pytest.raises(UnicodeError):+ _safe_hostname_bytes(hostname)++ @pytest.mark.parametrize(+ ("host", "expected"),+ [+ ("xn--i-7iq.ws", (False, b"xn--i-7iq.ws", "xn--i-7iq.ws")),+ ("i❤.ws", (False, b"xn--i-7iq.ws", "i❤.ws")),+ ("foo_bar.example", (False, b"foo_bar.example", "foo_bar.example")),+ ("example.com", (False, b"example.com", "example.com")),+ # bytes hostnames are decoded instead of encoded+ (b"xn--i-7iq.ws", (False, b"xn--i-7iq.ws", "i❤.ws")),+ (b"foo_bar.example", (False, b"foo_bar.example", "foo_bar.example")),+ (b"example.com", (False, b"example.com", "example.com")),+ # invalid hostnames must stay invalid, or Twisted resolves them+ # instead of failing early+ (":", (True, b":", ":")),+ ("[::1]", (True, b"[::1]", "[::1]")),+ (b":", (True, b":", ":")),+ ],+ )+ def test_host_as_bytes_and_text(+ self, host: bytes | str, expected: tuple[bool, bytes, str]+ ) -> None:+ """HostnameEndpoint should consider only invalid hostnames invalid."""+ assert HostnameEndpoint._hostAsBytesAndText(host) == expected++[email protected]_reactor+class TestIdnaRejectedEndpoints:+ """Endpoints for hostnames rejected by the idna package, such as emoji+ domains or domains with underscores, should be connectable."""++ @pytest.mark.parametrize("scheme", ["http", "https"])+ @pytest.mark.parametrize("hostname", IDNA_REJECTED_HOSTNAMES)+ def test_endpoint_for_uri(self, scheme: str, hostname: str) -> None:+ from twisted.internet import reactor++ crawler = get_crawler()+ endpoint_factory = _StandardEndpointFactory(+ reactor, _load_context_factory_from_settings(crawler), 10, None+ )+ endpoint = endpoint_factory.endpointForURI(+ URI.fromBytes(f"{scheme}://{hostname}/".encode())+ )+ # https endpoints are wrapped by wrapClientTLS()+ hostname_endpoint = getattr(endpoint, "_wrappedEndpoint", endpoint)+ assert hostname_endpoint._badHostname is False+ assert hostname_endpoint._hostBytes == hostname.encode("ascii")+ assert hostname_endpoint._hostText == hostname++@coroutine_testasync def test_fetch_deprecated_spider_arg():class CustomDownloader(Downloader):diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pyindex d4ea8e734..66a761f06 100644--- a/tests/test_downloader_handler_httpx.py+++ b/tests/test_downloader_handler_httpx.py@@ -9,14 +9,16 @@ from typing import TYPE_CHECKING, Any, ClassVarimport pytestfrom scrapy import Request-from scrapy.core.downloader.handlers._httpx import (+from scrapy.core.downloader.handlers._httpx import ( # type: ignore[attr-defined]HAS_HTTP2,HAS_SOCKS,HttpxDownloadHandler,+ httpx,)from scrapy.exceptions import DownloadFailedErrorfrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawler+from tests import IDNA_REJECTED_HOSTNAMESfrom tests.utils.bases.download_handlers_http import (TestHttpBase,TestHttpProxyBase,@@ -47,6 +49,18 @@ pytestmark = pytest.mark.only_asyncioif find_spec("httpx2") is None and find_spec("httpx") is None:pytest.skip("Neither httpx2 nor httpx are installed", allow_module_level=True)+# httpx2 < 2.4.0 reads URL.host through idna.decode() without display=True when+# building the Host header, which raises for punycode labels that IDNA 2008+# rejects. https://github.com/pydantic/httpx2/pull/1018+# This check can be dropped once the httpx2 requirement is bumped to >= 2.4.0.+try:+ for _hostname in IDNA_REJECTED_HOSTNAMES:+ _ = httpx.URL(f"http://{_hostname}").host+except Exception:+ HTTPX_SUPPORTS_IDNA_REJECTED_HOSTNAMES = False+else:+ HTTPX_SUPPORTS_IDNA_REJECTED_HOSTNAMES = True+class HttpxDownloadHandlerMixin:@property@@ -66,6 +80,7 @@ class HttpxDownloadHandlerMixin:class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):handler_supports_bindaddress_meta = Falsehandler_bad_header_handling = "fail"+ handler_supports_idna_rejected_hostnames = HTTPX_SUPPORTS_IDNA_REJECTED_HOSTNAMES@pytest.mark.skipif(sys.platform == "darwin",@@ -88,6 +103,7 @@ class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):handler_supports_bindaddress_meta = Falsehandler_bad_header_handling = "fail"+ handler_supports_idna_rejected_hostnames = HTTPX_SUPPORTS_IDNA_REJECTED_HOSTNAMEStls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher"@pytest.mark.skip(reason="The check is Twisted-specific")diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.pyindex acf00e791..a2d1d343b 100644--- a/tests/utils/bases/download_handlers_http.py+++ b/tests/utils/bases/download_handlers_http.py@@ -5,12 +5,12 @@ import jsonimport loggingimport osimport re+import socketimport sysfrom abc import ABC, abstractmethodfrom contextlib import asynccontextmanagerfrom http import HTTPStatusfrom ipaddress import IPv4Address-from socket import gethostbynamefrom typing import TYPE_CHECKING, Any, ClassVar, Literalfrom urllib.parse import urlparse@@ -35,9 +35,10 @@ from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponsefrom scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawler+from scrapy.utils.reactor import is_reactor_installedfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler-from tests import NON_EXISTING_RESOLVABLE+from tests import IDNA_REJECTED_HOSTNAMES, NON_EXISTING_RESOLVABLEfrom tests.mockserver.mitm_proxy import wrong_credentialsfrom tests.mockserver.proxy_echo import ProxyEchoMockServerfrom tests.mockserver.simple_https import SimpleMockServer@@ -74,6 +75,9 @@ class TestHttpBase(ABC):# h2.connection.H2Connection.receive_data()), thus closing all streams that# were using it, and we handle this as a normal exception.handler_supports_http2_dataloss: bool = True+ # whether the handler can request hostnames that the idna package rejects+ # (see IDNA_REJECTED_HOSTNAMES)+ handler_supports_idna_rejected_hostnames: bool = True# What the handler does with a bad response header line, e.g. one with no# colon in it:# "skip-bad": the bad line is skipped and the header lines that follow it@@ -407,6 +411,49 @@ class TestHttpBase(ABC):else:assert not request.headers+ @pytest.fixture+ def resolve_to_loopback(self, monkeypatch: pytest.MonkeyPatch) -> Generator[None]:+ """Resolve every hostname to 127.0.0.1 for the duration of a test."""+ real_getaddrinfo = socket.getaddrinfo++ def getaddrinfo(host, port, *args, **kwargs):+ return real_getaddrinfo("127.0.0.1", port, *args, **kwargs)++ # httpx goes through anyio and asyncio, which look socket.getaddrinfo+ # up when resolving.+ monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo)+ previous = None+ if is_reactor_installed():+ # Twisted resolves through a GAIResolver that bound+ # socket.getaddrinfo at import time, so it needs its own instance.+ from twisted.internet import reactor+ from twisted.internet._resolver import GAIResolver # noqa: PLC0415++ previous = reactor.installNameResolver(+ GAIResolver(reactor, getaddrinfo=getaddrinfo)+ )+ try:+ yield+ finally:+ if previous is not None:+ reactor.installNameResolver(previous)++ @pytest.mark.parametrize("hostname", IDNA_REJECTED_HOSTNAMES)+ @coroutine_test+ async def test_idna_rejected_hostname(+ self, hostname: str, mockserver: MockServer, resolve_to_loopback: None+ ) -> None:+ """Hostnames that the idna package rejects, such as punycode emoji+ domains or domains with underscores, should still be downloadable."""+ if not self.handler_supports_idna_rejected_hostnames:+ pytest.skip("This handler cannot request IDNA-rejected hostnames")+ scheme = "https" if self.is_secure else "http"+ port = mockserver.port(is_secure=self.is_secure)+ request = Request(f"{scheme}://{hostname}:{port}/text")+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert response.body == b"Works"+@coroutine_testasync def test_content_length_zero_bodyless_post_request_headers(self, mockserver: MockServer@@ -1330,7 +1377,7 @@ class TestHttpWithCrawlerBase(ABC):assert isinstance(crawler.spider, SingleRequestSpider)ip_address = crawler.spider.meta["responses"][0].ip_addressassert isinstance(ip_address, IPv4Address)- assert str(ip_address) == gethostbyname(expected_netloc)+ assert str(ip_address) == socket.gethostbyname(expected_netloc)@coroutine_testasync def test_bytes_received_stop_download_callback(