Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support IDNA-2003-only domains by monkey-patching Twisted IDNA helpers (#7846)

Laerte Pereirad12db698454f
diff --git a/scrapy/core/downloader/_idna_patch.py b/scrapy/core/downloader/_idna_patch.py
new file mode 100644
index 000000000..99db90c29
--- /dev/null
+++ b/scrapy/core/downloader/_idna_patch.py
@@ -0,0 +1,77 @@
+"""Fallbacks for hostnames that the ``idna`` package rejects but that resolve fine.
+
+Twisted encodes hostnames with ``idna``, which enforces IDNA 2008 and rejects
+emoji domains even in punycode (``xn--``) form
+(https://github.com/scrapy/scrapy/issues/4330) and domains with underscores
+(https://github.com/scrapy/scrapy/issues/3321). The standard library codec
+implements IDNA 2003 and handles both, so :func:`_install_twisted_idna_fallbacks`
+patches Twisted to fall back to it. Ideally Twisted would do this itself.
+"""
+
+from __future__ import annotations
+
+import re
+
+from twisted.internet import _idna
+
+_original_idna_bytes = _idna._idnaBytes
+_original_idna_text = _idna._idnaText
+
+# Letters, digits and hyphens, plus the underscore that the idna package
+# rejects. The stdlib codec passes any other ASCII through as well, e.g. the
+# ":" that Twisted parses out of a bracketless IPv6 netloc, and Twisted must
+# keep treating those as invalid hostnames instead of resolving them.
+_HOSTNAME = re.compile(rb"[a-zA-Z0-9._\-]+\Z")
+
+
+def _safe_hostname_bytes(hostname: str) -> bytes:
+ """Punycode *hostname* with IDNA 2003, which accepts emoji labels and passes
+ already-encoded (``xn--``) ones through.
+
+ Raises UnicodeError if it cannot be encoded into a hostname at all.
+ """
+ try:
+ encoded = hostname.encode("idna")
+ except UnicodeError:
+ # The stdlib codec also rejects some ASCII hostnames, e.g. overlong
+ # labels; non-ASCII ones raise UnicodeError again here.
+ encoded = hostname.encode("ascii")
+ if not _HOSTNAME.match(encoded):
+ raise UnicodeError(f"invalid hostname: {hostname}")
+ return encoded
+
+
+def _patched_idna_bytes(text: str) -> bytes:
+ try:
+ return _original_idna_bytes(text)
+ except UnicodeError: # parent class of idna.IDNAError
+ return _safe_hostname_bytes(text)
+
+
+def _patched_idna_text(octets: bytes) -> str:
+ try:
+ return _original_idna_text(octets)
+ except UnicodeError:
+ if not _HOSTNAME.match(octets):
+ raise
+ return octets.decode("idna")
+
+
+def _install_twisted_idna_fallbacks() -> None:
+ """Patch ``_idnaBytes()``/``_idnaText()`` wherever Twisted uses them for
+ client connections.
+
+ Each module binds them at import time, so every reference needs patching,
+ not just ``twisted.internet._idna``. Names that are missing or already
+ patched are left alone.
+ """
+ from twisted.internet import _resolver, _sslverify, endpoints # noqa: PLC0415
+
+ patches = (
+ ("_idnaBytes", _original_idna_bytes, _patched_idna_bytes),
+ ("_idnaText", _original_idna_text, _patched_idna_text),
+ )
+ for module in (_idna, _resolver, _sslverify, endpoints):
+ for name, original, patched in patches:
+ if getattr(module, name, None) is original:
+ setattr(module, name, patched)
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index c288ae792..da2174f6f 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -38,6 +38,7 @@ from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IRe
from zope.interface import implementer
from scrapy import Request, signals
+from scrapy.core.downloader._idna_patch import _install_twisted_idna_fallbacks
from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings
from scrapy.exceptions import (
DownloadCancelledError,
@@ -94,6 +95,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
def __init__(self, crawler: Crawler):
if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"):
raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")
+ _install_twisted_idna_fallbacks()
super().__init__(crawler)
self._crawler = crawler
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 055520f2b..654183741 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
from urllib.parse import urldefrag
from scrapy.core._http2.agent import H2Agent, H2ConnectionPool
+from scrapy.core.downloader._idna_patch import _install_twisted_idna_fallbacks
from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.exceptions import (
@@ -35,6 +36,7 @@ class H2DownloadHandler(BaseDownloadHandler):
def __init__(self, crawler: Crawler):
if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"):
raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")
+ _install_twisted_idna_fallbacks()
super().__init__(crawler)
self._crawler = crawler
diff --git a/tests/__init__.py b/tests/__init__.py
index cd52ade58..1e5ae1be7 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -27,6 +27,11 @@ except socket.gaierror:
NON_EXISTING_RESOLVABLE = False
+# Hostnames rejected by the idna package but resolvable in practice: the
+# punycode form of an emoji domain (i❤.ws) and a domain with an underscore.
+IDNA_REJECTED_HOSTNAMES = ["xn--i-7iq.ws", "foo_bar.example"]
+
+
def get_testdata(*paths: str) -> bytes:
"""Return test data"""
return Path(tests_datadir, *paths).read_bytes()
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index e613fd597..7e46f6c6e 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -6,16 +6,27 @@ from typing import TYPE_CHECKING, cast
import OpenSSL.SSL
import pytest
from pytest_twisted import async_yield_fixture
+from twisted.internet.endpoints import HostnameEndpoint
from twisted.internet.protocol import Factory
from twisted.internet.protocol import Protocol as TxProtocol
from twisted.internet.ssl import AcceptableCiphers, optionsForClientTLS
from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol
from twisted.web import server, static
-from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
+from twisted.web.client import (
+ URI,
+ Agent,
+ BrowserLikePolicyForHTTPS,
+ _StandardEndpointFactory,
+ readBody,
+)
from twisted.web.client import Response as TxResponse
from scrapy import Request, Spider
from scrapy.core.downloader import Downloader, Slot, tls
+from scrapy.core.downloader._idna_patch import (
+ _install_twisted_idna_fallbacks,
+ _safe_hostname_bytes,
+)
from scrapy.core.downloader.contextfactory import (
_get_creator_context,
_load_context_factory_from_settings,
@@ -29,6 +40,7 @@ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
+from tests import IDNA_REJECTED_HOSTNAMES
from tests.mockserver.http_resources import PayloadResource, put_child
from tests.mockserver.utils import ssl_context_factory
from tests.utils.cmdline import proc
@@ -45,6 +57,13 @@ if TYPE_CHECKING:
from tests.mockserver.http import MockServer
[email protected](autouse=True, scope="module")
+def _twisted_idna_fallbacks() -> None:
+ """The download handlers install these when built; the tests below exercise
+ the patched Twisted helpers directly, so they need them installed too."""
+ _install_twisted_idna_fallbacks()
+
+
class TestSlot:
def test_repr(self):
slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)
@@ -172,6 +191,31 @@ class TestContextFactory(TestContextFactoryBase):
)
factory.creatorForNetloc(b"website.tld", 443)
+ @pytest.mark.parametrize("hostname", [h.encode() for h in IDNA_REJECTED_HOSTNAMES])
+ def test_idna_rejected_hostname(
+ self, factory: _ScrapyClientContextFactory, hostname: bytes
+ ) -> None:
+ """Hostnames rejected by the idna package, such as punycode emoji
+ domains or domains with underscores, should work."""
+ creator = factory.creatorForNetloc(hostname, 443)
+ assert creator._hostnameBytes == hostname
+ assert creator._hostnameASCII == hostname.decode("ascii")
+ assert creator._hostnameIsDnsName is True
+ conn = creator.clientConnectionForTLS(self._get_dummy_protocol())
+ assert conn is not None
+
+ @pytest.mark.parametrize("hostname", [h.encode() for h in IDNA_REJECTED_HOSTNAMES])
+ def test_idna_rejected_hostname_verify_certificates(self, hostname: bytes) -> None:
+ """Same as test_idna_rejected_hostname() but with certificate
+ verification enabled, which uses plain optionsForClientTLS()."""
+ crawler = get_crawler(settings_dict={"DOWNLOAD_VERIFY_CERTIFICATES": True})
+ factory = cast(
+ "_ScrapyClientContextFactory", _load_context_factory_from_settings(crawler)
+ )
+ creator = factory.creatorForNetloc(hostname, 443)
+ assert creator._hostnameBytes == hostname
+ assert creator._hostnameASCII == hostname.decode("ascii")
+
@pytest.mark.parametrize("verify_certs", [False, True])
def test_keylog(
self,
@@ -348,6 +392,88 @@ def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:
downloader.close()
+class TestSafeHostnameBytes:
+ """Tests for the workarounds for hostnames rejected by the idna
+ package."""
+
+ @pytest.mark.parametrize(
+ ("hostname", "expected"),
+ [
+ ("example.com", b"example.com"),
+ ("xn--i-7iq.ws", b"xn--i-7iq.ws"), # i❤.ws
+ ("i❤.ws", b"xn--i-7iq.ws"),
+ ("ü.example", b"xn--tda.example"),
+ ("foo_bar.example", b"foo_bar.example"),
+ # overlong ASCII label, rejected by the stdlib codec
+ ("a" * 64 + ".com", b"a" * 64 + b".com"),
+ ],
+ )
+ def test_valid(self, hostname: str, expected: bytes) -> None:
+ assert _safe_hostname_bytes(hostname) == expected
+
+ @pytest.mark.parametrize(
+ "hostname",
+ [
+ "❤" * 200 + ".ws",
+ # not hostnames at all: Twisted parses ":" out of a bracketless
+ # IPv6 netloc on old versions
+ ":",
+ "[::1]",
+ ],
+ )
+ def test_invalid(self, hostname: str) -> None:
+ with pytest.raises(UnicodeError):
+ _safe_hostname_bytes(hostname)
+
+ @pytest.mark.parametrize(
+ ("host", "expected"),
+ [
+ ("xn--i-7iq.ws", (False, b"xn--i-7iq.ws", "xn--i-7iq.ws")),
+ ("i❤.ws", (False, b"xn--i-7iq.ws", "i❤.ws")),
+ ("foo_bar.example", (False, b"foo_bar.example", "foo_bar.example")),
+ ("example.com", (False, b"example.com", "example.com")),
+ # bytes hostnames are decoded instead of encoded
+ (b"xn--i-7iq.ws", (False, b"xn--i-7iq.ws", "i❤.ws")),
+ (b"foo_bar.example", (False, b"foo_bar.example", "foo_bar.example")),
+ (b"example.com", (False, b"example.com", "example.com")),
+ # invalid hostnames must stay invalid, or Twisted resolves them
+ # instead of failing early
+ (":", (True, b":", ":")),
+ ("[::1]", (True, b"[::1]", "[::1]")),
+ (b":", (True, b":", ":")),
+ ],
+ )
+ def test_host_as_bytes_and_text(
+ self, host: bytes | str, expected: tuple[bool, bytes, str]
+ ) -> None:
+ """HostnameEndpoint should consider only invalid hostnames invalid."""
+ assert HostnameEndpoint._hostAsBytesAndText(host) == expected
+
+
+class TestIdnaRejectedEndpoints:
+ """Endpoints for hostnames rejected by the idna package, such as emoji
+ domains or domains with underscores, should be connectable."""
+
+ @pytest.mark.parametrize("scheme", ["http", "https"])
+ @pytest.mark.parametrize("hostname", IDNA_REJECTED_HOSTNAMES)
+ def test_endpoint_for_uri(self, scheme: str, hostname: str) -> None:
+ from twisted.internet import reactor
+
+ crawler = get_crawler()
+ endpoint_factory = _StandardEndpointFactory(
+ reactor, _load_context_factory_from_settings(crawler), 10, None
+ )
+ endpoint = endpoint_factory.endpointForURI(
+ URI.fromBytes(f"{scheme}://{hostname}/".encode())
+ )
+ # https endpoints are wrapped by wrapClientTLS()
+ hostname_endpoint = getattr(endpoint, "_wrappedEndpoint", endpoint)
+ assert hostname_endpoint._badHostname is False
+ assert hostname_endpoint._hostBytes == hostname.encode("ascii")
+ assert hostname_endpoint._hostText == hostname
+
+
@coroutine_test
async def test_fetch_deprecated_spider_arg():
class CustomDownloader(Downloader):
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index d4ea8e734..66a761f06 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -9,14 +9,16 @@ from typing import TYPE_CHECKING, Any, ClassVar
import pytest
from scrapy import Request
-from scrapy.core.downloader.handlers._httpx import (
+from scrapy.core.downloader.handlers._httpx import ( # type: ignore[attr-defined]
HAS_HTTP2,
HAS_SOCKS,
HttpxDownloadHandler,
+ httpx,
)
from scrapy.exceptions import DownloadFailedError
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
+from tests import IDNA_REJECTED_HOSTNAMES
from tests.utils.bases.download_handlers_http import (
TestHttpBase,
TestHttpProxyBase,
@@ -47,6 +49,18 @@ pytestmark = pytest.mark.only_asyncio
if find_spec("httpx2") is None and find_spec("httpx") is None:
pytest.skip("Neither httpx2 nor httpx are installed", allow_module_level=True)
+# httpx2 < 2.4.0 reads URL.host through idna.decode() without display=True when
+# building the Host header, which raises for punycode labels that IDNA 2008
+# rejects. https://github.com/pydantic/httpx2/pull/1018
+# This check can be dropped once the httpx2 requirement is bumped to >= 2.4.0.
+try:
+ for _hostname in IDNA_REJECTED_HOSTNAMES:
+ _ = httpx.URL(f"http://{_hostname}").host
+except Exception:
+ HTTPX_SUPPORTS_IDNA_REJECTED_HOSTNAMES = False
+else:
+ HTTPX_SUPPORTS_IDNA_REJECTED_HOSTNAMES = True
+
class HttpxDownloadHandlerMixin:
@property
@@ -66,6 +80,7 @@ class HttpxDownloadHandlerMixin:
class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):
handler_supports_bindaddress_meta = False
handler_bad_header_handling = "fail"
+ handler_supports_idna_rejected_hostnames = HTTPX_SUPPORTS_IDNA_REJECTED_HOSTNAMES
@pytest.mark.skipif(
sys.platform == "darwin",
@@ -88,6 +103,7 @@ class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):
class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):
handler_supports_bindaddress_meta = False
handler_bad_header_handling = "fail"
+ handler_supports_idna_rejected_hostnames = HTTPX_SUPPORTS_IDNA_REJECTED_HOSTNAMES
tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher"
@pytest.mark.skip(reason="The check is Twisted-specific")
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index acf00e791..a2d1d343b 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -5,12 +5,12 @@ import json
import logging
import os
import re
+import socket
import sys
from abc import ABC, abstractmethod
from contextlib import asynccontextmanager
from http import HTTPStatus
from ipaddress import IPv4Address
-from socket import gethostbyname
from typing import TYPE_CHECKING, Any, ClassVar, Literal
from urllib.parse import urlparse
@@ -35,9 +35,10 @@ from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.reactor import is_reactor_installed
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
-from tests import NON_EXISTING_RESOLVABLE
+from tests import IDNA_REJECTED_HOSTNAMES, NON_EXISTING_RESOLVABLE
from tests.mockserver.mitm_proxy import wrong_credentials
from tests.mockserver.proxy_echo import ProxyEchoMockServer
from tests.mockserver.simple_https import SimpleMockServer
@@ -74,6 +75,9 @@ class TestHttpBase(ABC):
# h2.connection.H2Connection.receive_data()), thus closing all streams that
# were using it, and we handle this as a normal exception.
handler_supports_http2_dataloss: bool = True
+ # whether the handler can request hostnames that the idna package rejects
+ # (see IDNA_REJECTED_HOSTNAMES)
+ handler_supports_idna_rejected_hostnames: bool = True
# What the handler does with a bad response header line, e.g. one with no
# colon in it:
# "skip-bad": the bad line is skipped and the header lines that follow it
@@ -407,6 +411,49 @@ class TestHttpBase(ABC):
else:
assert not request.headers
+ @pytest.fixture
+ def resolve_to_loopback(self, monkeypatch: pytest.MonkeyPatch) -> Generator[None]:
+ """Resolve every hostname to 127.0.0.1 for the duration of a test."""
+ real_getaddrinfo = socket.getaddrinfo
+
+ def getaddrinfo(host, port, *args, **kwargs):
+ return real_getaddrinfo("127.0.0.1", port, *args, **kwargs)
+
+ # httpx goes through anyio and asyncio, which look socket.getaddrinfo
+ # up when resolving.
+ monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo)
+ previous = None
+ if is_reactor_installed():
+ # Twisted resolves through a GAIResolver that bound
+ # socket.getaddrinfo at import time, so it needs its own instance.
+ from twisted.internet import reactor
+ from twisted.internet._resolver import GAIResolver # noqa: PLC0415
+
+ previous = reactor.installNameResolver(
+ GAIResolver(reactor, getaddrinfo=getaddrinfo)
+ )
+ try:
+ yield
+ finally:
+ if previous is not None:
+ reactor.installNameResolver(previous)
+
+ @pytest.mark.parametrize("hostname", IDNA_REJECTED_HOSTNAMES)
+ @coroutine_test
+ async def test_idna_rejected_hostname(
+ self, hostname: str, mockserver: MockServer, resolve_to_loopback: None
+ ) -> None:
+ """Hostnames that the idna package rejects, such as punycode emoji
+ domains or domains with underscores, should still be downloadable."""
+ if not self.handler_supports_idna_rejected_hostnames:
+ pytest.skip("This handler cannot request IDNA-rejected hostnames")
+ scheme = "https" if self.is_secure else "http"
+ port = mockserver.port(is_secure=self.is_secure)
+ request = Request(f"{scheme}://{hostname}:{port}/text")
+ async with self.get_dh() as download_handler:
+ response = await download_handler.download_request(request)
+ assert response.body == b"Works"
+
@coroutine_test
async def test_content_length_zero_bodyless_post_request_headers(
self, mockserver: MockServer
@@ -1330,7 +1377,7 @@ class TestHttpWithCrawlerBase(ABC):
assert isinstance(crawler.spider, SingleRequestSpider)
ip_address = crawler.spider.meta["responses"][0].ip_address
assert isinstance(ip_address, IPv4Address)
- assert str(ip_address) == gethostbyname(expected_netloc)
+ assert str(ip_address) == socket.gethostbyname(expected_netloc)
@coroutine_test
async def test_bytes_received_stop_download_callback(