Skip to content

Building AI agents, atomically

Check Unicode host names in their IDNA-encoded form

Juampi2b21389f3569
diff --git a/atomic-forge/tools/firecrawl_search/README.md b/atomic-forge/tools/firecrawl_search/README.md
index fbd0ecd..598657f 100644
--- a/atomic-forge/tools/firecrawl_search/README.md
+++ b/atomic-forge/tools/firecrawl_search/README.md
@@ -31,7 +31,7 @@ Searches the web or recent news through the [Firecrawl Search](https://www.firec
- `include_content` (bool): also return each result page as Markdown (default `False`).
### Output Schema
-- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name, IPv4 or IPv6 address, a valid port, no whitespace or control characters) are skipped, and badly typed optional fields are left empty.
+- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name, IPv4 or IPv6 address, a valid port, no whitespace or control characters; Unicode host names are checked in their IDNA-encoded form) are skipped, and badly typed optional fields are left empty.
- `failures`: queries that failed while others returned results, each with `query` and `error` (the HTTP status and the API's error message). Keys are redacted from both as described for `api_key`. They are also logged as warnings.
## Usage
diff --git a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py
index 9086f63..27faded 100644
--- a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py
+++ b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py
@@ -331,6 +331,10 @@ def test_to_item_drops_badly_typed_fields(field: str, value: object, attribute:
f"https://{'a' * 63}.com/",
f"https://{'a' * 63}.{'b' * 63}.{'c' * 63}.{'d' * 61}/",
"https://1.2.3.4/",
+ "https://stra\u00dfe.de/",
+ "https://\u05e9\u05dc\u05d5\u05dd.co.il/",
+ "https://\u4f8b\u5b50.\u6d4b\u8bd5/",
+ f"https://{'\u00e9' * 20}.com/",
"https://3com.example.123abc/",
],
)
@@ -399,6 +403,9 @@ def test_to_item_keeps_valid_http_urls(url: str) -> None:
{"url": "https://1.2.3/"},
{"url": "https://999999/"},
{"url": "https://example.\u0661\u0662\u0663/"},
+ {"url": "https://a\u05d0.com/a"}, # mixes left-to-right and right-to-left in one label
+ {"url": f"https://{'\u00e9' * 63}.com/"}, # 63 characters, but over 63 bytes once IDNA-encoded
+ {"url": f"https://{('\u00e9' * 20 + '.') * 10}com/"}, # under 253 characters, over 253 once encoded
],
)
def test_to_item_without_usable_url_is_none(hit: object) -> None:
diff --git a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py
index a585af4..b6954fd 100644
--- a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py
+++ b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py
@@ -121,10 +121,11 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear
r"https?://(?:[^\s/?#@\\]*@)?(?:[^\s/?#@:\[\]]+|\[[0-9a-f:.]+\])(?::[^\s/?#@\\]*)?(?:[/?#]\S*)?", re.IGNORECASE
)
- # One DNS label: letters, digits, hyphens and underscores, starting and ending with a letter or digit. No DNS
- # lookup happens; this only rejects host names that cannot be valid, such as "example..com" or "a%.com".
- HOST_LABEL = r"[^\W_](?:[\w-]{0,61}[^\W_])?"
- HOST_NAME_PATTERN = re.compile(rf"(?:{HOST_LABEL}\.)*{HOST_LABEL}")
+ # One label of a host name in its ASCII (IDNA) form: letters, digits, hyphens and underscores, starting and ending
+ # with a letter or digit. The IDNA codec enforces the 63-byte label limit. No DNS lookup happens; this only
+ # rejects host names that cannot be valid, such as "example..com", "a%.com" or "aא.com".
+ HOST_LABEL = r"[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?"
+ HOST_NAME_PATTERN = re.compile(rf"(?:{HOST_LABEL}\.)*{HOST_LABEL}", re.IGNORECASE)
MAX_HOST_NAME_LENGTH = 253
def __init__(self, config: FirecrawlSearchToolConfig = FirecrawlSearchToolConfig()):
@@ -143,15 +144,28 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear
@classmethod
def _valid_host(cls, host: Optional[str]) -> bool:
- """True for an IPv6 literal (urlsplit has already validated it), an IPv4 address, or a host name of at most
- 253 characters made of labels that start and end with a letter or digit, with one optional trailing dot."""
+ """True for an IPv6 literal (urlsplit has already validated it), an IPv4 address, or a host name whose ASCII
+ (IDNA) form is at most 253 characters of labels that start and end with a letter or digit, with one optional
+ trailing dot. Unicode names are checked after encoding, which is what DNS sees."""
name = (host or "").removesuffix(".")
if ":" in name:
return True
if name.rsplit(".", 1)[-1].isdigit():
# A numeric last label is never a TLD, so only a real IPv4 address is usable (not 256.0.0.1 or 1.2.3).
return cls._is_ipv4(name)
- return len(name) <= cls.MAX_HOST_NAME_LENGTH and cls.HOST_NAME_PATTERN.fullmatch(name) is not None
+ ascii_name = cls._idna(name)
+ return ascii_name is not None and cls.HOST_NAME_PATTERN.fullmatch(ascii_name) is not None
+
+ @classmethod
+ def _idna(cls, name: str) -> Optional[str]:
+ """The ASCII (IDNA) form of a host name, or None when it is longer than 253 characters or cannot be encoded:
+ the codec rejects labels that are empty or over 63 bytes once encoded, and Unicode labels that break the IDNA
+ rules, such as mixing left-to-right and right-to-left scripts."""
+ try:
+ ascii_name = name.encode("idna").decode("ascii")
+ except UnicodeError:
+ return None
+ return ascii_name if len(ascii_name) <= cls.MAX_HOST_NAME_LENGTH else None
@staticmethod
def _is_ipv4(name: str) -> bool: