Building AI agents, atomically
Check Unicode host names in their IDNA-encoded form
Juampi2b21389f3569
diff --git a/atomic-forge/tools/firecrawl_search/README.md b/atomic-forge/tools/firecrawl_search/README.mdindex fbd0ecd..598657f 100644--- a/atomic-forge/tools/firecrawl_search/README.md+++ b/atomic-forge/tools/firecrawl_search/README.md@@ -31,7 +31,7 @@ Searches the web or recent news through the [Firecrawl Search](https://www.firec- `include_content` (bool): also return each result page as Markdown (default `False`).### Output Schema-- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name, IPv4 or IPv6 address, a valid port, no whitespace or control characters) are skipped, and badly typed optional fields are left empty.+- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name, IPv4 or IPv6 address, a valid port, no whitespace or control characters; Unicode host names are checked in their IDNA-encoded form) are skipped, and badly typed optional fields are left empty.- `failures`: queries that failed while others returned results, each with `query` and `error` (the HTTP status and the API's error message). Keys are redacted from both as described for `api_key`. They are also logged as warnings.## Usagediff --git a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.pyindex 9086f63..27faded 100644--- a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py+++ b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py@@ -331,6 +331,10 @@ def test_to_item_drops_badly_typed_fields(field: str, value: object, attribute:f"https://{'a' * 63}.com/",f"https://{'a' * 63}.{'b' * 63}.{'c' * 63}.{'d' * 61}/","https://1.2.3.4/",+ "https://stra\u00dfe.de/",+ "https://\u05e9\u05dc\u05d5\u05dd.co.il/",+ "https://\u4f8b\u5b50.\u6d4b\u8bd5/",+ f"https://{'\u00e9' * 20}.com/","https://3com.example.123abc/",],)@@ -399,6 +403,9 @@ def test_to_item_keeps_valid_http_urls(url: str) -> None:{"url": "https://1.2.3/"},{"url": "https://999999/"},{"url": "https://example.\u0661\u0662\u0663/"},+ {"url": "https://a\u05d0.com/a"}, # mixes left-to-right and right-to-left in one label+ {"url": f"https://{'\u00e9' * 63}.com/"}, # 63 characters, but over 63 bytes once IDNA-encoded+ {"url": f"https://{('\u00e9' * 20 + '.') * 10}com/"}, # under 253 characters, over 253 once encoded],)def test_to_item_without_usable_url_is_none(hit: object) -> None:diff --git a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.pyindex a585af4..b6954fd 100644--- a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py+++ b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py@@ -121,10 +121,11 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSearr"https?://(?:[^\s/?#@\\]*@)?(?:[^\s/?#@:\[\]]+|\[[0-9a-f:.]+\])(?::[^\s/?#@\\]*)?(?:[/?#]\S*)?", re.IGNORECASE)- # One DNS label: letters, digits, hyphens and underscores, starting and ending with a letter or digit. No DNS- # lookup happens; this only rejects host names that cannot be valid, such as "example..com" or "a%.com".- HOST_LABEL = r"[^\W_](?:[\w-]{0,61}[^\W_])?"- HOST_NAME_PATTERN = re.compile(rf"(?:{HOST_LABEL}\.)*{HOST_LABEL}")+ # One label of a host name in its ASCII (IDNA) form: letters, digits, hyphens and underscores, starting and ending+ # with a letter or digit. The IDNA codec enforces the 63-byte label limit. No DNS lookup happens; this only+ # rejects host names that cannot be valid, such as "example..com", "a%.com" or "aא.com".+ HOST_LABEL = r"[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?"+ HOST_NAME_PATTERN = re.compile(rf"(?:{HOST_LABEL}\.)*{HOST_LABEL}", re.IGNORECASE)MAX_HOST_NAME_LENGTH = 253def __init__(self, config: FirecrawlSearchToolConfig = FirecrawlSearchToolConfig()):@@ -143,15 +144,28 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear@classmethoddef _valid_host(cls, host: Optional[str]) -> bool:- """True for an IPv6 literal (urlsplit has already validated it), an IPv4 address, or a host name of at most- 253 characters made of labels that start and end with a letter or digit, with one optional trailing dot."""+ """True for an IPv6 literal (urlsplit has already validated it), an IPv4 address, or a host name whose ASCII+ (IDNA) form is at most 253 characters of labels that start and end with a letter or digit, with one optional+ trailing dot. Unicode names are checked after encoding, which is what DNS sees."""name = (host or "").removesuffix(".")if ":" in name:return Trueif name.rsplit(".", 1)[-1].isdigit():# A numeric last label is never a TLD, so only a real IPv4 address is usable (not 256.0.0.1 or 1.2.3).return cls._is_ipv4(name)- return len(name) <= cls.MAX_HOST_NAME_LENGTH and cls.HOST_NAME_PATTERN.fullmatch(name) is not None+ ascii_name = cls._idna(name)+ return ascii_name is not None and cls.HOST_NAME_PATTERN.fullmatch(ascii_name) is not None++ @classmethod+ def _idna(cls, name: str) -> Optional[str]:+ """The ASCII (IDNA) form of a host name, or None when it is longer than 253 characters or cannot be encoded:+ the codec rejects labels that are empty or over 63 bytes once encoded, and Unicode labels that break the IDNA+ rules, such as mixing left-to-right and right-to-left scripts."""+ try:+ ascii_name = name.encode("idna").decode("ascii")+ except UnicodeError:+ return None+ return ascii_name if len(ascii_name) <= cls.MAX_HOST_NAME_LENGTH else None@staticmethoddef _is_ipv4(name: str) -> bool: