Building AI agents, atomically
Encode host names before the numeric-label check so fullwidth dots cannot bypass it
Juampiae9c6514c86c
diff --git a/atomic-forge/tools/firecrawl_search/README.md b/atomic-forge/tools/firecrawl_search/README.mdindex 598657f..9910002 100644--- a/atomic-forge/tools/firecrawl_search/README.md+++ b/atomic-forge/tools/firecrawl_search/README.md@@ -31,7 +31,7 @@ Searches the web or recent news through the [Firecrawl Search](https://www.firec- `include_content` (bool): also return each result page as Markdown (default `False`).### Output Schema-- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name, IPv4 or IPv6 address, a valid port, no whitespace or control characters; Unicode host names are checked in their IDNA-encoded form) are skipped, and badly typed optional fields are left empty.+- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name, IPv4 or IPv6 address, a valid port, no whitespace or control characters; Unicode host names are checked in their IDNA 2003 encoded form, as Python's `idna` codec produces it) are skipped, and badly typed optional fields are left empty.- `failures`: queries that failed while others returned results, each with `query` and `error` (the HTTP status and the API's error message). Keys are redacted from both as described for `api_key`. They are also logged as warnings.## Usagediff --git a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.pyindex 27faded..4b63493 100644--- a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py+++ b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py@@ -335,6 +335,9 @@ def test_to_item_drops_badly_typed_fields(field: str, value: object, attribute:"https://\u05e9\u05dc\u05d5\u05dd.co.il/","https://\u4f8b\u5b50.\u6d4b\u8bd5/",f"https://{'\u00e9' * 20}.com/",+ "https://example\u3002com/",+ "https://example.com\u3002/",+ "https://1\uff0e2\uff0e3\uff0e4/","https://3com.example.123abc/",],)@@ -406,6 +409,9 @@ def test_to_item_keeps_valid_http_urls(url: str) -> None:{"url": "https://a\u05d0.com/a"}, # mixes left-to-right and right-to-left in one label{"url": f"https://{'\u00e9' * 63}.com/"}, # 63 characters, but over 63 bytes once IDNA-encoded{"url": f"https://{('\u00e9' * 20 + '.') * 10}com/"}, # under 253 characters, over 253 once encoded+ {"url": "https://256.0.0\uff0e1/"}, # fullwidth dot: encodes to 256.0.0.1+ {"url": "https://1.2\u30023/"}, # ideographic dot: encodes to 1.2.3+ {"url": "https://a\uff61999999/"}, # halfwidth ideographic dot: numeric last label],)def test_to_item_without_usable_url_is_none(hit: object) -> None:diff --git a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.pyindex b6954fd..f811b50 100644--- a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py+++ b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py@@ -147,22 +147,30 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear"""True for an IPv6 literal (urlsplit has already validated it), an IPv4 address, or a host name whose ASCII(IDNA) form is at most 253 characters of labels that start and end with a letter or digit, with one optionaltrailing dot. Unicode names are checked after encoding, which is what DNS sees."""- name = (host or "").removesuffix(".")+ name = host or ""if ":" in name:return True- if name.rsplit(".", 1)[-1].isdigit():- # A numeric last label is never a TLD, so only a real IPv4 address is usable (not 256.0.0.1 or 1.2.3).- return cls._is_ipv4(name)ascii_name = cls._idna(name)- return ascii_name is not None and cls.HOST_NAME_PATTERN.fullmatch(ascii_name) is not None+ if ascii_name is None:+ return False+ # A numeric last label is never a TLD, so only a real IPv4 address is usable (not 256.0.0.1 or 1.2.3). Both+ # forms are checked: encoding turns dots such as "。" into ".", and leaves digits such as "١٢٣" non-ASCII.+ if cls._numeric_last_label(name.removesuffix("."), ascii_name):+ return cls._is_ipv4(ascii_name)+ return cls.HOST_NAME_PATTERN.fullmatch(ascii_name) is not None++ @staticmethod+ def _numeric_last_label(*names: str) -> bool:+ return any(name.rsplit(".", 1)[-1].isdigit() for name in names)@classmethoddef _idna(cls, name: str) -> Optional[str]:- """The ASCII (IDNA) form of a host name, or None when it is longer than 253 characters or cannot be encoded:- the codec rejects labels that are empty or over 63 bytes once encoded, and Unicode labels that break the IDNA- rules, such as mixing left-to-right and right-to-left scripts."""+ """The ASCII (IDNA 2003, as Python's codec implements it) form of a host name without its trailing dot, or+ None when it is longer than 253 characters or cannot be encoded: the codec rejects labels that are empty or+ over 63 bytes once encoded, and Unicode labels that break the IDNA rules, such as mixing left-to-right and+ right-to-left scripts. Encoding also turns dots such as "。" into "."."""try:- ascii_name = name.encode("idna").decode("ascii")+ ascii_name = name.encode("idna").decode("ascii").removesuffix(".")except UnicodeError:return Nonereturn ascii_name if len(ascii_name) <= cls.MAX_HOST_NAME_LENGTH else None