Skip to content

Building AI agents, atomically

Encode host names before the numeric-label check so fullwidth dots cannot bypass it

Juampiae9c6514c86c
diff --git a/atomic-forge/tools/firecrawl_search/README.md b/atomic-forge/tools/firecrawl_search/README.md
index 598657f..9910002 100644
--- a/atomic-forge/tools/firecrawl_search/README.md
+++ b/atomic-forge/tools/firecrawl_search/README.md
@@ -31,7 +31,7 @@ Searches the web or recent news through the [Firecrawl Search](https://www.firec
- `include_content` (bool): also return each result page as Markdown (default `False`).
### Output Schema
-- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name, IPv4 or IPv6 address, a valid port, no whitespace or control characters; Unicode host names are checked in their IDNA-encoded form) are skipped, and badly typed optional fields are left empty.
+- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name, IPv4 or IPv6 address, a valid port, no whitespace or control characters; Unicode host names are checked in their IDNA 2003 encoded form, as Python's `idna` codec produces it) are skipped, and badly typed optional fields are left empty.
- `failures`: queries that failed while others returned results, each with `query` and `error` (the HTTP status and the API's error message). Keys are redacted from both as described for `api_key`. They are also logged as warnings.
## Usage
diff --git a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py
index 27faded..4b63493 100644
--- a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py
+++ b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py
@@ -335,6 +335,9 @@ def test_to_item_drops_badly_typed_fields(field: str, value: object, attribute:
"https://\u05e9\u05dc\u05d5\u05dd.co.il/",
"https://\u4f8b\u5b50.\u6d4b\u8bd5/",
f"https://{'\u00e9' * 20}.com/",
+ "https://example\u3002com/",
+ "https://example.com\u3002/",
+ "https://1\uff0e2\uff0e3\uff0e4/",
"https://3com.example.123abc/",
],
)
@@ -406,6 +409,9 @@ def test_to_item_keeps_valid_http_urls(url: str) -> None:
{"url": "https://a\u05d0.com/a"}, # mixes left-to-right and right-to-left in one label
{"url": f"https://{'\u00e9' * 63}.com/"}, # 63 characters, but over 63 bytes once IDNA-encoded
{"url": f"https://{('\u00e9' * 20 + '.') * 10}com/"}, # under 253 characters, over 253 once encoded
+ {"url": "https://256.0.0\uff0e1/"}, # fullwidth dot: encodes to 256.0.0.1
+ {"url": "https://1.2\u30023/"}, # ideographic dot: encodes to 1.2.3
+ {"url": "https://a\uff61999999/"}, # halfwidth ideographic dot: numeric last label
],
)
def test_to_item_without_usable_url_is_none(hit: object) -> None:
diff --git a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py
index b6954fd..f811b50 100644
--- a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py
+++ b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py
@@ -147,22 +147,30 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear
"""True for an IPv6 literal (urlsplit has already validated it), an IPv4 address, or a host name whose ASCII
(IDNA) form is at most 253 characters of labels that start and end with a letter or digit, with one optional
trailing dot. Unicode names are checked after encoding, which is what DNS sees."""
- name = (host or "").removesuffix(".")
+ name = host or ""
if ":" in name:
return True
- if name.rsplit(".", 1)[-1].isdigit():
- # A numeric last label is never a TLD, so only a real IPv4 address is usable (not 256.0.0.1 or 1.2.3).
- return cls._is_ipv4(name)
ascii_name = cls._idna(name)
- return ascii_name is not None and cls.HOST_NAME_PATTERN.fullmatch(ascii_name) is not None
+ if ascii_name is None:
+ return False
+ # A numeric last label is never a TLD, so only a real IPv4 address is usable (not 256.0.0.1 or 1.2.3). Both
+ # forms are checked: encoding turns dots such as "。" into ".", and leaves digits such as "١٢٣" non-ASCII.
+ if cls._numeric_last_label(name.removesuffix("."), ascii_name):
+ return cls._is_ipv4(ascii_name)
+ return cls.HOST_NAME_PATTERN.fullmatch(ascii_name) is not None
+
+ @staticmethod
+ def _numeric_last_label(*names: str) -> bool:
+ return any(name.rsplit(".", 1)[-1].isdigit() for name in names)
@classmethod
def _idna(cls, name: str) -> Optional[str]:
- """The ASCII (IDNA) form of a host name, or None when it is longer than 253 characters or cannot be encoded:
- the codec rejects labels that are empty or over 63 bytes once encoded, and Unicode labels that break the IDNA
- rules, such as mixing left-to-right and right-to-left scripts."""
+ """The ASCII (IDNA 2003, as Python's codec implements it) form of a host name without its trailing dot, or
+ None when it is longer than 253 characters or cannot be encoded: the codec rejects labels that are empty or
+ over 63 bytes once encoded, and Unicode labels that break the IDNA rules, such as mixing left-to-right and
+ right-to-left scripts. Encoding also turns dots such as "。" into "."."""
try:
- ascii_name = name.encode("idna").decode("ascii")
+ ascii_name = name.encode("idna").decode("ascii").removesuffix(".")
except UnicodeError:
return None
return ascii_name if len(ascii_name) <= cls.MAX_HOST_NAME_LENGTH else None