Building AI agents, atomically
Keep @ and backslash out of the port; accept numeric hosts only as IPv4; cap host names at 253 characters
Juampi9bb769e2a3e4
diff --git a/atomic-forge/tools/firecrawl_search/README.md b/atomic-forge/tools/firecrawl_search/README.mdindex 11d8710..fbd0ecd 100644--- a/atomic-forge/tools/firecrawl_search/README.md+++ b/atomic-forge/tools/firecrawl_search/README.md@@ -31,7 +31,7 @@ Searches the web or recent news through the [Firecrawl Search](https://www.firec- `include_content` (bool): also return each result page as Markdown (default `False`).### Output Schema-- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name or IPv6 address, a valid port, no whitespace or control characters) are skipped, and badly typed optional fields are left empty.+- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name, IPv4 or IPv6 address, a valid port, no whitespace or control characters) are skipped, and badly typed optional fields are left empty.- `failures`: queries that failed while others returned results, each with `query` and `error` (the HTTP status and the API's error message). Keys are redacted from both as described for `api_key`. They are also logged as warnings.## Usagediff --git a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.pyindex 6314df2..62ed9aa 100644--- a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py+++ b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py@@ -329,6 +329,9 @@ def test_to_item_drops_badly_typed_fields(field: str, value: object, attribute:"https://example.com./","https://my_host.example.com/",f"https://{'a' * 63}.com/",+ f"https://{'a' * 63}.{'b' * 63}.{'c' * 63}.{'d' * 61}/",+ "https://1.2.3.4/",+ "https://3com.example.123abc/",],)def test_to_item_keeps_valid_http_urls(url: str) -> None:@@ -387,6 +390,14 @@ def test_to_item_keeps_valid_http_urls(url: str) -> None:{"url": "https://-a.com/"},{"url": "https://a-.com/"},{"url": f"https://{'a' * 64}.com/"},+ {"url": "https://example.com../"},+ {"url": f"https://{('a' * 63 + '.') * 4}com/"},+ {"url": "https://a.com:1\\@b.com/"},+ {"url": "https://a.com:x@/"},+ {"url": "https://256.0.0.1/"},+ {"url": "https://1.2.3/"},+ {"url": "https://999999/"},+ {"url": "https://example.\u0661\u0662\u0663/"},],)def test_to_item_without_usable_url_is_none(hit: object) -> None:diff --git a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.pyindex c0cab88..a585af4 100644--- a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py+++ b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py@@ -1,4 +1,5 @@import asyncio+import ipaddressimport loggingimport osimport re@@ -112,17 +113,19 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear# when it is long enough that replacing it cannot mangle ordinary words.KEY_PATTERN = re.compile(r"\bfc-[0-9a-f]{32}\b", re.IGNORECASE)MIN_REDACTED_KEY_LENGTH = 8- # Shape of a usable result URL: http(s), optional userinfo without a backslash (parsers disagree on what host- # "a.com\@b.com" means), a host or IPv6 literal, an optional port, and no whitespace anywhere. urlsplit then- # validates the port and the IPv6 literal, and _valid_host the host name.+ # Shape of a usable result URL: http(s), optional userinfo, a host or IPv6 literal, an optional port, and no+ # whitespace anywhere. No backslash or extra "@" before the path, since parsers disagree on what host+ # "a.com\@b.com" or "a.com:1\@b.com" means. urlsplit then validates the port and the IPv6 literal, and+ # _valid_host the host name.RESULT_URL_PATTERN = re.compile(- r"https?://(?:[^\s/?#@\\]*@)?(?:[^\s/?#@:\[\]]+|\[[0-9a-f:.]+\])(?::[^\s/?#]*)?(?:[/?#]\S*)?", re.IGNORECASE+ r"https?://(?:[^\s/?#@\\]*@)?(?:[^\s/?#@:\[\]]+|\[[0-9a-f:.]+\])(?::[^\s/?#@\\]*)?(?:[/?#]\S*)?", re.IGNORECASE)# One DNS label: letters, digits, hyphens and underscores, starting and ending with a letter or digit. No DNS# lookup happens; this only rejects host names that cannot be valid, such as "example..com" or "a%.com".HOST_LABEL = r"[^\W_](?:[\w-]{0,61}[^\W_])?"- HOST_NAME_PATTERN = re.compile(rf"(?:{HOST_LABEL}\.)*{HOST_LABEL}\.?")+ HOST_NAME_PATTERN = re.compile(rf"(?:{HOST_LABEL}\.)*{HOST_LABEL}")+ MAX_HOST_NAME_LENGTH = 253def __init__(self, config: FirecrawlSearchToolConfig = FirecrawlSearchToolConfig()):super().__init__(config)@@ -140,9 +143,23 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear@classmethoddef _valid_host(cls, host: Optional[str]) -> bool:- """True for an IPv6 literal (urlsplit has already validated it) or a host name made of non-empty labels of- letters, digits, hyphens and underscores that start and end with a letter or digit."""- return host is not None and (":" in host or cls.HOST_NAME_PATTERN.fullmatch(host) is not None)+ """True for an IPv6 literal (urlsplit has already validated it), an IPv4 address, or a host name of at most+ 253 characters made of labels that start and end with a letter or digit, with one optional trailing dot."""+ name = (host or "").removesuffix(".")+ if ":" in name:+ return True+ if name.rsplit(".", 1)[-1].isdigit():+ # A numeric last label is never a TLD, so only a real IPv4 address is usable (not 256.0.0.1 or 1.2.3).+ return cls._is_ipv4(name)+ return len(name) <= cls.MAX_HOST_NAME_LENGTH and cls.HOST_NAME_PATTERN.fullmatch(name) is not None++ @staticmethod+ def _is_ipv4(name: str) -> bool:+ try:+ ipaddress.IPv4Address(name)+ except ValueError:+ return False+ return True@classmethoddef _usable_url(cls, url: str) -> bool: