Skip to content

Building AI agents, atomically

Keep @ and backslash out of the port; accept numeric hosts only as IPv4; cap host names at 253 characters

Juampi9bb769e2a3e4
diff --git a/atomic-forge/tools/firecrawl_search/README.md b/atomic-forge/tools/firecrawl_search/README.md
index 11d8710..fbd0ecd 100644
--- a/atomic-forge/tools/firecrawl_search/README.md
+++ b/atomic-forge/tools/firecrawl_search/README.md
@@ -31,7 +31,7 @@ Searches the web or recent news through the [Firecrawl Search](https://www.firec
- `include_content` (bool): also return each result page as Markdown (default `False`).
### Output Schema
-- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name or IPv6 address, a valid port, no whitespace or control characters) are skipped, and badly typed optional fields are left empty.
+- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name, IPv4 or IPv6 address, a valid port, no whitespace or control characters) are skipped, and badly typed optional fields are left empty.
- `failures`: queries that failed while others returned results, each with `query` and `error` (the HTTP status and the API's error message). Keys are redacted from both as described for `api_key`. They are also logged as warnings.
## Usage
diff --git a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py
index 6314df2..62ed9aa 100644
--- a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py
+++ b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py
@@ -329,6 +329,9 @@ def test_to_item_drops_badly_typed_fields(field: str, value: object, attribute:
"https://example.com./",
"https://my_host.example.com/",
f"https://{'a' * 63}.com/",
+ f"https://{'a' * 63}.{'b' * 63}.{'c' * 63}.{'d' * 61}/",
+ "https://1.2.3.4/",
+ "https://3com.example.123abc/",
],
)
def test_to_item_keeps_valid_http_urls(url: str) -> None:
@@ -387,6 +390,14 @@ def test_to_item_keeps_valid_http_urls(url: str) -> None:
{"url": "https://-a.com/"},
{"url": "https://a-.com/"},
{"url": f"https://{'a' * 64}.com/"},
+ {"url": "https://example.com../"},
+ {"url": f"https://{('a' * 63 + '.') * 4}com/"},
+ {"url": "https://a.com:1\\@b.com/"},
+ {"url": "https://a.com:x@/"},
+ {"url": "https://256.0.0.1/"},
+ {"url": "https://1.2.3/"},
+ {"url": "https://999999/"},
+ {"url": "https://example.\u0661\u0662\u0663/"},
],
)
def test_to_item_without_usable_url_is_none(hit: object) -> None:
diff --git a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py
index c0cab88..a585af4 100644
--- a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py
+++ b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py
@@ -1,4 +1,5 @@
import asyncio
+import ipaddress
import logging
import os
import re
@@ -112,17 +113,19 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear
# when it is long enough that replacing it cannot mangle ordinary words.
KEY_PATTERN = re.compile(r"\bfc-[0-9a-f]{32}\b", re.IGNORECASE)
MIN_REDACTED_KEY_LENGTH = 8
- # Shape of a usable result URL: http(s), optional userinfo without a backslash (parsers disagree on what host
- # "a.com\@b.com" means), a host or IPv6 literal, an optional port, and no whitespace anywhere. urlsplit then
- # validates the port and the IPv6 literal, and _valid_host the host name.
+ # Shape of a usable result URL: http(s), optional userinfo, a host or IPv6 literal, an optional port, and no
+ # whitespace anywhere. No backslash or extra "@" before the path, since parsers disagree on what host
+ # "a.com\@b.com" or "a.com:1\@b.com" means. urlsplit then validates the port and the IPv6 literal, and
+ # _valid_host the host name.
RESULT_URL_PATTERN = re.compile(
- r"https?://(?:[^\s/?#@\\]*@)?(?:[^\s/?#@:\[\]]+|\[[0-9a-f:.]+\])(?::[^\s/?#]*)?(?:[/?#]\S*)?", re.IGNORECASE
+ r"https?://(?:[^\s/?#@\\]*@)?(?:[^\s/?#@:\[\]]+|\[[0-9a-f:.]+\])(?::[^\s/?#@\\]*)?(?:[/?#]\S*)?", re.IGNORECASE
)
# One DNS label: letters, digits, hyphens and underscores, starting and ending with a letter or digit. No DNS
# lookup happens; this only rejects host names that cannot be valid, such as "example..com" or "a%.com".
HOST_LABEL = r"[^\W_](?:[\w-]{0,61}[^\W_])?"
- HOST_NAME_PATTERN = re.compile(rf"(?:{HOST_LABEL}\.)*{HOST_LABEL}\.?")
+ HOST_NAME_PATTERN = re.compile(rf"(?:{HOST_LABEL}\.)*{HOST_LABEL}")
+ MAX_HOST_NAME_LENGTH = 253
def __init__(self, config: FirecrawlSearchToolConfig = FirecrawlSearchToolConfig()):
super().__init__(config)
@@ -140,9 +143,23 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear
@classmethod
def _valid_host(cls, host: Optional[str]) -> bool:
- """True for an IPv6 literal (urlsplit has already validated it) or a host name made of non-empty labels of
- letters, digits, hyphens and underscores that start and end with a letter or digit."""
- return host is not None and (":" in host or cls.HOST_NAME_PATTERN.fullmatch(host) is not None)
+ """True for an IPv6 literal (urlsplit has already validated it), an IPv4 address, or a host name of at most
+ 253 characters made of labels that start and end with a letter or digit, with one optional trailing dot."""
+ name = (host or "").removesuffix(".")
+ if ":" in name:
+ return True
+ if name.rsplit(".", 1)[-1].isdigit():
+ # A numeric last label is never a TLD, so only a real IPv4 address is usable (not 256.0.0.1 or 1.2.3).
+ return cls._is_ipv4(name)
+ return len(name) <= cls.MAX_HOST_NAME_LENGTH and cls.HOST_NAME_PATTERN.fullmatch(name) is not None
+
+ @staticmethod
+ def _is_ipv4(name: str) -> bool:
+ try:
+ ipaddress.IPv4Address(name)
+ except ValueError:
+ return False
+ return True
@classmethod
def _usable_url(cls, url: str) -> bool: