Skip to content

Building AI agents, atomically

Tighten response validation, batch results and key redaction; test at the aiohttp transport

Juampi02336056967b
diff --git a/atomic-forge/tools/firecrawl_search/README.md b/atomic-forge/tools/firecrawl_search/README.md
index 1fbacb9..af7120b 100644
--- a/atomic-forge/tools/firecrawl_search/README.md
+++ b/atomic-forge/tools/firecrawl_search/README.md
@@ -15,7 +15,7 @@ Searches the web or recent news through the [Firecrawl Search](https://www.firec
2. Or copy the `tool/` folder directly into your project.
## Configuration
-- `api_key` (str): Firecrawl API key. Falls back to the `FIRECRAWL_API_KEY` environment variable when empty. Stored as a secret and kept out of reprs, raised errors, and logs.
+- `api_key` (`SecretStr`, a plain string is accepted): Firecrawl API key. Falls back to the `FIRECRAWL_API_KEY` environment variable when empty. It is only sent in the `Authorization` header and is redacted from raised errors and log messages.
- `base_url` (str): API base URL (default `https://api.firecrawl.dev/v2`).
- `location` (str, optional): location to localise results, e.g. `Germany`.
- `time_range` (str, optional): `day`, `week`, `month`, or `year`. Only applies to web results.
@@ -25,16 +25,17 @@ Searches the web or recent news through the [Firecrawl Search](https://www.firec
## Input & Output Structure
### Input Schema
-- `queries` (list[str]): search queries to run. Operators such as `site:`, `"exact phrase"`, `-term`, and `filetype:pdf` are supported.
+- `queries` (list[str], at least one): search queries to run. Operators such as `site:`, `"exact phrase"`, `-term`, and `filetype:pdf` are supported.
- `search_type` (str): `web` or `news`. Default `web`.
- `max_results_per_query` (int): 1-100 (default 5, kept low because each result can carry a full page with `include_content`).
- `include_content` (bool): also return each result page as Markdown (default `False`).
### Output Schema
-A list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set).
+- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a usable URL are skipped, and badly typed optional fields are left empty.
+- `failed_queries`: queries that failed while others returned results. The reason for each is logged as a warning.
## Usage
-If every query fails (for example a wrong API key), `run` raises the error. If only some fail, the others are returned and the failures are logged.
+If no query returns results and at least one failed (for example a wrong API key, no credits, or a malformed response), `run` raises the first error instead of returning an empty list. Redirects are not followed; they are reported as errors.
```python
from tool.firecrawl_search import FirecrawlSearchTool, FirecrawlSearchToolConfig, FirecrawlSearchToolInputSchema
diff --git a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py
index dba29bd..b7fae4f 100644
--- a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py
+++ b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py
@@ -1,13 +1,17 @@
+import asyncio
+import copy
import logging
import os
+import socket
import sys
+import traceback
from contextlib import asynccontextmanager
-from types import SimpleNamespace
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import AsyncMock, patch
import pytest
from aiohttp import web
from aiohttp.test_utils import TestServer
+from pydantic import ValidationError
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
@@ -20,6 +24,8 @@ from tool.firecrawl_search import ( # noqa: E402
)
+DUMMY_KEY = "fc-dummy0secret0key0123456789"
+
WEB_HIT = {
"url": "https://www.python.org/",
"title": "Welcome to Python.org",
@@ -38,14 +44,16 @@ NEWS_HIT = {
CONTENT_HIT = {
"url": "https://example.com/guide",
+ "title": "Agent guide",
"description": "A guide to agents.",
"position": 3,
"markdown": "# Agent guide\n\nBuild agents step by step.",
- "metadata": {"title": "Agent guide | Example", "description": "Metadata description", "statusCode": 200},
+ "metadata": {"title": "Agent guide | Example", "statusCode": 200},
}
-DUMMY_KEY = "fc-dummy-secret-0123456789"
+def ok(source: str, hits: list) -> web.Response:
+ return web.json_response({"success": True, "data": {source: hits}})
@asynccontextmanager
@@ -56,7 +64,8 @@ async def fake_firecrawl(respond):
async def handler(request):
body = await request.json()
seen.append({"headers": dict(request.headers), "body": body})
- return respond(body)
+ response = respond(body)
+ return await response if asyncio.iscoroutine(response) else response
app = web.Application()
app.router.add_post("/v2/search", handler)
@@ -68,327 +77,360 @@ async def fake_firecrawl(respond):
await server.close()
-def _http_tool(base_url: str, **config) -> FirecrawlSearchTool:
+def http_tool(base_url: str, **config) -> FirecrawlSearchTool:
return FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key=DUMMY_KEY, base_url=base_url, **config))
-def _mock_session(status: int, payload: dict, reason: str = "OK") -> MagicMock:
- session = MagicMock()
- response = SimpleNamespace(status=status, reason=reason, json=AsyncMock(return_value=payload))
- session.post.return_value.__aenter__.return_value = response
- return session
+async def search(respond, queries=("q",), **params):
+ """Run one search against a fake server and return (output, requests seen)."""
+ async with fake_firecrawl(respond) as (base_url, seen):
+ out = await http_tool(base_url).run_async(FirecrawlSearchToolInputSchema(queries=list(queries), **params))
+ return out, seen
+
+async def search_error(respond, queries=("q",), **params) -> Exception:
+ with pytest.raises(Exception) as raised:
+ await search(respond, queries, **params)
+ return raised.value
-def tool():
- return FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key="test-key"))
+def leak_surfaces(error: BaseException) -> str:
+ """Every text an error can surface through, including the locals of the tool's frames in its traceback."""
+ stack = traceback.TracebackException.from_exception(error, capture_locals=True).stack
+ tool_frames = [
+ repr(frame.locals) for frame in stack if frame.filename.endswith(os.path.join("tool", "firecrawl_search.py"))
+ ]
+ assert tool_frames, "expected the traceback to pass through the tool"
+ return "\n".join([str(error), repr(error), repr(error.__context__), repr(error.__cause__), *tool_frames])
+
+##################
+# Result parsing #
+##################
def test_to_item_web():
item = FirecrawlSearchTool._to_item(WEB_HIT, "python")
- assert isinstance(item, FirecrawlSearchResultItem)
- assert item.query == "python"
- assert item.title == "Welcome to Python.org"
- assert item.url == "https://www.python.org/"
- assert item.description.startswith("Python is a programming language")
- assert item.position == 1
- assert item.published is None
- assert item.content is None
-
-
-def test_to_item_news():
- """News results carry a snippet and a relative date instead of a description."""
+ assert item == FirecrawlSearchResultItem(
+ query="python",
+ title="Welcome to Python.org",
+ url="https://www.python.org/",
+ description="Python is a programming language that lets you work quickly.",
+ position=1,
+ )
+
+
+def test_to_item_news_uses_snippet_and_date():
item = FirecrawlSearchTool._to_item(NEWS_HIT, "python")
- assert item.description == "The Python core team shipped 3.14 today."
- assert item.published == "2 days ago"
- assert item.position == 2
+ assert (item.description, item.published, item.position) == ("The Python core team shipped 3.14 today.", "2 days ago", 2)
+
+
+def test_to_item_content_and_truncation():
+ assert FirecrawlSearchTool._to_item(CONTENT_HIT, "q").content == CONTENT_HIT["markdown"]
+ assert FirecrawlSearchTool._to_item(CONTENT_HIT, "q", max_content_chars=7).content == "# Agent"
-def test_to_item_with_content():
- """With scrapeOptions each result carries markdown and page metadata."""
- item = FirecrawlSearchTool._to_item(CONTENT_HIT, "agents")
- assert item.title == "Agent guide | Example"
- assert item.description == "A guide to agents."
- assert item.content.startswith("# Agent guide")
+ "field, value, attribute, expected",
+ [
+ ("title", {"nested": True}, "title", "https://example.com/a"),
+ ("title", 5, "title", "https://example.com/a"),
+ ("description", ["a", "b"], "description", None),
+ ("position", "N/A", "position", None),
+ ("position", 1.5, "position", None),
+ ("position", True, "position", None),
+ ("position", -3, "position", None),
+ ("date", 20250101, "published", None),
+ ("markdown", 42, "content", None),
+ ("markdown", {}, "content", None),
+ ],
+)
+def test_to_item_drops_badly_typed_fields(field, value, attribute, expected):
+ """One badly typed field must not discard the result or raise a raw ValidationError."""
+ item = FirecrawlSearchTool._to_item({"url": "https://example.com/a", field: value}, "q")
+ assert getattr(item, attribute) == expected
-def test_to_item_falls_back_to_url_for_title():
- item = FirecrawlSearchTool._to_item({"url": "https://example.com/untitled"}, "q")
- assert item.title == "https://example.com/untitled"
- assert item.description is None
[email protected]("hit", [1, "s", None, [], {"title": "no url"}, {"url": ""}, {"url": 123}])
+def test_to_item_without_usable_url_is_none(hit):
+ assert FirecrawlSearchTool._to_item(hit, "q") is None
-def test_build_body_defaults(tool):
- body = tool._build_body("python", FirecrawlSearchToolInputSchema(queries=["python"]))
- assert body == {"query": "python", "limit": 5, "sources": ["web"]}
+################
+# Request body #
+################
+def test_build_body_defaults():
+ tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key="k"))
+ assert tool._build_body("python", FirecrawlSearchToolInputSchema(queries=["python"])) == {
+ "query": "python",
+ "limit": 5,
+ "sources": ["web"],
+ }
def test_build_body_with_options():
tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key="k", location="Germany", time_range="week"))
params = FirecrawlSearchToolInputSchema(queries=["python"], max_results_per_query=3, include_content=True)
-
- body = tool._build_body("python", params)
-
- assert body["sources"] == ["web"]
- assert body["limit"] == 3
- assert body["location"] == "Germany"
- assert body["tbs"] == "qdr:w"
- assert body["scrapeOptions"] == {"formats": ["markdown"], "onlyMainContent": True}
+ assert tool._build_body("python", params) == {
+ "query": "python",
+ "limit": 3,
+ "sources": ["web"],
+ "location": "Germany",
+ "tbs": "qdr:w",
+ "scrapeOptions": {"formats": ["markdown"], "onlyMainContent": True},
+ }
def test_build_body_skips_time_range_for_news():
"""The API's tbs filter only applies to web results."""
tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key="k", time_range="day"))
-
body = tool._build_body("python", FirecrawlSearchToolInputSchema(queries=["python"], search_type="news"))
+ assert body == {"query": "python", "limit": 5, "sources": ["news"]}
- assert body["sources"] == ["news"]
- assert "tbs" not in body
-
-def test_to_item_truncates_content_and_ignores_bad_metadata():
- hit = {**CONTENT_HIT, "metadata": ["not", "a", "dict"]}
-
- item = FirecrawlSearchTool._to_item(hit, "agents", max_content_chars=7)
-
- assert item.content == "# Agent"
- assert item.title == "https://example.com/guide"
+def test_empty_queries_rejected():
+ with pytest.raises(ValidationError):
+ FirecrawlSearchToolInputSchema(queries=[])
+###################################
+# Round trips over real transport #
+###################################
@pytest.mark.asyncio
-async def test_fetch_builds_request(tool):
- session = _mock_session(200, {"success": True, "data": {"web": [WEB_HIT]}})
- params = FirecrawlSearchToolInputSchema(queries=["python"])
+async def test_web_round_trip():
+ out, seen = await search(lambda body: ok("web", [CONTENT_HIT]), queries=["agents"], include_content=True)
- items = await tool._fetch(session, "python", params)
-
- assert len(items) == 1
- assert items[0].url == "https://www.python.org/"
- call_args = session.post.call_args
- assert call_args[0][0] == "https://api.firecrawl.dev/v2/search"
- assert call_args[1]["json"]["query"] == "python"
+ assert out == FirecrawlSearchToolOutputSchema(results=[FirecrawlSearchTool._to_item(CONTENT_HIT, "agents")])
+ assert seen[0]["headers"]["Authorization"] == f"Bearer {DUMMY_KEY}"
+ assert seen[0]["body"] == {
+ "query": "agents",
+ "limit": 5,
+ "sources": ["web"],
+ "scrapeOptions": {"formats": ["markdown"], "onlyMainContent": True},
+ }
@pytest.mark.asyncio
-async def test_fetch_reads_news_results(tool):
- session = _mock_session(200, {"success": True, "data": {"news": [NEWS_HIT] * 4}})
- params = FirecrawlSearchToolInputSchema(queries=["python"], search_type="news", max_results_per_query=2)
-
- items = await tool._fetch(session, "python", params)
+async def test_news_round_trip_reads_news_key_only():
+ def respond(body):
+ return web.json_response({"success": True, "data": {"web": [WEB_HIT], "news": [NEWS_HIT]}})
- assert len(items) == 2
- assert items[0].published == "2 days ago"
+ out, _ = await search(respond, search_type="news")
+ assert [r.url for r in out.results] == [NEWS_HIT["url"]]
@pytest.mark.asyncio
-async def test_fetch_handles_missing_source_key(tool):
- """The API omits the source key entirely when there are no results."""
- session = _mock_session(200, {"success": True, "data": {}})
-
- items = await tool._fetch(session, "nothing", FirecrawlSearchToolInputSchema(queries=["nothing"]))
-
- assert items == []
+async def test_missing_source_key_is_an_empty_search():
+ """The API omits the source key when there are no results; that is a real empty result."""
+ out, _ = await search(lambda body: web.json_response({"success": True, "data": {}}))
+ assert out == FirecrawlSearchToolOutputSchema(results=[], failed_queries=[])
@pytest.mark.asyncio
-async def test_fetch_skips_hits_without_url(tool):
- session = _mock_session(200, {"success": True, "data": {"web": [WEB_HIT, {"title": "no url"}]}})
-
- items = await tool._fetch(session, "python", FirecrawlSearchToolInputSchema(queries=["python"]))
-
- assert [item.url for item in items] == ["https://www.python.org/"]
+async def test_results_trimmed_to_max_results():
+ hits = [{"url": f"https://example.com/{i}"} for i in range(5)]
+ out, _ = await search(lambda body: ok("web", hits), max_results_per_query=2)
+ assert [r.url for r in out.results] == ["https://example.com/0", "https://example.com/1"]
@pytest.mark.asyncio
-async def test_fetch_raises_on_http_error(tool):
- session = _mock_session(401, {"success": False, "error": "Unauthorized: Invalid token"}, reason="Unauthorized")
-
- with pytest.raises(Exception, match="Firecrawl search failed for 'python': 401 Unauthorized: Invalid token"):
- await tool._fetch(session, "python", FirecrawlSearchToolInputSchema(queries=["python"]))
+async def test_invalid_hits_skipped_valid_ones_kept():
+ hits = [1, "s", None, {"title": "x"}, {"url": ""}, WEB_HIT, {"url": "https://example.com/b", "position": "N/A"}]
+ out, _ = await search(lambda body: ok("web", hits))
+ assert [(r.url, r.position) for r in out.results] == [("https://www.python.org/", 1), ("https://example.com/b", None)]
@pytest.mark.asyncio
-async def test_fetch_raises_when_success_is_false(tool):
- session = _mock_session(200, {"success": False})
-
- with pytest.raises(Exception, match="Firecrawl search failed for 'python': 200 OK"):
- await tool._fetch(session, "python", FirecrawlSearchToolInputSchema(queries=["python"]))
+async def test_all_hits_invalid_raises():
+ error = await search_error(lambda body: ok("web", [1, None, {"title": "x"}, {"url": ""}]))
+ assert str(error) == "Firecrawl search failed for 'q': 200 none of the 4 'web' results had a usable url"
+#######################
+# Response validation #
+#######################
@pytest.mark.asyncio
-async def test_fetch_reports_status_for_non_json_body(tool):
- session = MagicMock()
- response = SimpleNamespace(status=502, reason="Bad Gateway", json=AsyncMock(side_effect=ValueError("not json")))
- session.post.return_value.__aenter__.return_value = response
-
- with pytest.raises(Exception, match="Firecrawl search failed for 'python': 502 Bad Gateway"):
- await tool._fetch(session, "python", FirecrawlSearchToolInputSchema(queries=["python"]))
+ "payload",
+ [
+ {"success": True, "data": None},
+ {"success": True},
+ {"success": True, "data": {"web": {"url": "https://example.com/not-a-list"}}},
+ {"success": True, "data": {"web": None}},
+ {"success": True, "data": [WEB_HIT]},
+ ],
+)
+async def test_malformed_success_response_raises(payload):
+ error = await search_error(lambda body: web.json_response(payload))
+ assert str(error) == "Firecrawl search failed for 'q': 200 unexpected response shape for 'web' results"
@pytest.mark.asyncio
-async def test_fetch_applies_max_content_chars():
- tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key="k", max_content_chars=5))
- session = _mock_session(200, {"success": True, "data": {"web": [CONTENT_HIT, "not a dict"]}})
-
- items = await tool._fetch(session, "agents", FirecrawlSearchToolInputSchema(queries=["agents"], include_content=True))
-
- assert [item.content for item in items] == ["# Age"]
[email protected]("payload", [{"success": "false"}, {"success": "true"}, {"success": 1}, {}])
+async def test_success_must_be_literally_true(payload):
+ error = await search_error(lambda body: web.json_response({**payload, "data": {"web": [WEB_HIT]}}))
+ assert str(error) == "Firecrawl search failed for 'q': 200 OK"
@pytest.mark.asyncio
-async def test_run_async_aggregates_results(tool):
- async def fake_fetch(self, session, query, params):
- return [FirecrawlSearchTool._to_item(WEB_HIT, query), FirecrawlSearchTool._to_item(NEWS_HIT, query)]
-
- with patch.object(FirecrawlSearchTool, "_fetch", fake_fetch):
- out = await tool.run_async(FirecrawlSearchToolInputSchema(queries=["q1", "q2"]))
[email protected]("status, reason", [(201, "Created"), (500, "Internal Server Error")])
+async def test_non_200_with_success_true_raises(status, reason):
+ def respond(body):
+ return web.json_response({"success": True, "data": {"web": [WEB_HIT]}}, status=status)
- assert isinstance(out, FirecrawlSearchToolOutputSchema)
- assert len(out.results) == 4
- assert {r.query for r in out.results} == {"q1", "q2"}
+ error = await search_error(respond)
+ assert str(error) == f"Firecrawl search failed for 'q': {status} {reason}"
@pytest.mark.asyncio
-async def test_run_async_skips_failing_query(tool):
- async def fake_fetch(self, session, query, params):
- if query == "bad":
- raise Exception("rate limited")
- return [FirecrawlSearchTool._to_item(WEB_HIT, query)]
+async def test_provider_error_message_is_reported():
+ error = await search_error(lambda body: web.json_response({"success": False, "error": "Payment required"}, status=402))
+ assert str(error) == "Firecrawl search failed for 'q': 402 Payment required"
- with patch.object(FirecrawlSearchTool, "_fetch", fake_fetch):
- out = await tool.run_async(FirecrawlSearchToolInputSchema(queries=["bad", "good"]))
- assert len(out.results) == 1
- assert out.results[0].query == "good"
+ "response, expected",
+ [
+ (lambda: web.Response(status=502, text="<html>Bad Gateway</html>", content_type="text/html"), "502 Bad Gateway"),
+ (lambda: web.Response(status=200, text=""), "200 OK"),
+ (lambda: web.Response(status=204), "204 No Content"),
+ ],
+)
+async def test_non_json_bodies_raise_with_status(response, expected):
+ error = await search_error(lambda body: response())
+ assert str(error) == f"Firecrawl search failed for 'q': {expected}"
@pytest.mark.asyncio
-async def test_run_async_raises_when_every_query_fails(tool):
- """A bad key or exhausted credits should surface as an error, not as an empty result."""
-
- async def fake_fetch(self, session, query, params):
- raise Exception("Firecrawl search failed for 'q': 401 Unauthorized: Invalid token")
+async def test_redirect_is_not_followed():
+ """A redirect, even to another host, is reported as an error; the query is never re-sent elsewhere."""
+ async with fake_firecrawl(lambda body: ok("web", [WEB_HIT])) as (other_url, other_seen):
+ error = await search_error(lambda body: web.Response(status=307, headers={"Location": f"{other_url}/search"}))
- with patch.object(FirecrawlSearchTool, "_fetch", fake_fetch):
- with pytest.raises(Exception, match="401"):
- await tool.run_async(FirecrawlSearchToolInputSchema(queries=["q1", "q2"]))
+ assert str(error) == "Firecrawl search failed for 'q': 307 Temporary Redirect"
+ assert other_seen == []
+###################
+# Batch semantics #
+###################
@pytest.mark.asyncio
-async def test_http_round_trip():
+async def test_partial_failure_reports_failed_queries(caplog):
def respond(body):
- return web.json_response(
- {"success": True, "data": {"web": [{**CONTENT_HIT, "url": f"https://example.com/{body['query']}"}]}}
- )
+ if body["query"] == "bad":
+ return web.json_response({"success": False, "error": "Rate limit exceeded"}, status=429)
+ return ok("web", [WEB_HIT])
- async with fake_firecrawl(respond) as (base_url, seen):
- out = await _http_tool(base_url).run_async(FirecrawlSearchToolInputSchema(queries=["agents"], include_content=True))
+ with caplog.at_level(logging.WARNING, logger="tool.firecrawl_search"):
+ out, _ = await search(respond, queries=["bad", "good"])
- assert [r.url for r in out.results] == ["https://example.com/agents"]
- assert out.results[0].content.startswith("# Agent guide")
- assert seen[0]["headers"]["Authorization"] == f"Bearer {DUMMY_KEY}"
- assert seen[0]["body"]["scrapeOptions"] == {"formats": ["markdown"], "onlyMainContent": True}
+ assert [r.query for r in out.results] == ["good"]
+ assert out.failed_queries == ["bad"]
+ assert "Firecrawl query 'bad' failed: Firecrawl search failed for 'bad': 429 Rate limit exceeded" in caplog.text
@pytest.mark.asyncio
@pytest.mark.parametrize(
- "payload",
+ "bad, expected",
[
- {"success": True, "data": None},
- {"success": True},
- {"success": True, "data": {"web": {"url": "https://example.com/not-a-list"}}},
- {"success": True, "data": {"web": None}},
- {"success": True, "data": [WEB_HIT]},
+ ({"success": True, "data": None}, "200 unexpected response shape for 'web' results"),
+ ({"success": False, "error": "Rate limit exceeded"}, "200 Rate limit exceeded"),
],
)
-async def test_http_malformed_success_response_raises(payload):
- """A 200 with success=true but no usable result list is an error, not an empty result."""
- async with fake_firecrawl(lambda body: web.json_response(payload)) as (base_url, _):
- with pytest.raises(Exception, match="unexpected response shape for 'web' results"):
- await _http_tool(base_url).run_async(FirecrawlSearchToolInputSchema(queries=["q"]))
-
+async def test_empty_plus_failed_query_raises(bad, expected):
+ """No results anywhere plus a failure must not look like an empty search."""
-async def test_http_non_json_error_page_reports_status():
def respond(body):
- return web.Response(status=502, text="<html>Bad Gateway</html>", content_type="text/html")
+ return web.json_response({"success": True, "data": {}} if body["query"] == "empty" else bad)
- async with fake_firecrawl(respond) as (base_url, _):
- with pytest.raises(Exception, match="Firecrawl search failed for 'q': 502 Bad Gateway"):
- await _http_tool(base_url).run_async(FirecrawlSearchToolInputSchema(queries=["q"]))
+ error = await search_error(respond, queries=["empty", "bad"])
+ assert str(error) == f"Firecrawl search failed for 'bad': {expected}"
@pytest.mark.asyncio
-async def test_http_transport_error_does_not_expose_key(caplog):
- """aiohttp's own errors (here a redirect loop) carry the request headers; the tool must not re-raise them as is."""
-
+async def test_every_query_failing_raises_first_error():
def respond(body):
- return web.Response(status=307, headers={"Location": "/v2/search"})
-
- async with fake_firecrawl(respond) as (base_url, seen):
- with caplog.at_level(logging.WARNING, logger="tool.firecrawl_search"):
- with pytest.raises(Exception) as raised:
- await _http_tool(base_url).run_async(FirecrawlSearchToolInputSchema(queries=["q"]))
+ return web.json_response({"success": False, "error": "Unauthorized"}, status=401)
- assert len(seen) > 1
- assert "Firecrawl search failed for 'q': TooManyRedirects" in str(raised.value)
- assert raised.value.__cause__ is None
- for text in (str(raised.value), repr(raised.value), caplog.text):
- assert DUMMY_KEY not in text
+ error = await search_error(respond, queries=["a", "b"])
+ assert str(error) == "Firecrawl search failed for 'a': 401 Unauthorized"
+####################
+# Transport errors #
+####################
@pytest.mark.asyncio
-async def test_http_dummy_key_kept_out_of_repr_errors_and_logs(caplog):
- """Even when the provider echoes the key back, it never reaches repr, raised errors, or logs."""
+async def test_connection_error_is_wrapped_without_context():
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ port = sock.getsockname()[1]
+ tool = http_tool(f"http://127.0.0.1:{port}/v2")
- def respond(body):
- if body["query"] == "bad":
- return web.json_response({"success": False, "error": f"Invalid token {DUMMY_KEY}"}, status=401)
- return web.json_response({"success": True, "data": {"web": [WEB_HIT]}})
+ with pytest.raises(Exception) as raised:
+ await tool.run_async(FirecrawlSearchToolInputSchema(queries=["q"]))
+
+ assert str(raised.value).startswith("Firecrawl search failed for 'q': request failed: ClientConnectorError")
+ assert raised.value.__context__ is None and raised.value.__cause__ is None
+ assert DUMMY_KEY not in leak_surfaces(raised.value)
- async with fake_firecrawl(respond) as (base_url, _):
- tool = _http_tool(base_url)
- with caplog.at_level(logging.WARNING, logger="tool.firecrawl_search"):
- out = await tool.run_async(FirecrawlSearchToolInputSchema(queries=["bad", "good"]))
+
+async def test_timeout_is_wrapped_with_query():
+ async def slow(body):
+ await asyncio.sleep(1)
+ return ok("web", [WEB_HIT])
+
+ async with fake_firecrawl(slow) as (base_url, _):
+ tool = http_tool(base_url)
+ tool.timeout = 0.05
with pytest.raises(Exception) as raised:
- await tool.run_async(FirecrawlSearchToolInputSchema(queries=["bad"]))
+ await tool.run_async(FirecrawlSearchToolInputSchema(queries=["q"]))
- assert [r.query for r in out.results] == ["good"]
- assert "401 Invalid token [redacted]" in caplog.text
- assert "401 Invalid token [redacted]" in str(raised.value)
- for text in (caplog.text, str(raised.value), repr(raised.value), repr(tool), repr(tool.config), str(tool.config)):
- assert DUMMY_KEY not in text
+ assert str(raised.value).startswith("Firecrawl search failed for 'q': request failed: ")
+ assert "Timeout" in str(raised.value)
+ assert raised.value.__context__ is None
+################
+# Key handling #
+################
@pytest.mark.asyncio
-async def test_run_async_sends_bearer_header(tool):
- captured = {}
+ "echo",
+ [DUMMY_KEY, f"Bearer {DUMMY_KEY}", DUMMY_KEY.upper(), {"token": DUMMY_KEY}, "fc-someotherkey12345"],
+)
+async def test_echoed_key_kept_out_of_errors_and_logs(caplog, echo):
+ """Even when the provider echoes a key back, it never reaches raised errors, tracebacks or logs."""
- class FakeSession:
- def __init__(self, headers=None, timeout=None):
- captured["headers"] = headers
+ def respond(body):
+ if body["query"] == "bad":
+ return web.json_response({"success": False, "error": echo}, status=401)
+ return ok("web", [WEB_HIT])
- async def __aenter__(self):
- return self
+ with caplog.at_level(logging.DEBUG):
+ out, _ = await search(respond, queries=["bad", "good"])
+ error = await search_error(respond, queries=["bad"])
- async def __aexit__(self, *args):
- return False
+ assert out.failed_queries == ["bad"]
+ assert "[redacted]" in str(error) and "[redacted]" in caplog.text
+ for text in (caplog.text, leak_surfaces(error)):
+ assert DUMMY_KEY not in text and DUMMY_KEY.upper() not in text and "fc-someotherkey12345" not in text
- async def fake_fetch(self, session, query, params):
- return []
- with (
- patch("tool.firecrawl_search.aiohttp.ClientSession", FakeSession),
- patch.object(FirecrawlSearchTool, "_fetch", fake_fetch),
- ):
- await tool.run_async(FirecrawlSearchToolInputSchema(queries=["q"]))
+def test_key_kept_out_of_tool_and_config_state():
+ tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key=DUMMY_KEY))
+ surfaces = [repr(tool), repr(tool.config), str(tool.config), tool.config.model_dump_json(), repr(vars(tool))]
+ surfaces.append(repr(vars(copy.copy(tool))))
+ assert all(DUMMY_KEY not in text for text in surfaces)
+ assert tool._api_key.get_secret_value() == DUMMY_KEY
+
- assert captured["headers"]["Authorization"] == "Bearer test-key"
- assert captured["headers"]["Content-Type"] == "application/json"
+def test_short_key_does_not_mangle_messages():
+ tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key="a"))
+ assert tool._redact("Firecrawl search failed") == "Firecrawl search failed"
@pytest.mark.asyncio
-async def test_run_async_requires_api_key(monkeypatch):
+async def test_missing_key_raises(monkeypatch):
monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False)
tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig())
@@ -399,14 +441,14 @@ async def test_run_async_requires_api_key(monkeypatch):
def test_api_key_falls_back_to_environment(monkeypatch):
monkeypatch.setenv("FIRECRAWL_API_KEY", "env-key")
tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig())
- assert tool._api_key == "env-key"
+ assert tool._api_key.get_secret_value() == "env-key"
-def test_run_invokes_run_async(tool):
+def test_run_invokes_run_async():
+ tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key="k"))
sentinel = FirecrawlSearchToolOutputSchema(results=[])
with patch.object(FirecrawlSearchTool, "run_async", AsyncMock(return_value=sentinel)):
- out = tool.run(FirecrawlSearchToolInputSchema(queries=["x"]))
- assert out is sentinel
+ assert tool.run(FirecrawlSearchToolInputSchema(queries=["x"])) is sentinel
if __name__ == "__main__":
diff --git a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py
index efdeed5..c91e57c 100644
--- a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py
+++ b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py
@@ -1,11 +1,12 @@
import asyncio
import logging
import os
+import re
from concurrent.futures import ThreadPoolExecutor
-from typing import List, Literal, Optional, Union
+from typing import Any, List, Literal, Optional
import aiohttp
-from pydantic import Field, SecretStr
+from pydantic import Field, SecretStr, field_validator
from atomic_agents import BaseIOSchema, BaseTool, BaseToolConfig
@@ -25,6 +26,7 @@ class FirecrawlSearchToolInputSchema(BaseIOSchema):
queries: List[str] = Field(
...,
+ min_length=1,
description='Search queries to run. Supports operators such as site:, "exact phrase", -term, and filetype:pdf.',
)
search_type: Literal["web", "news"] = Field(
@@ -52,11 +54,20 @@ class FirecrawlSearchResultItem(BaseIOSchema):
published: Optional[str] = Field(None, description="Publication date string (news only).")
content: Optional[str] = Field(None, description="Page content as Markdown (only when include_content is set).")
+ @field_validator("position", mode="before")
+ @classmethod
+ def _positive_rank_or_none(cls, value: Any) -> Optional[int]:
+ # Ranks come from the provider; anything other than a positive int (bool, float, "N/A") is dropped.
+ return value if type(value) is int and value > 0 else None
+
class FirecrawlSearchToolOutputSchema(BaseIOSchema):
"""Output of the Firecrawl search tool."""
results: List[FirecrawlSearchResultItem] = Field(..., description="Matching results across all queries.")
+ failed_queries: List[str] = Field(
+ default_factory=list, description="Queries that failed while others succeeded. Details are logged."
+ )
#################
@@ -89,10 +100,14 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear
"""Tool for searching the web and news via the Firecrawl Search API."""
TIME_RANGES = {"day": "qdr:d", "week": "qdr:w", "month": "qdr:m", "year": "qdr:y"}
+ # Anything shaped like a Firecrawl key is redacted from error and log text, plus the configured key itself
+ # when it is long enough that replacing it cannot mangle ordinary words.
+ KEY_PATTERN = re.compile(r"fc-[A-Za-z0-9]{8,}")
+ MIN_REDACTED_KEY_LENGTH = 8
def __init__(self, config: FirecrawlSearchToolConfig = FirecrawlSearchToolConfig()):
super().__init__(config)
- self._api_key = config.api_key.get_secret_value() or os.getenv("FIRECRAWL_API_KEY", "")
+ self._api_key = SecretStr(config.api_key.get_secret_value() or os.getenv("FIRECRAWL_API_KEY", ""))
self.base_url = config.base_url.rstrip("/")
self.location = config.location
self.time_range = config.time_range
@@ -100,30 +115,29 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear
self.timeout = config.timeout
@staticmethod
- def _first(*values):
- return next((value for value in values if value), None)
+ def _first_text(*values: Any) -> Optional[str]:
+ """Return the first non-empty string; result fields are provider data and may have any type."""
+ return next((value for value in values if isinstance(value, str) and value), None)
@classmethod
- def _to_item(cls, hit: dict, query: str, max_content_chars: Optional[int] = None) -> FirecrawlSearchResultItem:
- metadata = hit.get("metadata")
- metadata = metadata if isinstance(metadata, dict) else {}
- content = hit.get("markdown")
+ def _to_item(cls, hit: Any, query: str, max_content_chars: Optional[int] = None) -> Optional[FirecrawlSearchResultItem]:
+ """Normalise one result, or return None when it has no usable URL."""
+ url = cls._first_text(hit.get("url")) if isinstance(hit, dict) else None
+ if url is None:
+ return None
+ content = cls._first_text(hit.get("markdown"))
return FirecrawlSearchResultItem(
query=query,
- title=cls._first(hit.get("title"), metadata.get("title"), hit["url"]),
- url=hit["url"],
- description=cls._first(hit.get("description"), hit.get("snippet"), metadata.get("description")),
+ title=cls._first_text(hit.get("title"), url),
+ url=url,
+ description=cls._first_text(hit.get("description"), hit.get("snippet")),
position=hit.get("position"),
- published=hit.get("date"),
- content=content[:max_content_chars] if content else None,
+ published=cls._first_text(hit.get("date")),
+ content=content and content[:max_content_chars],
)
def _build_body(self, query: str, params: FirecrawlSearchToolInputSchema) -> dict:
- body = {
- "query": query,
- "limit": params.max_results_per_query,
- "sources": [params.search_type],
- }
+ body = {"query": query, "limit": params.max_results_per_query, "sources": [params.search_type]}
if self.location:
body["location"] = self.location
if self.time_range and params.search_type == "web":
@@ -133,29 +147,54 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear
return body
def _redact(self, text: str) -> str:
- return text.replace(self._api_key, "[redacted]") if self._api_key else text
+ key = self._api_key.get_secret_value()
+ if len(key) >= self.MIN_REDACTED_KEY_LENGTH:
+ text = re.sub(re.escape(key), "[redacted]", text, flags=re.IGNORECASE)
+ return self.KEY_PATTERN.sub("[redacted]", text)
- def _error(self, query: str, status: Union[int, str], detail: Optional[str]) -> Exception:
- return Exception(self._redact(f"Firecrawl search failed for '{query}': {status} {detail}"))
+ def _error(self, query: str, detail: str) -> Exception:
+ return Exception(self._redact(f"Firecrawl search failed for '{query}': {detail}"))
@staticmethod
- async def _read_json(resp: aiohttp.ClientResponse):
+ async def _read_json(resp: aiohttp.ClientResponse) -> Any:
try:
return await resp.json(content_type=None)
except ValueError:
return None
- def _check_success(self, query: str, status: int, reason: Optional[str], data) -> None:
+ def _failure(self, status: int, reason: Optional[str], data: Any) -> Optional[str]:
+ """Return why a response is not a successful search (only a 200 whose body says success is true is),
+ or None when it is."""
payload = data if isinstance(data, dict) else {}
- if status != 200 or not payload.get("success"):
- raise self._error(query, status, payload.get("error") or reason)
+ if status == 200 and payload.get("success") is True:
+ return None
+ return self._redact(f"{status} {payload.get('error') or reason}")
- def _extract_hits(self, query: str, data: dict, search_type: str) -> list:
+ async def _post(self, session: aiohttp.ClientSession, query: str, params: FirecrawlSearchToolInputSchema) -> dict:
+ """Send one search request and return the successful response body.
+
+ Errors are raised outside the except block, so the original aiohttp error (whose repr carries the request
+ headers) is not attached as context, and the provider body is dropped first, so a traceback that captures
+ locals cannot show a key the provider echoed back."""
+ try:
+ async with session.post(
+ f"{self.base_url}/search", json=self._build_body(query, params), allow_redirects=False
+ ) as resp:
+ payload = await self._read_json(resp)
+ failure = self._failure(resp.status, resp.reason, payload)
+ except (aiohttp.ClientError, asyncio.TimeoutError) as error:
+ payload, failure = None, f"request failed: {type(error).__name__} {error}"
+ if failure:
+ payload = None
+ raise self._error(query, failure)
+ return payload
+
+ def _extract_hits(self, query: str, payload: dict, search_type: str) -> list:
# The key for the requested source is absent when there are no results.
- groups = data.get("data")
+ groups = payload.get("data")
hits = groups.get(search_type, []) if isinstance(groups, dict) else None
if not isinstance(hits, list):
- raise self._error(query, 200, f"unexpected response shape for '{search_type}' results")
+ raise self._error(query, f"200 unexpected response shape for '{search_type}' results")
return hits
async def _fetch(
@@ -164,50 +203,46 @@ class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSear
query: str,
params: FirecrawlSearchToolInputSchema,
) -> List[FirecrawlSearchResultItem]:
- try:
- async with session.post(f"{self.base_url}/search", json=self._build_body(query, params)) as resp:
- data = await self._read_json(resp)
- self._check_success(query, resp.status, resp.reason, data)
- except aiohttp.ClientError as error:
- # aiohttp errors such as TooManyRedirects carry the request headers, Authorization included, in their repr.
- raise self._error(query, type(error).__name__, str(error)) from None
-
- hits = self._extract_hits(query, data, params.search_type)
- items = [self._to_item(hit, query, self.max_content_chars) for hit in hits if isinstance(hit, dict) and hit.get("url")]
+ payload = await self._post(session, query, params)
+ hits = self._extract_hits(query, payload, params.search_type)
+ items = list(filter(None, (self._to_item(hit, query, self.max_content_chars) for hit in hits)))
+ if hits and not items:
+ raise self._error(query, f"200 none of the {len(hits)} '{params.search_type}' results had a usable url")
return items[: params.max_results_per_query]
- def _log_failure(self, query: str, error: BaseException) -> None:
- logger.warning("Firecrawl query '%s' failed: %s", query, self._redact(str(error) or repr(error)))
-
- def _collect(self, queries: List[str], grouped: list) -> List[FirecrawlSearchResultItem]:
+ def _collect(self, queries: List[str], grouped: list) -> FirecrawlSearchToolOutputSchema:
+ """Merge per-query results. Raise when no query produced results and at least one failed, so a bad key,
+ exhausted credits or a malformed response never looks like an empty search."""
results: List[FirecrawlSearchResultItem] = []
- errors = []
+ failed_queries: List[str] = []
+ errors: List[Exception] = []
for query, group in zip(queries, grouped):
- if isinstance(group, BaseException):
+ if isinstance(group, Exception):
+ failed_queries.append(query)
errors.append(group)
- self._log_failure(query, group)
+ logger.warning("Firecrawl query '%s' failed: %s", query, self._redact(str(group)))
continue
results.extend(group)
- # A bad key or exhausted credits fails every query; raise instead of returning an empty list.
- if errors and len(errors) == len(grouped):
+ if errors and not results:
raise errors[0]
- return results
+ return FirecrawlSearchToolOutputSchema(results=results, failed_queries=failed_queries)
async def run_async(self, params: FirecrawlSearchToolInputSchema) -> FirecrawlSearchToolOutputSchema:
- if not self._api_key:
+ if not self._api_key.get_secret_value():
raise ValueError(
"Firecrawl API key is missing. Set FirecrawlSearchToolConfig.api_key or the FIRECRAWL_API_KEY "
"environment variable."
)
- headers = {"Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json"}
- timeout = aiohttp.ClientTimeout(total=self.timeout)
- async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
- tasks = [self._fetch(session, q, params) for q in params.queries]
- grouped = await asyncio.gather(*tasks, return_exceptions=True)
+ # The Authorization header is built inline so no local variable holds the key.
+ async with aiohttp.ClientSession(
+ headers={"Authorization": f"Bearer {self._api_key.get_secret_value()}", "Content-Type": "application/json"},
+ timeout=aiohttp.ClientTimeout(total=self.timeout),
+ ) as session:
+ grouped = await asyncio.gather(*(self._fetch(session, q, params) for q in params.queries), return_exceptions=True)
- return FirecrawlSearchToolOutputSchema(results=self._collect(params.queries, grouped))
+ return self._collect(params.queries, grouped)
def run(self, params: FirecrawlSearchToolInputSchema) -> FirecrawlSearchToolOutputSchema:
with ThreadPoolExecutor() as executor: