Skip to content
Python

Add Firecrawl as a web_search provider

Rakshith Ramprakashdeb68de4e958
diff --git a/coworker/config.py b/coworker/config.py
index 76837f1c..a85106c2 100644
--- a/coworker/config.py
+++ b/coworker/config.py
@@ -90,7 +90,7 @@ class Config:
host: str = "127.0.0.1"
port: int = 8765
# Web search provider: "duckduckgo" (keyless default) | "tavily" | "brave" | "serper"
- # | "exa" (the keyed ones need a key).
+ # | "exa" | "firecrawl" (the keyed ones need a key).
web_search_provider: str = "duckduckgo"
# OpenWorker Cloud (sign-in + managed connectors). Config, never constants:
# dev/staging/BYO-VPC deployments point these at their own instances.
diff --git a/coworker/web/__init__.py b/coworker/web/__init__.py
index 048df5b1..d171b521 100644
--- a/coworker/web/__init__.py
+++ b/coworker/web/__init__.py
@@ -6,6 +6,7 @@ from .providers import (
BraveProvider,
DuckDuckGoProvider,
ExaProvider,
+ FirecrawlProvider,
SearchResult,
SerperProvider,
TavilyProvider,
@@ -24,6 +25,7 @@ __all__ = [
"BraveProvider",
"SerperProvider",
"ExaProvider",
+ "FirecrawlProvider",
"build_provider",
"provider_names",
"make_web_search_tool",
diff --git a/coworker/web/providers.py b/coworker/web/providers.py
index 311c32e2..8363bce1 100644
--- a/coworker/web/providers.py
+++ b/coworker/web/providers.py
@@ -1,8 +1,9 @@
"""Web search providers — a keyless default + pluggable third-party services.
`duckduckgo` works with no API key (our "starting version of our own"). `tavily`, `brave`,
-`serper` and `exa` give better results but need a key (configured via the SecretStore / env).
-All providers return a uniform `list[SearchResult]`; the heavy client libs are lazy-imported.
+`serper`, `exa` and `firecrawl` give better results but need a key (configured via the
+SecretStore / env). All providers return a uniform `list[SearchResult]`; the heavy client libs
+are lazy-imported.
"""
from __future__ import annotations
@@ -195,12 +196,42 @@ class ExaProvider(WebSearchProvider):
]
+class FirecrawlProvider(WebSearchProvider):
+ """Firecrawl web search (https://docs.firecrawl.dev) via the v2 REST API."""
+
+ name = "firecrawl"
+ requires_key = True
+
+ def __init__(self, api_key: str) -> None:
+ self.api_key = api_key
+
+ def search(self, query: str, max_results: int = 5) -> list[SearchResult]:
+ import httpx
+
+ resp = httpx.post(
+ "https://api.firecrawl.dev/v2/search",
+ headers={"Authorization": f"Bearer {self.api_key}"},
+ json={"query": query, "limit": max_results},
+ timeout=_TIMEOUT,
+ )
+ data = resp.json()
+ return [
+ SearchResult(
+ title=r.get("title", ""),
+ url=r.get("url", ""),
+ snippet=r.get("description", ""),
+ )
+ for r in (data.get("data", {}) or {}).get("web", [])
+ ]
+
+
_PROVIDERS = {
"duckduckgo": DuckDuckGoProvider,
"tavily": TavilyProvider,
"brave": BraveProvider,
"serper": SerperProvider,
"exa": ExaProvider,
+ "firecrawl": FirecrawlProvider,
}
diff --git a/tests/test_web_search.py b/tests/test_web_search.py
index 866f876a..6b57d25c 100644
--- a/tests/test_web_search.py
+++ b/tests/test_web_search.py
@@ -1,7 +1,7 @@
"""Tests for web search — provider abstraction, the tool, and config resolution.
No network: a FakeProvider is injected; third-party key handling and the REST config path
-are exercised without hitting DuckDuckGo/Tavily/Brave/Serper/Exa.
+are exercised without hitting DuckDuckGo/Tavily/Brave/Serper/Exa/Firecrawl.
"""
from __future__ import annotations
@@ -21,6 +21,7 @@ from coworker.web.providers import (
BraveProvider,
DuckDuckGoProvider,
ExaProvider,
+ FirecrawlProvider,
SerperProvider,
TavilyProvider,
WebSearchProvider,
@@ -91,6 +92,9 @@ def test_build_provider_third_party_requires_key():
with pytest.raises(ValueError):
build_provider("exa") # no key
assert isinstance(build_provider("exa", "exa-x"), ExaProvider)
+ with pytest.raises(ValueError):
+ build_provider("firecrawl") # no key
+ assert isinstance(build_provider("firecrawl", "fc-x"), FirecrawlProvider)
def test_serper_sends_correct_request_and_maps_organic_results(monkeypatch):
@@ -174,6 +178,39 @@ def test_exa_provider_parses_highlights(monkeypatch):
]
+def test_firecrawl_maps_v2_search_response(monkeypatch):
+ import httpx
+
+ captured = {}
+
+ class _Resp:
+ def json(self):
+ return {
+ "success": True,
+ "data": {
+ "web": [
+ {"title": "t0", "url": "https://x/0", "description": "d0"},
+ {"title": "t1", "url": "https://x/1", "description": "d1"},
+ ]
+ },
+ }
+
+ def fake_post(url, headers=None, json=None, timeout=None):
+ captured.update(url=url, headers=headers, json=json)
+ return _Resp()
+
+ monkeypatch.setattr(httpx, "post", fake_post)
+ results = FirecrawlProvider("fc-key").search("claude api", max_results=2)
+
+ assert captured["url"] == "https://api.firecrawl.dev/v2/search"
+ assert captured["headers"]["Authorization"] == "Bearer fc-key"
+ assert captured["json"] == {"query": "claude api", "limit": 2}
+ assert [(r.title, r.url, r.snippet) for r in results] == [
+ ("t0", "https://x/0", "d0"),
+ ("t1", "https://x/1", "d1"),
+ ]
+
+
def test_keyed_providers_raise_on_http_errors(monkeypatch):
"""A 401/429/5xx surfaces as an error instead of an empty result list."""
import httpx