Python
Add Firecrawl as a web_search provider
Rakshith Ramprakashdeb68de4e958
diff --git a/coworker/config.py b/coworker/config.pyindex 76837f1c..a85106c2 100644--- a/coworker/config.py+++ b/coworker/config.py@@ -90,7 +90,7 @@ class Config:host: str = "127.0.0.1"port: int = 8765# Web search provider: "duckduckgo" (keyless default) | "tavily" | "brave" | "serper"- # | "exa" (the keyed ones need a key).+ # | "exa" | "firecrawl" (the keyed ones need a key).web_search_provider: str = "duckduckgo"# OpenWorker Cloud (sign-in + managed connectors). Config, never constants:# dev/staging/BYO-VPC deployments point these at their own instances.diff --git a/coworker/web/__init__.py b/coworker/web/__init__.pyindex 048df5b1..d171b521 100644--- a/coworker/web/__init__.py+++ b/coworker/web/__init__.py@@ -6,6 +6,7 @@ from .providers import (BraveProvider,DuckDuckGoProvider,ExaProvider,+ FirecrawlProvider,SearchResult,SerperProvider,TavilyProvider,@@ -24,6 +25,7 @@ __all__ = ["BraveProvider","SerperProvider","ExaProvider",+ "FirecrawlProvider","build_provider","provider_names","make_web_search_tool",diff --git a/coworker/web/providers.py b/coworker/web/providers.pyindex 311c32e2..8363bce1 100644--- a/coworker/web/providers.py+++ b/coworker/web/providers.py@@ -1,8 +1,9 @@"""Web search providers — a keyless default + pluggable third-party services.`duckduckgo` works with no API key (our "starting version of our own"). `tavily`, `brave`,-`serper` and `exa` give better results but need a key (configured via the SecretStore / env).-All providers return a uniform `list[SearchResult]`; the heavy client libs are lazy-imported.+`serper`, `exa` and `firecrawl` give better results but need a key (configured via the+SecretStore / env). All providers return a uniform `list[SearchResult]`; the heavy client libs+are lazy-imported."""from __future__ import annotations@@ -195,12 +196,42 @@ class ExaProvider(WebSearchProvider):]+class FirecrawlProvider(WebSearchProvider):+ """Firecrawl web search (https://docs.firecrawl.dev) via the v2 REST API."""++ name = "firecrawl"+ requires_key = True++ def __init__(self, api_key: str) -> None:+ self.api_key = api_key++ def search(self, query: str, max_results: int = 5) -> list[SearchResult]:+ import httpx++ resp = httpx.post(+ "https://api.firecrawl.dev/v2/search",+ headers={"Authorization": f"Bearer {self.api_key}"},+ json={"query": query, "limit": max_results},+ timeout=_TIMEOUT,+ )+ data = resp.json()+ return [+ SearchResult(+ title=r.get("title", ""),+ url=r.get("url", ""),+ snippet=r.get("description", ""),+ )+ for r in (data.get("data", {}) or {}).get("web", [])+ ]++_PROVIDERS = {"duckduckgo": DuckDuckGoProvider,"tavily": TavilyProvider,"brave": BraveProvider,"serper": SerperProvider,"exa": ExaProvider,+ "firecrawl": FirecrawlProvider,}diff --git a/tests/test_web_search.py b/tests/test_web_search.pyindex 866f876a..6b57d25c 100644--- a/tests/test_web_search.py+++ b/tests/test_web_search.py@@ -1,7 +1,7 @@"""Tests for web search — provider abstraction, the tool, and config resolution.No network: a FakeProvider is injected; third-party key handling and the REST config path-are exercised without hitting DuckDuckGo/Tavily/Brave/Serper/Exa.+are exercised without hitting DuckDuckGo/Tavily/Brave/Serper/Exa/Firecrawl."""from __future__ import annotations@@ -21,6 +21,7 @@ from coworker.web.providers import (BraveProvider,DuckDuckGoProvider,ExaProvider,+ FirecrawlProvider,SerperProvider,TavilyProvider,WebSearchProvider,@@ -91,6 +92,9 @@ def test_build_provider_third_party_requires_key():with pytest.raises(ValueError):build_provider("exa") # no keyassert isinstance(build_provider("exa", "exa-x"), ExaProvider)+ with pytest.raises(ValueError):+ build_provider("firecrawl") # no key+ assert isinstance(build_provider("firecrawl", "fc-x"), FirecrawlProvider)def test_serper_sends_correct_request_and_maps_organic_results(monkeypatch):@@ -174,6 +178,39 @@ def test_exa_provider_parses_highlights(monkeypatch):]+def test_firecrawl_maps_v2_search_response(monkeypatch):+ import httpx++ captured = {}++ class _Resp:+ def json(self):+ return {+ "success": True,+ "data": {+ "web": [+ {"title": "t0", "url": "https://x/0", "description": "d0"},+ {"title": "t1", "url": "https://x/1", "description": "d1"},+ ]+ },+ }++ def fake_post(url, headers=None, json=None, timeout=None):+ captured.update(url=url, headers=headers, json=json)+ return _Resp()++ monkeypatch.setattr(httpx, "post", fake_post)+ results = FirecrawlProvider("fc-key").search("claude api", max_results=2)++ assert captured["url"] == "https://api.firecrawl.dev/v2/search"+ assert captured["headers"]["Authorization"] == "Bearer fc-key"+ assert captured["json"] == {"query": "claude api", "limit": 2}+ assert [(r.title, r.url, r.snippet) for r in results] == [+ ("t0", "https://x/0", "d0"),+ ("t1", "https://x/1", "d1"),+ ]++def test_keyed_providers_raise_on_http_errors(monkeypatch):"""A 401/429/5xx surfaces as an error instead of an empty result list."""import httpx