Python
Register Firecrawl with the site wall
Rohit C Prasadc59c288fe32b
diff --git a/coworker/sandbox/network_profiles.py b/coworker/sandbox/network_profiles.pyindex be423708..026c24bb 100644--- a/coworker/sandbox/network_profiles.py+++ b/coworker/sandbox/network_profiles.py@@ -15,7 +15,7 @@ import sysCODE_HOSTS = ["github.com", "api.github.com", "codeload.github.com", "objects.githubusercontent.com", "raw.githubusercontent.com", "gitlab.com"]PACKAGE_REGISTRIES = ["pypi.org", "files.pythonhosted.org", "registry.npmjs.org", "crates.io", "static.crates.io", "index.crates.io", "proxy.golang.org", "sum.golang.org"]-SEARCH_APIS = ["api.search.brave.com", "api.tavily.com", "html.duckduckgo.com", "duckduckgo.com", "google.serper.dev", "api.exa.ai"]+SEARCH_APIS = ["api.search.brave.com", "api.tavily.com", "html.duckduckgo.com", "duckduckgo.com", "google.serper.dev", "api.exa.ai", "api.firecrawl.dev"]# The page's catalogue, by group. None is allowed until ticked.SITES: dict[str, list[str]] = {"code-hosts": CODE_HOSTS, "package-registries": PACKAGE_REGISTRIES, "search-apis": SEARCH_APIS}diff --git a/coworker/web/providers.py b/coworker/web/providers.pyindex 8363bce1..46ba7842 100644--- a/coworker/web/providers.py+++ b/coworker/web/providers.py@@ -200,6 +200,8 @@ class FirecrawlProvider(WebSearchProvider):"""Firecrawl web search (https://docs.firecrawl.dev) via the v2 REST API."""name = "firecrawl"+ display_name = "Firecrawl"+ key_url = "https://firecrawl.dev"requires_key = Truedef __init__(self, api_key: str) -> None:@@ -214,6 +216,7 @@ class FirecrawlProvider(WebSearchProvider):json={"query": query, "limit": max_results},timeout=_TIMEOUT,)+ resp.raise_for_status()data = resp.json()return [SearchResult(diff --git a/coworker/web/tool.py b/coworker/web/tool.pyindex df6849be..6346f53c 100644--- a/coworker/web/tool.py+++ b/coworker/web/tool.py@@ -59,6 +59,7 @@ PROVIDER_HOSTS = {"brave": "api.search.brave.com","serper": "google.serper.dev","exa": "api.exa.ai",+ "firecrawl": "api.firecrawl.dev",}diff --git a/docs/config.example.toml b/docs/config.example.tomlindex 68f047c2..f0ecef85 100644--- a/docs/config.example.toml+++ b/docs/config.example.toml@@ -19,7 +19,7 @@ max_iterations = 12 # max model<->tool iterations per turn before stopp# default (Anthropic high, Kimi K3 max). Env override:# COWORKER_REASONING_EFFORT.-# Web search backend: "duckduckgo" (keyless, default) | "tavily" | "brave" | "serper" | "exa"+# Web search backend: "duckduckgo" (keyless, default) | "tavily" | "brave" | "serper" | "exa" | "firecrawl"# (the keyed ones need an API key, set in the app's Settings or secret store).web_search_provider = "duckduckgo"diff --git a/tests/test_web_search.py b/tests/test_web_search.pyindex 6b57d25c..02a1352d 100644--- a/tests/test_web_search.py+++ b/tests/test_web_search.py@@ -184,6 +184,9 @@ def test_firecrawl_maps_v2_search_response(monkeypatch):captured = {}class _Resp:+ def raise_for_status(self):+ pass+def json(self):return {"success": True,@@ -220,7 +223,13 @@ def test_keyed_providers_raise_on_http_errors(monkeypatch):monkeypatch.setattr(httpx, "post", denied)monkeypatch.setattr(httpx, "get", denied)- for provider in (TavilyProvider("k"), BraveProvider("k"), SerperProvider("k"), ExaProvider("k")):+ for provider in (+ TavilyProvider("k"),+ BraveProvider("k"),+ SerperProvider("k"),+ ExaProvider("k"),+ FirecrawlProvider("k"),+ ):with pytest.raises(httpx.HTTPStatusError):provider.search("q")