Skip to content
Python

Register Firecrawl with the site wall

Rohit C Prasadc59c288fe32b
diff --git a/coworker/sandbox/network_profiles.py b/coworker/sandbox/network_profiles.py
index be423708..026c24bb 100644
--- a/coworker/sandbox/network_profiles.py
+++ b/coworker/sandbox/network_profiles.py
@@ -15,7 +15,7 @@ import sys
CODE_HOSTS = ["github.com", "api.github.com", "codeload.github.com", "objects.githubusercontent.com", "raw.githubusercontent.com", "gitlab.com"]
PACKAGE_REGISTRIES = ["pypi.org", "files.pythonhosted.org", "registry.npmjs.org", "crates.io", "static.crates.io", "index.crates.io", "proxy.golang.org", "sum.golang.org"]
-SEARCH_APIS = ["api.search.brave.com", "api.tavily.com", "html.duckduckgo.com", "duckduckgo.com", "google.serper.dev", "api.exa.ai"]
+SEARCH_APIS = ["api.search.brave.com", "api.tavily.com", "html.duckduckgo.com", "duckduckgo.com", "google.serper.dev", "api.exa.ai", "api.firecrawl.dev"]
# The page's catalogue, by group. None is allowed until ticked.
SITES: dict[str, list[str]] = {"code-hosts": CODE_HOSTS, "package-registries": PACKAGE_REGISTRIES, "search-apis": SEARCH_APIS}
diff --git a/coworker/web/providers.py b/coworker/web/providers.py
index 8363bce1..46ba7842 100644
--- a/coworker/web/providers.py
+++ b/coworker/web/providers.py
@@ -200,6 +200,8 @@ class FirecrawlProvider(WebSearchProvider):
"""Firecrawl web search (https://docs.firecrawl.dev) via the v2 REST API."""
name = "firecrawl"
+ display_name = "Firecrawl"
+ key_url = "https://firecrawl.dev"
requires_key = True
def __init__(self, api_key: str) -> None:
@@ -214,6 +216,7 @@ class FirecrawlProvider(WebSearchProvider):
json={"query": query, "limit": max_results},
timeout=_TIMEOUT,
)
+ resp.raise_for_status()
data = resp.json()
return [
SearchResult(
diff --git a/coworker/web/tool.py b/coworker/web/tool.py
index df6849be..6346f53c 100644
--- a/coworker/web/tool.py
+++ b/coworker/web/tool.py
@@ -59,6 +59,7 @@ PROVIDER_HOSTS = {
"brave": "api.search.brave.com",
"serper": "google.serper.dev",
"exa": "api.exa.ai",
+ "firecrawl": "api.firecrawl.dev",
}
diff --git a/docs/config.example.toml b/docs/config.example.toml
index 68f047c2..f0ecef85 100644
--- a/docs/config.example.toml
+++ b/docs/config.example.toml
@@ -19,7 +19,7 @@ max_iterations = 12 # max model<->tool iterations per turn before stopp
# default (Anthropic high, Kimi K3 max). Env override:
# COWORKER_REASONING_EFFORT.
-# Web search backend: "duckduckgo" (keyless, default) | "tavily" | "brave" | "serper" | "exa"
+# Web search backend: "duckduckgo" (keyless, default) | "tavily" | "brave" | "serper" | "exa" | "firecrawl"
# (the keyed ones need an API key, set in the app's Settings or secret store).
web_search_provider = "duckduckgo"
diff --git a/tests/test_web_search.py b/tests/test_web_search.py
index 6b57d25c..02a1352d 100644
--- a/tests/test_web_search.py
+++ b/tests/test_web_search.py
@@ -184,6 +184,9 @@ def test_firecrawl_maps_v2_search_response(monkeypatch):
captured = {}
class _Resp:
+ def raise_for_status(self):
+ pass
+
def json(self):
return {
"success": True,
@@ -220,7 +223,13 @@ def test_keyed_providers_raise_on_http_errors(monkeypatch):
monkeypatch.setattr(httpx, "post", denied)
monkeypatch.setattr(httpx, "get", denied)
- for provider in (TavilyProvider("k"), BraveProvider("k"), SerperProvider("k"), ExaProvider("k")):
+ for provider in (
+ TavilyProvider("k"),
+ BraveProvider("k"),
+ SerperProvider("k"),
+ ExaProvider("k"),
+ FirecrawlProvider("k"),
+ ):
with pytest.raises(httpx.HTTPStatusError):
provider.search("q")