Skip to content
Python

web_fetch can read pages through Exa, Firecrawl or Tavily

Rohit C Prasad3cc57ad0a0b7
diff --git a/coworker/agent.py b/coworker/agent.py
index 78920e1c..49ee0866 100644
--- a/coworker/agent.py
+++ b/coworker/agent.py
@@ -442,7 +442,7 @@ def build_engine(
# unless the caller withheld them (`openworker run --no-web-tools`).
if web_tools:
registry.register(make_web_search_tool(secrets))
- registry.register(make_web_fetch_tool())
+ registry.register(make_web_fetch_tool(secrets))
# ask_user: the universal human-in-the-loop Q&A primitive (every agent; engine-intercepted).
if question_asker is not None:
registry.register(ask_user_tool())
@@ -815,6 +815,13 @@ def build_engine(
from .web import provider_name
return {"search_provider": provider_name(secrets)}
+ if tool_name == "web_fetch":
+ # Same rule for web_fetch: when the page goes through the search service
+ # (Settings ▸ Customization), the card names it.
+ from .web import fetch_provider_name
+
+ via = fetch_provider_name(secrets)
+ return {"fetch_provider": via} if via else {}
return {}
engine.approval_extras = _approval_extras
diff --git a/coworker/web/fetch.py b/coworker/web/fetch.py
index b7199038..67d3dde5 100644
--- a/coworker/web/fetch.py
+++ b/coworker/web/fetch.py
@@ -3,17 +3,25 @@
Complements `web_search` (which returns snippets): this fetches one page over HTTP(S) and
returns a size-capped plain-text extraction (HTML stripped to text). External content — must
be treated as untrusted data to evaluate, not as instructions.
+
+With "Fetch pages through the search service" on (Settings ▸ Customization) and a provider
+that offers it (Exa, Firecrawl, Tavily), the page's address still passes the same checks as
+a local fetch — the address guard here, the sandbox's allowed-sites wall in the permission
+engine — and then the provider returns the page instead of this machine fetching it. Should
+the provider fail, the page is fetched locally and the result says so.
"""
from __future__ import annotations
import re
from html.parser import HTMLParser
-from typing import Any, Callable
+from typing import Any, Callable, Optional
import aisuite as ai
-from .guard import get_checked
+from ..secrets import SecretStore
+from .guard import check_url, get_checked
+from .tool import fetch_provider
_MAX = 20000 # default chars returned
@@ -75,7 +83,7 @@ def _html_to_text(html: str) -> str:
return re.sub(r"\n{3,}", "\n\n", "\n".join(parser.parts))
-def make_web_fetch_tool() -> Callable[..., Any]:
+def make_web_fetch_tool(secrets: Optional[SecretStore] = None) -> Callable[..., Any]:
def web_fetch(url: str, max_chars: int = _MAX) -> dict[str, Any]:
if not isinstance(url, str) or not url.lower().startswith(
("http://", "https://")
@@ -83,6 +91,30 @@ def make_web_fetch_tool() -> Callable[..., Any]:
return {"error": "url must start with http:// or https://"}
cap = max_chars if isinstance(max_chars, int) and max_chars > 0 else _MAX
cap = min(cap, 100000)
+ notice = ""
+ try:
+ provider = fetch_provider(secrets)
+ except Exception: # noqa: BLE001 - an unreadable profile means a local fetch
+ provider = None
+ if provider is not None:
+ # The provider sees the address, so the address is judged first, exactly as
+ # for a local fetch (loopback, private ranges, the metadata endpoint).
+ reason = check_url(url)
+ if reason:
+ return {"error": reason}
+ try:
+ text = provider.fetch_page(url)
+ except Exception as exc: # noqa: BLE001 - fall back to a local fetch
+ service = provider.display_name or provider.name
+ notice = f"{service} could not fetch the page ({exc}); fetched from this machine instead."
+ else:
+ return {
+ "url": url,
+ "content_type": "text/markdown",
+ "fetched_via": provider.name,
+ "truncated": len(text) > cap,
+ "text": text[:cap],
+ }
try:
import httpx
@@ -104,12 +136,15 @@ def make_web_fetch_tool() -> Callable[..., Any]:
except Exception as exc: # network / HTTP / TLS
return {"error": f"fetch failed: {exc}"}
text = _html_to_text(body) if "html" in ctype.lower() else body
- return {
+ out: dict[str, Any] = {
"url": final_url,
"content_type": ctype,
"truncated": len(text) > cap,
"text": text[:cap],
}
+ if notice:
+ out["notice"] = notice
+ return out
web_fetch.__name__ = "web_fetch"
web_fetch.__doc__ = _SCHEMA["function"]["description"]
diff --git a/tests/test_web_fetch_via_provider.py b/tests/test_web_fetch_via_provider.py
new file mode 100644
index 00000000..9d29e973
--- /dev/null
+++ b/tests/test_web_fetch_via_provider.py
@@ -0,0 +1,224 @@
+"""`web_fetch` through the search service (Settings ▸ Customization).
+
+No network: httpx is stubbed. Covers the three providers' request/response shapes
+(Exa /contents, Firecrawl /v2/scrape, Tavily /extract), when `fetch_provider` picks one,
+the address guard running before the provider is asked, the fallback to a local fetch
+with a notice, and the approval card's `fetch_provider` extra.
+"""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+from coworker.secrets import SecretStore
+from coworker.web import fetch_provider, fetch_provider_name
+from coworker.web.fetch import make_web_fetch_tool
+from coworker.web.providers import (
+ _PROVIDERS,
+ BraveProvider,
+ ExaProvider,
+ FirecrawlProvider,
+ TavilyProvider,
+)
+
+
+def _post_stub(monkeypatch, status, payload, captured):
+ def fake_post(url, headers=None, json=None, timeout=None):
+ captured.update(url=url, headers=headers, json=json, timeout=timeout)
+ return httpx.Response(status, request=httpx.Request("POST", url), json=payload)
+
+ monkeypatch.setattr(httpx, "post", fake_post)
+
+
+def test_only_exa_firecrawl_and_tavily_fetch_pages():
+ assert {n for n, c in _PROVIDERS.items() if c.supports_fetch} == {"exa", "firecrawl", "tavily"}
+ with pytest.raises(NotImplementedError):
+ BraveProvider("k").fetch_page("https://example.com")
+
+
+def test_exa_fetch_page_request_and_response(monkeypatch):
+ got = {}
+ _post_stub(monkeypatch, 200, {
+ "results": [{"title": "T", "url": "https://example.com", "text": "page text"}],
+ "statuses": [{"id": "https://example.com", "status": "success", "source": "crawled"}],
+ }, got)
+ assert ExaProvider("exa-k").fetch_page("https://example.com") == "page text"
+ assert got["url"] == "https://api.exa.ai/contents"
+ assert got["headers"]["x-api-key"] == "exa-k"
+ assert got["json"] == {"urls": ["https://example.com"], "text": True}
+
+ _post_stub(monkeypatch, 200, {
+ "results": [],
+ "statuses": [{"id": "x", "status": "error", "error": {"tag": "CRAWL_NOT_FOUND", "httpStatusCode": 404}}],
+ }, {})
+ with pytest.raises(RuntimeError, match="CRAWL_NOT_FOUND \\(HTTP 404\\)"):
+ ExaProvider("exa-k").fetch_page("https://example.com/missing")
+
+
+def test_firecrawl_fetch_page_request_and_response(monkeypatch):
+ got = {}
+ _post_stub(monkeypatch, 200, {"success": True, "data": {"markdown": "# Hi", "metadata": {"title": "Hi"}}}, got)
+ assert FirecrawlProvider("fc-k").fetch_page("https://example.com") == "# Hi"
+ assert got["url"] == "https://api.firecrawl.dev/v2/scrape"
+ assert got["headers"]["Authorization"] == "Bearer fc-k"
+ assert got["json"] == {"url": "https://example.com", "formats": ["markdown"]}
+
+ _post_stub(monkeypatch, 200, {"success": False, "error": "blocked by robots"}, {})
+ with pytest.raises(RuntimeError, match="blocked by robots"):
+ FirecrawlProvider("fc-k").fetch_page("https://example.com")
+
+
+def test_tavily_fetch_page_request_and_response(monkeypatch):
+ got = {}
+ _post_stub(monkeypatch, 200, {
+ "results": [{"url": "https://example.com", "raw_content": "body"}],
+ "failed_results": [],
+ }, got)
+ assert TavilyProvider("tv-k").fetch_page("https://example.com") == "body"
+ assert got["url"] == "https://api.tavily.com/extract"
+ assert got["headers"]["Authorization"] == "Bearer tv-k"
+ assert got["json"] == {"urls": ["https://example.com"]}
+
+ _post_stub(monkeypatch, 200, {"results": [], "failed_results": [{"url": "x", "error": "timeout"}]}, {})
+ with pytest.raises(RuntimeError, match="timeout"):
+ TavilyProvider("tv-k").fetch_page("https://example.com")
+
+ _post_stub(monkeypatch, 401, {}, {})
+ with pytest.raises(httpx.HTTPStatusError):
+ TavilyProvider("tv-k").fetch_page("https://example.com")
+
+
+def _secrets(tmp_path, monkeypatch, **profile):
+ for name in _PROVIDERS:
+ monkeypatch.delenv(f"{name.upper()}_API_KEY", raising=False)
+ secrets = SecretStore(tmp_path / "secrets.json")
+ secrets.put("web_search:default", profile)
+ return secrets
+
+
+def test_fetch_provider_needs_the_switch_support_and_a_key(tmp_path, monkeypatch):
+ # Off: local fetch, whatever the provider.
+ s = _secrets(tmp_path, monkeypatch, provider="exa", keys={"exa": "k"})
+ assert fetch_provider(s) is None and fetch_provider_name(s) == ""
+ # On with a provider that offers it and has a key.
+ s = _secrets(tmp_path, monkeypatch, provider="exa", keys={"exa": "k"}, fetch_via_provider=True)
+ assert isinstance(fetch_provider(s), ExaProvider) and fetch_provider_name(s) == "exa"
+ # On, but the active provider does not fetch pages.
+ s = _secrets(tmp_path, monkeypatch, provider="brave", keys={"brave": "k", "exa": "k"}, fetch_via_provider=True)
+ assert fetch_provider(s) is None
+ s = _secrets(tmp_path, monkeypatch, provider="duckduckgo", fetch_via_provider=True)
+ assert fetch_provider(s) is None
+ # On, a fetching provider, but no key anywhere: a local fetch rather than an error.
+ s = _secrets(tmp_path, monkeypatch, provider="tavily", fetch_via_provider=True)
+ assert fetch_provider(s) is None
+ monkeypatch.setenv("TAVILY_API_KEY", "env-k")
+ assert fetch_provider(s).api_key == "env-k"
+
+
+def test_web_fetch_goes_through_the_provider_after_the_address_check(tmp_path, monkeypatch):
+ import coworker.web.fetch as fetch_mod
+
+ s = _secrets(tmp_path, monkeypatch, provider="firecrawl", keys={"firecrawl": "k"}, fetch_via_provider=True)
+ got = {}
+ _post_stub(monkeypatch, 200, {"success": True, "data": {"markdown": "x" * 50}}, got)
+ checked = []
+ monkeypatch.setattr(fetch_mod, "check_url", lambda url: checked.append(url) or None)
+
+ def no_local(*_a, **_k):
+ raise AssertionError("the page must not be fetched locally")
+
+ monkeypatch.setattr(fetch_mod, "get_checked", no_local)
+ out = make_web_fetch_tool(s)("https://example.com/doc", max_chars=10)
+ assert checked == ["https://example.com/doc"]
+ assert got["json"]["url"] == "https://example.com/doc"
+ assert out == {
+ "url": "https://example.com/doc",
+ "content_type": "text/markdown",
+ "fetched_via": "firecrawl",
+ "truncated": True,
+ "text": "x" * 10,
+ }
+
+
+def test_web_fetch_refuses_blocked_addresses_before_asking_the_provider(tmp_path, monkeypatch):
+ s = _secrets(tmp_path, monkeypatch, provider="exa", keys={"exa": "k"}, fetch_via_provider=True)
+
+ def no_post(*_a, **_k):
+ raise AssertionError("the provider must not see a blocked address")
+
+ monkeypatch.setattr(httpx, "post", no_post)
+ out = make_web_fetch_tool(s)("http://127.0.0.1:11434/api/tags")
+ assert "loopback" in out["error"]
+ out = make_web_fetch_tool(s)("http://169.254.169.254/latest/meta-data/")
+ assert "link-local" in out["error"]
+
+
+def test_web_fetch_falls_back_to_a_local_fetch_with_a_notice(tmp_path, monkeypatch):
+ import coworker.web.fetch as fetch_mod
+
+ s = _secrets(tmp_path, monkeypatch, provider="tavily", keys={"tavily": "k"}, fetch_via_provider=True)
+ _post_stub(monkeypatch, 500, {}, {})
+ monkeypatch.setattr(fetch_mod, "check_url", lambda url: None)
+
+ def local(client, url):
+ resp = httpx.Response(
+ 200, request=httpx.Request("GET", url), text="<html><body><p>local</p></body></html>",
+ headers={"content-type": "text/html"},
+ )
+ resp.extensions["logical_url"] = url
+ return resp
+
+ monkeypatch.setattr(fetch_mod, "get_checked", local)
+ out = make_web_fetch_tool(s)("https://example.com/")
+ assert out["text"] == "local" and out["url"] == "https://example.com/"
+ assert "fetched_via" not in out
+ assert out["notice"].startswith("Tavily could not fetch the page (")
+ assert out["notice"].endswith("); fetched from this machine instead.")
+ assert "k" * 2 not in out["notice"]
+
+
+def test_web_fetch_without_the_switch_is_unchanged(tmp_path, monkeypatch):
+ import coworker.web.fetch as fetch_mod
+
+ s = _secrets(tmp_path, monkeypatch, provider="exa", keys={"exa": "k"})
+ monkeypatch.setattr(httpx, "post", lambda *a, **k: (_ for _ in ()).throw(AssertionError("no provider call")))
+
+ def local(client, url):
+ resp = httpx.Response(200, request=httpx.Request("GET", url), text="plain", headers={"content-type": "text/plain"})
+ resp.extensions["logical_url"] = url
+ return resp
+
+ monkeypatch.setattr(fetch_mod, "get_checked", local)
+ out = make_web_fetch_tool(s)("https://example.com/")
+ assert out == {"url": "https://example.com/", "content_type": "text/plain", "truncated": False, "text": "plain"}
+
+
+def test_web_fetch_approval_card_names_the_fetching_service(tmp_path, monkeypatch):
+ from coworker.agent import build_engine
+ from coworker.agents import chat_agent
+
+ class _Stub:
+ def complete(self, **_kw):
+ from coworker.providers import AssistantTurn
+
+ return AssistantTurn()
+
+ def capabilities(self, _model):
+ from coworker.providers.base import ModelCapabilities
+
+ return ModelCapabilities()
+
+ def stream(self, **_kw):
+ from coworker.providers.base import StreamChunk
+
+ yield StreamChunk(turn=self.complete())
+
+ s = _secrets(tmp_path, monkeypatch, provider="exa", keys={"exa": "k"})
+ eng = build_engine(agent=chat_agent(), provider=_Stub(), secrets=s)
+ assert "web_fetch" in eng.registry.names()
+ assert eng.approval_extras("web_fetch", {"url": "https://x"}) == {}
+ assert eng.approval_extras("web_search", {}) == {"search_provider": "exa"}
+ s.put("web_search:default", {"provider": "exa", "keys": {"exa": "k"}, "fetch_via_provider": True})
+ # Resolved at card time, so the Settings change shows through without a new session.
+ assert eng.approval_extras("web_fetch", {"url": "https://x"}) == {"fetch_provider": "exa"}