Python
web_fetch can read pages through Exa, Firecrawl or Tavily
Rohit C Prasad3cc57ad0a0b7
diff --git a/coworker/agent.py b/coworker/agent.pyindex 78920e1c..49ee0866 100644--- a/coworker/agent.py+++ b/coworker/agent.py@@ -442,7 +442,7 @@ def build_engine(# unless the caller withheld them (`openworker run --no-web-tools`).if web_tools:registry.register(make_web_search_tool(secrets))- registry.register(make_web_fetch_tool())+ registry.register(make_web_fetch_tool(secrets))# ask_user: the universal human-in-the-loop Q&A primitive (every agent; engine-intercepted).if question_asker is not None:registry.register(ask_user_tool())@@ -815,6 +815,13 @@ def build_engine(from .web import provider_namereturn {"search_provider": provider_name(secrets)}+ if tool_name == "web_fetch":+ # Same rule for web_fetch: when the page goes through the search service+ # (Settings ▸ Customization), the card names it.+ from .web import fetch_provider_name++ via = fetch_provider_name(secrets)+ return {"fetch_provider": via} if via else {}return {}engine.approval_extras = _approval_extrasdiff --git a/coworker/web/fetch.py b/coworker/web/fetch.pyindex b7199038..67d3dde5 100644--- a/coworker/web/fetch.py+++ b/coworker/web/fetch.py@@ -3,17 +3,25 @@Complements `web_search` (which returns snippets): this fetches one page over HTTP(S) andreturns a size-capped plain-text extraction (HTML stripped to text). External content — mustbe treated as untrusted data to evaluate, not as instructions.++With "Fetch pages through the search service" on (Settings ▸ Customization) and a provider+that offers it (Exa, Firecrawl, Tavily), the page's address still passes the same checks as+a local fetch — the address guard here, the sandbox's allowed-sites wall in the permission+engine — and then the provider returns the page instead of this machine fetching it. Should+the provider fail, the page is fetched locally and the result says so."""from __future__ import annotationsimport refrom html.parser import HTMLParser-from typing import Any, Callable+from typing import Any, Callable, Optionalimport aisuite as ai-from .guard import get_checked+from ..secrets import SecretStore+from .guard import check_url, get_checked+from .tool import fetch_provider_MAX = 20000 # default chars returned@@ -75,7 +83,7 @@ def _html_to_text(html: str) -> str:return re.sub(r"\n{3,}", "\n\n", "\n".join(parser.parts))-def make_web_fetch_tool() -> Callable[..., Any]:+def make_web_fetch_tool(secrets: Optional[SecretStore] = None) -> Callable[..., Any]:def web_fetch(url: str, max_chars: int = _MAX) -> dict[str, Any]:if not isinstance(url, str) or not url.lower().startswith(("http://", "https://")@@ -83,6 +91,30 @@ def make_web_fetch_tool() -> Callable[..., Any]:return {"error": "url must start with http:// or https://"}cap = max_chars if isinstance(max_chars, int) and max_chars > 0 else _MAXcap = min(cap, 100000)+ notice = ""+ try:+ provider = fetch_provider(secrets)+ except Exception: # noqa: BLE001 - an unreadable profile means a local fetch+ provider = None+ if provider is not None:+ # The provider sees the address, so the address is judged first, exactly as+ # for a local fetch (loopback, private ranges, the metadata endpoint).+ reason = check_url(url)+ if reason:+ return {"error": reason}+ try:+ text = provider.fetch_page(url)+ except Exception as exc: # noqa: BLE001 - fall back to a local fetch+ service = provider.display_name or provider.name+ notice = f"{service} could not fetch the page ({exc}); fetched from this machine instead."+ else:+ return {+ "url": url,+ "content_type": "text/markdown",+ "fetched_via": provider.name,+ "truncated": len(text) > cap,+ "text": text[:cap],+ }try:import httpx@@ -104,12 +136,15 @@ def make_web_fetch_tool() -> Callable[..., Any]:except Exception as exc: # network / HTTP / TLSreturn {"error": f"fetch failed: {exc}"}text = _html_to_text(body) if "html" in ctype.lower() else body- return {+ out: dict[str, Any] = {"url": final_url,"content_type": ctype,"truncated": len(text) > cap,"text": text[:cap],}+ if notice:+ out["notice"] = notice+ return outweb_fetch.__name__ = "web_fetch"web_fetch.__doc__ = _SCHEMA["function"]["description"]diff --git a/tests/test_web_fetch_via_provider.py b/tests/test_web_fetch_via_provider.pynew file mode 100644index 00000000..9d29e973--- /dev/null+++ b/tests/test_web_fetch_via_provider.py@@ -0,0 +1,224 @@+"""`web_fetch` through the search service (Settings ▸ Customization).++No network: httpx is stubbed. Covers the three providers' request/response shapes+(Exa /contents, Firecrawl /v2/scrape, Tavily /extract), when `fetch_provider` picks one,+the address guard running before the provider is asked, the fallback to a local fetch+with a notice, and the approval card's `fetch_provider` extra.+"""++from __future__ import annotations++import httpx+import pytest++from coworker.secrets import SecretStore+from coworker.web import fetch_provider, fetch_provider_name+from coworker.web.fetch import make_web_fetch_tool+from coworker.web.providers import (+ _PROVIDERS,+ BraveProvider,+ ExaProvider,+ FirecrawlProvider,+ TavilyProvider,+)+++def _post_stub(monkeypatch, status, payload, captured):+ def fake_post(url, headers=None, json=None, timeout=None):+ captured.update(url=url, headers=headers, json=json, timeout=timeout)+ return httpx.Response(status, request=httpx.Request("POST", url), json=payload)++ monkeypatch.setattr(httpx, "post", fake_post)+++def test_only_exa_firecrawl_and_tavily_fetch_pages():+ assert {n for n, c in _PROVIDERS.items() if c.supports_fetch} == {"exa", "firecrawl", "tavily"}+ with pytest.raises(NotImplementedError):+ BraveProvider("k").fetch_page("https://example.com")+++def test_exa_fetch_page_request_and_response(monkeypatch):+ got = {}+ _post_stub(monkeypatch, 200, {+ "results": [{"title": "T", "url": "https://example.com", "text": "page text"}],+ "statuses": [{"id": "https://example.com", "status": "success", "source": "crawled"}],+ }, got)+ assert ExaProvider("exa-k").fetch_page("https://example.com") == "page text"+ assert got["url"] == "https://api.exa.ai/contents"+ assert got["headers"]["x-api-key"] == "exa-k"+ assert got["json"] == {"urls": ["https://example.com"], "text": True}++ _post_stub(monkeypatch, 200, {+ "results": [],+ "statuses": [{"id": "x", "status": "error", "error": {"tag": "CRAWL_NOT_FOUND", "httpStatusCode": 404}}],+ }, {})+ with pytest.raises(RuntimeError, match="CRAWL_NOT_FOUND \\(HTTP 404\\)"):+ ExaProvider("exa-k").fetch_page("https://example.com/missing")+++def test_firecrawl_fetch_page_request_and_response(monkeypatch):+ got = {}+ _post_stub(monkeypatch, 200, {"success": True, "data": {"markdown": "# Hi", "metadata": {"title": "Hi"}}}, got)+ assert FirecrawlProvider("fc-k").fetch_page("https://example.com") == "# Hi"+ assert got["url"] == "https://api.firecrawl.dev/v2/scrape"+ assert got["headers"]["Authorization"] == "Bearer fc-k"+ assert got["json"] == {"url": "https://example.com", "formats": ["markdown"]}++ _post_stub(monkeypatch, 200, {"success": False, "error": "blocked by robots"}, {})+ with pytest.raises(RuntimeError, match="blocked by robots"):+ FirecrawlProvider("fc-k").fetch_page("https://example.com")+++def test_tavily_fetch_page_request_and_response(monkeypatch):+ got = {}+ _post_stub(monkeypatch, 200, {+ "results": [{"url": "https://example.com", "raw_content": "body"}],+ "failed_results": [],+ }, got)+ assert TavilyProvider("tv-k").fetch_page("https://example.com") == "body"+ assert got["url"] == "https://api.tavily.com/extract"+ assert got["headers"]["Authorization"] == "Bearer tv-k"+ assert got["json"] == {"urls": ["https://example.com"]}++ _post_stub(monkeypatch, 200, {"results": [], "failed_results": [{"url": "x", "error": "timeout"}]}, {})+ with pytest.raises(RuntimeError, match="timeout"):+ TavilyProvider("tv-k").fetch_page("https://example.com")++ _post_stub(monkeypatch, 401, {}, {})+ with pytest.raises(httpx.HTTPStatusError):+ TavilyProvider("tv-k").fetch_page("https://example.com")+++def _secrets(tmp_path, monkeypatch, **profile):+ for name in _PROVIDERS:+ monkeypatch.delenv(f"{name.upper()}_API_KEY", raising=False)+ secrets = SecretStore(tmp_path / "secrets.json")+ secrets.put("web_search:default", profile)+ return secrets+++def test_fetch_provider_needs_the_switch_support_and_a_key(tmp_path, monkeypatch):+ # Off: local fetch, whatever the provider.+ s = _secrets(tmp_path, monkeypatch, provider="exa", keys={"exa": "k"})+ assert fetch_provider(s) is None and fetch_provider_name(s) == ""+ # On with a provider that offers it and has a key.+ s = _secrets(tmp_path, monkeypatch, provider="exa", keys={"exa": "k"}, fetch_via_provider=True)+ assert isinstance(fetch_provider(s), ExaProvider) and fetch_provider_name(s) == "exa"+ # On, but the active provider does not fetch pages.+ s = _secrets(tmp_path, monkeypatch, provider="brave", keys={"brave": "k", "exa": "k"}, fetch_via_provider=True)+ assert fetch_provider(s) is None+ s = _secrets(tmp_path, monkeypatch, provider="duckduckgo", fetch_via_provider=True)+ assert fetch_provider(s) is None+ # On, a fetching provider, but no key anywhere: a local fetch rather than an error.+ s = _secrets(tmp_path, monkeypatch, provider="tavily", fetch_via_provider=True)+ assert fetch_provider(s) is None+ monkeypatch.setenv("TAVILY_API_KEY", "env-k")+ assert fetch_provider(s).api_key == "env-k"+++def test_web_fetch_goes_through_the_provider_after_the_address_check(tmp_path, monkeypatch):+ import coworker.web.fetch as fetch_mod++ s = _secrets(tmp_path, monkeypatch, provider="firecrawl", keys={"firecrawl": "k"}, fetch_via_provider=True)+ got = {}+ _post_stub(monkeypatch, 200, {"success": True, "data": {"markdown": "x" * 50}}, got)+ checked = []+ monkeypatch.setattr(fetch_mod, "check_url", lambda url: checked.append(url) or None)++ def no_local(*_a, **_k):+ raise AssertionError("the page must not be fetched locally")++ monkeypatch.setattr(fetch_mod, "get_checked", no_local)+ out = make_web_fetch_tool(s)("https://example.com/doc", max_chars=10)+ assert checked == ["https://example.com/doc"]+ assert got["json"]["url"] == "https://example.com/doc"+ assert out == {+ "url": "https://example.com/doc",+ "content_type": "text/markdown",+ "fetched_via": "firecrawl",+ "truncated": True,+ "text": "x" * 10,+ }+++def test_web_fetch_refuses_blocked_addresses_before_asking_the_provider(tmp_path, monkeypatch):+ s = _secrets(tmp_path, monkeypatch, provider="exa", keys={"exa": "k"}, fetch_via_provider=True)++ def no_post(*_a, **_k):+ raise AssertionError("the provider must not see a blocked address")++ monkeypatch.setattr(httpx, "post", no_post)+ out = make_web_fetch_tool(s)("http://127.0.0.1:11434/api/tags")+ assert "loopback" in out["error"]+ out = make_web_fetch_tool(s)("http://169.254.169.254/latest/meta-data/")+ assert "link-local" in out["error"]+++def test_web_fetch_falls_back_to_a_local_fetch_with_a_notice(tmp_path, monkeypatch):+ import coworker.web.fetch as fetch_mod++ s = _secrets(tmp_path, monkeypatch, provider="tavily", keys={"tavily": "k"}, fetch_via_provider=True)+ _post_stub(monkeypatch, 500, {}, {})+ monkeypatch.setattr(fetch_mod, "check_url", lambda url: None)++ def local(client, url):+ resp = httpx.Response(+ 200, request=httpx.Request("GET", url), text="<html><body><p>local</p></body></html>",+ headers={"content-type": "text/html"},+ )+ resp.extensions["logical_url"] = url+ return resp++ monkeypatch.setattr(fetch_mod, "get_checked", local)+ out = make_web_fetch_tool(s)("https://example.com/")+ assert out["text"] == "local" and out["url"] == "https://example.com/"+ assert "fetched_via" not in out+ assert out["notice"].startswith("Tavily could not fetch the page (")+ assert out["notice"].endswith("); fetched from this machine instead.")+ assert "k" * 2 not in out["notice"]+++def test_web_fetch_without_the_switch_is_unchanged(tmp_path, monkeypatch):+ import coworker.web.fetch as fetch_mod++ s = _secrets(tmp_path, monkeypatch, provider="exa", keys={"exa": "k"})+ monkeypatch.setattr(httpx, "post", lambda *a, **k: (_ for _ in ()).throw(AssertionError("no provider call")))++ def local(client, url):+ resp = httpx.Response(200, request=httpx.Request("GET", url), text="plain", headers={"content-type": "text/plain"})+ resp.extensions["logical_url"] = url+ return resp++ monkeypatch.setattr(fetch_mod, "get_checked", local)+ out = make_web_fetch_tool(s)("https://example.com/")+ assert out == {"url": "https://example.com/", "content_type": "text/plain", "truncated": False, "text": "plain"}+++def test_web_fetch_approval_card_names_the_fetching_service(tmp_path, monkeypatch):+ from coworker.agent import build_engine+ from coworker.agents import chat_agent++ class _Stub:+ def complete(self, **_kw):+ from coworker.providers import AssistantTurn++ return AssistantTurn()++ def capabilities(self, _model):+ from coworker.providers.base import ModelCapabilities++ return ModelCapabilities()++ def stream(self, **_kw):+ from coworker.providers.base import StreamChunk++ yield StreamChunk(turn=self.complete())++ s = _secrets(tmp_path, monkeypatch, provider="exa", keys={"exa": "k"})+ eng = build_engine(agent=chat_agent(), provider=_Stub(), secrets=s)+ assert "web_fetch" in eng.registry.names()+ assert eng.approval_extras("web_fetch", {"url": "https://x"}) == {}+ assert eng.approval_extras("web_search", {}) == {"search_provider": "exa"}+ s.put("web_search:default", {"provider": "exa", "keys": {"exa": "k"}, "fetch_via_provider": True})+ # Resolved at card time, so the Settings change shows through without a new session.+ assert eng.approval_extras("web_fetch", {"url": "https://x"}) == {"fetch_provider": "exa"}