Skip to content
Python

Register Serper with the site wall and name every provider

Rohit C Prasad2a19c61345b9
diff --git a/coworker/sandbox/network_profiles.py b/coworker/sandbox/network_profiles.py
index 88778dd0..49f1830f 100644
--- a/coworker/sandbox/network_profiles.py
+++ b/coworker/sandbox/network_profiles.py
@@ -15,7 +15,7 @@ import sys
CODE_HOSTS = ["github.com", "api.github.com", "codeload.github.com", "objects.githubusercontent.com", "raw.githubusercontent.com", "gitlab.com"]
PACKAGE_REGISTRIES = ["pypi.org", "files.pythonhosted.org", "registry.npmjs.org", "crates.io", "static.crates.io", "index.crates.io", "proxy.golang.org", "sum.golang.org"]
-SEARCH_APIS = ["api.search.brave.com", "api.tavily.com", "html.duckduckgo.com", "duckduckgo.com"]
+SEARCH_APIS = ["api.search.brave.com", "api.tavily.com", "html.duckduckgo.com", "duckduckgo.com", "google.serper.dev"]
# The page's catalogue, by group. None is allowed until ticked.
SITES: dict[str, list[str]] = {"code-hosts": CODE_HOSTS, "package-registries": PACKAGE_REGISTRIES, "search-apis": SEARCH_APIS}
diff --git a/coworker/web/providers.py b/coworker/web/providers.py
index 04d70682..d8af91c0 100644
--- a/coworker/web/providers.py
+++ b/coworker/web/providers.py
@@ -26,6 +26,10 @@ class SearchResult:
class WebSearchProvider(ABC):
name: str = "base"
+ # What Settings shows for this provider, and where a person gets a key (empty when the
+ # provider is keyless).
+ display_name: str = ""
+ key_url: str = ""
requires_key: bool = False
@abstractmethod
@@ -36,6 +40,7 @@ class DuckDuckGoProvider(WebSearchProvider):
"""Keyless default via the `ddgs` library."""
name = "duckduckgo"
+ display_name = "DuckDuckGo"
requires_key = False
def search(self, query: str, max_results: int = 5) -> list[SearchResult]:
@@ -54,6 +59,8 @@ class DuckDuckGoProvider(WebSearchProvider):
class TavilyProvider(WebSearchProvider):
name = "tavily"
+ display_name = "Tavily"
+ key_url = "https://tavily.com"
requires_key = True
def __init__(self, api_key: str) -> None:
@@ -67,6 +74,7 @@ class TavilyProvider(WebSearchProvider):
json={"api_key": self.api_key, "query": query, "max_results": max_results},
timeout=_TIMEOUT,
)
+ resp.raise_for_status()
data = resp.json()
return [
SearchResult(
@@ -80,6 +88,8 @@ class TavilyProvider(WebSearchProvider):
class BraveProvider(WebSearchProvider):
name = "brave"
+ display_name = "Brave Search"
+ key_url = "https://brave.com/search/api"
requires_key = True
def __init__(self, api_key: str) -> None:
@@ -97,6 +107,7 @@ class BraveProvider(WebSearchProvider):
params={"q": query, "count": max_results},
timeout=_TIMEOUT,
)
+ resp.raise_for_status()
data = resp.json()
return [
SearchResult(
@@ -117,6 +128,8 @@ class SerperProvider(WebSearchProvider):
"""
name = "serper"
+ display_name = "Serper"
+ key_url = "https://serper.dev"
requires_key = True
def __init__(self, api_key: str) -> None:
@@ -131,6 +144,7 @@ class SerperProvider(WebSearchProvider):
json={"q": query, "num": max_results},
timeout=_TIMEOUT,
)
+ resp.raise_for_status()
data = resp.json()
return [
SearchResult(
diff --git a/coworker/web/tool.py b/coworker/web/tool.py
index b24d73e6..61435c5c 100644
--- a/coworker/web/tool.py
+++ b/coworker/web/tool.py
@@ -57,6 +57,7 @@ PROVIDER_HOSTS = {
"duckduckgo": "duckduckgo.com",
"tavily": "api.tavily.com",
"brave": "api.search.brave.com",
+ "serper": "google.serper.dev",
}
diff --git a/docs/config.example.toml b/docs/config.example.toml
index 77637ad8..75a19a15 100644
--- a/docs/config.example.toml
+++ b/docs/config.example.toml
@@ -19,8 +19,8 @@ max_iterations = 12 # max model<->tool iterations per turn before stopp
# default (Anthropic high, Kimi K3 max). Env override:
# COWORKER_REASONING_EFFORT.
-# Web search backend: "duckduckgo" (keyless, default) | "tavily" | "brave"
-# (the latter two need an API key, set in the app's Settings or secret store).
+# Web search backend: "duckduckgo" (keyless, default) | "tavily" | "brave" | "serper"
+# (the keyed ones need an API key, set in the app's Settings or secret store).
web_search_provider = "duckduckgo"
# Commands auto-allowed without an approval prompt (prefix match).
diff --git a/tests/test_web_search.py b/tests/test_web_search.py
index 389b50f2..2cd3e27c 100644
--- a/tests/test_web_search.py
+++ b/tests/test_web_search.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import pytest
+from coworker.sandbox.network_profiles import SEARCH_APIS
from coworker.secrets import SecretStore
from coworker.web import (
SearchResult,
@@ -16,12 +17,14 @@ from coworker.web import (
provider_names,
)
from coworker.web.providers import (
+ _PROVIDERS,
BraveProvider,
DuckDuckGoProvider,
SerperProvider,
TavilyProvider,
WebSearchProvider,
)
+from coworker.web.tool import PROVIDER_HOSTS
class FakeProvider(WebSearchProvider):
@@ -93,6 +96,10 @@ def test_serper_sends_correct_request_and_maps_organic_results(monkeypatch):
sent = {}
class _Resp:
+ @staticmethod
+ def raise_for_status():
+ pass
+
@staticmethod
def json():
return {
@@ -123,6 +130,34 @@ def test_serper_sends_correct_request_and_maps_organic_results(monkeypatch):
assert results[0].snippet == "An example snippet."
+def test_keyed_providers_raise_on_http_errors(monkeypatch):
+ """A 401/429/5xx surfaces as an error instead of an empty result list."""
+ import httpx
+
+ def denied(url, **kwargs):
+ return httpx.Response(401, request=httpx.Request("POST", url))
+
+ monkeypatch.setattr(httpx, "post", denied)
+ monkeypatch.setattr(httpx, "get", denied)
+ for provider in (TavilyProvider("k"), BraveProvider("k"), SerperProvider("k")):
+ with pytest.raises(httpx.HTTPStatusError):
+ provider.search("q")
+
+
+def test_every_provider_has_a_name_and_a_key_page():
+ for name, cls in _PROVIDERS.items():
+ assert cls.name == name and cls.display_name, name
+ assert bool(cls.key_url) == cls.requires_key, name # keyless providers have no key page
+
+
+def test_every_provider_is_known_to_the_site_wall():
+ """`web_search` has no url argument: the sandbox's allowed-sites wall checks the
+ provider's host (PROVIDER_HOSTS), and the Access page offers it under Search."""
+ for name in provider_names():
+ host = PROVIDER_HOSTS[name]
+ assert host in SEARCH_APIS, (name, host)
+
+
def test_tool_surfaces_missing_key_error(tmp_path):
secrets = SecretStore(tmp_path / "secrets.json")
secrets.put("web_search:default", {"provider": "tavily"}) # no api_key