Python
Register You.com with the site wall and make key handling uniform
Rohit C Prasadd55ca630979a
diff --git a/coworker/sandbox/network_profiles.py b/coworker/sandbox/network_profiles.pyindex 026c24bb..4f253ac1 100644--- a/coworker/sandbox/network_profiles.py+++ b/coworker/sandbox/network_profiles.py@@ -15,7 +15,7 @@ import sysCODE_HOSTS = ["github.com", "api.github.com", "codeload.github.com", "objects.githubusercontent.com", "raw.githubusercontent.com", "gitlab.com"]PACKAGE_REGISTRIES = ["pypi.org", "files.pythonhosted.org", "registry.npmjs.org", "crates.io", "static.crates.io", "index.crates.io", "proxy.golang.org", "sum.golang.org"]-SEARCH_APIS = ["api.search.brave.com", "api.tavily.com", "html.duckduckgo.com", "duckduckgo.com", "google.serper.dev", "api.exa.ai", "api.firecrawl.dev"]+SEARCH_APIS = ["api.search.brave.com", "api.tavily.com", "html.duckduckgo.com", "duckduckgo.com", "google.serper.dev", "api.exa.ai", "api.firecrawl.dev", "api.you.com", "ydc-index.io"]# The page's catalogue, by group. None is allowed until ticked.SITES: dict[str, list[str]] = {"code-hosts": CODE_HOSTS, "package-registries": PACKAGE_REGISTRIES, "search-apis": SEARCH_APIS}diff --git a/coworker/web/providers.py b/coworker/web/providers.pyindex 6e7c7e92..a0caa44b 100644--- a/coworker/web/providers.py+++ b/coworker/web/providers.py@@ -9,7 +9,6 @@ lazy-imported.from __future__ import annotations-import osfrom abc import ABC, abstractmethodfrom dataclasses import dataclassfrom typing import Optional@@ -33,6 +32,10 @@ class WebSearchProvider(ABC):# provider is keyless).display_name: str = ""key_url: str = ""+ # Whether the provider takes a key: "none" (keyless), "optional" (works keyless, uses a+ # key when one is configured) or "required". `requires_key` / `accepts_key` say the same+ # thing for the callers that check one flag.+ key_mode: str = "none"requires_key: bool = False# Keyless provider that still uses a key when one is configured (better quota/features).# `build_provider` passes the key through instead of dropping it.@@ -47,6 +50,7 @@ class DuckDuckGoProvider(WebSearchProvider):name = "duckduckgo"display_name = "DuckDuckGo"+ key_mode = "none"requires_key = Falsedef search(self, query: str, max_results: int = 5) -> list[SearchResult]:@@ -66,13 +70,19 @@ class DuckDuckGoProvider(WebSearchProvider):class YouProvider(WebSearchProvider):"""You.com — keyless out of the box, better with a key.- Without a key it hits the free agents endpoint (100 searches/day, rate-limited per IP),- so it works in a fresh install like `duckduckgo` does. With a key it hits the Search API- (`YOU_API_KEY`, or `YDC_API_KEY` — the name You.com's own docs and SDKs use), which lifts- the quota. Both return the same `{"results": {"web": [...], "news": [...]}}` shape.+ Without a key it hits the agents endpoint, so it works in a fresh install like+ `duckduckgo` does. You.com does not document that endpoint and throttles it per IP+ (about 100 queries a day, per third-party docs), so results may stop without notice.+ With a key it hits the Search API, which lifts the quota. The key comes from the+ SecretStore or the environment (`YOU_API_KEY`, or `YDC_API_KEY` — the name You.com's+ own docs and SDKs use), resolved by `coworker.web.tool`, not here. Both endpoints+ return the same `{"results": {"web": [...], "news": [...]}}` shape."""name = "you"+ display_name = "You.com"+ key_url = "https://you.com/platform"+ key_mode = "optional"requires_key = Falseaccepts_key = True@@ -82,7 +92,7 @@ class YouProvider(WebSearchProvider):_USER_AGENT = "youdotcom-integration/andrewyng-openworker"def __init__(self, api_key: Optional[str] = None) -> None:- self.api_key = api_key or os.environ.get("YDC_API_KEY") or None+ self.api_key = api_key or Nonedef search(self, query: str, max_results: int = 5) -> list[SearchResult]:import httpx@@ -96,6 +106,7 @@ class YouProvider(WebSearchProvider):params={"query": query, "count": max_results},timeout=_TIMEOUT,)+ resp.raise_for_status()data = resp.json()results = data.get("results", {}) or {}# `count` applies per section; web first, news only fills any remainder.@@ -114,6 +125,7 @@ class TavilyProvider(WebSearchProvider):name = "tavily"display_name = "Tavily"key_url = "https://tavily.com"+ key_mode = "required"requires_key = Truedef __init__(self, api_key: str) -> None:@@ -143,6 +155,7 @@ class BraveProvider(WebSearchProvider):name = "brave"display_name = "Brave Search"key_url = "https://brave.com/search/api"+ key_mode = "required"requires_key = Truedef __init__(self, api_key: str) -> None:@@ -183,6 +196,7 @@ class SerperProvider(WebSearchProvider):name = "serper"display_name = "Serper"key_url = "https://serper.dev"+ key_mode = "required"requires_key = Truedef __init__(self, api_key: str) -> None:@@ -213,6 +227,7 @@ class ExaProvider(WebSearchProvider):name = "exa"display_name = "Exa"key_url = "https://exa.ai"+ key_mode = "required"requires_key = Truedef __init__(self, api_key: str) -> None:@@ -254,6 +269,7 @@ class FirecrawlProvider(WebSearchProvider):name = "firecrawl"display_name = "Firecrawl"key_url = "https://firecrawl.dev"+ key_mode = "required"requires_key = Truedef __init__(self, api_key: str) -> None:diff --git a/coworker/web/tool.py b/coworker/web/tool.pyindex 6346f53c..3ba299c2 100644--- a/coworker/web/tool.py+++ b/coworker/web/tool.py@@ -46,15 +46,15 @@ def provider_name("""The configured provider's NAME, without building (or validating) the provider.Same resolution order as `resolve_provider`. Used by the web_search approval card,which names the live destination (§1.9: "currently: ‹name›", never "default:")."""- secrets = secrets or SecretStore()- profile = secrets.get("web_search:default") or {}- return profile.get("provider") or _config_provider() or default+ return _lookup(secrets, default)[1]# Where each provider's queries go. The sandbox's allowed-sites wall checks this host for-# `web_search`, which has no url argument (OPE-219).-PROVIDER_HOSTS = {+# `web_search`, which has no url argument (OPE-219). A list means the destination depends+# on whether a key is configured: the keyless host first, the keyed host last.+PROVIDER_HOSTS: dict[str, str | list[str]] = {"duckduckgo": "duckduckgo.com",+ "you": ["api.you.com", "ydc-index.io"],"tavily": "api.tavily.com","brave": "api.search.brave.com","serper": "google.serper.dev",@@ -62,20 +62,40 @@ PROVIDER_HOSTS = {"firecrawl": "api.firecrawl.dev",}+# Other environment names a provider's key may come under, beside `{NAME}_API_KEY`: You.com's+# own docs and SDKs call it YDC_API_KEY.+_KEY_ENV_ALIASES = {"you": ("YDC_API_KEY",)}+def provider_host(secrets: Optional[SecretStore] = None) -> str:"""The host the configured search provider talks to; `''` for an unknown provider."""- return PROVIDER_HOSTS.get(provider_name(secrets), "")+ profile, name = _lookup(secrets, "duckduckgo")+ hosts = PROVIDER_HOSTS.get(name, "")+ if isinstance(hosts, str):+ return hosts+ return hosts[-1] if (profile.get("api_key") or _env_key(name)) else hosts[0]def resolve_provider(secrets: Optional[SecretStore] = None, *, default: str = "duckduckgo") -> WebSearchProvider:+ profile, name = _lookup(secrets, default)+ api_key = profile.get("api_key") or _env_key(name)+ return build_provider(name, api_key)+++def _lookup(secrets: Optional[SecretStore], default: str) -> tuple[dict, str]:+ """The `web_search:default` profile and the provider name it (or the config) selects."""secrets = secrets or SecretStore()profile = secrets.get("web_search:default") or {}- name = profile.get("provider") or _config_provider() or default- api_key = profile.get("api_key") or os.environ.get(f"{name.upper()}_API_KEY")- return build_provider(name, api_key)+ return profile, profile.get("provider") or _config_provider() or default+++def _env_key(name: str) -> Optional[str]:+ for var in (f"{name.upper()}_API_KEY", *_KEY_ENV_ALIASES.get(name, ())):+ if os.environ.get(var):+ return os.environ[var]+ return Nonedef _config_provider() -> Optional[str]:diff --git a/docs/config.example.toml b/docs/config.example.tomlindex f0ecef85..ea35d0ec 100644--- a/docs/config.example.toml+++ b/docs/config.example.toml@@ -19,7 +19,8 @@ max_iterations = 12 # max model<->tool iterations per turn before stopp# default (Anthropic high, Kimi K3 max). Env override:# COWORKER_REASONING_EFFORT.-# Web search backend: "duckduckgo" (keyless, default) | "tavily" | "brave" | "serper" | "exa" | "firecrawl"+# Web search backend: "duckduckgo" (keyless, default) | "you" (keyless, optional key)+# | "tavily" | "brave" | "serper" | "exa" | "firecrawl"# (the keyed ones need an API key, set in the app's Settings or secret store).web_search_provider = "duckduckgo"diff --git a/tests/test_web_search.py b/tests/test_web_search.pyindex 955bda38..7210daa3 100644--- a/tests/test_web_search.py+++ b/tests/test_web_search.py@@ -230,6 +230,8 @@ def test_keyed_providers_raise_on_http_errors(monkeypatch):SerperProvider("k"),ExaProvider("k"),FirecrawlProvider("k"),+ YouProvider(),+ YouProvider("k"),):with pytest.raises(httpx.HTTPStatusError):provider.search("q")@@ -238,31 +240,80 @@ def test_keyed_providers_raise_on_http_errors(monkeypatch):def test_every_provider_has_a_name_and_a_key_page():for name, cls in _PROVIDERS.items():assert cls.name == name and cls.display_name, name- assert bool(cls.key_url) == cls.requires_key, name # keyless providers have no key page+ assert cls.key_mode in ("none", "optional", "required"), name+ assert bool(cls.key_url) == (cls.key_mode != "none"), name # keyless: no key page+ # The one-flag views agree with the tri-state.+ assert cls.requires_key == (cls.key_mode == "required"), name+ assert cls.accepts_key == (cls.key_mode == "optional"), name+ assert DuckDuckGoProvider.key_mode == "none"+ assert YouProvider.key_mode == "optional"+ assert TavilyProvider.key_mode == "required"def test_every_provider_is_known_to_the_site_wall():"""`web_search` has no url argument: the sandbox's allowed-sites wall checks theprovider's host (PROVIDER_HOSTS), and the Access page offers it under Search."""for name in provider_names():- host = PROVIDER_HOSTS[name]- assert host in SEARCH_APIS, (name, host)+ hosts = PROVIDER_HOSTS[name]+ for host in [hosts] if isinstance(hosts, str) else hosts:+ assert host in SEARCH_APIS, (name, host)def test_build_provider_you_is_keyless_with_optional_key(monkeypatch):- monkeypatch.delenv("YDC_API_KEY", raising=False)+ monkeypatch.setenv("YDC_API_KEY", "ignored") # providers never read the environmentassert "you" in provider_names()keyless = build_provider("you") # no key neededassert isinstance(keyless, YouProvider) and keyless.api_key is Noneassert build_provider("you", "ydc-x").api_key == "ydc-x" # key is passed through- monkeypatch.setenv("YDC_API_KEY", "from-env")- assert build_provider("you").api_key == "from-env"+++def _you_env(monkeypatch, **values):+ for var in ("YOU_API_KEY", "YDC_API_KEY"):+ monkeypatch.delenv(var, raising=False)+ for var, value in values.items():+ monkeypatch.setenv(var, value)+++def test_resolve_provider_reads_the_you_key_from_either_env_name(tmp_path, monkeypatch):+ from coworker.web import resolve_provider++ secrets = SecretStore(tmp_path / "secrets.json")+ secrets.put("web_search:default", {"provider": "you"})+ _you_env(monkeypatch)+ assert resolve_provider(secrets).api_key is None+ _you_env(monkeypatch, YDC_API_KEY="from-ydc") # the name You.com's own docs use+ assert resolve_provider(secrets).api_key == "from-ydc"+ _you_env(monkeypatch, YOU_API_KEY="from-you", YDC_API_KEY="from-ydc")+ assert resolve_provider(secrets).api_key == "from-you" # the uniform name wins+ secrets.put("web_search:default", {"provider": "you", "api_key": "from-store"})+ assert resolve_provider(secrets).api_key == "from-store" # the store wins over both+++def test_provider_host_follows_the_you_key(tmp_path, monkeypatch):+ """You.com talks to one host keyless and another with a key; the site wall checks the+ one the next search will use."""+ from coworker.web import provider_host++ secrets = SecretStore(tmp_path / "secrets.json")+ secrets.put("web_search:default", {"provider": "you"})+ _you_env(monkeypatch)+ assert provider_host(secrets) == "api.you.com"+ _you_env(monkeypatch, YDC_API_KEY="k")+ assert provider_host(secrets) == "ydc-index.io"+ _you_env(monkeypatch)+ secrets.put("web_search:default", {"provider": "you", "api_key": "k"})+ assert provider_host(secrets) == "ydc-index.io"+ secrets.put("web_search:default", {"provider": "brave", "api_key": "k"})+ assert provider_host(secrets) == "api.search.brave.com" # single-host providers as beforedef _fake_you_get(captured, payload):"""Stand-in for httpx.get that records the request and replays a canned response."""class Resp:+ def raise_for_status(self):+ pass+def json(self):return payload