Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

Give the eval runner the same Anthropic client the app uses

Claude30d1da2f574d
diff --git a/evals/judges/executive_quality_judge.py b/evals/judges/executive_quality_judge.py
index 8ad33f7d..580956ca 100644
--- a/evals/judges/executive_quality_judge.py
+++ b/evals/judges/executive_quality_judge.py
@@ -1,8 +1,6 @@
"""LLM-as-judge for executive response quality."""
from __future__ import annotations
-import anthropic
-
JUDGE_SYSTEM_PROMPT = """You are a critical evaluator of AI executive advisory systems. Your job is to assess whether responses meet the standard of a senior business executive with 25+ years of experience.
You are harsh but fair. You penalize:
@@ -27,7 +25,9 @@ async def judge(
expected_topics: list[str] | None = None,
api_key: str | None = None,
) -> dict:
- client = anthropic.AsyncAnthropic(api_key=api_key)
+ from openexecutive.providers.anthropic_provider import configured_async_client
+
+ client = configured_async_client(api_key=api_key)
context_str = ""
if company_context:
diff --git a/evals/judges/triage_judge.py b/evals/judges/triage_judge.py
index 8446a288..318c25b6 100644
--- a/evals/judges/triage_judge.py
+++ b/evals/judges/triage_judge.py
@@ -63,7 +63,9 @@ async def judge_triage(
decision: dict[str, Any],
client: anthropic.AsyncAnthropic | None = None,
) -> dict[str, Any]:
- cli = client or anthropic.AsyncAnthropic()
+ from openexecutive.providers.anthropic_provider import configured_async_client
+
+ cli = client or configured_async_client()
judge_prompt = f"""You are an evaluator for a Triage agent that decides whether incoming
events (emails, Slack messages, documents) should fire proactive alerts to a
diff --git a/evals/run_evals.py b/evals/run_evals.py
index 3f14ba2d..e0bbf6b8 100644
--- a/evals/run_evals.py
+++ b/evals/run_evals.py
@@ -246,9 +246,12 @@ async def main() -> None:
initialize_overrides_db()
ReviewStore.initialize_db()
- import anthropic
+ # Built through the app's factory so an organisation-scoped key gets its
+ # `anthropic-workspace-id` header here too — a bare AsyncAnthropic() 400s
+ # on every judge call for those keys (#128).
+ from openexecutive.providers.anthropic_provider import configured_async_client
- client = anthropic.AsyncAnthropic()
+ client = configured_async_client()
scenario_dir = Path(args.scenarios)
output_dir = Path(args.output)
diff --git a/packages/core/openexecutive/providers/anthropic_provider.py b/packages/core/openexecutive/providers/anthropic_provider.py
index ce4dd1ab..2dd6c492 100644
--- a/packages/core/openexecutive/providers/anthropic_provider.py
+++ b/packages/core/openexecutive/providers/anthropic_provider.py
@@ -7,6 +7,54 @@ from typing import Any
import anthropic
+def _client_kwargs(
+ *, api_key: str | None, timeout: float | None, workspace_id: str | None
+) -> dict[str, Any]:
+ """The one place the Anthropic client's construction is defined.
+
+ Shared so that the header an organisation-scoped key needs cannot be
+ attached in one caller and forgotten in another — which is exactly how
+ #128 would have half-shipped, with chat fixed and `make eval` still 400ing.
+
+ An omitted key is left out entirely rather than passed as ``None``, so the
+ SDK's own ``ANTHROPIC_API_KEY`` lookup still applies.
+ """
+ kwargs: dict[str, Any] = {}
+ if api_key:
+ kwargs["api_key"] = api_key
+ if timeout is not None:
+ kwargs["timeout"] = timeout
+ if workspace_id:
+ # An organisation-scoped key carries no workspace of its own, so
+ # Anthropic 400s every call without this header. Set as a default
+ # header rather than per-request: it applies to create and stream alike.
+ kwargs["default_headers"] = {"anthropic-workspace-id": workspace_id}
+ return kwargs
+
+
+def configured_async_client(
+ *,
+ api_key: str | None = None,
+ timeout: float | None = None,
+) -> anthropic.AsyncAnthropic:
+ """An ``AsyncAnthropic`` carrying this install's workspace configuration.
+
+ For callers outside the request path — the eval runner and its judges —
+ which build their own client rather than going through the registry.
+ Anything omitted is resolved from ``Settings``.
+ """
+ from openexecutive.config import get_settings
+
+ settings = get_settings()
+ return anthropic.AsyncAnthropic(
+ **_client_kwargs(
+ api_key=api_key or settings.anthropic_api_key,
+ timeout=timeout,
+ workspace_id=getattr(settings, "anthropic_workspace_id", None),
+ )
+ )
+
+
class AnthropicProvider:
"""Thin delegate over ``anthropic.AsyncAnthropic``.
@@ -21,16 +69,12 @@ class AnthropicProvider:
timeout: float | None = None,
workspace_id: str | None = None,
) -> None:
- kwargs: dict[str, Any] = {"api_key": api_key}
- if timeout is not None:
- kwargs["timeout"] = timeout
- if workspace_id:
- # An organisation-scoped key carries no workspace of its own, so
- # Anthropic 400s every call without this header. Set as a default
- # header rather than per-request: it applies to create and stream
- # alike, and every Claude call in the app comes through here.
- kwargs["default_headers"] = {"anthropic-workspace-id": workspace_id}
- self._client = anthropic.AsyncAnthropic(**kwargs)
+ # Explicit, never settings-resolving: the registry passes what it
+ # read, and a test constructing this directly must not pick up an
+ # ambient workspace id from the environment.
+ self._client = anthropic.AsyncAnthropic(
+ **_client_kwargs(api_key=api_key, timeout=timeout, workspace_id=workspace_id)
+ )
def messages_create(self, **kwargs: Any) -> Awaitable[Any]:
return self._client.messages.create(**kwargs)
diff --git a/packages/core/tests/unit/test_anthropic_workspace_id.py b/packages/core/tests/unit/test_anthropic_workspace_id.py
index af346bdc..ce1e55c2 100644
--- a/packages/core/tests/unit/test_anthropic_workspace_id.py
+++ b/packages/core/tests/unit/test_anthropic_workspace_id.py
@@ -20,7 +20,10 @@ os.environ.setdefault("ANTHROPIC_API_KEY", "sk-test-not-used")
import pytest # noqa: E402
from openexecutive.providers import registry as registry_mod # noqa: E402
-from openexecutive.providers.anthropic_provider import AnthropicProvider # noqa: E402
+from openexecutive.providers.anthropic_provider import ( # noqa: E402
+ AnthropicProvider,
+ configured_async_client,
+)
@pytest.fixture(autouse=True)
@@ -178,3 +181,51 @@ def test_surrounding_whitespace_is_trimmed(monkeypatch: pytest.MonkeyPatch) -> N
monkeypatch.setenv("ANTHROPIC_WORKSPACE_ID", " wrkspc_123 ")
assert get_settings().anthropic_workspace_id == "wrkspc_123"
+
+
+# ── callers outside the request path ──────────────────────────────────────
+
+
+def test_configured_client_carries_the_workspace_header(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The eval runner and its judges build their own client rather than going
+ through the registry. Without this factory they would 400 on every judge
+ call for an organisation-scoped key, with chat working fine — the same bug
+ surviving in the one place nobody looks."""
+ monkeypatch.setenv("ANTHROPIC_WORKSPACE_ID", "wrkspc_evals")
+ monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test")
+
+ client = configured_async_client()
+
+ assert dict(client.default_headers)["anthropic-workspace-id"] == "wrkspc_evals"
+
+
+def test_configured_client_sends_no_header_when_unset(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("ANTHROPIC_WORKSPACE_ID", raising=False)
+ monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test")
+
+ client = configured_async_client()
+
+ assert "anthropic-workspace-id" not in dict(client.default_headers)
+
+
+def test_configured_client_keeps_an_explicit_api_key(monkeypatch: pytest.MonkeyPatch) -> None:
+ """`executive_quality_judge` passes its own key through."""
+ monkeypatch.setenv("ANTHROPIC_WORKSPACE_ID", "wrkspc_evals")
+ monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-from-env")
+
+ client = configured_async_client(api_key="sk-explicit")
+
+ assert client.api_key == "sk-explicit"
+ assert dict(client.default_headers)["anthropic-workspace-id"] == "wrkspc_evals"
+
+
+def test_provider_does_not_resolve_a_workspace_from_the_environment(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The provider stays explicit. Only the registry decides what it gets, so
+ a stray env var cannot change what a directly-constructed provider sends."""
+ monkeypatch.setenv("ANTHROPIC_WORKSPACE_ID", "wrkspc_ambient")
+
+ provider = AnthropicProvider(api_key="sk-test")
+
+ assert "anthropic-workspace-id" not in _headers(provider)