Python
feat(observability): decision-model reviewer checks get a model step with usage (OPE-244)
Devika Vermad57db12942c6
diff --git a/coworker/decision_reviewer.py b/coworker/decision_reviewer.pyindex c36f4d00..f35ebd4d 100644--- a/coworker/decision_reviewer.py+++ b/coworker/decision_reviewer.py@@ -1080,6 +1080,10 @@ BACKENDS: dict[str, type[DecisionBackend]] = {# -- the reviewer -----------------------------------------------------------------------------+# Backend name -> the recording's provider / wire name (recording format §6).+_WIRES = {"openai-decisions": "openai_decisions", "jev": "typesafe_jev"}++class DecisionReviewer:"""Same `review` signature and `Verdict` return as `coworker.reviewer.Reviewer`, so theengine, which duck-types its reviewer, can use either."""@@ -1108,9 +1112,13 @@ class DecisionReviewer:provenance: str = "",action_context: dict[str, Any] | None = None,facts: Sequence[str] = (),+ view: Any = None,) -> DecisionVerdict:"""Never raises. Every failure mode is an `unsure`.+ `view`, when given, records the request to the service as a model step nested in+ the check, with its usage and provider metadata (observability E23, TD-16).+`facts` are the engine's own lines (OPE-235/OPE-237, coworker/review_facts.py), thesame ones the session-model reviewer gets. They are used in code, not added to thestate the model reads, so the answers stay the ones the questions were measured on:@@ -1135,10 +1143,15 @@ class DecisionReviewer:if len(json.dumps(state, ensure_ascii=False)) > _STATE_CHAR_BUDGET:return self._count(_fail_closed("state exceeds the decision model's limit; a human must decide", error=True))questions = self.backend.questions++ def ask() -> Any:+ if view is not None and hasattr(view, "record_call"):+ return view.record_call(lambda: self.backend.ask_questions(state, questions),+ model=f"{_WIRES.get(self.backend.name, self.backend.name)}:{self.backend.model}")+ return self.backend.ask_questions(state, questions)+try:- answer = await asyncio.wait_for(- asyncio.to_thread(self.backend.ask_questions, state, questions), timeout=self.timeout- )+ answer = await asyncio.wait_for(asyncio.to_thread(ask), timeout=self.timeout)except asyncio.TimeoutError:return self._count(_fail_closed(f"{self.backend.label} timed out", error=True))except asyncio.CancelledError:diff --git a/coworker/engine.py b/coworker/engine.pyindex d6197b52..ce101c3d 100644--- a/coworker/engine.py+++ b/coworker/engine.py@@ -2040,7 +2040,10 @@ class TurnEngine:"model": getattr(reviewer, "model", None) if kind == "session_model" else None}},)if step_id is not None and _accepts_view(reviewer):- review_kw["view"] = _tag_provider(reviewer.provider, parent=step_id, call_id=tool_call.id,+ # A decision-model reviewer has no provider of its own: its request is recorded+ # through the engine's view (E23).+ review_kw["view"] = _tag_provider(getattr(reviewer, "provider", None) or self.provider,+ parent=step_id, call_id=tool_call.id,shadow=shadow, purpose="auto_approver")try:verdict = await reviewer.review(**review_kw)diff --git a/coworker/observability/provider_view.py b/coworker/observability/provider_view.pyindex b2626bb7..f9e1d6f4 100644--- a/coworker/observability/provider_view.py+++ b/coworker/observability/provider_view.py@@ -27,6 +27,7 @@ import loggingimport threadingimport timefrom dataclasses import dataclass, replace+from types import SimpleNamespacefrom typing import Any, Callable, Iterator, Optionalfrom ..providers.base import ProviderClient@@ -91,6 +92,23 @@ class ProviderView(ProviderClient):return getattr(self.base.inner, name)# -- calls --------------------------------------------------------------------------------+ def record_call(self, fn: Callable[[], Any], *, model: str) -> Any:+ """Record a model call made outside the provider router as a model step: a+ decision-model reviewer's request to its own service (E23, TD-16). `fn` makes the+ request on this thread; its HTTP exchange is captured, and the usage is read from+ the service's reply with the Responses-style mapping (input includes cached)."""+ call = _ModelCall.begin(self.tags, model=model, tools=None, streamed=False)+ if call is None:+ return fn()+ try:+ with call.http.active():+ result = fn()+ except BaseException as exc:+ call.failed(exc)+ raise+ call.finished(SimpleNamespace(finish_reason=None, usage=_reply_usage(call.http.entries()), served_by=None))+ return result+def complete(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any):call = _ModelCall.begin(self.tags, model=model, tools=tools, streamed=False)if call is None:@@ -324,6 +342,27 @@ class _ModelCall:log.exception("recording: could not record a failed model step")+def _reply_usage(entries: list[dict[str, Any]]) -> Any:+ """The `usage` object of the last JSON reply, as `TokenUsage`; None if there is none."""+ from ..providers.openai_responses import _usage_from++ for e in reversed(entries):+ try:+ body = json.loads(e.get("body") or "")+ except ValueError:+ continue+ usage = body.get("usage") if isinstance(body, dict) else None+ if isinstance(usage, dict):+ return _usage_from(_ns(usage))+ return None+++def _ns(value: Any) -> Any:+ if isinstance(value, dict):+ return SimpleNamespace(**{k: _ns(v) for k, v in value.items()})+ return value++def _is_stop(exc: BaseException) -> bool:return isinstance(exc, (GeneratorExit, KeyboardInterrupt))diff --git a/tests/test_observability_model_steps.py b/tests/test_observability_model_steps.pyindex 9e7d7b40..fd17ac8d 100644--- a/tests/test_observability_model_steps.py+++ b/tests/test_observability_model_steps.py@@ -254,3 +254,14 @@ def test_capture_is_off_outside_a_recorded_call(fake_server):assert getattr(capture._active, "log", None) is Noneinner.complete(model="model-a", messages=[{"role": "user", "content": "x"}])assert getattr(capture._active, "log", None) is None+++DECISION_VECTORS = [v for v in VECTORS if v["family"] in ("openai_decisions", "typesafe_jev")]++[email protected]("vector", DECISION_VECTORS, ids=[v["id"] for v in DECISION_VECTORS])+def test_decision_reviewer_usage_vectors(vector):+ from coworker.observability.provider_view import _reply_usage++ entries = [{"body": json.dumps({"usage": vector["raw"]})}]+ assert usage_attrs(_reply_usage(entries)) == {"reported": True, **vector["expected"]}diff --git a/tests/test_observability_reviewer_helper.py b/tests/test_observability_reviewer_helper.pyindex 56db0251..0b26126d 100644--- a/tests/test_observability_reviewer_helper.py+++ b/tests/test_observability_reviewer_helper.py@@ -192,3 +192,54 @@ def test_explore_helper_steps_are_nested_under_the_explore_step(tmp_path):assert len(d["turns"]) == 1mains = [s for s in d["steps"] if s["kind"] == "model" and s["attrs"]["purpose"] == "main"]assert len(mains) == 2+++# -- E23: a decision-model reviewer's request is a nested model step ----------------------------++import threading as _threading+from http.server import BaseHTTPRequestHandler, HTTPServer+++class _JevHandler(BaseHTTPRequestHandler):+ def do_POST(self): # noqa: N802+ body = json.loads(self.rfile.read(int(self.headers.get("content-length") or 0)))+ answers = {name: {"noul": 0.97, "probabilities": {"yes": 0.97, "no": 0.03}}+ for name in (body.get("questions") or {})}+ data = json.dumps({"model": "jev-test-1", "answers": answers,+ "usage": {"input_tokens": 6203, "output_tokens": 529}}).encode()+ self.send_response(200)+ self.send_header("content-type", "application/json")+ self.send_header("x-request-id", "req_jev")+ self.send_header("content-length", str(len(data)))+ self.end_headers()+ self.wfile.write(data)++ def log_message(self, *a):+ pass+++def test_a_decision_reviewer_check_has_a_model_step_with_usage(tmp_path):+ from coworker.decision_reviewer import build_decision_reviewer++ server = HTTPServer(("127.0.0.1", 0), _JevHandler)+ _threading.Thread(target=server.serve_forever, daemon=True).start()+ try:+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make clean"}),+ AssistantTurn(text="done")], [])+ engine.reviewer = build_decision_reviewer(+ "jev", api_key="test-key-not-real", base_url=f"http://127.0.0.1:{server.server_address[1]}")+ engine.reviewer_choice = "jev"+ _run(engine)+ finally:+ server.shutdown()+ steps = rec.as_dict()["steps"]+ [check] = [s for s in steps if s["kind"] == "auto_approver"]+ assert check["attrs"]["reviewer"] == {"kind": "typesafe_jev", "external": True, "model": None}+ [inner] = [s for s in steps if s["kind"] == "model" and s["parent_step_id"] == check["step_id"]]+ assert inner["attrs"]["purpose"] == "auto_approver" and inner["attrs"]["provider"] == "typesafe_jev"+ assert inner["attrs"]["usage"] == {"reported": True, "input_fresh": 6203, "cache_read": None,+ "cache_write": None, "output": 529, "reasoning": None}+ assert inner["attrs"]["provider_meta"][0]["wire"] == "typesafe_jev"+ assert inner["attrs"]["provider_facts"]["provider_request_id"] == "req_jev"+ assert check["attrs"]["usage_reported"] is True+ assert "test-key-not-real" not in json.dumps(steps)