Skip to content
Python

feat(observability): decision-model reviewer checks get a model step with usage (OPE-244)

Devika Vermad57db12942c6
diff --git a/coworker/decision_reviewer.py b/coworker/decision_reviewer.py
index c36f4d00..f35ebd4d 100644
--- a/coworker/decision_reviewer.py
+++ b/coworker/decision_reviewer.py
@@ -1080,6 +1080,10 @@ BACKENDS: dict[str, type[DecisionBackend]] = {
# -- the reviewer -----------------------------------------------------------------------------
+# Backend name -> the recording's provider / wire name (recording format §6).
+_WIRES = {"openai-decisions": "openai_decisions", "jev": "typesafe_jev"}
+
+
class DecisionReviewer:
"""Same `review` signature and `Verdict` return as `coworker.reviewer.Reviewer`, so the
engine, which duck-types its reviewer, can use either."""
@@ -1108,9 +1112,13 @@ class DecisionReviewer:
provenance: str = "",
action_context: dict[str, Any] | None = None,
facts: Sequence[str] = (),
+ view: Any = None,
) -> DecisionVerdict:
"""Never raises. Every failure mode is an `unsure`.
+ `view`, when given, records the request to the service as a model step nested in
+ the check, with its usage and provider metadata (observability E23, TD-16).
+
`facts` are the engine's own lines (OPE-235/OPE-237, coworker/review_facts.py), the
same ones the session-model reviewer gets. They are used in code, not added to the
state the model reads, so the answers stay the ones the questions were measured on:
@@ -1135,10 +1143,15 @@ class DecisionReviewer:
if len(json.dumps(state, ensure_ascii=False)) > _STATE_CHAR_BUDGET:
return self._count(_fail_closed("state exceeds the decision model's limit; a human must decide", error=True))
questions = self.backend.questions
+
+ def ask() -> Any:
+ if view is not None and hasattr(view, "record_call"):
+ return view.record_call(lambda: self.backend.ask_questions(state, questions),
+ model=f"{_WIRES.get(self.backend.name, self.backend.name)}:{self.backend.model}")
+ return self.backend.ask_questions(state, questions)
+
try:
- answer = await asyncio.wait_for(
- asyncio.to_thread(self.backend.ask_questions, state, questions), timeout=self.timeout
- )
+ answer = await asyncio.wait_for(asyncio.to_thread(ask), timeout=self.timeout)
except asyncio.TimeoutError:
return self._count(_fail_closed(f"{self.backend.label} timed out", error=True))
except asyncio.CancelledError:
diff --git a/coworker/engine.py b/coworker/engine.py
index d6197b52..ce101c3d 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -2040,7 +2040,10 @@ class TurnEngine:
"model": getattr(reviewer, "model", None) if kind == "session_model" else None}},
)
if step_id is not None and _accepts_view(reviewer):
- review_kw["view"] = _tag_provider(reviewer.provider, parent=step_id, call_id=tool_call.id,
+ # A decision-model reviewer has no provider of its own: its request is recorded
+ # through the engine's view (E23).
+ review_kw["view"] = _tag_provider(getattr(reviewer, "provider", None) or self.provider,
+ parent=step_id, call_id=tool_call.id,
shadow=shadow, purpose="auto_approver")
try:
verdict = await reviewer.review(**review_kw)
diff --git a/coworker/observability/provider_view.py b/coworker/observability/provider_view.py
index b2626bb7..f9e1d6f4 100644
--- a/coworker/observability/provider_view.py
+++ b/coworker/observability/provider_view.py
@@ -27,6 +27,7 @@ import logging
import threading
import time
from dataclasses import dataclass, replace
+from types import SimpleNamespace
from typing import Any, Callable, Iterator, Optional
from ..providers.base import ProviderClient
@@ -91,6 +92,23 @@ class ProviderView(ProviderClient):
return getattr(self.base.inner, name)
# -- calls --------------------------------------------------------------------------------
+ def record_call(self, fn: Callable[[], Any], *, model: str) -> Any:
+ """Record a model call made outside the provider router as a model step: a
+ decision-model reviewer's request to its own service (E23, TD-16). `fn` makes the
+ request on this thread; its HTTP exchange is captured, and the usage is read from
+ the service's reply with the Responses-style mapping (input includes cached)."""
+ call = _ModelCall.begin(self.tags, model=model, tools=None, streamed=False)
+ if call is None:
+ return fn()
+ try:
+ with call.http.active():
+ result = fn()
+ except BaseException as exc:
+ call.failed(exc)
+ raise
+ call.finished(SimpleNamespace(finish_reason=None, usage=_reply_usage(call.http.entries()), served_by=None))
+ return result
+
def complete(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any):
call = _ModelCall.begin(self.tags, model=model, tools=tools, streamed=False)
if call is None:
@@ -324,6 +342,27 @@ class _ModelCall:
log.exception("recording: could not record a failed model step")
+def _reply_usage(entries: list[dict[str, Any]]) -> Any:
+ """The `usage` object of the last JSON reply, as `TokenUsage`; None if there is none."""
+ from ..providers.openai_responses import _usage_from
+
+ for e in reversed(entries):
+ try:
+ body = json.loads(e.get("body") or "")
+ except ValueError:
+ continue
+ usage = body.get("usage") if isinstance(body, dict) else None
+ if isinstance(usage, dict):
+ return _usage_from(_ns(usage))
+ return None
+
+
+def _ns(value: Any) -> Any:
+ if isinstance(value, dict):
+ return SimpleNamespace(**{k: _ns(v) for k, v in value.items()})
+ return value
+
+
def _is_stop(exc: BaseException) -> bool:
return isinstance(exc, (GeneratorExit, KeyboardInterrupt))
diff --git a/tests/test_observability_model_steps.py b/tests/test_observability_model_steps.py
index 9e7d7b40..fd17ac8d 100644
--- a/tests/test_observability_model_steps.py
+++ b/tests/test_observability_model_steps.py
@@ -254,3 +254,14 @@ def test_capture_is_off_outside_a_recorded_call(fake_server):
assert getattr(capture._active, "log", None) is None
inner.complete(model="model-a", messages=[{"role": "user", "content": "x"}])
assert getattr(capture._active, "log", None) is None
+
+
+DECISION_VECTORS = [v for v in VECTORS if v["family"] in ("openai_decisions", "typesafe_jev")]
+
+
[email protected]("vector", DECISION_VECTORS, ids=[v["id"] for v in DECISION_VECTORS])
+def test_decision_reviewer_usage_vectors(vector):
+ from coworker.observability.provider_view import _reply_usage
+
+ entries = [{"body": json.dumps({"usage": vector["raw"]})}]
+ assert usage_attrs(_reply_usage(entries)) == {"reported": True, **vector["expected"]}
diff --git a/tests/test_observability_reviewer_helper.py b/tests/test_observability_reviewer_helper.py
index 56db0251..0b26126d 100644
--- a/tests/test_observability_reviewer_helper.py
+++ b/tests/test_observability_reviewer_helper.py
@@ -192,3 +192,54 @@ def test_explore_helper_steps_are_nested_under_the_explore_step(tmp_path):
assert len(d["turns"]) == 1
mains = [s for s in d["steps"] if s["kind"] == "model" and s["attrs"]["purpose"] == "main"]
assert len(mains) == 2
+
+
+# -- E23: a decision-model reviewer's request is a nested model step ----------------------------
+
+import threading as _threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+
+class _JevHandler(BaseHTTPRequestHandler):
+ def do_POST(self): # noqa: N802
+ body = json.loads(self.rfile.read(int(self.headers.get("content-length") or 0)))
+ answers = {name: {"noul": 0.97, "probabilities": {"yes": 0.97, "no": 0.03}}
+ for name in (body.get("questions") or {})}
+ data = json.dumps({"model": "jev-test-1", "answers": answers,
+ "usage": {"input_tokens": 6203, "output_tokens": 529}}).encode()
+ self.send_response(200)
+ self.send_header("content-type", "application/json")
+ self.send_header("x-request-id", "req_jev")
+ self.send_header("content-length", str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+
+ def log_message(self, *a):
+ pass
+
+
+def test_a_decision_reviewer_check_has_a_model_step_with_usage(tmp_path):
+ from coworker.decision_reviewer import build_decision_reviewer
+
+ server = HTTPServer(("127.0.0.1", 0), _JevHandler)
+ _threading.Thread(target=server.serve_forever, daemon=True).start()
+ try:
+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make clean"}),
+ AssistantTurn(text="done")], [])
+ engine.reviewer = build_decision_reviewer(
+ "jev", api_key="test-key-not-real", base_url=f"http://127.0.0.1:{server.server_address[1]}")
+ engine.reviewer_choice = "jev"
+ _run(engine)
+ finally:
+ server.shutdown()
+ steps = rec.as_dict()["steps"]
+ [check] = [s for s in steps if s["kind"] == "auto_approver"]
+ assert check["attrs"]["reviewer"] == {"kind": "typesafe_jev", "external": True, "model": None}
+ [inner] = [s for s in steps if s["kind"] == "model" and s["parent_step_id"] == check["step_id"]]
+ assert inner["attrs"]["purpose"] == "auto_approver" and inner["attrs"]["provider"] == "typesafe_jev"
+ assert inner["attrs"]["usage"] == {"reported": True, "input_fresh": 6203, "cache_read": None,
+ "cache_write": None, "output": 529, "reasoning": None}
+ assert inner["attrs"]["provider_meta"][0]["wire"] == "typesafe_jev"
+ assert inner["attrs"]["provider_facts"]["provider_request_id"] == "req_jev"
+ assert check["attrs"]["usage_reported"] is True
+ assert "test-key-not-real" not in json.dumps(steps)