Python
feat(observability): auto-approver checks and explore helper calls as nested steps (OPE-244)
Devika Verma075c80f9d194
diff --git a/coworker/engine.py b/coworker/engine.pyindex 149ede99..82f6ed5f 100644--- a/coworker/engine.py+++ b/coworker/engine.py@@ -34,7 +34,7 @@ from . import toolresultfrom . import unattended as _attendancefrom .events import Event, EventTypefrom .observability.decision import DecisionPath-from .observability.engine_hooks import TurnRecording+from .observability.engine_hooks import ToolRun, TurnRecording, set_tool_runfrom .observability.provider_view import tag as _tag_provider, view_for_enginefrom .observability.ids import new_ulid@@ -252,6 +252,23 @@ _WAIT_KINDS = {}+def _accepts_view(reviewer: Any) -> bool:+ """Whether this reviewer's `review` takes the provider view for its check (the session+ model reviewer does; decision-model reviewers call their own service, E23)."""+ try:+ import inspect++ return "view" in inspect.signature(reviewer.review).parameters+ except (TypeError, ValueError):+ return False+++def _verdict_matches(result: str, approved: bool) -> bool:+ """A shadow check matched you when it would have allowed what you approved, or denied+ what you denied; an unsure never matches."""+ return (result == "allow" and approved) or (result == "deny" and not approved)++def _reviewer_skip(engine: "TurnEngine", decision: Any) -> str:"""Why the auto-approver was not consulted on a call that needed a person (§8.2)."""if decision.human_only:@@ -545,6 +562,11 @@ class TurnEngine:# (observability E4, §8.3); and where its card was shown.self._decisions: dict[str, DecisionPath] = {}self._card_where: dict[str, str] = {}+ # Auto-approver check steps by tool call (E12), and shadow checks waiting to be+ # compared with your answer: call id -> (step id, verdict or None).+ self._review_steps: dict[str, str] = {}+ self._shadow_checks: dict[str, tuple[Optional[str], Any]] = {}+ self._user_answers: dict[str, bool] = {}if instructions and not (self.messages and self.messages[0].get("role") == "system"):@@ -1918,7 +1940,8 @@ class TurnEngine:settings_epoch = self.reviewer_settings_epochverdicts = await asyncio.gather(*[- consulted_reviewer.review(+ self._recorded_review(+ consulted_reviewer, tc,request=request,history=history,tool_name=tc.name,@@ -1936,9 +1959,65 @@ class TurnEngine:or not self._reviewer_active()or (request, history, context) != self._review_inputs()):verdict = _settings_changed(verdict)+ self._review_context_changed(tc.id)self._reviewer_verdicts[tc.id] = verdictself._reviewer_input_snapshots[tc.id] = (request, history, context)+ async def _recorded_review(self, reviewer: Any, tool_call: ToolCall, *, shadow: bool = False,+ **review_kw: Any) -> Any:+ """One auto-approver check, recorded as its own step (observability E12, §5.5),+ linked to the tool call by `call_id`; the reviewer's model call is nested in it."""+ kind = _REVIEWER_KINDS.get(getattr(self, "reviewer_choice", "session"), "session_model")+ step_id = self._obs.start_step(+ kind="auto_approver", name=tool_call.name, call_id=tool_call.id, shadow=shadow,+ attrs={"mode": "shadow" if shadow else "live",+ "reviewer": {"kind": kind, "external": kind != "session_model",+ "model": getattr(reviewer, "model", None) if kind == "session_model" else None}},+ )+ if step_id is not None and _accepts_view(reviewer):+ review_kw["view"] = _tag_provider(reviewer.provider, parent=step_id, call_id=tool_call.id,+ shadow=shadow, purpose="auto_approver")+ try:+ verdict = await reviewer.review(**review_kw)+ except BaseException:+ self._obs.end_step(step_id, status="error", attrs={"result": "could_not_run"})+ raise+ if step_id is not None:+ failed = bool(getattr(verdict, "error", False))+ result = "could_not_run" if failed else verdict.verdict+ attrs: dict[str, Any] = {"result": result,+ "usage_reported": self._obs.child_usage_reported(step_id)}+ for key in ("score", "cutoff"):+ if getattr(verdict, key, None) is not None:+ attrs[key] = getattr(verdict, key)+ if shadow:+ answer = self._user_answers.pop(tool_call.id, None)+ attrs["matched_user"] = None if failed or answer is None else _verdict_matches(result, answer)+ if answer is None and not failed:+ self._shadow_checks[tool_call.id] = (step_id, result)+ else:+ denials = self._reviewer_denials + 1 if result == "deny" else 0+ attrs["breaker"] = {"denials_in_row": denials, "tripped": denials >= _REVIEWER_TRIP}+ self._review_steps[tool_call.id] = step_id+ reason = getattr(verdict, "reason", "") or ""+ self._obs.end_step(step_id, status="error" if failed else "ok", attrs=attrs,+ content={"reason": reason} if reason else {})+ return verdict++ def _review_context_changed(self, call_id: str) -> None:+ """The request, history or settings changed while the check ran: the engine treats+ it as unsure, and the step's result says so (§8.2)."""+ self._obs.update_step(self._review_steps.pop(call_id, None), attrs={"result": "context_changed"})++ def _note_user_answer(self, call_id: str, *, approved: bool) -> None:+ """Your answer on a card, compared with a shadow check's verdict (`matched_user`)."""+ waiting = self._shadow_checks.pop(call_id, None)+ if waiting is not None:+ step_id, result = waiting+ self._obs.update_step(step_id, attrs={"matched_user": _verdict_matches(result, approved)})+ elif self.reviewer_shadow:+ self._user_answers[call_id] = approved+async def _consult_reviewer(self, tool_call: ToolCall) -> Any:"""The parked verdict from `_preconsult_reviewer`, or a fresh single call."""verdict = self._reviewer_verdicts.pop(tool_call.id, None)@@ -1946,6 +2025,7 @@ class TurnEngine:if verdict is not None:if snapshot is not None and snapshot != self._review_inputs():from .reviewer import Verdict+ self._review_context_changed(tool_call.id)return Verdict("unsure", "Approval context changed since review; a human must decide.")return verdictrequest, history, context = self._review_inputs()@@ -1954,7 +2034,8 @@ class TurnEngine:from .reviewer import Verdictif delegated.get("hard_deny") or delegated.get("human_only"):return Verdict("unsure", delegated["reason"])- verdict = await self.reviewer.review(+ verdict = await self._recorded_review(+ self.reviewer, tool_call,request=request, history=history,tool_name=delegated["tool"], arguments=delegated["arguments"],provenance=delegated.get("provenance", ""),@@ -1963,7 +2044,8 @@ class TurnEngine:if delegated != self._delegated_context(tool_call) or (request, history, context) != self._review_inputs():return Verdict("unsure", "The worker request or its permissions changed during review. Review the current request.")return verdict- result = await self.reviewer.review(+ result = await self._recorded_review(+ self.reviewer, tool_call,request=request,history=history,tool_name=tool_call.name,@@ -1974,6 +2056,7 @@ class TurnEngine:)if (request, history, context) != self._review_inputs():from .reviewer import Verdict+ self._review_context_changed(tool_call.id)return Verdict("unsure", "Approval context changed during review; review the current request.")return result@@ -2047,7 +2130,8 @@ class TurnEngine:async def _shadow() -> None:try:- verdict = await self.reviewer.review(+ verdict = await self._recorded_review(+ self.reviewer, tool_call, shadow=True,request=request,history=history,provenance=prov,@@ -2259,6 +2343,7 @@ class TurnEngine:or not self._reviewer_active())if settings_changed:verdict = _settings_changed(verdict)+ self._review_context_changed(tool_call.id)path.add("auto_approver", "context_changed" if settings_changedelse "could_not_run" if getattr(verdict, "error", False) else verdict.verdict)self._audit(@@ -2457,6 +2542,8 @@ class TurnEngine:outcome.value if isinstance(outcome, ApprovalOutcome) else str(outcome),interrupted=outcome is ApprovalOutcome.DENY and self._cancel.is_set(),)+ if not self._cancel.is_set() and outcome is not ApprovalOutcome.SUPERSEDED:+ self._note_user_answer(tool_call.id, approved=outcome is not ApprovalOutcome.DENY)if outcome is ApprovalOutcome.SUPERSEDED:result = {"skipped": True,@@ -2609,7 +2696,13 @@ class TurnEngine:kind="tool", name=tool_call.name, call_id=tool_call.id,call_index=self._call_index.get(tool_call.id),)- result, status = self._execute_call(tool_call)+ # A tool that starts its own work (the explore helper, E6) records it under this step.+ outer = set_tool_run(ToolRun(step_id=step_id, turn_id=self._obs.turn_id,+ view=self.provider, recording=self._obs))+ try:+ result, status = self._execute_call(tool_call)+ finally:+ set_tool_run(outer)if step_id is not None:self._obs.end_step(step_id, status=status, attrs=self._tool_step_attrs(tool_call, result),diff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.pyindex d4cc9469..6576a9fd 100644--- a/coworker/observability/engine_hooks.py+++ b/coworker/observability/engine_hooks.py@@ -51,6 +51,7 @@ class TurnRecording:self._waits: dict[str, str] = {} # call_id -> its open wait step (E20)self._reserved: dict[str, dict[str, Any]] = {} # summary steps written at their end (E4)self._tokens_after_for: Optional[str] = None # a summary waiting for the next main call+ self._child_usage: dict[str, bool] = {} # parent step -> its model call reported usage@propertydef enabled(self) -> bool:@@ -207,6 +208,10 @@ class TurnRecording:ev["content"] = contentself._send(ev)+ def child_usage_reported(self, step_id: Optional[str]) -> bool:+ with self._lock:+ return self._child_usage.pop(step_id or "", False)+# -- wait steps (E20) -----------------------------------------------------------------------def track_wait(self, call_id: str, step_id: Optional[str]) -> None:if step_id:@@ -302,6 +307,8 @@ class TurnRecording:ev["error"] = errorev.update({k: idn[k] for k in ("kind", "name", "parent_step_id", "call_id","call_index", "shadow", "after_turn", "started_at")})+ if idn["kind"] == "model" and idn["parent_step_id"]:+ self._child_usage[idn["parent_step_id"]] = bool(((attrs or {}).get("usage") or {}).get("reported"))waiting_turn = idn["turn_id"] is not None and idn["turn_id"] == self.turn_id and not self._endedif wait_for_tool_message and idn["call_id"] and waiting_turn:self._tool_ends[idn["call_id"]] = ev@@ -357,3 +364,119 @@ class TurnRecording:self.instant_step(kind="tool", name=str(data.get("name") or ""), status=status,call_id=call_id, call_index=call_index, attrs=attrs,message_id=message_id)+++# -- the explore helper (E6) --------------------------------------------------------------------++_tool_run = threading.local()+++class ToolRun:+ """What a running tool may know about itself: its step, its turn, and the engine's+ provider view and recording. Published for the tool's thread by `_execute_sync`."""++ def __init__(self, *, step_id: Optional[str], turn_id: Optional[str], view: Any,+ recording: "TurnRecording") -> None:+ self.step_id, self.turn_id, self.view, self.recording = step_id, turn_id, view, recording+++def current_tool_run() -> Optional[ToolRun]:+ return getattr(_tool_run, "run", None)+++def set_tool_run(run: Optional[ToolRun]) -> Optional[ToolRun]:+ """Publish `run` for this thread; returns the one it replaces."""+ outer = getattr(_tool_run, "run", None)+ _tool_run.run = run+ return outer+++class HelperRecording:+ """The recording of an explore helper's engine (§5.3, E6). Its steps go to the parent's+ recording, on the parent's turn, nested under the `explore` step. Its own turn events are+ not recorded as turns. Its tools run read-only in Plan mode, so their decision is+ `{helper: true}` instead of a full path."""++ def __init__(self, parent: TurnRecording, parent_step_id: str, turn_id: Optional[str]) -> None:+ self.parent, self.parent_step_id = parent, parent_step_id+ self.turn_id = turn_id++ @property+ def recorder(self) -> Any:+ return self.parent.recorder++ @property+ def enabled(self) -> bool:+ return self.parent.enabled++ @property+ def in_turn(self) -> bool:+ return True++ def begin_turn(self, **_kw: Any) -> Optional[str]:+ return self.turn_id++ def end_turn(self, status: str) -> None:+ pass++ def note_message(self, message: dict[str, Any]) -> None:+ pass++ def note_main_usage(self, usage: dict[str, Any]) -> None:+ pass++ def start_step(self, **kw: Any) -> Optional[str]:+ kw.setdefault("parent_step_id", None)+ kw["parent_step_id"] = kw["parent_step_id"] or self.parent_step_id+ kw["turn_id"] = self.turn_id+ return self.parent.start_step(**kw)++ def end_step(self, step_id: Optional[str], **kw: Any) -> None:+ kw.pop("wait_for_tool_message", None)+ kw.pop("wait_for_reply_message", None)+ attrs = kw.get("attrs")+ if attrs is not None and "decision" in attrs:+ attrs["decision"] = {"helper": True}+ self.parent.end_step(step_id, **kw)++ def instant_step(self, **kw: Any) -> None:+ attrs = kw.get("attrs")+ if attrs is not None and "decision" in attrs:+ attrs["decision"] = {"helper": True}+ step_id = self.start_step(kind=kw["kind"], name=kw["name"], call_id=kw.get("call_id"),+ call_index=kw.get("call_index"))+ self.parent.end_step(step_id, status=kw["status"], attrs=attrs,+ message_id=kw.get("message_id"))++ def tool_finished(self, data: dict[str, Any], **kw: Any) -> None:+ pass # a helper's refused calls stay in its own transcript++ def step_for_call(self, call_id: str) -> Optional[str]:+ return None++ def update_step(self, step_id: Optional[str], **kw: Any) -> None:+ self.parent.update_step(step_id, **kw)++ def track_wait(self, call_id: str, step_id: Optional[str]) -> None:+ pass++ def end_wait(self, call_id: str) -> Optional[str]:+ return None++ def wait_where(self, call_id: Optional[str], where: str) -> None:+ pass++ def reserve_step(self) -> Optional[str]:+ return None++ def model_children(self, step_id: Optional[str]) -> list[str]:+ return []++ def finish_reserved(self, *a: Any, **kw: Any) -> None:+ pass++ def drop_reserved(self, step_id: Optional[str]) -> None:+ pass++ def child_usage_reported(self, step_id: Optional[str]) -> bool:+ return self.parent.child_usage_reported(step_id)diff --git a/coworker/reviewer.py b/coworker/reviewer.pyindex d121ce1c..d6d69379 100644--- a/coworker/reviewer.py+++ b/coworker/reviewer.py@@ -449,8 +449,11 @@ class Reviewer:provenance: str = "",action_context: dict[str, Any] | None = None,facts: Sequence[str] = (),+ view: Any = None,) -> Verdict:- """Never raises. Every failure mode is an `unsure` (§8.5)."""+ """Never raises. Every failure mode is an `unsure` (§8.5). `view`, when given, is+ the provider view this check's model call goes through (observability E12: its+ parent is the check's step)."""# Never quietly truncate restrictions out of the owner's words. This is a# deterministic size limit, not a scope classifier; oversized input asks a human.if action_context and action_context.get("context_unavailable"):@@ -470,7 +473,7 @@ class Reviewer:try:turn = await asyncio.wait_for(asyncio.to_thread(- self.provider.complete,+ (view or self.provider).complete,model=self.model,messages=messages,# One JSON line comes back. Without an explicit cap the provider'sdiff --git a/coworker/tools/subagent.py b/coworker/tools/subagent.pyindex 2b0f98de..ce5c8b70 100644--- a/coworker/tools/subagent.py+++ b/coworker/tools/subagent.py@@ -19,6 +19,8 @@ from typing import Any, Optionalimport aisuite as ai+from ..observability.engine_hooks import HelperRecording, current_tool_run+from ..observability.provider_view import tag as tag_providerfrom ..engine import TurnEnginefrom ..events import EventTypefrom ..permissions import Mode, PermissionEngine@@ -96,12 +98,21 @@ def explorer_tools(task (str): The research question, with any constraints and the expectedshape of the report."""+ # Observability E6: the helper's model and tool calls are recorded under this+ # `explore` step, on the parent's turn, with purpose `helper`.+ run = current_tool_run()+ helper_provider = provider+ if run is not None and run.step_id is not None:+ helper_provider = tag_provider(run.view, purpose="helper", parent=run.step_id,+ turn=run.turn_id)engine = build_explorer_engine(workspace=workspace,- provider=provider,+ provider=helper_provider,model=model,model_settings=model_settings,)+ if run is not None and run.step_id is not None:+ engine._obs = HelperRecording(run.recording, run.step_id, run.turn_id)async def _run() -> tuple[str, str]:report, status = "", "unknown"diff --git a/tests/test_observability_reviewer_helper.py b/tests/test_observability_reviewer_helper.pynew file mode 100644index 00000000..56db0251--- /dev/null+++ b/tests/test_observability_reviewer_helper.py@@ -0,0 +1,194 @@+"""Observability E12 and E6: auto-approver checks and the explore helper, recorded.++- E12: each auto-approver check is its own step, linked to the tool call by `call_id`,+ with the reviewer's model call nested inside (purpose `auto_approver`); live checks carry+ the breaker, shadow checks whether they matched your answer.+- E6: the explore helper's model and tool calls are recorded on the parent's turn, nested+ under the `explore` tool step, with purpose `helper` and decision `{helper: true}`.+"""++from __future__ import annotations++import asyncio+import json++import aisuite as ai++from coworker.engine import ApprovalOutcome, TurnEngine+from coworker.observability import Recorder+from coworker.observability.provider_view import RecordingProvider+from coworker.permissions import Mode, PermissionEngine+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall+from coworker.providers.base import TokenUsage+from coworker.reviewer import Reviewer+from coworker.tools import ToolRegistry+from coworker.tools.subagent import explorer_tools+++class _Meta:+ category: str = ""+ risk_level: str = "high"+ requires_approval: bool = False+++class _Scripted(ProviderClient):+ def __init__(self, turns):+ self._turns = list(turns)+ self.lock = __import__("threading").Lock()++ def complete(self, *, model, messages, tools=None, **settings):+ with self.lock:+ turn = self._turns.pop(0)+ if isinstance(turn, Exception):+ raise turn+ return AssistantTurn(text=turn, usage=TokenUsage(input=50, output=5)) if isinstance(turn, str) else turn++ def capabilities(self, model):+ return ModelCapabilities()+++def run_shell(command: str) -> str:+ """Run a command.++ Args:+ command: the command line+ """+ return f"ran: {command}"+++def _call(cid, name, args):+ return AssistantTurn(tool_calls=[ToolCall(id=cid, name=name, arguments=args)], finish_reason="tool_calls")+++def _engine(tmp_path, main_turns, review_turns, *, shadow=False, answer=ApprovalOutcome.ONCE):+ registry = ToolRegistry()+ registry.register(run_shell, metadata=_Meta())+ base = RecordingProvider(_Scripted(main_turns))++ async def approver(request):+ return answer++ engine = TurnEngine(+ provider=base,+ registry=registry,+ permissions=PermissionEngine(workspace_root=tmp_path,+ mode=Mode.INTERACTIVE if shadow else Mode.AUTO_APPROVE),+ model="openai:model-a",+ approver=approver,+ )+ engine.audit_context = {"session_id": "sess_r"}+ engine.is_attended = lambda: True+ rec = Recorder()+ engine.recorder = rec+ # The reviewer's own provider: the engine's view tagged auto_approver, as the server does.+ reviewer_base = RecordingProvider(_Scripted(review_turns))+ engine.reviewer = Reviewer(+ provider=reviewer_base.tagged(purpose="auto_approver", sink=engine._obs), model="openai:model-r"+ )+ engine.reviewer_choice = "session"+ engine.reviewer_enabled = not shadow+ engine.reviewer_shadow = shadow+ return engine, rec+++def _run(engine, text="clean the build"):+ async def go():+ events = [ev async for ev in engine.run(text, trigger="user")]+ await engine.drain_shadow_reviews()+ return events++ return asyncio.run(go())+++def test_a_live_check_is_a_step_with_its_model_call_inside(tmp_path):+ engine, rec = _engine(+ tmp_path,+ [_call("c1", "run_shell", {"command": "make clean"}), AssistantTurn(text="done")],+ [json.dumps({"verdict": "allow", "reason": "matches the request"})],+ )+ _run(engine)+ steps = rec.as_dict()["steps"]+ [check] = [s for s in steps if s["kind"] == "auto_approver"]+ assert (check["name"], check["call_id"], check["shadow"], check["status"]) == ("run_shell", "c1", False, "ok")+ assert check["attrs"] == {+ "mode": "live", "result": "allow", "usage_reported": True,+ "reviewer": {"kind": "session_model", "external": False, "model": "openai:model-r"},+ "breaker": {"denials_in_row": 0, "tripped": False},+ }+ assert check["content"] == {"reason": "matches the request"}+ [inner] = [s for s in steps if s["kind"] == "model" and s["parent_step_id"] == check["step_id"]]+ assert inner["attrs"]["purpose"] == "auto_approver" and inner["call_id"] == "c1"+ [tool] = [s for s in steps if s["kind"] == "tool"]+ assert tool["attrs"]["decision"]["origin"] == "reviewer"+++def test_a_denied_check_counts_toward_the_breaker(tmp_path):+ engine, rec = _engine(+ tmp_path,+ [_call("c1", "run_shell", {"command": "rm -rf /"}), AssistantTurn(text="ok")],+ [json.dumps({"verdict": "deny", "reason": "destroys the machine"})],+ )+ _run(engine)+ [check] = [s for s in rec.as_dict()["steps"] if s["kind"] == "auto_approver"]+ assert check["attrs"]["result"] == "deny"+ assert check["attrs"]["breaker"] == {"denials_in_row": 1, "tripped": False}+ [tool] = [s for s in rec.as_dict()["steps"] if s["kind"] == "tool"]+ assert tool["status"] == "denied" and tool["attrs"]["decision"]["origin"] == "reviewer_denied"+++def test_a_check_that_could_not_run_is_an_error_step(tmp_path):+ engine, rec = _engine(+ tmp_path,+ [_call("c1", "run_shell", {"command": "deploy"}), AssistantTurn(text="ok")],+ [RuntimeError("no key")],+ )+ _run(engine)+ [check] = [s for s in rec.as_dict()["steps"] if s["kind"] == "auto_approver"]+ assert (check["status"], check["attrs"]["result"]) == ("error", "could_not_run")+++def test_a_shadow_check_records_whether_it_matched_you(tmp_path):+ engine, rec = _engine(+ tmp_path,+ [_call("c1", "run_shell", {"command": "make"}), AssistantTurn(text="done")],+ [json.dumps({"verdict": "allow", "reason": "fine"})],+ shadow=True, answer=ApprovalOutcome.DENY,+ )+ _run(engine)+ [check] = [s for s in rec.as_dict()["steps"] if s["kind"] == "auto_approver"]+ assert check["shadow"] is True and check["attrs"]["mode"] == "shadow"+ assert check["attrs"]["matched_user"] is False # it would have allowed what you denied+ assert "breaker" not in check["attrs"]+++def test_explore_helper_steps_are_nested_under_the_explore_step(tmp_path):+ (tmp_path / "notes.txt").write_text("Friday release", encoding="utf-8")+ scripted = _Scripted([+ _call("c1", "explore", {"task": "what is in notes.txt?"}), # parent+ _call("h1", "read_file", {"path": "notes.txt"}), # helper+ AssistantTurn(text="It says Friday release"), # helper report+ AssistantTurn(text="The notes say Friday."), # parent+ ])+ base = RecordingProvider(scripted)+ registry = ToolRegistry()+ registry.register_all(explorer_tools(workspace=tmp_path, provider=base, model="openai:model-a"))+ engine = TurnEngine(provider=base, registry=registry,+ permissions=PermissionEngine(workspace_root=tmp_path, mode=Mode.BYPASS_APPROVALS),+ model="openai:model-a")+ engine.audit_context = {"session_id": "sess_h"}+ rec = Recorder()+ engine.recorder = rec+ _run(engine, "use explore")+ d = rec.as_dict()+ [turn] = d["turns"]+ [explore] = [s for s in d["steps"] if s["kind"] == "tool" and s["name"] == "explore"]+ nested = [s for s in d["steps"] if s["parent_step_id"] == explore["step_id"]]+ assert sorted((s["kind"], s["attrs"].get("purpose")) for s in nested) == [+ ("model", "helper"), ("model", "helper"), ("tool", None)]+ [read] = [s for s in nested if s["kind"] == "tool"]+ assert read["name"] == "read_file" and read["attrs"]["decision"] == {"helper": True}+ assert all(s["turn_id"] == turn["turn_id"] for s in nested)+ # The helper's own run is not a second turn.+ assert len(d["turns"]) == 1+ mains = [s for s in d["steps"] if s["kind"] == "model" and s["attrs"]["purpose"] == "main"]+ assert len(mains) == 2