Python
"""Observability Phase 2, OB-7: helpers and team workers in the timeline (design §13).
- E6b: the `explore` step carries `attrs.helper`, `{index, status, model_calls, tool_calls,
tokens}`, for display only. The pill still adds up the helper's own model steps.
- E19a: team tool steps carry `attrs.team` (ids only); worker turns carry `woken_by`; a
`steer_worker` delivery is `trigger: team_wake` with `via: steer`, not `channel`.
- E19b: the read API lists, under each lead turn, the worker turns that lead woke during it.
Their tokens are display only, never in the lead's pill.
- E19c: TeamView's totals come from the recording, so helper tokens count (V2-29).
- Export: a worker turn's span carries `openworker.team.id` and `openworker.lead.session.id`.
"""
from __future__ import annotations
import asyncio
import importlib
import json
import threading
from pathlib import Path
import jsonschema
from coworker.engine import TurnEngine
from coworker.observability import Recorder, calculator, export_spans, read_api
from coworker.observability.engine_hooks import ToolRun, note_team, set_tool_run
from coworker.observability.provider_view import RecordingProvider
from coworker.permissions import Mode, PermissionEngine
from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall
from coworker.providers.base import TokenUsage
from coworker.teams import Actor, Role, TeamStore, board_tools
from coworker.teams.registry import TeamWorker
from coworker.teams.tools import propose_team_tool
from coworker.tools import ToolRegistry
from coworker.tools.subagent import explorer_tools
proposals = importlib.import_module("proposal_fixtures")
SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))
def _check(recording):
validator = jsonschema.validators.validator_for(SCHEMA)(SCHEMA)
errors = sorted(validator.iter_errors(recording), key=lambda e: list(e.absolute_path))
assert not errors, "\n".join(f"{list(e.absolute_path)}: {e.message[:300]}" for e in errors[:8])
# -- E6b: the helper's summary on the explore step ----------------------------------------------
class _ByTask(ProviderClient):
"""Answers each conversation by its first user message, so parallel helpers each get
their own script whichever thread asks first."""
def __init__(self, scripts):
self._scripts = {k: list(v) for k, v in scripts.items()}
self._lock = threading.Lock()
def complete(self, *, model, messages, tools=None, **settings):
first = next(m["content"] for m in messages if m.get("role") == "user")
key = next(k for k in self._scripts if k in str(first))
with self._lock:
turn = self._scripts[key].pop(0)
return AssistantTurn(text=turn, usage=TokenUsage(input=50, output=5)) if isinstance(turn, str) else turn
def capabilities(self, model):
return ModelCapabilities()
def _calls(*calls):
return AssistantTurn(tool_calls=[ToolCall(id=c, name=n, arguments=a) for c, n, a in calls],
finish_reason="tool_calls", usage=TokenUsage(input=100, output=10))
def _explore_engine(tmp_path, scripts):
(tmp_path / "notes.txt").write_text("Friday release", encoding="utf-8")
base = RecordingProvider(_ByTask(scripts))
registry = ToolRegistry()
registry.register_all(explorer_tools(workspace=tmp_path, provider=base, model="openai:model-x"))
engine = TurnEngine(provider=base, registry=registry,
permissions=PermissionEngine(workspace_root=tmp_path, mode=Mode.BYPASS_APPROVALS),
model="openai:model-x")
engine.audit_context = {"session_id": "sess_lead"}
rec = Recorder()
engine.recorder = rec
return engine, rec
def _run(engine, text):
async def go():
return [ev async for ev in engine.run(text, trigger="user")]
return asyncio.run(go())
def test_the_explore_step_says_what_its_helper_did(tmp_path):
engine, rec = _explore_engine(tmp_path, {
"use explore": [_calls(("c1", "explore", {"task": "alpha: what is in notes.txt?"})),
AssistantTurn(text="Friday.", usage=TokenUsage(input=10, output=1))],
"alpha": [_calls(("h1", "read_file", {"path": "notes.txt"}), ("h2", "list_files", {})),
"It says Friday release"],
})
_run(engine, "use explore")
d = rec.as_dict()
[explore] = [s for s in d["steps"] if s["name"] == "explore"]
# Two helper model calls: 110 + 55 tokens; two tool calls.
assert explore["attrs"]["helper"] == {"index": 1, "status": "ok", "model_calls": 2,
"tool_calls": 2, "tokens": 165}
# The pill counts the helper's own model steps once, never the summary as well:
# lead 110 + 11, helper 110 + 55.
assert calculator.usage(d)["pill"] == 110 + 11 + 110 + 55
_check(d)
def test_two_helpers_in_parallel_are_numbered_in_the_models_order(tmp_path):
engine, rec = _explore_engine(tmp_path, {
"use explore": [_calls(("c1", "explore", {"task": "alpha"}), ("c2", "explore", {"task": "beta"})),
AssistantTurn(text="done")],
"alpha": ["alpha report"],
"beta": [_calls(("b1", "read_file", {"path": "notes.txt"})), "beta report"],
})
_run(engine, "use explore")
d = rec.as_dict()
by_call = {s["call_id"]: s["attrs"]["helper"] for s in d["steps"] if s["name"] == "explore"}
assert by_call["c1"]["index"] == 1 and by_call["c1"]["model_calls"] == 1
assert by_call["c2"]["index"] == 2 and by_call["c2"]["tool_calls"] == 1
_check(d)
def test_a_helper_that_hits_its_step_limit_says_so(tmp_path):
loop = [_calls((f"h{i}", "list_files", {})) for i in range(12)]
engine, rec = _explore_engine(tmp_path, {
"use explore": [_calls(("c1", "explore", {"task": "alpha"})), AssistantTurn(text="partial")],
"alpha": loop,
})
_run(engine, "use explore")
d = rec.as_dict()
[explore] = [s for s in d["steps"] if s["name"] == "explore"]
helper = explore["attrs"]["helper"]
assert helper["status"] == "step_limit"
assert helper["model_calls"] == 10 # the helper's limit
_check(d)
# -- E19a: ids on the team tool steps -----------------------------------------------------------
class _Script(ProviderClient):
def __init__(self, turns):
self._turns = list(turns)
def complete(self, *, model, messages, tools=None, **settings):
turn = self._turns.pop(0)
return AssistantTurn(text=turn, usage=TokenUsage(input=10, output=1)) if isinstance(turn, str) else turn
def capabilities(self, model):
return ModelCapabilities()
def _team_engine(tmp_path, turns, tools):
registry = ToolRegistry()
registry.register_all(tools)
engine = TurnEngine(provider=_Script(turns), registry=registry,
permissions=PermissionEngine(workspace_root=tmp_path, mode=Mode.BYPASS_APPROVALS),
model="openai:model-x")
engine.audit_context = {"session_id": "sess_lead"}
rec = Recorder()
engine.recorder = rec
return engine, rec
def test_the_leads_board_calls_carry_the_team_and_item_ids(tmp_path):
store = TeamStore(tmp_path / "teams.db")
lead = Actor(id="lead", role=Role.LEAD, session_id="sess_lead")
tools = board_tools(store, space="proj", actor=lead, team_id=lambda: "team-1",
worker_session=lambda actor: {"sam": "sess_sam"}.get(actor))
engine, rec = _team_engine(tmp_path, [
_calls(("c1", "create_item", {"title": "Invoice PDF", "criteria": "tests pass"})),
_calls(("c2", "assign", {"item": 1, "assignee": "sam"}),
("c3", "comment", {"item": 1, "body": "use the new template"}),
("c4", "list_items", {})),
"assigned",
], tools)
_run(engine, "plan the work")
store.close()
d = rec.as_dict()
team = {s["name"]: s["attrs"].get("team") for s in d["steps"] if s["kind"] == "tool"}
assert team["create_item"] == {"team_id": "team-1", "item_id": "1"}
assert team["assign"] == {"team_id": "team-1", "item_id": "1", "worker_session_id": "sess_sam"}
assert team["comment"] == {"team_id": "team-1", "item_id": "1"}
assert team["list_items"] is None # a read, not a team action
# Ids only: no title, criteria or comment text in the step.
assert "Invoice" not in json.dumps([s["attrs"] for s in d["steps"]])
_check(d)
def test_an_approved_propose_team_names_the_worker_sessions_it_created(tmp_path):
engine, rec = _team_engine(tmp_path, [
_calls(("p1", "propose_team", proposals.team_proposal())), "staffed",
], [propose_team_tool()])
async def approve(args, call_id):
return {"approved": True, "team_id": "team-1",
"workers": [{"actor": "sam", "session_id": "sess_sam"}, {"actor": "kim", "session_id": "sess_kim"}]}
engine.team_approver = approve
_run(engine, "staff it")
d = rec.as_dict()
[step] = [s for s in d["steps"] if s["kind"] == "tool" and s["name"] == "propose_team"]
assert step["attrs"]["team"] == {"team_id": "team-1", "worker_session_ids": ["sess_sam", "sess_kim"]}
_check(d)
class _Steps:
def __init__(self):
self.updates = []
def update_step(self, step_id, attrs=None, content=None):
self.updates.append((step_id, attrs))
def _in_tool_run(fn):
rec = _Steps()
outer = set_tool_run(ToolRun(step_id="step-1", turn_id="turn-1", view=None, recording=rec))
try:
result = fn()
finally:
set_tool_run(outer)
return result, [a["team"] for _, a in rec.updates]
def test_note_team_does_nothing_outside_a_recorded_tool_call():
note_team("team-1", item_id=3) # no tool run on this thread: nothing to write to
_, seen = _in_tool_run(lambda: note_team("", item_id=3))
assert seen == [] # no team: nothing recorded
def _manager(tmp_path, monkeypatch, provider=None):
from coworker.server.manager import SessionManager
monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))
(tmp_path / "repo").mkdir(exist_ok=True)
return SessionManager(workspace=str(tmp_path / "repo"), data_dir=tmp_path / "data", provider=provider)
def _team(mgr):
return mgr.teams.create(space="proj", lead_session="sess_lead", lead_actor="lead",
workers=[TeamWorker(actor="sam", persona="swe-worker", session_id="sess_sam"),
TeamWorker(actor="kim", persona="swe-worker", session_id="sess_kim")])
def test_steer_worker_and_decide_worker_call_name_the_worker_session(tmp_path, monkeypatch):
mgr = _manager(tmp_path, monkeypatch)
team = _team(mgr)
steer = mgr._steer_tool("sess_lead")
_, seen = _in_tool_run(lambda: steer(worker="kim", message="switch to the API docs"))
assert seen == [{"team_id": team.team_id, "worker_session_id": "sess_kim"}]
_, seen = _in_tool_run(lambda: steer(worker="nobody", message="x"))
assert seen == [{"team_id": team.team_id}] # the team, but no such worker
result, seen = _in_tool_run(lambda: mgr.decide_worker_call("sess_lead", "sam", "call-9", "allow"))
assert "error" in result # no parked call: the ids are still on the step
assert seen == [{"team_id": team.team_id, "worker_session_id": "sess_sam"}]
asyncio.run(mgr.aclose())
def test_post_chat_carries_the_team_and_the_item_it_records_on(tmp_path, monkeypatch):
mgr = _manager(tmp_path, monkeypatch)
team = _team(mgr)
post = mgr._post_chat_tool("sess_sam", "worker")
_, seen = _in_tool_run(lambda: post(text="@lead done?", record_on_item=4))
assert seen == [{"team_id": team.team_id, "item_id": "4"}]
asyncio.run(mgr.aclose())
# -- E19a: who woke a worker's turn -------------------------------------------------------------
def _woken(mgr, session_id, message, source, wake=None):
asyncio.run(mgr.deliver_to_session(session_id, message, source=source, wake=wake))
turns = mgr.recordings_store.load_recording(session_id)["turns"]
return turns[-1]
def test_a_worker_turn_records_who_woke_it(tmp_path, monkeypatch):
mgr = _manager(tmp_path, monkeypatch, provider=_Script(["on it", "ok", "seen", "hi"]))
team = _team(mgr)
mgr.get_engine("sess_sam", agent="chat")
steer = _woken(mgr, "sess_sam", "[Lead] switch to the API docs",
{"connector": "team", "sender_name": "lead"})
assert steer["trigger"] == "team_wake" # V2-30: not `channel`
assert steer["woken_by"] == {"lead_session_id": "sess_lead", "team_id": team.team_id,
"item_ids": [], "via": "steer"}
rows = [{"item": 3, "kind": "assigned"}, {"item": 5, "kind": "comment"}, {"item": 3, "kind": "moved"}]
board = mgr._woken_by("sess_sam", {"connector": "board", "board": {"rows": rows}})
assert board == {"lead_session_id": "sess_lead", "team_id": team.team_id,
"item_ids": ["3", "5"], "via": "board"} # batched: every item, once
mention = mgr._woken_by("sess_sam", {"connector": "board", "board": {"rows": [{"kind": "chat"}]}})
assert mention["via"] == "mention"
assert mgr._woken_by("sess_sam", None, wake=object())["via"] == "timer"
# A lead's own turns, and sessions outside a team, have none.
assert mgr._woken_by("sess_lead", {"connector": "team"}) is None
mgr.get_engine("sess_solo", agent="chat")
solo = _woken(mgr, "sess_solo", "hello", {"connector": "slack", "sender_name": "pat"})
assert solo["trigger"] == "channel" and "woken_by" not in solo
_check(mgr.recordings_store.load_recording("sess_sam"))
asyncio.run(mgr.aclose())
# -- E19b: worker rows under the lead's turns ---------------------------------------------------
def _turn(tid, sid, start, dur, *, status="completed", woken_by=None, seq=1):
t = {"turn_id": tid, "session_id": sid, "seq": seq, "kind": "run", "trigger": "user", "started_at": start,
"duration_ms": dur, "status": status, "incomplete": False, "approval_mode": "interactive",
"attended": True, "reviewer": {"active": False, "shadow": False, "kind": None, "model": None},
"stamp": {}, "first_message_id": None, "last_message_id": None, "schema_version": 1}
if woken_by:
t["trigger"], t["woken_by"] = "team_wake", woken_by
return t
def _model(step_id, tid, sid, start, tokens, purpose="main"):
return {"step_id": step_id, "turn_id": tid, "session_id": sid, "parent_step_id": None, "seq": 1,
"kind": "model", "name": "model", "started_at": start, "duration_ms": 10, "status": "ok",
"call_id": None, "call_index": None, "message_id": None, "shadow": False, "after_turn": False,
"attrs": {"purpose": purpose, "model": "openai:model-x",
"usage": {"reported": True, "input_fresh": tokens - 1, "output": 1}},
"content": {}}
def _by(team_id="team-1", items=("3",), via="board"):
return {"lead_session_id": "sess_lead", "team_id": team_id, "item_ids": list(items), "via": via}
def test_worker_turns_are_listed_under_the_lead_turn_they_started_in():
lead = [_turn("L1", "sess_lead", 1000, 1000, seq=1), _turn("L2", "sess_lead", 5000, 1000, seq=2),
_turn("L3", "sess_lead", 9000, None, status="running", seq=3)]
workers = [
(_turn("W1", "sess_sam", 1500, 400, woken_by=_by(), seq=3), [_model("m1", "W1", "sess_sam", 1500, 12000)]),
(_turn("W2", "sess_kim", 1600, 400, woken_by=_by(items=("3", "4"))), []), # one action, two workers
(_turn("W3", "sess_sam", 3000, 100, woken_by=_by(), seq=4), []), # after L1 ended: in L2
(_turn("W4", "sess_sam", 9500, None, status="running", woken_by=_by(via="steer"), seq=5), []),
]
rows = read_api.worker_rows(lead, workers, names={"sess_sam": "sam", "sess_kim": "kim"}.get)
assert [r["turn_id"] for r in rows["L1"]] == ["W1", "W2"]
assert [r["turn_id"] for r in rows["L2"]] == ["W3"]
assert [r["turn_id"] for r in rows["L3"]] == ["W4"] # the running lead turn is open-ended
assert rows["L1"][0] == {"session_id": "sess_sam", "turn_id": "W1", "seq": 3, "name": "sam",
"started_at": 1500, "duration_ms": 400, "status": "completed", "tokens": 12000,
"at_least": False, "team_id": "team-1", "item_ids": ["3"], "via": "board"}
assert rows["L1"][1]["item_ids"] == ["3", "4"]
def test_a_worker_turn_after_the_last_lead_turn_waits_for_the_next():
lead = [_turn("L1", "sess_lead", 1000, 1000)]
rows = read_api.worker_rows(lead, [(_turn("W1", "sess_sam", 2500, 10, woken_by=_by()), [])])
assert rows == {"L1": []}
def test_the_read_api_lists_worker_rows_and_keeps_their_tokens_out_of_the_leads_pill(tmp_path, monkeypatch):
mgr = _manager(tmp_path, monkeypatch)
_team(mgr)
store = mgr.recordings_store
store.write_turn(_turn("L1", "sess_lead", 1000, 5000))
store.write_step(_model("lm", "L1", "sess_lead", 1100, 300))
store.write_turn(_turn("W1", "sess_sam", 2000, 1000, woken_by=_by()))
store.write_step(_model("wm", "W1", "sess_sam", 2100, 12000))
store.write_turn(_turn("X1", "sess_other", 2000, 1000, woken_by={**_by(), "lead_session_id": "sess_x"}))
[turn] = mgr.recording_turns("sess_lead")["turns"]
[row] = turn["workers"]
assert (row["name"], row["session_id"], row["tokens"]) == ("sam", "sess_sam", 12000)
assert turn["usage"]["pill"] == 300 # the worker's 12000 are display only
assert mgr.recording_usage("sess_lead")["pill"] == 300
assert "workers" not in mgr.recording_turns("sess_sam")["turns"][0]
asyncio.run(mgr.aclose())
# -- E19c: TeamView's totals from the recording (V2-29) -----------------------------------------
def test_team_view_totals_count_helper_tokens_from_the_recording(tmp_path, monkeypatch):
from coworker.sessions import SessionRecord
mgr = _manager(tmp_path, monkeypatch)
mgr.session_store.save(SessionRecord(session_id="sess_lead", workspace=str(tmp_path / "repo"),
model="model-x", mode="interactive", agent="swe-lead"))
team = mgr.teams.create(space=mgr._board_space("sess_lead"), lead_session="sess_lead", lead_actor="lead",
workers=[TeamWorker(actor="sam", persona="swe-worker", session_id="sess_sam")])
use = {"input": 29, "cache_read": 0, "output": 1, "cache_write": 0}
mgr.session_store.save(SessionRecord(
session_id="sess_sam", workspace=str(tmp_path / "repo"), model="model-x", mode="interactive",
agent="swe-worker", messages=[
{"role": "assistant", "content": "before recording", "ts": 10, "usage": use}, # 30
{"role": "assistant", "content": "recorded", "ts": 25, "usage": use}, # 30, in the recording
]))
store = mgr.recordings_store
store.write_turn(_turn("W1", "sess_sam", 20_000, 10_000, woken_by=_by()))
store.write_step(_model("wm", "W1", "sess_sam", 25_000, 30))
store.write_step({**_model("wh", "W1", "sess_sam", 25_500, 100, purpose="helper"), "seq": 2})
summary = mgr.team_summary(team.team_id)
sam = next(w for w in summary["workers"] if w["actor"] == "sam")
# 30 from before the first recorded turn (transcript), 30 main + 100 helper (recording).
assert sum(sam["tokens"].values()) == 160
assert sum(sam["tokens"].values()) == mgr.recording_usage("sess_sam")["pill"] + 30
asyncio.run(mgr.aclose())
# -- export: a worker turn's span joins the lead's trace ----------------------------------------
def _root_span(turn):
spans = export_spans.build_turn_spans(turn, [], sw=export_spans.switches_from({}), dialect="otlp",
get_piece=lambda k: None, masker=export_spans.Masker())
return next(s for s in spans if s["parent_span_id"] is None)
def test_a_worker_turns_span_names_its_team_and_lead():
root = _root_span(_turn("W1", "sess_sam", 2000, 1000, woken_by=_by()))
assert root["attrs"]["openworker.team.id"] == "team-1"
assert root["attrs"]["openworker.lead.session.id"] == "sess_lead"
assert "openworker.team.id" not in _root_span(_turn("L1", "sess_lead", 1000, 5000))["attrs"]