Python
feat(observability): every model call is a model step with its exact exchange and correct token counts (OPE-244)
Devika Verma1e010d19966f
diff --git a/coworker/agent.py b/coworker/agent.pyindex e9278f00..299e128c 100644--- a/coworker/agent.py+++ b/coworker/agent.py@@ -36,6 +36,7 @@ from .permissions import Mode, PermissionEnginefrom .project import load_agents_mdfrom . import session_factsfrom .roots import RootDir, normalize_roots, render_context+from .observability.provider_view import tag as tag_providerfrom .providers import ProviderClient, ProviderRouterfrom .overrides import RiskOverrideStorefrom .secrets import SecretStore, state_dir@@ -876,7 +877,7 @@ def build_engine()engine.reviewer = make_auto_reviewer(choice,- provider=provider,+ provider=tag_provider(engine.provider, purpose="auto_approver"),model=model,known_world=engine.session_facts.world.render() + "\nRUNTIME FACTS (availability, not access grants)\n" + json.dumps(engine.runtime_facts),)diff --git a/coworker/engine.py b/coworker/engine.pyindex d5d6ec4a..c5d0d028 100644--- a/coworker/engine.py+++ b/coworker/engine.py@@ -34,6 +34,7 @@ from . import toolresultfrom . import unattended as _attendancefrom .events import Event, EventTypefrom .observability.engine_hooks import TurnRecording+from .observability.provider_view import tag as _tag_provider, view_for_enginefrom .observability.ids import new_ulid# §8.4 retry guard: the reviewer pauses for the rest of the turn after this many denials@@ -484,6 +485,9 @@ class TurnEngine:# Observability (E1): the turn and step events this engine sends to its recorder.# Off until a recorder is set (`engine.recorder = ...`), like the other surface hooks.self._obs = TurnRecording(None, lambda: str(self.audit_context.get("session_id") or ""))+ # The engine's own tagged view of its provider (observability E2, §3.3): every call+ # it makes is recorded as a model step of the current turn, purpose `main`.+ self.provider = view_for_engine(self.provider, self._obs)self._call_index: dict[str, int] = {}if instructions and not (self.messages and self.messages[0].get("role") == "system"@@ -1248,7 +1252,7 @@ class TurnEngine:def _build() -> Optional[_compaction.CompactionState]:return _compaction.build_state(self.messages,- provider=self.provider,+ provider=_tag_provider(self.provider, purpose="summary"),model=model,keep_tokens=keep,prior=self.compaction_state,diff --git a/coworker/observability/capture.py b/coworker/observability/capture.pynew file mode 100644index 00000000..0b689106--- /dev/null+++ b/coworker/observability/capture.py@@ -0,0 +1,567 @@+"""Everything sent and received on a model call (observability E22, technical design §5.2.1).++For every model call the step keeps three things:+- the exchange: the exact request (method, URL, headers, body) and the exact response+ (status, headers, the full body or every stream event), content included, minus secrets.+ Stored as pieces (TD-17): each message, tool definition or response body once, by its+ SHA-256, and the request as a manifest that points at them;+- `provider_meta`: the same responses with content removed, every other field as sent;+- `provider_facts`: fixed names read from `provider_meta`, the same for every provider.++How it is captured: both SDKs the app uses, and the Ollama transport, send their requests+through httpx's `HTTPTransport` (or `httpx2`'s, the fork newer SDKs ship on). Its+`handle_request` is wrapped once. While a model call is being recorded on the current+thread, each request and response it makes is copied into that call's list; on any other+thread, or when nothing is recorded, the wrapper only checks a thread-local and passes the+request through. Calls are recorded on the thread that makes them (the stream thread for a+streamed reply), so two sessions calling at once never mix their requests.++Ported from the Phase 1 lab (`phase-1/lab/raw_capture.py`), where the exact rebuild and the+"nothing missing from the metadata view" checks passed on every live call.+"""++from __future__ import annotations++import contextlib+import hashlib+import json+import threading+import zlib+from typing import Any, Callable, Iterator, Optional++import httpx++# -- capture ------------------------------------------------------------------------------++_active = threading.local()+_install_lock = threading.Lock()+_installed = False+++def _http_modules() -> list[Any]:+ """httpx, and httpx2 where installed (the fork newer anthropic and openai SDKs use)."""+ mods = [httpx]+ try:+ import httpx2 # type: ignore[import-not-found]++ mods.append(httpx2)+ except ImportError:+ pass+ return mods+++def _tee_class(mod: Any) -> type:+ class Tee(mod.SyncByteStream): # type: ignore[misc, name-defined]+ def __init__(self, inner: Any, sink: bytearray) -> None:+ self.inner, self.sink = inner, sink++ def __iter__(self) -> Iterator[bytes]:+ for chunk in self.inner:+ self.sink.extend(chunk)+ yield chunk++ def close(self) -> None:+ close = getattr(self.inner, "close", None)+ if close:+ close()++ return Tee+++def install() -> None:+ """Wrap `HTTPTransport.handle_request` once per process. Safe to call many times."""+ global _installed+ with _install_lock:+ if _installed:+ return+ for mod in _http_modules():+ original = mod.HTTPTransport.handle_request+ tee = _tee_class(mod)++ def handle(self: Any, request: Any, _original: Any = original, _tee: type = tee) -> Any:+ log = getattr(_active, "log", None)+ response = _original(self, request)+ if log is None:+ return response+ try:+ body = bytes(request.content or b"")+ except Exception: # noqa: BLE001 - a streamed request body cannot be re-read+ body = b""+ entry: dict[str, Any] = {+ "url": str(request.url),+ "method": request.method,+ "status": response.status_code,+ "request_headers": {+ k: v for k, v in request.headers.items() if not is_secret_header(k)+ },+ "request_body": body,+ "response_headers": dict(response.headers.items()),+ "_body": bytearray(),+ }+ response.stream = _tee(response.stream, entry["_body"])+ log.append(entry)+ return response++ mod.HTTPTransport.handle_request = handle+ _installed = True+++class HttpCapture:+ """The HTTP exchanges of one model call. Capture is switched on only inside `active()`,+ so a stream left half-read (Stop pressed) never keeps capturing on its thread: the+ recording view reads each stream piece inside `active()`."""++ def __init__(self) -> None:+ install()+ self._log: list[dict[str, Any]] = []++ @contextlib.contextmanager+ def active(self) -> Iterator[None]:+ outer = getattr(_active, "log", None)+ _active.log = self._log+ try:+ yield+ finally:+ _active.log = outer++ def entries(self) -> list[dict[str, Any]]:+ """Every exchange so far, with each response body decoded to text."""+ for e in self._log:+ if "_body" in e:+ e["body"] = _decode(bytes(e.pop("_body")), e["response_headers"]).decode(+ "utf-8", "replace"+ )+ return self._log+++def _decode(data: bytes, headers: dict[str, str]) -> bytes:+ """The transport sees the body as sent on the wire, before content decoding."""+ enc = next((v for k, v in headers.items() if k.lower() == "content-encoding"), "").lower()+ try:+ if enc == "gzip":+ return zlib.decompress(data, 16 + zlib.MAX_WBITS)+ if enc == "deflate":+ return zlib.decompress(data)+ if enc == "br":+ import brotli # type: ignore[import-not-found]++ return brotli.decompress(data)+ if enc == "zstd":+ import zstandard # type: ignore[import-not-found]++ return zstandard.ZstdDecompressor().decompressobj().decompress(data)+ except Exception: # noqa: BLE001 - keep the raw bytes+ return data+ return data+++# -- secrets --------------------------------------------------------------------------------++SECRET_HEADERS = {+ "set-cookie", "cookie", "authorization", "x-api-key", "x-goog-api-key", "api-key",+ "proxy-authorization",+}+++def is_secret_header(name: str) -> bool:+ n = name.lower()+ return n in SECRET_HEADERS or "token" in n or n.endswith("-key") or "secret" in n+++def _public_headers(headers: dict[str, str]) -> dict[str, str]:+ return {k: v for k, v in headers.items() if not is_secret_header(k)}+++# -- which wire -----------------------------------------------------------------------------++def wire_of_url(url: str) -> str:+ u = url.split("?", 1)[0].rstrip("/")+ if u.endswith("/v1/messages"):+ return "anthropic"+ if u.endswith("/responses"):+ return "openai_responses"+ if u.endswith("/v1/decisions"):+ return "openai_decisions"+ if u.endswith("/v1/systemone"):+ return "typesafe_jev"+ if u.endswith("/api/chat") or u.endswith("/api/show"):+ return "ollama"+ if ":generateContent" in u or ":streamGenerateContent" in u:+ return "gemini"+ return "openai_chat"+++def is_auxiliary(url: str) -> bool:+ """A helper request made before the call, not an attempt of it (M-R9): Ollama's /api/show."""+ return url.split("?", 1)[0].rstrip("/").endswith("/api/show")+++# -- provider_meta: everything sent, minus content and secrets ------------------------------+#+# A short DENY list, not a list of fields to keep: whatever a provider sends that is not on+# it is saved, including fields it adds later. Paths match an event's flattened field paths;+# "*" applies to every event kind of that wire.++CONTENT_PATHS: dict[str, dict[str, list[str]]] = {+ "anthropic": {+ "message_start": ["message.content"],+ "content_block_start": ["content_block.text", "content_block.thinking",+ "content_block.signature", "content_block.data",+ "content_block.input"],+ "content_block_delta": ["delta.text", "delta.partial_json", "delta.thinking",+ "delta.signature"],+ "message": ["content"],+ },+ "openai_responses": {+ "*": ["delta", "text", "arguments", "part.text", "item.content", "item.arguments",+ "item.summary", "item.encrypted_content", "response.output",+ "response.instructions", "response.tools", "response.text.format.schema",+ "output", "instructions", "tools"],+ },+ "openai_decisions": {"*": ["answers"]},+ "typesafe_jev": {"*": ["answers"]},+ "ollama": {"*": ["message.content", "message.thinking", "message.tool_calls",+ "message.images"]},+ "openai_chat": {+ "*": ["choices[].delta.content", "choices[].delta.reasoning",+ "choices[].delta.reasoning_content", "choices[].delta.reasoning_details",+ "choices[].delta.tool_calls", "choices[].message", "choices[].logprobs"],+ },+ "gemini": {"*": ["candidates[].content"]},+}++# Request fields that are content (the conversation, tool definitions, system text, a+# decision reviewer's action description and question wording).+_REQUEST_CONTENT = ("messages", "input", "tools", "system", "instructions", "state",+ "questions", "contents", "systemInstruction")+++def request_params(body: bytes) -> dict[str, Any]:+ """Every request parameter except content; `tools_offered` as a count."""+ try:+ parsed = json.loads(body or b"{}")+ except Exception: # noqa: BLE001 - not JSON+ return {}+ if not isinstance(parsed, dict):+ return {}+ out = {k: v for k, v in parsed.items() if k not in _REQUEST_CONTENT}+ out["tools_offered"] = len(parsed.get("tools") or [])+ if isinstance(parsed.get("questions"), list):+ out["questions_asked"] = len(parsed["questions"])+ out["question_names"] = [+ q.get("name") for q in parsed["questions"] if isinstance(q, dict) and q.get("name")+ ] or None+ return out+++def parse_events(body: str) -> list[Any]:+ """Every JSON object in a body: plain JSON, server-sent events, or one object per line."""+ body = body.strip()+ if body.startswith("{") or body.startswith("["):+ try:+ parsed = json.loads(body)+ return parsed if isinstance(parsed, list) else [parsed]+ except ValueError:+ pass+ lines = [ln.strip() for ln in body.splitlines() if ln.strip()]+ if all(ln.startswith("{") for ln in lines):+ events: list[Any] = []+ for ln in lines:+ try:+ events.append(json.loads(ln))+ except ValueError:+ events.append({"_unparsed": ln[:200]})+ return events+ events = []+ for line in body.splitlines():+ line = line.strip()+ if line.startswith("data:"):+ data = line[5:].strip()+ if data and data != "[DONE]":+ try:+ events.append(json.loads(data))+ except ValueError:+ events.append({"_unparsed": data[:200]})+ return events+++def event_kind_of(wire: str, ev: dict[str, Any]) -> str:+ if wire == "openai_chat":+ return "chunk with usage" if ev.get("usage") else "chunk"+ if wire == "ollama":+ return "done" if ev.get("done") else "chunk"+ if wire == "gemini":+ return "chunk with usage" if ev.get("usageMetadata") else "chunk"+ return str(ev.get("type", "response"))+++def is_content(wire: str, kind: str, path: str) -> bool:+ rules = CONTENT_PATHS.get(wire, {})+ for pat in rules.get("*", []) + rules.get(kind, []):+ if path == pat or path.startswith(pat + ".") or path.startswith(pat + "["):+ return True+ return False+++def _strip(obj: Any, wire: str, kind: str, prefix: str = "") -> Any:+ if isinstance(obj, dict):+ out = {}+ for k, v in obj.items():+ p = f"{prefix}.{k}" if prefix else k+ if is_content(wire, kind, p):+ continue+ out[k] = _strip(v, wire, kind, p)+ return out+ if isinstance(obj, list):+ return [_strip(v, wire, kind, f"{prefix}[]") for v in obj]+ return obj+++def _merge(into: dict[str, Any], new: dict[str, Any]) -> None:+ """Deep merge, later values win (stream counters end at their final value)."""+ for k, v in new.items():+ old = into.get(k)+ if isinstance(v, dict) and isinstance(old, dict):+ _merge(old, v)+ elif isinstance(v, list) and isinstance(old, list) and all(isinstance(x, dict) for x in old + v):+ # Stream pieces repeat lists such as `choices`: merge item by item, so a field+ # sent only in the first piece is not lost to a later one.+ for i, item in enumerate(v):+ if i < len(old):+ _merge(old[i], item)+ else:+ old.append(item)+ elif v is not None or k not in into:+ into[k] = v+++def provider_meta(entry: dict[str, Any]) -> dict[str, Any]:+ """Everything one HTTP response told us, as sent, minus content and secrets."""+ wire = wire_of_url(entry["url"])+ aux = is_auxiliary(entry["url"])+ events: dict[str, Any] = {}+ counts: dict[str, int] = {}+ for ev in parse_events(entry.get("body", "")):+ if not isinstance(ev, dict):+ continue+ kind = "response" if aux else event_kind_of(wire, ev)+ counts[kind] = counts.get(kind, 0) + 1+ _merge(events.setdefault(kind, {}), _strip(ev, wire, kind))+ out: dict[str, Any] = {+ "wire": wire,+ "status": entry["status"],+ "request": request_params(entry.get("request_body") or b""),+ "headers": _public_headers(entry.get("response_headers", {})),+ "event_counts": counts,+ "events": events,+ }+ if aux:+ out["auxiliary"] = True+ return out+++# -- provider_facts: the same names for every provider (§5.2.1, recording-format §6.4) ------++FACT_NAMES = (+ "served_model", "provider_request_id", "provider_response_id", "served_by", "reasoning",+ "cache_read", "cache_write", "cache_write_by_ttl", "provider_total", "requested",+ "applied", "stop", "provider_timing", "rate_limit", "hosted_tools", "cost_reported",+)+++def _get(d: Any, path: str) -> Any:+ for part in path.split("."):+ if isinstance(d, list):+ d = d[0] if d else None+ if not isinstance(d, dict):+ return None+ d = d.get(part)+ return d+++def provider_facts(meta: Optional[dict[str, Any]]) -> dict[str, Any]:+ """Fixed-name facts read from one `provider_meta` entry. None = the provider did not send it."""+ if not meta:+ return {}+ w, ev, h, req = meta.get("wire"), meta.get("events", {}), meta.get("headers", {}), meta.get("request", {})+ hl = {k.lower(): v for k, v in h.items()}+ if w == "anthropic":+ msg = _get(ev, "message_start.message") or ev.get("message") or {}+ fin = ev.get("message_delta", {})+ u = {**(msg.get("usage") or {}), **(fin.get("usage") or {})}+ stop_src = fin.get("delta") or msg+ facts = {+ "served_model": msg.get("model"), "provider_response_id": msg.get("id"),+ "provider_request_id": hl.get("request-id"),+ "reasoning": _get(u, "output_tokens_details.thinking_tokens"),+ "cache_read": u.get("cache_read_input_tokens"),+ "cache_write": u.get("cache_creation_input_tokens"),+ "cache_write_by_ttl": u.get("cache_creation"), "provider_total": None,+ "requested": {"thinking": req.get("thinking"), "effort": _get(req, "output_config.effort"),+ "max_tokens": req.get("max_tokens"), "temperature": req.get("temperature"),+ "service_tier": req.get("service_tier")},+ "applied": {"service_tier": u.get("service_tier"), "speed": u.get("speed"),+ "inference_geo": u.get("inference_geo")},+ "stop": {"reason": stop_src.get("stop_reason"), "sequence": stop_src.get("stop_sequence"),+ "details": stop_src.get("stop_details")},+ "provider_timing": {"server_timing": hl.get("server-timing")},+ "hosted_tools": u.get("server_tool_use"), "cost_reported": None,+ }+ elif w == "openai_responses":+ r = (ev.get("response.completed") or ev.get("response.incomplete")+ or ev.get("response.failed") or {}).get("response") or ev.get("response") or {}+ u = r.get("usage") or {}+ facts = {+ "served_model": r.get("model"), "provider_response_id": r.get("id"),+ "provider_request_id": hl.get("x-request-id"),+ "reasoning": _get(u, "output_tokens_details.reasoning_tokens"),+ "cache_read": _get(u, "input_tokens_details.cached_tokens"),+ "cache_write": _get(u, "input_tokens_details.cache_write_tokens"),+ "cache_write_by_ttl": None, "provider_total": u.get("total_tokens"),+ "requested": {"effort": _get(req, "reasoning.effort"), "max_tokens": req.get("max_output_tokens"),+ "temperature": req.get("temperature"), "service_tier": req.get("service_tier")},+ "applied": {"effort": _get(r, "reasoning.effort"), "service_tier": r.get("service_tier"),+ "prompt_cache_retention": r.get("prompt_cache_retention"),+ "truncation": r.get("truncation")},+ "stop": {"reason": r.get("status"), "details": r.get("incomplete_details")},+ "provider_timing": {"processing_ms": hl.get("openai-processing-ms"),+ "created_at": r.get("created_at"), "completed_at": r.get("completed_at")},+ "hosted_tools": r.get("tool_usage"), "cost_reported": None,+ }+ elif w in ("openai_decisions", "typesafe_jev"):+ r = ev.get("response", {})+ u = r.get("usage") or {}+ facts = {+ "served_model": r.get("model"), "provider_response_id": r.get("id"),+ "provider_request_id": hl.get("x-request-id") or hl.get("x-typesafe-request-id"),+ "cache_read": _get(u, "input_tokens_details.cached_tokens"),+ "cache_write": _get(u, "input_tokens_details.cache_write_tokens"),+ "reasoning": _get(u, "output_tokens_details.reasoning_tokens"),+ "provider_total": u.get("total_tokens"),+ "requested": {"model": req.get("model"), "questions_asked": req.get("questions_asked")},+ "applied": {}, "stop": {}, "cost_reported": None, "hosted_tools": None,+ "provider_timing": {"processing_ms": hl.get("openai-processing-ms"),+ "server_timing_total": hl.get("server-timing-total")},+ }+ elif w == "ollama":+ # Native /api/chat: the last line (`done: true`) carries the counts and timings, in+ # nanoseconds. `prompt_eval_cached_count` is the part of the prompt reused (V-20).+ d = ev.get("done", {}) or ev.get("response", {})+ opts = req.get("options") or {}+ facts = {+ "served_model": d.get("model"), "provider_response_id": None, "provider_request_id": None,+ "cache_read": d.get("prompt_eval_cached_count"),+ "reasoning": None, "cache_write": None, "cache_write_by_ttl": None, "provider_total": None,+ "requested": {"num_ctx": opts.get("num_ctx"), "max_tokens": opts.get("num_predict"),+ "temperature": opts.get("temperature"), "think": req.get("think"),+ "shift": req.get("shift"), "keep_alive": req.get("keep_alive")},+ "applied": {}, "stop": {"reason": d.get("done_reason")},+ "provider_timing": {k: d.get(k) for k in ("total_duration", "load_duration",+ "prompt_eval_duration", "eval_duration",+ "created_at")},+ "hosted_tools": None, "cost_reported": None,+ }+ elif w == "gemini":+ c = {**ev.get("chunk", {}), **ev.get("chunk with usage", {})}+ u = c.get("usageMetadata") or {}+ facts = {+ "served_model": c.get("modelVersion"), "provider_response_id": c.get("responseId"),+ "provider_request_id": None, "reasoning": u.get("thoughtsTokenCount"),+ "cache_read": u.get("cachedContentTokenCount"), "cache_write": None,+ "cache_write_by_ttl": None, "provider_total": u.get("totalTokenCount"),+ "requested": {"generation_config": req.get("generationConfig")}, "applied": {},+ "stop": {"reason": _get(c, "candidates.finishReason")}, "provider_timing": {},+ "hosted_tools": None, "cost_reported": None,+ }+ else:+ c = {**ev.get("chunk", {}), **ev.get("chunk with usage", {}), **ev.get("response", {})}+ u = c.get("usage") or {}+ facts = {+ "served_model": c.get("model"), "provider_response_id": c.get("id"),+ "provider_request_id": hl.get("x-request-id") or hl.get("x-generation-id"),+ "served_by": c.get("provider"),+ "reasoning": _get(u, "completion_tokens_details.reasoning_tokens"),+ "cache_read": _get(u, "prompt_tokens_details.cached_tokens"),+ "cache_write": _get(u, "prompt_tokens_details.cache_write_tokens"),+ "cache_write_by_ttl": None, "provider_total": u.get("total_tokens"),+ "requested": {"effort": req.get("reasoning_effort") or _get(req, "reasoning.effort"),+ "max_tokens": req.get("max_tokens") or req.get("max_completion_tokens"),+ "temperature": req.get("temperature"), "service_tier": req.get("service_tier")},+ "applied": {"service_tier": c.get("service_tier"),+ "system_fingerprint": c.get("system_fingerprint")},+ "stop": {"reason": _get(c, "choices.finish_reason"),+ "native_reason": _get(c, "choices.native_finish_reason")},+ "provider_timing": {"processing_ms": hl.get("openai-processing-ms")},+ "hosted_tools": None,+ "cost_reported": ({"amount": u.get("cost"), "details": u.get("cost_details"),+ "is_byok": u.get("is_byok")} if u.get("cost") is not None else None),+ }+ facts["rate_limit"] = {k: v for k, v in h.items() if "ratelimit" in k.lower()} or None+ return {k: facts.get(k) for k in FACT_NAMES}+++# -- the exchange, stored as pieces (TD-17) ---------------------------------------------------++# Request fields whose items are stored as separate pieces, shared across a session's calls.+PIECE_FIELDS = ("messages", "input", "tools", "system", "instructions", "contents",+ "questions", "state")+++def _piece(value: Any, put: Callable[[bytes], str]) -> dict[str, str]:+ data = json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)+ return {"$blob": put(data.encode("utf-8"))}+++def store_exchange(entry: dict[str, Any], put: Callable[[bytes], str]) -> dict[str, Any]:+ """One exchange entry (recording-format §6): the request as a manifest of pieces, the+ response body as one piece. `put(bytes) -> sha256` stores a piece once."""+ body: bytes = entry.get("request_body") or b""+ try:+ parsed = json.loads(body) if body else None+ except ValueError:+ parsed = None+ req_body: dict[str, Any]+ if isinstance(parsed, dict):+ manifest = {}+ for k, v in parsed.items():+ if k in PIECE_FIELDS and isinstance(v, list):+ manifest[k] = [_piece(x, put) for x in v]+ elif k in PIECE_FIELDS:+ manifest[k] = _piece(v, put)+ else:+ manifest[k] = v+ req_body = {"manifest": manifest}+ else:+ req_body = {"blob": put(body)} if body else {}+ resp = (entry.get("body") or "").encode("utf-8")+ out: dict[str, Any] = {+ "request": {+ "method": entry["method"], "url": entry["url"],+ "headers": _public_headers(entry.get("request_headers", {})),+ "body_sha256": hashlib.sha256(body).hexdigest(), "body_size": len(body), **req_body,+ },+ "response": {+ "status": entry["status"],+ "headers": _public_headers(entry.get("response_headers", {})),+ "body_blob": put(resp), "body_size": len(resp),+ },+ }+ if is_auxiliary(entry["url"]):+ out = {"auxiliary": True, **out}+ return out+++def rebuild_request(exchange: dict[str, Any], get: Callable[[str], Optional[bytes]]) -> Any:+ """The request JSON exactly as sent (or the raw body bytes, for a non-JSON body)."""+ req = exchange["request"]+ if "blob" in req:+ return get(req["blob"])++ def un(v: Any) -> Any:+ if isinstance(v, dict) and set(v) == {"$blob"}:+ return json.loads(get(v["$blob"]) or b"null")+ if isinstance(v, list):+ return [un(x) for x in v]+ return v++ return {k: un(v) for k, v in req.get("manifest", {}).items()}diff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.pyindex 26a2a94e..7836fafd 100644--- a/coworker/observability/engine_hooks.py+++ b/coworker/observability/engine_hooks.py@@ -23,6 +23,9 @@ from typing import Any, Callable, Optionalfrom .ids import new_ulidfrom .recorder import EVENT_V, Recorder, now_ms+# `start_step(turn_id=CURRENT)`: the engine's current turn.+CURRENT = object()+# TOOL_FINISHED status -> tool step status (technical design §5.1)._TOOL_STATUS = {"ok": "ok", "error": "error", "denied": "denied", "interrupted": "interrupted","skipped": "skipped"}@@ -44,6 +47,7 @@ class TurnRecording:self._ident: dict[str, dict[str, Any]] = {} # step_id -> identity copyself._tool_steps: dict[str, str] = {} # call_id -> step_id, for this turn's toolsself._tool_ends: dict[str, dict[str, Any]] = {} # call_id -> step_ended awaiting its mid+ self._reply_end: Optional[dict[str, Any]] = None # a main model step awaiting its reply's mid@propertydef enabled(self) -> bool:@@ -84,6 +88,7 @@ class TurnRecording:self._last_mid = Noneself._tool_steps.clear()self._tool_ends.clear()+ self._reply_end = Noneev = self._base("turn_started", self.turn_id, self._turn_session)ev.update({"kind": kind, "trigger": trigger, "first_message_id": first_message_id,"attrs": copy.deepcopy(attrs), "stamp": copy.deepcopy(stamp)})@@ -102,6 +107,7 @@ class TurnRecording:for ev in self._tool_ends.values(): # a tool whose result message never cameself.recorder.emit(ev)self._tool_ends.clear()+ self._flush_reply(None)ev = self._base("turn_ended", self.turn_id, self._turn_session)ev.update({"status": status, "last_message_id": self._last_mid})self.recorder.emit(ev)@@ -132,28 +138,47 @@ class TurnRecording:if ev is not None:ev["message_id"] = midself._send(ev)+ elif message.get("role") == "assistant":+ self._flush_reply(mid)# -- steps ----------------------------------------------------------------------------------def start_step(self, *, kind: str, name: str, call_id: Optional[str] = None,call_index: Optional[int] = None, parent_step_id: Optional[str] = None,attrs: Optional[dict[str, Any]] = None,- content: Optional[dict[str, Any]] = None) -> Optional[str]:+ content: Optional[dict[str, Any]] = None,+ turn_id: Any = CURRENT, shadow: bool = False) -> Optional[str]:+ """`step_started`. The step belongs to the engine's current turn, or with `turn_id` to+ that turn: a step tagged with a turn that already ended is `after_turn` (§3.3, an+ after-turn title). A model step outside any turn is recorded with no turn; a tool+ step outside a turn is not recorded."""if not self.enabled:return Nonewith self._lock:- if not self.in_turn:- return None+ after = False+ if turn_id is CURRENT:+ if self.in_turn:+ tid, sid = self.turn_id, self._turn_session+ elif kind == "tool":+ return None+ else:+ tid, sid = None, self._session_id() or ""+ else:+ tid = turn_id or None+ same = tid is not None and tid == self.turn_id+ sid = self._turn_session if same else (self._session_id() or "")+ # Turns of one engine run one after another: another turn of this engine has+ # ended, and so has this one once its end was sent.+ after = tid is not None and (not same or self._ended)step_id = new_ulid()- ev = self._base("step_started", self.turn_id, self._turn_session)+ ev = self._base("step_started", tid, sid)ev.update({"step_id": step_id, "parent_step_id": parent_step_id, "kind": kind,"name": name, "call_id": call_id, "call_index": call_index,- "shadow": False, "after_turn": False, "attrs": attrs or {},+ "shadow": bool(shadow), "after_turn": after, "attrs": attrs or {},"content": content or {}})self._ident[step_id] = {"kind": kind, "name": name, "parent_step_id": parent_step_id, "call_id": call_id,- "call_index": call_index, "shadow": False, "after_turn": False,- "started_at": self._started_at(ev), "turn_id": self.turn_id,- "session_id": self._turn_session,+ "call_index": call_index, "shadow": bool(shadow), "after_turn": after,+ "started_at": self._started_at(ev), "turn_id": tid, "session_id": sid,}if kind == "tool" and call_id:self._tool_steps[call_id] = step_id@@ -168,7 +193,10 @@ class TurnRecording:def end_step(self, step_id: Optional[str], *, status: str = "ok",attrs: Optional[dict[str, Any]] = None, content: Optional[dict[str, Any]] = None,message_id: Optional[str] = None, error: Optional[dict[str, Any]] = None,- wait_for_tool_message: bool = False) -> None:+ wait_for_tool_message: bool = False,+ wait_for_reply_message: bool = False) -> None:+ """`step_ended`. A tool step's end waits for its result message, a main model step's+ end for the reply message the engine adds next, so each carries that message's `mid`."""if not self.enabled or step_id is None:returnwith self._lock:@@ -182,11 +210,21 @@ class TurnRecording:ev["error"] = errorev.update({k: idn[k] for k in ("kind", "name", "parent_step_id", "call_id","call_index", "shadow", "after_turn", "started_at")})- if wait_for_tool_message and idn["call_id"] and not self._ended:+ waiting_turn = idn["turn_id"] is not None and idn["turn_id"] == self.turn_id and not self._ended+ if wait_for_tool_message and idn["call_id"] and waiting_turn:self._tool_ends[idn["call_id"]] = ev+ elif wait_for_reply_message and waiting_turn:+ self._flush_reply(None) # an earlier reply that never got a message+ self._reply_end = evelse:self._send(ev)+ def _flush_reply(self, mid: Optional[str]) -> None:+ ev, self._reply_end = self._reply_end, None+ if ev is not None:+ ev["message_id"] = mid+ self._send(ev)+def instant_step(self, *, kind: str, name: str, status: str, call_id: Optional[str] = None,call_index: Optional[int] = None, attrs: Optional[dict[str, Any]] = None,message_id: Optional[str] = None) -> None:diff --git a/coworker/observability/provider_view.py b/coworker/observability/provider_view.pynew file mode 100644index 00000000..62772c71--- /dev/null+++ b/coworker/observability/provider_view.py@@ -0,0 +1,366 @@+"""Tagged provider views: every model call recorded as a model step (observability E2, E22).++Every model call leaves the app through one provider router, shared by every session's+engine and by the OpenAI-compatible proxy. A wrapper around the router alone cannot know+which session, turn or purpose a call belongs to, so each caller holds its own **tagged+view** (technical design §3.3, TD-2):++ view = provider.tagged(purpose="summary", parent=step_id)+ turn = view.complete(model=..., messages=...) # recorded as a model step++- `RecordingProvider` wraps the router once, where it is made. A call made through it with+ no tags is recorded as `unattributed`, so a missing tag shows instead of counting as main.+- `tagged()` on a view returns a new view: fields passed replace the old, the rest are kept.+- A view records through a `sink`: the engine's turn recording (`TurnRecording`), which knows+ the turn, or for calls outside any engine (the proxy) a sink on the server's recorder.++Each model step carries the call's purpose, provider, model, tool count, time to the first+streamed piece, finish reason and token counts (§5.2), and, from the HTTP capture+(`capture.py`), the exact exchange, `provider_meta` and `provider_facts` (§5.2.1). Recording+never breaks a call: any recording error is logged and the call goes on.+"""++from __future__ import annotations++import json+import logging+import threading+import time+from dataclasses import dataclass, replace+from typing import Any, Callable, Iterator, Optional++from ..providers.base import ProviderClient+from . import capture+from .engine_hooks import CURRENT, TurnRecording+from .recorder import short_hash++log = logging.getLogger(__name__)+++@dataclass(frozen=True)+class Tags:+ purpose: str = "unattributed"+ parent: Optional[str] = None # the step this call is nested in+ call_id: Optional[str] = None # for auto-approver checks: the tool call checked+ shadow: bool = False+ turn: Any = CURRENT # CURRENT = the sink's current turn; a turn id; None = no turn+ sink: Optional[TurnRecording] = None+++class RecordingProvider(ProviderClient):+ """The router, wrapped once. Untagged calls are recorded as `unattributed` when a recorder+ is given; everything else (`invalidate`, `_secrets`…) passes through to the router."""++ def __init__(self, inner: ProviderClient, recorder: Any = None) -> None:+ self.inner = inner+ self.recorder = recorder+ self._sink = TurnRecording(recorder, lambda: "") if recorder is not None else None++ def tagged(self, **tags: Any) -> "ProviderView":+ if "sink" not in tags:+ tags["sink"] = self._sink+ return ProviderView(self, Tags(**tags))++ def complete(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any):+ return self.tagged().complete(model=model, messages=messages, tools=tools, **settings)++ def stream(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any):+ return self.tagged().stream(model=model, messages=messages, tools=tools, **settings)++ def capabilities(self, model: str):+ return self.inner.capabilities(model)++ def __getattr__(self, name: str) -> Any:+ return getattr(self.inner, name)+++class ProviderView(ProviderClient):+ """A tagged handle on the shared provider. Tags are explicit; nothing relies on context+ variables crossing threads."""++ def __init__(self, base: RecordingProvider, tags: Tags) -> None:+ self.base, self.tags = base, tags++ def tagged(self, **overrides: Any) -> "ProviderView":+ return ProviderView(self.base, replace(self.tags, **overrides))++ def capabilities(self, model: str):+ return self.base.inner.capabilities(model)++ def __getattr__(self, name: str) -> Any:+ return getattr(self.base.inner, name)++ # -- calls --------------------------------------------------------------------------------+ def complete(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any):+ call = _ModelCall.begin(self.tags, model=model, tools=tools, streamed=False)+ if call is None:+ return self.base.inner.complete(model=model, messages=messages, tools=tools, **settings)+ try:+ with call.http.active():+ turn = self.base.inner.complete(model=model, messages=messages, tools=tools, **settings)+ except BaseException as exc:+ call.failed(exc)+ raise+ call.finished(turn)+ return turn++ def stream(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any) -> Iterator[Any]:+ call = _ModelCall.begin(self.tags, model=model, tools=tools, streamed=True)+ if call is None:+ yield from self.base.inner.stream(model=model, messages=messages, tools=tools, **settings)+ return+ it = iter(())+ turn = None+ try:+ with call.http.active():+ it = iter(self.base.inner.stream(model=model, messages=messages, tools=tools, **settings))+ while True:+ # Capture is on only while the next piece is read, so a stream left half-read+ # (Stop pressed) never keeps capturing on this thread. Such a step stays open+ # and is closed by the turn's end as `interrupted` (§7.4).+ with call.http.active():+ try:+ chunk = next(it)+ except StopIteration:+ break+ call.saw(chunk)+ if getattr(chunk, "turn", None) is not None:+ turn = chunk.turn+ yield chunk+ except GeneratorExit:+ raise+ except BaseException as exc:+ call.failed(exc)+ raise+ if turn is not None:+ call.finished(turn)+++# -- one recorded call ----------------------------------------------------------------------------++_tools_hash_cache: dict[int, tuple[Any, Optional[str]]] = {}+_tools_hash_lock = threading.Lock()+++def tools_hash(tools: Any) -> Optional[str]:+ """A short code of the tool definitions, to spot tool changes; cached per tool list (P-6)."""+ if not tools:+ return None+ key = id(tools)+ with _tools_hash_lock:+ hit = _tools_hash_cache.get(key)+ if hit is not None and hit[0] is tools:+ return hit[1]+ code = short_hash(json.dumps(tools, sort_keys=True, default=str))+ with _tools_hash_lock:+ if len(_tools_hash_cache) > 64:+ _tools_hash_cache.clear()+ _tools_hash_cache[key] = (tools, code)+ return code+++def _provider_of(model: str) -> str:+ return (model or "").split(":", 1)[0] if ":" in (model or "") else "openai"+++def _provider_title(provider: str) -> Optional[str]:+ try:+ from ..providers.registry import get_descriptor++ d = get_descriptor(provider)+ return getattr(d, "title", None) if d is not None else None+ except Exception: # noqa: BLE001+ return None+++def _context_window(model: str) -> Optional[int]:+ try:+ from ..providers.matrix import model_context_windows++ return model_context_windows().get(model)+ except Exception: # noqa: BLE001+ return None+++def root_cause(exc: BaseException) -> BaseException:+ seen = set()+ while id(exc) not in seen and (exc.__cause__ is not None or exc.__context__ is not None):+ seen.add(id(exc))+ exc = exc.__cause__ or exc.__context__ # type: ignore[assignment]+ return exc+++def classify_error(exc: BaseException) -> str:+ """`error.type` from the failure's root cause (§5.2, recording-format §5)."""+ try:+ from ..compaction import is_context_overflow++ if is_context_overflow(exc):+ return "context_length"+ except Exception: # noqa: BLE001+ pass+ root = root_cause(exc)+ text = f"{type(exc).__name__} {exc} {type(root).__name__} {root}".lower()+ status = getattr(exc, "status_code", None) or getattr(root, "status_code", None)+ if status in (529, 503) or "overload" in text:+ return "overloaded"+ if status == 429 or "rate limit" in text or "ratelimit" in text:+ return "rate_limited"+ if status in (401, 403) or "api key" in text or "authentication" in text:+ return "auth"+ if "timeout" in text or "timed out" in text:+ return "timeout"+ if "cancel" in text:+ return "cancelled"+ if any(k in text for k in ("connect", "dns", "ssl", "tls", "proxy", "name resolution")):+ return "network"+ return "provider_error"+++def usage_attrs(usage: Any) -> dict[str, Any]:+ """The step's `usage` from the provider's `TokenUsage` (§5.2, §6.2). A count the provider+ did not send is `null`, never 0; a call with no counts is `{"reported": false}`."""+ if usage is None:+ return {"reported": False}+ return {+ "reported": True,+ "input_fresh": usage.input,+ "cache_read": usage.cache_read if getattr(usage, "cache_read_reported", True) else None,+ "cache_write": usage.cache_write if getattr(usage, "cache_write_reported", True) else None,+ "output": usage.output,+ "reasoning": getattr(usage, "reasoning", None),+ }+++class _ModelCall:+ """The state of one recorded model call, from start to end."""++ def __init__(self, tags: Tags, sink: TurnRecording, step_id: str, streamed: bool) -> None:+ self.tags, self.sink, self.step_id, self.streamed = tags, sink, step_id, streamed+ self.http = capture.HttpCapture()+ self.t0 = time.perf_counter()+ self.ttft_ms: Optional[int] = None++ @classmethod+ def begin(cls, tags: Tags, *, model: str, tools: Any, streamed: bool) -> Optional["_ModelCall"]:+ sink = tags.sink+ if sink is None or not sink.enabled:+ return None+ try:+ provider = _provider_of(model)+ step_id = sink.start_step(+ kind="model", name="model", turn_id=tags.turn, shadow=tags.shadow,+ parent_step_id=tags.parent,+ call_id=tags.call_id if tags.purpose == "auto_approver" else None,+ attrs={+ "purpose": tags.purpose, "provider": provider,+ "provider_title": _provider_title(provider), "model": model,+ "stream": streamed, "context_window": _context_window(model),+ "tools_offered": len(tools or []), "tools_hash": tools_hash(tools),+ },+ )+ except Exception: # noqa: BLE001 - recording never breaks a call+ log.exception("recording: could not start a model step")+ return None+ if step_id is None:+ return None+ return cls(tags, sink, step_id, streamed)++ def saw(self, chunk: Any) -> None:+ # The first streamed piece of the reply: text or thinking (§7.2).+ if self.ttft_ms is None and (getattr(chunk, "text_delta", None) or getattr(chunk, "reasoning_delta", None)):+ self.ttft_ms = int((time.perf_counter() - self.t0) * 1000)++ def _exchange(self) -> tuple[dict[str, Any], dict[str, Any]]:+ """(attrs, content) from the HTTP capture: provider_meta, provider_facts, the exchange."""+ attrs: dict[str, Any] = {}+ content: dict[str, Any] = {}+ entries = [e for e in self.http.entries() if e.get("method") == "POST"]+ if not entries:+ return attrs, content+ recorder = self.sink.recorder+ content["exchange"] = [capture.store_exchange(e, recorder.put_piece) for e in entries]+ meta = [capture.provider_meta(e) for e in entries]+ attrs["provider_meta"] = meta+ reply = [m for m in meta if not m.get("auxiliary")] or meta+ facts = capture.provider_facts(reply[-1])+ attrs["provider_facts"] = facts+ if facts.get("served_by") is not None:+ attrs["served_by"] = facts["served_by"]+ return attrs, content++ def finished(self, turn: Any) -> None:+ try:+ attrs: dict[str, Any] = {+ "finish_reason": getattr(turn, "finish_reason", None),+ "ttft_ms": self.ttft_ms if self.streamed else None,+ "usage": usage_attrs(getattr(turn, "usage", None)),+ }+ if getattr(turn, "served_by", None):+ attrs["served_by"] = turn.served_by+ more, content = self._exchange()+ attrs.update(more)+ _fill_from_facts(attrs)+ self.sink.end_step(+ self.step_id, status="ok", attrs=attrs, content=content,+ wait_for_reply_message=self.tags.purpose == "main",+ )+ except Exception: # noqa: BLE001+ log.exception("recording: could not end a model step")++ def failed(self, exc: BaseException) -> None:+ try:+ if _is_stop(exc):+ return # Stop: the turn's end closes the step as interrupted+ more, content = self._exchange()+ attrs: dict[str, Any] = {"finish_reason": None, "ttft_ms": self.ttft_ms if self.streamed else None,+ "usage": {"reported": False}, **more}+ text = str(root_cause(exc)).strip()[:200] or None+ self.sink.end_step(self.step_id, status="error", attrs=attrs, content=content,+ error={"type": classify_error(exc), "text": text})+ except Exception: # noqa: BLE001+ log.exception("recording: could not record a failed model step")+++def _is_stop(exc: BaseException) -> bool:+ return isinstance(exc, (GeneratorExit, KeyboardInterrupt))+++def _fill_from_facts(attrs: dict[str, Any]) -> None:+ """A count the provider sent is recorded even where the provider code dropped it+ (V-13, V-15), and impossible numbers are flagged, never corrected (V-19)."""+ use, facts = attrs.get("usage") or {}, attrs.get("provider_facts") or {}+ if not use.get("reported"):+ return+ if use.get("reasoning") is None and facts.get("reasoning") is not None:+ use["reasoning"] = facts["reasoning"]+ if use.get("cache_write") is None and facts.get("cache_write") is not None:+ use["cache_write"] = facts["cache_write"]+ flags = []+ if use.get("reasoning") is not None and use.get("output") is not None and use["reasoning"] > use["output"]:+ flags.append("reasoning_exceeds_output")+ total = facts.get("provider_total")+ parts = (use.get("input_fresh") or 0) + (use.get("cache_read") or 0) + (use.get("cache_write") or 0) + (use.get("output") or 0)+ if total is not None and total != parts:+ flags.append("provider_total_differs")+ if flags:+ use["inconsistent"] = flags+++def view_for_engine(provider: Any, sink: TurnRecording) -> "ProviderView":+ """The engine's own view (§3.3): tags its creator already set are kept, so a helper's+ engine keeps `helper`; a plain provider is wrapped first."""+ if isinstance(provider, ProviderView):+ tags = provider.tags+ return provider.tagged(+ purpose="main" if tags.purpose == "unattributed" else tags.purpose,+ sink=sink if tags.sink is None or tags.sink is provider.base._sink else tags.sink,+ )+ base = provider if isinstance(provider, RecordingProvider) else RecordingProvider(provider)+ return base.tagged(purpose="main", sink=sink)+++def tag(provider: Any, **tags: Any) -> Any:+ """`provider.tagged(**tags)` for a recording provider or view; any other provider as is."""+ tagged = getattr(type(provider), "tagged", None)+ return provider.tagged(**tags) if tagged is not None else providerdiff --git a/coworker/observability/recorder.py b/coworker/observability/recorder.pyindex 11012e46..271b11a5 100644--- a/coworker/observability/recorder.py+++ b/coworker/observability/recorder.py@@ -20,6 +20,7 @@ Rust engine too.from __future__ import annotationsimport copy+import hashlibimport loggingimport threadingimport time@@ -129,6 +130,11 @@ def _merge_top(into: dict[str, Any], new: Optional[dict[str, Any]]) -> None:into[k] = copy.deepcopy(v)+def short_hash(text: str) -> str:+ """First 6 hex characters of the SHA-256: the short codes of §5.2 and §9.4."""+ return hashlib.sha256(text.encode("utf-8")).hexdigest()[:6]++def now_ms() -> tuple[int, int]:"""(at_ms, mono_ms) for an event sent now.@@ -160,6 +166,23 @@ class Recorder:self._lock = threading.RLock()self._turn_count: dict[str, int] = {} # session_id -> turns seen, for `seq`self._listeners: list[Callable[[dict[str, Any]], None]] = []+ self._pieces: dict[str, bytes] = {} # exchange pieces when there is no store++ # -- exchange pieces (TD-17) ------------------------------------------------------------+ def put_piece(self, data: bytes) -> str:+ """Store one exchange piece once, by its SHA-256: in the store, or in memory."""+ if self.store is not None:+ return self.store.put_piece(data)+ key = hashlib.sha256(data).hexdigest()+ with self._lock:+ self._pieces.setdefault(key, data)+ return key++ def get_piece(self, key: str) -> Optional[bytes]:+ if self.store is not None:+ return self.store.get_piece(key)+ with self._lock:+ return self._pieces.get(key)# -- listeners ---------------------------------------------------------------------------def add_listener(self, fn: Callable[[dict[str, Any]], None]) -> Callable[[], None]:diff --git a/coworker/observability/store.py b/coworker/observability/store.pyindex fd017a2a..b279d12f 100644--- a/coworker/observability/store.py+++ b/coworker/observability/store.py@@ -134,7 +134,8 @@ def _step_row(s: dict[str, Any]) -> tuple:meta = attrs.pop("provider_meta", None)usage = attrs.get("usage") if isinstance(attrs.get("usage"), dict) else Noneif s.get("kind") == "model":- reported: Optional[int] = int(usage is not None)+ # A call the provider gave no counts for is `usage: {"reported": false}` (§5.2).+ reported: Optional[int] = int(bool(usage) and usage.get("reported", True) is not False)else:reported = _bool(attrs.get("usage_reported"))facts = attrs.get("provider_facts") or {}diff --git a/coworker/providers/anthropic_provider.py b/coworker/providers/anthropic_provider.pyindex 25c1d88e..629150b9 100644--- a/coworker/providers/anthropic_provider.py+++ b/coworker/providers/anthropic_provider.py@@ -42,13 +42,41 @@ def _usage_from(usage: Any) -> Optional[TokenUsage]:"""Messages-API usage object → normalized counts (input_tokens excludes cache)."""if usage is None:return None+ read = getattr(usage, "cache_read_input_tokens", None)+ write = getattr(usage, "cache_creation_input_tokens", None)+ thinking = getattr(getattr(usage, "output_tokens_details", None), "thinking_tokens", None)return TokenUsage(input=int(getattr(usage, "input_tokens", 0) or 0),output=int(getattr(usage, "output_tokens", 0) or 0),- cache_read=int(getattr(usage, "cache_read_input_tokens", 0) or 0),- cache_write=int(getattr(usage, "cache_creation_input_tokens", 0) or 0),+ cache_read=int(read or 0),+ cache_write=int(write or 0),+ reasoning=None if thinking is None else int(thinking),+ cache_read_reported=read is not None,+ cache_write_reported=write is not None,)++def _merge_final_usage(usage: Optional[TokenUsage], final: Any) -> Optional[TokenUsage]:+ """The stream's last `message_delta` usage holds the call's totals (observability E3):+ every count it carries replaces the opening event's."""+ if final is None:+ return usage+ usage = usage or TokenUsage()+ for attr, field in (("input_tokens", "input"), ("output_tokens", "output")):+ value = getattr(final, attr, None)+ if value:+ setattr(usage, field, int(value))+ read = getattr(final, "cache_read_input_tokens", None)+ if read is not None:+ usage.cache_read, usage.cache_read_reported = int(read), True+ write = getattr(final, "cache_creation_input_tokens", None)+ if write is not None:+ usage.cache_write, usage.cache_write_reported = int(write), True+ thinking = getattr(getattr(final, "output_tokens_details", None), "thinking_tokens", None)+ if thinking is not None:+ usage.reasoning = int(thinking)+ return usage+# Required by the Messages API; a ceiling, not a spend target. Sized for file# generation, not just chat: a coworker writing a self-contained HTML report ships the# whole file inside one tool call's arguments, and 16k proved too small in the field@@ -788,13 +816,8 @@ class AnthropicProvider(ProviderClient):reason = getattr(last_message_delta, "stop_reason", None)if reason:stop_reason = reason- # Final (cumulative) output-token count rides message_delta.usage.- out = int(- getattr(getattr(event, "usage", None), "output_tokens", 0) or 0- )- if out:- usage = usage or TokenUsage()- usage.output = out+ # The final (cumulative) counts ride message_delta.usage.+ usage = _merge_final_usage(usage, getattr(event, "usage", None))_raise_on_refusal(stop_reason, last_message_delta)tool_calls = []diff --git a/coworker/providers/base.py b/coworker/providers/base.pyindex c197793b..3b11a4c3 100644--- a/coworker/providers/base.py+++ b/coworker/providers/base.py@@ -30,12 +30,21 @@ class TokenUsage:split into `cache_read`/`cache_write`. Providers that don't report a cachesplit (Ollama, most compat vendors) leave the cache fields at 0. `output`includes thinking tokens where the vendor bills them as output (Gemini).++ For observability (E3) a count the provider did not send must read as unknown, not+ 0: `cache_read_reported` / `cache_write_reported` say whether it was sent, and+ `reasoning` (thinking tokens within `output`) is None when not sent. The int fields+ stay ints, so existing sums are unchanged."""input: int = 0output: int = 0cache_read: int = 0cache_write: int = 0+ # Not part of equality: two usages with the same counts are equal, as before.+ reasoning: Optional[int] = field(default=None, compare=False)+ cache_read_reported: bool = field(default=True, compare=False)+ cache_write_reported: bool = field(default=True, compare=False)@propertydef context_tokens(self) -> int:diff --git a/coworker/providers/bedrock_provider.py b/coworker/providers/bedrock_provider.pyindex 6be7c4e4..3505d4e8 100644--- a/coworker/providers/bedrock_provider.py+++ b/coworker/providers/bedrock_provider.py@@ -58,6 +58,8 @@ def _usage_from(usage: Any) -> Optional[TokenUsage]:output=int(usage.get("outputTokens") or 0),cache_read=int(usage.get("cacheReadInputTokens") or 0),cache_write=int(usage.get("cacheWriteInputTokens") or 0),+ cache_read_reported="cacheReadInputTokens" in usage,+ cache_write_reported="cacheWriteInputTokens" in usage,)# Converse has no required max token param but per-model defaults vary wildly (Meta's isdiff --git a/coworker/providers/gemini_provider.py b/coworker/providers/gemini_provider.pyindex 9e58bbe6..9453faf4 100644--- a/coworker/providers/gemini_provider.py+++ b/coworker/providers/gemini_provider.py@@ -42,12 +42,15 @@ def _usage_from(meta: Any) -> Optional[TokenUsage]:if meta is None:return Noneprompt = int(getattr(meta, "prompt_token_count", 0) or 0)- cached = int(getattr(meta, "cached_content_token_count", 0) or 0)+ cached = getattr(meta, "cached_content_token_count", None)+ thoughts = getattr(meta, "thoughts_token_count", None)return TokenUsage(- input=max(prompt - cached, 0),- output=int(getattr(meta, "candidates_token_count", 0) or 0)- + int(getattr(meta, "thoughts_token_count", 0) or 0),- cache_read=cached,+ input=max(prompt - int(cached or 0), 0),+ output=int(getattr(meta, "candidates_token_count", 0) or 0) + int(thoughts or 0),+ cache_read=int(cached or 0),+ reasoning=None if thoughts is None else int(thoughts),+ cache_read_reported=cached is not None,+ cache_write_reported=False,)# Gemini finishReason → the engine's OpenAI-shaped finish_reason vocabulary. STOP maps todiff --git a/coworker/providers/ollama_context.py b/coworker/providers/ollama_context.pyindex a2090e04..3147bc8a 100644--- a/coworker/providers/ollama_context.py+++ b/coworker/providers/ollama_context.py@@ -437,6 +437,7 @@ def to_openai_completion(native: dict[str, Any]) -> dict[str, Any]:"prompt_tokens": prompt,"completion_tokens": completion,"total_tokens": prompt + completion,+ **_cached_details(native),},}return body@@ -477,6 +478,7 @@ def to_openai_stream_events("prompt_tokens": prompt,"completion_tokens": completion,"total_tokens": prompt + completion,+ **_cached_details(native),},})@@ -523,3 +525,13 @@ def _finish_reason(reason: str, has_tools: bool) -> str:def _usage_counts(native: dict[str, Any]) -> tuple[int, int]:return int(native.get("prompt_eval_count") or 0), int(native.get("eval_count") or 0)+++def _cached_details(native: dict[str, Any]) -> dict[str, Any]:+ """Ollama's `prompt_eval_cached_count`, the part of the prompt reused from its cache, as+ the OpenAI-shaped `cached_tokens`, so fresh input is the rest (V-20, observability E3).+ An older Ollama that does not send it leaves the cached share unknown."""+ cached = native.get("prompt_eval_cached_count")+ if cached is None:+ return {}+ return {"prompt_tokens_details": {"cached_tokens": int(cached)}}diff --git a/coworker/providers/openai_provider.py b/coworker/providers/openai_provider.pyindex b9db639f..b7be89a8 100644--- a/coworker/providers/openai_provider.py+++ b/coworker/providers/openai_provider.py@@ -203,11 +203,19 @@ def _usage_from(usage: Any) -> Optional[TokenUsage]:return Noneprompt = int(getattr(usage, "prompt_tokens", 0) or 0)details = getattr(usage, "prompt_tokens_details", None)- cached = int(getattr(details, "cached_tokens", 0) or 0)+ cached = getattr(details, "cached_tokens", None)+ # The cache write is inside the prompt count too (V-21), when the server sends it+ # (OpenRouter does, V-13). A count not sent is unknown, not 0 (observability E3).+ written = getattr(details, "cache_write_tokens", None)+ reasoning = getattr(getattr(usage, "completion_tokens_details", None), "reasoning_tokens", None)return TokenUsage(- input=max(prompt - cached, 0),+ input=max(prompt - int(cached or 0) - int(written or 0), 0),output=int(getattr(usage, "completion_tokens", 0) or 0),- cache_read=cached,+ cache_read=int(cached or 0),+ cache_write=int(written or 0),+ reasoning=None if reasoning is None else int(reasoning),+ cache_read_reported=cached is not None,+ cache_write_reported=written is not None,)diff --git a/coworker/providers/openai_responses.py b/coworker/providers/openai_responses.pyindex be9522fd..f2382e81 100644--- a/coworker/providers/openai_responses.py+++ b/coworker/providers/openai_responses.py@@ -245,11 +245,19 @@ def _usage_from(usage: Any) -> Optional[TokenUsage]:return Noneprompt = int(getattr(usage, "input_tokens", 0) or 0)details = getattr(usage, "input_tokens_details", None)- cached = int(getattr(details, "cached_tokens", 0) or 0)+ cached = getattr(details, "cached_tokens", None)+ # The cache write is inside the prompt count too (V-21), when the server sends it+ # (OpenRouter does, V-13). A count not sent is unknown, not 0 (observability E3).+ written = getattr(details, "cache_write_tokens", None)+ reasoning = getattr(getattr(usage, "output_tokens_details", None), "reasoning_tokens", None)return TokenUsage(- input=max(prompt - cached, 0),+ input=max(prompt - int(cached or 0) - int(written or 0), 0),output=int(getattr(usage, "output_tokens", 0) or 0),- cache_read=cached,+ cache_read=int(cached or 0),+ cache_write=int(written or 0),+ reasoning=None if reasoning is None else int(reasoning),+ cache_read_reported=cached is not None,+ cache_write_reported=written is not None,)diff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex de48be02..24ae59a3 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -44,6 +44,7 @@ from ..unrouted import UnroutedStorefrom ..unattended import UnattendedRegistryfrom ..audit import AuditStorefrom ..observability import Recorder, RecordingStore+from ..observability.provider_view import RecordingProvider, tag as tag_providerfrom ..config import load_config, workspace_allowed_commandsfrom ..conversations import ConversationStore, title_fromfrom ..engine import ApprovalOutcome, Approver, TurnEngine@@ -430,6 +431,10 @@ class SessionManager:self.provider = ProviderRouter(self.secrets, default_provider="openai", on_use=self._note_provider_use)+ # Observability (E2): every model call goes through one recording wrapper; each+ # caller tags its own view (engine, summary, reviewer, title, proxy).+ if not isinstance(self.provider, RecordingProvider):+ self.provider = RecordingProvider(self.provider, recorder=self.recorder)self.mcp = MCPManager(secrets=self.secrets)# OAuth MCP servers with a sign-in in flight / their last connect error —# feeds list_mcp's status so the GUI can show "authorizing…" and failures.@@ -1299,7 +1304,7 @@ class SessionManager:elif engine.reviewer is None or getattr(engine, "reviewer_choice", "session") != choice:from ..decision_reviewer import make_auto_reviewerengine.reviewer = make_auto_reviewer(- choice, provider=engine.provider, model=engine.model,+ choice, provider=tag_provider(engine.provider, purpose="auto_approver"), model=engine.model,known_world=(engine.session_facts.world.render() if engine.session_facts else "") + "\nRUNTIME FACTS (availability, not access grants)\n" + json.dumps(getattr(engine, "runtime_facts", {})),)engine.reviewer_choice = choice@@ -8192,8 +8197,11 @@ class SessionManager:self._autotitle_inflight.add(session_id)# Retain the task: the loop holds only a weak ref, and a GC'd task would both# kill the title mid-flight and strand the inflight guard.+ # The title call is recorded on the turn running now, or the one that just ended+ # (observability §3.3): captured here, not when the call starts.+ turn_id = getattr(getattr(engine, "_obs", None), "turn_id", None)task = loop.create_task(- self._generate_autotitle(session_id, engine, openers, assistant)+ self._generate_autotitle(session_id, engine, openers, assistant, turn_id=turn_id))self._autotitle_tasks.add(task)task.add_done_callback(self._autotitle_tasks.discard)@@ -8204,6 +8212,7 @@ class SessionManager:engine: TurnEngine,openers: list[str],assistant: str = "",+ turn_id: Optional[str] = None,) -> None:"""One cheap non-streaming completion on the session's own provider/model. Everyfailure (provider error, empty, absurdly long) is swallowed — the title_from@@ -8211,7 +8220,7 @@ class SessionManager:retry can run."""try:turn = await asyncio.to_thread(- engine.provider.complete,+ tag_provider(engine.provider, purpose="title", turn=turn_id).complete,model=engine.model,messages=[{"role": "system", "content": self._AUTOTITLE_PROMPT},@@ -8690,7 +8699,9 @@ class SessionManager:# -- provider proxy ---------------------------------------------------------def provider_complete(self, model, messages, tools=None):- return self.provider.complete(model=model, messages=messages, tools=tools)+ return tag_provider(self.provider, purpose="proxy", turn=None).complete(+ model=model, messages=messages, tools=tools+ )def _refresh_provider(self, name: Optional[str] = None) -> None:"""Drop the router's cached client(s) so the next turn rebuilds with fresh config.diff --git a/tests/test_observability_model_steps.py b/tests/test_observability_model_steps.pynew file mode 100644index 00000000..906bad2b--- /dev/null+++ b/tests/test_observability_model_steps.py@@ -0,0 +1,256 @@+"""Observability E2, E22, E3: every model call is a model step, with its exact exchange and+correct token counts.++- E3: the shared usage vectors (`golden-phase1/vectors/usage.json`, recording format 1.0+ section 5) go through each provider's own usage code and come out as the step's `usage`.+- E2: a real engine turn records one model step per call, purpose `main`, pointing at the+ reply message it produced; untagged and proxy calls are told apart.+- E22: a call to a local fake server is captured below the SDK: the request rebuilds+ exactly from its pieces, no credential is stored, and `provider_facts` are read.+"""++from __future__ import annotations++import asyncio+import json+import threading+from http.server import BaseHTTPRequestHandler, HTTPServer+from pathlib import Path+from types import SimpleNamespace++import aisuite as ai+import pytest++from coworker.engine import TurnEngine+from coworker.observability import Recorder+from coworker.observability import capture+from coworker.observability.engine_hooks import TurnRecording+from coworker.observability.provider_view import RecordingProvider, usage_attrs+from coworker.permissions import PermissionEngine+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall+from coworker.providers import anthropic_provider, bedrock_provider, gemini_provider+from coworker.providers import openai_provider, openai_responses+from coworker.providers.base import TokenUsage+from coworker.providers.ollama_context import to_openai_completion+from coworker.tools import ToolRegistry++VECTORS = json.loads(+ (Path(__file__).parent / "corpora/observability/golden-phase1/vectors/usage.json").read_text("utf-8")+)+++def _ns(v):+ if isinstance(v, dict):+ return SimpleNamespace(**{k: _ns(x) for k, x in v.items()})+ return v+++def _provider_usage(family, raw):+ if family == "anthropic":+ return anthropic_provider._usage_from(_ns(raw))+ if family == "openai_chat":+ return openai_provider._usage_from(_ns(raw))+ if family == "openai_responses":+ return openai_responses._usage_from(_ns(raw))+ if family == "gemini":+ return gemini_provider._usage_from(_ns(raw))+ if family == "bedrock_converse":+ return bedrock_provider._usage_from(raw)+ if family == "ollama":+ # The app turns Ollama's native reply into an OpenAI-shaped one (ollama_context.py).+ return openai_provider._usage_from(_ns(to_openai_completion({"message": {}, **raw})["usage"]))+ raise AssertionError(family)+++# The decision reviewers call their own service directly; their usage comes with E23.+PROVIDER_VECTORS = [v for v in VECTORS if v["family"] not in ("openai_decisions", "typesafe_jev")]++[email protected]("vector", PROVIDER_VECTORS, ids=[v["id"] for v in PROVIDER_VECTORS])+def test_usage_vectors(vector):+ got = usage_attrs(_provider_usage(vector["family"], vector["raw"]))+ assert got == {"reported": True, **vector["expected"]}+++def test_a_call_with_no_usage_is_not_reported():+ assert usage_attrs(None) == {"reported": False}+ assert usage_attrs(TokenUsage(input=5, output=1, cache_write_reported=False))["cache_write"] is None+++def test_anthropic_stream_totals_come_from_the_final_event():+ start = _ns({"input_tokens": 5, "output_tokens": 1, "cache_read_input_tokens": 5194,+ "cache_creation_input_tokens": 130})+ final = _ns({"output_tokens": 23, "output_tokens_details": {"thinking_tokens": 0}})+ u = anthropic_provider._merge_final_usage(anthropic_provider._usage_from(start), final)+ assert usage_attrs(u) == {"reported": True, "input_fresh": 5, "cache_read": 5194,+ "cache_write": 130, "output": 23, "reasoning": 0}+++# -- E2: model steps from a real engine turn -------------------------------------------------++class ScriptedProvider(ProviderClient):+ def __init__(self, turns):+ self._turns = list(turns)++ def complete(self, *, model, messages, tools=None, **settings):+ turn = self._turns.pop(0)+ if isinstance(turn, Exception):+ raise turn+ return turn++ def capabilities(self, model):+ return ModelCapabilities()+++def _engine(tmp_path, turns):+ registry = ToolRegistry()+ registry.register_all(ai.toolkits.files(root=str(tmp_path), allow_write=True))+ engine = TurnEngine(+ provider=ScriptedProvider(turns),+ registry=registry,+ permissions=PermissionEngine(workspace_root=tmp_path),+ model="openai:model-a",+ )+ engine.audit_context = {"session_id": "sess_1"}+ rec = Recorder()+ engine.recorder = rec+ return engine, rec+++def _run(engine, text):+ async def go():+ return [ev async for ev in engine.run(text, trigger="user")]++ return asyncio.run(go())+++def test_every_model_call_is_a_main_model_step(tmp_path):+ (tmp_path / "a.txt").write_text("hello", encoding="utf-8")+ engine, rec = _engine(tmp_path, [+ AssistantTurn(tool_calls=[ToolCall(id="c1", name="read_file", arguments={"path": "a.txt"})],+ finish_reason="tool_calls",+ usage=TokenUsage(input=100, output=10, cache_read=50, cache_write_reported=False)),+ AssistantTurn(text="it says hello", finish_reason="stop", usage=None),+ ])+ _run(engine, "read it")+ d = rec.as_dict()+ models = [s for s in d["steps"] if s["kind"] == "model"]+ assert [s["attrs"]["purpose"] for s in models] == ["main", "main"]+ assistants = [m for m in engine.messages if m["role"] == "assistant"]+ assert [s["message_id"] for s in models] == [m["mid"] for m in assistants]+ first, second = models+ assert first["attrs"]["finish_reason"] == "tool_calls"+ assert first["attrs"]["usage"] == {"reported": True, "input_fresh": 100, "cache_read": 50,+ "cache_write": None, "output": 10, "reasoning": None}+ assert second["attrs"]["usage"] == {"reported": False}+ assert first["attrs"]["model"] == "openai:model-a" and first["attrs"]["provider"] == "openai"+ assert first["attrs"]["tools_offered"] > 0 and first["attrs"]["tools_hash"]+ assert all(s["turn_id"] == d["turns"][0]["turn_id"] for s in models)+ # canonical order: model, tool, model+ assert [s["kind"] for s in d["steps"]] == ["model", "tool", "model"]+++def test_a_failed_model_call_is_an_error_step(tmp_path):+ engine, rec = _engine(tmp_path, [RuntimeError("upstream 529 overloaded")])+ _run(engine, "hi")+ [step] = [s for s in rec.as_dict()["steps"] if s["kind"] == "model"]+ assert step["status"] == "error"+ assert step["attrs"]["error"]["type"] == "overloaded"+++def test_untagged_and_proxy_calls_are_told_apart():+ rec = Recorder()+ base = RecordingProvider(ScriptedProvider([AssistantTurn(text="a"), AssistantTurn(text="b")]), recorder=rec)+ base.complete(model="openai:model-a", messages=[{"role": "user", "content": "x"}])+ base.tagged(purpose="proxy", turn=None).complete(model="openai:model-a", messages=[])+ purposes = sorted(s["attrs"]["purpose"] for s in rec.as_dict()["steps"])+ assert purposes == ["proxy", "unattributed"]+ assert all(s["turn_id"] in ("", None) for s in rec.as_dict()["steps"])+++def test_a_title_tagged_with_an_ended_turn_is_after_turn(tmp_path):+ engine, rec = _engine(tmp_path, [AssistantTurn(text="done"), AssistantTurn(text="Title")])+ _run(engine, "hello")+ turn_id = engine._obs.turn_id+ engine.provider.tagged(purpose="title", turn=turn_id).complete(model=engine.model, messages=[])+ title = next(s for s in rec.as_dict()["steps"] if s["attrs"].get("purpose") == "title")+ assert title["turn_id"] == turn_id and title["after_turn"] is True+++# -- E22: the exact exchange, captured below the SDK ------------------------------------------++_REPLY = {+ "id": "chatcmpl-1", "object": "chat.completion", "created": 1, "model": "model-a-2026",+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi there"},+ "finish_reason": "stop"}],+ "usage": {"prompt_tokens": 30, "completion_tokens": 3, "total_tokens": 33,+ "prompt_tokens_details": {"cached_tokens": 20}},+}+++class _Handler(BaseHTTPRequestHandler):+ seen: list = []++ def do_POST(self): # noqa: N802+ body = self.rfile.read(int(self.headers.get("content-length") or 0))+ _Handler.seen.append((dict(self.headers), body))+ data = json.dumps(_REPLY).encode()+ self.send_response(200)+ self.send_header("content-type", "application/json")+ self.send_header("x-request-id", "req_abc")+ self.send_header("content-length", str(len(data)))+ self.end_headers()+ self.wfile.write(data)++ def log_message(self, *a):+ pass+++def fake_server():+ server = HTTPServer(("127.0.0.1", 0), _Handler)+ t = threading.Thread(target=server.serve_forever, daemon=True)+ t.start()+ yield f"http://127.0.0.1:{server.server_address[1]}/v1"+ server.shutdown()+++def test_the_exact_exchange_is_kept_and_no_key_is_stored(fake_server):+ rec = Recorder()+ sink = TurnRecording(rec, lambda: "sess_x")+ inner = openai_provider.OpenAIProvider(api_key="sk-test-not-real", base_url=fake_server)+ view = RecordingProvider(inner).tagged(purpose="main", sink=sink)+ messages = [{"role": "system", "content": "be brief"}, {"role": "user", "content": "say hi"}]+ turn = view.complete(model="model-a", messages=messages)+ assert turn.text == "hi there"++ [step] = rec.as_dict()["steps"]+ [exchange] = step["content"]["exchange"]+ sent_headers, sent_body = _Handler.seen[-1]+ # Rebuilds exactly, and the stored hash is the hash of the bytes sent.+ assert capture.rebuild_request(exchange, rec.get_piece) == json.loads(sent_body)+ import hashlib+ assert exchange["request"]["body_sha256"] == hashlib.sha256(sent_body).hexdigest()+ assert json.loads(rec.get_piece(exchange["response"]["body_blob"])) == _REPLY+ # No credential anywhere in what is stored.+ stored = json.dumps(step)+ assert "sk-test-not-real" not in stored+ assert "authorization" not in {k.lower() for k in exchange["request"]["headers"]}+ # Metadata without content; facts with fixed names.+ meta = step["attrs"]["provider_meta"][0]+ assert meta["wire"] == "openai_chat" and "messages" not in meta["request"]+ assert "hi there" not in json.dumps(step["attrs"])+ facts = step["attrs"]["provider_facts"]+ assert facts["served_model"] == "model-a-2026"+ assert facts["provider_request_id"] == "req_abc"+ assert facts["provider_total"] == 33+ assert step["attrs"]["usage"] == {"reported": True, "input_fresh": 10, "cache_read": 20,+ "cache_write": None, "output": 3, "reasoning": None}+++def test_capture_is_off_outside_a_recorded_call(fake_server):+ inner = openai_provider.OpenAIProvider(api_key="sk-test-not-real", base_url=fake_server)+ capture.install()+ assert getattr(capture._active, "log", None) is None+ inner.complete(model="model-a", messages=[{"role": "user", "content": "x"}])+ assert getattr(capture._active, "log", None) is Nonediff --git a/tests/test_observability_store.py b/tests/test_observability_store.pyindex 6c22d960..54eefebd 100644--- a/tests/test_observability_store.py+++ b/tests/test_observability_store.py@@ -92,10 +92,11 @@ def test_a_model_call_without_usage_is_marked_unreported(store):rec = Recorder(store=store)_turn(rec, "t1")_model_step(rec, "s1", "t1", usage=None)+ _model_step(rec, "s2", "t1", usage={"reported": False}, at=70, mono=70, end=80)_end_turn(rec, "t1")store.flush()- row = store._conn.execute("SELECT usage_reported, output FROM steps").fetchone()- assert (row["usage_reported"], row["output"]) == (0, None)+ rows = store._conn.execute("SELECT usage_reported, output FROM steps ORDER BY seq").fetchall()+ assert [(r["usage_reported"], r["output"]) for r in rows] == [(0, None), (0, None)]def test_a_step_written_many_times_is_one_row_with_its_last_state(store):