Skip to content
Python

feat(observability): every model call is a model step with its exact exchange and correct token counts (OPE-244)

Devika Verma1e010d19966f
diff --git a/coworker/agent.py b/coworker/agent.py
index e9278f00..299e128c 100644
--- a/coworker/agent.py
+++ b/coworker/agent.py
@@ -36,6 +36,7 @@ from .permissions import Mode, PermissionEngine
from .project import load_agents_md
from . import session_facts
from .roots import RootDir, normalize_roots, render_context
+from .observability.provider_view import tag as tag_provider
from .providers import ProviderClient, ProviderRouter
from .overrides import RiskOverrideStore
from .secrets import SecretStore, state_dir
@@ -876,7 +877,7 @@ def build_engine(
)
engine.reviewer = make_auto_reviewer(
choice,
- provider=provider,
+ provider=tag_provider(engine.provider, purpose="auto_approver"),
model=model,
known_world=engine.session_facts.world.render() + "\nRUNTIME FACTS (availability, not access grants)\n" + json.dumps(engine.runtime_facts),
)
diff --git a/coworker/engine.py b/coworker/engine.py
index d5d6ec4a..c5d0d028 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -34,6 +34,7 @@ from . import toolresult
from . import unattended as _attendance
from .events import Event, EventType
from .observability.engine_hooks import TurnRecording
+from .observability.provider_view import tag as _tag_provider, view_for_engine
from .observability.ids import new_ulid
# §8.4 retry guard: the reviewer pauses for the rest of the turn after this many denials
@@ -484,6 +485,9 @@ class TurnEngine:
# Observability (E1): the turn and step events this engine sends to its recorder.
# Off until a recorder is set (`engine.recorder = ...`), like the other surface hooks.
self._obs = TurnRecording(None, lambda: str(self.audit_context.get("session_id") or ""))
+ # The engine's own tagged view of its provider (observability E2, §3.3): every call
+ # it makes is recorded as a model step of the current turn, purpose `main`.
+ self.provider = view_for_engine(self.provider, self._obs)
self._call_index: dict[str, int] = {}
if instructions and not (
self.messages and self.messages[0].get("role") == "system"
@@ -1248,7 +1252,7 @@ class TurnEngine:
def _build() -> Optional[_compaction.CompactionState]:
return _compaction.build_state(
self.messages,
- provider=self.provider,
+ provider=_tag_provider(self.provider, purpose="summary"),
model=model,
keep_tokens=keep,
prior=self.compaction_state,
diff --git a/coworker/observability/capture.py b/coworker/observability/capture.py
new file mode 100644
index 00000000..0b689106
--- /dev/null
+++ b/coworker/observability/capture.py
@@ -0,0 +1,567 @@
+"""Everything sent and received on a model call (observability E22, technical design §5.2.1).
+
+For every model call the step keeps three things:
+- the exchange: the exact request (method, URL, headers, body) and the exact response
+ (status, headers, the full body or every stream event), content included, minus secrets.
+ Stored as pieces (TD-17): each message, tool definition or response body once, by its
+ SHA-256, and the request as a manifest that points at them;
+- `provider_meta`: the same responses with content removed, every other field as sent;
+- `provider_facts`: fixed names read from `provider_meta`, the same for every provider.
+
+How it is captured: both SDKs the app uses, and the Ollama transport, send their requests
+through httpx's `HTTPTransport` (or `httpx2`'s, the fork newer SDKs ship on). Its
+`handle_request` is wrapped once. While a model call is being recorded on the current
+thread, each request and response it makes is copied into that call's list; on any other
+thread, or when nothing is recorded, the wrapper only checks a thread-local and passes the
+request through. Calls are recorded on the thread that makes them (the stream thread for a
+streamed reply), so two sessions calling at once never mix their requests.
+
+Ported from the Phase 1 lab (`phase-1/lab/raw_capture.py`), where the exact rebuild and the
+"nothing missing from the metadata view" checks passed on every live call.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import json
+import threading
+import zlib
+from typing import Any, Callable, Iterator, Optional
+
+import httpx
+
+# -- capture ------------------------------------------------------------------------------
+
+_active = threading.local()
+_install_lock = threading.Lock()
+_installed = False
+
+
+def _http_modules() -> list[Any]:
+ """httpx, and httpx2 where installed (the fork newer anthropic and openai SDKs use)."""
+ mods = [httpx]
+ try:
+ import httpx2 # type: ignore[import-not-found]
+
+ mods.append(httpx2)
+ except ImportError:
+ pass
+ return mods
+
+
+def _tee_class(mod: Any) -> type:
+ class Tee(mod.SyncByteStream): # type: ignore[misc, name-defined]
+ def __init__(self, inner: Any, sink: bytearray) -> None:
+ self.inner, self.sink = inner, sink
+
+ def __iter__(self) -> Iterator[bytes]:
+ for chunk in self.inner:
+ self.sink.extend(chunk)
+ yield chunk
+
+ def close(self) -> None:
+ close = getattr(self.inner, "close", None)
+ if close:
+ close()
+
+ return Tee
+
+
+def install() -> None:
+ """Wrap `HTTPTransport.handle_request` once per process. Safe to call many times."""
+ global _installed
+ with _install_lock:
+ if _installed:
+ return
+ for mod in _http_modules():
+ original = mod.HTTPTransport.handle_request
+ tee = _tee_class(mod)
+
+ def handle(self: Any, request: Any, _original: Any = original, _tee: type = tee) -> Any:
+ log = getattr(_active, "log", None)
+ response = _original(self, request)
+ if log is None:
+ return response
+ try:
+ body = bytes(request.content or b"")
+ except Exception: # noqa: BLE001 - a streamed request body cannot be re-read
+ body = b""
+ entry: dict[str, Any] = {
+ "url": str(request.url),
+ "method": request.method,
+ "status": response.status_code,
+ "request_headers": {
+ k: v for k, v in request.headers.items() if not is_secret_header(k)
+ },
+ "request_body": body,
+ "response_headers": dict(response.headers.items()),
+ "_body": bytearray(),
+ }
+ response.stream = _tee(response.stream, entry["_body"])
+ log.append(entry)
+ return response
+
+ mod.HTTPTransport.handle_request = handle
+ _installed = True
+
+
+class HttpCapture:
+ """The HTTP exchanges of one model call. Capture is switched on only inside `active()`,
+ so a stream left half-read (Stop pressed) never keeps capturing on its thread: the
+ recording view reads each stream piece inside `active()`."""
+
+ def __init__(self) -> None:
+ install()
+ self._log: list[dict[str, Any]] = []
+
+ @contextlib.contextmanager
+ def active(self) -> Iterator[None]:
+ outer = getattr(_active, "log", None)
+ _active.log = self._log
+ try:
+ yield
+ finally:
+ _active.log = outer
+
+ def entries(self) -> list[dict[str, Any]]:
+ """Every exchange so far, with each response body decoded to text."""
+ for e in self._log:
+ if "_body" in e:
+ e["body"] = _decode(bytes(e.pop("_body")), e["response_headers"]).decode(
+ "utf-8", "replace"
+ )
+ return self._log
+
+
+def _decode(data: bytes, headers: dict[str, str]) -> bytes:
+ """The transport sees the body as sent on the wire, before content decoding."""
+ enc = next((v for k, v in headers.items() if k.lower() == "content-encoding"), "").lower()
+ try:
+ if enc == "gzip":
+ return zlib.decompress(data, 16 + zlib.MAX_WBITS)
+ if enc == "deflate":
+ return zlib.decompress(data)
+ if enc == "br":
+ import brotli # type: ignore[import-not-found]
+
+ return brotli.decompress(data)
+ if enc == "zstd":
+ import zstandard # type: ignore[import-not-found]
+
+ return zstandard.ZstdDecompressor().decompressobj().decompress(data)
+ except Exception: # noqa: BLE001 - keep the raw bytes
+ return data
+ return data
+
+
+# -- secrets --------------------------------------------------------------------------------
+
+SECRET_HEADERS = {
+ "set-cookie", "cookie", "authorization", "x-api-key", "x-goog-api-key", "api-key",
+ "proxy-authorization",
+}
+
+
+def is_secret_header(name: str) -> bool:
+ n = name.lower()
+ return n in SECRET_HEADERS or "token" in n or n.endswith("-key") or "secret" in n
+
+
+def _public_headers(headers: dict[str, str]) -> dict[str, str]:
+ return {k: v for k, v in headers.items() if not is_secret_header(k)}
+
+
+# -- which wire -----------------------------------------------------------------------------
+
+def wire_of_url(url: str) -> str:
+ u = url.split("?", 1)[0].rstrip("/")
+ if u.endswith("/v1/messages"):
+ return "anthropic"
+ if u.endswith("/responses"):
+ return "openai_responses"
+ if u.endswith("/v1/decisions"):
+ return "openai_decisions"
+ if u.endswith("/v1/systemone"):
+ return "typesafe_jev"
+ if u.endswith("/api/chat") or u.endswith("/api/show"):
+ return "ollama"
+ if ":generateContent" in u or ":streamGenerateContent" in u:
+ return "gemini"
+ return "openai_chat"
+
+
+def is_auxiliary(url: str) -> bool:
+ """A helper request made before the call, not an attempt of it (M-R9): Ollama's /api/show."""
+ return url.split("?", 1)[0].rstrip("/").endswith("/api/show")
+
+
+# -- provider_meta: everything sent, minus content and secrets ------------------------------
+#
+# A short DENY list, not a list of fields to keep: whatever a provider sends that is not on
+# it is saved, including fields it adds later. Paths match an event's flattened field paths;
+# "*" applies to every event kind of that wire.
+
+CONTENT_PATHS: dict[str, dict[str, list[str]]] = {
+ "anthropic": {
+ "message_start": ["message.content"],
+ "content_block_start": ["content_block.text", "content_block.thinking",
+ "content_block.signature", "content_block.data",
+ "content_block.input"],
+ "content_block_delta": ["delta.text", "delta.partial_json", "delta.thinking",
+ "delta.signature"],
+ "message": ["content"],
+ },
+ "openai_responses": {
+ "*": ["delta", "text", "arguments", "part.text", "item.content", "item.arguments",
+ "item.summary", "item.encrypted_content", "response.output",
+ "response.instructions", "response.tools", "response.text.format.schema",
+ "output", "instructions", "tools"],
+ },
+ "openai_decisions": {"*": ["answers"]},
+ "typesafe_jev": {"*": ["answers"]},
+ "ollama": {"*": ["message.content", "message.thinking", "message.tool_calls",
+ "message.images"]},
+ "openai_chat": {
+ "*": ["choices[].delta.content", "choices[].delta.reasoning",
+ "choices[].delta.reasoning_content", "choices[].delta.reasoning_details",
+ "choices[].delta.tool_calls", "choices[].message", "choices[].logprobs"],
+ },
+ "gemini": {"*": ["candidates[].content"]},
+}
+
+# Request fields that are content (the conversation, tool definitions, system text, a
+# decision reviewer's action description and question wording).
+_REQUEST_CONTENT = ("messages", "input", "tools", "system", "instructions", "state",
+ "questions", "contents", "systemInstruction")
+
+
+def request_params(body: bytes) -> dict[str, Any]:
+ """Every request parameter except content; `tools_offered` as a count."""
+ try:
+ parsed = json.loads(body or b"{}")
+ except Exception: # noqa: BLE001 - not JSON
+ return {}
+ if not isinstance(parsed, dict):
+ return {}
+ out = {k: v for k, v in parsed.items() if k not in _REQUEST_CONTENT}
+ out["tools_offered"] = len(parsed.get("tools") or [])
+ if isinstance(parsed.get("questions"), list):
+ out["questions_asked"] = len(parsed["questions"])
+ out["question_names"] = [
+ q.get("name") for q in parsed["questions"] if isinstance(q, dict) and q.get("name")
+ ] or None
+ return out
+
+
+def parse_events(body: str) -> list[Any]:
+ """Every JSON object in a body: plain JSON, server-sent events, or one object per line."""
+ body = body.strip()
+ if body.startswith("{") or body.startswith("["):
+ try:
+ parsed = json.loads(body)
+ return parsed if isinstance(parsed, list) else [parsed]
+ except ValueError:
+ pass
+ lines = [ln.strip() for ln in body.splitlines() if ln.strip()]
+ if all(ln.startswith("{") for ln in lines):
+ events: list[Any] = []
+ for ln in lines:
+ try:
+ events.append(json.loads(ln))
+ except ValueError:
+ events.append({"_unparsed": ln[:200]})
+ return events
+ events = []
+ for line in body.splitlines():
+ line = line.strip()
+ if line.startswith("data:"):
+ data = line[5:].strip()
+ if data and data != "[DONE]":
+ try:
+ events.append(json.loads(data))
+ except ValueError:
+ events.append({"_unparsed": data[:200]})
+ return events
+
+
+def event_kind_of(wire: str, ev: dict[str, Any]) -> str:
+ if wire == "openai_chat":
+ return "chunk with usage" if ev.get("usage") else "chunk"
+ if wire == "ollama":
+ return "done" if ev.get("done") else "chunk"
+ if wire == "gemini":
+ return "chunk with usage" if ev.get("usageMetadata") else "chunk"
+ return str(ev.get("type", "response"))
+
+
+def is_content(wire: str, kind: str, path: str) -> bool:
+ rules = CONTENT_PATHS.get(wire, {})
+ for pat in rules.get("*", []) + rules.get(kind, []):
+ if path == pat or path.startswith(pat + ".") or path.startswith(pat + "["):
+ return True
+ return False
+
+
+def _strip(obj: Any, wire: str, kind: str, prefix: str = "") -> Any:
+ if isinstance(obj, dict):
+ out = {}
+ for k, v in obj.items():
+ p = f"{prefix}.{k}" if prefix else k
+ if is_content(wire, kind, p):
+ continue
+ out[k] = _strip(v, wire, kind, p)
+ return out
+ if isinstance(obj, list):
+ return [_strip(v, wire, kind, f"{prefix}[]") for v in obj]
+ return obj
+
+
+def _merge(into: dict[str, Any], new: dict[str, Any]) -> None:
+ """Deep merge, later values win (stream counters end at their final value)."""
+ for k, v in new.items():
+ old = into.get(k)
+ if isinstance(v, dict) and isinstance(old, dict):
+ _merge(old, v)
+ elif isinstance(v, list) and isinstance(old, list) and all(isinstance(x, dict) for x in old + v):
+ # Stream pieces repeat lists such as `choices`: merge item by item, so a field
+ # sent only in the first piece is not lost to a later one.
+ for i, item in enumerate(v):
+ if i < len(old):
+ _merge(old[i], item)
+ else:
+ old.append(item)
+ elif v is not None or k not in into:
+ into[k] = v
+
+
+def provider_meta(entry: dict[str, Any]) -> dict[str, Any]:
+ """Everything one HTTP response told us, as sent, minus content and secrets."""
+ wire = wire_of_url(entry["url"])
+ aux = is_auxiliary(entry["url"])
+ events: dict[str, Any] = {}
+ counts: dict[str, int] = {}
+ for ev in parse_events(entry.get("body", "")):
+ if not isinstance(ev, dict):
+ continue
+ kind = "response" if aux else event_kind_of(wire, ev)
+ counts[kind] = counts.get(kind, 0) + 1
+ _merge(events.setdefault(kind, {}), _strip(ev, wire, kind))
+ out: dict[str, Any] = {
+ "wire": wire,
+ "status": entry["status"],
+ "request": request_params(entry.get("request_body") or b""),
+ "headers": _public_headers(entry.get("response_headers", {})),
+ "event_counts": counts,
+ "events": events,
+ }
+ if aux:
+ out["auxiliary"] = True
+ return out
+
+
+# -- provider_facts: the same names for every provider (§5.2.1, recording-format §6.4) ------
+
+FACT_NAMES = (
+ "served_model", "provider_request_id", "provider_response_id", "served_by", "reasoning",
+ "cache_read", "cache_write", "cache_write_by_ttl", "provider_total", "requested",
+ "applied", "stop", "provider_timing", "rate_limit", "hosted_tools", "cost_reported",
+)
+
+
+def _get(d: Any, path: str) -> Any:
+ for part in path.split("."):
+ if isinstance(d, list):
+ d = d[0] if d else None
+ if not isinstance(d, dict):
+ return None
+ d = d.get(part)
+ return d
+
+
+def provider_facts(meta: Optional[dict[str, Any]]) -> dict[str, Any]:
+ """Fixed-name facts read from one `provider_meta` entry. None = the provider did not send it."""
+ if not meta:
+ return {}
+ w, ev, h, req = meta.get("wire"), meta.get("events", {}), meta.get("headers", {}), meta.get("request", {})
+ hl = {k.lower(): v for k, v in h.items()}
+ if w == "anthropic":
+ msg = _get(ev, "message_start.message") or ev.get("message") or {}
+ fin = ev.get("message_delta", {})
+ u = {**(msg.get("usage") or {}), **(fin.get("usage") or {})}
+ stop_src = fin.get("delta") or msg
+ facts = {
+ "served_model": msg.get("model"), "provider_response_id": msg.get("id"),
+ "provider_request_id": hl.get("request-id"),
+ "reasoning": _get(u, "output_tokens_details.thinking_tokens"),
+ "cache_read": u.get("cache_read_input_tokens"),
+ "cache_write": u.get("cache_creation_input_tokens"),
+ "cache_write_by_ttl": u.get("cache_creation"), "provider_total": None,
+ "requested": {"thinking": req.get("thinking"), "effort": _get(req, "output_config.effort"),
+ "max_tokens": req.get("max_tokens"), "temperature": req.get("temperature"),
+ "service_tier": req.get("service_tier")},
+ "applied": {"service_tier": u.get("service_tier"), "speed": u.get("speed"),
+ "inference_geo": u.get("inference_geo")},
+ "stop": {"reason": stop_src.get("stop_reason"), "sequence": stop_src.get("stop_sequence"),
+ "details": stop_src.get("stop_details")},
+ "provider_timing": {"server_timing": hl.get("server-timing")},
+ "hosted_tools": u.get("server_tool_use"), "cost_reported": None,
+ }
+ elif w == "openai_responses":
+ r = (ev.get("response.completed") or ev.get("response.incomplete")
+ or ev.get("response.failed") or {}).get("response") or ev.get("response") or {}
+ u = r.get("usage") or {}
+ facts = {
+ "served_model": r.get("model"), "provider_response_id": r.get("id"),
+ "provider_request_id": hl.get("x-request-id"),
+ "reasoning": _get(u, "output_tokens_details.reasoning_tokens"),
+ "cache_read": _get(u, "input_tokens_details.cached_tokens"),
+ "cache_write": _get(u, "input_tokens_details.cache_write_tokens"),
+ "cache_write_by_ttl": None, "provider_total": u.get("total_tokens"),
+ "requested": {"effort": _get(req, "reasoning.effort"), "max_tokens": req.get("max_output_tokens"),
+ "temperature": req.get("temperature"), "service_tier": req.get("service_tier")},
+ "applied": {"effort": _get(r, "reasoning.effort"), "service_tier": r.get("service_tier"),
+ "prompt_cache_retention": r.get("prompt_cache_retention"),
+ "truncation": r.get("truncation")},
+ "stop": {"reason": r.get("status"), "details": r.get("incomplete_details")},
+ "provider_timing": {"processing_ms": hl.get("openai-processing-ms"),
+ "created_at": r.get("created_at"), "completed_at": r.get("completed_at")},
+ "hosted_tools": r.get("tool_usage"), "cost_reported": None,
+ }
+ elif w in ("openai_decisions", "typesafe_jev"):
+ r = ev.get("response", {})
+ u = r.get("usage") or {}
+ facts = {
+ "served_model": r.get("model"), "provider_response_id": r.get("id"),
+ "provider_request_id": hl.get("x-request-id") or hl.get("x-typesafe-request-id"),
+ "cache_read": _get(u, "input_tokens_details.cached_tokens"),
+ "cache_write": _get(u, "input_tokens_details.cache_write_tokens"),
+ "reasoning": _get(u, "output_tokens_details.reasoning_tokens"),
+ "provider_total": u.get("total_tokens"),
+ "requested": {"model": req.get("model"), "questions_asked": req.get("questions_asked")},
+ "applied": {}, "stop": {}, "cost_reported": None, "hosted_tools": None,
+ "provider_timing": {"processing_ms": hl.get("openai-processing-ms"),
+ "server_timing_total": hl.get("server-timing-total")},
+ }
+ elif w == "ollama":
+ # Native /api/chat: the last line (`done: true`) carries the counts and timings, in
+ # nanoseconds. `prompt_eval_cached_count` is the part of the prompt reused (V-20).
+ d = ev.get("done", {}) or ev.get("response", {})
+ opts = req.get("options") or {}
+ facts = {
+ "served_model": d.get("model"), "provider_response_id": None, "provider_request_id": None,
+ "cache_read": d.get("prompt_eval_cached_count"),
+ "reasoning": None, "cache_write": None, "cache_write_by_ttl": None, "provider_total": None,
+ "requested": {"num_ctx": opts.get("num_ctx"), "max_tokens": opts.get("num_predict"),
+ "temperature": opts.get("temperature"), "think": req.get("think"),
+ "shift": req.get("shift"), "keep_alive": req.get("keep_alive")},
+ "applied": {}, "stop": {"reason": d.get("done_reason")},
+ "provider_timing": {k: d.get(k) for k in ("total_duration", "load_duration",
+ "prompt_eval_duration", "eval_duration",
+ "created_at")},
+ "hosted_tools": None, "cost_reported": None,
+ }
+ elif w == "gemini":
+ c = {**ev.get("chunk", {}), **ev.get("chunk with usage", {})}
+ u = c.get("usageMetadata") or {}
+ facts = {
+ "served_model": c.get("modelVersion"), "provider_response_id": c.get("responseId"),
+ "provider_request_id": None, "reasoning": u.get("thoughtsTokenCount"),
+ "cache_read": u.get("cachedContentTokenCount"), "cache_write": None,
+ "cache_write_by_ttl": None, "provider_total": u.get("totalTokenCount"),
+ "requested": {"generation_config": req.get("generationConfig")}, "applied": {},
+ "stop": {"reason": _get(c, "candidates.finishReason")}, "provider_timing": {},
+ "hosted_tools": None, "cost_reported": None,
+ }
+ else:
+ c = {**ev.get("chunk", {}), **ev.get("chunk with usage", {}), **ev.get("response", {})}
+ u = c.get("usage") or {}
+ facts = {
+ "served_model": c.get("model"), "provider_response_id": c.get("id"),
+ "provider_request_id": hl.get("x-request-id") or hl.get("x-generation-id"),
+ "served_by": c.get("provider"),
+ "reasoning": _get(u, "completion_tokens_details.reasoning_tokens"),
+ "cache_read": _get(u, "prompt_tokens_details.cached_tokens"),
+ "cache_write": _get(u, "prompt_tokens_details.cache_write_tokens"),
+ "cache_write_by_ttl": None, "provider_total": u.get("total_tokens"),
+ "requested": {"effort": req.get("reasoning_effort") or _get(req, "reasoning.effort"),
+ "max_tokens": req.get("max_tokens") or req.get("max_completion_tokens"),
+ "temperature": req.get("temperature"), "service_tier": req.get("service_tier")},
+ "applied": {"service_tier": c.get("service_tier"),
+ "system_fingerprint": c.get("system_fingerprint")},
+ "stop": {"reason": _get(c, "choices.finish_reason"),
+ "native_reason": _get(c, "choices.native_finish_reason")},
+ "provider_timing": {"processing_ms": hl.get("openai-processing-ms")},
+ "hosted_tools": None,
+ "cost_reported": ({"amount": u.get("cost"), "details": u.get("cost_details"),
+ "is_byok": u.get("is_byok")} if u.get("cost") is not None else None),
+ }
+ facts["rate_limit"] = {k: v for k, v in h.items() if "ratelimit" in k.lower()} or None
+ return {k: facts.get(k) for k in FACT_NAMES}
+
+
+# -- the exchange, stored as pieces (TD-17) ---------------------------------------------------
+
+# Request fields whose items are stored as separate pieces, shared across a session's calls.
+PIECE_FIELDS = ("messages", "input", "tools", "system", "instructions", "contents",
+ "questions", "state")
+
+
+def _piece(value: Any, put: Callable[[bytes], str]) -> dict[str, str]:
+ data = json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
+ return {"$blob": put(data.encode("utf-8"))}
+
+
+def store_exchange(entry: dict[str, Any], put: Callable[[bytes], str]) -> dict[str, Any]:
+ """One exchange entry (recording-format §6): the request as a manifest of pieces, the
+ response body as one piece. `put(bytes) -> sha256` stores a piece once."""
+ body: bytes = entry.get("request_body") or b""
+ try:
+ parsed = json.loads(body) if body else None
+ except ValueError:
+ parsed = None
+ req_body: dict[str, Any]
+ if isinstance(parsed, dict):
+ manifest = {}
+ for k, v in parsed.items():
+ if k in PIECE_FIELDS and isinstance(v, list):
+ manifest[k] = [_piece(x, put) for x in v]
+ elif k in PIECE_FIELDS:
+ manifest[k] = _piece(v, put)
+ else:
+ manifest[k] = v
+ req_body = {"manifest": manifest}
+ else:
+ req_body = {"blob": put(body)} if body else {}
+ resp = (entry.get("body") or "").encode("utf-8")
+ out: dict[str, Any] = {
+ "request": {
+ "method": entry["method"], "url": entry["url"],
+ "headers": _public_headers(entry.get("request_headers", {})),
+ "body_sha256": hashlib.sha256(body).hexdigest(), "body_size": len(body), **req_body,
+ },
+ "response": {
+ "status": entry["status"],
+ "headers": _public_headers(entry.get("response_headers", {})),
+ "body_blob": put(resp), "body_size": len(resp),
+ },
+ }
+ if is_auxiliary(entry["url"]):
+ out = {"auxiliary": True, **out}
+ return out
+
+
+def rebuild_request(exchange: dict[str, Any], get: Callable[[str], Optional[bytes]]) -> Any:
+ """The request JSON exactly as sent (or the raw body bytes, for a non-JSON body)."""
+ req = exchange["request"]
+ if "blob" in req:
+ return get(req["blob"])
+
+ def un(v: Any) -> Any:
+ if isinstance(v, dict) and set(v) == {"$blob"}:
+ return json.loads(get(v["$blob"]) or b"null")
+ if isinstance(v, list):
+ return [un(x) for x in v]
+ return v
+
+ return {k: un(v) for k, v in req.get("manifest", {}).items()}
diff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.py
index 26a2a94e..7836fafd 100644
--- a/coworker/observability/engine_hooks.py
+++ b/coworker/observability/engine_hooks.py
@@ -23,6 +23,9 @@ from typing import Any, Callable, Optional
from .ids import new_ulid
from .recorder import EVENT_V, Recorder, now_ms
+# `start_step(turn_id=CURRENT)`: the engine's current turn.
+CURRENT = object()
+
# TOOL_FINISHED status -> tool step status (technical design §5.1).
_TOOL_STATUS = {"ok": "ok", "error": "error", "denied": "denied", "interrupted": "interrupted",
"skipped": "skipped"}
@@ -44,6 +47,7 @@ class TurnRecording:
self._ident: dict[str, dict[str, Any]] = {} # step_id -> identity copy
self._tool_steps: dict[str, str] = {} # call_id -> step_id, for this turn's tools
self._tool_ends: dict[str, dict[str, Any]] = {} # call_id -> step_ended awaiting its mid
+ self._reply_end: Optional[dict[str, Any]] = None # a main model step awaiting its reply's mid
@property
def enabled(self) -> bool:
@@ -84,6 +88,7 @@ class TurnRecording:
self._last_mid = None
self._tool_steps.clear()
self._tool_ends.clear()
+ self._reply_end = None
ev = self._base("turn_started", self.turn_id, self._turn_session)
ev.update({"kind": kind, "trigger": trigger, "first_message_id": first_message_id,
"attrs": copy.deepcopy(attrs), "stamp": copy.deepcopy(stamp)})
@@ -102,6 +107,7 @@ class TurnRecording:
for ev in self._tool_ends.values(): # a tool whose result message never came
self.recorder.emit(ev)
self._tool_ends.clear()
+ self._flush_reply(None)
ev = self._base("turn_ended", self.turn_id, self._turn_session)
ev.update({"status": status, "last_message_id": self._last_mid})
self.recorder.emit(ev)
@@ -132,28 +138,47 @@ class TurnRecording:
if ev is not None:
ev["message_id"] = mid
self._send(ev)
+ elif message.get("role") == "assistant":
+ self._flush_reply(mid)
# -- steps ----------------------------------------------------------------------------------
def start_step(self, *, kind: str, name: str, call_id: Optional[str] = None,
call_index: Optional[int] = None, parent_step_id: Optional[str] = None,
attrs: Optional[dict[str, Any]] = None,
- content: Optional[dict[str, Any]] = None) -> Optional[str]:
+ content: Optional[dict[str, Any]] = None,
+ turn_id: Any = CURRENT, shadow: bool = False) -> Optional[str]:
+ """`step_started`. The step belongs to the engine's current turn, or with `turn_id` to
+ that turn: a step tagged with a turn that already ended is `after_turn` (§3.3, an
+ after-turn title). A model step outside any turn is recorded with no turn; a tool
+ step outside a turn is not recorded."""
if not self.enabled:
return None
with self._lock:
- if not self.in_turn:
- return None
+ after = False
+ if turn_id is CURRENT:
+ if self.in_turn:
+ tid, sid = self.turn_id, self._turn_session
+ elif kind == "tool":
+ return None
+ else:
+ tid, sid = None, self._session_id() or ""
+ else:
+ tid = turn_id or None
+ same = tid is not None and tid == self.turn_id
+ sid = self._turn_session if same else (self._session_id() or "")
+ # Turns of one engine run one after another: another turn of this engine has
+ # ended, and so has this one once its end was sent.
+ after = tid is not None and (not same or self._ended)
step_id = new_ulid()
- ev = self._base("step_started", self.turn_id, self._turn_session)
+ ev = self._base("step_started", tid, sid)
ev.update({"step_id": step_id, "parent_step_id": parent_step_id, "kind": kind,
"name": name, "call_id": call_id, "call_index": call_index,
- "shadow": False, "after_turn": False, "attrs": attrs or {},
+ "shadow": bool(shadow), "after_turn": after, "attrs": attrs or {},
"content": content or {}})
self._ident[step_id] = {
"kind": kind, "name": name, "parent_step_id": parent_step_id, "call_id": call_id,
- "call_index": call_index, "shadow": False, "after_turn": False,
- "started_at": self._started_at(ev), "turn_id": self.turn_id,
- "session_id": self._turn_session,
+ "call_index": call_index, "shadow": bool(shadow), "after_turn": after,
+ "started_at": self._started_at(ev), "turn_id": tid, "session_id": sid,
}
if kind == "tool" and call_id:
self._tool_steps[call_id] = step_id
@@ -168,7 +193,10 @@ class TurnRecording:
def end_step(self, step_id: Optional[str], *, status: str = "ok",
attrs: Optional[dict[str, Any]] = None, content: Optional[dict[str, Any]] = None,
message_id: Optional[str] = None, error: Optional[dict[str, Any]] = None,
- wait_for_tool_message: bool = False) -> None:
+ wait_for_tool_message: bool = False,
+ wait_for_reply_message: bool = False) -> None:
+ """`step_ended`. A tool step's end waits for its result message, a main model step's
+ end for the reply message the engine adds next, so each carries that message's `mid`."""
if not self.enabled or step_id is None:
return
with self._lock:
@@ -182,11 +210,21 @@ class TurnRecording:
ev["error"] = error
ev.update({k: idn[k] for k in ("kind", "name", "parent_step_id", "call_id",
"call_index", "shadow", "after_turn", "started_at")})
- if wait_for_tool_message and idn["call_id"] and not self._ended:
+ waiting_turn = idn["turn_id"] is not None and idn["turn_id"] == self.turn_id and not self._ended
+ if wait_for_tool_message and idn["call_id"] and waiting_turn:
self._tool_ends[idn["call_id"]] = ev
+ elif wait_for_reply_message and waiting_turn:
+ self._flush_reply(None) # an earlier reply that never got a message
+ self._reply_end = ev
else:
self._send(ev)
+ def _flush_reply(self, mid: Optional[str]) -> None:
+ ev, self._reply_end = self._reply_end, None
+ if ev is not None:
+ ev["message_id"] = mid
+ self._send(ev)
+
def instant_step(self, *, kind: str, name: str, status: str, call_id: Optional[str] = None,
call_index: Optional[int] = None, attrs: Optional[dict[str, Any]] = None,
message_id: Optional[str] = None) -> None:
diff --git a/coworker/observability/provider_view.py b/coworker/observability/provider_view.py
new file mode 100644
index 00000000..62772c71
--- /dev/null
+++ b/coworker/observability/provider_view.py
@@ -0,0 +1,366 @@
+"""Tagged provider views: every model call recorded as a model step (observability E2, E22).
+
+Every model call leaves the app through one provider router, shared by every session's
+engine and by the OpenAI-compatible proxy. A wrapper around the router alone cannot know
+which session, turn or purpose a call belongs to, so each caller holds its own **tagged
+view** (technical design §3.3, TD-2):
+
+ view = provider.tagged(purpose="summary", parent=step_id)
+ turn = view.complete(model=..., messages=...) # recorded as a model step
+
+- `RecordingProvider` wraps the router once, where it is made. A call made through it with
+ no tags is recorded as `unattributed`, so a missing tag shows instead of counting as main.
+- `tagged()` on a view returns a new view: fields passed replace the old, the rest are kept.
+- A view records through a `sink`: the engine's turn recording (`TurnRecording`), which knows
+ the turn, or for calls outside any engine (the proxy) a sink on the server's recorder.
+
+Each model step carries the call's purpose, provider, model, tool count, time to the first
+streamed piece, finish reason and token counts (§5.2), and, from the HTTP capture
+(`capture.py`), the exact exchange, `provider_meta` and `provider_facts` (§5.2.1). Recording
+never breaks a call: any recording error is logged and the call goes on.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import threading
+import time
+from dataclasses import dataclass, replace
+from typing import Any, Callable, Iterator, Optional
+
+from ..providers.base import ProviderClient
+from . import capture
+from .engine_hooks import CURRENT, TurnRecording
+from .recorder import short_hash
+
+log = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class Tags:
+ purpose: str = "unattributed"
+ parent: Optional[str] = None # the step this call is nested in
+ call_id: Optional[str] = None # for auto-approver checks: the tool call checked
+ shadow: bool = False
+ turn: Any = CURRENT # CURRENT = the sink's current turn; a turn id; None = no turn
+ sink: Optional[TurnRecording] = None
+
+
+class RecordingProvider(ProviderClient):
+ """The router, wrapped once. Untagged calls are recorded as `unattributed` when a recorder
+ is given; everything else (`invalidate`, `_secrets`…) passes through to the router."""
+
+ def __init__(self, inner: ProviderClient, recorder: Any = None) -> None:
+ self.inner = inner
+ self.recorder = recorder
+ self._sink = TurnRecording(recorder, lambda: "") if recorder is not None else None
+
+ def tagged(self, **tags: Any) -> "ProviderView":
+ if "sink" not in tags:
+ tags["sink"] = self._sink
+ return ProviderView(self, Tags(**tags))
+
+ def complete(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any):
+ return self.tagged().complete(model=model, messages=messages, tools=tools, **settings)
+
+ def stream(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any):
+ return self.tagged().stream(model=model, messages=messages, tools=tools, **settings)
+
+ def capabilities(self, model: str):
+ return self.inner.capabilities(model)
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(self.inner, name)
+
+
+class ProviderView(ProviderClient):
+ """A tagged handle on the shared provider. Tags are explicit; nothing relies on context
+ variables crossing threads."""
+
+ def __init__(self, base: RecordingProvider, tags: Tags) -> None:
+ self.base, self.tags = base, tags
+
+ def tagged(self, **overrides: Any) -> "ProviderView":
+ return ProviderView(self.base, replace(self.tags, **overrides))
+
+ def capabilities(self, model: str):
+ return self.base.inner.capabilities(model)
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(self.base.inner, name)
+
+ # -- calls --------------------------------------------------------------------------------
+ def complete(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any):
+ call = _ModelCall.begin(self.tags, model=model, tools=tools, streamed=False)
+ if call is None:
+ return self.base.inner.complete(model=model, messages=messages, tools=tools, **settings)
+ try:
+ with call.http.active():
+ turn = self.base.inner.complete(model=model, messages=messages, tools=tools, **settings)
+ except BaseException as exc:
+ call.failed(exc)
+ raise
+ call.finished(turn)
+ return turn
+
+ def stream(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any) -> Iterator[Any]:
+ call = _ModelCall.begin(self.tags, model=model, tools=tools, streamed=True)
+ if call is None:
+ yield from self.base.inner.stream(model=model, messages=messages, tools=tools, **settings)
+ return
+ it = iter(())
+ turn = None
+ try:
+ with call.http.active():
+ it = iter(self.base.inner.stream(model=model, messages=messages, tools=tools, **settings))
+ while True:
+ # Capture is on only while the next piece is read, so a stream left half-read
+ # (Stop pressed) never keeps capturing on this thread. Such a step stays open
+ # and is closed by the turn's end as `interrupted` (§7.4).
+ with call.http.active():
+ try:
+ chunk = next(it)
+ except StopIteration:
+ break
+ call.saw(chunk)
+ if getattr(chunk, "turn", None) is not None:
+ turn = chunk.turn
+ yield chunk
+ except GeneratorExit:
+ raise
+ except BaseException as exc:
+ call.failed(exc)
+ raise
+ if turn is not None:
+ call.finished(turn)
+
+
+# -- one recorded call ----------------------------------------------------------------------------
+
+_tools_hash_cache: dict[int, tuple[Any, Optional[str]]] = {}
+_tools_hash_lock = threading.Lock()
+
+
+def tools_hash(tools: Any) -> Optional[str]:
+ """A short code of the tool definitions, to spot tool changes; cached per tool list (P-6)."""
+ if not tools:
+ return None
+ key = id(tools)
+ with _tools_hash_lock:
+ hit = _tools_hash_cache.get(key)
+ if hit is not None and hit[0] is tools:
+ return hit[1]
+ code = short_hash(json.dumps(tools, sort_keys=True, default=str))
+ with _tools_hash_lock:
+ if len(_tools_hash_cache) > 64:
+ _tools_hash_cache.clear()
+ _tools_hash_cache[key] = (tools, code)
+ return code
+
+
+def _provider_of(model: str) -> str:
+ return (model or "").split(":", 1)[0] if ":" in (model or "") else "openai"
+
+
+def _provider_title(provider: str) -> Optional[str]:
+ try:
+ from ..providers.registry import get_descriptor
+
+ d = get_descriptor(provider)
+ return getattr(d, "title", None) if d is not None else None
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _context_window(model: str) -> Optional[int]:
+ try:
+ from ..providers.matrix import model_context_windows
+
+ return model_context_windows().get(model)
+ except Exception: # noqa: BLE001
+ return None
+
+
+def root_cause(exc: BaseException) -> BaseException:
+ seen = set()
+ while id(exc) not in seen and (exc.__cause__ is not None or exc.__context__ is not None):
+ seen.add(id(exc))
+ exc = exc.__cause__ or exc.__context__ # type: ignore[assignment]
+ return exc
+
+
+def classify_error(exc: BaseException) -> str:
+ """`error.type` from the failure's root cause (§5.2, recording-format §5)."""
+ try:
+ from ..compaction import is_context_overflow
+
+ if is_context_overflow(exc):
+ return "context_length"
+ except Exception: # noqa: BLE001
+ pass
+ root = root_cause(exc)
+ text = f"{type(exc).__name__} {exc} {type(root).__name__} {root}".lower()
+ status = getattr(exc, "status_code", None) or getattr(root, "status_code", None)
+ if status in (529, 503) or "overload" in text:
+ return "overloaded"
+ if status == 429 or "rate limit" in text or "ratelimit" in text:
+ return "rate_limited"
+ if status in (401, 403) or "api key" in text or "authentication" in text:
+ return "auth"
+ if "timeout" in text or "timed out" in text:
+ return "timeout"
+ if "cancel" in text:
+ return "cancelled"
+ if any(k in text for k in ("connect", "dns", "ssl", "tls", "proxy", "name resolution")):
+ return "network"
+ return "provider_error"
+
+
+def usage_attrs(usage: Any) -> dict[str, Any]:
+ """The step's `usage` from the provider's `TokenUsage` (§5.2, §6.2). A count the provider
+ did not send is `null`, never 0; a call with no counts is `{"reported": false}`."""
+ if usage is None:
+ return {"reported": False}
+ return {
+ "reported": True,
+ "input_fresh": usage.input,
+ "cache_read": usage.cache_read if getattr(usage, "cache_read_reported", True) else None,
+ "cache_write": usage.cache_write if getattr(usage, "cache_write_reported", True) else None,
+ "output": usage.output,
+ "reasoning": getattr(usage, "reasoning", None),
+ }
+
+
+class _ModelCall:
+ """The state of one recorded model call, from start to end."""
+
+ def __init__(self, tags: Tags, sink: TurnRecording, step_id: str, streamed: bool) -> None:
+ self.tags, self.sink, self.step_id, self.streamed = tags, sink, step_id, streamed
+ self.http = capture.HttpCapture()
+ self.t0 = time.perf_counter()
+ self.ttft_ms: Optional[int] = None
+
+ @classmethod
+ def begin(cls, tags: Tags, *, model: str, tools: Any, streamed: bool) -> Optional["_ModelCall"]:
+ sink = tags.sink
+ if sink is None or not sink.enabled:
+ return None
+ try:
+ provider = _provider_of(model)
+ step_id = sink.start_step(
+ kind="model", name="model", turn_id=tags.turn, shadow=tags.shadow,
+ parent_step_id=tags.parent,
+ call_id=tags.call_id if tags.purpose == "auto_approver" else None,
+ attrs={
+ "purpose": tags.purpose, "provider": provider,
+ "provider_title": _provider_title(provider), "model": model,
+ "stream": streamed, "context_window": _context_window(model),
+ "tools_offered": len(tools or []), "tools_hash": tools_hash(tools),
+ },
+ )
+ except Exception: # noqa: BLE001 - recording never breaks a call
+ log.exception("recording: could not start a model step")
+ return None
+ if step_id is None:
+ return None
+ return cls(tags, sink, step_id, streamed)
+
+ def saw(self, chunk: Any) -> None:
+ # The first streamed piece of the reply: text or thinking (§7.2).
+ if self.ttft_ms is None and (getattr(chunk, "text_delta", None) or getattr(chunk, "reasoning_delta", None)):
+ self.ttft_ms = int((time.perf_counter() - self.t0) * 1000)
+
+ def _exchange(self) -> tuple[dict[str, Any], dict[str, Any]]:
+ """(attrs, content) from the HTTP capture: provider_meta, provider_facts, the exchange."""
+ attrs: dict[str, Any] = {}
+ content: dict[str, Any] = {}
+ entries = [e for e in self.http.entries() if e.get("method") == "POST"]
+ if not entries:
+ return attrs, content
+ recorder = self.sink.recorder
+ content["exchange"] = [capture.store_exchange(e, recorder.put_piece) for e in entries]
+ meta = [capture.provider_meta(e) for e in entries]
+ attrs["provider_meta"] = meta
+ reply = [m for m in meta if not m.get("auxiliary")] or meta
+ facts = capture.provider_facts(reply[-1])
+ attrs["provider_facts"] = facts
+ if facts.get("served_by") is not None:
+ attrs["served_by"] = facts["served_by"]
+ return attrs, content
+
+ def finished(self, turn: Any) -> None:
+ try:
+ attrs: dict[str, Any] = {
+ "finish_reason": getattr(turn, "finish_reason", None),
+ "ttft_ms": self.ttft_ms if self.streamed else None,
+ "usage": usage_attrs(getattr(turn, "usage", None)),
+ }
+ if getattr(turn, "served_by", None):
+ attrs["served_by"] = turn.served_by
+ more, content = self._exchange()
+ attrs.update(more)
+ _fill_from_facts(attrs)
+ self.sink.end_step(
+ self.step_id, status="ok", attrs=attrs, content=content,
+ wait_for_reply_message=self.tags.purpose == "main",
+ )
+ except Exception: # noqa: BLE001
+ log.exception("recording: could not end a model step")
+
+ def failed(self, exc: BaseException) -> None:
+ try:
+ if _is_stop(exc):
+ return # Stop: the turn's end closes the step as interrupted
+ more, content = self._exchange()
+ attrs: dict[str, Any] = {"finish_reason": None, "ttft_ms": self.ttft_ms if self.streamed else None,
+ "usage": {"reported": False}, **more}
+ text = str(root_cause(exc)).strip()[:200] or None
+ self.sink.end_step(self.step_id, status="error", attrs=attrs, content=content,
+ error={"type": classify_error(exc), "text": text})
+ except Exception: # noqa: BLE001
+ log.exception("recording: could not record a failed model step")
+
+
+def _is_stop(exc: BaseException) -> bool:
+ return isinstance(exc, (GeneratorExit, KeyboardInterrupt))
+
+
+def _fill_from_facts(attrs: dict[str, Any]) -> None:
+ """A count the provider sent is recorded even where the provider code dropped it
+ (V-13, V-15), and impossible numbers are flagged, never corrected (V-19)."""
+ use, facts = attrs.get("usage") or {}, attrs.get("provider_facts") or {}
+ if not use.get("reported"):
+ return
+ if use.get("reasoning") is None and facts.get("reasoning") is not None:
+ use["reasoning"] = facts["reasoning"]
+ if use.get("cache_write") is None and facts.get("cache_write") is not None:
+ use["cache_write"] = facts["cache_write"]
+ flags = []
+ if use.get("reasoning") is not None and use.get("output") is not None and use["reasoning"] > use["output"]:
+ flags.append("reasoning_exceeds_output")
+ total = facts.get("provider_total")
+ parts = (use.get("input_fresh") or 0) + (use.get("cache_read") or 0) + (use.get("cache_write") or 0) + (use.get("output") or 0)
+ if total is not None and total != parts:
+ flags.append("provider_total_differs")
+ if flags:
+ use["inconsistent"] = flags
+
+
+def view_for_engine(provider: Any, sink: TurnRecording) -> "ProviderView":
+ """The engine's own view (§3.3): tags its creator already set are kept, so a helper's
+ engine keeps `helper`; a plain provider is wrapped first."""
+ if isinstance(provider, ProviderView):
+ tags = provider.tags
+ return provider.tagged(
+ purpose="main" if tags.purpose == "unattributed" else tags.purpose,
+ sink=sink if tags.sink is None or tags.sink is provider.base._sink else tags.sink,
+ )
+ base = provider if isinstance(provider, RecordingProvider) else RecordingProvider(provider)
+ return base.tagged(purpose="main", sink=sink)
+
+
+def tag(provider: Any, **tags: Any) -> Any:
+ """`provider.tagged(**tags)` for a recording provider or view; any other provider as is."""
+ tagged = getattr(type(provider), "tagged", None)
+ return provider.tagged(**tags) if tagged is not None else provider
diff --git a/coworker/observability/recorder.py b/coworker/observability/recorder.py
index 11012e46..271b11a5 100644
--- a/coworker/observability/recorder.py
+++ b/coworker/observability/recorder.py
@@ -20,6 +20,7 @@ Rust engine too.
from __future__ import annotations
import copy
+import hashlib
import logging
import threading
import time
@@ -129,6 +130,11 @@ def _merge_top(into: dict[str, Any], new: Optional[dict[str, Any]]) -> None:
into[k] = copy.deepcopy(v)
+def short_hash(text: str) -> str:
+ """First 6 hex characters of the SHA-256: the short codes of §5.2 and §9.4."""
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()[:6]
+
+
def now_ms() -> tuple[int, int]:
"""(at_ms, mono_ms) for an event sent now.
@@ -160,6 +166,23 @@ class Recorder:
self._lock = threading.RLock()
self._turn_count: dict[str, int] = {} # session_id -> turns seen, for `seq`
self._listeners: list[Callable[[dict[str, Any]], None]] = []
+ self._pieces: dict[str, bytes] = {} # exchange pieces when there is no store
+
+ # -- exchange pieces (TD-17) ------------------------------------------------------------
+ def put_piece(self, data: bytes) -> str:
+ """Store one exchange piece once, by its SHA-256: in the store, or in memory."""
+ if self.store is not None:
+ return self.store.put_piece(data)
+ key = hashlib.sha256(data).hexdigest()
+ with self._lock:
+ self._pieces.setdefault(key, data)
+ return key
+
+ def get_piece(self, key: str) -> Optional[bytes]:
+ if self.store is not None:
+ return self.store.get_piece(key)
+ with self._lock:
+ return self._pieces.get(key)
# -- listeners ---------------------------------------------------------------------------
def add_listener(self, fn: Callable[[dict[str, Any]], None]) -> Callable[[], None]:
diff --git a/coworker/observability/store.py b/coworker/observability/store.py
index fd017a2a..b279d12f 100644
--- a/coworker/observability/store.py
+++ b/coworker/observability/store.py
@@ -134,7 +134,8 @@ def _step_row(s: dict[str, Any]) -> tuple:
meta = attrs.pop("provider_meta", None)
usage = attrs.get("usage") if isinstance(attrs.get("usage"), dict) else None
if s.get("kind") == "model":
- reported: Optional[int] = int(usage is not None)
+ # A call the provider gave no counts for is `usage: {"reported": false}` (§5.2).
+ reported: Optional[int] = int(bool(usage) and usage.get("reported", True) is not False)
else:
reported = _bool(attrs.get("usage_reported"))
facts = attrs.get("provider_facts") or {}
diff --git a/coworker/providers/anthropic_provider.py b/coworker/providers/anthropic_provider.py
index 25c1d88e..629150b9 100644
--- a/coworker/providers/anthropic_provider.py
+++ b/coworker/providers/anthropic_provider.py
@@ -42,13 +42,41 @@ def _usage_from(usage: Any) -> Optional[TokenUsage]:
"""Messages-API usage object → normalized counts (input_tokens excludes cache)."""
if usage is None:
return None
+ read = getattr(usage, "cache_read_input_tokens", None)
+ write = getattr(usage, "cache_creation_input_tokens", None)
+ thinking = getattr(getattr(usage, "output_tokens_details", None), "thinking_tokens", None)
return TokenUsage(
input=int(getattr(usage, "input_tokens", 0) or 0),
output=int(getattr(usage, "output_tokens", 0) or 0),
- cache_read=int(getattr(usage, "cache_read_input_tokens", 0) or 0),
- cache_write=int(getattr(usage, "cache_creation_input_tokens", 0) or 0),
+ cache_read=int(read or 0),
+ cache_write=int(write or 0),
+ reasoning=None if thinking is None else int(thinking),
+ cache_read_reported=read is not None,
+ cache_write_reported=write is not None,
)
+
+def _merge_final_usage(usage: Optional[TokenUsage], final: Any) -> Optional[TokenUsage]:
+ """The stream's last `message_delta` usage holds the call's totals (observability E3):
+ every count it carries replaces the opening event's."""
+ if final is None:
+ return usage
+ usage = usage or TokenUsage()
+ for attr, field in (("input_tokens", "input"), ("output_tokens", "output")):
+ value = getattr(final, attr, None)
+ if value:
+ setattr(usage, field, int(value))
+ read = getattr(final, "cache_read_input_tokens", None)
+ if read is not None:
+ usage.cache_read, usage.cache_read_reported = int(read), True
+ write = getattr(final, "cache_creation_input_tokens", None)
+ if write is not None:
+ usage.cache_write, usage.cache_write_reported = int(write), True
+ thinking = getattr(getattr(final, "output_tokens_details", None), "thinking_tokens", None)
+ if thinking is not None:
+ usage.reasoning = int(thinking)
+ return usage
+
# Required by the Messages API; a ceiling, not a spend target. Sized for file
# generation, not just chat: a coworker writing a self-contained HTML report ships the
# whole file inside one tool call's arguments, and 16k proved too small in the field
@@ -788,13 +816,8 @@ class AnthropicProvider(ProviderClient):
reason = getattr(last_message_delta, "stop_reason", None)
if reason:
stop_reason = reason
- # Final (cumulative) output-token count rides message_delta.usage.
- out = int(
- getattr(getattr(event, "usage", None), "output_tokens", 0) or 0
- )
- if out:
- usage = usage or TokenUsage()
- usage.output = out
+ # The final (cumulative) counts ride message_delta.usage.
+ usage = _merge_final_usage(usage, getattr(event, "usage", None))
_raise_on_refusal(stop_reason, last_message_delta)
tool_calls = []
diff --git a/coworker/providers/base.py b/coworker/providers/base.py
index c197793b..3b11a4c3 100644
--- a/coworker/providers/base.py
+++ b/coworker/providers/base.py
@@ -30,12 +30,21 @@ class TokenUsage:
split into `cache_read`/`cache_write`. Providers that don't report a cache
split (Ollama, most compat vendors) leave the cache fields at 0. `output`
includes thinking tokens where the vendor bills them as output (Gemini).
+
+ For observability (E3) a count the provider did not send must read as unknown, not
+ 0: `cache_read_reported` / `cache_write_reported` say whether it was sent, and
+ `reasoning` (thinking tokens within `output`) is None when not sent. The int fields
+ stay ints, so existing sums are unchanged.
"""
input: int = 0
output: int = 0
cache_read: int = 0
cache_write: int = 0
+ # Not part of equality: two usages with the same counts are equal, as before.
+ reasoning: Optional[int] = field(default=None, compare=False)
+ cache_read_reported: bool = field(default=True, compare=False)
+ cache_write_reported: bool = field(default=True, compare=False)
@property
def context_tokens(self) -> int:
diff --git a/coworker/providers/bedrock_provider.py b/coworker/providers/bedrock_provider.py
index 6be7c4e4..3505d4e8 100644
--- a/coworker/providers/bedrock_provider.py
+++ b/coworker/providers/bedrock_provider.py
@@ -58,6 +58,8 @@ def _usage_from(usage: Any) -> Optional[TokenUsage]:
output=int(usage.get("outputTokens") or 0),
cache_read=int(usage.get("cacheReadInputTokens") or 0),
cache_write=int(usage.get("cacheWriteInputTokens") or 0),
+ cache_read_reported="cacheReadInputTokens" in usage,
+ cache_write_reported="cacheWriteInputTokens" in usage,
)
# Converse has no required max token param but per-model defaults vary wildly (Meta's is
diff --git a/coworker/providers/gemini_provider.py b/coworker/providers/gemini_provider.py
index 9e58bbe6..9453faf4 100644
--- a/coworker/providers/gemini_provider.py
+++ b/coworker/providers/gemini_provider.py
@@ -42,12 +42,15 @@ def _usage_from(meta: Any) -> Optional[TokenUsage]:
if meta is None:
return None
prompt = int(getattr(meta, "prompt_token_count", 0) or 0)
- cached = int(getattr(meta, "cached_content_token_count", 0) or 0)
+ cached = getattr(meta, "cached_content_token_count", None)
+ thoughts = getattr(meta, "thoughts_token_count", None)
return TokenUsage(
- input=max(prompt - cached, 0),
- output=int(getattr(meta, "candidates_token_count", 0) or 0)
- + int(getattr(meta, "thoughts_token_count", 0) or 0),
- cache_read=cached,
+ input=max(prompt - int(cached or 0), 0),
+ output=int(getattr(meta, "candidates_token_count", 0) or 0) + int(thoughts or 0),
+ cache_read=int(cached or 0),
+ reasoning=None if thoughts is None else int(thoughts),
+ cache_read_reported=cached is not None,
+ cache_write_reported=False,
)
# Gemini finishReason → the engine's OpenAI-shaped finish_reason vocabulary. STOP maps to
diff --git a/coworker/providers/ollama_context.py b/coworker/providers/ollama_context.py
index a2090e04..3147bc8a 100644
--- a/coworker/providers/ollama_context.py
+++ b/coworker/providers/ollama_context.py
@@ -437,6 +437,7 @@ def to_openai_completion(native: dict[str, Any]) -> dict[str, Any]:
"prompt_tokens": prompt,
"completion_tokens": completion,
"total_tokens": prompt + completion,
+ **_cached_details(native),
},
}
return body
@@ -477,6 +478,7 @@ def to_openai_stream_events(
"prompt_tokens": prompt,
"completion_tokens": completion,
"total_tokens": prompt + completion,
+ **_cached_details(native),
},
}
)
@@ -523,3 +525,13 @@ def _finish_reason(reason: str, has_tools: bool) -> str:
def _usage_counts(native: dict[str, Any]) -> tuple[int, int]:
return int(native.get("prompt_eval_count") or 0), int(native.get("eval_count") or 0)
+
+
+def _cached_details(native: dict[str, Any]) -> dict[str, Any]:
+ """Ollama's `prompt_eval_cached_count`, the part of the prompt reused from its cache, as
+ the OpenAI-shaped `cached_tokens`, so fresh input is the rest (V-20, observability E3).
+ An older Ollama that does not send it leaves the cached share unknown."""
+ cached = native.get("prompt_eval_cached_count")
+ if cached is None:
+ return {}
+ return {"prompt_tokens_details": {"cached_tokens": int(cached)}}
diff --git a/coworker/providers/openai_provider.py b/coworker/providers/openai_provider.py
index b9db639f..b7be89a8 100644
--- a/coworker/providers/openai_provider.py
+++ b/coworker/providers/openai_provider.py
@@ -203,11 +203,19 @@ def _usage_from(usage: Any) -> Optional[TokenUsage]:
return None
prompt = int(getattr(usage, "prompt_tokens", 0) or 0)
details = getattr(usage, "prompt_tokens_details", None)
- cached = int(getattr(details, "cached_tokens", 0) or 0)
+ cached = getattr(details, "cached_tokens", None)
+ # The cache write is inside the prompt count too (V-21), when the server sends it
+ # (OpenRouter does, V-13). A count not sent is unknown, not 0 (observability E3).
+ written = getattr(details, "cache_write_tokens", None)
+ reasoning = getattr(getattr(usage, "completion_tokens_details", None), "reasoning_tokens", None)
return TokenUsage(
- input=max(prompt - cached, 0),
+ input=max(prompt - int(cached or 0) - int(written or 0), 0),
output=int(getattr(usage, "completion_tokens", 0) or 0),
- cache_read=cached,
+ cache_read=int(cached or 0),
+ cache_write=int(written or 0),
+ reasoning=None if reasoning is None else int(reasoning),
+ cache_read_reported=cached is not None,
+ cache_write_reported=written is not None,
)
diff --git a/coworker/providers/openai_responses.py b/coworker/providers/openai_responses.py
index be9522fd..f2382e81 100644
--- a/coworker/providers/openai_responses.py
+++ b/coworker/providers/openai_responses.py
@@ -245,11 +245,19 @@ def _usage_from(usage: Any) -> Optional[TokenUsage]:
return None
prompt = int(getattr(usage, "input_tokens", 0) or 0)
details = getattr(usage, "input_tokens_details", None)
- cached = int(getattr(details, "cached_tokens", 0) or 0)
+ cached = getattr(details, "cached_tokens", None)
+ # The cache write is inside the prompt count too (V-21), when the server sends it
+ # (OpenRouter does, V-13). A count not sent is unknown, not 0 (observability E3).
+ written = getattr(details, "cache_write_tokens", None)
+ reasoning = getattr(getattr(usage, "output_tokens_details", None), "reasoning_tokens", None)
return TokenUsage(
- input=max(prompt - cached, 0),
+ input=max(prompt - int(cached or 0) - int(written or 0), 0),
output=int(getattr(usage, "output_tokens", 0) or 0),
- cache_read=cached,
+ cache_read=int(cached or 0),
+ cache_write=int(written or 0),
+ reasoning=None if reasoning is None else int(reasoning),
+ cache_read_reported=cached is not None,
+ cache_write_reported=written is not None,
)
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index de48be02..24ae59a3 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -44,6 +44,7 @@ from ..unrouted import UnroutedStore
from ..unattended import UnattendedRegistry
from ..audit import AuditStore
from ..observability import Recorder, RecordingStore
+from ..observability.provider_view import RecordingProvider, tag as tag_provider
from ..config import load_config, workspace_allowed_commands
from ..conversations import ConversationStore, title_from
from ..engine import ApprovalOutcome, Approver, TurnEngine
@@ -430,6 +431,10 @@ class SessionManager:
self.provider = ProviderRouter(
self.secrets, default_provider="openai", on_use=self._note_provider_use
)
+ # Observability (E2): every model call goes through one recording wrapper; each
+ # caller tags its own view (engine, summary, reviewer, title, proxy).
+ if not isinstance(self.provider, RecordingProvider):
+ self.provider = RecordingProvider(self.provider, recorder=self.recorder)
self.mcp = MCPManager(secrets=self.secrets)
# OAuth MCP servers with a sign-in in flight / their last connect error —
# feeds list_mcp's status so the GUI can show "authorizing…" and failures.
@@ -1299,7 +1304,7 @@ class SessionManager:
elif engine.reviewer is None or getattr(engine, "reviewer_choice", "session") != choice:
from ..decision_reviewer import make_auto_reviewer
engine.reviewer = make_auto_reviewer(
- choice, provider=engine.provider, model=engine.model,
+ choice, provider=tag_provider(engine.provider, purpose="auto_approver"), model=engine.model,
known_world=(engine.session_facts.world.render() if engine.session_facts else "") + "\nRUNTIME FACTS (availability, not access grants)\n" + json.dumps(getattr(engine, "runtime_facts", {})),
)
engine.reviewer_choice = choice
@@ -8192,8 +8197,11 @@ class SessionManager:
self._autotitle_inflight.add(session_id)
# Retain the task: the loop holds only a weak ref, and a GC'd task would both
# kill the title mid-flight and strand the inflight guard.
+ # The title call is recorded on the turn running now, or the one that just ended
+ # (observability §3.3): captured here, not when the call starts.
+ turn_id = getattr(getattr(engine, "_obs", None), "turn_id", None)
task = loop.create_task(
- self._generate_autotitle(session_id, engine, openers, assistant)
+ self._generate_autotitle(session_id, engine, openers, assistant, turn_id=turn_id)
)
self._autotitle_tasks.add(task)
task.add_done_callback(self._autotitle_tasks.discard)
@@ -8204,6 +8212,7 @@ class SessionManager:
engine: TurnEngine,
openers: list[str],
assistant: str = "",
+ turn_id: Optional[str] = None,
) -> None:
"""One cheap non-streaming completion on the session's own provider/model. Every
failure (provider error, empty, absurdly long) is swallowed — the title_from
@@ -8211,7 +8220,7 @@ class SessionManager:
retry can run."""
try:
turn = await asyncio.to_thread(
- engine.provider.complete,
+ tag_provider(engine.provider, purpose="title", turn=turn_id).complete,
model=engine.model,
messages=[
{"role": "system", "content": self._AUTOTITLE_PROMPT},
@@ -8690,7 +8699,9 @@ class SessionManager:
# -- provider proxy ---------------------------------------------------------
def provider_complete(self, model, messages, tools=None):
- return self.provider.complete(model=model, messages=messages, tools=tools)
+ return tag_provider(self.provider, purpose="proxy", turn=None).complete(
+ model=model, messages=messages, tools=tools
+ )
def _refresh_provider(self, name: Optional[str] = None) -> None:
"""Drop the router's cached client(s) so the next turn rebuilds with fresh config.
diff --git a/tests/test_observability_model_steps.py b/tests/test_observability_model_steps.py
new file mode 100644
index 00000000..906bad2b
--- /dev/null
+++ b/tests/test_observability_model_steps.py
@@ -0,0 +1,256 @@
+"""Observability E2, E22, E3: every model call is a model step, with its exact exchange and
+correct token counts.
+
+- E3: the shared usage vectors (`golden-phase1/vectors/usage.json`, recording format 1.0
+ section 5) go through each provider's own usage code and come out as the step's `usage`.
+- E2: a real engine turn records one model step per call, purpose `main`, pointing at the
+ reply message it produced; untagged and proxy calls are told apart.
+- E22: a call to a local fake server is captured below the SDK: the request rebuilds
+ exactly from its pieces, no credential is stored, and `provider_facts` are read.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
+from types import SimpleNamespace
+
+import aisuite as ai
+import pytest
+
+from coworker.engine import TurnEngine
+from coworker.observability import Recorder
+from coworker.observability import capture
+from coworker.observability.engine_hooks import TurnRecording
+from coworker.observability.provider_view import RecordingProvider, usage_attrs
+from coworker.permissions import PermissionEngine
+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall
+from coworker.providers import anthropic_provider, bedrock_provider, gemini_provider
+from coworker.providers import openai_provider, openai_responses
+from coworker.providers.base import TokenUsage
+from coworker.providers.ollama_context import to_openai_completion
+from coworker.tools import ToolRegistry
+
+VECTORS = json.loads(
+ (Path(__file__).parent / "corpora/observability/golden-phase1/vectors/usage.json").read_text("utf-8")
+)
+
+
+def _ns(v):
+ if isinstance(v, dict):
+ return SimpleNamespace(**{k: _ns(x) for k, x in v.items()})
+ return v
+
+
+def _provider_usage(family, raw):
+ if family == "anthropic":
+ return anthropic_provider._usage_from(_ns(raw))
+ if family == "openai_chat":
+ return openai_provider._usage_from(_ns(raw))
+ if family == "openai_responses":
+ return openai_responses._usage_from(_ns(raw))
+ if family == "gemini":
+ return gemini_provider._usage_from(_ns(raw))
+ if family == "bedrock_converse":
+ return bedrock_provider._usage_from(raw)
+ if family == "ollama":
+ # The app turns Ollama's native reply into an OpenAI-shaped one (ollama_context.py).
+ return openai_provider._usage_from(_ns(to_openai_completion({"message": {}, **raw})["usage"]))
+ raise AssertionError(family)
+
+
+# The decision reviewers call their own service directly; their usage comes with E23.
+PROVIDER_VECTORS = [v for v in VECTORS if v["family"] not in ("openai_decisions", "typesafe_jev")]
+
+
[email protected]("vector", PROVIDER_VECTORS, ids=[v["id"] for v in PROVIDER_VECTORS])
+def test_usage_vectors(vector):
+ got = usage_attrs(_provider_usage(vector["family"], vector["raw"]))
+ assert got == {"reported": True, **vector["expected"]}
+
+
+def test_a_call_with_no_usage_is_not_reported():
+ assert usage_attrs(None) == {"reported": False}
+ assert usage_attrs(TokenUsage(input=5, output=1, cache_write_reported=False))["cache_write"] is None
+
+
+def test_anthropic_stream_totals_come_from_the_final_event():
+ start = _ns({"input_tokens": 5, "output_tokens": 1, "cache_read_input_tokens": 5194,
+ "cache_creation_input_tokens": 130})
+ final = _ns({"output_tokens": 23, "output_tokens_details": {"thinking_tokens": 0}})
+ u = anthropic_provider._merge_final_usage(anthropic_provider._usage_from(start), final)
+ assert usage_attrs(u) == {"reported": True, "input_fresh": 5, "cache_read": 5194,
+ "cache_write": 130, "output": 23, "reasoning": 0}
+
+
+# -- E2: model steps from a real engine turn -------------------------------------------------
+
+class ScriptedProvider(ProviderClient):
+ def __init__(self, turns):
+ self._turns = list(turns)
+
+ def complete(self, *, model, messages, tools=None, **settings):
+ turn = self._turns.pop(0)
+ if isinstance(turn, Exception):
+ raise turn
+ return turn
+
+ def capabilities(self, model):
+ return ModelCapabilities()
+
+
+def _engine(tmp_path, turns):
+ registry = ToolRegistry()
+ registry.register_all(ai.toolkits.files(root=str(tmp_path), allow_write=True))
+ engine = TurnEngine(
+ provider=ScriptedProvider(turns),
+ registry=registry,
+ permissions=PermissionEngine(workspace_root=tmp_path),
+ model="openai:model-a",
+ )
+ engine.audit_context = {"session_id": "sess_1"}
+ rec = Recorder()
+ engine.recorder = rec
+ return engine, rec
+
+
+def _run(engine, text):
+ async def go():
+ return [ev async for ev in engine.run(text, trigger="user")]
+
+ return asyncio.run(go())
+
+
+def test_every_model_call_is_a_main_model_step(tmp_path):
+ (tmp_path / "a.txt").write_text("hello", encoding="utf-8")
+ engine, rec = _engine(tmp_path, [
+ AssistantTurn(tool_calls=[ToolCall(id="c1", name="read_file", arguments={"path": "a.txt"})],
+ finish_reason="tool_calls",
+ usage=TokenUsage(input=100, output=10, cache_read=50, cache_write_reported=False)),
+ AssistantTurn(text="it says hello", finish_reason="stop", usage=None),
+ ])
+ _run(engine, "read it")
+ d = rec.as_dict()
+ models = [s for s in d["steps"] if s["kind"] == "model"]
+ assert [s["attrs"]["purpose"] for s in models] == ["main", "main"]
+ assistants = [m for m in engine.messages if m["role"] == "assistant"]
+ assert [s["message_id"] for s in models] == [m["mid"] for m in assistants]
+ first, second = models
+ assert first["attrs"]["finish_reason"] == "tool_calls"
+ assert first["attrs"]["usage"] == {"reported": True, "input_fresh": 100, "cache_read": 50,
+ "cache_write": None, "output": 10, "reasoning": None}
+ assert second["attrs"]["usage"] == {"reported": False}
+ assert first["attrs"]["model"] == "openai:model-a" and first["attrs"]["provider"] == "openai"
+ assert first["attrs"]["tools_offered"] > 0 and first["attrs"]["tools_hash"]
+ assert all(s["turn_id"] == d["turns"][0]["turn_id"] for s in models)
+ # canonical order: model, tool, model
+ assert [s["kind"] for s in d["steps"]] == ["model", "tool", "model"]
+
+
+def test_a_failed_model_call_is_an_error_step(tmp_path):
+ engine, rec = _engine(tmp_path, [RuntimeError("upstream 529 overloaded")])
+ _run(engine, "hi")
+ [step] = [s for s in rec.as_dict()["steps"] if s["kind"] == "model"]
+ assert step["status"] == "error"
+ assert step["attrs"]["error"]["type"] == "overloaded"
+
+
+def test_untagged_and_proxy_calls_are_told_apart():
+ rec = Recorder()
+ base = RecordingProvider(ScriptedProvider([AssistantTurn(text="a"), AssistantTurn(text="b")]), recorder=rec)
+ base.complete(model="openai:model-a", messages=[{"role": "user", "content": "x"}])
+ base.tagged(purpose="proxy", turn=None).complete(model="openai:model-a", messages=[])
+ purposes = sorted(s["attrs"]["purpose"] for s in rec.as_dict()["steps"])
+ assert purposes == ["proxy", "unattributed"]
+ assert all(s["turn_id"] in ("", None) for s in rec.as_dict()["steps"])
+
+
+def test_a_title_tagged_with_an_ended_turn_is_after_turn(tmp_path):
+ engine, rec = _engine(tmp_path, [AssistantTurn(text="done"), AssistantTurn(text="Title")])
+ _run(engine, "hello")
+ turn_id = engine._obs.turn_id
+ engine.provider.tagged(purpose="title", turn=turn_id).complete(model=engine.model, messages=[])
+ title = next(s for s in rec.as_dict()["steps"] if s["attrs"].get("purpose") == "title")
+ assert title["turn_id"] == turn_id and title["after_turn"] is True
+
+
+# -- E22: the exact exchange, captured below the SDK ------------------------------------------
+
+_REPLY = {
+ "id": "chatcmpl-1", "object": "chat.completion", "created": 1, "model": "model-a-2026",
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi there"},
+ "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 30, "completion_tokens": 3, "total_tokens": 33,
+ "prompt_tokens_details": {"cached_tokens": 20}},
+}
+
+
+class _Handler(BaseHTTPRequestHandler):
+ seen: list = []
+
+ def do_POST(self): # noqa: N802
+ body = self.rfile.read(int(self.headers.get("content-length") or 0))
+ _Handler.seen.append((dict(self.headers), body))
+ data = json.dumps(_REPLY).encode()
+ self.send_response(200)
+ self.send_header("content-type", "application/json")
+ self.send_header("x-request-id", "req_abc")
+ self.send_header("content-length", str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+
+ def log_message(self, *a):
+ pass
+
+
+def fake_server():
+ server = HTTPServer(("127.0.0.1", 0), _Handler)
+ t = threading.Thread(target=server.serve_forever, daemon=True)
+ t.start()
+ yield f"http://127.0.0.1:{server.server_address[1]}/v1"
+ server.shutdown()
+
+
+def test_the_exact_exchange_is_kept_and_no_key_is_stored(fake_server):
+ rec = Recorder()
+ sink = TurnRecording(rec, lambda: "sess_x")
+ inner = openai_provider.OpenAIProvider(api_key="sk-test-not-real", base_url=fake_server)
+ view = RecordingProvider(inner).tagged(purpose="main", sink=sink)
+ messages = [{"role": "system", "content": "be brief"}, {"role": "user", "content": "say hi"}]
+ turn = view.complete(model="model-a", messages=messages)
+ assert turn.text == "hi there"
+
+ [step] = rec.as_dict()["steps"]
+ [exchange] = step["content"]["exchange"]
+ sent_headers, sent_body = _Handler.seen[-1]
+ # Rebuilds exactly, and the stored hash is the hash of the bytes sent.
+ assert capture.rebuild_request(exchange, rec.get_piece) == json.loads(sent_body)
+ import hashlib
+ assert exchange["request"]["body_sha256"] == hashlib.sha256(sent_body).hexdigest()
+ assert json.loads(rec.get_piece(exchange["response"]["body_blob"])) == _REPLY
+ # No credential anywhere in what is stored.
+ stored = json.dumps(step)
+ assert "sk-test-not-real" not in stored
+ assert "authorization" not in {k.lower() for k in exchange["request"]["headers"]}
+ # Metadata without content; facts with fixed names.
+ meta = step["attrs"]["provider_meta"][0]
+ assert meta["wire"] == "openai_chat" and "messages" not in meta["request"]
+ assert "hi there" not in json.dumps(step["attrs"])
+ facts = step["attrs"]["provider_facts"]
+ assert facts["served_model"] == "model-a-2026"
+ assert facts["provider_request_id"] == "req_abc"
+ assert facts["provider_total"] == 33
+ assert step["attrs"]["usage"] == {"reported": True, "input_fresh": 10, "cache_read": 20,
+ "cache_write": None, "output": 3, "reasoning": None}
+
+
+def test_capture_is_off_outside_a_recorded_call(fake_server):
+ inner = openai_provider.OpenAIProvider(api_key="sk-test-not-real", base_url=fake_server)
+ capture.install()
+ assert getattr(capture._active, "log", None) is None
+ inner.complete(model="model-a", messages=[{"role": "user", "content": "x"}])
+ assert getattr(capture._active, "log", None) is None
diff --git a/tests/test_observability_store.py b/tests/test_observability_store.py
index 6c22d960..54eefebd 100644
--- a/tests/test_observability_store.py
+++ b/tests/test_observability_store.py
@@ -92,10 +92,11 @@ def test_a_model_call_without_usage_is_marked_unreported(store):
rec = Recorder(store=store)
_turn(rec, "t1")
_model_step(rec, "s1", "t1", usage=None)
+ _model_step(rec, "s2", "t1", usage={"reported": False}, at=70, mono=70, end=80)
_end_turn(rec, "t1")
store.flush()
- row = store._conn.execute("SELECT usage_reported, output FROM steps").fetchone()
- assert (row["usage_reported"], row["output"]) == (0, None)
+ rows = store._conn.execute("SELECT usage_reported, output FROM steps ORDER BY seq").fetchall()
+ assert [(r["usage_reported"], r["output"]) for r in rows] == [(0, None), (0, None)]
def test_a_step_written_many_times_is_one_row_with_its_last_state(store):