Python
feat(observability): OB-6 export of recordings to Langfuse or an OTLP endpoint
Devika Verma0937e5dd630c
diff --git a/coworker/conversations.py b/coworker/conversations.pyindex a52db466..9144d0c1 100644--- a/coworker/conversations.py+++ b/coworker/conversations.py@@ -635,6 +635,15 @@ class ConversationStore:self._conn.commit()return cur.rowcount > 0+ def display_title(self, session_id: str) -> Optional[str]:+ """The title the sidebar shows, or None when the session has no row yet."""+ with self._lock:+ row = self._conn.execute(+ "SELECT title, auto_title, renamed FROM sessions WHERE session_id = ?",+ (session_id,),+ ).fetchone()+ return None if row is None else _display_title(row)+def title_state(self, session_id: str) -> Optional[dict]:"""The auto-title guard inputs: whether the user renamed and whether a generatedtitle already exists. None when the session has no row yet."""diff --git a/coworker/observability/export.py b/coworker/observability/export.pynew file mode 100644index 00000000..bc5ee3ec--- /dev/null+++ b/coworker/observability/export.py@@ -0,0 +1,682 @@+"""Sending recordings to Langfuse or an OpenTelemetry endpoint (OB-6, phase-2 design §10).++Runs on the machine that ran the work, beside the recording store. Off by default: with no+destination the exporter is not created, so there is no thread, no DNS lookup and no socket.++- `ExportSettings`: the page's settings, in `observability.json` in the machine's state folder+ (never in a project's files, never in `config.toml`). Keys and header values are not here:+ they live in the app's `SecretStore`, profile `observability-export`.+- `managed_export`: the org policy's `observability_export` field. When a policy sets it, the+ policy's destination and switches are used and the page is read-only (TD2-11).+- `Exporter`: one thread that reads finished turns from the store, builds their spans+ (`export_spans.py`), sends them in batches and marks each step sent for the destination.+ The queue is the store: a restart resumes from the marks, and nothing is sent twice.+- `ObservabilityExport`: what the server holds. It answers the settings, status and test+ endpoints and creates or stops the exporter when the settings or the policy change.+"""++from __future__ import annotations++import hashlib+import json+import logging+import threading+import time+from dataclasses import dataclass, field, replace+from pathlib import Path+from typing import Any, Callable, Optional+from urllib.parse import urlparse++from . import export_spans as spans_mod+from .export_spans import Masker, build_turn_spans, encode_otlp, resource_attrs, switches_from++log = logging.getLogger(__name__)++SETTINGS_FILE = "observability.json"+PROFILE = "observability-export"+LANGFUSE_PATH = "/api/public/otel/v1/traces"+KINDS = ("off", "langfuse", "otlp")+RETENTION_CHOICES = (7, 30, 90)+MAX_HEADERS = 8+INCLUDE_KEYS = ("messages", "tool_io", "context", "names")++BATCH_SPANS = 200+FLUSH_SECONDS = 5.0+BACKOFF_START = 1.0+BACKOFF_CAP = 300.0+TIMEOUT_SECONDS = 20.0++REFUSED_KEY = "The destination refused the key"+NOT_FOUND = "Address not found"+++# -- what is chosen ----------------------------------------------------------------------------+@dataclass(frozen=True)+class Destination:+ kind: str = "off"+ host: str = "" # Langfuse+ url: str = "" # OpenTelemetry endpoint+ header_names: tuple[str, ...] = () # OpenTelemetry endpoint; values are secrets++ @property+ def endpoint(self) -> str:+ if self.kind == "langfuse":+ return self.host.rstrip("/") + LANGFUSE_PATH+ return self.url if self.kind == "otlp" else ""++ @property+ def dialect(self) -> str:+ return "langfuse" if self.kind == "langfuse" else "semconv"++ @property+ def key(self) -> str:+ """Identifies the destination for the sent marks: kind and address."""+ return f"{self.kind}:{self.endpoint}" if self.kind != "off" else ""++ @property+ def place(self) -> str:+ """The address's host, for the page and the session notice."""+ return urlparse(self.endpoint).netloc or self.endpoint++ def public(self) -> dict[str, Any]:+ out: dict[str, Any] = {"kind": self.kind}+ if self.kind == "langfuse":+ out["host"] = self.host+ elif self.kind == "otlp":+ out["url"] = self.url+ out["header_names"] = list(self.header_names)+ return out+++def _include(raw: Any) -> dict[str, bool]:+ raw = raw if isinstance(raw, dict) else {}+ return {k: bool(raw.get(k)) for k in INCLUDE_KEYS}+++def _http_url(value: Any) -> str:+ v = str(value or "").strip()+ return v if urlparse(v).scheme in ("http", "https") and urlparse(v).netloc else ""+++def parse_destination(raw: Any) -> Optional[Destination]:+ """A destination from settings or a policy; None when it does not make sense."""+ if not isinstance(raw, dict):+ return None+ kind = str(raw.get("kind") or "off").lower()+ if kind == "off":+ return Destination()+ if kind == "langfuse":+ host = _http_url(raw.get("host"))+ return Destination(kind="langfuse", host=host.rstrip("/")) if host else None+ if kind == "otlp":+ url = _http_url(raw.get("url"))+ names = raw.get("header_names")+ if names is None and isinstance(raw.get("headers"), dict):+ names = list(raw["headers"])+ names = [str(n).strip() for n in (names or []) if str(n).strip()]+ if not url or len(names) > MAX_HEADERS:+ return None+ return Destination(kind="otlp", url=url, header_names=tuple(dict.fromkeys(names)))+ return None+++@dataclass+class ExportSettings:+ record: bool = True+ retention_days: int = 30+ destination: Destination = field(default_factory=Destination)+ include: dict[str, bool] = field(default_factory=lambda: _include({}))+ # When each destination was chosen: only turns that end after it are sent (TD2-12).+ since: dict[str, int] = field(default_factory=dict)++ @classmethod+ def load(cls, path: Path) -> "ExportSettings":+ try:+ raw = json.loads(Path(path).read_text(encoding="utf-8"))+ except (OSError, ValueError):+ return cls()+ if not isinstance(raw, dict):+ return cls()+ days = raw.get("retention_days")+ return cls(+ record=raw.get("record", True) is not False,+ retention_days=days if days in RETENTION_CHOICES else 30,+ destination=parse_destination(raw.get("destination")) or Destination(),+ include=_include(raw.get("include")),+ since={str(k): int(v) for k, v in (raw.get("since") or {}).items() if isinstance(v, (int, float))},+ )++ def save(self, path: Path) -> None:+ path = Path(path)+ path.parent.mkdir(parents=True, exist_ok=True)+ data = {"record": self.record, "retention_days": self.retention_days,+ "destination": self.destination.public(), "include": self.include, "since": self.since}+ tmp = path.with_name(path.name + ".tmp")+ tmp.write_text(json.dumps(data, indent=2), encoding="utf-8")+ tmp.replace(path)+++# -- company control (TD2-11) ------------------------------------------------------------------+@dataclass+class Managed:+ by: str+ name: str+ at: str+ destination: Destination+ include: dict[str, bool]+++POLICY_FIELD = "observability_export"+_POLICY_SECRET_KEYS = ("public_key", "secret_key", "headers")+++def managed_export(policy: Any) -> Optional[Managed]:+ """The org policy's `observability_export` field, read like `audit_export`: anything+ malformed reads as not set, so governance never turns on by accident.++ Shape: `{"enabled": true, "name": "...", "set_at": "...", "destination": {"kind":+ "langfuse", "host": ...} | {"kind": "otlp", "url": ..., "header_names": [...]} |+ {"kind": "off"}, "include": {"messages", "tool_io", "context", "names"}}`. The keys+ and header values travel in the same field and are moved to the secret store+ (`take_policy_secrets`) before the policy is saved."""+ raw = policy.get(POLICY_FIELD) if isinstance(policy, dict) else None+ if not isinstance(raw, dict) or raw.get("enabled") is not True:+ return None+ dest = parse_destination(raw.get("destination"))+ if dest is None:+ return None+ by = str(policy.get("org_name") or policy.get("org_id") or "your organisation")+ return Managed(by=by, name=str(raw.get("name") or ""), at=str(raw.get("set_at") or ""),+ destination=dest, include=_include(raw.get("include")))+++def take_policy_secrets(policy: dict[str, Any], secrets: Any) -> dict[str, Any]:+ """Move the policy's export keys into the secret store (owned by the policy) and return+ the policy without them, so `policy.json` never holds them. A policy without the field+ removes the keys it had stored."""+ if not isinstance(policy, dict):+ return policy+ raw = policy.get(POLICY_FIELD)+ held = dict(secrets.get(PROFILE) or {}) if secrets is not None else {}+ if not isinstance(raw, dict):+ if secrets is not None and "policy" in held:+ held.pop("policy")+ _put_profile(secrets, held)+ return policy+ dest = raw.get("destination") if isinstance(raw.get("destination"), dict) else {}+ found = {k: dest[k] for k in _POLICY_SECRET_KEYS if k in dest}+ if secrets is not None and found:+ if isinstance(found.get("headers"), dict):+ found["headers"] = {str(k): str(v) for k, v in found["headers"].items()}+ dest = {**dest, "header_names": list(found["headers"])}+ held["policy"] = {k: v for k, v in found.items()}+ _put_profile(secrets, held)+ clean_dest = {k: v for k, v in dest.items() if k not in _POLICY_SECRET_KEYS}+ return {**policy, POLICY_FIELD: {**raw, "destination": clean_dest}}+++def _put_profile(secrets: Any, data: dict[str, Any]) -> None:+ if data:+ secrets.put(PROFILE, data)+ else:+ secrets.delete(PROFILE)+++def _mask_hint(value: str, *, keep_end: bool) -> str:+ if not value:+ return ""+ prefix = value[:6] if value[:3] in ("pk-", "sk-") else ""+ tail = value[-4:] if keep_end and len(value) >= 16 else ""+ return f"{prefix}{'•' * 8}{tail}"+++# -- sending -----------------------------------------------------------------------------------+Post = Callable[[str, bytes, dict[str, str], float], tuple[int, str]]+++def http_post(url: str, body: bytes, headers: dict[str, str], timeout: float) -> tuple[int, str]:+ """POST one OTLP/HTTP JSON body. A network error is status 0 with its kind as the text+ (never the URL or anything from the request)."""+ import httpx++ try:+ with httpx.Client(timeout=timeout) as client:+ resp = client.post(url, content=body, headers=headers)+ return resp.status_code, resp.text[:500]+ except httpx.TimeoutException:+ return 0, "timed out"+ except httpx.HTTPError as exc:+ return 0, type(exc).__name__+++class _Retry(Exception):+ pass+++class _Refused(Exception):+ pass+++@dataclass+class ExportConfig:+ destination: Destination+ headers: dict[str, str] # auth and the endpoint's own headers: secrets+ include: dict[str, bool]+ since_ms: int+ managed: bool = False+++def _reason(status: int, text: str) -> str:+ if status == 0:+ return f"no answer ({text})" if text else "no answer"+ return f"HTTP {status}" + (f": {text.strip()[:200]}" if text.strip() else "")+++class Exporter:+ """Reads finished turns from the store and sends them, each step once."""++ def __init__(self, store: Any, config: ExportConfig, *, post: Post = http_post,+ secret_values: Callable[[], list[str]] = list,+ session_title: Callable[[str], Optional[str]] = lambda _sid: None,+ batch_spans: int = BATCH_SPANS, flush_seconds: float = FLUSH_SECONDS,+ backoff_start: float = BACKOFF_START, backoff_cap: float = BACKOFF_CAP) -> None:+ self.store = store+ self.config = config+ self.post = post+ self.secret_values = secret_values+ self.session_title = session_title+ self.batch_spans = batch_spans+ self.flush_seconds = flush_seconds+ self.backoff_start, self.backoff_cap = backoff_start, backoff_cap+ self.state = "sending"+ self.last_error = ""+ self.last_sent_at: Optional[float] = None+ self._wake = threading.Event()+ self._stop = threading.Event()+ self._thread: Optional[threading.Thread] = None+ store.export_destination = (config.destination.key, config.since_ms)++ # -- the thread ------------------------------------------------------------------------------+ def start(self) -> None:+ self._thread = threading.Thread(target=self._run, name="observability-export", daemon=True)+ self._thread.start()++ def wake(self) -> None:+ self._wake.set()++ def stop(self, timeout: float = 5.0) -> None:+ self._stop.set()+ self._wake.set()+ if self._thread is not None and self._thread is not threading.current_thread():+ self._thread.join(timeout)+ if getattr(self.store, "export_destination", None) == (self.config.destination.key, self.config.since_ms):+ self.store.export_destination = None++ def set_include(self, include: dict[str, bool]) -> None:+ """Switches changed: applied per turn, when the turn is sent (§10.9)."""+ self.config = replace(self.config, include=dict(include))++ def _run(self) -> None:+ backoff = self.backoff_start+ while not self._stop.is_set():+ try:+ self.flush_once()+ backoff = self.backoff_start+ self.state, self.last_error = "sending", ""+ self._wake.wait(self.flush_seconds)+ self._wake.clear()+ except _Retry as exc:+ self.state, self.last_error = "retrying", str(exc)+ self._stop.wait(backoff)+ backoff = min(backoff * 2, self.backoff_cap)+ except _Refused as exc:+ # Stops until the settings change (a new exporter is made then).+ self.state, self.last_error = "failed", str(exc)+ self.store.count_export(self.config.destination.key, failed=1)+ self._stop.wait()+ except Exception as exc: # noqa: BLE001 - never let the thread die quietly+ log.exception("observability export: unexpected failure")+ self.state, self.last_error = "retrying", type(exc).__name__+ self._stop.wait(backoff)+ backoff = min(backoff * 2, self.backoff_cap)++ # -- one pass --------------------------------------------------------------------------------+ def flush_once(self) -> int:+ """Send everything ready, in batches. Returns the spans sent. Raises `_Retry` (keep+ the batch, back off) or `_Refused` (stop until the settings change)."""+ cfg = self.config+ dest = cfg.destination.key+ sw = switches_from(cfg.include)+ masker = Masker(self.secret_values()) if any(sw.values()) else Masker()+ sent = 0+ done: set[str] = set() # turns with nothing left to send in this pass+ while not self._stop.is_set():+ queue = [t for t in self.store.export_queue(dest, cfg.since_ms, limit=50) if t not in done]+ if not queue:+ return sent+ pending: list[tuple[dict[str, Any], list[tuple[str, str]]]] = []+ for tid in queue:+ built = self._turn_spans(tid, sw, masker)+ if not built:+ done.add(tid)+ pending.extend(built)+ for i in range(0, len(pending), self.batch_spans):+ sent += self._send(pending[i:i + self.batch_spans])+ if not pending:+ return sent++ def _turn_spans(self, turn_id: str, sw: dict[str, bool], masker: Masker) -> list[tuple[dict[str, Any], list[tuple[str, str]]]]:+ cfg = self.config+ dest = cfg.destination.key+ loaded = self.store.load_turn(turn_id)+ if loaded is None:+ return []+ turn, steps = loaded+ marked = self.store.export_marked(dest, turn_id)+ ready = {s["step_id"] for s in steps+ if s.get("status") not in (None, "running") and s["step_id"] not in marked}+ include_root = turn_id not in marked+ late_notices = [s["step_id"] for s in steps if s.get("kind") == "notice" and s["step_id"] in ready]+ if not include_root and late_notices:+ # A notice is an event on the turn's span, which has gone already.+ self.store.mark_export(dest, [(n, turn_id) for n in late_notices], "skipped")+ title = self.session_title(turn["session_id"]) if sw.get("names") else None+ built = build_turn_spans(turn, steps, sw=sw, dialect=cfg.destination.dialect,+ get_piece=self.store.get_piece, masker=masker,+ include_root=include_root, only=ready, session_title=title)+ stamp = turn.get("stamp") or {}+ machine = stamp.get("machine") or {}+ resource = resource_attrs(app_version=str(machine.get("app_version") or ""),+ machine_name=str(machine.get("name") or "") if sw.get("names") else None)+ out = []+ for span in built:+ marks = [(span["_mark"], turn_id)] + [(n, turn_id) for n in span.get("_notices") or []]+ out.append(({**span, "_resource": resource}, marks))+ return out++ def _encode(self, items: list[tuple[dict[str, Any], list[tuple[str, str]]]]) -> bytes:+ groups: dict[str, tuple[dict[str, Any], list[dict[str, Any]]]] = {}+ for span, _ in items:+ res = span["_resource"]+ groups.setdefault(json.dumps(res, sort_keys=True), (res, []))[1].append(span)+ return json.dumps(encode_otlp(list(groups.values())), ensure_ascii=False).encode("utf-8")++ def _post(self, items: list[tuple[dict[str, Any], list[tuple[str, str]]]]) -> tuple[int, str]:+ headers = {"Content-Type": "application/json", **self.config.headers}+ return self.post(self.config.destination.endpoint, self._encode(items), headers, TIMEOUT_SECONDS)++ def _send(self, batch: list[tuple[dict[str, Any], list[tuple[str, str]]]]) -> int:+ status, text = self._post(batch)+ if 200 <= status < 300:+ self._mark(batch, "sent")+ return len(batch)+ self._raise_for(status, text)+ # 400 and the other 4xx: split the batch once; a span refused on its own is dropped.+ sent = 0+ for item in batch:+ status, text = self._post([item])+ if 200 <= status < 300:+ self._mark([item], "sent")+ sent += 1+ continue+ self._raise_for(status, text)+ self._mark([item], "dropped")+ self.last_error = _reason(status, text)+ log.info("observability export: a span was refused on its own (%s)", status)+ return sent++ def _raise_for(self, status: int, text: str) -> None:+ if status in (401, 403):+ raise _Refused(REFUSED_KEY)+ if status == 404:+ raise _Refused(NOT_FOUND)+ if status == 0 or status == 429 or status >= 500:+ raise _Retry(_reason(status, text))++ def _mark(self, batch: list[tuple[dict[str, Any], list[tuple[str, str]]]], state: str) -> None:+ marks = [m for _, ms in batch for m in ms]+ self.store.mark_export(self.config.destination.key, marks, state)+ if state == "sent":+ self.last_sent_at = time.time()+++# -- what the server holds ---------------------------------------------------------------------+class ObservabilityExport:+ """Settings, company control, status and the exporter's life (design §10.5 to §10.8)."""++ def __init__(self, state: Path, secrets: Any, store: Any, *,+ session_title: Callable[[str], Optional[str]] = lambda _sid: None,+ post: Post = http_post, exporter_options: Optional[dict[str, Any]] = None,+ clock: Callable[[], float] = time.time) -> None:+ self.path = Path(state) / SETTINGS_FILE+ self.secrets = secrets+ self.store = store+ self.session_title = session_title+ self.post = post+ self.exporter_options = dict(exporter_options or {})+ self.clock = clock+ self.settings = ExportSettings.load(self.path)+ self.managed: Optional[Managed] = None+ self.exporter: Optional[Exporter] = None+ self._lock = threading.RLock()+ if store is not None:+ store.retention_days = self.settings.retention_days++ # -- what is in force --------------------------------------------------------------------------+ def effective(self) -> tuple[Destination, dict[str, bool]]:+ if self.managed is not None:+ return self.managed.destination, self.managed.include+ return self.settings.destination, self.settings.include++ def _held(self) -> dict[str, Any]:+ data = (self.secrets.get(PROFILE) if self.secrets is not None else None) or {}+ return data if isinstance(data, dict) else {}++ def _keys(self) -> dict[str, Any]:+ """The keys for the destination in force: the policy's when it manages the page."""+ held = self._held()+ own = held.get("policy" if self.managed is not None else "user") or {}+ return own if isinstance(own, dict) else {}++ def _headers(self, dest: Destination) -> dict[str, str]:+ keys = self._keys()+ if dest.kind == "langfuse":+ import base64++ pair = f"{keys.get('public_key') or ''}:{keys.get('secret_key') or ''}"+ return {"Authorization": "Basic " + base64.b64encode(pair.encode()).decode()}+ if dest.kind == "otlp":+ values = keys.get("headers") if isinstance(keys.get("headers"), dict) else {}+ return {n: str(values[n]) for n in dest.header_names if values.get(n)}+ return {}++ def secret_values(self) -> list[str]:+ """Masking (R4): every value in the secret store, and credential-looking env vars."""+ data = self.secrets._read() if self.secrets is not None and hasattr(self.secrets, "_read") else {}+ return spans_mod.store_secret_values(data) + spans_mod.env_secret_values()++ def content_notice(self) -> Optional[str]:+ """When the company's policy sends content: what goes where, for each new session."""+ if self.managed is None or self.managed.destination.kind == "off":+ return None+ inc = self.managed.include+ what = [w for k, w in (("messages", "messages"), ("tool_io", "tool inputs and results"),+ ("context", "the context OpenWorker adds"), ("names", "names")) if inc.get(k)]+ if not what:+ return None+ listed = what[0] if len(what) == 1 else ", ".join(what[:-1]) + " and " + what[-1]+ dest = self.managed.destination+ where = "Langfuse" if dest.kind == "langfuse" else "OpenTelemetry"+ return f"{self.managed.by} receives this session's {listed} ({where}, {dest.place})."++ # -- the exporter's life ------------------------------------------------------------------------+ def reconcile(self) -> None:+ """Create, keep or stop the exporter for what is in force. Off: none at all."""+ with self._lock:+ dest, include = self.effective()+ if dest.kind == "off" or not self.settings.record:+ self._stop_exporter()+ return+ since = self._since(dest)+ cfg = ExportConfig(destination=dest, headers=self._headers(dest), include=dict(include),+ since_ms=since, managed=self.managed is not None)+ old = self.exporter+ if old is not None and old.state != "failed" and old.config.destination == dest and old.config.headers == cfg.headers \+ and old.config.since_ms == since:+ old.set_include(include)+ old.wake()+ return+ self._stop_exporter()+ self.exporter = Exporter(self.store, cfg, post=self.post, secret_values=self.secret_values,+ session_title=self.session_title, **self.exporter_options)+ self.exporter.start()++ def _since(self, dest: Destination) -> int:+ """When this destination was chosen: older turns are not sent (TD2-12)."""+ since = self.settings.since.get(dest.key)+ if since is None:+ since = int(self.clock() * 1000)+ self.settings.since[dest.key] = since+ self.settings.save(self.path)+ return since++ def _stop_exporter(self) -> None:+ if self.exporter is not None:+ self.exporter.stop()+ self.exporter = None++ def wake(self) -> None:+ """A turn ended: send soon."""+ ex = self.exporter+ if ex is not None:+ ex.wake()++ def close(self) -> None:+ with self._lock:+ self._stop_exporter()++ # -- the policy -----------------------------------------------------------------------------------+ def apply_policy(self, policy: Optional[dict[str, Any]]) -> None:+ with self._lock:+ self.managed = managed_export(policy or {})+ self.reconcile()++ # -- GET/PUT settings (§10.8) ---------------------------------------------------------------------+ def get_settings(self) -> dict[str, Any]:+ dest, include = self.effective()+ keys = self._keys()+ secrets: dict[str, Any] = {}+ if dest.kind == "langfuse":+ secrets = {"public_key": _mask_hint(str(keys.get("public_key") or ""), keep_end=True),+ "secret_key": _mask_hint(str(keys.get("secret_key") or ""), keep_end=False)}+ elif dest.kind == "otlp":+ values = keys.get("headers") if isinstance(keys.get("headers"), dict) else {}+ secrets = {"headers": {n: ("•" * 8 if values.get(n) else "") for n in dest.header_names}}+ out = {+ "record": self.settings.record,+ "retention_days": self.settings.retention_days,+ "destination": dest.public(),+ "include": dict(include),+ "secrets": secrets, # masked hints only: values never leave+ "managed": None,+ "notice": self.content_notice(),+ }+ if self.managed is not None:+ out["managed"] = {"by": self.managed.by, "name": self.managed.name, "at": self.managed.at}+ return out++ def put_settings(self, body: dict[str, Any]) -> tuple[int, dict[str, Any]]:+ """Returns (HTTP status, answer). 409 when the page is managed by a policy."""+ if not isinstance(body, dict):+ return 400, {"ok": False, "error": "expected an object"}+ with self._lock:+ s = self.settings+ if "record" in body:+ s.record = bool(body["record"])+ if "retention_days" in body:+ if body["retention_days"] not in RETENTION_CHOICES:+ return 400, {"ok": False, "error": "retention_days must be 7, 30 or 90"}+ s.retention_days = int(body["retention_days"])+ if self.store is not None:+ self.store.retention_days = s.retention_days+ touches_export = any(k in body for k in ("destination", "include", "secrets"))+ if touches_export and self.managed is not None:+ self.settings.save(self.path)+ return 409, {"ok": False, "error": f"Managed by {self.managed.by}"}+ if "destination" in body:+ dest = parse_destination(body["destination"])+ if dest is None:+ return 400, {"ok": False, "error": "check the address (http or https) and at most 8 headers"}+ if dest != s.destination:+ s.destination = dest+ if dest.kind != "off": # a new choice starts from now (TD2-12)+ s.since[dest.key] = int(self.clock() * 1000)+ if "include" in body:+ s.include = _include({**s.include, **(body["include"] if isinstance(body["include"], dict) else {})})+ if isinstance(body.get("secrets"), dict):+ self._put_user_secrets(body["secrets"], s.destination)+ s.save(self.path)+ self.reconcile()+ return 200, {"ok": True, **self.get_settings()}++ def _put_user_secrets(self, given: dict[str, Any], dest: Destination) -> None:+ held = self._held()+ user = dict(held.get("user") or {})+ for k in ("public_key", "secret_key"):+ v = given.get(k)+ if isinstance(v, str) and v.strip():+ user[k] = v.strip()+ if isinstance(given.get("headers"), dict):+ headers = dict(user.get("headers") or {})+ for n, v in given["headers"].items():+ if isinstance(v, str) and v:+ headers[str(n)] = v+ user["headers"] = headers+ if dest.kind == "otlp" and isinstance(user.get("headers"), dict):+ # A header taken off the list takes its value with it.+ user["headers"] = {n: v for n, v in user["headers"].items() if n in dest.header_names}+ held["user"] = user+ _put_profile(self.secrets, {k: v for k, v in held.items() if v})++ # -- status (§10.5) ----------------------------------------------------------------------------------+ def status(self) -> dict[str, Any]:+ dest, _ = self.effective()+ ex = self.exporter+ if dest.kind == "off" or ex is None:+ return {"state": "off", "last_sent_at": None, "sent": 0, "sent_today": 0, "failed": 0,+ "dropped": 0, "last_error": "", "destination": dest.public()}+ midnight = time.mktime(time.localtime(self.clock())[:3] + (0, 0, 0, 0, 0, -1)) * 1000+ counts = self.store.export_counts(dest.key)+ today = self.store.export_counts(dest.key, since_ms=int(midnight))["sent"]+ state = ex.state+ if state == "sending" and self.managed is not None:+ state = "managed"+ return {"state": state, "last_sent_at": ex.last_sent_at, "sent": counts["sent"], "sent_today": today,+ "failed": counts["failed"], "dropped": counts["dropped"], "last_error": ex.last_error,+ "destination": dest.public()}++ # -- the test recording (E9c) ---------------------------------------------------------------------+ def send_test(self) -> dict[str, Any]:+ dest, _ = self.effective()+ if dest.kind == "off":+ return {"ok": False, "status": 0, "text": "Choose where to send recordings first."}+ spans = spans_mod.sample_spans(int(self.clock() * 1000), dest.dialect)+ from .stamps import machine_stamp++ res = resource_attrs(app_version=str(machine_stamp().get("app_version") or ""))+ body = json.dumps(encode_otlp([(res, spans)])).encode("utf-8")+ started = time.monotonic()+ status, text = self.post(dest.endpoint, body, {"Content-Type": "application/json", **self._headers(dest)},+ TIMEOUT_SECONDS)+ ms = int((time.monotonic() - started) * 1000)+ if 200 <= status < 300:+ answer = "Sent. Open Langfuse to see it." if dest.kind == "langfuse" else "Sent."+ return {"ok": True, "status": status, "text": answer, "ms": ms}+ if status in (401, 403):+ return {"ok": False, "status": status, "text": REFUSED_KEY, "ms": ms}+ return {"ok": False, "status": status, "text": _reason(status, text), "ms": ms}+++def destination_digest(dest: Destination) -> str:+ """A short, stable label for a destination (logs never show its address)."""+ return hashlib.sha256(dest.key.encode()).hexdigest()[:8]diff --git a/coworker/observability/export_spans.py b/coworker/observability/export_spans.pynew file mode 100644index 00000000..987ba4d6--- /dev/null+++ b/coworker/observability/export_spans.py@@ -0,0 +1,811 @@+"""Recorded turns as OpenTelemetry spans, with the content rules (OB-6, phase-2 design §10.4).++Pure functions, no network: the exporter (`export.py`) reads the store, calls+`build_turn_spans` for a turn's unsent steps and encodes the result with `encode_otlp`.++The mapping is the frozen 1.x one (tracing-spec §9 and §10, attributes.yaml part 2): one trace+per turn, the turn as the root span, each step a span below it, notices as events on the turn.+The Phase 1 lab exporter (`phase-1/lab/export_otel.py`) was the starting point; it was checked+against a local Langfuse. This module writes OTLP/HTTP JSON itself, so the app needs no+OpenTelemetry SDK.++Content (tracing-spec §11). Metadata is always sent. Each content attribute has its switch,+and five rules apply on top:+- R1. Tool parts inside exported messages follow the tool switches, wherever they appear.+- R2. OpenWorker's per-turn `<system-context>` block is cut out of user text unless+ `capture.context` is on.+- R3. The auto-approver's and the summary's calls need the tool switches as well.+- R4. Secrets are masked, then each text part is cut at 8 KB (`Masker`).+- R5. Never exported: reasoning text, provider-reported cost, `error.text`, `questions`,+ `decision_details`, `agents_md`, `blocked`. None of them has a name here.+"""++from __future__ import annotations++import datetime as _dt+import hashlib+import json+import os+import re+from typing import Any, Callable, Iterable, Optional++from . import calculator+from .capture import is_auxiliary, parse_events, rebuild_request, wire_of_url++ATTRIBUTES_VERSION = "1.2"+SCHEMA = f"openworker/{ATTRIBUTES_VERSION}"+SERVICE_NAME = "openworker-desktop"+SCOPE_NAME = "openworker.observability"++# Markers (tracing-spec §11).+REASONING = "[reasoning text not exported]"+TOOL_ARGS_OUT = "[tool arguments not exported]"+TOOL_RESULT_OUT = "[tool result not exported]"+CONTEXT_OUT = "[OpenWorker context not exported]"+SECRET = "[secret masked]"+CUT_BYTES = 8192+CUT_MARKER = "[cut: {n} more bytes]"++SWITCHES = ("prompts", "responses", "tool_args", "tool_results", "system_instructions",+ "context", "names")++# attributes.yaml export.value_maps.gen_ai_provider_name+PROVIDER_NAME = {"anthropic": "anthropic", "openai": "openai", "gemini": "gcp.gemini",+ "bedrock": "aws.bedrock", "openrouter": "openrouter", "ollama": "ollama",+ "together": "together", "openai_decisions": "openai", "typesafe_jev": "typesafe"}++SPAN_INTERNAL, SPAN_CLIENT = 1, 3 # OTLP SpanKind+STATUS_ERROR = 2+++# -- R4: mask, then cut -----------------------------------------------------------------------+# attributes.yaml export.content_switches.masking+MASK_PATTERNS = [+ r"sk-[A-Za-z0-9_-]{16,}",+ r"\b(AKIA|ASIA)[A-Z0-9]{16}\b",+ r"\bgh[pousr]_[A-Za-z0-9]{30,}\b",+ r"\bgithub_pat_[A-Za-z0-9_]{30,}\b",+ r"\bxox[abp]-[A-Za-z0-9-]{10,}",+ r"\bAIza[0-9A-Za-z_-]{35}\b",+ r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----",+]+_PATTERNS = [re.compile(p) for p in MASK_PATTERNS]+MIN_SECRET_LEN = 8+_ENV_SECRET_PARTS = ("_API_KEY", "_TOKEN", "_SECRET", "PASSWORD")+++def env_secret_values(environ: Optional[dict[str, str]] = None) -> list[str]:+ """Credential-looking environment variables (masking.values)."""+ env = os.environ if environ is None else environ+ return [v for k, v in env.items()+ if any(p in k.upper() for p in _ENV_SECRET_PARTS) and len(v or "") >= MIN_SECRET_LEN]+++def store_secret_values(data: Any) -> list[str]:+ """Every string value in the secret store's profiles."""+ out: list[str] = []++ def walk(v: Any) -> None:+ if isinstance(v, str):+ out.append(v)+ elif isinstance(v, dict):+ for x in v.values():+ walk(x)+ elif isinstance(v, list):+ for x in v:+ walk(x)++ walk(data)+ return out+++def cut(text: str, limit: int = CUT_BYTES) -> str:+ """Cut at `limit` bytes of UTF-8, on a character boundary, and say how much was left out."""+ data = text.encode("utf-8")+ if len(data) <= limit:+ return text+ n = limit+ while n > 0 and (data[n] & 0xC0) == 0x80: # never split a character+ n -= 1+ return data[:n].decode("utf-8") + CUT_MARKER.format(n=len(data) - n)+++class Masker:+ """R4: stored secret values (longest first), then the key patterns; then the cut."""++ def __init__(self, secrets: Iterable[str] = ()) -> None:+ values = {s for s in secrets if isinstance(s, str) and len(s) >= MIN_SECRET_LEN}+ self.values = sorted(values, key=len, reverse=True)++ def mask(self, text: str) -> str:+ for v in self.values:+ if v in text:+ text = text.replace(v, SECRET)+ for p in _PATTERNS:+ text = p.sub(SECRET, text)+ return text++ def clean(self, value: Any) -> str:+ """One exported text part: as text (JSON for anything else), masked, then cut."""+ text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)+ return cut(self.mask(text))+++# -- content: the exact request and reply, in the GenAI message shape -------------------------+def _text_parts(content: Any) -> list[dict[str, Any]]:+ if content is None:+ return []+ if isinstance(content, str):+ return [{"type": "text", "content": content}] if content else []+ parts: list[dict[str, Any]] = []+ for b in content if isinstance(content, list) else [content]:+ if not isinstance(b, dict):+ parts.append({"type": "text", "content": str(b)})+ continue+ t = b.get("type")+ if t in ("text", "input_text", "output_text"):+ parts.append({"type": "text", "content": b.get("text", "")})+ elif t == "tool_use":+ parts.append({"type": "tool_call", "id": b.get("id"), "name": b.get("name"), "arguments": b.get("input")})+ elif t == "tool_result":+ parts.append({"type": "tool_call_response", "id": b.get("tool_use_id"), "response": b.get("content")})+ elif t in ("thinking", "redacted_thinking", "reasoning"):+ parts.append({"type": "reasoning", "content": REASONING})+ else:+ parts.append({"type": t or "unknown"}) # images and the like: the kind only+ return parts+++def _gemini_parts(parts: Any) -> list[dict[str, Any]]:+ out: list[dict[str, Any]] = []+ for p in parts or []:+ if not isinstance(p, dict):+ continue+ if p.get("thought"):+ out.append({"type": "reasoning", "content": REASONING})+ elif "text" in p:+ out.append({"type": "text", "content": p.get("text") or ""})+ elif "functionCall" in p:+ fc = p["functionCall"] or {}+ out.append({"type": "tool_call", "id": fc.get("id"), "name": fc.get("name"), "arguments": fc.get("args")})+ elif "functionResponse" in p:+ fr = p["functionResponse"] or {}+ out.append({"type": "tool_call_response", "id": fr.get("id") or fr.get("name"), "response": fr.get("response")})+ else:+ out.append({"type": next(iter(p), "unknown")})+ return out+++def request_messages(wire: str, req: dict[str, Any]) -> tuple[list, list, list]:+ """(system instructions, input messages, tool definitions) from a rebuilt request."""+ system: list[dict[str, Any]] = []+ msgs: list[dict[str, Any]] = []+ tools = req.get("tools") or []+ if wire == "anthropic":+ system = _text_parts(req.get("system"))+ for m in req.get("messages") or []:+ msgs.append({"role": m.get("role"), "parts": _text_parts(m.get("content"))})+ elif wire == "openai_responses":+ if req.get("instructions"):+ system = [{"type": "text", "content": req["instructions"]}]+ items = req.get("input") or []+ for it in [{"role": "user", "content": items}] if isinstance(items, str) else items:+ t = it.get("type")+ if t == "function_call":+ msgs.append({"role": "assistant", "parts": [{"type": "tool_call", "id": it.get("call_id"),+ "name": it.get("name"), "arguments": it.get("arguments")}]})+ elif t == "function_call_output":+ msgs.append({"role": "tool", "parts": [{"type": "tool_call_response", "id": it.get("call_id"),+ "response": it.get("output")}]})+ elif t == "reasoning":+ msgs.append({"role": "assistant", "parts": [{"type": "reasoning", "content": REASONING}]})+ elif it.get("role") in ("system", "developer"):+ system += _text_parts(it.get("content"))+ else:+ msgs.append({"role": it.get("role", "user"), "parts": _text_parts(it.get("content"))})+ elif wire == "gemini":+ system = _gemini_parts((req.get("systemInstruction") or {}).get("parts"))+ for c in req.get("contents") or []:+ msgs.append({"role": c.get("role", "user"), "parts": _gemini_parts(c.get("parts"))})+ else: # openai_chat and ollama: a messages list, system first+ for m in req.get("messages") or []:+ if m.get("role") in ("system", "developer"):+ system += _text_parts(m.get("content"))+ continue+ parts = _text_parts(m.get("content"))+ if m.get("reasoning_content") or m.get("reasoning") or m.get("thinking"):+ parts.insert(0, {"type": "reasoning", "content": REASONING})+ for tc in m.get("tool_calls") or []:+ fn = tc.get("function") or {}+ parts.append({"type": "tool_call", "id": tc.get("id"), "name": fn.get("name"),+ "arguments": fn.get("arguments")})+ if m.get("role") == "tool":+ parts = [{"type": "tool_call_response", "id": m.get("tool_call_id") or m.get("tool_name"),+ "response": m.get("content")}]+ msgs.append({"role": m.get("role"), "parts": parts})+ return system, msgs, tools+++def response_message(wire: str, body: str) -> dict[str, Any]:+ """The assistant's reply, put together from the body exactly as received."""+ evs = [e for e in parse_events(body) if isinstance(e, dict)]+ parts: list[dict[str, Any]] = []+ finish = None+ if wire == "anthropic":+ blocks: dict[int, dict[str, Any]] = {}+ for e in evs:+ t = e.get("type")+ if t == "message" and isinstance(e.get("content"), list): # not streamed+ for i, b in enumerate(e["content"]):+ blocks[i] = dict(b, _json="")+ finish = e.get("stop_reason")+ elif t == "content_block_start":+ blocks[e.get("index", 0)] = dict(e.get("content_block") or {}, _json="")+ elif t == "content_block_delta":+ b = blocks.setdefault(e.get("index", 0), {"type": "text", "text": "", "_json": ""})+ d = e.get("delta") or {}+ if d.get("type") == "text_delta":+ b["text"] = b.get("text", "") + d.get("text", "")+ elif d.get("type") == "input_json_delta":+ b["_json"] += d.get("partial_json", "")+ elif t == "message_delta":+ finish = (e.get("delta") or {}).get("stop_reason")+ for i in sorted(blocks):+ b = blocks[i]+ if b.get("type") == "tool_use":+ try:+ args = json.loads(b["_json"]) if b["_json"] else b.get("input")+ except ValueError:+ args = b["_json"]+ parts.append({"type": "tool_call", "id": b.get("id"), "name": b.get("name"), "arguments": args})+ elif b.get("type") == "text":+ parts.append({"type": "text", "content": b.get("text", "")})+ else:+ parts.append({"type": "reasoning", "content": REASONING})+ elif wire == "openai_responses":+ done = [e for e in evs if e.get("type") == "response.completed"]+ resp = done[-1].get("response") if done else (evs[-1] if evs and "output" in evs[-1] else {})+ finish = (resp or {}).get("status")+ for it in (resp or {}).get("output") or []:+ if it.get("type") == "function_call":+ parts.append({"type": "tool_call", "id": it.get("call_id"), "name": it.get("name"),+ "arguments": it.get("arguments")})+ elif it.get("type") == "message":+ parts += _text_parts(it.get("content"))+ elif it.get("type") == "reasoning":+ parts.append({"type": "reasoning", "content": REASONING})+ elif wire == "gemini":+ for e in evs:+ for c in e.get("candidates") or []:+ parts += _gemini_parts((c.get("content") or {}).get("parts"))+ finish = c.get("finishReason") or finish+ elif wire == "ollama":+ text, reasoning = "", False+ for e in evs:+ m = e.get("message") or {}+ text += m.get("content") or ""+ reasoning = reasoning or bool(m.get("thinking"))+ for tc in m.get("tool_calls") or []:+ fn = tc.get("function") or {}+ parts.append({"type": "tool_call", "id": tc.get("id"), "name": fn.get("name"),+ "arguments": fn.get("arguments")})+ if e.get("done"):+ finish = e.get("done_reason")+ if text:+ parts.insert(0, {"type": "text", "content": text})+ if reasoning:+ parts.insert(0, {"type": "reasoning", "content": REASONING})+ else: # openai_chat, streamed or not+ text, calls, reasoning = "", {}, False+ for e in evs:+ for ch in e.get("choices") or []:+ d = ch.get("delta") or ch.get("message") or {}+ text += d.get("content") or ""+ reasoning = reasoning or bool(d.get("reasoning") or d.get("reasoning_content"))+ for n, tc in enumerate(d.get("tool_calls") or []):+ c = calls.setdefault(tc.get("index", n), {"id": None, "name": None, "arguments": ""})+ c["id"] = tc.get("id") or c["id"]+ fn = tc.get("function") or {}+ c["name"] = fn.get("name") or c["name"]+ c["arguments"] += fn.get("arguments") or ""+ finish = ch.get("finish_reason") or finish+ if reasoning:+ parts.append({"type": "reasoning", "content": REASONING})+ if text:+ parts.append({"type": "text", "content": text})+ parts += [dict(type="tool_call", **calls[i]) for i in sorted(calls)]+ return {"role": "assistant", "parts": parts, "finish_reason": finish}+++def step_exchange(step: dict[str, Any], get_piece: Callable[[str], Optional[bytes]]) -> Optional[tuple[str, dict, str]]:+ """(wire, rebuilt request, response body) of a model step's last real request."""+ for ex in reversed((step.get("content") or {}).get("exchange") or []):+ url = (ex.get("request") or {}).get("url") or ""+ if ex.get("auxiliary") or is_auxiliary(url):+ continue # a helper lookup (Ollama's /api/show), not the call+ try:+ req = rebuild_request(ex, get_piece)+ if isinstance(req, (bytes, bytearray)):+ req = json.loads(req)+ except Exception: # noqa: BLE001 - a piece already swept: nothing to rebuild+ return None+ if not isinstance(req, dict):+ return None+ body = get_piece((ex.get("response") or {}).get("body_blob") or "") or b""+ return wire_of_url(url), req, body.decode("utf-8", "replace")+ return None+++# -- the content rules -------------------------------------------------------------------------+_CONTEXT_BLOCK = re.compile(r"<system-context>.*?(</system-context>|\Z)", re.S)+++def cut_context(text: str) -> str:+ """R2: OpenWorker's per-turn block, replaced by its marker."""+ return _CONTEXT_BLOCK.sub(CONTEXT_OUT, text)+++def apply_rules(messages: list[dict[str, Any]], sw: dict[str, bool], masker: Masker) -> list[dict[str, Any]]:+ """R1, R2 and R4 over a list of exported messages (a copy)."""+ out = []+ for m in messages:+ parts = []+ for p in m.get("parts") or []:+ p = dict(p)+ t = p.get("type")+ if t == "text":+ text = p.get("content") or ""+ if m.get("role") == "user" and not sw.get("context"):+ text = cut_context(text)+ p["content"] = masker.clean(text)+ elif t == "tool_call":+ p["arguments"] = masker.clean(p.get("arguments")) if sw.get("tool_args") else TOOL_ARGS_OUT+ elif t == "tool_call_response":+ p["response"] = masker.clean(p.get("response")) if sw.get("tool_results") else TOOL_RESULT_OUT+ parts.append(p)+ out.append({k: v for k, v in m.items() if k != "parts"} | {"parts": parts})+ return out+++def content_allowed(purpose: str, sw: dict[str, bool]) -> tuple[bool, bool]:+ """(input, output) of a model call may go out (R3)."""+ if purpose == "auto_approver":+ return sw["prompts"] and sw["tool_args"], sw["responses"] and sw["tool_args"]+ if purpose == "summary":+ tools = sw["tool_args"] and sw["tool_results"]+ return sw["prompts"] and tools, sw["responses"] and tools+ return sw["prompts"], sw["responses"]+++# -- tokens -------------------------------------------------------------------------------------+def _usage(step: dict[str, Any]) -> Optional[dict[str, Any]]:+ u = (step.get("attrs") or {}).get("usage") or {}+ if not u.get("reported"):+ return None+ return {k: u.get(k) for k in ("input_fresh", "cache_read", "cache_write", "output", "reasoning")}+++def _semconv_usage(u: dict[str, Any]) -> dict[str, int]:+ a = {"gen_ai.usage.input_tokens": calculator.input_total(u), "gen_ai.usage.output_tokens": u.get("output") or 0}+ for part, name in (("cache_read", "gen_ai.usage.cache_read.input_tokens"),+ ("cache_write", "gen_ai.usage.cache_creation.input_tokens"),+ ("reasoning", "gen_ai.usage.reasoning.output_tokens")):+ if u.get(part) is not None: # a part not reported is left out, never 0+ a[name] = u[part]+ return a+++def _langfuse_usage(u: dict[str, Any]) -> dict[str, int]:+ """Buckets that do not overlap: they add up to the call's total."""+ d = {"input": u.get("input_fresh") or 0}+ if u.get("cache_read") is not None:+ d["input_cache_read"] = u["cache_read"]+ if u.get("cache_write") is not None:+ d["input_cache_creation"] = u["cache_write"]+ out = u.get("output") or 0+ if u.get("reasoning") is not None:+ d["output"] = max(out - u["reasoning"], 0)+ d["output_reasoning"] = u["reasoning"]+ else:+ d["output"] = out+ return d+++# -- ids -----------------------------------------------------------------------------------------+def _hex(text: str, n: int) -> str:+ h = hashlib.sha256(text.encode("utf-8")).hexdigest()[:n]+ return h if int(h, 16) else "1".rjust(n, "0") # an all-zero id is invalid in OTLP+++def trace_id(turn_id: str) -> str:+ return _hex(f"trace|{turn_id}", 32)+++def root_span_id(turn_id: str) -> str:+ return _hex(f"turn|{turn_id}", 16)+++def step_span_id(step_id: str) -> str:+ return _hex(f"step|{step_id}", 16)+++def _iso(ms: int) -> str:+ return (_dt.datetime.fromtimestamp(ms / 1000, _dt.timezone.utc)+ .isoformat(timespec="milliseconds").replace("+00:00", "Z"))+++def _js(v: Any) -> str:+ return json.dumps(v, ensure_ascii=False, default=str)+++# -- one turn -> spans -----------------------------------------------------------------------------+def switches_from(include: dict[str, bool]) -> dict[str, bool]:+ """The page's switches to the shared `capture.*` ones (design §10.4)."""+ return {+ "prompts": bool(include.get("messages")), "responses": bool(include.get("messages")),+ "tool_args": bool(include.get("tool_io")), "tool_results": bool(include.get("tool_io")),+ "system_instructions": bool(include.get("context")), "context": bool(include.get("context")),+ "names": bool(include.get("names")),+ }+++def turn_end_ms(turn: dict[str, Any], steps: list[dict[str, Any]]) -> int:+ if turn.get("duration_ms") is not None:+ return turn["started_at"] + turn["duration_ms"]+ ends = [s["started_at"] + (s.get("duration_ms") or 0) for s in steps if s.get("started_at") is not None]+ return max([turn.get("started_at") or 0] + ends)+++def build_turn_spans(+ turn: dict[str, Any],+ steps: list[dict[str, Any]],+ *,+ sw: dict[str, bool],+ dialect: str,+ get_piece: Callable[[str], Optional[bytes]],+ masker: Masker,+ include_root: bool = True,+ only: Optional[set[str]] = None,+ session_title: Optional[str] = None,+) -> list[dict[str, Any]]:+ """The spans of one turn, as plain dicts (`encode_otlp` turns them into OTLP JSON).++ `steps` is every step of the turn (parents and content are looked up among them);+ `only` limits the spans built to these step ids (a late `after_turn` step), and+ `include_root` says whether the turn's own span is built. Each span dict carries+ `_mark`: the id the exporter marks as sent (the step id, or the turn id for the root).+ """+ lf = dialect == "langfuse"+ tid = turn["turn_id"]+ by_id = {s["step_id"]: s for s in steps}+ end_ms = turn_end_ms(turn, steps)+ common = {"session.id": turn["session_id"], "gen_ai.conversation.id": turn["session_id"],+ "openworker.turn.id": tid}+ any_content = any(sw.get(k) for k in SWITCHES if k != "names")++ # Content for tool spans comes from the exchanges around them: the arguments from the+ # reply that asked for the call, the result from the next request that carried it back.+ exchanges: dict[str, tuple[str, dict, str]] = {}+ tool_args: dict[str, Any] = {}+ tool_results: dict[str, Any] = {}+ if any_content:+ for s in steps:+ if s.get("kind") != "model":+ continue+ x = step_exchange(s, get_piece)+ if x is None:+ continue+ exchanges[s["step_id"]] = x+ wire, req, body = x+ for m in request_messages(wire, req)[1]:+ for p in m["parts"]:+ if p.get("type") == "tool_call_response" and p.get("id"):+ tool_results[p["id"]] = p.get("response")+ for p in response_message(wire, body)["parts"]:+ if p.get("type") == "tool_call" and p.get("id"):+ tool_args[p["id"]] = p.get("arguments")++ spans: list[dict[str, Any]] = []+ trace = trace_id(tid)+ root_id = root_span_id(tid)++ def depth(s: dict[str, Any]) -> int:+ d, p, seen = 0, s.get("parent_step_id"), set()+ while p and p in by_id and p not in seen:+ seen.add(p)+ d, p = d + 1, by_id[p].get("parent_step_id")+ return d++ first_user_text = last_reply_text = None+ for s in sorted(steps, key=lambda s: (depth(s), s.get("seq") or 0)):+ if s.get("kind") == "notice" or (only is not None and s["step_id"] not in only):+ continue+ a = s.get("attrs") or {}+ attrs: dict[str, Any] = dict(common, **{+ "openworker.step.id": s["step_id"], "openworker.step.kind": s["kind"],+ "openworker.step.status": s.get("status") or "unknown", "openworker.step.seq": s.get("seq") or 0})+ if s.get("after_turn"):+ attrs["openworker.after_turn"] = True+ if s.get("shadow"):+ attrs["openworker.shadow"] = True+ call_id = s.get("call_id") or (a.get("for_call_id") if s["kind"] == "wait" else None)+ if call_id:+ # Only a tool's own span carries gen_ai.tool.call.id: viewers type any span with it+ # as a tool (V-22). Waits, checks and the reviewer's model call use our own name.+ attrs["gen_ai.tool.call.id" if s["kind"] == "tool" else "openworker.tool.call_id"] = call_id+ kind = SPAN_INTERNAL+ events: list[dict[str, Any]] = []+ lf_type = "span"+ usage = None+ if s["kind"] == "model":+ model = a.get("model") or ""+ prov, _, short = model.partition(":")+ short = short or model+ prov = a.get("provider") or prov+ facts = a.get("provider_facts") or {}+ name = f"chat {short}"+ kind, lf_type = SPAN_CLIENT, "generation"+ attrs.update({"gen_ai.operation.name": "chat", "gen_ai.provider.name": PROVIDER_NAME.get(prov, prov),+ "gen_ai.request.model": short, "openworker.model": model,+ "openworker.llm.purpose": a.get("purpose") or "unattributed",+ "openworker.stream": bool(a.get("stream"))})+ if facts.get("served_model"):+ attrs["gen_ai.response.model"] = facts["served_model"]+ if facts.get("provider_response_id"):+ attrs["gen_ai.response.id"] = facts["provider_response_id"]+ if facts.get("provider_request_id"):+ attrs["openworker.provider.request_id"] = facts["provider_request_id"]+ if facts.get("served_by") or a.get("served_by"):+ attrs["openworker.provider.served_by"] = facts.get("served_by") or a.get("served_by")+ if a.get("finish_reason"):+ attrs["gen_ai.response.finish_reasons"] = [a["finish_reason"]]+ if a.get("tools_offered") is not None:+ attrs["openworker.tools_offered"] = a["tools_offered"]+ usage = _usage(s)+ attrs["openworker.usage.reported"] = usage is not None+ if usage:+ attrs.update(_semconv_usage(usage))+ if lf:+ attrs["langfuse.observation.usage_details"] = _js(_langfuse_usage(usage))+ if a.get("ttft_ms") is not None and s.get("started_at") is not None:+ first = s["started_at"] + a["ttft_ms"]+ attrs["gen_ai.client.operation.time_to_first_chunk"] = a["ttft_ms"] / 1000+ events.append({"name": "gen_ai.first_chunk", "at": first, "attrs": {}})+ if lf:+ attrs["langfuse.observation.completion_start_time"] = _iso(first)+ err = a.get("error")+ if err:+ attrs["error.type"] = err.get("type") or "error" # never error.text (R5)+ x = exchanges.get(s["step_id"])+ if x is not None:+ wire, req, body = x+ system, msgs, tools = request_messages(wire, req)+ out = response_message(wire, body)+ purpose = a.get("purpose") or ""+ in_ok, out_ok = content_allowed(purpose, sw)+ if sw.get("system_instructions"):+ attrs["gen_ai.system_instructions"] = _js(apply_rules([{"role": "system", "parts": system}],+ {**sw, "context": True}, masker)[0]["parts"])+ attrs["gen_ai.tool.definitions"] = masker.clean(tools)+ if in_ok:+ attrs["gen_ai.input.messages"] = _js(apply_rules(msgs, sw, masker))+ if out_ok:+ attrs["gen_ai.output.messages"] = _js(apply_rules([out], sw, masker))+ if purpose == "main":+ users = [p["content"] for m in msgs if m.get("role") == "user"+ for p in m["parts"] if p.get("type") == "text"]+ first_user_text = first_user_text or (users[-1] if users else None)+ texts = [p["content"] for p in out["parts"] if p.get("type") == "text"]+ last_reply_text = texts[-1] if texts else last_reply_text+ elif s["kind"] == "tool":+ name = f"execute_tool {s.get('name')}"+ lf_type = "tool"+ attrs.update({"gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": s.get("name") or "",+ "gen_ai.tool.type": "function"})+ dec = a.get("decision") or {}+ if dec.get("origin"):+ attrs["openworker.decision.origin"] = dec["origin"]+ if dec.get("path"):+ attrs["openworker.decision.path"] = [f"{p.get('stage')}:{p.get('outcome')}" for p in dec["path"]]+ if dec.get("helper"):+ attrs["openworker.decision.helper"] = True+ for k in ("risk_class", "exit_code", "result_bytes"):+ if a.get(k) is not None:+ attrs[f"openworker.tool.{k}"] = a[k]+ cid = s.get("call_id")+ if sw.get("tool_args") and cid in tool_args:+ attrs["gen_ai.tool.call.arguments"] = masker.clean(tool_args[cid])+ if sw.get("tool_results") and cid in tool_results:+ attrs["gen_ai.tool.call.result"] = masker.clean(tool_results[cid])+ elif s["kind"] == "wait":+ wk = a.get("kind") or "approval"+ name = f"openworker.{wk}.wait"+ for k in ("kind", "where", "outcome", "attended"):+ if a.get(k) is not None:+ attrs[f"openworker.wait.{k}"] = a[k]+ resolved = (s.get("content") or {}).get("resolved_by")+ if sw.get("names") and resolved:+ attrs["openworker.wait.resolved_by"] = masker.clean(resolved)+ elif s["kind"] == "auto_approver":+ name = f"openworker.policy.review {s.get('name')}"+ lf_type = "guardrail"+ rv = a.get("reviewer") or {}+ attrs.update({"openworker.reviewer.kind": rv.get("kind") or "",+ "openworker.reviewer.external": bool(rv.get("external")),+ "openworker.reviewer.mode": a.get("mode") or "",+ "openworker.reviewer.result": a.get("result") or ""})+ if rv.get("model"):+ attrs["openworker.reviewer.model"] = rv["model"]+ for k in ("score", "cutoff"):+ if a.get(k) is not None:+ attrs[f"openworker.reviewer.{k}"] = a[k]+ reason = (s.get("content") or {}).get("reason")+ if reason and sw.get("responses") and sw.get("tool_args"): # R3: it quotes the action+ attrs["openworker.reviewer.reason"] = masker.clean(reason)+ elif s["kind"] == "summary":+ name = "openworker.compaction" if s.get("name") == "summary" else "openworker.compaction.trim"+ for k in ("reason", "tokens_before", "tokens_after"):+ if a.get(k) is not None:+ attrs[f"openworker.compaction.{k}"] = a[k]+ elif s["kind"] == "context":+ name = "openworker.context.build"+ for k in ("size_chars", "session_prompt_hash", "block_hash"):+ if a.get(k) is not None:+ attrs[f"openworker.context.{k}"] = a[k]+ elif s["kind"] == "sandbox":+ name = "openworker.sandbox.setup"+ else:+ name = f"openworker.{s['kind']}"++ status = s.get("status") or "unknown"+ has_end = s.get("duration_ms") is not None+ level = message = None+ if status == "error":+ level = "ERROR"+ message = (a.get("error") or {}).get("type") or "error"+ if s["kind"] == "model":+ message += "; usage unknown"+ elif status in ("denied", "interrupted", "unknown") or not has_end:+ level, message = "WARNING", status if has_end else "unknown"+ elif s["kind"] == "model" and usage is None:+ level, message = "WARNING", "usage not reported"+ if lf:+ attrs["langfuse.observation.type"] = lf_type+ if level:+ attrs["langfuse.observation.level"] = level+ attrs["langfuse.observation.status_message"] = message+ parent = s.get("parent_step_id")+ start = s.get("started_at") if s.get("started_at") is not None else turn["started_at"]+ spans.append({+ "_mark": s["step_id"], "trace_id": trace,+ "span_id": step_span_id(s["step_id"]),+ "parent_span_id": step_span_id(parent) if parent and parent in by_id else root_id,+ "name": name, "kind": kind, "start_ms": start,+ "end_ms": start + s["duration_ms"] if has_end else max(end_ms, start),+ "attrs": attrs, "events": events,+ "error": ((a.get("error") or {}).get("type") or "error") if status == "error" else None,+ })++ if include_root:+ usage = calculator.usage({"steps": steps}, turn_id=tid)+ root_attrs = dict(common, **{+ "gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "coworker",+ "openworker.turn.seq": turn.get("seq") or 0, "openworker.turn.kind": turn.get("kind") or "",+ "openworker.turn.trigger": turn.get("trigger") or "", "openworker.turn.status": turn.get("status") or "",+ "openworker.approval_mode": turn.get("approval_mode") or "",+ "openworker.attended": bool(turn.get("attended")),+ # Never a gen_ai.usage.* name on a parent span: viewers would count it twice.+ "openworker.usage.total_tokens": usage["pill"],+ "openworker.usage.at_least": usage["at_least"],+ "openworker.usage.unreported_calls": usage["unreported_calls"],+ "openworker.usage.failed_calls": usage["failed_calls"],+ "openworker.usage.context_limit_calls": usage["overflow_calls"]})+ woken = turn.get("woken_by") or {}+ if isinstance(woken, dict):+ if woken.get("team_id"):+ root_attrs["openworker.team.id"] = woken["team_id"]+ if woken.get("lead_session_id"):+ root_attrs["openworker.lead.session.id"] = woken["lead_session_id"]+ if sw.get("names") and session_title:+ root_attrs["openworker.session.title"] = masker.clean(session_title)+ if lf:+ root_attrs.update({"langfuse.observation.type": "agent", "langfuse.trace.name": "invoke_agent coworker"})+ if usage["unreported_calls"] or usage["failed_calls"]:+ root_attrs["langfuse.trace.tags"] = ["usage-incomplete"]+ if sw.get("prompts") and first_user_text:+ text = first_user_text if sw.get("context") else cut_context(first_user_text)+ root_attrs["langfuse.trace.input"] = root_attrs["langfuse.observation.input"] = masker.clean(text)+ if sw.get("responses") and last_reply_text:+ root_attrs["langfuse.trace.output"] = root_attrs["langfuse.observation.output"] = masker.clean(last_reply_text)+ events = [{"name": f"openworker.notice.{s.get('name')}", "at": s.get("started_at") or turn["started_at"],+ "attrs": {"openworker.step.id": s["step_id"]}}+ for s in sorted(steps, key=lambda s: s.get("seq") or 0) if s.get("kind") == "notice"]+ spans.insert(0, {+ "_mark": tid, "_notices": [s["step_id"] for s in steps if s.get("kind") == "notice"],+ "trace_id": trace, "span_id": root_id, "parent_span_id": None,+ "name": "invoke_agent coworker", "kind": SPAN_INTERNAL, "start_ms": turn["started_at"],+ "end_ms": end_ms, "attrs": root_attrs, "events": events,+ "error": turn.get("status") if turn.get("status") == "error" else None,+ })+ return spans+++# -- the made-up test recording (E9c) ------------------------------------------------------------+TEST_SESSION = "openworker-test"+++def sample_spans(now_ms: int, dialect: str) -> list[dict[str, Any]]:+ """One trace with one model span and one tool span, fixed made-up values, tagged+ `openworker-test`. Nothing in it comes from a session."""+ turn_id = f"openworker-test-{now_ms}"+ turn = {"turn_id": turn_id, "session_id": TEST_SESSION, "seq": 1, "kind": "run", "trigger": "user",+ "status": "completed", "approval_mode": "ask", "attended": True,+ "started_at": now_ms - 1500, "duration_ms": 1500}+ steps = [+ {"step_id": f"{turn_id}-model", "turn_id": turn_id, "session_id": TEST_SESSION, "seq": 1,+ "kind": "model", "name": "model", "started_at": now_ms - 1400, "duration_ms": 900, "status": "ok",+ "attrs": {"model": "openworker:test-model", "provider": "openworker", "purpose": "main",+ "stream": False, "finish_reason": "tool_calls",+ "usage": {"reported": True, "input_fresh": 100, "cache_read": None, "cache_write": None,+ "output": 10, "reasoning": None}}},+ {"step_id": f"{turn_id}-tool", "turn_id": turn_id, "session_id": TEST_SESSION, "seq": 2,+ "kind": "tool", "name": "test_tool", "call_id": "test-call", "started_at": now_ms - 400,+ "duration_ms": 200, "status": "ok", "attrs": {"result_bytes": 2}},+ ]+ sw = {k: False for k in SWITCHES}+ spans = build_turn_spans(turn, steps, sw=sw, dialect=dialect, get_piece=lambda _k: None, masker=Masker())+ if dialect == "langfuse":+ spans[0]["attrs"]["langfuse.trace.tags"] = [TEST_SESSION]+ return spans+++# -- OTLP/HTTP JSON ------------------------------------------------------------------------------+def _value(v: Any) -> dict[str, Any]:+ if isinstance(v, bool):+ return {"boolValue": v}+ if isinstance(v, int):+ return {"intValue": str(v)}+ if isinstance(v, float):+ return {"doubleValue": v}+ if isinstance(v, (list, tuple)):+ return {"arrayValue": {"values": [_value(x) for x in v]}}+ return {"stringValue": str(v)}+++def _attrs(d: dict[str, Any]) -> list[dict[str, Any]]:+ return [{"key": k, "value": _value(v)} for k, v in d.items() if v is not None]+++def _ns(ms: int) -> str:+ return str(int(ms) * 1_000_000)+++def resource_attrs(*, app_version: str, machine_name: Optional[str] = None) -> dict[str, Any]:+ out = {"service.name": SERVICE_NAME, "service.version": app_version or "unknown",+ "openworker.export.schema": SCHEMA}+ if machine_name: # only with capture.names+ out["host.name"] = machine_name+ return out+++def encode_otlp(groups: list[tuple[dict[str, Any], list[dict[str, Any]]]]) -> dict[str, Any]:+ """`[(resource attributes, spans)]` -> the OTLP/HTTP JSON body."""+ out = []+ for resource, spans in groups:+ enc = []+ for s in spans:+ span = {+ "traceId": s["trace_id"], "spanId": s["span_id"], "name": s["name"], "kind": s["kind"],+ "startTimeUnixNano": _ns(s["start_ms"]), "endTimeUnixNano": _ns(s["end_ms"]),+ "attributes": _attrs(s["attrs"]),+ "events": [{"timeUnixNano": _ns(e["at"]), "name": e["name"], "attributes": _attrs(e["attrs"])}+ for e in s.get("events") or []],+ }+ if s.get("parent_span_id"):+ span["parentSpanId"] = s["parent_span_id"]+ if s.get("error"):+ span["status"] = {"code": STATUS_ERROR, "message": s["error"]}+ enc.append(span)+ out.append({"resource": {"attributes": _attrs(resource)},+ "scopeSpans": [{"scope": {"name": SCOPE_NAME, "version": ATTRIBUTES_VERSION}, "spans": enc}]})+ return {"resourceSpans": out}diff --git a/coworker/observability/store.py b/coworker/observability/store.pyindex 59664091..cb1f9473 100644--- a/coworker/observability/store.py+++ b/coworker/observability/store.py@@ -65,6 +65,12 @@ CREATE TABLE IF NOT EXISTS exchange_pieces (CREATE TABLE IF NOT EXISTS step_pieces (step_id TEXT NOT NULL, sha256 TEXT NOT NULL, PRIMARY KEY (step_id, sha256));CREATE INDEX IF NOT EXISTS step_pieces_sha ON step_pieces(sha256);+CREATE TABLE IF NOT EXISTS export_marks (+ destination TEXT NOT NULL, item_id TEXT NOT NULL, turn_id TEXT, state TEXT NOT NULL, at INTEGER NOT NULL,+ PRIMARY KEY (destination, item_id));+CREATE INDEX IF NOT EXISTS export_marks_turn ON export_marks(turn_id);+CREATE TABLE IF NOT EXISTS export_counters (+ destination TEXT PRIMARY KEY, failed INTEGER NOT NULL DEFAULT 0, dropped INTEGER NOT NULL DEFAULT 0);"""_TURN_COLUMNS = (@@ -195,6 +201,9 @@ class RecordingStore:self._closed = Falseself._last_sweep = 0.0self._background = background+ # OB-6: the destination being exported to and the time it was chosen, so a turn the+ # retention sweep deletes before it was sent is counted as dropped (design §10.5).+ self.export_destination: Optional[tuple[str, int]] = Noneif background:_writer.add(self)@@ -352,6 +361,10 @@ class RecordingStore:"(SELECT step_id FROM steps WHERE session_id = ?)",(session_id,),)+ self._conn.execute(+ "DELETE FROM export_marks WHERE turn_id IN (SELECT turn_id FROM turns WHERE session_id = ?)",+ (session_id,),+ )self._conn.execute("DELETE FROM steps WHERE session_id = ?", (session_id,))self._conn.execute("DELETE FROM turns WHERE session_id = ?", (session_id,))self._delete_unused_pieces()@@ -378,6 +391,8 @@ class RecordingStore:]if ids:marks = ", ".join("?" * len(ids))+ self._count_unsent_dropped(ids, marks)+ self._conn.execute(f"DELETE FROM export_marks WHERE turn_id IN ({marks})", ids)self._conn.execute("DELETE FROM step_pieces WHERE step_id IN "f"(SELECT step_id FROM steps WHERE turn_id IN ({marks}))",@@ -422,6 +437,107 @@ class RecordingStore:breaktime.sleep(SWEEP_PAUSE_S)+ # -- export marks (OB-6, phase-2 design §10.2, §10.5) ----------------------------------------+ # The queue is the store itself: a finished turn's root and steps with no mark for the+ # destination are still to send. A mark is `sent`, `dropped` (refused on its own) or+ # `skipped` (a notice that came after its turn's span had gone). Marks are per destination,+ # so turning export off and on again never re-sends a step.++ def _count_unsent_dropped(self, ids: list[str], marks: str) -> None:+ """Inside the sweep's transaction: turns about to be deleted that ended after the+ destination was chosen and were never sent."""+ dest = self.export_destination+ if dest is None:+ return+ key, since = dest+ n = self._conn.execute(+ f"SELECT COUNT(*) AS n FROM turns t WHERE t.turn_id IN ({marks}) "+ "AND t.status IS NOT NULL AND t.status != 'running' "+ "AND t.started_at + COALESCE(t.duration_ms, 0) >= ? AND NOT EXISTS "+ "(SELECT 1 FROM export_marks m WHERE m.destination = ? AND m.item_id = t.turn_id)",+ (*ids, since, key),+ ).fetchone()["n"]+ if n:+ self._bump(key, dropped=n)++ def _bump(self, destination: str, *, failed: int = 0, dropped: int = 0) -> None:+ self._conn.execute(+ "INSERT INTO export_counters (destination, failed, dropped) VALUES (?, ?, ?) "+ "ON CONFLICT(destination) DO UPDATE SET failed = failed + excluded.failed, "+ "dropped = dropped + excluded.dropped",+ (destination, failed, dropped),+ )++ def count_export(self, destination: str, *, failed: int = 0, dropped: int = 0) -> None:+ with self._lock, self._conn:+ self._bump(destination, failed=failed, dropped=dropped)++ def export_queue(self, destination: str, since_ms: int, limit: int = 50) -> list[str]:+ """Finished turns, ended at or after `since_ms`, with something not yet marked for+ this destination: the turn itself, or a step that has ended (an `after_turn` step+ is sent alone when it ends). Oldest first."""+ self.flush()+ with self._lock:+ rows = self._conn.execute(+ "SELECT t.turn_id FROM turns t "+ "WHERE t.status IS NOT NULL AND t.status != 'running' "+ "AND t.started_at + COALESCE(t.duration_ms, 0) >= ? AND ("+ " NOT EXISTS (SELECT 1 FROM export_marks m WHERE m.destination = ? AND m.item_id = t.turn_id)"+ " OR EXISTS (SELECT 1 FROM steps s WHERE s.turn_id = t.turn_id"+ " AND s.status IS NOT NULL AND s.status != 'running' AND NOT EXISTS"+ " (SELECT 1 FROM export_marks m WHERE m.destination = ? AND m.item_id = s.step_id))) "+ "ORDER BY t.started_at, t.turn_id LIMIT ?",+ (since_ms, destination, destination, limit),+ ).fetchall()+ return [r["turn_id"] for r in rows]++ def export_marked(self, destination: str, turn_id: str) -> set[str]:+ with self._lock:+ rows = self._conn.execute(+ "SELECT item_id FROM export_marks WHERE destination = ? AND turn_id = ?",+ (destination, turn_id),+ ).fetchall()+ return {r["item_id"] for r in rows}++ def mark_export(self, destination: str, items: list[tuple[str, str]], state: str) -> None:+ """`items`: (item id, turn id) pairs. A mark, once written, is kept."""+ now = int(time.time() * 1000)+ with self._lock, self._conn:+ self._conn.executemany(+ "INSERT OR IGNORE INTO export_marks (destination, item_id, turn_id, state, at) "+ "VALUES (?, ?, ?, ?, ?)",+ [(destination, item, turn, state, now) for item, turn in items],+ )++ def export_counts(self, destination: str, *, since_ms: int = 0) -> dict[str, int]:+ """Steps sent (since `since_ms`), steps dropped, and batches failed for good."""+ with self._lock:+ sent = self._conn.execute(+ "SELECT COUNT(*) AS n FROM export_marks WHERE destination = ? AND state = 'sent' AND at >= ?",+ (destination, since_ms),+ ).fetchone()["n"]+ dropped = self._conn.execute(+ "SELECT COUNT(*) AS n FROM export_marks WHERE destination = ? AND state = 'dropped'",+ (destination,),+ ).fetchone()["n"]+ row = self._conn.execute(+ "SELECT failed, dropped FROM export_counters WHERE destination = ?", (destination,)+ ).fetchone()+ return {"sent": int(sent), "dropped": int(dropped) + (int(row["dropped"]) if row else 0),+ "failed": int(row["failed"]) if row else 0}++ def load_turn(self, turn_id: str) -> Optional[tuple[dict[str, Any], list[dict[str, Any]]]]:+ """One turn and its steps (by `seq`), content included."""+ self.flush()+ with self._lock:+ t = self._conn.execute("SELECT * FROM turns WHERE turn_id = ?", (turn_id,)).fetchone()+ if t is None:+ return None+ steps = self._conn.execute(+ "SELECT * FROM steps WHERE turn_id = ? ORDER BY seq IS NULL, seq, started_at", (turn_id,)+ ).fetchall()+ return _turn_record(t), [_step_record(s) for s in steps]+# -- the Issues result (phase-2 design §5): derived, stored beside the turn ---------------def write_issues(self, turn_id: str, result: dict[str, Any], *, seen: bool = False) -> None:"""Store a turn's Issues result. `seen` is false only for a result worked out when thediff --git a/coworker/remote/joiner.py b/coworker/remote/joiner.pyindex fa7c423b..6cb9d417 100644--- a/coworker/remote/joiner.py+++ b/coworker/remote/joiner.py@@ -402,21 +402,28 @@ def apply_policy(app, policy: dict[str, Any], *, state: Optional[Path] = None, exporter=None) -> tuple[bool, int]:"""Persist + apply an org policy DOCUMENT on this box (spec §Versioned org- policy). Today one field acts: `audit_export` (remote/audit_export.py).+ policy). Two fields act: `audit_export` (remote/audit_export.py) and+ `observability_export` (observability/export.py).Unknown keys are kept verbatim so the machine can display them; nothingturns on by accident. A document OLDER than the one held is ignored — astale controller never rolls a machine back. Returns (applied, versionnow held)."""+ from ..observability.export import take_policy_secretsfrom .audit_export import ExportPolicy, save_policyincoming, current = policy_version(policy), policy_version(held_policy(app))if incoming and current and incoming < current:return False, current+ manager = getattr(app.state, "manager", None)+ # OB-6: the observability export's keys go to the secret store, never policy.json.+ policy = take_policy_secrets(policy, getattr(manager, "secrets", None))if state is not None:save_policy(state, policy)- manager = getattr(app.state, "manager", None)if manager is not None:manager.org_policy = dict(policy)+ apply = getattr(manager, "apply_org_policy", None)+ if apply is not None:+ apply(policy)if exporter is not None:exporter.set_policy(ExportPolicy.from_policy(policy))return True, incoming@@ -443,6 +450,9 @@ def _build_exporter(state: Path, app, welcome: dict[str, Any]):stored = load_policy(state)if stored and not getattr(manager, "org_policy", None):manager.org_policy = stored+ apply = getattr(manager, "apply_org_policy", None)+ if apply is not None:+ apply(stored)return exporterdiff --git a/coworker/server/app.py b/coworker/server/app.pyindex 0d11f0af..13ce8a78 100644--- a/coworker/server/app.py+++ b/coworker/server/app.py@@ -23,7 +23,7 @@ from typing import Any, Optionalfrom fastapi import FastAPI, Request, WebSocket, WebSocketDisconnectfrom fastapi.middleware.cors import CORSMiddleware-from fastapi.responses import JSONResponse+from fastapi.responses import JSONResponse, Responselogger = logging.getLogger("coworker.server.app")@@ -852,6 +852,33 @@ def create_app(manager: SessionManager) -> FastAPI:def session_issues_seen(session_id: str) -> dict[str, Any]:return manager.mark_recording_issues_seen(session_id)+ # OB-6: one session's recording as a file, content included (phase-2 design §10.6).+ @app.get("/v1/sessions/{session_id}/recording")+ def session_recording(session_id: str):+ body = json.dumps(manager.recording_file(session_id), ensure_ascii=False)+ safe = re.sub(r"[^A-Za-z0-9_.-]", "_", session_id)[:80] or "session"+ return Response(content=body, media_type="application/json", headers={+ "Content-Disposition": f'attachment; filename="recording-{safe}.json"'})++ # OB-6: Settings ▸ Machine ▸ Observability. Keys are write-only: answers carry masked+ # hints, never a value (phase-2 design §10.8).+ @app.get("/v1/observability/settings")+ def observability_settings() -> dict[str, Any]:+ return manager.observability.get_settings()++ @app.put("/v1/observability/settings")+ def observability_settings_put(body: dict):+ code, answer = manager.observability.put_settings(body or {})+ return JSONResponse(answer, status_code=code)++ @app.post("/v1/observability/test")+ async def observability_test() -> dict[str, Any]:+ return await asyncio.to_thread(manager.observability.send_test)++ @app.get("/v1/observability/status")+ def observability_status() -> dict[str, Any]:+ return manager.observability.status()+@app.patch("/v1/sessions/{session_id}")def session_patch(session_id: str, body: dict) -> dict[str, Any]:body = body or {}@@ -3309,6 +3336,16 @@ def create_app(manager: SessionManager) -> FastAPI:},})+ # OB-6 (TD2-11): the company's policy sends this machine's content somewhere. Each+ # new session says what goes where, once, before its first message.+ export_notice = manager.observability.content_notice()+ if export_notice and not any(+ m.get("role") in ("user", "assistant") or m.get("kind") == "export_notice"+ for m in engine.messages+ ):+ engine._append_notice("export_notice", export_notice)+ manager.save(session_id, engine, touch=False)+ await ws.send_json({"type": "export_notice", "data": {"text": export_notice}})inbound_times: deque[float] = deque()async def reject_input(reason: str) -> None:diff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex 8bba3f7d..d010cd3b 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -46,7 +46,9 @@ from ..audit import AuditStorefrom ..observability import Recorder, RecordingStorefrom ..observability import read_api as recording_apifrom ..observability import signals as recording_signals+from ..observability.export import ObservabilityExportfrom ..observability.provider_view import RecordingProvider, tag as tag_provider+from ..remote.audit_export import load_policyfrom ..config import load_config, workspace_allowed_commandsfrom ..conversations import ConversationStore, title_fromfrom ..engine import ApprovalOutcome, Approver, TurnEngine@@ -431,6 +433,15 @@ class SessionManager:self._autotitle_sig: dict[str, int] = {}self.workspace_trust = WorkspaceTrustStore()self.secrets = SecretStore()+ # OB-6: record on or off, keep for, and sending recordings to Langfuse or an+ # OpenTelemetry endpoint. Settings live in the machine's state folder (beside the+ # secrets), never in a project. Off by default: then no exporter exists at all.+ self.observability = ObservabilityExport(+ self.secrets.path.parent, self.secrets, self.recordings_store,+ session_title=self.session_store.display_title,+ )+ self.observability.apply_policy(load_policy(self.secrets.path.parent))+ self.recorder.add_listener(self._export_on_turn_end)# No explicit provider injected → route by the model's `provider:` prefix (OpenAI default,# Ollama, …). Tests inject a provider directly and bypass the router. The same router is# shared by every engine and the `/v1/chat/completions` proxy.@@ -1019,8 +1030,9 @@ class SessionManager:auto_approve_shadow=self.auto_approve_shadow(),auto_approve_reviewer=self.auto_approve_reviewer(),)- # Observability (E1): this engine's turns and steps go to the server's recorder.- engine.recorder = self.recorder+ # Observability (E1): this engine's turns and steps go to the server's recorder,+ # unless recording is off on this machine (OB-6): then new engines record nothing.+ engine.recorder = self.recorder if self.observability.settings.record else Noneengine.resolver_of = lambda call_id, _sid=session_id: self.inbox.resolver_of(_sid, call_id)engine.delegated_approval = lambda args: self.worker_review_context(session_id, args)engine.reviewer_context = lambda: self.approval_context(session_id)@@ -6379,6 +6391,35 @@ class SessionManager:def recording_usage(self, session_id: str) -> dict[str, Any]:return recording_api.usage(self.recordings_store.load_recording(session_id), session_id)+ # -- observability: export and its settings (OB-6, phase-2 design §10) ------------------+ def _export_on_turn_end(self, event: dict[str, Any]) -> None:+ """A turn ended: the exporter sends soon. A late step (the session title) goes with+ the exporter's next pass, within its flush interval."""+ if event.get("type") == "turn_ended":+ self.observability.wake()++ def apply_org_policy(self, policy: dict[str, Any]) -> None:+ """The org policy changed: its `observability_export` field manages the page."""+ self.observability.apply_policy(policy)++ def recording_file(self, session_id: str) -> dict[str, Any]:+ """Everything recorded for one session, content included, in the shape the read+ endpoints serve: turns, each turn's steps, and each step in full (§10.6)."""+ rec = self.recordings_store.load_recording(session_id)+ messages = self.session_messages(session_id)+ details = {}+ for s in rec.get("steps") or []:+ details[s["step_id"]] = {"step": recording_api.step(+ rec, s["step_id"], get_piece=self.recordings_store.get_piece, messages=messages,+ audit_rows=lambda call_id: self.audit_store.list(session_id=session_id, call_id=call_id))}+ return {+ "format": "openworker-recording", "version": "1.2", "session_id": session_id,+ "exported_at": int(time.time() * 1000),+ "turns": recording_api.turns(rec),+ "steps": {t["turn_id"]: recording_api.turn_steps(rec, t["turn_id"]) for t in rec.get("turns") or []},+ "details": details,+ }+def _forward_recording_event(self, event: dict[str, Any]) -> None:"""Queue one recording event for the session's sockets; a batch goes out every100 ms as `{"type": "step", "events": [...]}`, so a burst of steps does not flood@@ -6517,6 +6558,7 @@ class SessionManager:await self.stop_gateway()await self.mcp.aclose()self.audit_store.close()+ self.observability.close()self.recordings_store.close()from ..providers import llamacpp_setup@@ -7080,8 +7122,9 @@ class SessionManager:[d] if (d := self.persona_skill_scope(task.agent)[0]) is not None else None),)- # Observability (E1): this engine's turns and steps go to the server's recorder.- engine.recorder = self.recorder+ # Observability (E1): this engine's turns and steps go to the server's recorder,+ # unless recording is off on this machine (OB-6): then new engines record nothing.+ engine.recorder = self.recorder if self.observability.settings.record else Noneengine.resolver_of = lambda call_id, _sid=session_id: self.inbox.resolver_of(_sid, call_id)self._seed_task_permissions(engine, task)return enginediff --git a/tests/test_observability_export.py b/tests/test_observability_export.pynew file mode 100644index 00000000..843e892c--- /dev/null+++ b/tests/test_observability_export.py@@ -0,0 +1,618 @@+"""OB-6: sending recordings to Langfuse or an OpenTelemetry endpoint (phase-2 design §10).++A recording is built in a real store with content planted everywhere a real session puts it:+the user's text, OpenWorker's per-turn context block, the system prompt and tool list, the+AI's answer and its thinking, a tool call's arguments and result (inside the next call's+history too), the auto-approver's input, answer and reason, a summary, the session title+call, a failed call's error text and cost, who answered a card, the machine's name, the+session title, and two secrets. Every planted text holds `PLANT`.++- Metadata only: no planted text in anything sent; a value planted in an exported field is+ sent (the leak test of phase-2/lab/leak_test.py, as a repo test).+- Each switch alone lets out only its own content, and rules R1 to R5 hold.+- Sending: each step once per destination (marks survive a restart), retry and back-off,+ a refused key, a refused span, against a fake OTLP endpoint on 127.0.0.1.+"""++from __future__ import annotations++import json+import threading+import time+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from pathlib import Path++import pytest++from coworker.observability import export as ex+from coworker.observability import export_spans as sp+from coworker.observability.capture import store_exchange+from coworker.observability.store import RecordingStore++VECTORS = Path(__file__).parent / "corpora" / "observability" / "golden-phase2" / "vectors" / "masking.json"++STORED_SECRET = "hunter2-correct-horse-PLANT"+KEY_SHAPED = "sk-PLANTabcdefghijklmnopqrstu"+T0 = 1_790_000_000_000+++# -- a recording with content planted everywhere ------------------------------------------------+def _exchange(store: RecordingStore, request: dict, response: dict) -> dict:+ return store_exchange({+ "method": "POST", "url": "https://api.anthropic.com/v1/messages",+ "request_headers": {"content-type": "application/json"},+ "request_body": json.dumps(request).encode("utf-8"),+ "status": 200, "response_headers": {}, "body": json.dumps(response),+ }, store.put_piece)+++def _reply(*blocks: dict, stop: str = "end_turn") -> dict:+ return {"type": "message", "role": "assistant", "content": list(blocks), "stop_reason": stop}+++def _usage(i: int = 100, o: int = 10) -> dict:+ return {"reported": True, "input_fresh": i, "cache_read": 0, "cache_write": None, "output": o, "reasoning": None}+++def _model(sid, tid, step_id, seq, at, purpose, exchange, *, parent=None, call_id=None, after_turn=False, status="ok",+ model="anthropic:claude-EXPORTED-zq81", **attrs):+ a = {"model": model, "provider": "anthropic", "purpose": purpose, "stream": False,+ "finish_reason": "end_turn", "tools_offered": 1, "usage": _usage(),+ "provider_facts": {"cost_reported": 98765.4321, "served_model": "claude-x"}}+ a.update(attrs)+ return {"step_id": step_id, "turn_id": tid, "session_id": sid, "parent_step_id": parent, "seq": seq,+ "kind": "model", "name": "model", "started_at": at, "duration_ms": 400, "status": status,+ "call_id": call_id, "after_turn": after_turn, "attrs": a,+ "content": {"exchange": [exchange]} if exchange else {}}+++def plant_turn(store: RecordingStore, sid: str = "sess-1", tid: str = "turn-1", at: int = T0,+ *, long_text: str = "") -> None:+ user = (f"USER-PLANT-u1 my key {KEY_SHAPED} and {STORED_SECRET} {long_text}"+ "<system-context>CTX-PLANT-folders C:\\Users\\PLANT-home</system-context>")+ system = "SYSTEM-PLANT-instructions"+ tools = [{"name": "run_shell", "description": "TOOLDEF-PLANT", "input_schema": {"type": "object"}}]+ m1 = _exchange(store, {"model": "claude-x", "system": system, "tools": tools,+ "messages": [{"role": "user", "content": user}]},+ _reply({"type": "thinking", "thinking": "THINK-PLANT-reasoning"},+ {"type": "text", "text": "ASSIST-PLANT-a1"},+ {"type": "tool_use", "id": "c1", "name": "run_shell", "input": {"command": "ARGS-PLANT-cmd"}},+ stop="tool_use"))+ rv = _exchange(store, {"model": "claude-x", "system": "You review actions.",+ "messages": [{"role": "user", "content": "REVIEWER-IN-PLANT run_shell ARGS-PLANT-cmd"}]},+ _reply({"type": "text", "text": "REVIEWER-OUT-PLANT allow"}))+ m2 = _exchange(store, {"model": "claude-x", "system": system, "tools": tools, "messages": [+ {"role": "user", "content": user},+ {"role": "assistant", "content": [{"type": "tool_use", "id": "c1", "name": "run_shell",+ "input": {"command": "ARGS-PLANT-cmd"}}]},+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "c1", "content": "RESULT-PLANT-out"}]}]},+ _reply({"type": "text", "text": "ASSIST-PLANT-a2 final"}))+ sm = _exchange(store, {"model": "claude-x", "messages": [+ {"role": "user", "content": "SUMMARY-IN-PLANT summarise RESULT-PLANT-out"}]},+ _reply({"type": "text", "text": "SUMMARY-OUT-PLANT"}))+ ti = _exchange(store, {"model": "claude-x", "messages": [{"role": "user", "content": "TITLE-IN-PLANT"}]},+ _reply({"type": "text", "text": "TITLE-OUT-PLANT"}))+ failed = _exchange(store, {"model": "claude-x", "messages": [{"role": "user", "content": user}]}, {})+ store.write_turn({+ "turn_id": tid, "session_id": sid, "seq": 1, "kind": "run", "trigger": "user", "started_at": at,+ "duration_ms": 9000, "status": "completed", "approval_mode": "ask", "attended": True,+ "stamp": {"machine": {"name": "PLANT-machine-name", "app_version": "9.9.9"}}, "schema_version": 1,+ })+ steps = [+ {"step_id": f"{tid}-ctx", "kind": "context", "name": "context", "seq": 1, "started_at": at, "duration_ms": 1,+ "status": "ok", "attrs": {"size_chars": 120, "block_hash": "abc"},+ "content": {"agents_md": "AGENTS-PLANT"}},+ _model(sid, tid, f"{tid}-m1", 2, at + 10, "main", m1),+ {"step_id": f"{tid}-aa", "kind": "auto_approver", "name": "run_shell", "seq": 3, "started_at": at + 500,+ "duration_ms": 300, "status": "ok", "call_id": "c1",+ "attrs": {"mode": "live", "result": "allow", "reviewer": {"kind": "session_model", "model": "anthropic:claude-x"}},+ "content": {"reason": "REVIEWER-REASON-PLANT", "questions": [{"name": "QUESTIONS-PLANT"}]}},+ _model(sid, tid, f"{tid}-rm", 4, at + 510, "auto_approver", rv, parent=f"{tid}-aa", call_id="c1"),+ {"step_id": f"{tid}-w", "kind": "wait", "name": "approval", "seq": 5, "started_at": at + 900,+ "duration_ms": 1000, "status": "ok",+ "attrs": {"kind": "approval", "where": "session", "outcome": "approved", "attended": True, "for_call_id": "c1"},+ "content": {"resolved_by": "PLANT-person-bob"}},+ {"step_id": f"{tid}-t", "kind": "tool", "name": "run_shell", "seq": 6, "started_at": at + 2000,+ "duration_ms": 500, "status": "ok", "call_id": "c1",+ "attrs": {"risk_class": "exec", "exit_code": 0, "result_bytes": 16,+ "decision": {"origin": "user", "path": [{"stage": "permission", "outcome": "asked"}]}},+ "content": {"decision_details": "DECISION-PLANT", "blocked": "BLOCKED-PLANT"}},+ _model(sid, tid, f"{tid}-m2", 7, at + 2600, "main", m2),+ {"step_id": f"{tid}-sum", "kind": "summary", "name": "summary", "seq": 8, "started_at": at + 3200,+ "duration_ms": 900, "status": "ok", "attrs": {"reason": "overflow", "tokens_after": 700}},+ _model(sid, tid, f"{tid}-sm", 9, at + 3210, "summary", sm, parent=f"{tid}-sum"),+ _model(sid, tid, f"{tid}-err", 10, at + 4200, "main", failed, status="error",+ error={"type": "rate_limit", "text": "ERRTEXT-PLANT https://PLANT.example/v1"}, usage={"reported": False}),+ {"step_id": f"{tid}-n", "kind": "notice", "name": "interrupted", "seq": 11, "started_at": at + 4700,+ "duration_ms": 0, "status": "ok", "attrs": {}},+ _model(sid, tid, f"{tid}-title", 12, at + 8000, "title", ti, after_turn=True),+ ]+ for s in steps:+ s.setdefault("turn_id", tid)+ s.setdefault("session_id", sid)+ store.write_step(s)+ store.flush()+++class Captured:+ """A `post` for the exporter that keeps what it was given and answers 200."""++ def __init__(self, answers=None):+ self.bodies: list[dict] = []+ self.headers: list[dict] = []+ self.answers = list(answers or [])++ def __call__(self, url, body, headers, timeout):+ self.bodies.append(json.loads(body))+ self.headers.append(headers)+ return self.answers.pop(0) if self.answers else (200, "")++ def text(self) -> str:+ return json.dumps(self.bodies, ensure_ascii=False)++ def spans(self) -> list[dict]:+ return [s for b in self.bodies for r in b["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]+++def _attr(span: dict, key: str):+ for a in span["attributes"]:+ if a["key"] == key:+ v = a["value"]+ return next(iter(v.values()))+ return None+++def export_once(tmp_path, include: dict, *, dest=None, long_text: str = "") -> Captured:+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ plant_turn(store, long_text=long_text)+ cap = Captured()+ cfg = ex.ExportConfig(destination=dest or ex.Destination(kind="langfuse", host="http://127.0.0.1:9"),+ headers={"Authorization": "Basic x"}, include=include, since_ms=0)+ exporter = ex.Exporter(store, cfg, post=cap, secret_values=lambda: [STORED_SECRET],+ session_title=lambda _sid: "TITLE-PLANT-session")+ exporter.flush_once()+ store.close()+ return cap+++OFF = {"messages": False, "tool_io": False, "context": False, "names": False}+++def on(*keys: str) -> dict:+ return {**OFF, **{k: True for k in keys}}+++# -- 2. metadata only carries no content --------------------------------------------------------------[email protected]("dest", [ex.Destination(kind="langfuse", host="http://127.0.0.1:9"),+ ex.Destination(kind="otlp", url="http://127.0.0.1:9/v1/traces")])+def test_metadata_only_export_carries_no_content(tmp_path, dest):+ cap = export_once(tmp_path, OFF, dest=dest)+ sent = cap.text()+ assert len(cap.spans()) == 12 # the turn and 11 steps; the notice is an event on the turn+ # The positive control: a value planted in an exported field is found by the same search.+ assert "claude-EXPORTED-zq81" in sent+ # Nothing planted in content is.+ assert "PLANT" not in sent+ assert "98765" not in sent # provider-reported cost (R5)+ assert "C:\\\\Users" not in sent and "Users" not in sent+ # Metadata is there: names, kinds, counts, the verdict, the notice.+ spans = cap.spans()+ names = {s["name"] for s in spans}+ assert {"invoke_agent coworker", "openworker.context.build", "chat claude-EXPORTED-zq81",+ "openworker.policy.review run_shell", "openworker.approval.wait", "execute_tool run_shell",+ "openworker.compaction"} <= names+ review = next(s for s in spans if s["name"] == "openworker.policy.review run_shell")+ assert _attr(review, "openworker.reviewer.result") == "allow"+ root = next(s for s in spans if s["name"] == "invoke_agent coworker")+ assert [e["name"] for e in root["events"]] == ["openworker.notice.interrupted"]+ failed = next(s for s in spans if _attr(s, "openworker.step.id") == "turn-1-err")+ assert _attr(failed, "error.type") == "rate_limit" and failed["status"]["code"] == 2+++# -- 3. each switch lets out only its own content --------------------------------------------------------+ALL_PLANTS = ["USER-PLANT", "ASSIST-PLANT", "CTX-PLANT", "SYSTEM-PLANT", "TOOLDEF-PLANT", "ARGS-PLANT",+ "RESULT-PLANT", "REVIEWER-IN-PLANT", "REVIEWER-OUT-PLANT", "REVIEWER-REASON-PLANT",+ "SUMMARY-IN-PLANT", "SUMMARY-OUT-PLANT", "TITLE-IN-PLANT", "TITLE-OUT-PLANT",+ "TITLE-PLANT-session", "PLANT-person-bob", "PLANT-machine-name"]+NEVER = ["THINK-PLANT", "ERRTEXT-PLANT", "98765", "QUESTIONS-PLANT", "DECISION-PLANT", "AGENTS-PLANT",+ "BLOCKED-PLANT", STORED_SECRET, KEY_SHAPED]++CASES = {+ # R1, R2, R3: the user's and the AI's own text, the title call; no tool parts, no context+ # block, no reviewer or summary text.+ "messages": (on("messages"), {"USER-PLANT", "ASSIST-PLANT", "TITLE-IN-PLANT", "TITLE-OUT-PLANT"}),+ # Tool spans only: the reviewer's input and reason need the messages switch too (R3).+ "tool_io": (on("tool_io"), {"ARGS-PLANT", "RESULT-PLANT"}),+ "context": (on("context"), {"SYSTEM-PLANT", "TOOLDEF-PLANT"}),+ "names": (on("names"), {"TITLE-PLANT-session", "PLANT-person-bob", "PLANT-machine-name"}),+ "messages+tool_io": (on("messages", "tool_io"),+ {"USER-PLANT", "ASSIST-PLANT", "ARGS-PLANT", "RESULT-PLANT", "REVIEWER-IN-PLANT",+ "REVIEWER-OUT-PLANT", "REVIEWER-REASON-PLANT", "SUMMARY-IN-PLANT", "SUMMARY-OUT-PLANT",+ "TITLE-IN-PLANT", "TITLE-OUT-PLANT"}),+ "messages+context": (on("messages", "context"),+ {"USER-PLANT", "ASSIST-PLANT", "CTX-PLANT", "SYSTEM-PLANT", "TOOLDEF-PLANT",+ "TITLE-IN-PLANT", "TITLE-OUT-PLANT"}),+ "everything": (on("messages", "tool_io", "context", "names"), set(ALL_PLANTS)),+}++[email protected]("case", list(CASES))+def test_each_switch_lets_out_only_its_content(tmp_path, case):+ include, allowed = CASES[case]+ sent = export_once(tmp_path, include).text()+ for plant in ALL_PLANTS:+ assert (plant in sent) == (plant in allowed), f"{case}: {plant}"+ for plant in NEVER: # R5 and R4, whatever the switches say+ assert plant not in sent, f"{case}: {plant}"+++def test_tool_parts_inside_messages_are_masked_unless_the_tool_switch_is_on(tmp_path):+ cap = export_once(tmp_path, on("messages"))+ main = [s for s in cap.spans() if _attr(s, "openworker.step.id") == "turn-1-m2"][0]+ history = json.loads(_attr(main, "gen_ai.input.messages"))+ parts = [p for m in history for p in m["parts"]]+ call = next(p for p in parts if p["type"] == "tool_call")+ result = next(p for p in parts if p["type"] == "tool_call_response")+ # R1: the part, its id and its tool name stay; its content does not.+ assert (call["id"], call["name"], call["arguments"]) == ("c1", "run_shell", sp.TOOL_ARGS_OUT)+ assert (result["id"], result["response"]) == ("c1", sp.TOOL_RESULT_OUT)+ # R2: the context block is cut out of the user's text, the rest kept.+ user = next(p["content"] for m in history if m["role"] == "user" for p in m["parts"] if p["type"] == "text")+ assert user.startswith("USER-PLANT-u1") and user.endswith(sp.CONTEXT_OUT)+ # R4: the stored secret and the key-shaped one are masked.+ assert user.count(sp.SECRET) == 2+ # Reasoning keeps its marker, never its text (R5).+ reply = json.loads(_attr([s for s in cap.spans() if _attr(s, "openworker.step.id") == "turn-1-m1"][0],+ "gen_ai.output.messages"))+ assert {"type": "reasoning", "content": sp.REASONING} in reply[0]["parts"]+ # The turn's own input and output (Langfuse).+ root = cap.spans()[0]+ assert _attr(root, "langfuse.trace.input").endswith(sp.CONTEXT_OUT)+ assert _attr(root, "langfuse.trace.output") == "ASSIST-PLANT-a2 final"+++def test_tool_switch_with_messages_lets_tool_parts_into_messages(tmp_path):+ cap = export_once(tmp_path, on("messages", "tool_io"))+ main = [s for s in cap.spans() if _attr(s, "openworker.step.id") == "turn-1-m2"][0]+ history = _attr(main, "gen_ai.input.messages")+ assert "ARGS-PLANT-cmd" in history and "RESULT-PLANT-out" in history+ tool = [s for s in cap.spans() if s["name"] == "execute_tool run_shell"][0]+ assert json.loads(_attr(tool, "gen_ai.tool.call.arguments")) == {"command": "ARGS-PLANT-cmd"}+ assert _attr(tool, "gen_ai.tool.call.result") == "RESULT-PLANT-out"+ # gen_ai.tool.call.id only on the tool's own span (V-22).+ assert [s["name"] for s in cap.spans() if _attr(s, "gen_ai.tool.call.id")] == ["execute_tool run_shell"]+++def test_exported_text_is_cut_at_8_kb(tmp_path):+ cap = export_once(tmp_path, on("messages"), long_text="x" * 20_000)+ main = [s for s in cap.spans() if _attr(s, "openworker.step.id") == "turn-1-m1"][0]+ user = json.loads(_attr(main, "gen_ai.input.messages"))[0]["parts"][0]["content"]+ assert user.endswith(" more bytes]") and "[cut: " in user+ kept = user[: user.index("[cut: ")]+ assert len(kept.encode("utf-8")) == sp.CUT_BYTES+++def test_masking_and_cut_match_the_shared_vectors():+ vectors = json.loads(VECTORS.read_text(encoding="utf-8"))+ for v in vectors["mask"]:+ assert sp.Masker(v["secrets"]).mask(v["text"]) == v["expected"], v["text"]+ for v in vectors["cut"]:+ text = "a" * 8191 + "é" + "b" * 10+ assert len(text.encode("utf-8")) == v["text_bytes"]+ out = sp.cut(text)+ kept = out[: out.index("[cut:")]+ assert len(kept.encode("utf-8")) == v["expected_kept_bytes"]+ assert out.endswith(v["expected_tail"])+ # The patterns are the ones in attributes.yaml export.content_switches.masking.+ assert sp.CUT_BYTES == 8192 and len(sp.MASK_PATTERNS) == 7+++def test_credential_looking_environment_variables_are_masked():+ env = {"MY_API_KEY": "PLANT-envkey-123", "SHORT_TOKEN": "abc", "HOME": "PLANT-not-a-secret"}+ m = sp.Masker(sp.env_secret_values(env))+ assert m.mask("a PLANT-envkey-123 b abc PLANT-not-a-secret") == "a [secret masked] b abc PLANT-not-a-secret"+++def test_token_numbers_follow_the_verified_rules(tmp_path):+ cap = export_once(tmp_path, OFF)+ spans = cap.spans()+ m1 = [s for s in spans if _attr(s, "openworker.step.id") == "turn-1-m1"][0]+ assert _attr(m1, "gen_ai.usage.input_tokens") == "100"+ assert _attr(m1, "gen_ai.usage.cache_read.input_tokens") == "0"+ assert _attr(m1, "gen_ai.usage.cache_creation.input_tokens") is None # not reported: left out+ assert json.loads(_attr(m1, "langfuse.observation.usage_details")) == {"input": 100, "input_cache_read": 0, "output": 10}+ root = spans[0]+ assert _attr(root, "gen_ai.usage.input_tokens") is None # never on a parent span+ assert _attr(root, "openworker.usage.failed_calls") == "1"+ assert _attr(root, "langfuse.trace.tags") == {"values": [{"stringValue": "usage-incomplete"}]}+ err = [s for s in spans if _attr(s, "openworker.step.id") == "turn-1-err"][0]+ assert _attr(err, "langfuse.observation.level") == "ERROR"+ assert _attr(err, "langfuse.observation.status_message") == "rate_limit; usage unknown"+ # One trace per turn, children under their parents.+ assert len({s["traceId"] for s in spans}) == 1+ rm = [s for s in spans if _attr(s, "openworker.step.id") == "turn-1-rm"][0]+ aa = [s for s in spans if _attr(s, "openworker.step.id") == "turn-1-aa"][0]+ assert rm["parentSpanId"] == aa["spanId"] and aa["parentSpanId"] == root["spanId"]+ resource = cap.bodies[0]["resourceSpans"][0]["resource"]["attributes"]+ assert {"key": "openworker.export.schema", "value": {"stringValue": "openworker/1.2"}} in resource+ assert not any(a["key"] == "host.name" for a in resource) # the machine name: names switch only+++# -- 4. each step once; retry; refused key; refused span ------------------------------------------------+class FakeOtlp:+ """A local OTLP/HTTP endpoint. `answers` is a list of status codes for the next posts;+ `refuse` is a set of span ids refused (400) whenever they are in a post."""++ def __init__(self):+ self.answers: list[int] = []+ self.refuse: set[str] = set()+ self.received: list[dict] = []+ self.auth: list[str] = []+ outer = self++ class Handler(BaseHTTPRequestHandler):+ def do_POST(self): # noqa: N802+ body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))+ outer.auth.append(self.headers.get("Authorization") or "")+ ids = [s["spanId"] for r in body["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]+ code = outer.answers.pop(0) if outer.answers else 200+ if code == 200 and outer.refuse & set(ids):+ code = 400+ if code == 200:+ outer.received.append(body)+ self.send_response(code)+ self.end_headers()+ self.wfile.write(b'{"message":"refused here"}' if code != 200 else b"{}")++ def log_message(self, *a):+ pass++ self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ self.url = f"http://127.0.0.1:{self.server.server_address[1]}/v1/traces"+ threading.Thread(target=self.server.serve_forever, daemon=True).start()++ def span_ids(self) -> list[str]:+ return [s["spanId"] for b in self.received for r in b["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]++ def close(self):+ self.server.shutdown()+++def otlp():+ f = FakeOtlp()+ yield f+ f.close()+++def _cfg(url: str, since: int = 0) -> ex.ExportConfig:+ return ex.ExportConfig(destination=ex.Destination(kind="otlp", url=url), headers={"Authorization": "Bearer t"},+ include=OFF, since_ms=since)+++def test_each_step_is_sent_once_and_the_marks_survive_a_restart(tmp_path, otlp):+ db = tmp_path / "recordings.db"+ store = RecordingStore(db, background=False)+ plant_turn(store, tid="turn-1")+ plant_turn(store, tid="turn-2", at=T0 + 20_000)+ exporter = ex.Exporter(store, _cfg(otlp.url))+ assert exporter.flush_once() == 24+ assert exporter.flush_once() == 0 # nothing new+ store.close()+ # A restart: a new store on the same file, a new exporter.+ store = RecordingStore(db, background=False)+ assert ex.Exporter(store, _cfg(otlp.url)).flush_once() == 0+ ids = otlp.span_ids()+ assert len(ids) == len(set(ids)) == 24+ # A step that ends after its turn was sent (a late title call) is sent alone, once.+ store.write_step({"step_id": "turn-1-late", "turn_id": "turn-1", "session_id": "sess-1", "seq": 13,+ "kind": "model", "name": "model", "started_at": T0 + 9500, "duration_ms": 100, "status": "ok",+ "after_turn": True, "attrs": {"model": "anthropic:claude-x", "purpose": "title", "usage": _usage()}})+ store.flush()+ assert ex.Exporter(store, _cfg(otlp.url)).flush_once() == 1+ assert len(otlp.span_ids()) == 25+ assert store.export_counts("otlp:" + otlp.url)["sent"] == 2 * 12 + 2 * 1 + 1 # notices ride with their turn+ store.close()+++def test_a_new_destination_starts_from_turns_that_end_after_the_change(tmp_path, otlp):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ plant_turn(store, tid="old", at=T0)+ plant_turn(store, tid="new", at=T0 + 60_000)+ ex.Exporter(store, _cfg(otlp.url, since=T0 + 30_000)).flush_once()+ assert {s["traceId"] for b in otlp.received for r in b["resourceSpans"] for ss in r["scopeSpans"]+ for s in ss["spans"]} == {sp.trace_id("new")}+ store.close()+++def _wait_for(cond, timeout=10.0):+ end = time.time() + timeout+ while time.time() < end:+ if cond():+ return True+ time.sleep(0.02)+ return False+++def test_retry_with_back_off_then_sent(tmp_path, otlp):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ plant_turn(store)+ otlp.answers = [503, 503, 200]+ exporter = ex.Exporter(store, _cfg(otlp.url), backoff_start=0.05, flush_seconds=0.05)+ seen: list[tuple[str, str]] = []+ exporter.start()+ try:+ assert _wait_for(lambda: seen.append((exporter.state, exporter.last_error)) or len(otlp.span_ids()) == 12)+ finally:+ exporter.stop()+ assert ("retrying", "HTTP 503: {\"message\":\"refused here\"}") in seen+ assert exporter.state == "sending" and exporter.last_error == ""+ store.close()+++def test_a_network_error_is_retried(tmp_path):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ plant_turn(store)+ exporter = ex.Exporter(store, _cfg("http://127.0.0.1:1/v1/traces"), backoff_start=0.05)+ with pytest.raises(ex._Retry):+ exporter.flush_once()+ assert store.export_marked("otlp:http://127.0.0.1:1/v1/traces", "turn-1") == set()+ store.close()+++def test_a_refused_key_stops_sending_until_the_settings_change(tmp_path, otlp):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ plant_turn(store)+ otlp.answers = [401]+ exporter = ex.Exporter(store, _cfg(otlp.url), backoff_start=0.05, flush_seconds=0.05)+ exporter.start()+ try:+ assert _wait_for(lambda: exporter.state == "failed")+ time.sleep(0.2)+ assert otlp.span_ids() == [] # it stopped: no second try with the same key+ assert exporter.last_error == "The destination refused the key"+ assert store.export_counts("otlp:" + otlp.url)["failed"] == 1+ finally:+ exporter.stop()+ store.close()+++def test_a_span_refused_on_its_own_is_dropped_and_the_rest_sent(tmp_path, otlp):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ plant_turn(store)+ otlp.refuse = {sp.step_span_id("turn-1-t")}+ ex.Exporter(store, _cfg(otlp.url)).flush_once()+ ids = otlp.span_ids()+ assert len(ids) == 11 and sp.step_span_id("turn-1-t") not in ids+ assert store.export_counts("otlp:" + otlp.url)["dropped"] == 1+ store.close()+++def test_a_turn_deleted_before_it_was_sent_counts_as_dropped(tmp_path):+ store = RecordingStore(tmp_path / "recordings.db", background=False, retention_days=7)+ plant_turn(store, at=T0)+ store.export_destination = ("otlp:http://x/v1/traces", 0)+ store.sweep(now_ms=T0 + 8 * 24 * 3600 * 1000)+ assert store.export_counts("otlp:http://x/v1/traces")["dropped"] == 1+ store.close()+++def test_batches_hold_at_most_the_batch_size(tmp_path):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ for i in range(3):+ plant_turn(store, tid=f"t{i}", at=T0 + i * 20_000)+ cap = Captured()+ ex.Exporter(store, _cfg("http://127.0.0.1:9/v1/traces"), post=cap, batch_spans=5).flush_once()+ sizes = [len([s for r in b["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]) for b in cap.bodies]+ assert max(sizes) <= 5 and sum(sizes) == 36+ store.close()+++# -- settings, keys and the test recording ------------------------------------------------------------+class FakeSecrets:+ def __init__(self):+ self.data: dict = {}++ def get(self, profile):+ return self.data.get(profile)++ def put(self, profile, value):+ self.data[profile] = json.loads(json.dumps(value))++ def delete(self, profile):+ return self.data.pop(profile, None) is not None++ def _read(self):+ return self.data+++def test_settings_keep_keys_out_of_the_settings_file_and_answers(tmp_path):+ secrets = FakeSecrets()+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ obs = ex.ObservabilityExport(tmp_path, secrets, store, post=Captured())+ try:+ assert obs.exporter is None and obs.status()["state"] == "off"+ code, answer = obs.put_settings({+ "destination": {"kind": "langfuse", "host": "https://langfuse.example.com"},+ "secrets": {"public_key": "pk-lf-PLANT-public-3a91", "secret_key": "sk-lf-PLANT-secret-key"},+ "include": {"messages": True}})+ assert code == 200 and answer["ok"]+ assert answer["secrets"] == {"public_key": "pk-lf-••••••••3a91", "secret_key": "sk-lf-••••••••"}+ assert "PLANT" not in json.dumps(answer) and "PLANT" not in json.dumps(obs.get_settings())+ assert "PLANT" not in (tmp_path / ex.SETTINGS_FILE).read_text(encoding="utf-8")+ assert secrets.data[ex.PROFILE]["user"]["secret_key"] == "sk-lf-PLANT-secret-key"+ assert obs.exporter is not None and obs.exporter.config.destination.endpoint == \+ "https://langfuse.example.com/api/public/otel/v1/traces"+ # Off again: the exporter goes away, its thread with it.+ obs.put_settings({"destination": {"kind": "off"}})+ assert obs.exporter is None+ assert not any(t.name == "observability-export" for t in threading.enumerate())+ # Bad input is refused, not saved.+ assert obs.put_settings({"destination": {"kind": "otlp", "url": "ftp://x"}})[0] == 400+ assert obs.put_settings({"retention_days": 12})[0] == 400+ finally:+ obs.close()+ store.close()+++def test_the_test_recording_reports_a_refused_key(tmp_path, otlp):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ obs = ex.ObservabilityExport(tmp_path, FakeSecrets(), store)+ try:+ obs.put_settings({"destination": {"kind": "otlp", "url": otlp.url, "header_names": ["Authorization"]},+ "secrets": {"headers": {"Authorization": "Bearer PLANT-token"}}})+ otlp.answers = [200]+ ok = obs.send_test()+ assert ok["ok"] and ok["text"] == "Sent." and ok["status"] == 200+ assert otlp.auth[-1] == "Bearer PLANT-token"+ [span_names] = [[s["name"] for r in b["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]+ for b in otlp.received[-1:]]+ assert span_names == ["invoke_agent coworker", "chat test-model", "execute_tool test_tool"]+ otlp.answers = [401]+ refused = obs.send_test()+ assert refused == {"ok": False, "status": 401, "text": "The destination refused the key", "ms": refused["ms"]}+ otlp.answers = [500]+ assert obs.send_test()["text"].startswith("HTTP 500")+ finally:+ obs.close()+ store.close()+++POLICY = {"org_id": "org_1", "org_name": "Acme Corp", "version": 3, "observability_export": {+ "enabled": True, "name": "Security team default", "set_at": "2026-10-12",+ "destination": {"kind": "langfuse", "host": "https://langfuse.acme.internal",+ "public_key": "pk-lf-POLICY-PLANT-0001", "secret_key": "sk-lf-POLICY-PLANT"},+ "include": {"messages": True, "names": True}}}+++def test_a_company_policy_manages_the_page(tmp_path):+ secrets = FakeSecrets()+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ obs = ex.ObservabilityExport(tmp_path, secrets, store, post=Captured())+ try:+ stripped = ex.take_policy_secrets(POLICY, secrets)+ assert "PLANT" not in json.dumps(stripped) # what policy.json will hold+ assert secrets.data[ex.PROFILE]["policy"]["secret_key"] == "sk-lf-POLICY-PLANT"+ obs.apply_policy(stripped)+ got = obs.get_settings()+ assert got["managed"] == {"by": "Acme Corp", "name": "Security team default", "at": "2026-10-12"}+ assert got["destination"] == {"kind": "langfuse", "host": "https://langfuse.acme.internal"}+ assert got["include"] == {"messages": True, "tool_io": False, "context": False, "names": True}+ assert got["notice"] == "Acme Corp receives this session's messages and names (Langfuse, langfuse.acme.internal)."+ assert "PLANT" not in json.dumps(got)+ assert obs.put_settings({"include": {"tool_io": True}})[0] == 409+ assert obs.put_settings({"destination": {"kind": "off"}})[0] == 409+ assert obs.put_settings({"retention_days": 7})[0] == 200 # on this machine stays the person's+ assert obs.exporter is not None and obs.exporter.config.managed+ assert obs.status()["state"] in ("managed", "sending", "retrying")+ # Anything malformed reads as not managed.+ for bad in ({"observability_export": {"enabled": "yes"}},+ {"observability_export": {"enabled": True, "destination": {"kind": "smtp"}}},+ {"observability_export": []}):+ assert ex.managed_export(bad) is None+ # The policy taken away: the person's own settings are back, the policy's keys gone.+ obs.apply_policy(ex.take_policy_secrets({"org_id": "org_1"}, secrets))+ assert obs.get_settings()["managed"] is None and "policy" not in (secrets.data.get(ex.PROFILE) or {})+ finally:+ obs.close()+ store.close()diff --git a/tests/test_observability_export_server.py b/tests/test_observability_export_server.pynew file mode 100644index 00000000..4af2cc4a--- /dev/null+++ b/tests/test_observability_export_server.py@@ -0,0 +1,141 @@+"""OB-6 through the real server: the settings, status and test endpoints, the managed state,+the session notice, Record steps off, the recording file, and Off meaning no exporter and no+connection at all (phase-2 design §10.3, §10.6 to §10.8)."""++from __future__ import annotations++import json+import threading++from fastapi.testclient import TestClient++from coworker.observability import export as ex+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient+from coworker.providers.base import TokenUsage+from coworker.remote import joiner+from coworker.server import SessionManager, create_app+++class _Scripted(ProviderClient):+ def complete(self, *, model, messages, tools=None, **settings):+ if str(messages[0].get("content") or "").startswith("You title"):+ return AssistantTurn(text="Notes", usage=TokenUsage(input=40, output=2))+ return AssistantTurn(text="Done.", finish_reason="stop", usage=TokenUsage(input=20, output=5))++ def capabilities(self, model):+ return ModelCapabilities()+++def _turn(client, sid: str, text: str = "hello") -> None:+ with client.websocket_connect(f"/ws/session/{sid}") as ws:+ assert ws.receive_json()["type"] in ("ready", "export_notice")+ ws.send_json({"type": "user_message", "text": text})+ while ws.receive_json()["type"] != "turn_done":+ pass+++def _app(tmp_path, monkeypatch, posts: list):+ def post(url, body, headers, timeout):+ posts.append(url)+ return 200, ""++ # Every send the exporter or the test button could make goes through `post`; with Off it+ # must never be called.+ manager = SessionManager(workspace=tmp_path, provider=_Scripted())+ manager.observability.post = post+ return manager, TestClient(create_app(manager))+++def test_off_creates_no_exporter_and_sends_nothing(tmp_path, monkeypatch):+ posts: list = []+ manager, client = _app(tmp_path, monkeypatch, posts)+ _turn(client, "s1")+ assert client.get("/v1/sessions/s1/turns").json()["turns"] # recording works+ assert manager.observability.exporter is None+ assert not any(t.name == "observability-export" for t in threading.enumerate())+ assert client.get("/v1/observability/status").json()["state"] == "off"+ test = client.post("/v1/observability/test").json()+ assert test["ok"] is False and posts == []+++def test_settings_endpoints_are_write_only_for_keys(tmp_path, monkeypatch):+ posts: list = []+ manager, client = _app(tmp_path, monkeypatch, posts)+ got = client.get("/v1/observability/settings").json()+ assert got == {"record": True, "retention_days": 30, "destination": {"kind": "off"},+ "include": {"messages": False, "tool_io": False, "context": False, "names": False},+ "secrets": {}, "managed": None, "notice": None}+ r = client.put("/v1/observability/settings", json={+ "destination": {"kind": "otlp", "url": "http://127.0.0.1:9/v1/traces", "header_names": ["x-honeycomb-team"]},+ "secrets": {"headers": {"x-honeycomb-team": "PLANT-header-value"}}, "retention_days": 7})+ assert r.status_code == 200+ body = r.json()+ assert body["secrets"] == {"headers": {"x-honeycomb-team": "••••••••"}}+ for answer in (body, client.get("/v1/observability/settings").json(), client.get("/v1/observability/status").json()):+ assert "PLANT" not in json.dumps(answer)+ state = manager.secrets.path.parent+ assert "PLANT" not in (state / ex.SETTINGS_FILE).read_text(encoding="utf-8")+ assert manager.secrets.get(ex.PROFILE)["user"]["headers"] == {"x-honeycomb-team": "PLANT-header-value"}+ assert manager.recordings_store.retention_days == 7+ assert manager.observability.exporter is not None+ # A turn after the change is sent; the status counts it.+ _turn(client, "s2")+ manager.observability.exporter.flush_once()+ status = client.get("/v1/observability/status").json()+ assert status["state"] == "sending" and status["sent"] > 0 and status["last_sent_at"]+ assert set(status) >= {"state", "last_sent_at", "sent", "failed", "dropped", "last_error"}+ assert posts and all(u == "http://127.0.0.1:9/v1/traces" for u in posts)+ manager.observability.close()+++def test_a_company_policy_makes_the_page_read_only_and_tells_new_sessions(tmp_path, monkeypatch):+ posts: list = []+ manager, client = _app(tmp_path, monkeypatch, posts)+ policy = {"org_id": "org_1", "org_name": "Acme Corp", "version": 2, "observability_export": {+ "enabled": True, "name": "Security team default", "set_at": "2026-10-12",+ "destination": {"kind": "langfuse", "host": "https://langfuse.acme.internal",+ "public_key": "pk-lf-PLANT-0001", "secret_key": "sk-lf-PLANT"},+ "include": {"messages": True}}}++ class _App:+ class state: # noqa: N801+ pass+ _App.state.manager = manager+ state = manager.secrets.path.parent+ applied, _ = joiner.apply_policy(_App, policy, state=state)+ assert applied+ assert "PLANT" not in (state / "policy.json").read_text(encoding="utf-8")+ got = client.get("/v1/observability/settings").json()+ assert got["managed"]["by"] == "Acme Corp" and got["destination"]["host"] == "https://langfuse.acme.internal"+ assert "PLANT" not in json.dumps(got)+ r = client.put("/v1/observability/settings", json={"include": {"tool_io": True}})+ assert r.status_code == 409 and r.json()["error"] == "Managed by Acme Corp"+ # Each new session says what goes where, once.+ with client.websocket_connect("/ws/session/fresh") as ws:+ msgs = [ws.receive_json() for _ in range(2)]+ notice = next(m for m in msgs if m["type"] == "export_notice")+ assert notice["data"]["text"] == "Acme Corp receives this session's messages (Langfuse, langfuse.acme.internal)."+ kinds = [m.get("kind") for m in client.get("/v1/sessions/fresh/messages").json()["messages"]]+ assert kinds.count("export_notice") == 1+ manager.observability.close()+++def test_record_steps_off_records_nothing_for_new_sessions(tmp_path, monkeypatch):+ manager, client = _app(tmp_path, monkeypatch, [])+ assert client.put("/v1/observability/settings", json={"record": False}).status_code == 200+ _turn(client, "quiet")+ assert client.get("/v1/sessions/quiet/turns").json()["turns"] == []+ assert client.get("/v1/observability/settings").json()["record"] is False+++def test_the_recording_file_holds_the_whole_session(tmp_path, monkeypatch):+ manager, client = _app(tmp_path, monkeypatch, [])+ _turn(client, "s9", "what is in notes?")+ r = client.get("/v1/sessions/s9/recording")+ assert r.status_code == 200+ assert 'attachment; filename="recording-s9.json"' == r.headers["content-disposition"]+ data = r.json()+ assert data["session_id"] == "s9" and data["version"] == "1.2"+ [turn] = data["turns"]+ steps = data["steps"][turn["turn_id"]]+ assert steps and all(s["step_id"] in data["details"] for s in steps)