Skip to content
Python

feat(observability): OB-6 export of recordings to Langfuse or an OTLP endpoint

Devika Verma0937e5dd630c
diff --git a/coworker/conversations.py b/coworker/conversations.py
index a52db466..9144d0c1 100644
--- a/coworker/conversations.py
+++ b/coworker/conversations.py
@@ -635,6 +635,15 @@ class ConversationStore:
self._conn.commit()
return cur.rowcount > 0
+ def display_title(self, session_id: str) -> Optional[str]:
+ """The title the sidebar shows, or None when the session has no row yet."""
+ with self._lock:
+ row = self._conn.execute(
+ "SELECT title, auto_title, renamed FROM sessions WHERE session_id = ?",
+ (session_id,),
+ ).fetchone()
+ return None if row is None else _display_title(row)
+
def title_state(self, session_id: str) -> Optional[dict]:
"""The auto-title guard inputs: whether the user renamed and whether a generated
title already exists. None when the session has no row yet."""
diff --git a/coworker/observability/export.py b/coworker/observability/export.py
new file mode 100644
index 00000000..bc5ee3ec
--- /dev/null
+++ b/coworker/observability/export.py
@@ -0,0 +1,682 @@
+"""Sending recordings to Langfuse or an OpenTelemetry endpoint (OB-6, phase-2 design §10).
+
+Runs on the machine that ran the work, beside the recording store. Off by default: with no
+destination the exporter is not created, so there is no thread, no DNS lookup and no socket.
+
+- `ExportSettings`: the page's settings, in `observability.json` in the machine's state folder
+ (never in a project's files, never in `config.toml`). Keys and header values are not here:
+ they live in the app's `SecretStore`, profile `observability-export`.
+- `managed_export`: the org policy's `observability_export` field. When a policy sets it, the
+ policy's destination and switches are used and the page is read-only (TD2-11).
+- `Exporter`: one thread that reads finished turns from the store, builds their spans
+ (`export_spans.py`), sends them in batches and marks each step sent for the destination.
+ The queue is the store: a restart resumes from the marks, and nothing is sent twice.
+- `ObservabilityExport`: what the server holds. It answers the settings, status and test
+ endpoints and creates or stops the exporter when the settings or the policy change.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import threading
+import time
+from dataclasses import dataclass, field, replace
+from pathlib import Path
+from typing import Any, Callable, Optional
+from urllib.parse import urlparse
+
+from . import export_spans as spans_mod
+from .export_spans import Masker, build_turn_spans, encode_otlp, resource_attrs, switches_from
+
+log = logging.getLogger(__name__)
+
+SETTINGS_FILE = "observability.json"
+PROFILE = "observability-export"
+LANGFUSE_PATH = "/api/public/otel/v1/traces"
+KINDS = ("off", "langfuse", "otlp")
+RETENTION_CHOICES = (7, 30, 90)
+MAX_HEADERS = 8
+INCLUDE_KEYS = ("messages", "tool_io", "context", "names")
+
+BATCH_SPANS = 200
+FLUSH_SECONDS = 5.0
+BACKOFF_START = 1.0
+BACKOFF_CAP = 300.0
+TIMEOUT_SECONDS = 20.0
+
+REFUSED_KEY = "The destination refused the key"
+NOT_FOUND = "Address not found"
+
+
+# -- what is chosen ----------------------------------------------------------------------------
+@dataclass(frozen=True)
+class Destination:
+ kind: str = "off"
+ host: str = "" # Langfuse
+ url: str = "" # OpenTelemetry endpoint
+ header_names: tuple[str, ...] = () # OpenTelemetry endpoint; values are secrets
+
+ @property
+ def endpoint(self) -> str:
+ if self.kind == "langfuse":
+ return self.host.rstrip("/") + LANGFUSE_PATH
+ return self.url if self.kind == "otlp" else ""
+
+ @property
+ def dialect(self) -> str:
+ return "langfuse" if self.kind == "langfuse" else "semconv"
+
+ @property
+ def key(self) -> str:
+ """Identifies the destination for the sent marks: kind and address."""
+ return f"{self.kind}:{self.endpoint}" if self.kind != "off" else ""
+
+ @property
+ def place(self) -> str:
+ """The address's host, for the page and the session notice."""
+ return urlparse(self.endpoint).netloc or self.endpoint
+
+ def public(self) -> dict[str, Any]:
+ out: dict[str, Any] = {"kind": self.kind}
+ if self.kind == "langfuse":
+ out["host"] = self.host
+ elif self.kind == "otlp":
+ out["url"] = self.url
+ out["header_names"] = list(self.header_names)
+ return out
+
+
+def _include(raw: Any) -> dict[str, bool]:
+ raw = raw if isinstance(raw, dict) else {}
+ return {k: bool(raw.get(k)) for k in INCLUDE_KEYS}
+
+
+def _http_url(value: Any) -> str:
+ v = str(value or "").strip()
+ return v if urlparse(v).scheme in ("http", "https") and urlparse(v).netloc else ""
+
+
+def parse_destination(raw: Any) -> Optional[Destination]:
+ """A destination from settings or a policy; None when it does not make sense."""
+ if not isinstance(raw, dict):
+ return None
+ kind = str(raw.get("kind") or "off").lower()
+ if kind == "off":
+ return Destination()
+ if kind == "langfuse":
+ host = _http_url(raw.get("host"))
+ return Destination(kind="langfuse", host=host.rstrip("/")) if host else None
+ if kind == "otlp":
+ url = _http_url(raw.get("url"))
+ names = raw.get("header_names")
+ if names is None and isinstance(raw.get("headers"), dict):
+ names = list(raw["headers"])
+ names = [str(n).strip() for n in (names or []) if str(n).strip()]
+ if not url or len(names) > MAX_HEADERS:
+ return None
+ return Destination(kind="otlp", url=url, header_names=tuple(dict.fromkeys(names)))
+ return None
+
+
+@dataclass
+class ExportSettings:
+ record: bool = True
+ retention_days: int = 30
+ destination: Destination = field(default_factory=Destination)
+ include: dict[str, bool] = field(default_factory=lambda: _include({}))
+ # When each destination was chosen: only turns that end after it are sent (TD2-12).
+ since: dict[str, int] = field(default_factory=dict)
+
+ @classmethod
+ def load(cls, path: Path) -> "ExportSettings":
+ try:
+ raw = json.loads(Path(path).read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ return cls()
+ if not isinstance(raw, dict):
+ return cls()
+ days = raw.get("retention_days")
+ return cls(
+ record=raw.get("record", True) is not False,
+ retention_days=days if days in RETENTION_CHOICES else 30,
+ destination=parse_destination(raw.get("destination")) or Destination(),
+ include=_include(raw.get("include")),
+ since={str(k): int(v) for k, v in (raw.get("since") or {}).items() if isinstance(v, (int, float))},
+ )
+
+ def save(self, path: Path) -> None:
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ data = {"record": self.record, "retention_days": self.retention_days,
+ "destination": self.destination.public(), "include": self.include, "since": self.since}
+ tmp = path.with_name(path.name + ".tmp")
+ tmp.write_text(json.dumps(data, indent=2), encoding="utf-8")
+ tmp.replace(path)
+
+
+# -- company control (TD2-11) ------------------------------------------------------------------
+@dataclass
+class Managed:
+ by: str
+ name: str
+ at: str
+ destination: Destination
+ include: dict[str, bool]
+
+
+POLICY_FIELD = "observability_export"
+_POLICY_SECRET_KEYS = ("public_key", "secret_key", "headers")
+
+
+def managed_export(policy: Any) -> Optional[Managed]:
+ """The org policy's `observability_export` field, read like `audit_export`: anything
+ malformed reads as not set, so governance never turns on by accident.
+
+ Shape: `{"enabled": true, "name": "...", "set_at": "...", "destination": {"kind":
+ "langfuse", "host": ...} | {"kind": "otlp", "url": ..., "header_names": [...]} |
+ {"kind": "off"}, "include": {"messages", "tool_io", "context", "names"}}`. The keys
+ and header values travel in the same field and are moved to the secret store
+ (`take_policy_secrets`) before the policy is saved."""
+ raw = policy.get(POLICY_FIELD) if isinstance(policy, dict) else None
+ if not isinstance(raw, dict) or raw.get("enabled") is not True:
+ return None
+ dest = parse_destination(raw.get("destination"))
+ if dest is None:
+ return None
+ by = str(policy.get("org_name") or policy.get("org_id") or "your organisation")
+ return Managed(by=by, name=str(raw.get("name") or ""), at=str(raw.get("set_at") or ""),
+ destination=dest, include=_include(raw.get("include")))
+
+
+def take_policy_secrets(policy: dict[str, Any], secrets: Any) -> dict[str, Any]:
+ """Move the policy's export keys into the secret store (owned by the policy) and return
+ the policy without them, so `policy.json` never holds them. A policy without the field
+ removes the keys it had stored."""
+ if not isinstance(policy, dict):
+ return policy
+ raw = policy.get(POLICY_FIELD)
+ held = dict(secrets.get(PROFILE) or {}) if secrets is not None else {}
+ if not isinstance(raw, dict):
+ if secrets is not None and "policy" in held:
+ held.pop("policy")
+ _put_profile(secrets, held)
+ return policy
+ dest = raw.get("destination") if isinstance(raw.get("destination"), dict) else {}
+ found = {k: dest[k] for k in _POLICY_SECRET_KEYS if k in dest}
+ if secrets is not None and found:
+ if isinstance(found.get("headers"), dict):
+ found["headers"] = {str(k): str(v) for k, v in found["headers"].items()}
+ dest = {**dest, "header_names": list(found["headers"])}
+ held["policy"] = {k: v for k, v in found.items()}
+ _put_profile(secrets, held)
+ clean_dest = {k: v for k, v in dest.items() if k not in _POLICY_SECRET_KEYS}
+ return {**policy, POLICY_FIELD: {**raw, "destination": clean_dest}}
+
+
+def _put_profile(secrets: Any, data: dict[str, Any]) -> None:
+ if data:
+ secrets.put(PROFILE, data)
+ else:
+ secrets.delete(PROFILE)
+
+
+def _mask_hint(value: str, *, keep_end: bool) -> str:
+ if not value:
+ return ""
+ prefix = value[:6] if value[:3] in ("pk-", "sk-") else ""
+ tail = value[-4:] if keep_end and len(value) >= 16 else ""
+ return f"{prefix}{'•' * 8}{tail}"
+
+
+# -- sending -----------------------------------------------------------------------------------
+Post = Callable[[str, bytes, dict[str, str], float], tuple[int, str]]
+
+
+def http_post(url: str, body: bytes, headers: dict[str, str], timeout: float) -> tuple[int, str]:
+ """POST one OTLP/HTTP JSON body. A network error is status 0 with its kind as the text
+ (never the URL or anything from the request)."""
+ import httpx
+
+ try:
+ with httpx.Client(timeout=timeout) as client:
+ resp = client.post(url, content=body, headers=headers)
+ return resp.status_code, resp.text[:500]
+ except httpx.TimeoutException:
+ return 0, "timed out"
+ except httpx.HTTPError as exc:
+ return 0, type(exc).__name__
+
+
+class _Retry(Exception):
+ pass
+
+
+class _Refused(Exception):
+ pass
+
+
+@dataclass
+class ExportConfig:
+ destination: Destination
+ headers: dict[str, str] # auth and the endpoint's own headers: secrets
+ include: dict[str, bool]
+ since_ms: int
+ managed: bool = False
+
+
+def _reason(status: int, text: str) -> str:
+ if status == 0:
+ return f"no answer ({text})" if text else "no answer"
+ return f"HTTP {status}" + (f": {text.strip()[:200]}" if text.strip() else "")
+
+
+class Exporter:
+ """Reads finished turns from the store and sends them, each step once."""
+
+ def __init__(self, store: Any, config: ExportConfig, *, post: Post = http_post,
+ secret_values: Callable[[], list[str]] = list,
+ session_title: Callable[[str], Optional[str]] = lambda _sid: None,
+ batch_spans: int = BATCH_SPANS, flush_seconds: float = FLUSH_SECONDS,
+ backoff_start: float = BACKOFF_START, backoff_cap: float = BACKOFF_CAP) -> None:
+ self.store = store
+ self.config = config
+ self.post = post
+ self.secret_values = secret_values
+ self.session_title = session_title
+ self.batch_spans = batch_spans
+ self.flush_seconds = flush_seconds
+ self.backoff_start, self.backoff_cap = backoff_start, backoff_cap
+ self.state = "sending"
+ self.last_error = ""
+ self.last_sent_at: Optional[float] = None
+ self._wake = threading.Event()
+ self._stop = threading.Event()
+ self._thread: Optional[threading.Thread] = None
+ store.export_destination = (config.destination.key, config.since_ms)
+
+ # -- the thread ------------------------------------------------------------------------------
+ def start(self) -> None:
+ self._thread = threading.Thread(target=self._run, name="observability-export", daemon=True)
+ self._thread.start()
+
+ def wake(self) -> None:
+ self._wake.set()
+
+ def stop(self, timeout: float = 5.0) -> None:
+ self._stop.set()
+ self._wake.set()
+ if self._thread is not None and self._thread is not threading.current_thread():
+ self._thread.join(timeout)
+ if getattr(self.store, "export_destination", None) == (self.config.destination.key, self.config.since_ms):
+ self.store.export_destination = None
+
+ def set_include(self, include: dict[str, bool]) -> None:
+ """Switches changed: applied per turn, when the turn is sent (§10.9)."""
+ self.config = replace(self.config, include=dict(include))
+
+ def _run(self) -> None:
+ backoff = self.backoff_start
+ while not self._stop.is_set():
+ try:
+ self.flush_once()
+ backoff = self.backoff_start
+ self.state, self.last_error = "sending", ""
+ self._wake.wait(self.flush_seconds)
+ self._wake.clear()
+ except _Retry as exc:
+ self.state, self.last_error = "retrying", str(exc)
+ self._stop.wait(backoff)
+ backoff = min(backoff * 2, self.backoff_cap)
+ except _Refused as exc:
+ # Stops until the settings change (a new exporter is made then).
+ self.state, self.last_error = "failed", str(exc)
+ self.store.count_export(self.config.destination.key, failed=1)
+ self._stop.wait()
+ except Exception as exc: # noqa: BLE001 - never let the thread die quietly
+ log.exception("observability export: unexpected failure")
+ self.state, self.last_error = "retrying", type(exc).__name__
+ self._stop.wait(backoff)
+ backoff = min(backoff * 2, self.backoff_cap)
+
+ # -- one pass --------------------------------------------------------------------------------
+ def flush_once(self) -> int:
+ """Send everything ready, in batches. Returns the spans sent. Raises `_Retry` (keep
+ the batch, back off) or `_Refused` (stop until the settings change)."""
+ cfg = self.config
+ dest = cfg.destination.key
+ sw = switches_from(cfg.include)
+ masker = Masker(self.secret_values()) if any(sw.values()) else Masker()
+ sent = 0
+ done: set[str] = set() # turns with nothing left to send in this pass
+ while not self._stop.is_set():
+ queue = [t for t in self.store.export_queue(dest, cfg.since_ms, limit=50) if t not in done]
+ if not queue:
+ return sent
+ pending: list[tuple[dict[str, Any], list[tuple[str, str]]]] = []
+ for tid in queue:
+ built = self._turn_spans(tid, sw, masker)
+ if not built:
+ done.add(tid)
+ pending.extend(built)
+ for i in range(0, len(pending), self.batch_spans):
+ sent += self._send(pending[i:i + self.batch_spans])
+ if not pending:
+ return sent
+
+ def _turn_spans(self, turn_id: str, sw: dict[str, bool], masker: Masker) -> list[tuple[dict[str, Any], list[tuple[str, str]]]]:
+ cfg = self.config
+ dest = cfg.destination.key
+ loaded = self.store.load_turn(turn_id)
+ if loaded is None:
+ return []
+ turn, steps = loaded
+ marked = self.store.export_marked(dest, turn_id)
+ ready = {s["step_id"] for s in steps
+ if s.get("status") not in (None, "running") and s["step_id"] not in marked}
+ include_root = turn_id not in marked
+ late_notices = [s["step_id"] for s in steps if s.get("kind") == "notice" and s["step_id"] in ready]
+ if not include_root and late_notices:
+ # A notice is an event on the turn's span, which has gone already.
+ self.store.mark_export(dest, [(n, turn_id) for n in late_notices], "skipped")
+ title = self.session_title(turn["session_id"]) if sw.get("names") else None
+ built = build_turn_spans(turn, steps, sw=sw, dialect=cfg.destination.dialect,
+ get_piece=self.store.get_piece, masker=masker,
+ include_root=include_root, only=ready, session_title=title)
+ stamp = turn.get("stamp") or {}
+ machine = stamp.get("machine") or {}
+ resource = resource_attrs(app_version=str(machine.get("app_version") or ""),
+ machine_name=str(machine.get("name") or "") if sw.get("names") else None)
+ out = []
+ for span in built:
+ marks = [(span["_mark"], turn_id)] + [(n, turn_id) for n in span.get("_notices") or []]
+ out.append(({**span, "_resource": resource}, marks))
+ return out
+
+ def _encode(self, items: list[tuple[dict[str, Any], list[tuple[str, str]]]]) -> bytes:
+ groups: dict[str, tuple[dict[str, Any], list[dict[str, Any]]]] = {}
+ for span, _ in items:
+ res = span["_resource"]
+ groups.setdefault(json.dumps(res, sort_keys=True), (res, []))[1].append(span)
+ return json.dumps(encode_otlp(list(groups.values())), ensure_ascii=False).encode("utf-8")
+
+ def _post(self, items: list[tuple[dict[str, Any], list[tuple[str, str]]]]) -> tuple[int, str]:
+ headers = {"Content-Type": "application/json", **self.config.headers}
+ return self.post(self.config.destination.endpoint, self._encode(items), headers, TIMEOUT_SECONDS)
+
+ def _send(self, batch: list[tuple[dict[str, Any], list[tuple[str, str]]]]) -> int:
+ status, text = self._post(batch)
+ if 200 <= status < 300:
+ self._mark(batch, "sent")
+ return len(batch)
+ self._raise_for(status, text)
+ # 400 and the other 4xx: split the batch once; a span refused on its own is dropped.
+ sent = 0
+ for item in batch:
+ status, text = self._post([item])
+ if 200 <= status < 300:
+ self._mark([item], "sent")
+ sent += 1
+ continue
+ self._raise_for(status, text)
+ self._mark([item], "dropped")
+ self.last_error = _reason(status, text)
+ log.info("observability export: a span was refused on its own (%s)", status)
+ return sent
+
+ def _raise_for(self, status: int, text: str) -> None:
+ if status in (401, 403):
+ raise _Refused(REFUSED_KEY)
+ if status == 404:
+ raise _Refused(NOT_FOUND)
+ if status == 0 or status == 429 or status >= 500:
+ raise _Retry(_reason(status, text))
+
+ def _mark(self, batch: list[tuple[dict[str, Any], list[tuple[str, str]]]], state: str) -> None:
+ marks = [m for _, ms in batch for m in ms]
+ self.store.mark_export(self.config.destination.key, marks, state)
+ if state == "sent":
+ self.last_sent_at = time.time()
+
+
+# -- what the server holds ---------------------------------------------------------------------
+class ObservabilityExport:
+ """Settings, company control, status and the exporter's life (design §10.5 to §10.8)."""
+
+ def __init__(self, state: Path, secrets: Any, store: Any, *,
+ session_title: Callable[[str], Optional[str]] = lambda _sid: None,
+ post: Post = http_post, exporter_options: Optional[dict[str, Any]] = None,
+ clock: Callable[[], float] = time.time) -> None:
+ self.path = Path(state) / SETTINGS_FILE
+ self.secrets = secrets
+ self.store = store
+ self.session_title = session_title
+ self.post = post
+ self.exporter_options = dict(exporter_options or {})
+ self.clock = clock
+ self.settings = ExportSettings.load(self.path)
+ self.managed: Optional[Managed] = None
+ self.exporter: Optional[Exporter] = None
+ self._lock = threading.RLock()
+ if store is not None:
+ store.retention_days = self.settings.retention_days
+
+ # -- what is in force --------------------------------------------------------------------------
+ def effective(self) -> tuple[Destination, dict[str, bool]]:
+ if self.managed is not None:
+ return self.managed.destination, self.managed.include
+ return self.settings.destination, self.settings.include
+
+ def _held(self) -> dict[str, Any]:
+ data = (self.secrets.get(PROFILE) if self.secrets is not None else None) or {}
+ return data if isinstance(data, dict) else {}
+
+ def _keys(self) -> dict[str, Any]:
+ """The keys for the destination in force: the policy's when it manages the page."""
+ held = self._held()
+ own = held.get("policy" if self.managed is not None else "user") or {}
+ return own if isinstance(own, dict) else {}
+
+ def _headers(self, dest: Destination) -> dict[str, str]:
+ keys = self._keys()
+ if dest.kind == "langfuse":
+ import base64
+
+ pair = f"{keys.get('public_key') or ''}:{keys.get('secret_key') or ''}"
+ return {"Authorization": "Basic " + base64.b64encode(pair.encode()).decode()}
+ if dest.kind == "otlp":
+ values = keys.get("headers") if isinstance(keys.get("headers"), dict) else {}
+ return {n: str(values[n]) for n in dest.header_names if values.get(n)}
+ return {}
+
+ def secret_values(self) -> list[str]:
+ """Masking (R4): every value in the secret store, and credential-looking env vars."""
+ data = self.secrets._read() if self.secrets is not None and hasattr(self.secrets, "_read") else {}
+ return spans_mod.store_secret_values(data) + spans_mod.env_secret_values()
+
+ def content_notice(self) -> Optional[str]:
+ """When the company's policy sends content: what goes where, for each new session."""
+ if self.managed is None or self.managed.destination.kind == "off":
+ return None
+ inc = self.managed.include
+ what = [w for k, w in (("messages", "messages"), ("tool_io", "tool inputs and results"),
+ ("context", "the context OpenWorker adds"), ("names", "names")) if inc.get(k)]
+ if not what:
+ return None
+ listed = what[0] if len(what) == 1 else ", ".join(what[:-1]) + " and " + what[-1]
+ dest = self.managed.destination
+ where = "Langfuse" if dest.kind == "langfuse" else "OpenTelemetry"
+ return f"{self.managed.by} receives this session's {listed} ({where}, {dest.place})."
+
+ # -- the exporter's life ------------------------------------------------------------------------
+ def reconcile(self) -> None:
+ """Create, keep or stop the exporter for what is in force. Off: none at all."""
+ with self._lock:
+ dest, include = self.effective()
+ if dest.kind == "off" or not self.settings.record:
+ self._stop_exporter()
+ return
+ since = self._since(dest)
+ cfg = ExportConfig(destination=dest, headers=self._headers(dest), include=dict(include),
+ since_ms=since, managed=self.managed is not None)
+ old = self.exporter
+ if old is not None and old.state != "failed" and old.config.destination == dest and old.config.headers == cfg.headers \
+ and old.config.since_ms == since:
+ old.set_include(include)
+ old.wake()
+ return
+ self._stop_exporter()
+ self.exporter = Exporter(self.store, cfg, post=self.post, secret_values=self.secret_values,
+ session_title=self.session_title, **self.exporter_options)
+ self.exporter.start()
+
+ def _since(self, dest: Destination) -> int:
+ """When this destination was chosen: older turns are not sent (TD2-12)."""
+ since = self.settings.since.get(dest.key)
+ if since is None:
+ since = int(self.clock() * 1000)
+ self.settings.since[dest.key] = since
+ self.settings.save(self.path)
+ return since
+
+ def _stop_exporter(self) -> None:
+ if self.exporter is not None:
+ self.exporter.stop()
+ self.exporter = None
+
+ def wake(self) -> None:
+ """A turn ended: send soon."""
+ ex = self.exporter
+ if ex is not None:
+ ex.wake()
+
+ def close(self) -> None:
+ with self._lock:
+ self._stop_exporter()
+
+ # -- the policy -----------------------------------------------------------------------------------
+ def apply_policy(self, policy: Optional[dict[str, Any]]) -> None:
+ with self._lock:
+ self.managed = managed_export(policy or {})
+ self.reconcile()
+
+ # -- GET/PUT settings (§10.8) ---------------------------------------------------------------------
+ def get_settings(self) -> dict[str, Any]:
+ dest, include = self.effective()
+ keys = self._keys()
+ secrets: dict[str, Any] = {}
+ if dest.kind == "langfuse":
+ secrets = {"public_key": _mask_hint(str(keys.get("public_key") or ""), keep_end=True),
+ "secret_key": _mask_hint(str(keys.get("secret_key") or ""), keep_end=False)}
+ elif dest.kind == "otlp":
+ values = keys.get("headers") if isinstance(keys.get("headers"), dict) else {}
+ secrets = {"headers": {n: ("•" * 8 if values.get(n) else "") for n in dest.header_names}}
+ out = {
+ "record": self.settings.record,
+ "retention_days": self.settings.retention_days,
+ "destination": dest.public(),
+ "include": dict(include),
+ "secrets": secrets, # masked hints only: values never leave
+ "managed": None,
+ "notice": self.content_notice(),
+ }
+ if self.managed is not None:
+ out["managed"] = {"by": self.managed.by, "name": self.managed.name, "at": self.managed.at}
+ return out
+
+ def put_settings(self, body: dict[str, Any]) -> tuple[int, dict[str, Any]]:
+ """Returns (HTTP status, answer). 409 when the page is managed by a policy."""
+ if not isinstance(body, dict):
+ return 400, {"ok": False, "error": "expected an object"}
+ with self._lock:
+ s = self.settings
+ if "record" in body:
+ s.record = bool(body["record"])
+ if "retention_days" in body:
+ if body["retention_days"] not in RETENTION_CHOICES:
+ return 400, {"ok": False, "error": "retention_days must be 7, 30 or 90"}
+ s.retention_days = int(body["retention_days"])
+ if self.store is not None:
+ self.store.retention_days = s.retention_days
+ touches_export = any(k in body for k in ("destination", "include", "secrets"))
+ if touches_export and self.managed is not None:
+ self.settings.save(self.path)
+ return 409, {"ok": False, "error": f"Managed by {self.managed.by}"}
+ if "destination" in body:
+ dest = parse_destination(body["destination"])
+ if dest is None:
+ return 400, {"ok": False, "error": "check the address (http or https) and at most 8 headers"}
+ if dest != s.destination:
+ s.destination = dest
+ if dest.kind != "off": # a new choice starts from now (TD2-12)
+ s.since[dest.key] = int(self.clock() * 1000)
+ if "include" in body:
+ s.include = _include({**s.include, **(body["include"] if isinstance(body["include"], dict) else {})})
+ if isinstance(body.get("secrets"), dict):
+ self._put_user_secrets(body["secrets"], s.destination)
+ s.save(self.path)
+ self.reconcile()
+ return 200, {"ok": True, **self.get_settings()}
+
+ def _put_user_secrets(self, given: dict[str, Any], dest: Destination) -> None:
+ held = self._held()
+ user = dict(held.get("user") or {})
+ for k in ("public_key", "secret_key"):
+ v = given.get(k)
+ if isinstance(v, str) and v.strip():
+ user[k] = v.strip()
+ if isinstance(given.get("headers"), dict):
+ headers = dict(user.get("headers") or {})
+ for n, v in given["headers"].items():
+ if isinstance(v, str) and v:
+ headers[str(n)] = v
+ user["headers"] = headers
+ if dest.kind == "otlp" and isinstance(user.get("headers"), dict):
+ # A header taken off the list takes its value with it.
+ user["headers"] = {n: v for n, v in user["headers"].items() if n in dest.header_names}
+ held["user"] = user
+ _put_profile(self.secrets, {k: v for k, v in held.items() if v})
+
+ # -- status (§10.5) ----------------------------------------------------------------------------------
+ def status(self) -> dict[str, Any]:
+ dest, _ = self.effective()
+ ex = self.exporter
+ if dest.kind == "off" or ex is None:
+ return {"state": "off", "last_sent_at": None, "sent": 0, "sent_today": 0, "failed": 0,
+ "dropped": 0, "last_error": "", "destination": dest.public()}
+ midnight = time.mktime(time.localtime(self.clock())[:3] + (0, 0, 0, 0, 0, -1)) * 1000
+ counts = self.store.export_counts(dest.key)
+ today = self.store.export_counts(dest.key, since_ms=int(midnight))["sent"]
+ state = ex.state
+ if state == "sending" and self.managed is not None:
+ state = "managed"
+ return {"state": state, "last_sent_at": ex.last_sent_at, "sent": counts["sent"], "sent_today": today,
+ "failed": counts["failed"], "dropped": counts["dropped"], "last_error": ex.last_error,
+ "destination": dest.public()}
+
+ # -- the test recording (E9c) ---------------------------------------------------------------------
+ def send_test(self) -> dict[str, Any]:
+ dest, _ = self.effective()
+ if dest.kind == "off":
+ return {"ok": False, "status": 0, "text": "Choose where to send recordings first."}
+ spans = spans_mod.sample_spans(int(self.clock() * 1000), dest.dialect)
+ from .stamps import machine_stamp
+
+ res = resource_attrs(app_version=str(machine_stamp().get("app_version") or ""))
+ body = json.dumps(encode_otlp([(res, spans)])).encode("utf-8")
+ started = time.monotonic()
+ status, text = self.post(dest.endpoint, body, {"Content-Type": "application/json", **self._headers(dest)},
+ TIMEOUT_SECONDS)
+ ms = int((time.monotonic() - started) * 1000)
+ if 200 <= status < 300:
+ answer = "Sent. Open Langfuse to see it." if dest.kind == "langfuse" else "Sent."
+ return {"ok": True, "status": status, "text": answer, "ms": ms}
+ if status in (401, 403):
+ return {"ok": False, "status": status, "text": REFUSED_KEY, "ms": ms}
+ return {"ok": False, "status": status, "text": _reason(status, text), "ms": ms}
+
+
+def destination_digest(dest: Destination) -> str:
+ """A short, stable label for a destination (logs never show its address)."""
+ return hashlib.sha256(dest.key.encode()).hexdigest()[:8]
diff --git a/coworker/observability/export_spans.py b/coworker/observability/export_spans.py
new file mode 100644
index 00000000..987ba4d6
--- /dev/null
+++ b/coworker/observability/export_spans.py
@@ -0,0 +1,811 @@
+"""Recorded turns as OpenTelemetry spans, with the content rules (OB-6, phase-2 design §10.4).
+
+Pure functions, no network: the exporter (`export.py`) reads the store, calls
+`build_turn_spans` for a turn's unsent steps and encodes the result with `encode_otlp`.
+
+The mapping is the frozen 1.x one (tracing-spec §9 and §10, attributes.yaml part 2): one trace
+per turn, the turn as the root span, each step a span below it, notices as events on the turn.
+The Phase 1 lab exporter (`phase-1/lab/export_otel.py`) was the starting point; it was checked
+against a local Langfuse. This module writes OTLP/HTTP JSON itself, so the app needs no
+OpenTelemetry SDK.
+
+Content (tracing-spec §11). Metadata is always sent. Each content attribute has its switch,
+and five rules apply on top:
+- R1. Tool parts inside exported messages follow the tool switches, wherever they appear.
+- R2. OpenWorker's per-turn `<system-context>` block is cut out of user text unless
+ `capture.context` is on.
+- R3. The auto-approver's and the summary's calls need the tool switches as well.
+- R4. Secrets are masked, then each text part is cut at 8 KB (`Masker`).
+- R5. Never exported: reasoning text, provider-reported cost, `error.text`, `questions`,
+ `decision_details`, `agents_md`, `blocked`. None of them has a name here.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import hashlib
+import json
+import os
+import re
+from typing import Any, Callable, Iterable, Optional
+
+from . import calculator
+from .capture import is_auxiliary, parse_events, rebuild_request, wire_of_url
+
+ATTRIBUTES_VERSION = "1.2"
+SCHEMA = f"openworker/{ATTRIBUTES_VERSION}"
+SERVICE_NAME = "openworker-desktop"
+SCOPE_NAME = "openworker.observability"
+
+# Markers (tracing-spec §11).
+REASONING = "[reasoning text not exported]"
+TOOL_ARGS_OUT = "[tool arguments not exported]"
+TOOL_RESULT_OUT = "[tool result not exported]"
+CONTEXT_OUT = "[OpenWorker context not exported]"
+SECRET = "[secret masked]"
+CUT_BYTES = 8192
+CUT_MARKER = "[cut: {n} more bytes]"
+
+SWITCHES = ("prompts", "responses", "tool_args", "tool_results", "system_instructions",
+ "context", "names")
+
+# attributes.yaml export.value_maps.gen_ai_provider_name
+PROVIDER_NAME = {"anthropic": "anthropic", "openai": "openai", "gemini": "gcp.gemini",
+ "bedrock": "aws.bedrock", "openrouter": "openrouter", "ollama": "ollama",
+ "together": "together", "openai_decisions": "openai", "typesafe_jev": "typesafe"}
+
+SPAN_INTERNAL, SPAN_CLIENT = 1, 3 # OTLP SpanKind
+STATUS_ERROR = 2
+
+
+# -- R4: mask, then cut -----------------------------------------------------------------------
+# attributes.yaml export.content_switches.masking
+MASK_PATTERNS = [
+ r"sk-[A-Za-z0-9_-]{16,}",
+ r"\b(AKIA|ASIA)[A-Z0-9]{16}\b",
+ r"\bgh[pousr]_[A-Za-z0-9]{30,}\b",
+ r"\bgithub_pat_[A-Za-z0-9_]{30,}\b",
+ r"\bxox[abp]-[A-Za-z0-9-]{10,}",
+ r"\bAIza[0-9A-Za-z_-]{35}\b",
+ r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----",
+]
+_PATTERNS = [re.compile(p) for p in MASK_PATTERNS]
+MIN_SECRET_LEN = 8
+_ENV_SECRET_PARTS = ("_API_KEY", "_TOKEN", "_SECRET", "PASSWORD")
+
+
+def env_secret_values(environ: Optional[dict[str, str]] = None) -> list[str]:
+ """Credential-looking environment variables (masking.values)."""
+ env = os.environ if environ is None else environ
+ return [v for k, v in env.items()
+ if any(p in k.upper() for p in _ENV_SECRET_PARTS) and len(v or "") >= MIN_SECRET_LEN]
+
+
+def store_secret_values(data: Any) -> list[str]:
+ """Every string value in the secret store's profiles."""
+ out: list[str] = []
+
+ def walk(v: Any) -> None:
+ if isinstance(v, str):
+ out.append(v)
+ elif isinstance(v, dict):
+ for x in v.values():
+ walk(x)
+ elif isinstance(v, list):
+ for x in v:
+ walk(x)
+
+ walk(data)
+ return out
+
+
+def cut(text: str, limit: int = CUT_BYTES) -> str:
+ """Cut at `limit` bytes of UTF-8, on a character boundary, and say how much was left out."""
+ data = text.encode("utf-8")
+ if len(data) <= limit:
+ return text
+ n = limit
+ while n > 0 and (data[n] & 0xC0) == 0x80: # never split a character
+ n -= 1
+ return data[:n].decode("utf-8") + CUT_MARKER.format(n=len(data) - n)
+
+
+class Masker:
+ """R4: stored secret values (longest first), then the key patterns; then the cut."""
+
+ def __init__(self, secrets: Iterable[str] = ()) -> None:
+ values = {s for s in secrets if isinstance(s, str) and len(s) >= MIN_SECRET_LEN}
+ self.values = sorted(values, key=len, reverse=True)
+
+ def mask(self, text: str) -> str:
+ for v in self.values:
+ if v in text:
+ text = text.replace(v, SECRET)
+ for p in _PATTERNS:
+ text = p.sub(SECRET, text)
+ return text
+
+ def clean(self, value: Any) -> str:
+ """One exported text part: as text (JSON for anything else), masked, then cut."""
+ text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
+ return cut(self.mask(text))
+
+
+# -- content: the exact request and reply, in the GenAI message shape -------------------------
+def _text_parts(content: Any) -> list[dict[str, Any]]:
+ if content is None:
+ return []
+ if isinstance(content, str):
+ return [{"type": "text", "content": content}] if content else []
+ parts: list[dict[str, Any]] = []
+ for b in content if isinstance(content, list) else [content]:
+ if not isinstance(b, dict):
+ parts.append({"type": "text", "content": str(b)})
+ continue
+ t = b.get("type")
+ if t in ("text", "input_text", "output_text"):
+ parts.append({"type": "text", "content": b.get("text", "")})
+ elif t == "tool_use":
+ parts.append({"type": "tool_call", "id": b.get("id"), "name": b.get("name"), "arguments": b.get("input")})
+ elif t == "tool_result":
+ parts.append({"type": "tool_call_response", "id": b.get("tool_use_id"), "response": b.get("content")})
+ elif t in ("thinking", "redacted_thinking", "reasoning"):
+ parts.append({"type": "reasoning", "content": REASONING})
+ else:
+ parts.append({"type": t or "unknown"}) # images and the like: the kind only
+ return parts
+
+
+def _gemini_parts(parts: Any) -> list[dict[str, Any]]:
+ out: list[dict[str, Any]] = []
+ for p in parts or []:
+ if not isinstance(p, dict):
+ continue
+ if p.get("thought"):
+ out.append({"type": "reasoning", "content": REASONING})
+ elif "text" in p:
+ out.append({"type": "text", "content": p.get("text") or ""})
+ elif "functionCall" in p:
+ fc = p["functionCall"] or {}
+ out.append({"type": "tool_call", "id": fc.get("id"), "name": fc.get("name"), "arguments": fc.get("args")})
+ elif "functionResponse" in p:
+ fr = p["functionResponse"] or {}
+ out.append({"type": "tool_call_response", "id": fr.get("id") or fr.get("name"), "response": fr.get("response")})
+ else:
+ out.append({"type": next(iter(p), "unknown")})
+ return out
+
+
+def request_messages(wire: str, req: dict[str, Any]) -> tuple[list, list, list]:
+ """(system instructions, input messages, tool definitions) from a rebuilt request."""
+ system: list[dict[str, Any]] = []
+ msgs: list[dict[str, Any]] = []
+ tools = req.get("tools") or []
+ if wire == "anthropic":
+ system = _text_parts(req.get("system"))
+ for m in req.get("messages") or []:
+ msgs.append({"role": m.get("role"), "parts": _text_parts(m.get("content"))})
+ elif wire == "openai_responses":
+ if req.get("instructions"):
+ system = [{"type": "text", "content": req["instructions"]}]
+ items = req.get("input") or []
+ for it in [{"role": "user", "content": items}] if isinstance(items, str) else items:
+ t = it.get("type")
+ if t == "function_call":
+ msgs.append({"role": "assistant", "parts": [{"type": "tool_call", "id": it.get("call_id"),
+ "name": it.get("name"), "arguments": it.get("arguments")}]})
+ elif t == "function_call_output":
+ msgs.append({"role": "tool", "parts": [{"type": "tool_call_response", "id": it.get("call_id"),
+ "response": it.get("output")}]})
+ elif t == "reasoning":
+ msgs.append({"role": "assistant", "parts": [{"type": "reasoning", "content": REASONING}]})
+ elif it.get("role") in ("system", "developer"):
+ system += _text_parts(it.get("content"))
+ else:
+ msgs.append({"role": it.get("role", "user"), "parts": _text_parts(it.get("content"))})
+ elif wire == "gemini":
+ system = _gemini_parts((req.get("systemInstruction") or {}).get("parts"))
+ for c in req.get("contents") or []:
+ msgs.append({"role": c.get("role", "user"), "parts": _gemini_parts(c.get("parts"))})
+ else: # openai_chat and ollama: a messages list, system first
+ for m in req.get("messages") or []:
+ if m.get("role") in ("system", "developer"):
+ system += _text_parts(m.get("content"))
+ continue
+ parts = _text_parts(m.get("content"))
+ if m.get("reasoning_content") or m.get("reasoning") or m.get("thinking"):
+ parts.insert(0, {"type": "reasoning", "content": REASONING})
+ for tc in m.get("tool_calls") or []:
+ fn = tc.get("function") or {}
+ parts.append({"type": "tool_call", "id": tc.get("id"), "name": fn.get("name"),
+ "arguments": fn.get("arguments")})
+ if m.get("role") == "tool":
+ parts = [{"type": "tool_call_response", "id": m.get("tool_call_id") or m.get("tool_name"),
+ "response": m.get("content")}]
+ msgs.append({"role": m.get("role"), "parts": parts})
+ return system, msgs, tools
+
+
+def response_message(wire: str, body: str) -> dict[str, Any]:
+ """The assistant's reply, put together from the body exactly as received."""
+ evs = [e for e in parse_events(body) if isinstance(e, dict)]
+ parts: list[dict[str, Any]] = []
+ finish = None
+ if wire == "anthropic":
+ blocks: dict[int, dict[str, Any]] = {}
+ for e in evs:
+ t = e.get("type")
+ if t == "message" and isinstance(e.get("content"), list): # not streamed
+ for i, b in enumerate(e["content"]):
+ blocks[i] = dict(b, _json="")
+ finish = e.get("stop_reason")
+ elif t == "content_block_start":
+ blocks[e.get("index", 0)] = dict(e.get("content_block") or {}, _json="")
+ elif t == "content_block_delta":
+ b = blocks.setdefault(e.get("index", 0), {"type": "text", "text": "", "_json": ""})
+ d = e.get("delta") or {}
+ if d.get("type") == "text_delta":
+ b["text"] = b.get("text", "") + d.get("text", "")
+ elif d.get("type") == "input_json_delta":
+ b["_json"] += d.get("partial_json", "")
+ elif t == "message_delta":
+ finish = (e.get("delta") or {}).get("stop_reason")
+ for i in sorted(blocks):
+ b = blocks[i]
+ if b.get("type") == "tool_use":
+ try:
+ args = json.loads(b["_json"]) if b["_json"] else b.get("input")
+ except ValueError:
+ args = b["_json"]
+ parts.append({"type": "tool_call", "id": b.get("id"), "name": b.get("name"), "arguments": args})
+ elif b.get("type") == "text":
+ parts.append({"type": "text", "content": b.get("text", "")})
+ else:
+ parts.append({"type": "reasoning", "content": REASONING})
+ elif wire == "openai_responses":
+ done = [e for e in evs if e.get("type") == "response.completed"]
+ resp = done[-1].get("response") if done else (evs[-1] if evs and "output" in evs[-1] else {})
+ finish = (resp or {}).get("status")
+ for it in (resp or {}).get("output") or []:
+ if it.get("type") == "function_call":
+ parts.append({"type": "tool_call", "id": it.get("call_id"), "name": it.get("name"),
+ "arguments": it.get("arguments")})
+ elif it.get("type") == "message":
+ parts += _text_parts(it.get("content"))
+ elif it.get("type") == "reasoning":
+ parts.append({"type": "reasoning", "content": REASONING})
+ elif wire == "gemini":
+ for e in evs:
+ for c in e.get("candidates") or []:
+ parts += _gemini_parts((c.get("content") or {}).get("parts"))
+ finish = c.get("finishReason") or finish
+ elif wire == "ollama":
+ text, reasoning = "", False
+ for e in evs:
+ m = e.get("message") or {}
+ text += m.get("content") or ""
+ reasoning = reasoning or bool(m.get("thinking"))
+ for tc in m.get("tool_calls") or []:
+ fn = tc.get("function") or {}
+ parts.append({"type": "tool_call", "id": tc.get("id"), "name": fn.get("name"),
+ "arguments": fn.get("arguments")})
+ if e.get("done"):
+ finish = e.get("done_reason")
+ if text:
+ parts.insert(0, {"type": "text", "content": text})
+ if reasoning:
+ parts.insert(0, {"type": "reasoning", "content": REASONING})
+ else: # openai_chat, streamed or not
+ text, calls, reasoning = "", {}, False
+ for e in evs:
+ for ch in e.get("choices") or []:
+ d = ch.get("delta") or ch.get("message") or {}
+ text += d.get("content") or ""
+ reasoning = reasoning or bool(d.get("reasoning") or d.get("reasoning_content"))
+ for n, tc in enumerate(d.get("tool_calls") or []):
+ c = calls.setdefault(tc.get("index", n), {"id": None, "name": None, "arguments": ""})
+ c["id"] = tc.get("id") or c["id"]
+ fn = tc.get("function") or {}
+ c["name"] = fn.get("name") or c["name"]
+ c["arguments"] += fn.get("arguments") or ""
+ finish = ch.get("finish_reason") or finish
+ if reasoning:
+ parts.append({"type": "reasoning", "content": REASONING})
+ if text:
+ parts.append({"type": "text", "content": text})
+ parts += [dict(type="tool_call", **calls[i]) for i in sorted(calls)]
+ return {"role": "assistant", "parts": parts, "finish_reason": finish}
+
+
+def step_exchange(step: dict[str, Any], get_piece: Callable[[str], Optional[bytes]]) -> Optional[tuple[str, dict, str]]:
+ """(wire, rebuilt request, response body) of a model step's last real request."""
+ for ex in reversed((step.get("content") or {}).get("exchange") or []):
+ url = (ex.get("request") or {}).get("url") or ""
+ if ex.get("auxiliary") or is_auxiliary(url):
+ continue # a helper lookup (Ollama's /api/show), not the call
+ try:
+ req = rebuild_request(ex, get_piece)
+ if isinstance(req, (bytes, bytearray)):
+ req = json.loads(req)
+ except Exception: # noqa: BLE001 - a piece already swept: nothing to rebuild
+ return None
+ if not isinstance(req, dict):
+ return None
+ body = get_piece((ex.get("response") or {}).get("body_blob") or "") or b""
+ return wire_of_url(url), req, body.decode("utf-8", "replace")
+ return None
+
+
+# -- the content rules -------------------------------------------------------------------------
+_CONTEXT_BLOCK = re.compile(r"<system-context>.*?(</system-context>|\Z)", re.S)
+
+
+def cut_context(text: str) -> str:
+ """R2: OpenWorker's per-turn block, replaced by its marker."""
+ return _CONTEXT_BLOCK.sub(CONTEXT_OUT, text)
+
+
+def apply_rules(messages: list[dict[str, Any]], sw: dict[str, bool], masker: Masker) -> list[dict[str, Any]]:
+ """R1, R2 and R4 over a list of exported messages (a copy)."""
+ out = []
+ for m in messages:
+ parts = []
+ for p in m.get("parts") or []:
+ p = dict(p)
+ t = p.get("type")
+ if t == "text":
+ text = p.get("content") or ""
+ if m.get("role") == "user" and not sw.get("context"):
+ text = cut_context(text)
+ p["content"] = masker.clean(text)
+ elif t == "tool_call":
+ p["arguments"] = masker.clean(p.get("arguments")) if sw.get("tool_args") else TOOL_ARGS_OUT
+ elif t == "tool_call_response":
+ p["response"] = masker.clean(p.get("response")) if sw.get("tool_results") else TOOL_RESULT_OUT
+ parts.append(p)
+ out.append({k: v for k, v in m.items() if k != "parts"} | {"parts": parts})
+ return out
+
+
+def content_allowed(purpose: str, sw: dict[str, bool]) -> tuple[bool, bool]:
+ """(input, output) of a model call may go out (R3)."""
+ if purpose == "auto_approver":
+ return sw["prompts"] and sw["tool_args"], sw["responses"] and sw["tool_args"]
+ if purpose == "summary":
+ tools = sw["tool_args"] and sw["tool_results"]
+ return sw["prompts"] and tools, sw["responses"] and tools
+ return sw["prompts"], sw["responses"]
+
+
+# -- tokens -------------------------------------------------------------------------------------
+def _usage(step: dict[str, Any]) -> Optional[dict[str, Any]]:
+ u = (step.get("attrs") or {}).get("usage") or {}
+ if not u.get("reported"):
+ return None
+ return {k: u.get(k) for k in ("input_fresh", "cache_read", "cache_write", "output", "reasoning")}
+
+
+def _semconv_usage(u: dict[str, Any]) -> dict[str, int]:
+ a = {"gen_ai.usage.input_tokens": calculator.input_total(u), "gen_ai.usage.output_tokens": u.get("output") or 0}
+ for part, name in (("cache_read", "gen_ai.usage.cache_read.input_tokens"),
+ ("cache_write", "gen_ai.usage.cache_creation.input_tokens"),
+ ("reasoning", "gen_ai.usage.reasoning.output_tokens")):
+ if u.get(part) is not None: # a part not reported is left out, never 0
+ a[name] = u[part]
+ return a
+
+
+def _langfuse_usage(u: dict[str, Any]) -> dict[str, int]:
+ """Buckets that do not overlap: they add up to the call's total."""
+ d = {"input": u.get("input_fresh") or 0}
+ if u.get("cache_read") is not None:
+ d["input_cache_read"] = u["cache_read"]
+ if u.get("cache_write") is not None:
+ d["input_cache_creation"] = u["cache_write"]
+ out = u.get("output") or 0
+ if u.get("reasoning") is not None:
+ d["output"] = max(out - u["reasoning"], 0)
+ d["output_reasoning"] = u["reasoning"]
+ else:
+ d["output"] = out
+ return d
+
+
+# -- ids -----------------------------------------------------------------------------------------
+def _hex(text: str, n: int) -> str:
+ h = hashlib.sha256(text.encode("utf-8")).hexdigest()[:n]
+ return h if int(h, 16) else "1".rjust(n, "0") # an all-zero id is invalid in OTLP
+
+
+def trace_id(turn_id: str) -> str:
+ return _hex(f"trace|{turn_id}", 32)
+
+
+def root_span_id(turn_id: str) -> str:
+ return _hex(f"turn|{turn_id}", 16)
+
+
+def step_span_id(step_id: str) -> str:
+ return _hex(f"step|{step_id}", 16)
+
+
+def _iso(ms: int) -> str:
+ return (_dt.datetime.fromtimestamp(ms / 1000, _dt.timezone.utc)
+ .isoformat(timespec="milliseconds").replace("+00:00", "Z"))
+
+
+def _js(v: Any) -> str:
+ return json.dumps(v, ensure_ascii=False, default=str)
+
+
+# -- one turn -> spans -----------------------------------------------------------------------------
+def switches_from(include: dict[str, bool]) -> dict[str, bool]:
+ """The page's switches to the shared `capture.*` ones (design §10.4)."""
+ return {
+ "prompts": bool(include.get("messages")), "responses": bool(include.get("messages")),
+ "tool_args": bool(include.get("tool_io")), "tool_results": bool(include.get("tool_io")),
+ "system_instructions": bool(include.get("context")), "context": bool(include.get("context")),
+ "names": bool(include.get("names")),
+ }
+
+
+def turn_end_ms(turn: dict[str, Any], steps: list[dict[str, Any]]) -> int:
+ if turn.get("duration_ms") is not None:
+ return turn["started_at"] + turn["duration_ms"]
+ ends = [s["started_at"] + (s.get("duration_ms") or 0) for s in steps if s.get("started_at") is not None]
+ return max([turn.get("started_at") or 0] + ends)
+
+
+def build_turn_spans(
+ turn: dict[str, Any],
+ steps: list[dict[str, Any]],
+ *,
+ sw: dict[str, bool],
+ dialect: str,
+ get_piece: Callable[[str], Optional[bytes]],
+ masker: Masker,
+ include_root: bool = True,
+ only: Optional[set[str]] = None,
+ session_title: Optional[str] = None,
+) -> list[dict[str, Any]]:
+ """The spans of one turn, as plain dicts (`encode_otlp` turns them into OTLP JSON).
+
+ `steps` is every step of the turn (parents and content are looked up among them);
+ `only` limits the spans built to these step ids (a late `after_turn` step), and
+ `include_root` says whether the turn's own span is built. Each span dict carries
+ `_mark`: the id the exporter marks as sent (the step id, or the turn id for the root).
+ """
+ lf = dialect == "langfuse"
+ tid = turn["turn_id"]
+ by_id = {s["step_id"]: s for s in steps}
+ end_ms = turn_end_ms(turn, steps)
+ common = {"session.id": turn["session_id"], "gen_ai.conversation.id": turn["session_id"],
+ "openworker.turn.id": tid}
+ any_content = any(sw.get(k) for k in SWITCHES if k != "names")
+
+ # Content for tool spans comes from the exchanges around them: the arguments from the
+ # reply that asked for the call, the result from the next request that carried it back.
+ exchanges: dict[str, tuple[str, dict, str]] = {}
+ tool_args: dict[str, Any] = {}
+ tool_results: dict[str, Any] = {}
+ if any_content:
+ for s in steps:
+ if s.get("kind") != "model":
+ continue
+ x = step_exchange(s, get_piece)
+ if x is None:
+ continue
+ exchanges[s["step_id"]] = x
+ wire, req, body = x
+ for m in request_messages(wire, req)[1]:
+ for p in m["parts"]:
+ if p.get("type") == "tool_call_response" and p.get("id"):
+ tool_results[p["id"]] = p.get("response")
+ for p in response_message(wire, body)["parts"]:
+ if p.get("type") == "tool_call" and p.get("id"):
+ tool_args[p["id"]] = p.get("arguments")
+
+ spans: list[dict[str, Any]] = []
+ trace = trace_id(tid)
+ root_id = root_span_id(tid)
+
+ def depth(s: dict[str, Any]) -> int:
+ d, p, seen = 0, s.get("parent_step_id"), set()
+ while p and p in by_id and p not in seen:
+ seen.add(p)
+ d, p = d + 1, by_id[p].get("parent_step_id")
+ return d
+
+ first_user_text = last_reply_text = None
+ for s in sorted(steps, key=lambda s: (depth(s), s.get("seq") or 0)):
+ if s.get("kind") == "notice" or (only is not None and s["step_id"] not in only):
+ continue
+ a = s.get("attrs") or {}
+ attrs: dict[str, Any] = dict(common, **{
+ "openworker.step.id": s["step_id"], "openworker.step.kind": s["kind"],
+ "openworker.step.status": s.get("status") or "unknown", "openworker.step.seq": s.get("seq") or 0})
+ if s.get("after_turn"):
+ attrs["openworker.after_turn"] = True
+ if s.get("shadow"):
+ attrs["openworker.shadow"] = True
+ call_id = s.get("call_id") or (a.get("for_call_id") if s["kind"] == "wait" else None)
+ if call_id:
+ # Only a tool's own span carries gen_ai.tool.call.id: viewers type any span with it
+ # as a tool (V-22). Waits, checks and the reviewer's model call use our own name.
+ attrs["gen_ai.tool.call.id" if s["kind"] == "tool" else "openworker.tool.call_id"] = call_id
+ kind = SPAN_INTERNAL
+ events: list[dict[str, Any]] = []
+ lf_type = "span"
+ usage = None
+ if s["kind"] == "model":
+ model = a.get("model") or ""
+ prov, _, short = model.partition(":")
+ short = short or model
+ prov = a.get("provider") or prov
+ facts = a.get("provider_facts") or {}
+ name = f"chat {short}"
+ kind, lf_type = SPAN_CLIENT, "generation"
+ attrs.update({"gen_ai.operation.name": "chat", "gen_ai.provider.name": PROVIDER_NAME.get(prov, prov),
+ "gen_ai.request.model": short, "openworker.model": model,
+ "openworker.llm.purpose": a.get("purpose") or "unattributed",
+ "openworker.stream": bool(a.get("stream"))})
+ if facts.get("served_model"):
+ attrs["gen_ai.response.model"] = facts["served_model"]
+ if facts.get("provider_response_id"):
+ attrs["gen_ai.response.id"] = facts["provider_response_id"]
+ if facts.get("provider_request_id"):
+ attrs["openworker.provider.request_id"] = facts["provider_request_id"]
+ if facts.get("served_by") or a.get("served_by"):
+ attrs["openworker.provider.served_by"] = facts.get("served_by") or a.get("served_by")
+ if a.get("finish_reason"):
+ attrs["gen_ai.response.finish_reasons"] = [a["finish_reason"]]
+ if a.get("tools_offered") is not None:
+ attrs["openworker.tools_offered"] = a["tools_offered"]
+ usage = _usage(s)
+ attrs["openworker.usage.reported"] = usage is not None
+ if usage:
+ attrs.update(_semconv_usage(usage))
+ if lf:
+ attrs["langfuse.observation.usage_details"] = _js(_langfuse_usage(usage))
+ if a.get("ttft_ms") is not None and s.get("started_at") is not None:
+ first = s["started_at"] + a["ttft_ms"]
+ attrs["gen_ai.client.operation.time_to_first_chunk"] = a["ttft_ms"] / 1000
+ events.append({"name": "gen_ai.first_chunk", "at": first, "attrs": {}})
+ if lf:
+ attrs["langfuse.observation.completion_start_time"] = _iso(first)
+ err = a.get("error")
+ if err:
+ attrs["error.type"] = err.get("type") or "error" # never error.text (R5)
+ x = exchanges.get(s["step_id"])
+ if x is not None:
+ wire, req, body = x
+ system, msgs, tools = request_messages(wire, req)
+ out = response_message(wire, body)
+ purpose = a.get("purpose") or ""
+ in_ok, out_ok = content_allowed(purpose, sw)
+ if sw.get("system_instructions"):
+ attrs["gen_ai.system_instructions"] = _js(apply_rules([{"role": "system", "parts": system}],
+ {**sw, "context": True}, masker)[0]["parts"])
+ attrs["gen_ai.tool.definitions"] = masker.clean(tools)
+ if in_ok:
+ attrs["gen_ai.input.messages"] = _js(apply_rules(msgs, sw, masker))
+ if out_ok:
+ attrs["gen_ai.output.messages"] = _js(apply_rules([out], sw, masker))
+ if purpose == "main":
+ users = [p["content"] for m in msgs if m.get("role") == "user"
+ for p in m["parts"] if p.get("type") == "text"]
+ first_user_text = first_user_text or (users[-1] if users else None)
+ texts = [p["content"] for p in out["parts"] if p.get("type") == "text"]
+ last_reply_text = texts[-1] if texts else last_reply_text
+ elif s["kind"] == "tool":
+ name = f"execute_tool {s.get('name')}"
+ lf_type = "tool"
+ attrs.update({"gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": s.get("name") or "",
+ "gen_ai.tool.type": "function"})
+ dec = a.get("decision") or {}
+ if dec.get("origin"):
+ attrs["openworker.decision.origin"] = dec["origin"]
+ if dec.get("path"):
+ attrs["openworker.decision.path"] = [f"{p.get('stage')}:{p.get('outcome')}" for p in dec["path"]]
+ if dec.get("helper"):
+ attrs["openworker.decision.helper"] = True
+ for k in ("risk_class", "exit_code", "result_bytes"):
+ if a.get(k) is not None:
+ attrs[f"openworker.tool.{k}"] = a[k]
+ cid = s.get("call_id")
+ if sw.get("tool_args") and cid in tool_args:
+ attrs["gen_ai.tool.call.arguments"] = masker.clean(tool_args[cid])
+ if sw.get("tool_results") and cid in tool_results:
+ attrs["gen_ai.tool.call.result"] = masker.clean(tool_results[cid])
+ elif s["kind"] == "wait":
+ wk = a.get("kind") or "approval"
+ name = f"openworker.{wk}.wait"
+ for k in ("kind", "where", "outcome", "attended"):
+ if a.get(k) is not None:
+ attrs[f"openworker.wait.{k}"] = a[k]
+ resolved = (s.get("content") or {}).get("resolved_by")
+ if sw.get("names") and resolved:
+ attrs["openworker.wait.resolved_by"] = masker.clean(resolved)
+ elif s["kind"] == "auto_approver":
+ name = f"openworker.policy.review {s.get('name')}"
+ lf_type = "guardrail"
+ rv = a.get("reviewer") or {}
+ attrs.update({"openworker.reviewer.kind": rv.get("kind") or "",
+ "openworker.reviewer.external": bool(rv.get("external")),
+ "openworker.reviewer.mode": a.get("mode") or "",
+ "openworker.reviewer.result": a.get("result") or ""})
+ if rv.get("model"):
+ attrs["openworker.reviewer.model"] = rv["model"]
+ for k in ("score", "cutoff"):
+ if a.get(k) is not None:
+ attrs[f"openworker.reviewer.{k}"] = a[k]
+ reason = (s.get("content") or {}).get("reason")
+ if reason and sw.get("responses") and sw.get("tool_args"): # R3: it quotes the action
+ attrs["openworker.reviewer.reason"] = masker.clean(reason)
+ elif s["kind"] == "summary":
+ name = "openworker.compaction" if s.get("name") == "summary" else "openworker.compaction.trim"
+ for k in ("reason", "tokens_before", "tokens_after"):
+ if a.get(k) is not None:
+ attrs[f"openworker.compaction.{k}"] = a[k]
+ elif s["kind"] == "context":
+ name = "openworker.context.build"
+ for k in ("size_chars", "session_prompt_hash", "block_hash"):
+ if a.get(k) is not None:
+ attrs[f"openworker.context.{k}"] = a[k]
+ elif s["kind"] == "sandbox":
+ name = "openworker.sandbox.setup"
+ else:
+ name = f"openworker.{s['kind']}"
+
+ status = s.get("status") or "unknown"
+ has_end = s.get("duration_ms") is not None
+ level = message = None
+ if status == "error":
+ level = "ERROR"
+ message = (a.get("error") or {}).get("type") or "error"
+ if s["kind"] == "model":
+ message += "; usage unknown"
+ elif status in ("denied", "interrupted", "unknown") or not has_end:
+ level, message = "WARNING", status if has_end else "unknown"
+ elif s["kind"] == "model" and usage is None:
+ level, message = "WARNING", "usage not reported"
+ if lf:
+ attrs["langfuse.observation.type"] = lf_type
+ if level:
+ attrs["langfuse.observation.level"] = level
+ attrs["langfuse.observation.status_message"] = message
+ parent = s.get("parent_step_id")
+ start = s.get("started_at") if s.get("started_at") is not None else turn["started_at"]
+ spans.append({
+ "_mark": s["step_id"], "trace_id": trace,
+ "span_id": step_span_id(s["step_id"]),
+ "parent_span_id": step_span_id(parent) if parent and parent in by_id else root_id,
+ "name": name, "kind": kind, "start_ms": start,
+ "end_ms": start + s["duration_ms"] if has_end else max(end_ms, start),
+ "attrs": attrs, "events": events,
+ "error": ((a.get("error") or {}).get("type") or "error") if status == "error" else None,
+ })
+
+ if include_root:
+ usage = calculator.usage({"steps": steps}, turn_id=tid)
+ root_attrs = dict(common, **{
+ "gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "coworker",
+ "openworker.turn.seq": turn.get("seq") or 0, "openworker.turn.kind": turn.get("kind") or "",
+ "openworker.turn.trigger": turn.get("trigger") or "", "openworker.turn.status": turn.get("status") or "",
+ "openworker.approval_mode": turn.get("approval_mode") or "",
+ "openworker.attended": bool(turn.get("attended")),
+ # Never a gen_ai.usage.* name on a parent span: viewers would count it twice.
+ "openworker.usage.total_tokens": usage["pill"],
+ "openworker.usage.at_least": usage["at_least"],
+ "openworker.usage.unreported_calls": usage["unreported_calls"],
+ "openworker.usage.failed_calls": usage["failed_calls"],
+ "openworker.usage.context_limit_calls": usage["overflow_calls"]})
+ woken = turn.get("woken_by") or {}
+ if isinstance(woken, dict):
+ if woken.get("team_id"):
+ root_attrs["openworker.team.id"] = woken["team_id"]
+ if woken.get("lead_session_id"):
+ root_attrs["openworker.lead.session.id"] = woken["lead_session_id"]
+ if sw.get("names") and session_title:
+ root_attrs["openworker.session.title"] = masker.clean(session_title)
+ if lf:
+ root_attrs.update({"langfuse.observation.type": "agent", "langfuse.trace.name": "invoke_agent coworker"})
+ if usage["unreported_calls"] or usage["failed_calls"]:
+ root_attrs["langfuse.trace.tags"] = ["usage-incomplete"]
+ if sw.get("prompts") and first_user_text:
+ text = first_user_text if sw.get("context") else cut_context(first_user_text)
+ root_attrs["langfuse.trace.input"] = root_attrs["langfuse.observation.input"] = masker.clean(text)
+ if sw.get("responses") and last_reply_text:
+ root_attrs["langfuse.trace.output"] = root_attrs["langfuse.observation.output"] = masker.clean(last_reply_text)
+ events = [{"name": f"openworker.notice.{s.get('name')}", "at": s.get("started_at") or turn["started_at"],
+ "attrs": {"openworker.step.id": s["step_id"]}}
+ for s in sorted(steps, key=lambda s: s.get("seq") or 0) if s.get("kind") == "notice"]
+ spans.insert(0, {
+ "_mark": tid, "_notices": [s["step_id"] for s in steps if s.get("kind") == "notice"],
+ "trace_id": trace, "span_id": root_id, "parent_span_id": None,
+ "name": "invoke_agent coworker", "kind": SPAN_INTERNAL, "start_ms": turn["started_at"],
+ "end_ms": end_ms, "attrs": root_attrs, "events": events,
+ "error": turn.get("status") if turn.get("status") == "error" else None,
+ })
+ return spans
+
+
+# -- the made-up test recording (E9c) ------------------------------------------------------------
+TEST_SESSION = "openworker-test"
+
+
+def sample_spans(now_ms: int, dialect: str) -> list[dict[str, Any]]:
+ """One trace with one model span and one tool span, fixed made-up values, tagged
+ `openworker-test`. Nothing in it comes from a session."""
+ turn_id = f"openworker-test-{now_ms}"
+ turn = {"turn_id": turn_id, "session_id": TEST_SESSION, "seq": 1, "kind": "run", "trigger": "user",
+ "status": "completed", "approval_mode": "ask", "attended": True,
+ "started_at": now_ms - 1500, "duration_ms": 1500}
+ steps = [
+ {"step_id": f"{turn_id}-model", "turn_id": turn_id, "session_id": TEST_SESSION, "seq": 1,
+ "kind": "model", "name": "model", "started_at": now_ms - 1400, "duration_ms": 900, "status": "ok",
+ "attrs": {"model": "openworker:test-model", "provider": "openworker", "purpose": "main",
+ "stream": False, "finish_reason": "tool_calls",
+ "usage": {"reported": True, "input_fresh": 100, "cache_read": None, "cache_write": None,
+ "output": 10, "reasoning": None}}},
+ {"step_id": f"{turn_id}-tool", "turn_id": turn_id, "session_id": TEST_SESSION, "seq": 2,
+ "kind": "tool", "name": "test_tool", "call_id": "test-call", "started_at": now_ms - 400,
+ "duration_ms": 200, "status": "ok", "attrs": {"result_bytes": 2}},
+ ]
+ sw = {k: False for k in SWITCHES}
+ spans = build_turn_spans(turn, steps, sw=sw, dialect=dialect, get_piece=lambda _k: None, masker=Masker())
+ if dialect == "langfuse":
+ spans[0]["attrs"]["langfuse.trace.tags"] = [TEST_SESSION]
+ return spans
+
+
+# -- OTLP/HTTP JSON ------------------------------------------------------------------------------
+def _value(v: Any) -> dict[str, Any]:
+ if isinstance(v, bool):
+ return {"boolValue": v}
+ if isinstance(v, int):
+ return {"intValue": str(v)}
+ if isinstance(v, float):
+ return {"doubleValue": v}
+ if isinstance(v, (list, tuple)):
+ return {"arrayValue": {"values": [_value(x) for x in v]}}
+ return {"stringValue": str(v)}
+
+
+def _attrs(d: dict[str, Any]) -> list[dict[str, Any]]:
+ return [{"key": k, "value": _value(v)} for k, v in d.items() if v is not None]
+
+
+def _ns(ms: int) -> str:
+ return str(int(ms) * 1_000_000)
+
+
+def resource_attrs(*, app_version: str, machine_name: Optional[str] = None) -> dict[str, Any]:
+ out = {"service.name": SERVICE_NAME, "service.version": app_version or "unknown",
+ "openworker.export.schema": SCHEMA}
+ if machine_name: # only with capture.names
+ out["host.name"] = machine_name
+ return out
+
+
+def encode_otlp(groups: list[tuple[dict[str, Any], list[dict[str, Any]]]]) -> dict[str, Any]:
+ """`[(resource attributes, spans)]` -> the OTLP/HTTP JSON body."""
+ out = []
+ for resource, spans in groups:
+ enc = []
+ for s in spans:
+ span = {
+ "traceId": s["trace_id"], "spanId": s["span_id"], "name": s["name"], "kind": s["kind"],
+ "startTimeUnixNano": _ns(s["start_ms"]), "endTimeUnixNano": _ns(s["end_ms"]),
+ "attributes": _attrs(s["attrs"]),
+ "events": [{"timeUnixNano": _ns(e["at"]), "name": e["name"], "attributes": _attrs(e["attrs"])}
+ for e in s.get("events") or []],
+ }
+ if s.get("parent_span_id"):
+ span["parentSpanId"] = s["parent_span_id"]
+ if s.get("error"):
+ span["status"] = {"code": STATUS_ERROR, "message": s["error"]}
+ enc.append(span)
+ out.append({"resource": {"attributes": _attrs(resource)},
+ "scopeSpans": [{"scope": {"name": SCOPE_NAME, "version": ATTRIBUTES_VERSION}, "spans": enc}]})
+ return {"resourceSpans": out}
diff --git a/coworker/observability/store.py b/coworker/observability/store.py
index 59664091..cb1f9473 100644
--- a/coworker/observability/store.py
+++ b/coworker/observability/store.py
@@ -65,6 +65,12 @@ CREATE TABLE IF NOT EXISTS exchange_pieces (
CREATE TABLE IF NOT EXISTS step_pieces (
step_id TEXT NOT NULL, sha256 TEXT NOT NULL, PRIMARY KEY (step_id, sha256));
CREATE INDEX IF NOT EXISTS step_pieces_sha ON step_pieces(sha256);
+CREATE TABLE IF NOT EXISTS export_marks (
+ destination TEXT NOT NULL, item_id TEXT NOT NULL, turn_id TEXT, state TEXT NOT NULL, at INTEGER NOT NULL,
+ PRIMARY KEY (destination, item_id));
+CREATE INDEX IF NOT EXISTS export_marks_turn ON export_marks(turn_id);
+CREATE TABLE IF NOT EXISTS export_counters (
+ destination TEXT PRIMARY KEY, failed INTEGER NOT NULL DEFAULT 0, dropped INTEGER NOT NULL DEFAULT 0);
"""
_TURN_COLUMNS = (
@@ -195,6 +201,9 @@ class RecordingStore:
self._closed = False
self._last_sweep = 0.0
self._background = background
+ # OB-6: the destination being exported to and the time it was chosen, so a turn the
+ # retention sweep deletes before it was sent is counted as dropped (design §10.5).
+ self.export_destination: Optional[tuple[str, int]] = None
if background:
_writer.add(self)
@@ -352,6 +361,10 @@ class RecordingStore:
"(SELECT step_id FROM steps WHERE session_id = ?)",
(session_id,),
)
+ self._conn.execute(
+ "DELETE FROM export_marks WHERE turn_id IN (SELECT turn_id FROM turns WHERE session_id = ?)",
+ (session_id,),
+ )
self._conn.execute("DELETE FROM steps WHERE session_id = ?", (session_id,))
self._conn.execute("DELETE FROM turns WHERE session_id = ?", (session_id,))
self._delete_unused_pieces()
@@ -378,6 +391,8 @@ class RecordingStore:
]
if ids:
marks = ", ".join("?" * len(ids))
+ self._count_unsent_dropped(ids, marks)
+ self._conn.execute(f"DELETE FROM export_marks WHERE turn_id IN ({marks})", ids)
self._conn.execute(
"DELETE FROM step_pieces WHERE step_id IN "
f"(SELECT step_id FROM steps WHERE turn_id IN ({marks}))",
@@ -422,6 +437,107 @@ class RecordingStore:
break
time.sleep(SWEEP_PAUSE_S)
+ # -- export marks (OB-6, phase-2 design §10.2, §10.5) ----------------------------------------
+ # The queue is the store itself: a finished turn's root and steps with no mark for the
+ # destination are still to send. A mark is `sent`, `dropped` (refused on its own) or
+ # `skipped` (a notice that came after its turn's span had gone). Marks are per destination,
+ # so turning export off and on again never re-sends a step.
+
+ def _count_unsent_dropped(self, ids: list[str], marks: str) -> None:
+ """Inside the sweep's transaction: turns about to be deleted that ended after the
+ destination was chosen and were never sent."""
+ dest = self.export_destination
+ if dest is None:
+ return
+ key, since = dest
+ n = self._conn.execute(
+ f"SELECT COUNT(*) AS n FROM turns t WHERE t.turn_id IN ({marks}) "
+ "AND t.status IS NOT NULL AND t.status != 'running' "
+ "AND t.started_at + COALESCE(t.duration_ms, 0) >= ? AND NOT EXISTS "
+ "(SELECT 1 FROM export_marks m WHERE m.destination = ? AND m.item_id = t.turn_id)",
+ (*ids, since, key),
+ ).fetchone()["n"]
+ if n:
+ self._bump(key, dropped=n)
+
+ def _bump(self, destination: str, *, failed: int = 0, dropped: int = 0) -> None:
+ self._conn.execute(
+ "INSERT INTO export_counters (destination, failed, dropped) VALUES (?, ?, ?) "
+ "ON CONFLICT(destination) DO UPDATE SET failed = failed + excluded.failed, "
+ "dropped = dropped + excluded.dropped",
+ (destination, failed, dropped),
+ )
+
+ def count_export(self, destination: str, *, failed: int = 0, dropped: int = 0) -> None:
+ with self._lock, self._conn:
+ self._bump(destination, failed=failed, dropped=dropped)
+
+ def export_queue(self, destination: str, since_ms: int, limit: int = 50) -> list[str]:
+ """Finished turns, ended at or after `since_ms`, with something not yet marked for
+ this destination: the turn itself, or a step that has ended (an `after_turn` step
+ is sent alone when it ends). Oldest first."""
+ self.flush()
+ with self._lock:
+ rows = self._conn.execute(
+ "SELECT t.turn_id FROM turns t "
+ "WHERE t.status IS NOT NULL AND t.status != 'running' "
+ "AND t.started_at + COALESCE(t.duration_ms, 0) >= ? AND ("
+ " NOT EXISTS (SELECT 1 FROM export_marks m WHERE m.destination = ? AND m.item_id = t.turn_id)"
+ " OR EXISTS (SELECT 1 FROM steps s WHERE s.turn_id = t.turn_id"
+ " AND s.status IS NOT NULL AND s.status != 'running' AND NOT EXISTS"
+ " (SELECT 1 FROM export_marks m WHERE m.destination = ? AND m.item_id = s.step_id))) "
+ "ORDER BY t.started_at, t.turn_id LIMIT ?",
+ (since_ms, destination, destination, limit),
+ ).fetchall()
+ return [r["turn_id"] for r in rows]
+
+ def export_marked(self, destination: str, turn_id: str) -> set[str]:
+ with self._lock:
+ rows = self._conn.execute(
+ "SELECT item_id FROM export_marks WHERE destination = ? AND turn_id = ?",
+ (destination, turn_id),
+ ).fetchall()
+ return {r["item_id"] for r in rows}
+
+ def mark_export(self, destination: str, items: list[tuple[str, str]], state: str) -> None:
+ """`items`: (item id, turn id) pairs. A mark, once written, is kept."""
+ now = int(time.time() * 1000)
+ with self._lock, self._conn:
+ self._conn.executemany(
+ "INSERT OR IGNORE INTO export_marks (destination, item_id, turn_id, state, at) "
+ "VALUES (?, ?, ?, ?, ?)",
+ [(destination, item, turn, state, now) for item, turn in items],
+ )
+
+ def export_counts(self, destination: str, *, since_ms: int = 0) -> dict[str, int]:
+ """Steps sent (since `since_ms`), steps dropped, and batches failed for good."""
+ with self._lock:
+ sent = self._conn.execute(
+ "SELECT COUNT(*) AS n FROM export_marks WHERE destination = ? AND state = 'sent' AND at >= ?",
+ (destination, since_ms),
+ ).fetchone()["n"]
+ dropped = self._conn.execute(
+ "SELECT COUNT(*) AS n FROM export_marks WHERE destination = ? AND state = 'dropped'",
+ (destination,),
+ ).fetchone()["n"]
+ row = self._conn.execute(
+ "SELECT failed, dropped FROM export_counters WHERE destination = ?", (destination,)
+ ).fetchone()
+ return {"sent": int(sent), "dropped": int(dropped) + (int(row["dropped"]) if row else 0),
+ "failed": int(row["failed"]) if row else 0}
+
+ def load_turn(self, turn_id: str) -> Optional[tuple[dict[str, Any], list[dict[str, Any]]]]:
+ """One turn and its steps (by `seq`), content included."""
+ self.flush()
+ with self._lock:
+ t = self._conn.execute("SELECT * FROM turns WHERE turn_id = ?", (turn_id,)).fetchone()
+ if t is None:
+ return None
+ steps = self._conn.execute(
+ "SELECT * FROM steps WHERE turn_id = ? ORDER BY seq IS NULL, seq, started_at", (turn_id,)
+ ).fetchall()
+ return _turn_record(t), [_step_record(s) for s in steps]
+
# -- the Issues result (phase-2 design §5): derived, stored beside the turn ---------------
def write_issues(self, turn_id: str, result: dict[str, Any], *, seen: bool = False) -> None:
"""Store a turn's Issues result. `seen` is false only for a result worked out when the
diff --git a/coworker/remote/joiner.py b/coworker/remote/joiner.py
index fa7c423b..6cb9d417 100644
--- a/coworker/remote/joiner.py
+++ b/coworker/remote/joiner.py
@@ -402,21 +402,28 @@ def apply_policy(
app, policy: dict[str, Any], *, state: Optional[Path] = None, exporter=None
) -> tuple[bool, int]:
"""Persist + apply an org policy DOCUMENT on this box (spec §Versioned org
- policy). Today one field acts: `audit_export` (remote/audit_export.py).
+ policy). Two fields act: `audit_export` (remote/audit_export.py) and
+ `observability_export` (observability/export.py).
Unknown keys are kept verbatim so the machine can display them; nothing
turns on by accident. A document OLDER than the one held is ignored — a
stale controller never rolls a machine back. Returns (applied, version
now held)."""
+ from ..observability.export import take_policy_secrets
from .audit_export import ExportPolicy, save_policy
incoming, current = policy_version(policy), policy_version(held_policy(app))
if incoming and current and incoming < current:
return False, current
+ manager = getattr(app.state, "manager", None)
+ # OB-6: the observability export's keys go to the secret store, never policy.json.
+ policy = take_policy_secrets(policy, getattr(manager, "secrets", None))
if state is not None:
save_policy(state, policy)
- manager = getattr(app.state, "manager", None)
if manager is not None:
manager.org_policy = dict(policy)
+ apply = getattr(manager, "apply_org_policy", None)
+ if apply is not None:
+ apply(policy)
if exporter is not None:
exporter.set_policy(ExportPolicy.from_policy(policy))
return True, incoming
@@ -443,6 +450,9 @@ def _build_exporter(state: Path, app, welcome: dict[str, Any]):
stored = load_policy(state)
if stored and not getattr(manager, "org_policy", None):
manager.org_policy = stored
+ apply = getattr(manager, "apply_org_policy", None)
+ if apply is not None:
+ apply(stored)
return exporter
diff --git a/coworker/server/app.py b/coworker/server/app.py
index 0d11f0af..13ce8a78 100644
--- a/coworker/server/app.py
+++ b/coworker/server/app.py
@@ -23,7 +23,7 @@ from typing import Any, Optional
from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect
from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse
+from fastapi.responses import JSONResponse, Response
logger = logging.getLogger("coworker.server.app")
@@ -852,6 +852,33 @@ def create_app(manager: SessionManager) -> FastAPI:
def session_issues_seen(session_id: str) -> dict[str, Any]:
return manager.mark_recording_issues_seen(session_id)
+ # OB-6: one session's recording as a file, content included (phase-2 design §10.6).
+ @app.get("/v1/sessions/{session_id}/recording")
+ def session_recording(session_id: str):
+ body = json.dumps(manager.recording_file(session_id), ensure_ascii=False)
+ safe = re.sub(r"[^A-Za-z0-9_.-]", "_", session_id)[:80] or "session"
+ return Response(content=body, media_type="application/json", headers={
+ "Content-Disposition": f'attachment; filename="recording-{safe}.json"'})
+
+ # OB-6: Settings ▸ Machine ▸ Observability. Keys are write-only: answers carry masked
+ # hints, never a value (phase-2 design §10.8).
+ @app.get("/v1/observability/settings")
+ def observability_settings() -> dict[str, Any]:
+ return manager.observability.get_settings()
+
+ @app.put("/v1/observability/settings")
+ def observability_settings_put(body: dict):
+ code, answer = manager.observability.put_settings(body or {})
+ return JSONResponse(answer, status_code=code)
+
+ @app.post("/v1/observability/test")
+ async def observability_test() -> dict[str, Any]:
+ return await asyncio.to_thread(manager.observability.send_test)
+
+ @app.get("/v1/observability/status")
+ def observability_status() -> dict[str, Any]:
+ return manager.observability.status()
+
@app.patch("/v1/sessions/{session_id}")
def session_patch(session_id: str, body: dict) -> dict[str, Any]:
body = body or {}
@@ -3309,6 +3336,16 @@ def create_app(manager: SessionManager) -> FastAPI:
},
}
)
+ # OB-6 (TD2-11): the company's policy sends this machine's content somewhere. Each
+ # new session says what goes where, once, before its first message.
+ export_notice = manager.observability.content_notice()
+ if export_notice and not any(
+ m.get("role") in ("user", "assistant") or m.get("kind") == "export_notice"
+ for m in engine.messages
+ ):
+ engine._append_notice("export_notice", export_notice)
+ manager.save(session_id, engine, touch=False)
+ await ws.send_json({"type": "export_notice", "data": {"text": export_notice}})
inbound_times: deque[float] = deque()
async def reject_input(reason: str) -> None:
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index 8bba3f7d..d010cd3b 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -46,7 +46,9 @@ from ..audit import AuditStore
from ..observability import Recorder, RecordingStore
from ..observability import read_api as recording_api
from ..observability import signals as recording_signals
+from ..observability.export import ObservabilityExport
from ..observability.provider_view import RecordingProvider, tag as tag_provider
+from ..remote.audit_export import load_policy
from ..config import load_config, workspace_allowed_commands
from ..conversations import ConversationStore, title_from
from ..engine import ApprovalOutcome, Approver, TurnEngine
@@ -431,6 +433,15 @@ class SessionManager:
self._autotitle_sig: dict[str, int] = {}
self.workspace_trust = WorkspaceTrustStore()
self.secrets = SecretStore()
+ # OB-6: record on or off, keep for, and sending recordings to Langfuse or an
+ # OpenTelemetry endpoint. Settings live in the machine's state folder (beside the
+ # secrets), never in a project. Off by default: then no exporter exists at all.
+ self.observability = ObservabilityExport(
+ self.secrets.path.parent, self.secrets, self.recordings_store,
+ session_title=self.session_store.display_title,
+ )
+ self.observability.apply_policy(load_policy(self.secrets.path.parent))
+ self.recorder.add_listener(self._export_on_turn_end)
# No explicit provider injected → route by the model's `provider:` prefix (OpenAI default,
# Ollama, …). Tests inject a provider directly and bypass the router. The same router is
# shared by every engine and the `/v1/chat/completions` proxy.
@@ -1019,8 +1030,9 @@ class SessionManager:
auto_approve_shadow=self.auto_approve_shadow(),
auto_approve_reviewer=self.auto_approve_reviewer(),
)
- # Observability (E1): this engine's turns and steps go to the server's recorder.
- engine.recorder = self.recorder
+ # Observability (E1): this engine's turns and steps go to the server's recorder,
+ # unless recording is off on this machine (OB-6): then new engines record nothing.
+ engine.recorder = self.recorder if self.observability.settings.record else None
engine.resolver_of = lambda call_id, _sid=session_id: self.inbox.resolver_of(_sid, call_id)
engine.delegated_approval = lambda args: self.worker_review_context(session_id, args)
engine.reviewer_context = lambda: self.approval_context(session_id)
@@ -6379,6 +6391,35 @@ class SessionManager:
def recording_usage(self, session_id: str) -> dict[str, Any]:
return recording_api.usage(self.recordings_store.load_recording(session_id), session_id)
+ # -- observability: export and its settings (OB-6, phase-2 design §10) ------------------
+ def _export_on_turn_end(self, event: dict[str, Any]) -> None:
+ """A turn ended: the exporter sends soon. A late step (the session title) goes with
+ the exporter's next pass, within its flush interval."""
+ if event.get("type") == "turn_ended":
+ self.observability.wake()
+
+ def apply_org_policy(self, policy: dict[str, Any]) -> None:
+ """The org policy changed: its `observability_export` field manages the page."""
+ self.observability.apply_policy(policy)
+
+ def recording_file(self, session_id: str) -> dict[str, Any]:
+ """Everything recorded for one session, content included, in the shape the read
+ endpoints serve: turns, each turn's steps, and each step in full (§10.6)."""
+ rec = self.recordings_store.load_recording(session_id)
+ messages = self.session_messages(session_id)
+ details = {}
+ for s in rec.get("steps") or []:
+ details[s["step_id"]] = {"step": recording_api.step(
+ rec, s["step_id"], get_piece=self.recordings_store.get_piece, messages=messages,
+ audit_rows=lambda call_id: self.audit_store.list(session_id=session_id, call_id=call_id))}
+ return {
+ "format": "openworker-recording", "version": "1.2", "session_id": session_id,
+ "exported_at": int(time.time() * 1000),
+ "turns": recording_api.turns(rec),
+ "steps": {t["turn_id"]: recording_api.turn_steps(rec, t["turn_id"]) for t in rec.get("turns") or []},
+ "details": details,
+ }
+
def _forward_recording_event(self, event: dict[str, Any]) -> None:
"""Queue one recording event for the session's sockets; a batch goes out every
100 ms as `{"type": "step", "events": [...]}`, so a burst of steps does not flood
@@ -6517,6 +6558,7 @@ class SessionManager:
await self.stop_gateway()
await self.mcp.aclose()
self.audit_store.close()
+ self.observability.close()
self.recordings_store.close()
from ..providers import llamacpp_setup
@@ -7080,8 +7122,9 @@ class SessionManager:
[d] if (d := self.persona_skill_scope(task.agent)[0]) is not None else None
),
)
- # Observability (E1): this engine's turns and steps go to the server's recorder.
- engine.recorder = self.recorder
+ # Observability (E1): this engine's turns and steps go to the server's recorder,
+ # unless recording is off on this machine (OB-6): then new engines record nothing.
+ engine.recorder = self.recorder if self.observability.settings.record else None
engine.resolver_of = lambda call_id, _sid=session_id: self.inbox.resolver_of(_sid, call_id)
self._seed_task_permissions(engine, task)
return engine
diff --git a/tests/test_observability_export.py b/tests/test_observability_export.py
new file mode 100644
index 00000000..843e892c
--- /dev/null
+++ b/tests/test_observability_export.py
@@ -0,0 +1,618 @@
+"""OB-6: sending recordings to Langfuse or an OpenTelemetry endpoint (phase-2 design §10).
+
+A recording is built in a real store with content planted everywhere a real session puts it:
+the user's text, OpenWorker's per-turn context block, the system prompt and tool list, the
+AI's answer and its thinking, a tool call's arguments and result (inside the next call's
+history too), the auto-approver's input, answer and reason, a summary, the session title
+call, a failed call's error text and cost, who answered a card, the machine's name, the
+session title, and two secrets. Every planted text holds `PLANT`.
+
+- Metadata only: no planted text in anything sent; a value planted in an exported field is
+ sent (the leak test of phase-2/lab/leak_test.py, as a repo test).
+- Each switch alone lets out only its own content, and rules R1 to R5 hold.
+- Sending: each step once per destination (marks survive a restart), retry and back-off,
+ a refused key, a refused span, against a fake OTLP endpoint on 127.0.0.1.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+import pytest
+
+from coworker.observability import export as ex
+from coworker.observability import export_spans as sp
+from coworker.observability.capture import store_exchange
+from coworker.observability.store import RecordingStore
+
+VECTORS = Path(__file__).parent / "corpora" / "observability" / "golden-phase2" / "vectors" / "masking.json"
+
+STORED_SECRET = "hunter2-correct-horse-PLANT"
+KEY_SHAPED = "sk-PLANTabcdefghijklmnopqrstu"
+T0 = 1_790_000_000_000
+
+
+# -- a recording with content planted everywhere ------------------------------------------------
+def _exchange(store: RecordingStore, request: dict, response: dict) -> dict:
+ return store_exchange({
+ "method": "POST", "url": "https://api.anthropic.com/v1/messages",
+ "request_headers": {"content-type": "application/json"},
+ "request_body": json.dumps(request).encode("utf-8"),
+ "status": 200, "response_headers": {}, "body": json.dumps(response),
+ }, store.put_piece)
+
+
+def _reply(*blocks: dict, stop: str = "end_turn") -> dict:
+ return {"type": "message", "role": "assistant", "content": list(blocks), "stop_reason": stop}
+
+
+def _usage(i: int = 100, o: int = 10) -> dict:
+ return {"reported": True, "input_fresh": i, "cache_read": 0, "cache_write": None, "output": o, "reasoning": None}
+
+
+def _model(sid, tid, step_id, seq, at, purpose, exchange, *, parent=None, call_id=None, after_turn=False, status="ok",
+ model="anthropic:claude-EXPORTED-zq81", **attrs):
+ a = {"model": model, "provider": "anthropic", "purpose": purpose, "stream": False,
+ "finish_reason": "end_turn", "tools_offered": 1, "usage": _usage(),
+ "provider_facts": {"cost_reported": 98765.4321, "served_model": "claude-x"}}
+ a.update(attrs)
+ return {"step_id": step_id, "turn_id": tid, "session_id": sid, "parent_step_id": parent, "seq": seq,
+ "kind": "model", "name": "model", "started_at": at, "duration_ms": 400, "status": status,
+ "call_id": call_id, "after_turn": after_turn, "attrs": a,
+ "content": {"exchange": [exchange]} if exchange else {}}
+
+
+def plant_turn(store: RecordingStore, sid: str = "sess-1", tid: str = "turn-1", at: int = T0,
+ *, long_text: str = "") -> None:
+ user = (f"USER-PLANT-u1 my key {KEY_SHAPED} and {STORED_SECRET} {long_text}"
+ "<system-context>CTX-PLANT-folders C:\\Users\\PLANT-home</system-context>")
+ system = "SYSTEM-PLANT-instructions"
+ tools = [{"name": "run_shell", "description": "TOOLDEF-PLANT", "input_schema": {"type": "object"}}]
+ m1 = _exchange(store, {"model": "claude-x", "system": system, "tools": tools,
+ "messages": [{"role": "user", "content": user}]},
+ _reply({"type": "thinking", "thinking": "THINK-PLANT-reasoning"},
+ {"type": "text", "text": "ASSIST-PLANT-a1"},
+ {"type": "tool_use", "id": "c1", "name": "run_shell", "input": {"command": "ARGS-PLANT-cmd"}},
+ stop="tool_use"))
+ rv = _exchange(store, {"model": "claude-x", "system": "You review actions.",
+ "messages": [{"role": "user", "content": "REVIEWER-IN-PLANT run_shell ARGS-PLANT-cmd"}]},
+ _reply({"type": "text", "text": "REVIEWER-OUT-PLANT allow"}))
+ m2 = _exchange(store, {"model": "claude-x", "system": system, "tools": tools, "messages": [
+ {"role": "user", "content": user},
+ {"role": "assistant", "content": [{"type": "tool_use", "id": "c1", "name": "run_shell",
+ "input": {"command": "ARGS-PLANT-cmd"}}]},
+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "c1", "content": "RESULT-PLANT-out"}]}]},
+ _reply({"type": "text", "text": "ASSIST-PLANT-a2 final"}))
+ sm = _exchange(store, {"model": "claude-x", "messages": [
+ {"role": "user", "content": "SUMMARY-IN-PLANT summarise RESULT-PLANT-out"}]},
+ _reply({"type": "text", "text": "SUMMARY-OUT-PLANT"}))
+ ti = _exchange(store, {"model": "claude-x", "messages": [{"role": "user", "content": "TITLE-IN-PLANT"}]},
+ _reply({"type": "text", "text": "TITLE-OUT-PLANT"}))
+ failed = _exchange(store, {"model": "claude-x", "messages": [{"role": "user", "content": user}]}, {})
+ store.write_turn({
+ "turn_id": tid, "session_id": sid, "seq": 1, "kind": "run", "trigger": "user", "started_at": at,
+ "duration_ms": 9000, "status": "completed", "approval_mode": "ask", "attended": True,
+ "stamp": {"machine": {"name": "PLANT-machine-name", "app_version": "9.9.9"}}, "schema_version": 1,
+ })
+ steps = [
+ {"step_id": f"{tid}-ctx", "kind": "context", "name": "context", "seq": 1, "started_at": at, "duration_ms": 1,
+ "status": "ok", "attrs": {"size_chars": 120, "block_hash": "abc"},
+ "content": {"agents_md": "AGENTS-PLANT"}},
+ _model(sid, tid, f"{tid}-m1", 2, at + 10, "main", m1),
+ {"step_id": f"{tid}-aa", "kind": "auto_approver", "name": "run_shell", "seq": 3, "started_at": at + 500,
+ "duration_ms": 300, "status": "ok", "call_id": "c1",
+ "attrs": {"mode": "live", "result": "allow", "reviewer": {"kind": "session_model", "model": "anthropic:claude-x"}},
+ "content": {"reason": "REVIEWER-REASON-PLANT", "questions": [{"name": "QUESTIONS-PLANT"}]}},
+ _model(sid, tid, f"{tid}-rm", 4, at + 510, "auto_approver", rv, parent=f"{tid}-aa", call_id="c1"),
+ {"step_id": f"{tid}-w", "kind": "wait", "name": "approval", "seq": 5, "started_at": at + 900,
+ "duration_ms": 1000, "status": "ok",
+ "attrs": {"kind": "approval", "where": "session", "outcome": "approved", "attended": True, "for_call_id": "c1"},
+ "content": {"resolved_by": "PLANT-person-bob"}},
+ {"step_id": f"{tid}-t", "kind": "tool", "name": "run_shell", "seq": 6, "started_at": at + 2000,
+ "duration_ms": 500, "status": "ok", "call_id": "c1",
+ "attrs": {"risk_class": "exec", "exit_code": 0, "result_bytes": 16,
+ "decision": {"origin": "user", "path": [{"stage": "permission", "outcome": "asked"}]}},
+ "content": {"decision_details": "DECISION-PLANT", "blocked": "BLOCKED-PLANT"}},
+ _model(sid, tid, f"{tid}-m2", 7, at + 2600, "main", m2),
+ {"step_id": f"{tid}-sum", "kind": "summary", "name": "summary", "seq": 8, "started_at": at + 3200,
+ "duration_ms": 900, "status": "ok", "attrs": {"reason": "overflow", "tokens_after": 700}},
+ _model(sid, tid, f"{tid}-sm", 9, at + 3210, "summary", sm, parent=f"{tid}-sum"),
+ _model(sid, tid, f"{tid}-err", 10, at + 4200, "main", failed, status="error",
+ error={"type": "rate_limit", "text": "ERRTEXT-PLANT https://PLANT.example/v1"}, usage={"reported": False}),
+ {"step_id": f"{tid}-n", "kind": "notice", "name": "interrupted", "seq": 11, "started_at": at + 4700,
+ "duration_ms": 0, "status": "ok", "attrs": {}},
+ _model(sid, tid, f"{tid}-title", 12, at + 8000, "title", ti, after_turn=True),
+ ]
+ for s in steps:
+ s.setdefault("turn_id", tid)
+ s.setdefault("session_id", sid)
+ store.write_step(s)
+ store.flush()
+
+
+class Captured:
+ """A `post` for the exporter that keeps what it was given and answers 200."""
+
+ def __init__(self, answers=None):
+ self.bodies: list[dict] = []
+ self.headers: list[dict] = []
+ self.answers = list(answers or [])
+
+ def __call__(self, url, body, headers, timeout):
+ self.bodies.append(json.loads(body))
+ self.headers.append(headers)
+ return self.answers.pop(0) if self.answers else (200, "")
+
+ def text(self) -> str:
+ return json.dumps(self.bodies, ensure_ascii=False)
+
+ def spans(self) -> list[dict]:
+ return [s for b in self.bodies for r in b["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]
+
+
+def _attr(span: dict, key: str):
+ for a in span["attributes"]:
+ if a["key"] == key:
+ v = a["value"]
+ return next(iter(v.values()))
+ return None
+
+
+def export_once(tmp_path, include: dict, *, dest=None, long_text: str = "") -> Captured:
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ plant_turn(store, long_text=long_text)
+ cap = Captured()
+ cfg = ex.ExportConfig(destination=dest or ex.Destination(kind="langfuse", host="http://127.0.0.1:9"),
+ headers={"Authorization": "Basic x"}, include=include, since_ms=0)
+ exporter = ex.Exporter(store, cfg, post=cap, secret_values=lambda: [STORED_SECRET],
+ session_title=lambda _sid: "TITLE-PLANT-session")
+ exporter.flush_once()
+ store.close()
+ return cap
+
+
+OFF = {"messages": False, "tool_io": False, "context": False, "names": False}
+
+
+def on(*keys: str) -> dict:
+ return {**OFF, **{k: True for k in keys}}
+
+
+# -- 2. metadata only carries no content --------------------------------------------------------------
[email protected]("dest", [ex.Destination(kind="langfuse", host="http://127.0.0.1:9"),
+ ex.Destination(kind="otlp", url="http://127.0.0.1:9/v1/traces")])
+def test_metadata_only_export_carries_no_content(tmp_path, dest):
+ cap = export_once(tmp_path, OFF, dest=dest)
+ sent = cap.text()
+ assert len(cap.spans()) == 12 # the turn and 11 steps; the notice is an event on the turn
+ # The positive control: a value planted in an exported field is found by the same search.
+ assert "claude-EXPORTED-zq81" in sent
+ # Nothing planted in content is.
+ assert "PLANT" not in sent
+ assert "98765" not in sent # provider-reported cost (R5)
+ assert "C:\\\\Users" not in sent and "Users" not in sent
+ # Metadata is there: names, kinds, counts, the verdict, the notice.
+ spans = cap.spans()
+ names = {s["name"] for s in spans}
+ assert {"invoke_agent coworker", "openworker.context.build", "chat claude-EXPORTED-zq81",
+ "openworker.policy.review run_shell", "openworker.approval.wait", "execute_tool run_shell",
+ "openworker.compaction"} <= names
+ review = next(s for s in spans if s["name"] == "openworker.policy.review run_shell")
+ assert _attr(review, "openworker.reviewer.result") == "allow"
+ root = next(s for s in spans if s["name"] == "invoke_agent coworker")
+ assert [e["name"] for e in root["events"]] == ["openworker.notice.interrupted"]
+ failed = next(s for s in spans if _attr(s, "openworker.step.id") == "turn-1-err")
+ assert _attr(failed, "error.type") == "rate_limit" and failed["status"]["code"] == 2
+
+
+# -- 3. each switch lets out only its own content --------------------------------------------------------
+ALL_PLANTS = ["USER-PLANT", "ASSIST-PLANT", "CTX-PLANT", "SYSTEM-PLANT", "TOOLDEF-PLANT", "ARGS-PLANT",
+ "RESULT-PLANT", "REVIEWER-IN-PLANT", "REVIEWER-OUT-PLANT", "REVIEWER-REASON-PLANT",
+ "SUMMARY-IN-PLANT", "SUMMARY-OUT-PLANT", "TITLE-IN-PLANT", "TITLE-OUT-PLANT",
+ "TITLE-PLANT-session", "PLANT-person-bob", "PLANT-machine-name"]
+NEVER = ["THINK-PLANT", "ERRTEXT-PLANT", "98765", "QUESTIONS-PLANT", "DECISION-PLANT", "AGENTS-PLANT",
+ "BLOCKED-PLANT", STORED_SECRET, KEY_SHAPED]
+
+CASES = {
+ # R1, R2, R3: the user's and the AI's own text, the title call; no tool parts, no context
+ # block, no reviewer or summary text.
+ "messages": (on("messages"), {"USER-PLANT", "ASSIST-PLANT", "TITLE-IN-PLANT", "TITLE-OUT-PLANT"}),
+ # Tool spans only: the reviewer's input and reason need the messages switch too (R3).
+ "tool_io": (on("tool_io"), {"ARGS-PLANT", "RESULT-PLANT"}),
+ "context": (on("context"), {"SYSTEM-PLANT", "TOOLDEF-PLANT"}),
+ "names": (on("names"), {"TITLE-PLANT-session", "PLANT-person-bob", "PLANT-machine-name"}),
+ "messages+tool_io": (on("messages", "tool_io"),
+ {"USER-PLANT", "ASSIST-PLANT", "ARGS-PLANT", "RESULT-PLANT", "REVIEWER-IN-PLANT",
+ "REVIEWER-OUT-PLANT", "REVIEWER-REASON-PLANT", "SUMMARY-IN-PLANT", "SUMMARY-OUT-PLANT",
+ "TITLE-IN-PLANT", "TITLE-OUT-PLANT"}),
+ "messages+context": (on("messages", "context"),
+ {"USER-PLANT", "ASSIST-PLANT", "CTX-PLANT", "SYSTEM-PLANT", "TOOLDEF-PLANT",
+ "TITLE-IN-PLANT", "TITLE-OUT-PLANT"}),
+ "everything": (on("messages", "tool_io", "context", "names"), set(ALL_PLANTS)),
+}
+
+
[email protected]("case", list(CASES))
+def test_each_switch_lets_out_only_its_content(tmp_path, case):
+ include, allowed = CASES[case]
+ sent = export_once(tmp_path, include).text()
+ for plant in ALL_PLANTS:
+ assert (plant in sent) == (plant in allowed), f"{case}: {plant}"
+ for plant in NEVER: # R5 and R4, whatever the switches say
+ assert plant not in sent, f"{case}: {plant}"
+
+
+def test_tool_parts_inside_messages_are_masked_unless_the_tool_switch_is_on(tmp_path):
+ cap = export_once(tmp_path, on("messages"))
+ main = [s for s in cap.spans() if _attr(s, "openworker.step.id") == "turn-1-m2"][0]
+ history = json.loads(_attr(main, "gen_ai.input.messages"))
+ parts = [p for m in history for p in m["parts"]]
+ call = next(p for p in parts if p["type"] == "tool_call")
+ result = next(p for p in parts if p["type"] == "tool_call_response")
+ # R1: the part, its id and its tool name stay; its content does not.
+ assert (call["id"], call["name"], call["arguments"]) == ("c1", "run_shell", sp.TOOL_ARGS_OUT)
+ assert (result["id"], result["response"]) == ("c1", sp.TOOL_RESULT_OUT)
+ # R2: the context block is cut out of the user's text, the rest kept.
+ user = next(p["content"] for m in history if m["role"] == "user" for p in m["parts"] if p["type"] == "text")
+ assert user.startswith("USER-PLANT-u1") and user.endswith(sp.CONTEXT_OUT)
+ # R4: the stored secret and the key-shaped one are masked.
+ assert user.count(sp.SECRET) == 2
+ # Reasoning keeps its marker, never its text (R5).
+ reply = json.loads(_attr([s for s in cap.spans() if _attr(s, "openworker.step.id") == "turn-1-m1"][0],
+ "gen_ai.output.messages"))
+ assert {"type": "reasoning", "content": sp.REASONING} in reply[0]["parts"]
+ # The turn's own input and output (Langfuse).
+ root = cap.spans()[0]
+ assert _attr(root, "langfuse.trace.input").endswith(sp.CONTEXT_OUT)
+ assert _attr(root, "langfuse.trace.output") == "ASSIST-PLANT-a2 final"
+
+
+def test_tool_switch_with_messages_lets_tool_parts_into_messages(tmp_path):
+ cap = export_once(tmp_path, on("messages", "tool_io"))
+ main = [s for s in cap.spans() if _attr(s, "openworker.step.id") == "turn-1-m2"][0]
+ history = _attr(main, "gen_ai.input.messages")
+ assert "ARGS-PLANT-cmd" in history and "RESULT-PLANT-out" in history
+ tool = [s for s in cap.spans() if s["name"] == "execute_tool run_shell"][0]
+ assert json.loads(_attr(tool, "gen_ai.tool.call.arguments")) == {"command": "ARGS-PLANT-cmd"}
+ assert _attr(tool, "gen_ai.tool.call.result") == "RESULT-PLANT-out"
+ # gen_ai.tool.call.id only on the tool's own span (V-22).
+ assert [s["name"] for s in cap.spans() if _attr(s, "gen_ai.tool.call.id")] == ["execute_tool run_shell"]
+
+
+def test_exported_text_is_cut_at_8_kb(tmp_path):
+ cap = export_once(tmp_path, on("messages"), long_text="x" * 20_000)
+ main = [s for s in cap.spans() if _attr(s, "openworker.step.id") == "turn-1-m1"][0]
+ user = json.loads(_attr(main, "gen_ai.input.messages"))[0]["parts"][0]["content"]
+ assert user.endswith(" more bytes]") and "[cut: " in user
+ kept = user[: user.index("[cut: ")]
+ assert len(kept.encode("utf-8")) == sp.CUT_BYTES
+
+
+def test_masking_and_cut_match_the_shared_vectors():
+ vectors = json.loads(VECTORS.read_text(encoding="utf-8"))
+ for v in vectors["mask"]:
+ assert sp.Masker(v["secrets"]).mask(v["text"]) == v["expected"], v["text"]
+ for v in vectors["cut"]:
+ text = "a" * 8191 + "é" + "b" * 10
+ assert len(text.encode("utf-8")) == v["text_bytes"]
+ out = sp.cut(text)
+ kept = out[: out.index("[cut:")]
+ assert len(kept.encode("utf-8")) == v["expected_kept_bytes"]
+ assert out.endswith(v["expected_tail"])
+ # The patterns are the ones in attributes.yaml export.content_switches.masking.
+ assert sp.CUT_BYTES == 8192 and len(sp.MASK_PATTERNS) == 7
+
+
+def test_credential_looking_environment_variables_are_masked():
+ env = {"MY_API_KEY": "PLANT-envkey-123", "SHORT_TOKEN": "abc", "HOME": "PLANT-not-a-secret"}
+ m = sp.Masker(sp.env_secret_values(env))
+ assert m.mask("a PLANT-envkey-123 b abc PLANT-not-a-secret") == "a [secret masked] b abc PLANT-not-a-secret"
+
+
+def test_token_numbers_follow_the_verified_rules(tmp_path):
+ cap = export_once(tmp_path, OFF)
+ spans = cap.spans()
+ m1 = [s for s in spans if _attr(s, "openworker.step.id") == "turn-1-m1"][0]
+ assert _attr(m1, "gen_ai.usage.input_tokens") == "100"
+ assert _attr(m1, "gen_ai.usage.cache_read.input_tokens") == "0"
+ assert _attr(m1, "gen_ai.usage.cache_creation.input_tokens") is None # not reported: left out
+ assert json.loads(_attr(m1, "langfuse.observation.usage_details")) == {"input": 100, "input_cache_read": 0, "output": 10}
+ root = spans[0]
+ assert _attr(root, "gen_ai.usage.input_tokens") is None # never on a parent span
+ assert _attr(root, "openworker.usage.failed_calls") == "1"
+ assert _attr(root, "langfuse.trace.tags") == {"values": [{"stringValue": "usage-incomplete"}]}
+ err = [s for s in spans if _attr(s, "openworker.step.id") == "turn-1-err"][0]
+ assert _attr(err, "langfuse.observation.level") == "ERROR"
+ assert _attr(err, "langfuse.observation.status_message") == "rate_limit; usage unknown"
+ # One trace per turn, children under their parents.
+ assert len({s["traceId"] for s in spans}) == 1
+ rm = [s for s in spans if _attr(s, "openworker.step.id") == "turn-1-rm"][0]
+ aa = [s for s in spans if _attr(s, "openworker.step.id") == "turn-1-aa"][0]
+ assert rm["parentSpanId"] == aa["spanId"] and aa["parentSpanId"] == root["spanId"]
+ resource = cap.bodies[0]["resourceSpans"][0]["resource"]["attributes"]
+ assert {"key": "openworker.export.schema", "value": {"stringValue": "openworker/1.2"}} in resource
+ assert not any(a["key"] == "host.name" for a in resource) # the machine name: names switch only
+
+
+# -- 4. each step once; retry; refused key; refused span ------------------------------------------------
+class FakeOtlp:
+ """A local OTLP/HTTP endpoint. `answers` is a list of status codes for the next posts;
+ `refuse` is a set of span ids refused (400) whenever they are in a post."""
+
+ def __init__(self):
+ self.answers: list[int] = []
+ self.refuse: set[str] = set()
+ self.received: list[dict] = []
+ self.auth: list[str] = []
+ outer = self
+
+ class Handler(BaseHTTPRequestHandler):
+ def do_POST(self): # noqa: N802
+ body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+ outer.auth.append(self.headers.get("Authorization") or "")
+ ids = [s["spanId"] for r in body["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]
+ code = outer.answers.pop(0) if outer.answers else 200
+ if code == 200 and outer.refuse & set(ids):
+ code = 400
+ if code == 200:
+ outer.received.append(body)
+ self.send_response(code)
+ self.end_headers()
+ self.wfile.write(b'{"message":"refused here"}' if code != 200 else b"{}")
+
+ def log_message(self, *a):
+ pass
+
+ self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self.url = f"http://127.0.0.1:{self.server.server_address[1]}/v1/traces"
+ threading.Thread(target=self.server.serve_forever, daemon=True).start()
+
+ def span_ids(self) -> list[str]:
+ return [s["spanId"] for b in self.received for r in b["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]
+
+ def close(self):
+ self.server.shutdown()
+
+
+def otlp():
+ f = FakeOtlp()
+ yield f
+ f.close()
+
+
+def _cfg(url: str, since: int = 0) -> ex.ExportConfig:
+ return ex.ExportConfig(destination=ex.Destination(kind="otlp", url=url), headers={"Authorization": "Bearer t"},
+ include=OFF, since_ms=since)
+
+
+def test_each_step_is_sent_once_and_the_marks_survive_a_restart(tmp_path, otlp):
+ db = tmp_path / "recordings.db"
+ store = RecordingStore(db, background=False)
+ plant_turn(store, tid="turn-1")
+ plant_turn(store, tid="turn-2", at=T0 + 20_000)
+ exporter = ex.Exporter(store, _cfg(otlp.url))
+ assert exporter.flush_once() == 24
+ assert exporter.flush_once() == 0 # nothing new
+ store.close()
+ # A restart: a new store on the same file, a new exporter.
+ store = RecordingStore(db, background=False)
+ assert ex.Exporter(store, _cfg(otlp.url)).flush_once() == 0
+ ids = otlp.span_ids()
+ assert len(ids) == len(set(ids)) == 24
+ # A step that ends after its turn was sent (a late title call) is sent alone, once.
+ store.write_step({"step_id": "turn-1-late", "turn_id": "turn-1", "session_id": "sess-1", "seq": 13,
+ "kind": "model", "name": "model", "started_at": T0 + 9500, "duration_ms": 100, "status": "ok",
+ "after_turn": True, "attrs": {"model": "anthropic:claude-x", "purpose": "title", "usage": _usage()}})
+ store.flush()
+ assert ex.Exporter(store, _cfg(otlp.url)).flush_once() == 1
+ assert len(otlp.span_ids()) == 25
+ assert store.export_counts("otlp:" + otlp.url)["sent"] == 2 * 12 + 2 * 1 + 1 # notices ride with their turn
+ store.close()
+
+
+def test_a_new_destination_starts_from_turns_that_end_after_the_change(tmp_path, otlp):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ plant_turn(store, tid="old", at=T0)
+ plant_turn(store, tid="new", at=T0 + 60_000)
+ ex.Exporter(store, _cfg(otlp.url, since=T0 + 30_000)).flush_once()
+ assert {s["traceId"] for b in otlp.received for r in b["resourceSpans"] for ss in r["scopeSpans"]
+ for s in ss["spans"]} == {sp.trace_id("new")}
+ store.close()
+
+
+def _wait_for(cond, timeout=10.0):
+ end = time.time() + timeout
+ while time.time() < end:
+ if cond():
+ return True
+ time.sleep(0.02)
+ return False
+
+
+def test_retry_with_back_off_then_sent(tmp_path, otlp):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ plant_turn(store)
+ otlp.answers = [503, 503, 200]
+ exporter = ex.Exporter(store, _cfg(otlp.url), backoff_start=0.05, flush_seconds=0.05)
+ seen: list[tuple[str, str]] = []
+ exporter.start()
+ try:
+ assert _wait_for(lambda: seen.append((exporter.state, exporter.last_error)) or len(otlp.span_ids()) == 12)
+ finally:
+ exporter.stop()
+ assert ("retrying", "HTTP 503: {\"message\":\"refused here\"}") in seen
+ assert exporter.state == "sending" and exporter.last_error == ""
+ store.close()
+
+
+def test_a_network_error_is_retried(tmp_path):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ plant_turn(store)
+ exporter = ex.Exporter(store, _cfg("http://127.0.0.1:1/v1/traces"), backoff_start=0.05)
+ with pytest.raises(ex._Retry):
+ exporter.flush_once()
+ assert store.export_marked("otlp:http://127.0.0.1:1/v1/traces", "turn-1") == set()
+ store.close()
+
+
+def test_a_refused_key_stops_sending_until_the_settings_change(tmp_path, otlp):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ plant_turn(store)
+ otlp.answers = [401]
+ exporter = ex.Exporter(store, _cfg(otlp.url), backoff_start=0.05, flush_seconds=0.05)
+ exporter.start()
+ try:
+ assert _wait_for(lambda: exporter.state == "failed")
+ time.sleep(0.2)
+ assert otlp.span_ids() == [] # it stopped: no second try with the same key
+ assert exporter.last_error == "The destination refused the key"
+ assert store.export_counts("otlp:" + otlp.url)["failed"] == 1
+ finally:
+ exporter.stop()
+ store.close()
+
+
+def test_a_span_refused_on_its_own_is_dropped_and_the_rest_sent(tmp_path, otlp):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ plant_turn(store)
+ otlp.refuse = {sp.step_span_id("turn-1-t")}
+ ex.Exporter(store, _cfg(otlp.url)).flush_once()
+ ids = otlp.span_ids()
+ assert len(ids) == 11 and sp.step_span_id("turn-1-t") not in ids
+ assert store.export_counts("otlp:" + otlp.url)["dropped"] == 1
+ store.close()
+
+
+def test_a_turn_deleted_before_it_was_sent_counts_as_dropped(tmp_path):
+ store = RecordingStore(tmp_path / "recordings.db", background=False, retention_days=7)
+ plant_turn(store, at=T0)
+ store.export_destination = ("otlp:http://x/v1/traces", 0)
+ store.sweep(now_ms=T0 + 8 * 24 * 3600 * 1000)
+ assert store.export_counts("otlp:http://x/v1/traces")["dropped"] == 1
+ store.close()
+
+
+def test_batches_hold_at_most_the_batch_size(tmp_path):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ for i in range(3):
+ plant_turn(store, tid=f"t{i}", at=T0 + i * 20_000)
+ cap = Captured()
+ ex.Exporter(store, _cfg("http://127.0.0.1:9/v1/traces"), post=cap, batch_spans=5).flush_once()
+ sizes = [len([s for r in b["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]) for b in cap.bodies]
+ assert max(sizes) <= 5 and sum(sizes) == 36
+ store.close()
+
+
+# -- settings, keys and the test recording ------------------------------------------------------------
+class FakeSecrets:
+ def __init__(self):
+ self.data: dict = {}
+
+ def get(self, profile):
+ return self.data.get(profile)
+
+ def put(self, profile, value):
+ self.data[profile] = json.loads(json.dumps(value))
+
+ def delete(self, profile):
+ return self.data.pop(profile, None) is not None
+
+ def _read(self):
+ return self.data
+
+
+def test_settings_keep_keys_out_of_the_settings_file_and_answers(tmp_path):
+ secrets = FakeSecrets()
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ obs = ex.ObservabilityExport(tmp_path, secrets, store, post=Captured())
+ try:
+ assert obs.exporter is None and obs.status()["state"] == "off"
+ code, answer = obs.put_settings({
+ "destination": {"kind": "langfuse", "host": "https://langfuse.example.com"},
+ "secrets": {"public_key": "pk-lf-PLANT-public-3a91", "secret_key": "sk-lf-PLANT-secret-key"},
+ "include": {"messages": True}})
+ assert code == 200 and answer["ok"]
+ assert answer["secrets"] == {"public_key": "pk-lf-••••••••3a91", "secret_key": "sk-lf-••••••••"}
+ assert "PLANT" not in json.dumps(answer) and "PLANT" not in json.dumps(obs.get_settings())
+ assert "PLANT" not in (tmp_path / ex.SETTINGS_FILE).read_text(encoding="utf-8")
+ assert secrets.data[ex.PROFILE]["user"]["secret_key"] == "sk-lf-PLANT-secret-key"
+ assert obs.exporter is not None and obs.exporter.config.destination.endpoint == \
+ "https://langfuse.example.com/api/public/otel/v1/traces"
+ # Off again: the exporter goes away, its thread with it.
+ obs.put_settings({"destination": {"kind": "off"}})
+ assert obs.exporter is None
+ assert not any(t.name == "observability-export" for t in threading.enumerate())
+ # Bad input is refused, not saved.
+ assert obs.put_settings({"destination": {"kind": "otlp", "url": "ftp://x"}})[0] == 400
+ assert obs.put_settings({"retention_days": 12})[0] == 400
+ finally:
+ obs.close()
+ store.close()
+
+
+def test_the_test_recording_reports_a_refused_key(tmp_path, otlp):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ obs = ex.ObservabilityExport(tmp_path, FakeSecrets(), store)
+ try:
+ obs.put_settings({"destination": {"kind": "otlp", "url": otlp.url, "header_names": ["Authorization"]},
+ "secrets": {"headers": {"Authorization": "Bearer PLANT-token"}}})
+ otlp.answers = [200]
+ ok = obs.send_test()
+ assert ok["ok"] and ok["text"] == "Sent." and ok["status"] == 200
+ assert otlp.auth[-1] == "Bearer PLANT-token"
+ [span_names] = [[s["name"] for r in b["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]
+ for b in otlp.received[-1:]]
+ assert span_names == ["invoke_agent coworker", "chat test-model", "execute_tool test_tool"]
+ otlp.answers = [401]
+ refused = obs.send_test()
+ assert refused == {"ok": False, "status": 401, "text": "The destination refused the key", "ms": refused["ms"]}
+ otlp.answers = [500]
+ assert obs.send_test()["text"].startswith("HTTP 500")
+ finally:
+ obs.close()
+ store.close()
+
+
+POLICY = {"org_id": "org_1", "org_name": "Acme Corp", "version": 3, "observability_export": {
+ "enabled": True, "name": "Security team default", "set_at": "2026-10-12",
+ "destination": {"kind": "langfuse", "host": "https://langfuse.acme.internal",
+ "public_key": "pk-lf-POLICY-PLANT-0001", "secret_key": "sk-lf-POLICY-PLANT"},
+ "include": {"messages": True, "names": True}}}
+
+
+def test_a_company_policy_manages_the_page(tmp_path):
+ secrets = FakeSecrets()
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ obs = ex.ObservabilityExport(tmp_path, secrets, store, post=Captured())
+ try:
+ stripped = ex.take_policy_secrets(POLICY, secrets)
+ assert "PLANT" not in json.dumps(stripped) # what policy.json will hold
+ assert secrets.data[ex.PROFILE]["policy"]["secret_key"] == "sk-lf-POLICY-PLANT"
+ obs.apply_policy(stripped)
+ got = obs.get_settings()
+ assert got["managed"] == {"by": "Acme Corp", "name": "Security team default", "at": "2026-10-12"}
+ assert got["destination"] == {"kind": "langfuse", "host": "https://langfuse.acme.internal"}
+ assert got["include"] == {"messages": True, "tool_io": False, "context": False, "names": True}
+ assert got["notice"] == "Acme Corp receives this session's messages and names (Langfuse, langfuse.acme.internal)."
+ assert "PLANT" not in json.dumps(got)
+ assert obs.put_settings({"include": {"tool_io": True}})[0] == 409
+ assert obs.put_settings({"destination": {"kind": "off"}})[0] == 409
+ assert obs.put_settings({"retention_days": 7})[0] == 200 # on this machine stays the person's
+ assert obs.exporter is not None and obs.exporter.config.managed
+ assert obs.status()["state"] in ("managed", "sending", "retrying")
+ # Anything malformed reads as not managed.
+ for bad in ({"observability_export": {"enabled": "yes"}},
+ {"observability_export": {"enabled": True, "destination": {"kind": "smtp"}}},
+ {"observability_export": []}):
+ assert ex.managed_export(bad) is None
+ # The policy taken away: the person's own settings are back, the policy's keys gone.
+ obs.apply_policy(ex.take_policy_secrets({"org_id": "org_1"}, secrets))
+ assert obs.get_settings()["managed"] is None and "policy" not in (secrets.data.get(ex.PROFILE) or {})
+ finally:
+ obs.close()
+ store.close()
diff --git a/tests/test_observability_export_server.py b/tests/test_observability_export_server.py
new file mode 100644
index 00000000..4af2cc4a
--- /dev/null
+++ b/tests/test_observability_export_server.py
@@ -0,0 +1,141 @@
+"""OB-6 through the real server: the settings, status and test endpoints, the managed state,
+the session notice, Record steps off, the recording file, and Off meaning no exporter and no
+connection at all (phase-2 design §10.3, §10.6 to §10.8)."""
+
+from __future__ import annotations
+
+import json
+import threading
+
+from fastapi.testclient import TestClient
+
+from coworker.observability import export as ex
+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient
+from coworker.providers.base import TokenUsage
+from coworker.remote import joiner
+from coworker.server import SessionManager, create_app
+
+
+class _Scripted(ProviderClient):
+ def complete(self, *, model, messages, tools=None, **settings):
+ if str(messages[0].get("content") or "").startswith("You title"):
+ return AssistantTurn(text="Notes", usage=TokenUsage(input=40, output=2))
+ return AssistantTurn(text="Done.", finish_reason="stop", usage=TokenUsage(input=20, output=5))
+
+ def capabilities(self, model):
+ return ModelCapabilities()
+
+
+def _turn(client, sid: str, text: str = "hello") -> None:
+ with client.websocket_connect(f"/ws/session/{sid}") as ws:
+ assert ws.receive_json()["type"] in ("ready", "export_notice")
+ ws.send_json({"type": "user_message", "text": text})
+ while ws.receive_json()["type"] != "turn_done":
+ pass
+
+
+def _app(tmp_path, monkeypatch, posts: list):
+ def post(url, body, headers, timeout):
+ posts.append(url)
+ return 200, ""
+
+ # Every send the exporter or the test button could make goes through `post`; with Off it
+ # must never be called.
+ manager = SessionManager(workspace=tmp_path, provider=_Scripted())
+ manager.observability.post = post
+ return manager, TestClient(create_app(manager))
+
+
+def test_off_creates_no_exporter_and_sends_nothing(tmp_path, monkeypatch):
+ posts: list = []
+ manager, client = _app(tmp_path, monkeypatch, posts)
+ _turn(client, "s1")
+ assert client.get("/v1/sessions/s1/turns").json()["turns"] # recording works
+ assert manager.observability.exporter is None
+ assert not any(t.name == "observability-export" for t in threading.enumerate())
+ assert client.get("/v1/observability/status").json()["state"] == "off"
+ test = client.post("/v1/observability/test").json()
+ assert test["ok"] is False and posts == []
+
+
+def test_settings_endpoints_are_write_only_for_keys(tmp_path, monkeypatch):
+ posts: list = []
+ manager, client = _app(tmp_path, monkeypatch, posts)
+ got = client.get("/v1/observability/settings").json()
+ assert got == {"record": True, "retention_days": 30, "destination": {"kind": "off"},
+ "include": {"messages": False, "tool_io": False, "context": False, "names": False},
+ "secrets": {}, "managed": None, "notice": None}
+ r = client.put("/v1/observability/settings", json={
+ "destination": {"kind": "otlp", "url": "http://127.0.0.1:9/v1/traces", "header_names": ["x-honeycomb-team"]},
+ "secrets": {"headers": {"x-honeycomb-team": "PLANT-header-value"}}, "retention_days": 7})
+ assert r.status_code == 200
+ body = r.json()
+ assert body["secrets"] == {"headers": {"x-honeycomb-team": "••••••••"}}
+ for answer in (body, client.get("/v1/observability/settings").json(), client.get("/v1/observability/status").json()):
+ assert "PLANT" not in json.dumps(answer)
+ state = manager.secrets.path.parent
+ assert "PLANT" not in (state / ex.SETTINGS_FILE).read_text(encoding="utf-8")
+ assert manager.secrets.get(ex.PROFILE)["user"]["headers"] == {"x-honeycomb-team": "PLANT-header-value"}
+ assert manager.recordings_store.retention_days == 7
+ assert manager.observability.exporter is not None
+ # A turn after the change is sent; the status counts it.
+ _turn(client, "s2")
+ manager.observability.exporter.flush_once()
+ status = client.get("/v1/observability/status").json()
+ assert status["state"] == "sending" and status["sent"] > 0 and status["last_sent_at"]
+ assert set(status) >= {"state", "last_sent_at", "sent", "failed", "dropped", "last_error"}
+ assert posts and all(u == "http://127.0.0.1:9/v1/traces" for u in posts)
+ manager.observability.close()
+
+
+def test_a_company_policy_makes_the_page_read_only_and_tells_new_sessions(tmp_path, monkeypatch):
+ posts: list = []
+ manager, client = _app(tmp_path, monkeypatch, posts)
+ policy = {"org_id": "org_1", "org_name": "Acme Corp", "version": 2, "observability_export": {
+ "enabled": True, "name": "Security team default", "set_at": "2026-10-12",
+ "destination": {"kind": "langfuse", "host": "https://langfuse.acme.internal",
+ "public_key": "pk-lf-PLANT-0001", "secret_key": "sk-lf-PLANT"},
+ "include": {"messages": True}}}
+
+ class _App:
+ class state: # noqa: N801
+ pass
+ _App.state.manager = manager
+ state = manager.secrets.path.parent
+ applied, _ = joiner.apply_policy(_App, policy, state=state)
+ assert applied
+ assert "PLANT" not in (state / "policy.json").read_text(encoding="utf-8")
+ got = client.get("/v1/observability/settings").json()
+ assert got["managed"]["by"] == "Acme Corp" and got["destination"]["host"] == "https://langfuse.acme.internal"
+ assert "PLANT" not in json.dumps(got)
+ r = client.put("/v1/observability/settings", json={"include": {"tool_io": True}})
+ assert r.status_code == 409 and r.json()["error"] == "Managed by Acme Corp"
+ # Each new session says what goes where, once.
+ with client.websocket_connect("/ws/session/fresh") as ws:
+ msgs = [ws.receive_json() for _ in range(2)]
+ notice = next(m for m in msgs if m["type"] == "export_notice")
+ assert notice["data"]["text"] == "Acme Corp receives this session's messages (Langfuse, langfuse.acme.internal)."
+ kinds = [m.get("kind") for m in client.get("/v1/sessions/fresh/messages").json()["messages"]]
+ assert kinds.count("export_notice") == 1
+ manager.observability.close()
+
+
+def test_record_steps_off_records_nothing_for_new_sessions(tmp_path, monkeypatch):
+ manager, client = _app(tmp_path, monkeypatch, [])
+ assert client.put("/v1/observability/settings", json={"record": False}).status_code == 200
+ _turn(client, "quiet")
+ assert client.get("/v1/sessions/quiet/turns").json()["turns"] == []
+ assert client.get("/v1/observability/settings").json()["record"] is False
+
+
+def test_the_recording_file_holds_the_whole_session(tmp_path, monkeypatch):
+ manager, client = _app(tmp_path, monkeypatch, [])
+ _turn(client, "s9", "what is in notes?")
+ r = client.get("/v1/sessions/s9/recording")
+ assert r.status_code == 200
+ assert 'attachment; filename="recording-s9.json"' == r.headers["content-disposition"]
+ data = r.json()
+ assert data["session_id"] == "s9" and data["version"] == "1.2"
+ [turn] = data["turns"]
+ steps = data["steps"][turn["turn_id"]]
+ assert steps and all(s["step_id"] in data["details"] for s in steps)