Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

12.8 KB
Raw Blame
"""Audit recorded voice sessions against the known failure classes.

Reads the flight recorder (``data/flight_recorder/YYYY-MM-DD.jsonl``), splits
it into voice sessions, and checks every session for the failure patterns
confirmed in the 2026-07-13 realtime forensics (BUG-047..BUG-050):

- ``promise-without-action`` — a turn ends on a deferred-action claim while
  its ``tool_calls`` are empty (the model promised and nothing ran).
- ``voice-identity-break`` — a non-realtime layer spoke (classic pipeline
  TTS) while a live realtime session owned the voice.
- ``tool-retry-loop`` — the same tool failed three or more times inside one
  session (the model grinding against a broken/lying tool).
- ``exhausted-brain-turn`` — a brain turn died on ``budget_exceeded`` or
  returned no text at all (outside the legitimate suppress paths).
- ``silent-turn`` — a user turn completed without any assistant text.

Plus every realtime transport-health class from the ``RealtimeSessionPostmortem``
event (splice seams, ungrounded caption/refusal storms, event-loop stalls,
sender-behind-wall-clock, rebuild loops, mute emergency releases, language
flips, spawn budget) — the verdict logic is shared with the codex live-call
probe via ``dodaios.diagnostics.realtime_forensics`` so the two can never
disagree.

Usage::

    python scripts/diag_voice_sessions.py            # today's sessions
    python scripts/diag_voice_sessions.py --date 2026-07-13
    python scripts/diag_voice_sessions.py --last 3   # newest N sessions only
    python scripts/diag_voice_sessions.py --harness data/diagnostics/codex_live/<ts>/round.jsonl

Read-only developer diagnostic: no app, network, or config access. The
``--harness`` mode exits 1 when any HIGH finding survives, so probe rounds
can gate on it.
"""
from __future__ import annotations

import argparse
import json
import sys
from collections import Counter
from dataclasses import dataclass, field
from datetime import date
from pathlib import Path
from typing import Any

REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT))

from dodaios.brain.action_honesty import has_unbacked_action_claim  # noqa: E402
from dodaios.diagnostics.realtime_forensics import (  # noqa: E402
    evaluate_postmortem,
    extract_postmortems,
)

_MAX_PREVIEW_CHARS = 90
_RETRY_LOOP_THRESHOLD = 3
# Brain turns that legitimately produce no user-facing text.
_SUPPRESSED_FINISH_REASONS = frozenset({"suppress_response", "voice_confirm_pending"})


@dataclass(slots=True)
class Finding:
    severity: str  # "high" | "warn" | "info"
    kind: str
    detail: str


@dataclass(slots=True)
class Session:
    session_id: str
    started_ts_ns: int
    events: list[dict[str, Any]] = field(default_factory=list)
    findings: list[Finding] = field(default_factory=list)

    @property
    def turn_count(self) -> int:
        return sum(1 for e in self.events if e.get("event") == "VoiceTurnCompleted")


def _preview(text: Any) -> str:
    cleaned = " ".join(str(text or "").split())
    if len(cleaned) <= _MAX_PREVIEW_CHARS:
        return cleaned
    return cleaned[: _MAX_PREVIEW_CHARS - 1] + "…"


def _split_sessions(events: list[dict[str, Any]]) -> list[Session]:
    """Assign events to sessions by id where present, else by time window."""
    sessions: list[Session] = []
    active: Session | None = None
    for event in events:
        payload = event.get("payload") or {}
        name = event.get("event", "")
        if name == "VoiceSessionStarted":
            active = Session(
                session_id=str(payload.get("session_id", "?")),
                started_ts_ns=int(event.get("ts_ns", 0)),
            )
            sessions.append(active)
        if active is not None:
            event_session = payload.get("session_id")
            if event_session is None or event_session == active.session_id:
                active.events.append(event)
        if name == "VoiceSessionEnded" and active is not None:
            if payload.get("session_id") in (None, active.session_id):
                active = None
    return sessions


def _check_promise_without_action(session: Session) -> None:
    for event in session.events:
        if event.get("event") != "VoiceTurnCompleted":
            continue
        payload = event.get("payload") or {}
        jarvis_text = str(payload.get("jarvis_text", "") or "")
        tool_calls = payload.get("tool_calls") or ()
        if jarvis_text and not tool_calls and has_unbacked_action_claim(jarvis_text):
            session.findings.append(
                Finding(
                    severity="high",
                    kind="promise-without-action",
                    detail=(
                        "turn ended on a future-work claim with no tool call: "
                        f"\"{_preview(jarvis_text)}\""
                    ),
                )
            )


def _check_voice_identity(session: Session) -> None:
    realtime_live = False
    for event in session.events:
        name = event.get("event", "")
        if name == "RealtimeSessionReady":
            realtime_live = True
        elif name == "VoiceSessionEnded":
            realtime_live = False
        elif name == "SpeechSpoken" and realtime_live:
            layer = str(event.get("layer", "") or "")
            if layer and not layer.startswith("realtime."):
                payload = event.get("payload") or {}
                session.findings.append(
                    Finding(
                        severity="high",
                        kind="voice-identity-break",
                        detail=(
                            f"layer {layer!r} spoke into the live realtime call: "
                            f"\"{_preview(payload.get('text'))}\""
                        ),
                    )
                )


def _check_tool_retry_loops(session: Session) -> None:
    failures: Counter[str] = Counter()
    last_error: dict[str, str] = {}
    for event in session.events:
        if event.get("event") != "ActionExecuted":
            continue
        payload = event.get("payload") or {}
        if payload.get("success"):
            continue
        tool = str(payload.get("tool_name", "?"))
        failures[tool] += 1
        last_error[tool] = _preview(payload.get("error"))
    for tool, count in failures.items():
        if count >= _RETRY_LOOP_THRESHOLD:
            session.findings.append(
                Finding(
                    severity="warn",
                    kind="tool-retry-loop",
                    detail=(
                        f"{tool} failed {count}x (last error: "
                        f"{last_error.get(tool, '?')})"
                    ),
                )
            )


def _check_exhausted_brain_turns(session: Session) -> None:
    for event in session.events:
        if event.get("event") != "BrainTurnCompleted":
            continue
        payload = event.get("payload") or {}
        finish = str(payload.get("finish_reason", "") or "")
        text_len = int(payload.get("text_len", 0) or 0)
        if finish == "budget_exceeded":
            session.findings.append(
                Finding(
                    severity="high",
                    kind="exhausted-brain-turn",
                    detail=(
                        "iteration budget exhausted before an answer "
                        f"(tokens_in={payload.get('tokens_in')})"
                    ),
                )
            )
        elif text_len == 0 and finish not in _SUPPRESSED_FINISH_REASONS:
            session.findings.append(
                Finding(
                    severity="warn",
                    kind="empty-brain-turn",
                    detail=f"brain turn produced no text (finish_reason={finish!r})",
                )
            )


def _check_silent_turns(session: Session) -> None:
    for event in session.events:
        if event.get("event") != "VoiceTurnCompleted":
            continue
        payload = event.get("payload") or {}
        user_text = str(payload.get("user_text", "") or "").strip()
        jarvis_text = str(payload.get("jarvis_text", "") or "").strip()
        # A one/two-word interjection ("Hm?") legitimately gets no reply.
        if jarvis_text or len(user_text.split()) < 3:
            continue
        session.findings.append(
            Finding(
                severity="info",
                kind="silent-turn",
                detail=f"no assistant text for: \"{_preview(user_text)}\"",
            )
        )


def _check_realtime_postmortem(session: Session) -> None:
    """Fold the shared transport-health verdicts into this session's findings."""
    for payload in extract_postmortems(session.events):
        for verdict in evaluate_postmortem(payload):
            session.findings.append(
                Finding(
                    severity=verdict.severity,
                    kind=verdict.kind,
                    detail=verdict.detail,
                )
            )


_CHECKS = (
    _check_promise_without_action,
    _check_voice_identity,
    _check_tool_retry_loops,
    _check_exhausted_brain_turns,
    _check_silent_turns,
    _check_realtime_postmortem,
)


def audit_events(events: list[dict[str, Any]]) -> list[Session]:
    """Split raw flight-recorder events and run every check per session."""
    sessions = _split_sessions(events)
    for session in sessions:
        for check in _CHECKS:
            check(session)
    return sessions


def _load_events(path: Path) -> list[dict[str, Any]]:
    events: list[dict[str, Any]] = []
    with path.open(encoding="utf-8") as handle:
        for line in handle:
            try:
                events.append(json.loads(line))
            except json.JSONDecodeError:
                continue
    return events


def _format_report(sessions: list[Session]) -> str:
    lines: list[str] = []
    total_findings = 0
    for session in sessions:
        header = (
            f"session {session.session_id[:8]}  turns={session.turn_count}  "
            f"findings={len(session.findings)}"
        )
        lines.append(header)
        for finding in session.findings:
            total_findings += 1
            lines.append(f"  [{finding.severity:4}] {finding.kind}: {finding.detail}")
    lines.append(
        f"\n{len(sessions)} session(s), {total_findings} finding(s)."
    )
    return "\n".join(lines)


def audit_harness_capture(rows: list[dict[str, Any]]) -> tuple[str, int]:
    """Verdict table for a probe round capture; exit 1 on any HIGH finding.

    A probe round is one session (occasionally more after a rebuild), so the
    report is per postmortem rather than per reconstructed session timeline.
    """
    postmortems = extract_postmortems(rows)
    if not postmortems:
        return "no RealtimeSessionPostmortem in the capture", 2
    lines: list[str] = []
    worst_is_high = False
    for payload in postmortems:
        findings = evaluate_postmortem(payload)
        verdict = "PASS"
        if any(f.severity == "high" for f in findings):
            verdict, worst_is_high = "FAIL", True
        elif findings:
            verdict = "WARN"
        lines.append(
            f"postmortem {str(payload.get('session_id', '?'))[:12]}  "
            f"provider={payload.get('provider', '?')}  "
            f"turns={payload.get('turns_completed', 0)}  "
            f"ready={payload.get('ready_ms', 0)}ms  "
            f"first_audio={payload.get('first_audio_ms', 0)}ms  -> {verdict}"
        )
        for finding in findings:
            lines.append(f"  [{finding.severity:4}] {finding.kind}: {finding.detail}")
    return "\n".join(lines), (1 if worst_is_high else 0)


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument(
        "--date",
        default=date.today().isoformat(),
        help="flight-recorder day to audit (YYYY-MM-DD, default: today)",
    )
    parser.add_argument(
        "--last",
        type=int,
        default=0,
        help="audit only the newest N sessions (default: all)",
    )
    parser.add_argument(
        "--harness",
        type=Path,
        default=None,
        help=(
            "audit a codex live-probe round capture (round.jsonl) instead of "
            "the flight recorder; exits 1 on any HIGH finding"
        ),
    )
    args = parser.parse_args(argv)

    if args.harness is not None:
        if not args.harness.exists():
            print(f"no harness capture at {args.harness}")
            return 2
        report, code = audit_harness_capture(_load_events(args.harness))
        print(report)
        return code

    recorder_path = REPO_ROOT / "data" / "flight_recorder" / f"{args.date}.jsonl"
    if not recorder_path.exists():
        print(f"no flight recorder file for {args.date}: {recorder_path}")
        return 2

    sessions = audit_events(_load_events(recorder_path))
    if args.last > 0:
        sessions = sessions[-args.last :]
    print(_format_report(sessions))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())