Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

51.4 KB
Raw Blame
#!/usr/bin/env python3
"""
AiSOC Pillar-1 Unified Eval Runner
===================================
Runs all four offline evaluation suites against the 200-incident benchmark
dataset and emits a single JSON report (and a human-readable summary).

Suites:
    1. MITRE ATT&CK tactic accuracy   (services/agents/tests/test_mitre_accuracy.py)
    2. Alert reduction ratio          (services/agents/tests/test_alert_reduction.py)
    3. Investigation completeness     (services/agents/tests/test_investigation_completeness.py)
    4. Response-plan quality          (services/agents/tests/test_response_quality.py)
    5. Hunt corpus coverage           (services/agents/tests/test_hunt_corpus.py)
    6. AI-vs-AI adversary degradation (services/agents/tests/test_adversary_eval.py)
    7. Confidence calibration         (services/agents/tests/test_confidence_calibration.py)
    8. Memory recall fidelity         (services/agents/tests/test_memory_recall.py)
    9. Override accuracy              (services/agents/tests/test_override_accuracy.py)
   10. Playbook completion rate       (services/agents/tests/test_playbook_completion_rate.py)
   11. Per-rule cross-fire FP rate    (services/agents/tests/test_detection_fp_rate.py)

Each substrate suite reports two metrics:

  * **Per-case mean**     – classic average across all 200 incidents.
  * **Per-template macro** – equal-weight average across the ~55 distinct
    incident templates.  Because each template is cycled 3-4× through
    `{user}/{host}/{ip}/{campaign}` permutations, a single broken template
    moves per-case mean by ~0.5% but per-template macro by ~1.5-1.8% — so
    the macro is the regression-signal-preserving metric we gate on.

The runner also dumps the synthetic-telemetry coverage summary so connector
and Sigma-rule PRs can pin against a stable corpus.

Usage:
    python3 scripts/run_evals.py                       # human-readable + writes report
    python3 scripts/run_evals.py --suite all           # run every suite (default)
    python3 scripts/run_evals.py --suite mitre_accuracy
                                                       # run a single suite by name
    python3 scripts/run_evals.py --json                # JSON to stdout
    python3 scripts/run_evals.py --out path.json       # write to a custom path
    python3 scripts/run_evals.py --ci                  # exit non-zero on regression
    python3 scripts/run_evals.py \
        --baseline eval_baseline.json \
        --max-regression-pp 1.0                        # gate against a saved baseline

Exit codes:
    0  All gates passed (or --ci not set)
    1  At least one suite below its target floor (only with --ci)
    2  MITRE accuracy regressed by ≥ --max-regression-pp vs baseline (w2-dac)
    3  Eval substrate imports failed (services/agents deps not installed)
"""

from __future__ import annotations

import argparse
import json
import os
import sys
import time
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, cast

# `scripts/` is on sys.path when this file is run as a program, but not when a
# test loads it by path with importlib. gate_toolkit sits beside it either way.
sys.path.insert(0, str(Path(__file__).resolve().parent))

from gate_toolkit import repo_root, self_test_if_requested

self_test_if_requested(__file__)

_REPO_ROOT = repo_root()
_AGENTS_ROOT = _REPO_ROOT / "services" / "agents"
# Order matters: scripts/ also contains a tests/ package (scripts/tests/) for
# the CLI smoke test, and would shadow services/agents/tests/ if it landed at
# position 0 of sys.path. Insert scripts/ first, then agents/ on top, so that
# `from tests.test_adversary_eval import ...` resolves to the substrate tests
# under services/agents/tests/.
sys.path.insert(0, str(_REPO_ROOT / "scripts"))
sys.path.insert(0, str(_AGENTS_ROOT))

# The per-investigation token/USD/latency telemetry block (T2.4) is stdlib-only
# and lives in ``scripts/eval_telemetry.py`` so it can run on hosts that
# don't have the full agent dev dependency stack (pydantic, langgraph, ...).
from eval_telemetry import (  # type: ignore  # noqa: E402
    DEFAULT_INCIDENTS_PATH,
    compute_per_investigation_telemetry,
)
from eval_telemetry import (  # noqa: E402  # same reason as the import above
    DEFAULT_MODEL as _TELEMETRY_DEFAULT_MODEL,
)

# Wet-eval shim (T5.5). Dry-run path is stdlib-only; live path imports the
# agent stack lazily and degrades cleanly if it isn't available. Lives in
# ``scripts/wet_eval.py`` to keep the wet-eval shape decoupled from the
# substrate-suite plumbing below.
from wet_eval import compute_wet_eval  # type: ignore  # noqa: E402

# The substrate-suite imports below pull in the agent runtime (pydantic etc).
# Wrap them in a try/except so ``--telemetry-only`` can still run on a bare
# Python install — the new T2.4 token/USD/latency block doesn't need them.
_SUBSTRATE_IMPORT_ERROR: Exception | None = None
try:
    from tests.test_adversary_eval import (
        _HEAVY_BUCKET_CEILING as _ADVERSARY_HEAVY_CEILING,
    )
    from tests.test_adversary_eval import (
        _LIGHT_BUCKET_FLOOR as _ADVERSARY_LIGHT_FLOOR,
    )
    from tests.test_adversary_eval import (
        _OVERALL_FLOOR as _ADVERSARY_OVERALL_FLOOR,
    )
    from tests.test_adversary_eval import (  # type: ignore
        evaluate_adversary_accuracy,
    )
    from tests.test_alert_reduction import (  # type: ignore
        compute_reduction,
        fuse_alerts,
        generate_noisy_alert_stream,
    )
    from tests.test_confidence_calibration import (  # type: ignore
        BRIER_THRESHOLD_INVESTIGATION as _CALIB_BRIER_INV,
    )
    from tests.test_confidence_calibration import (
        BRIER_THRESHOLD_TRIAGE as _CALIB_BRIER_TRIAGE,
    )
    from tests.test_confidence_calibration import (
        ECE_THRESHOLD_INVESTIGATION as _CALIB_ECE_INV,
    )
    from tests.test_confidence_calibration import (
        ECE_THRESHOLD_TRIAGE as _CALIB_ECE_TRIAGE,
    )
    from tests.test_confidence_calibration import (
        run_evaluation as _run_calibration_eval,
    )
    from tests.test_detection_fp_rate import (  # type: ignore
        MAX_PER_RULE_FPR as _DETECTION_FP_CEILING,
    )
    from tests.test_detection_fp_rate import (
        evaluate_per_rule_fp,
    )
    from tests.test_hunt_corpus import (
        _NEGATIVE_CEILING as _HUNT_NEGATIVE_CEILING,
    )
    from tests.test_hunt_corpus import (
        _POSITIVE_FLOOR as _HUNT_POSITIVE_FLOOR,
    )
    from tests.test_hunt_corpus import (  # type: ignore
        evaluate_hunt_corpus,
    )
    from tests.test_investigation_completeness import (  # type: ignore
        evaluate_completeness,
    )
    from tests.test_memory_recall import (  # type: ignore
        RECALL_ACCURACY_FLOOR as _RECALL_FLOOR,
    )
    from tests.test_memory_recall import (
        run_evaluation as _run_memory_recall_eval,
    )
    from tests.test_mitre_accuracy import evaluate_mitre_accuracy  # type: ignore
    from tests.test_override_accuracy import (  # type: ignore
        OVERRIDE_ACCURACY_FLOOR as _OVERRIDE_FLOOR,
    )
    from tests.test_override_accuracy import (
        run_evaluation as _run_override_accuracy_eval,
    )
    from tests.test_playbook_completion_rate import (
        ACTION_ALIGNMENT_FLOOR as _PLAYBOOK_ALIGN_FLOOR,
    )
    from tests.test_playbook_completion_rate import (
        HIGH_CRIT_MAPPED_FLOOR as _PLAYBOOK_HIGH_CRIT_FLOOR,
    )
    from tests.test_playbook_completion_rate import (  # type: ignore
        OVERALL_COMPLETION_FLOOR as _PLAYBOOK_OVERALL_FLOOR,
    )
    from tests.test_playbook_completion_rate import (
        evaluate_playbook_completion,
    )
    from tests.test_response_quality import (  # type: ignore
        evaluate_response_quality,
    )

    _SUBSTRATE_AVAILABLE = True
except Exception as _exc:  # pragma: no cover - degraded mode for telemetry-only
    _SUBSTRATE_IMPORT_ERROR = _exc
    _SUBSTRATE_AVAILABLE = False
    # Fallback constants so the module-level ``_TARGETS`` dict still constructs
    # cleanly. Real values come from the test modules when ``--telemetry-only``
    # is *not* in play. These match the floors hard-coded in the test files.
    _ADVERSARY_HEAVY_CEILING = 0.50
    _ADVERSARY_LIGHT_FLOOR = 0.85
    _ADVERSARY_OVERALL_FLOOR = 0.40
    _CALIB_BRIER_INV = 0.20
    _CALIB_BRIER_TRIAGE = 0.20
    _CALIB_ECE_INV = 0.10
    _CALIB_ECE_TRIAGE = 0.10
    _DETECTION_FP_CEILING = 0.05
    _HUNT_NEGATIVE_CEILING = 0.0
    _HUNT_POSITIVE_FLOOR = 1.0
    _RECALL_FLOOR = 0.95
    _OVERRIDE_FLOOR = 0.95
    _PLAYBOOK_ALIGN_FLOOR = 0.85
    _PLAYBOOK_HIGH_CRIT_FLOOR = 0.95
    _PLAYBOOK_OVERALL_FLOOR = 0.50

# Per-suite floors (must match what tests assert)
_TARGETS = {
    "mitre_accuracy": 0.80,
    "alert_reduction": 0.70,
    "investigation_completeness": 0.85,
    "response_quality": 0.80,
    "hunt_corpus": _HUNT_POSITIVE_FLOOR,
    "adversary_eval": _ADVERSARY_OVERALL_FLOOR,
    # Calibration target is "Brier score *at or below* this value", so the
    # display logic in main() inverts the pass condition for this suite.
    "confidence_calibration": _CALIB_BRIER_INV,
    "memory_recall": _RECALL_FLOOR,
    "override_accuracy": _OVERRIDE_FLOOR,
    # Playbook completion gate (WS-C3): we surface the *overall* coverage
    # rate as the headline metric but the pass condition combines three
    # sub-gates (overall, mapped high+critical, action alignment, no
    # orphan playbooks/templates). See _run_playbook_completion().
    "playbook_completion_rate": _PLAYBOOK_OVERALL_FLOOR,
    # Per-rule cross-fire FP gate (Issue #5): lower-is-better. Each
    # native detection rule is replayed against every other rule's
    # positive fixture; "target" here is the *ceiling* on per-rule FPR.
    # See services/agents/tests/test_detection_fp_rate.py.
    "detection_fp_rate": _DETECTION_FP_CEILING,
}

# Per-template macro floors (kept slightly below per-case floors because each
# template contributes ~1/55 of the macro vs. ~1/200 of the per-case mean).
_TEMPLATE_TARGETS = {
    "mitre_accuracy": 0.80,
    "investigation_completeness": 0.80,
    "response_quality": 0.75,
}

_TELEMETRY_PATH = _AGENTS_ROOT / "tests" / "eval_data" / "synthetic_telemetry.jsonl"

# Canonical suite name list. Kept in lock-step with the keys of
# ``summary["suites"]`` built in ``main()`` and the per-suite ``_run_*``
# helpers above. ``--suite all`` is the default; passing one of these names
# runs that suite in isolation (used by the per-axis CI matrix and by
# operators bisecting a regression to a single gate).
_SUITE_NAMES: tuple[str, ...] = (
    "mitre_accuracy",
    "alert_reduction",
    "investigation_completeness",
    "response_quality",
    "hunt_corpus",
    "adversary_eval",
    "confidence_calibration",
    "memory_recall",
    "override_accuracy",
    "playbook_completion_rate",
    "detection_fp_rate",
)


def _run_mitre() -> dict:
    t0 = time.perf_counter()
    res = evaluate_mitre_accuracy(threshold=_TARGETS["mitre_accuracy"])
    dur = (time.perf_counter() - t0) * 1000
    tpl = res.per_template_summary() if hasattr(res, "per_template_summary") else None
    out: dict = {
        "metric": "accuracy",
        "value": round(res.accuracy, 4),
        "target": _TARGETS["mitre_accuracy"],
        "passed": res.accuracy >= _TARGETS["mitre_accuracy"],
        "duration_ms": round(dur, 1),
        "details": {
            "incidents": res.total,
            "correct": res.correct,
            "precision": round(res.precision, 4),
            "recall": round(res.recall, 4),
            "f1": round(res.f1, 4),
        },
    }
    if tpl:
        macro = tpl.get("template_macro_accuracy", 0.0)
        target = _TEMPLATE_TARGETS["mitre_accuracy"]
        out["per_template"] = {
            "metric": "macro_accuracy",
            "value": round(macro, 4),
            "target": target,
            "passed": macro >= target,
            "template_count": tpl.get("template_count", 0),
            "failing_templates": [t["template_id"] for t in tpl.get("failing_templates", [])],
        }
        out["passed"] = out["passed"] and out["per_template"]["passed"]
    return out


def _run_alert_reduction(stream_size: int = 1000) -> dict:
    t0 = time.perf_counter()
    alerts = generate_noisy_alert_stream(count=stream_size)
    incidents = fuse_alerts(alerts)
    metrics = compute_reduction(alerts, incidents)
    dur = (time.perf_counter() - t0) * 1000
    storm = sum(1 for i in incidents if i.host.startswith("<storm:"))
    return {
        "metric": "reduction_ratio",
        "value": float(metrics["reduction"]),
        "target": _TARGETS["alert_reduction"],
        "passed": metrics["reduction"] >= _TARGETS["alert_reduction"],
        "duration_ms": round(dur, 1),
        "details": {
            "alerts_in": metrics["alerts_in"],
            "incidents_out": metrics["incidents_out"],
            "reduction_pct": metrics["reduction_pct"],
            "storm_incidents": storm,
        },
    }


def _run_completeness() -> dict:
    t0 = time.perf_counter()
    res = evaluate_completeness(keep_per_incident=True)
    dur = (time.perf_counter() - t0) * 1000
    out: dict = {
        "metric": "mean_keyword_coverage",
        "value": round(res.mean, 4),
        "target": _TARGETS["investigation_completeness"],
        "passed": res.mean >= _TARGETS["investigation_completeness"],
        "duration_ms": round(dur, 1),
        "details": {
            "incidents": res.incidents,
            "fully_covered": res.full_coverage,
            "fully_covered_pct": round(res.full_coverage_pct, 4),
        },
    }
    tpl = res.per_template_summary()
    macro = tpl.get("template_macro_mean", 0.0)
    target = _TEMPLATE_TARGETS["investigation_completeness"]
    out["per_template"] = {
        "metric": "macro_completeness",
        "value": round(macro, 4),
        "target": target,
        "passed": macro >= target,
        "template_count": tpl.get("template_count", 0),
        "failing_templates": [t["template_id"] for t in tpl.get("failing_templates", [])],
    }
    out["passed"] = out["passed"] and out["per_template"]["passed"]
    return out


def _run_response_quality() -> dict:
    t0 = time.perf_counter()
    res = evaluate_response_quality(keep_per_incident=True)
    dur = (time.perf_counter() - t0) * 1000
    out: dict = {
        "metric": "mean_rubric_score",
        "value": round(res.mean_score, 4),
        "target": _TARGETS["response_quality"],
        "passed": res.mean_score >= _TARGETS["response_quality"],
        "duration_ms": round(dur, 1),
        "details": {
            "incidents": res.incidents,
            "criteria": {k: round(res.crit_mean(k), 4) for k in res.crit_sum},
        },
    }
    tpl = res.per_template_summary()
    macro = tpl.get("template_macro_score", 0.0)
    target = _TEMPLATE_TARGETS["response_quality"]
    out["per_template"] = {
        "metric": "macro_score",
        "value": round(macro, 4),
        "target": target,
        "passed": macro >= target,
        "template_count": tpl.get("template_count", 0),
        "failing_templates": [t["template_id"] for t in tpl.get("failing_templates", [])],
    }
    out["passed"] = out["passed"] and out["per_template"]["passed"]
    return out


def _run_hunt_corpus() -> dict:
    """Fifth gate (w2-hac): hunt-as-code coverage against the synthetic
    hunt telemetry corpus.

    The hunt corpus is small and hand-authored, so we hold ourselves to
    perfect scenario coverage rather than a percentage floor:

      * every hunt MUST fire on its declared positive scenario (positive
        rate at or above ``_HUNT_POSITIVE_FLOOR``);
      * no hunt may fire on its declared negative scenario (false-positive
        rate at or below ``_HUNT_NEGATIVE_CEILING``);
      * every ``INC-HUNT-*`` event in the telemetry must be referenced by
        at least one hunt YAML — orphan telemetry is a regression.
    """
    t0 = time.perf_counter()
    res = evaluate_hunt_corpus()
    dur = (time.perf_counter() - t0) * 1000

    positive_pass = res.positive_rate >= _HUNT_POSITIVE_FLOOR
    negative_pass = res.false_positive_rate <= _HUNT_NEGATIVE_CEILING
    no_orphans = not res.orphan_incident_ids

    return {
        "metric": "positive_scenario_catch_rate",
        "value": round(res.positive_rate, 4),
        "target": _HUNT_POSITIVE_FLOOR,
        "passed": positive_pass and negative_pass and no_orphans,
        "duration_ms": round(dur, 1),
        "details": {
            "hunts": res.hunts_total,
            "positives_expected": res.positives_expected,
            "positives_caught": res.positives_caught,
            "negatives_expected": res.negatives_expected,
            "false_positives": res.false_positives,
            "false_positive_rate": round(res.false_positive_rate, 4),
            "negative_ceiling": _HUNT_NEGATIVE_CEILING,
            "orphan_incident_ids": res.orphan_incident_ids,
            "misses": res.misses,
            "false_positive_details": res.false_positive_details,
            "positive_pass": positive_pass,
            "negative_pass": negative_pass,
            "no_orphans": no_orphans,
        },
    }


def _run_detection_fp_rate() -> dict:
    """Per-rule cross-fire FP gate (Issue #5).

    ``scripts/validate_detections.py`` replays each native rule against its
    own positive + negative fixture (TP / TN gate). That misses the failure
    mode operators feel hardest in production: a rule that fires on events
    intended for a *different* rule. This gate replays every native rule's
    ``match_when`` against every other rule's positive fixture and trips
    when any single rule's cross-fire FPR exceeds ``MAX_PER_RULE_FPR``
    (default 5%).

    Headline metric is "lower-is-better" — we surface the worst per-rule
    FPR and the count of failing rules. The "target" field carries the
    ceiling so the JSON consumer (CI / dashboard) can render it the same
    way as the standard floor-based gates.
    """
    t0 = time.perf_counter()
    rep = evaluate_per_rule_fp()
    dur = (time.perf_counter() - t0) * 1000
    return {
        "metric": "worst_per_rule_fp_rate",
        "value": round(rep.worst_fpr, 4),
        "target": rep.max_per_rule_fpr,
        "passed": rep.passed,
        "duration_ms": round(dur, 1),
        "details": {
            "lower_is_better": True,
            "rules_evaluated": rep.rules_total,
            "rules_passed": rep.rules_passed,
            "rules_failed": rep.rules_failed,
            "mean_fpr": round(rep.mean_fpr, 4),
            "worst_fpr": round(rep.worst_fpr, 4),
            "ceiling": rep.max_per_rule_fpr,
            # Cap the failing-rule list so a globally-broken rule pack
            # doesn't bloat the eval JSON. The full list is always
            # available by re-running the dedicated test module.
            "failing_rules_sample": rep.failing_rules[:20],
        },
    }


def _run_adversary() -> dict:
    """Sixth gate (w2-aivai): graceful degradation under attacker-LLM mutation.

    The adversary corpus is generated by ``scripts/generate_adversary_incidents.py``
    and rewrites every defender-known keyword into evasive synonyms, character
    obfuscation, and fragmentation across heavy/medium/light buckets. The gate
    enforces three things at once:

      * overall catch rate stays at or above ``_ADVERSARY_OVERALL_FLOOR`` —
        the substrate must degrade gracefully, not fall off a cliff;
      * the light (control) bucket stays at or above ``_ADVERSARY_LIGHT_FLOOR``
        — leetspeak alone shouldn't break the keyword extractor;
      * the heavy bucket stays *at or below* ``_ADVERSARY_HEAVY_CEILING`` —
        if heavy is catching too much, the dataset isn't actually adversarial
        and the suite has lost its signal.
    """
    t0 = time.perf_counter()
    res = evaluate_adversary_accuracy()
    dur = (time.perf_counter() - t0) * 1000

    light_acc = res.bucket_accuracy("light")
    heavy_acc = res.bucket_accuracy("heavy")
    overall_pass = res.accuracy >= _ADVERSARY_OVERALL_FLOOR
    light_pass = light_acc >= _ADVERSARY_LIGHT_FLOOR
    heavy_pass = heavy_acc <= _ADVERSARY_HEAVY_CEILING

    return {
        "metric": "graceful_degradation_catch_rate",
        "value": round(res.accuracy, 4),
        "target": _ADVERSARY_OVERALL_FLOOR,
        "passed": overall_pass and light_pass and heavy_pass,
        "duration_ms": round(dur, 1),
        "details": {
            "incidents": res.total,
            "correct": res.correct,
            "lost_all_tactics": res.lost_all_tactics,
            "buckets": res.to_summary()["buckets"],
            "light_floor": _ADVERSARY_LIGHT_FLOOR,
            "heavy_ceiling": _ADVERSARY_HEAVY_CEILING,
            "light_pass": light_pass,
            "heavy_pass": heavy_pass,
        },
    }


def _run_confidence_calibration() -> dict:
    """Seventh gate (tier1-confidence): Brier-score gate over 200 positive +
    200 deterministic benign cases.

    Calibration is a *lower-is-better* metric, so unlike the other suites the
    pass condition is "value <= target" for both Brier score and ECE. We
    surface the investigation Brier as the headline number because it's the
    end-to-end agent verdict; triage gates are exposed via ``details``.
    """
    t0 = time.perf_counter()
    res = _run_calibration_eval()
    dur = (time.perf_counter() - t0) * 1000

    inv_brier = res["investigation"]["brier"]
    inv_ece = res["investigation"]["ece"]
    triage_brier = res["triage"]["brier"]
    triage_ece = res["triage"]["ece"]

    inv_brier_pass = inv_brier <= _CALIB_BRIER_INV
    inv_ece_pass = inv_ece <= _CALIB_ECE_INV
    triage_brier_pass = triage_brier <= _CALIB_BRIER_TRIAGE
    triage_ece_pass = triage_ece <= _CALIB_ECE_TRIAGE

    return {
        "metric": "investigation_brier_score",
        "value": inv_brier,
        "target": _CALIB_BRIER_INV,
        # Lower-is-better: passed iff *all* sub-gates pass.
        "passed": (inv_brier_pass and inv_ece_pass and triage_brier_pass and triage_ece_pass and bool(res.get("passed", False))),
        "duration_ms": round(dur, 1),
        "details": {
            "lower_is_better": True,
            "investigation_brier": inv_brier,
            "investigation_brier_max": _CALIB_BRIER_INV,
            "investigation_ece": inv_ece,
            "investigation_ece_max": _CALIB_ECE_INV,
            "triage_brier": triage_brier,
            "triage_brier_max": _CALIB_BRIER_TRIAGE,
            "triage_ece": triage_ece,
            "triage_ece_max": _CALIB_ECE_TRIAGE,
            "investigation_separation": res["investigation"]["separation"],
            "triage_separation": res["triage"]["separation"],
            "mean_confidence_positive_investigation": res["investigation"]["mean_confidence_positive"],
            "mean_confidence_benign_investigation": res["investigation"]["mean_confidence_benign"],
            "investigation_brier_pass": inv_brier_pass,
            "investigation_ece_pass": inv_ece_pass,
            "triage_brier_pass": triage_brier_pass,
            "triage_ece_pass": triage_ece_pass,
        },
    }


def _run_memory_recall() -> dict:
    """Memory-recall gate: validates three-tier memory fidelity, priority,
    isolation, and override ingestion using offline in-process fallbacks."""
    t0 = time.perf_counter()
    res = _run_memory_recall_eval()
    dur = (time.perf_counter() - t0) * 1000
    return {
        "metric": "memory_recall_accuracy",
        "value": res["recall_accuracy"],
        "target": _RECALL_FLOOR,
        "passed": res["passed"],
        "duration_ms": round(dur, 1),
        "details": {
            "total_cases": res["total_cases"],
            "passed_cases": res["passed_cases"],
            "failed_cases": res.get("failed_cases", []),
        },
    }


def _run_override_accuracy() -> dict:
    """Override-accuracy gate: validates analyst-feedback ingestion, retrieval,
    idempotent upsert, and cross-tenant isolation using offline fallbacks."""
    t0 = time.perf_counter()
    res = _run_override_accuracy_eval()
    dur = (time.perf_counter() - t0) * 1000
    return {
        "metric": "override_accuracy",
        "value": res["override_accuracy"],
        "target": _OVERRIDE_FLOOR,
        "passed": res["passed"],
        "duration_ms": round(dur, 1),
        "details": {
            "total_cases": res["total_cases"],
            "passed_cases": res["passed_cases"],
            "failed_cases": res.get("failed_cases", []),
        },
    }


def _run_playbook_completion() -> dict:
    """WS-C3 gate: playbook eval-harness validation.

    Measures playbook coverage against the deterministic 200-incident
    benchmark dataset. Reports the *overall* completion rate as the headline
    metric for diff-friendly trend tracking, but the suite passes only when
    *all* sub-gates pass:

      * overall completion rate ≥ ``OVERALL_COMPLETION_FLOOR`` —
        catches accidental playbook deletions / wholesale regressions;
      * high+critical completion rate over *mapped* templates ≥
        ``HIGH_CRIT_MAPPED_FLOOR`` — every severe incident the pack claims
        to cover must have a containment playbook;
      * action alignment rate ≥ ``ACTION_ALIGNMENT_FLOOR`` — among matched
        incidents, the playbook's first-line steps align with the dataset's
        ``response_class`` (block / quarantine / disable / etc.);
      * no orphan playbooks (firing on zero incidents, except the documented
        allowlist of medium-severity / off-corpus playbooks);
      * no orphan templates (mapped templates with zero playbook hits).

    The high+critical gate intentionally measures *mapped* coverage rather
    than raw severity coverage — the dataset includes ~22 endpoint-compromise
    / persistence / defense-evasion templates that are documented v1 coverage
    gaps, and gating on them would either force inflating coverage with
    mismatched playbooks or punish CI for known scope decisions.
    """
    t0 = time.perf_counter()
    res = evaluate_playbook_completion(keep_per_incident=True)
    dur = (time.perf_counter() - t0) * 1000

    per_incident = res.per_incident or []
    hc_rate, hc_covered, hc_total = res.severity_completion_rate_mapped(("high", "critical"), per_incident)
    hc_raw_rate = res.severity_completion_rate(("high", "critical"))
    overall_pass = res.completion_rate >= _PLAYBOOK_OVERALL_FLOOR
    high_crit_pass = hc_rate >= _PLAYBOOK_HIGH_CRIT_FLOOR
    align_pass = res.action_alignment_rate >= _PLAYBOOK_ALIGN_FLOOR
    no_orphan_playbooks = not res.orphan_playbooks
    no_orphan_templates = not res.orphan_templates

    return {
        "metric": "completion_rate",
        "value": round(res.completion_rate, 4),
        "target": _PLAYBOOK_OVERALL_FLOOR,
        "passed": (overall_pass and high_crit_pass and align_pass and no_orphan_playbooks and no_orphan_templates),
        "duration_ms": round(dur, 1),
        "details": {
            "incidents": res.incidents,
            "covered": res.covered,
            "aligned": res.aligned,
            "overall_completion_rate": round(res.completion_rate, 4),
            "overall_completion_pass": overall_pass,
            "high_critical_completion_rate_mapped": round(hc_rate, 4),
            "high_critical_completion_rate_raw": round(hc_raw_rate, 4),
            "high_critical_covered_mapped": hc_covered,
            "high_critical_total_mapped": hc_total,
            "high_critical_floor": _PLAYBOOK_HIGH_CRIT_FLOOR,
            "high_critical_pass": high_crit_pass,
            "action_alignment_rate": round(res.action_alignment_rate, 4),
            "action_alignment_floor": _PLAYBOOK_ALIGN_FLOOR,
            "action_alignment_pass": align_pass,
            "per_severity": res.per_severity,
            "per_category": res.per_category,
            "orphan_templates": res.orphan_templates,
            "orphan_playbooks": res.orphan_playbooks,
            "no_orphan_playbooks": no_orphan_playbooks,
            "no_orphan_templates": no_orphan_templates,
        },
    }


def _summarise_telemetry() -> dict:
    """Summarise the synthetic-telemetry corpus produced alongside the dataset.

    Returned shape stays small (no per-event payloads) so the eval report
    remains diff-friendly. If the JSONL is missing, returns a stub so the
    runner stays usable for hosts that strip out telemetry artefacts.
    """
    if not _TELEMETRY_PATH.exists():
        return {"present": False, "events": 0, "sources": {}, "incidents_with_telemetry": 0}
    sources: dict[str, int] = {}
    incidents: set[str] = set()
    total = 0
    for line in _TELEMETRY_PATH.read_text().splitlines():
        line = line.strip()
        if not line:
            continue
        try:
            evt = json.loads(line)
        except json.JSONDecodeError:
            continue
        total += 1
        src = evt.get("source", "unknown")
        sources[src] = sources.get(src, 0) + 1
        inc_id = evt.get("incident_id")
        if inc_id:
            incidents.add(inc_id)
    return {
        "present": True,
        "events": total,
        "sources": dict(sorted(sources.items())),
        "incidents_with_telemetry": len(incidents),
        "path": str(_TELEMETRY_PATH.relative_to(_REPO_ROOT)),
    }


def _build_per_investigation_block(model: str, *, keep_records: bool) -> dict:
    """Compute the T2.4 per-investigation telemetry block.

    The substrate doesn't call an LLM, so these are *deterministic budget
    projections*. They are flat-out marked ``mode: deterministic_substrate``
    in the JSON; the docs/UI must not present them as wet-eval ground truth.
    Wet-eval (real LLM) replaces them in T5.5.

    The headline keys ``tokens_per_investigation`` and
    ``usd_per_investigation`` mirror the field names used by the Track 5 docs
    page so a single JSON consumer can render either source.
    """
    rep = compute_per_investigation_telemetry(
        DEFAULT_INCIDENTS_PATH,
        model=model,
        keep_records=keep_records,
    )
    block = rep.to_dict(include_records=keep_records)
    # Hoist the most-asked-for headline numbers up to the block root so
    # external consumers don't have to know about ``aggregate.tokens.total``.
    total = block["aggregate"]["tokens"]["total"]
    usd = block["aggregate"]["usd"]
    latency = block["aggregate"]["latency_ms"]
    block["tokens_per_investigation"] = {
        "mean": total["mean"],
        "median": total["median"],
        "p95": total["p95"],
        "p99": total["p99"],
        "prompt_mean": block["aggregate"]["tokens"]["prompt"]["mean"],
        "completion_mean": block["aggregate"]["tokens"]["completion"]["mean"],
    }
    block["usd_per_investigation"] = {
        "mean": usd["mean"],
        "median": usd["median"],
        "p95": usd["p95"],
        "p99": usd["p99"],
    }
    block["latency_per_investigation_ms"] = {
        "p50": latency["p50"],
        "p95": latency["p95"],
        "p99": latency["p99"],
        "mean": latency["mean"],
    }
    return block


def grade_agent_quality(runs: list[Any]) -> dict[str, Any]:
    """Tool selection, evidence completeness and time to verdict.

    Wired into the harness because a metric nothing computes is a
    module. These three had zero hits anywhere in the tree: the eleven
    existing suites grade outcomes, and an agent that reaches the
    right verdict by reading the alert title scores identically to one
    that investigated.
    """
    sys.path.insert(0, str(repo_root() / "services" / "agents"))
    from app.eval.agent_quality import score_agent_quality  # noqa: PLC0415

    return score_agent_quality(runs).as_dict()


def main() -> None:
    parser = argparse.ArgumentParser(description="AiSOC Pillar-1 unified evaluation runner.")
    parser.add_argument(
        "--suite",
        choices=("all", *_SUITE_NAMES),
        default="all",
        help=(
            "Which eval suite to run. 'all' (default) executes every suite; "
            "pass a single suite name to run it in isolation. This is the "
            "flag the AiSOC demo video uses (`--suite all`)."
        ),
    )
    parser.add_argument("--json", action="store_true", help="Print JSON report to stdout.")
    parser.add_argument(
        "--out",
        type=Path,
        default=_REPO_ROOT / "eval_report.json",
        help="Write JSON report to this path.",
    )
    parser.add_argument(
        "--ci",
        action="store_true",
        help="Exit non-zero if any suite is below its target floor.",
    )
    parser.add_argument(
        "--baseline",
        type=Path,
        default=None,
        help=(
            "Compare results against a saved baseline JSON (Wave 2 — w2-dac). "
            "When set, fail with exit code 2 if MITRE accuracy regresses by "
            "≥ --max-regression-pp percentage points."
        ),
    )
    parser.add_argument(
        "--max-regression-pp",
        type=float,
        default=1.0,
        help="Allowed MITRE accuracy regression vs baseline, in percentage points.",
    )
    parser.add_argument(
        "--telemetry-only",
        action="store_true",
        help=(
            "Skip the substrate-suite gates and only emit the per-investigation "
            "token/USD/latency telemetry block (T2.4). Useful on hosts without "
            "the full agent dev dependency stack (pydantic, langgraph, ...)."
        ),
    )
    parser.add_argument(
        "--telemetry-model",
        default=_TELEMETRY_DEFAULT_MODEL,
        help=("Model name to apply against the rate card when computing the per-investigation USD projection. Default: gpt-4o."),
    )
    parser.add_argument(
        "--no-telemetry-records",
        action="store_true",
        help=("Drop the per-incident telemetry array from the JSON report. Aggregate + per-template stats are always kept."),
    )
    parser.add_argument(
        "--wet",
        action="store_true",
        help=(
            "Run the live-LLM wet-eval harness (T5.5) over the 200-incident "
            "corpus. Requires WET_EVAL_OPENAI_KEY in the environment. The "
            "weekly cron in ``.github/workflows/wet-eval.yml`` is the only "
            "place this should run unattended; the preflight in "
            "``scripts/wet_eval_check.py`` no-ops the workflow on forks "
            "where the secret isn't set."
        ),
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help=(
            "Only meaningful with --wet. Synthesise the wet-eval JSON shape "
            "from the deterministic substrate budget projection (no live "
            "LLM calls). Used by CI to validate the report shape on every "
            "push and by the test suite."
        ),
    )
    parser.add_argument(
        "--wet-out",
        type=Path,
        default=None,
        help=(
            "Write the wet-eval block to this path in addition to the main "
            "--out report. The weekly workflow uses this to feed "
            "``scripts/wet_eval_update_benchmark.py`` without re-parsing "
            "the substrate suites."
        ),
    )
    parser.add_argument(
        "--wet-limit",
        type=int,
        default=None,
        help=(
            "Only meaningful with --wet. Dispatch at most N incidents instead "
            "of the full 200. A CPU-hosted local model cannot finish the whole "
            "corpus inside a CI budget; the slice is a deterministic prefix so "
            "runs stay comparable, and the report records how many were used."
        ),
    )
    parser.add_argument(
        "--wet-model",
        default=None,
        help=(
            "Only meaningful with --wet. The model the run is actually served "
            "by, written into the report's `model` field. Without it a local "
            "Ollama run labels itself with the headline rate-card model, so "
            "the report names a model that was never called and prices the "
            "tokens against that model's list."
        ),
    )
    parser.add_argument(
        "--wet-require-live",
        action="store_true",
        help=(
            "Only meaningful with --wet. Fail instead of degrading to dry-run "
            "numbers when the live agent cannot be reached. Degrading is right "
            "for a reporting job, but fatal for a gate: a degraded run would "
            "publish substrate numbers as live-agent performance. Any job that "
            "asserts on these numbers must pass this."
        ),
    )
    args = parser.parse_args()

    # Wet-eval mode short-circuits the substrate gates entirely (T5.5).
    # ``--wet --dry-run`` is the path the test and the workflow's
    # PR-time validation step exercise; ``--wet`` without ``--dry-run``
    # is the weekly cron job's path and refuses to start without the
    # API key (the preflight should have caught it earlier, but we
    # belt-and-braces here so a manual ``run_evals.py --wet`` invocation
    # never silently degrades).
    if args.wet:
        wet_mode = "dry_run" if args.dry_run else "live"
        if wet_mode == "live" and not os.environ.get("WET_EVAL_OPENAI_KEY"):
            print(
                "[run_evals] --wet requires WET_EVAL_OPENAI_KEY in the "
                "environment. Pass --dry-run for the no-API-call shape "
                "check, or run scripts/wet_eval_check.py first.",
                file=sys.stderr,
            )
            sys.exit(2)
        try:
            wet_report = compute_wet_eval(
                mode=wet_mode,
                model=args.wet_model or _TELEMETRY_DEFAULT_MODEL,
                harness_version=(f"scripts/run_evals.py @ {os.environ.get('GITHUB_SHA', 'local')}"),
                limit=args.wet_limit,
                require_live=args.wet_require_live,
            )
        except RuntimeError as exc:
            # Only reachable under --wet-require-live. A caller that asked for
            # live numbers and got a degraded run needs a non-zero exit, not a
            # well-formed report full of substrate estimates.
            print(f"[run_evals] live wet-eval required but unavailable: {exc}", file=sys.stderr)
            sys.exit(3)
        wet_block = wet_report.to_dict(include_records=False)
        summary = {
            "generated_at": datetime.now(UTC).isoformat(),
            "dataset": "synthetic_incidents.json (200 cases, deterministic)",
            "wet_eval": wet_block,
            "all_passed": True,  # wet-eval reports performance, not pass/fail.
        }
        args.out.write_text(json.dumps(summary, indent=2))
        if args.wet_out is not None:
            args.wet_out.parent.mkdir(parents=True, exist_ok=True)
            args.wet_out.write_text(json.dumps(wet_block, indent=2))
        if args.json:
            print(json.dumps(summary, indent=2))
        else:
            print()
            print("=" * 78)
            label = "DRY RUN" if wet_mode == "dry_run" else "LIVE"
            # The slice, not the corpus. The banner said "200-incident" while
            # --wet-limit was dispatching 20, so the sample size a reader took
            # away was whichever number the header happened to hardcode.
            print(f"  AiSOC wet-eval ({label}) — {wet_block['incidents']} of 200 synthetic incidents")
            print("=" * 78)
            print(f"  Mode:           {wet_block['mode']}")
            print(f"  Model:          {wet_block['model']}")
            print(f"  Incidents:      {wet_block['incidents']}")
            print(f"  Templates:      {wet_block['templates']}")
            lat = wet_block["latency_seconds"]
            print(f"  Latency (s):    p50={lat['p50']:.2f}  p95={lat['p95']:.2f}  p99={lat['p99']:.2f}  mean={lat['mean']:.2f}")
            tot = wet_block["tokens"]["total"]
            print(f"  Tokens / inv:   mean={tot['mean']:.0f}  median={tot['median']:.0f}  p95={tot['p95']:.0f}  p99={tot['p99']:.0f}")
            usd = wet_block["usd"]
            if usd.get("measured") is False:
                print(f"  USD / inv:      {usd.get('note', 'not measured')}")
            else:
                print(
                    f"  USD / inv:      mean=${usd['mean']:.5f}  median=${usd['median']:.5f}  p95=${usd['p95']:.5f}  p99=${usd['p99']:.5f}"
                )
            print(f"  MITRE accuracy: {wet_block['mitre_accuracy']:.4f}")
            grounded = wet_block.get("groundedness") or {}
            if grounded.get("measured"):
                print(
                    f"  Groundedness:   mean={grounded['mean']:.4f}  median={grounded['median']:.4f}  "
                    f"min={grounded['min']:.4f}  p05={grounded['p05']:.4f}  (n={grounded['scored_incidents']})"
                )
            else:
                print(f"  Groundedness:   {grounded.get('note', 'not measured')}")
            if wet_block.get("warnings"):
                print("-" * 78)
                print("  Warnings:")
                for w in wet_block["warnings"]:
                    print(f"    - {w}")
            print("=" * 78)
        sys.exit(0)

    if not args.telemetry_only and not _SUBSTRATE_AVAILABLE:
        # Substrate suites need pydantic / langchain etc. If they're not
        # installed we can still emit the telemetry block — surface the
        # original ImportError so operators know what to fix.
        #
        # Exit 3, which is what this file's own "Exit codes" block has always
        # said a substrate-import failure returns. It exited 2, and 2 means
        # "MITRE accuracy regressed against the baseline" — so a fresh clone
        # missing a dependency reported an accuracy regression, and the one
        # actionable instruction (install the deps) was not in the message.
        msg = (
            "ERROR: substrate-suite imports failed (likely missing agent dev deps "
            f"such as pydantic): {_SUBSTRATE_IMPORT_ERROR!r}.\n"
            "  Fix:  pip install -e services/agents\n"
            "  Or:   pass --telemetry-only to emit just the T2.4 token/USD/latency block."
        )
        print(msg, file=sys.stderr)
        sys.exit(3)

    keep_records = not args.no_telemetry_records
    per_investigation = _build_per_investigation_block(
        args.telemetry_model,
        keep_records=keep_records,
    )

    if args.telemetry_only:
        summary: dict = {
            "generated_at": datetime.now(UTC).isoformat(),
            "dataset": "synthetic_incidents.json (200 cases, deterministic)",
            "telemetry_only": True,
            "per_investigation": per_investigation,
        }
        summary["all_passed"] = True  # telemetry-only never gates substrate
    else:
        # Only the requested suites are *called*. The previous form built this
        # dict with all eleven calls inline, so `--suite mitre_accuracy` ran
        # every suite, took the full runtime, and then printed
        # "PASS — mitre_accuracy green" — a verdict naming one suite and
        # decided by eleven. `--ci` would fail a single-suite run on an
        # unrelated regression, which is the opposite of what an operator
        # bisecting to one gate is asking for. `args.suite` reached nothing
        # but the banner wording.
        runners = {
            "mitre_accuracy": _run_mitre,
            "alert_reduction": _run_alert_reduction,
            "investigation_completeness": _run_completeness,
            "response_quality": _run_response_quality,
            "hunt_corpus": _run_hunt_corpus,
            "adversary_eval": _run_adversary,
            "confidence_calibration": _run_confidence_calibration,
            "memory_recall": _run_memory_recall,
            "override_accuracy": _run_override_accuracy,
            "playbook_completion_rate": _run_playbook_completion,
            "detection_fp_rate": _run_detection_fp_rate,
        }
        # The registry and the name list are the same set, asserted rather than
        # assumed: argparse accepts a name from `_SUITE_NAMES`, so one missing
        # here would be a KeyError at the moment somebody bisects a regression.
        if set(runners) != set(_SUITE_NAMES):
            raise AssertionError(f"suite registry and _SUITE_NAMES disagree: {set(runners) ^ set(_SUITE_NAMES)}")
        selected = _SUITE_NAMES if args.suite == "all" else (args.suite,)
        summary = {
            "generated_at": datetime.now(UTC).isoformat(),
            "dataset": "synthetic_incidents.json (200 cases, deterministic)",
            # Which suites this report describes. Without it a single-suite
            # report is indistinguishable from a full one that lost ten suites.
            "suite_filter": args.suite,
            "suites": {name: runners[name]() for name in selected},
            "telemetry": _summarise_telemetry(),
            "per_investigation": per_investigation,
        }
        summary["all_passed"] = all(s["passed"] for s in summary["suites"].values())

    regression_failure = False
    if args.baseline is not None and not args.telemetry_only:
        if not args.baseline.exists():
            summary["baseline_compare"] = {
                "baseline_path": str(args.baseline),
                "available": False,
                "note": "baseline file not found; treating as no-op",
            }
        else:
            try:
                baseline = json.loads(args.baseline.read_text())
            except json.JSONDecodeError as exc:
                summary["baseline_compare"] = {
                    "baseline_path": str(args.baseline),
                    "available": False,
                    "error": f"invalid baseline JSON: {exc}",
                }
            else:
                deltas: dict[str, dict] = {}
                worst_mitre_drop_pp = 0.0
                for name, suite in summary["suites"].items():
                    base_suite = baseline.get("suites", {}).get(name) or {}
                    base_value = float(base_suite.get("value", suite["value"]))
                    delta_pp = round((suite["value"] - base_value) * 100, 4)
                    deltas[name] = {
                        "candidate": suite["value"],
                        "baseline": base_value,
                        "delta_pp": delta_pp,
                    }
                    if name == "mitre_accuracy" and delta_pp < -worst_mitre_drop_pp:
                        worst_mitre_drop_pp = -delta_pp
                regression_failure = worst_mitre_drop_pp >= args.max_regression_pp
                summary["baseline_compare"] = {
                    "baseline_path": str(args.baseline),
                    "available": True,
                    "max_regression_pp": args.max_regression_pp,
                    "mitre_drop_pp": round(worst_mitre_drop_pp, 4),
                    "regressed": regression_failure,
                    "deltas": deltas,
                }

    args.out.write_text(json.dumps(summary, indent=2))

    if args.json:
        print(json.dumps(summary, indent=2))
    elif args.telemetry_only:
        pi = cast(dict[str, Any], summary["per_investigation"])
        print()
        print("=" * 78)
        print("  AiSOC Eval - per-investigation telemetry (deterministic substrate)")
        print("=" * 78)
        print(f"  Mode:          {pi['mode']}")
        print(f"  Model:         {pi['model']}")
        print(
            f"  Rate (USD/M):  input ${pi['rate_card_per_m_tokens_usd']['input']:.2f}  "
            f"output ${pi['rate_card_per_m_tokens_usd']['output']:.2f}  (illustrative)"
        )
        print(f"  Incidents:     {pi['incidents']}  Templates: {pi['templates']}")
        print("-" * 78)
        tok = cast(dict[str, Any], pi["tokens_per_investigation"])
        print(f"  Tokens / investigation:  mean={tok['mean']:.0f}  median={tok['median']:.0f}  p95={tok['p95']:.0f}  p99={tok['p99']:.0f}")
        print(f"      prompt mean={tok['prompt_mean']:.0f}    completion mean={tok['completion_mean']:.0f}")
        usd = cast(dict[str, Any], pi["usd_per_investigation"])
        print(
            f"  USD / investigation:     mean=${usd['mean']:.5f}  median=${usd['median']:.5f}  p95=${usd['p95']:.5f}  p99=${usd['p99']:.5f}"
        )
        lat = cast(dict[str, Any], pi["latency_per_investigation_ms"])
        print(f"  Latency (ms / inv):      p50={lat['p50']:.4f}  p95={lat['p95']:.4f}  p99={lat['p99']:.4f}  (substrate-only path)")
        print("=" * 78)
        try:
            rel = args.out.relative_to(_REPO_ROOT)
        except ValueError:
            rel = args.out
        print(f"  Report written to: {rel}")
        print()
    else:
        print()
        print("=" * 78)
        print("  AiSOC Pillar-1 Eval - 200-incident synthetic benchmark")
        print("=" * 78)
        for name, suite in summary["suites"].items():
            mark = "PASS" if suite["passed"] else "FAIL"
            lower_is_better = bool(suite.get("details", {}).get("lower_is_better"))
            comparator = "<=" if lower_is_better else ">="
            print(f"  [{mark}] {name:<28} {suite['metric']:<28} {suite['value']:.3f}  (target {comparator} {suite['target']:.2f})")
            tpl = suite.get("per_template")
            if tpl:
                tpl_mark = "PASS" if tpl["passed"] else "FAIL"
                fail_count = len(tpl.get("failing_templates", []))
                fail_note = f" ({fail_count} failing templates)" if fail_count else ""
                print(
                    f"         per-template macro       "
                    f"{tpl['value']:.3f}  (target >= {tpl['target']:.2f}, "
                    f"n={tpl.get('template_count', 0)} templates) [{tpl_mark}]"
                    f"{fail_note}"
                )
                if fail_count:
                    failing = ", ".join(tpl["failing_templates"][:5])
                    suffix = "..." if fail_count > 5 else ""
                    print(f"           regressions: {failing}{suffix}")
        print("-" * 78)
        tele = cast(dict[str, Any], summary["telemetry"])
        if tele.get("present"):
            print(
                f"  Synthetic telemetry: {tele['events']} events across "
                f"{len(tele['sources'])} sources, "
                f"{tele['incidents_with_telemetry']} incidents wired up "
                f"({tele['path']})"
            )
        else:
            print("  Synthetic telemetry: <not generated>")
        print("-" * 78)
        pi = cast(dict[str, Any], summary.get("per_investigation") or {})
        if pi:
            tok = pi.get("tokens_per_investigation", {})
            usd = pi.get("usd_per_investigation", {})
            lat = pi.get("latency_per_investigation_ms", {})
            print(f"  Per-investigation budget (deterministic substrate, model={pi.get('model', '?')})")
            print(
                f"    tokens   mean={tok.get('mean', 0):.0f}  median={tok.get('median', 0):.0f}  "
                f"p95={tok.get('p95', 0):.0f}  p99={tok.get('p99', 0):.0f}"
            )
            print(
                f"    USD      mean=${usd.get('mean', 0):.5f}  median=${usd.get('median', 0):.5f}  "
                f"p95=${usd.get('p95', 0):.5f}  p99=${usd.get('p99', 0):.5f}"
            )
            print(
                f"    latency  p50={lat.get('p50', 0):.4f} ms  p95={lat.get('p95', 0):.4f} ms  "
                f"p99={lat.get('p99', 0):.4f} ms  (substrate path)"
            )
        print("=" * 78)
        if summary["all_passed"]:
            verdict = "PASS — ALL GATES GREEN" if args.suite == "all" else f"PASS — {args.suite} green"
        else:
            verdict = "FAIL — REGRESSION DETECTED" if args.suite == "all" else f"FAIL — {args.suite} regressed"
        print(f"  {verdict}")
        cmp = cast(dict[str, Any] | None, summary.get("baseline_compare"))
        if cmp and cmp.get("available"):
            arrow = "DROP" if cmp["regressed"] else "OK"
            print(
                f"  Baseline compare: MITRE Δ = "
                f"{cmp['deltas']['mitre_accuracy']['delta_pp']:+.2f} pp "
                f"(allowed drop ≤ {cmp['max_regression_pp']:.2f} pp) [{arrow}]"
            )
        try:
            rel = args.out.relative_to(_REPO_ROOT)
        except ValueError:
            rel = args.out
        print(f"  Report written to: {rel}")
        print()

    if regression_failure:
        sys.exit(2)
    sys.exit(0 if (summary["all_passed"] or not args.ci) else 1)


if __name__ == "__main__":
    main()