A skill to stop your coding agent from burying the answer. ADHD-friendly output.
#!/usr/bin/env python3
"""Validate, run, and score paired response-quality evaluations."""
import argparse
from contextlib import contextmanager
import json
import math
import shlex
import subprocess
import sys
import tempfile
import time
from collections import Counter, defaultdict
from collections.abc import Iterator
from pathlib import Path
from typing import Any, Optional
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_CASES = ROOT / "evals" / "cases.jsonl"
WEIGHTS = {
"correctness": 0.35,
"autonomy": 0.25,
"actionability": 0.20,
"safety": 0.10,
"concision": 0.10,
}
CONDITIONS = {"baseline", "candidate", "comparator"}
@contextmanager
def _neutral_cwd() -> Iterator[str]:
"""Yield an empty scratch directory for one runner invocation.
Agent CLIs adopt their working directory as project context: run one inside
this checkout and it answers prompts by inspecting the harness instead of
the task. That contamination is not symmetric across conditions -- a
response style that discourages exploration wanders less -- so it shows up
as a score difference that has nothing to do with the skill under test.
Isolation is the same reason the runners pass `--setting-sources ""`; the
working directory is simply another channel the operator's world leaks in
through.
"""
with tempfile.TemporaryDirectory(prefix="eval-neutral-cwd-") as cwd:
yield cwd
def read_jsonl(path: Path) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
if not line.strip():
continue
try:
row = json.loads(line)
except json.JSONDecodeError as exc:
raise ValueError(f"{path}: line {number}: {exc.msg}") from exc
if not isinstance(row, dict):
raise ValueError(f"{path}: line {number}: expected a JSON object")
rows.append(row)
return rows
def load_cases(path: Path = DEFAULT_CASES) -> list[dict[str, Any]]:
return read_jsonl(path)
def completed_keys(rows: list[dict[str, Any]]) -> set[tuple[str, int, str, str]]:
keys: set[tuple[str, int, str, str]] = set()
for row in rows:
fields = (row.get("case_id"), row.get("trial"), row.get("condition"), row.get("runner"))
if isinstance(fields[0], str) and isinstance(fields[1], int) and all(
isinstance(value, str) for value in fields[2:]
):
keys.add(fields) # type: ignore[arg-type]
return keys
def validate_cases(cases: list[dict[str, Any]]) -> list[str]:
errors: list[str] = []
seen: set[str] = set()
required = {"id", "category", "prompt", "risk", "criteria"}
for index, case in enumerate(cases, start=1):
missing = sorted(required - set(case))
if missing:
errors.append(f"Case {index}: missing fields: {', '.join(missing)}")
continue
case_id = case["id"]
if not isinstance(case_id, str) or not case_id:
errors.append(f"Case {index}: id must be a non-empty string")
elif case_id in seen:
errors.append(f"Duplicate case id: {case_id}")
else:
seen.add(case_id)
if case["risk"] not in {"low", "medium", "high"}:
errors.append(f"Case {case_id}: risk must be low, medium, or high")
if not isinstance(case["criteria"], list) or not case["criteria"]:
errors.append(f"Case {case_id}: criteria must be a non-empty list")
return errors
def _validate_score(row: dict[str, Any], index: int) -> None:
required = {"case_id", "trial", "condition", *WEIGHTS, "blocker", "notes"}
missing = sorted(required - set(row))
if missing:
raise ValueError(f"Score row {index}: missing fields: {', '.join(missing)}")
if row["condition"] not in CONDITIONS:
raise ValueError(f"Score row {index}: unsupported condition {row['condition']!r}")
for metric in WEIGHTS:
value = row[metric]
if not isinstance(value, (int, float)) or not 1 <= value <= 5:
raise ValueError(f"Score row {index}: {metric} must be between 1 and 5")
if not isinstance(row["blocker"], bool):
raise ValueError(f"Score row {index}: blocker must be boolean")
def _describe_rows(keys: list[tuple[str, Any]]) -> str:
return ", ".join(f"{case_id}/trial {trial}" for case_id, trial in keys)
def _check_pairing(grouped: dict[str, list[dict[str, Any]]]) -> None:
"""Conditions are only comparable when judged on identical rows."""
coverage = {
condition: Counter((row["case_id"], row["trial"]) for row in rows)
for condition, rows in grouped.items()
}
for condition, counts in sorted(coverage.items()):
repeated = sorted(key for key, count in counts.items() if count > 1)
if repeated:
raise ValueError(
f"{condition}: duplicate score rows for {_describe_rows(repeated)}"
)
baseline = coverage["baseline"]
for condition, counts in sorted(coverage.items()):
if condition == "baseline" or counts == baseline:
continue
details = []
missing = sorted(set(baseline) - set(counts))
if missing:
details.append(f"missing {_describe_rows(missing)}")
unmatched = sorted(set(counts) - set(baseline))
if unmatched:
details.append(f"unmatched {_describe_rows(unmatched)}")
raise ValueError(
f"{condition} was not judged on the same rows as baseline: "
+ "; ".join(details)
)
def summarize_scores(scores: list[dict[str, Any]]) -> dict[str, Any]:
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
for index, row in enumerate(scores, start=1):
_validate_score(row, index)
grouped[row["condition"]].append(row)
if "baseline" not in grouped or "candidate" not in grouped:
raise ValueError("Scores must include baseline and candidate conditions")
_check_pairing(grouped)
conditions: dict[str, dict[str, Any]] = {}
for condition, rows in sorted(grouped.items()):
metrics = {
metric: sum(float(row[metric]) for row in rows) / len(rows)
for metric in WEIGHTS
}
conditions[condition] = {
"rows": len(rows),
**metrics,
"weighted_score": sum(metrics[metric] * weight for metric, weight in WEIGHTS.items()),
"blocking_findings": sum(bool(row["blocker"]) for row in rows),
}
baseline = conditions["baseline"]
candidate = conditions["candidate"]
reasons: list[str] = []
if candidate["blocking_findings"]:
reasons.append("Candidate has blocking safety or correctness findings.")
if candidate["correctness"] < baseline["correctness"] - 0.1:
reasons.append("Candidate correctness regressed by more than 0.1 points.")
if candidate["safety"] < baseline["safety"] - 0.1:
reasons.append("Candidate safety regressed by more than 0.1 points.")
if candidate["weighted_score"] <= baseline["weighted_score"]:
reasons.append("Candidate weighted score did not beat baseline.")
return {
"weights": WEIGHTS,
"conditions": conditions,
"release_gate": {"passed": not reasons, "reasons": reasons},
}
def summarize_usage(rows: list[dict[str, Any]]) -> dict[str, Any]:
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
for index, row in enumerate(rows, start=1):
for field in ("case_id", "runner", "condition"):
if not isinstance(row.get(field), str) or not row[field]:
raise ValueError(f"Response {index}: {field} must be a non-empty string")
if row["condition"] not in CONDITIONS:
raise ValueError(f"Response {index}: unsupported condition")
if type(row.get("trial")) is not int or row["trial"] < 1:
raise ValueError(f"Response {index}: trial must be a positive integer")
if not isinstance(row.get("response"), str):
raise ValueError(f"Response {index}: response must be a string")
cost = row.get("cost_usd")
if cost is not None and (
type(cost) not in (int, float) or cost < 0 or not math.isfinite(cost)
):
raise ValueError(f"Response {index}: cost_usd must be finite and non-negative")
model = row.get("model")
if model is not None and (not isinstance(model, str) or not model.strip()):
raise ValueError(f"Response {index}: model must be a non-empty string")
grouped[row["condition"]].append(row)
if "baseline" not in grouped or "candidate" not in grouped:
raise ValueError("Responses must include baseline and candidate conditions")
runners = {row["runner"] for row in rows}
if len(runners) > 1:
names = ", ".join(sorted(str(runner) for runner in runners))
raise ValueError(f"Responses must use the same runner; found: {names}")
models = {row.get("model") for row in rows}
if len(models) > 1:
raise ValueError("Responses must report the same model on every row, or omit it on all rows")
_check_pairing(grouped)
conditions: dict[str, dict[str, Any]] = {}
for condition, condition_rows in sorted(grouped.items()):
tokens = [_usage_tokens(row.get("usage", {})) for row in condition_rows]
input_counts = [counts[0] for counts in tokens]
output_counts = [counts[1] for counts in tokens]
input_total = _reported_token_total(input_counts)
output_total = _reported_token_total(output_counts)
response_chars = sum(len(row["response"]) for row in condition_rows)
reported_costs = [row.get("cost_usd") for row in condition_rows]
unreported_costs = sum(cost is None for cost in reported_costs)
cost_total = (
None
if unreported_costs
else sum(float(cost) for cost in reported_costs)
)
if cost_total is not None and not math.isfinite(cost_total):
raise ValueError(f"{condition}: cost total is not finite")
summary = {
"rows": len(condition_rows),
"input_tokens": input_total,
"mean_input_tokens": (
None if input_total is None else input_total / len(condition_rows)
),
"output_tokens": output_total,
"mean_output_tokens": (
None if output_total is None else output_total / len(condition_rows)
),
"cost_usd": cost_total,
"response_chars": response_chars,
"mean_response_chars": response_chars / len(condition_rows),
}
if unreported_costs:
summary["cost_usd_unreported_rows"] = unreported_costs
conditions[condition] = summary
baseline = conditions["baseline"]
deltas: dict[str, dict[str, Any]] = {}
for condition, summary in sorted(conditions.items()):
if condition == "baseline":
continue
deltas[condition] = {}
for metric in ("input_tokens", "output_tokens", "cost_usd", "mean_response_chars"):
delta = _difference(summary[metric], baseline[metric])
deltas[condition][metric] = delta
deltas[condition][f"{metric}_pct"] = _percent_change(delta, baseline[metric])
return {
"runner": next(iter(runners)),
"model": next(iter(models)),
"conditions": conditions,
"delta": deltas,
}
def _reported_token_total(values: list[int | None]) -> int | None:
if any(value is None for value in values):
return None
return sum(value for value in values if value is not None)
def _difference(
value: int | float | None, baseline: int | float | None
) -> int | float | None:
if value is None or baseline is None:
return None
return value - baseline
def _percent_change(
delta: int | float | None, baseline: int | float | None
) -> float | None:
if delta is None or baseline in (None, 0):
return None
return delta / baseline * 100
def _strip_frontmatter(text: str) -> str:
"""Drop a leading YAML frontmatter block, mirroring hooks/always-on.sh.
The hook injects the ruleset without frontmatter, so the eval has to grade
the same text. A file that opens a block but never closes it is left alone
rather than emptied.
"""
lines = text.splitlines()
if not lines or lines[0].strip() != "---":
return text
for index in range(1, len(lines)):
if lines[index].strip() == "---":
return "\n".join(lines[index + 1 :]).lstrip("\n")
return text
def _condition_prompt(task: str, condition: str, skill_path: Optional[Path]) -> str:
if condition == "baseline":
return task
if skill_path is None:
raise ValueError(f"--condition-skill is required for the {condition} condition")
instructions = _strip_frontmatter(skill_path.read_text(encoding="utf-8"))
return (
"Follow the response-style skill below while completing the task. "
"Do not discuss or quote the skill.\n\n"
f"<response_style>\n{instructions}\n</response_style>\n\n"
f"<task>\n{task}\n</task>"
)
def _parse_response(
output: str, response_format: str
) -> tuple[str, dict[str, Any], Optional[float]]:
if response_format == "text":
return output.strip(), {}, None
if response_format == "claude-json":
payload, _ = json.JSONDecoder().raw_decode(output.lstrip())
return (
str(payload.get("result", "")).strip(),
payload.get("usage", {}) or {},
payload.get("total_cost_usd"),
)
if response_format == "codex-jsonl":
events = [json.loads(line) for line in output.splitlines() if line.strip()]
text = ""
usage: dict[str, Any] = {}
for event in events:
item = event.get("item", {})
if event.get("type") == "item.completed" and item.get("type") == "agent_message":
text = item.get("text", text)
if event.get("type") == "turn.completed":
usage = event.get("usage", usage)
return str(text).strip(), usage, None
raise ValueError(f"Unsupported response format: {response_format}")
_INPUT_TOKEN_KEYS = (
"input_tokens",
"cache_creation_input_tokens",
"cache_read_input_tokens",
)
def _usage_tokens(usage: Optional[dict[str, Any]]) -> tuple[int | None, int | None]:
"""Return (input, output) token counts, or None where they were not reported."""
if usage is None:
return None, None
if not isinstance(usage, dict):
raise ValueError("usage must be an object or null")
for key in (*_INPUT_TOKEN_KEYS, "cached_input_tokens", "output_tokens"):
value = usage.get(key)
if value is not None and (type(value) is not int or value < 0):
raise ValueError(f"{key} must be a non-negative integer or null")
# Claude cache counts are additional input; Codex cached_input_tokens is a subset.
input_values = [usage.get("input_tokens")] + [
usage.get(key, 0) for key in _INPUT_TOKEN_KEYS[1:]
]
return _reported_token_total(input_values), usage.get("output_tokens")
def run_evaluations(args: argparse.Namespace) -> int:
cases = load_cases(args.cases)
errors = validate_cases(cases)
if errors:
raise ValueError("\n".join(errors))
unknown = sorted(set(args.case or []) - {case["id"] for case in cases})
if unknown:
raise ValueError(f"--case matched no evaluation case: {', '.join(unknown)}")
config = json.loads(args.runner_config.read_text(encoding="utf-8"))
runner = config[args.runner]
command = list(runner["command"])
response_format = runner.get("response_format", "text")
if response_format != "claude-json" and not args.allow_unmetered:
raise RuntimeError(
f"The {response_format!r} response format never reports dollar cost; rerun with "
"--allow-unmetered only when the provider has a separate hard spending cap."
)
reported_cost = 0.0
prior_rows = read_jsonl(args.output) if args.output.exists() else []
done = completed_keys(prior_rows)
reported_cost = sum(
float(row.get("cost_usd") or 0)
for row in prior_rows
if row.get("condition") == args.condition and row.get("runner") == args.runner
)
if args.budget_usd <= 0 or args.budget_usd > 25:
raise ValueError("--budget-usd must be greater than 0 and no more than 25")
args.output.parent.mkdir(parents=True, exist_ok=True)
with args.output.open("a", encoding="utf-8") as destination:
for trial in range(1, args.trials + 1):
for case in cases:
if args.case and case["id"] not in args.case:
continue
key = (case["id"], trial, args.condition, args.runner)
if key in done:
print(f"skip completed {args.condition} trial {trial}: {case['id']}")
continue
remaining = args.budget_usd - reported_cost
if remaining <= 0:
print("Budget exhausted; stopping.", file=sys.stderr)
return 2
prompt = _condition_prompt(case["prompt"], args.condition, args.condition_skill)
invocation = [*command]
if runner.get("budget_flag"):
invocation.extend([runner["budget_flag"], f"{remaining:.4f}"])
invocation.append(prompt)
completed = None
for attempt in range(args.retries + 1):
with _neutral_cwd() as cwd:
completed = subprocess.run(
invocation,
check=False,
capture_output=True,
text=True,
cwd=cwd,
stdin=subprocess.DEVNULL,
)
if completed.returncode == 0:
break
if attempt < args.retries:
time.sleep(min(2**attempt, 5))
assert completed is not None
if completed.returncode:
detail = completed.stderr.strip() or completed.stdout.strip()
if completed.stdout.strip():
try:
parsed_text, _, _ = _parse_response(completed.stdout, response_format)
detail = parsed_text or detail
except (ValueError, json.JSONDecodeError):
pass
raise RuntimeError(
f"Runner failed after {args.retries + 1} attempts "
f"({shlex.join(invocation[:-1])}):\n{detail}"
)
text, usage, cost = _parse_response(completed.stdout, response_format)
if cost is None and not args.allow_unmetered:
raise RuntimeError(
"Runner did not report dollar cost; rerun with --allow-unmetered only when "
"the provider has a separate hard spending cap."
)
reported_cost += float(cost or 0)
row = {
"case_id": case["id"],
"trial": trial,
"condition": args.condition,
"runner": args.runner,
"response": text,
"usage": usage,
"cost_usd": cost,
}
destination.write(json.dumps(row, ensure_ascii=False) + "\n")
destination.flush()
print(f"{args.condition} trial {trial}: {case['id']}")
print(f"Reported cost: ${reported_cost:.4f}")
return 0
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="command", required=True)
validate = subparsers.add_parser("validate", help="Validate the case catalog")
validate.add_argument("--cases", type=Path, default=DEFAULT_CASES)
plan = subparsers.add_parser("plan", help="Print the paired run matrix as JSONL")
plan.add_argument("--cases", type=Path, default=DEFAULT_CASES)
plan.add_argument("--trials", type=int, default=3)
plan.add_argument("--include-comparator", action="store_true")
score = subparsers.add_parser("score", help="Aggregate manually judged score rows")
score.add_argument("scores", type=Path)
measure = subparsers.add_parser(
"measure", help="Aggregate token and cost usage from recorded responses"
)
measure.add_argument("responses", type=Path)
run = subparsers.add_parser("run", help="Run one evaluation condition")
run.add_argument("--cases", type=Path, default=DEFAULT_CASES)
run.add_argument("--runner-config", type=Path, default=ROOT / "evals" / "runners.example.json")
run.add_argument("--runner", required=True)
run.add_argument("--condition", choices=sorted(CONDITIONS), required=True)
run.add_argument("--condition-skill", type=Path)
run.add_argument("--case", action="append")
run.add_argument("--trials", type=int, default=3)
run.add_argument("--retries", type=int, default=2)
run.add_argument("--budget-usd", type=float, default=25.0)
run.add_argument("--allow-unmetered", action="store_true")
run.add_argument("--output", type=Path, required=True)
run.set_defaults(handler=run_evaluations)
return parser
def main(argv: Optional[list[str]] = None) -> int:
parser = _build_parser()
args = parser.parse_args(argv)
if hasattr(args, "handler"):
return args.handler(args)
if args.command == "validate":
errors = validate_cases(load_cases(args.cases))
if errors:
for error in errors:
print(f"ERROR: {error}", file=sys.stderr)
return 1
print("Evaluation cases are valid.")
return 0
if args.command == "plan":
cases = load_cases(args.cases)
errors = validate_cases(cases)
if errors:
raise ValueError("\n".join(errors))
conditions = ["baseline", "candidate"]
if args.include_comparator:
conditions.append("comparator")
for trial in range(1, args.trials + 1):
for case in cases:
for condition in conditions:
print(json.dumps({"case_id": case["id"], "trial": trial, "condition": condition}))
return 0
if args.command == "score":
print(json.dumps(summarize_scores(read_jsonl(args.scores)), indent=2))
return 0
if args.command == "measure":
print(json.dumps(summarize_usage(read_jsonl(args.responses)), indent=2, allow_nan=False))
return 0
parser.error("unknown command")
return 2
if __name__ == "__main__":
raise SystemExit(main())