Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

7.7 KB
Raw Blame
"""Evaluate a fine-tuned checkpoint on the typed-decisions test split.

Mirrors cell 14 of the published notebook, restricted to the three metrics the Step-4 contract
names: **accuracy**, **ECE** and **Brier**. The 20-option order-flip surface is deliberately not
here: it belongs to `massive_intent.en`, not typed-decisions (see README).

Calibration is not byte-comparable across contracts. `laya.train` fits temperatures with
`fit_temperature_map` (per type, and per option-count bucket above a floor) under the runtime
clamp `[0.5, 5.0]`; the old notebook fitted one LBFGS temperature per type under `[0.1, 10]`.
So ECE and Brier support either path, one per invocation:

- ``--temperatures shipped`` (default) — the checkpoint's own fitted `temperature` and
  `temperature_by_options`, which is what the notebook's number also used;
- ``--temperatures raw`` — temperature 1.0 everywhere, isolating the training from the
  calibration contract.

Each run evaluates exactly one of the two and records which in the output; the committed Gate D
table is the shipped path.

Accuracy is unaffected by temperature (it only rescales the logits), so it is the clean
reproduction signal.

Usage:
    python evaluate.py --checkpoint runs/step4/rlcd-seed0 \
        --test-meta data/typed_decisions/test_meta.json \
        --out runs/step4/rlcd-seed0/eval_shipped.json
"""
from __future__ import annotations

import argparse
import json
import os
import sys
from typing import Any, Dict, List

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from metrics import (  # noqa: E402
    accuracy,
    brier_score,
    expected_calibration_error,
    soft_accuracy,
)


def _normalise(values: List[float]) -> List[float]:
    total = sum(values)
    if total > 0:
        return [v / total for v in values]
    return values


def evaluate_checkpoint(checkpoint_dir: str, test_meta: List[Dict[str, Any]],
                        device: str = "auto", temperatures: str = "shipped") -> Dict[str, Any]:
    import laya

    agent = laya.load(checkpoint_dir, device=device)
    if temperatures == "raw":
        agent.temperature = [1.0, 1.0, 1.0]
        agent.temperature_by_options = {}
    elif temperatures != "shipped":
        raise ValueError("temperatures must be 'shipped' or 'raw', got %r" % (temperatures,))

    corrects: List[float] = []
    confidences: List[float] = []
    briers: List[float] = []
    softs: List[float] = []
    per_type: Dict[str, List[float]] = {"choice": [], "noul": [], "score": []}
    score_maes: List[float] = []
    within_one: List[float] = []

    for i, case in enumerate(test_meta):
        state = case["state"]
        questions = case["questions"]
        gold = case["gold"]
        answers = agent.predict(state, questions)["answers"]

        for qid, qdef in questions.items():
            if qid not in gold:
                continue
            g = gold[qid]
            qtype = qdef["type"]
            p_ans = answers[qid]

            if qtype == "choice":
                keys = list(qdef["criteria"].keys())
                pred = p_ans["choice"]
                gold_label = str(g["label"])
                is_corr = float(pred == gold_label)

                p_probs = _normalise([p_ans["probabilities"].get(k, 1e-6) for k in keys])
                g_probs = _normalise([g["probabilities"].get(k, 1e-6) for k in keys])

                corrects.append(is_corr)
                confidences.append(max(p_probs))
                briers.append(brier_score(p_probs, g_probs))
                softs.append(soft_accuracy(p_probs, g_probs))
                per_type["choice"].append(is_corr)

            elif qtype == "noul":
                p_val = float(p_ans["noul"])
                g_val = float(g.get("noul", g.get("probabilities", {}).get("true", 0.5)))
                gold_label = str(g["label"]).lower()
                pred_label = "true" if p_val >= 0.5 else "false"
                is_corr = float(pred_label == gold_label)

                p_dist = [1.0 - p_val, p_val]
                g_dist = [1.0 - g_val, g_val]

                corrects.append(is_corr)
                confidences.append(max(p_val, 1.0 - p_val))
                briers.append(brier_score(p_dist, g_dist))
                softs.append(soft_accuracy(p_dist, g_dist))
                per_type["noul"].append(is_corr)

            elif qtype == "score":
                p_score = float(p_ans["score"])
                g_score = float(g.get("score", 0.0))
                score_maes.append(abs(p_score - g_score))
                within_one.append(float(abs(p_score - g_score) <= 1.0))

                n_levels = len(qdef.get("criteria", []))
                p_probs = [p_ans["probabilities"].get(str(i), 0.0) for i in range(n_levels)]
                if sum(p_probs) > 0:
                    p_probs = _normalise(p_probs)
                    p_lvl = int(max(range(len(p_probs)), key=lambda j: p_probs[j]))
                    confidences.append(max(p_probs))
                else:
                    p_lvl = int(round(p_score))
                    confidences.append(0.5)

                g_lvl = int(g.get("label", int(round(g_score))))
                is_corr = float(p_lvl == g_lvl)
                corrects.append(is_corr)
                per_type["score"].append(is_corr)

        if (i + 1) % 100 == 0:
            print(f"  {i + 1}/{len(test_meta)} cases", flush=True)

    result = {
        "checkpoint": checkpoint_dir,
        "temperatures": temperatures,
        "n_cases": len(test_meta),
        "n_decisions": len(corrects),
        "accuracy": round(accuracy(corrects), 4),
        "ece": round(expected_calibration_error(confidences, corrects), 4),
        "brier": round(float(sum(briers) / len(briers)) if briers else float("nan"), 4),
        "soft_accuracy": round(float(sum(softs) / len(softs)) if softs else float("nan"), 4),
        "score_mae": round(float(sum(score_maes) / len(score_maes)) if score_maes else float("nan"), 4),
        "within_one_level": round(float(sum(within_one) / len(within_one)) if within_one else float("nan"), 4),
        "per_type_accuracy": {k: round(accuracy(v), 4) for k, v in per_type.items() if v},
    }
    # `brier`/`soft_accuracy`/`ece` mix question types; record the counts they rest on.
    result["n_brier_decisions"] = len(briers)
    return result


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--checkpoint", required=True)
    parser.add_argument("--test-meta", required=True)
    parser.add_argument("--device", default="auto")
    parser.add_argument("--temperatures", choices=["shipped", "raw"], default="shipped")
    parser.add_argument("--out", default=None,
                        help="Output JSON (default: <checkpoint>/eval_<temperatures>.json)")
    args = parser.parse_args()

    with open(args.test_meta, encoding="utf-8") as f:
        test_meta = json.load(f)
    print(f"loaded {len(test_meta)} test cases", flush=True)

    result = evaluate_checkpoint(args.checkpoint, test_meta, device=args.device,
                                 temperatures=args.temperatures)

    print("\n" + "=" * 60)
    print(f"evaluation: {args.checkpoint} ({args.temperatures} temperatures)")
    print("=" * 60)
    print(f"  accuracy:     {result['accuracy']:.4f}")
    print(f"  ECE:          {result['ece']:.4f}")
    print(f"  Brier:        {result['brier']:.4f}")
    print(f"  soft acc:     {result['soft_accuracy']:.4f}")
    print(f"  per type:     {result['per_type_accuracy']}")
    print("=" * 60)

    out_path = args.out or os.path.join(args.checkpoint, f"eval_{args.temperatures}.json")
    os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
    with open(out_path, "w", encoding="utf-8") as f:
        json.dump(result, f, indent=2)
    print(f"wrote {out_path}")
    return 0


if __name__ == "__main__":
    sys.exit(main())