Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
research: recompute ECE under both definitions of confidence (#327)
君莫88fef3338dcf
diff --git a/research/results/confidence_definitions.json b/research/results/confidence_definitions.jsonnew file mode 100644index 00000000..dc7b64af--- /dev/null+++ b/research/results/confidence_definitions.json@@ -0,0 +1,350 @@+{+ "bins": 15,+ "models": {+ "jev": {+ "accuracy": 0.9948,+ "answer_confidence": {+ "acc_at_50_coverage": 1.0,+ "auroc": 1.0,+ "ece": 0.0274,+ "gate_at_0.85": {+ "accuracy_of_automated": 1.0,+ "automated": 181,+ "automated_share": 0.9427,+ "threshold": 0.85+ },+ "mean_confidence": 0.9674,+ "reliability": [+ {+ "accuracy": 0.6667,+ "bin": "0.47-0.53",+ "gap": -0.1533,+ "mean_confidence": 0.5133,+ "n": 3+ },+ {+ "accuracy": 1.0,+ "bin": "0.73-0.80",+ "gap": -0.234,+ "mean_confidence": 0.766,+ "n": 5+ },+ {+ "accuracy": 1.0,+ "bin": "0.80-0.87",+ "gap": -0.1633,+ "mean_confidence": 0.8367,+ "n": 6+ },+ {+ "accuracy": 1.0,+ "bin": "0.87-0.93",+ "gap": -0.0918,+ "mean_confidence": 0.9082,+ "n": 11+ },+ {+ "accuracy": 1.0,+ "bin": "0.93-1.00",+ "gap": -0.0098,+ "mean_confidence": 0.9902,+ "n": 167+ }+ ]+ },+ "entropy_confidence": {+ "acc_at_50_coverage": 1.0,+ "auroc": 1.0,+ "ece": 0.0678,+ "gate_at_0.85": {+ "accuracy_of_automated": 1.0,+ "automated": 160,+ "automated_share": 0.8333,+ "threshold": 0.85+ },+ "mean_confidence": 0.9269,+ "reliability": [+ {+ "accuracy": 0.5,+ "bin": "0.33-0.40",+ "gap": -0.1168,+ "mean_confidence": 0.3832,+ "n": 2+ },+ {+ "accuracy": 1.0,+ "bin": "0.40-0.47",+ "gap": -0.5792,+ "mean_confidence": 0.4208,+ "n": 1+ },+ {+ "accuracy": 1.0,+ "bin": "0.53-0.60",+ "gap": -0.4056,+ "mean_confidence": 0.5944,+ "n": 2+ },+ {+ "accuracy": 1.0,+ "bin": "0.60-0.67",+ "gap": -0.368,+ "mean_confidence": 0.632,+ "n": 5+ },+ {+ "accuracy": 1.0,+ "bin": "0.67-0.73",+ "gap": -0.3004,+ "mean_confidence": 0.6996,+ "n": 6+ },+ {+ "accuracy": 1.0,+ "bin": "0.73-0.80",+ "gap": -0.2503,+ "mean_confidence": 0.7497,+ "n": 7+ },+ {+ "accuracy": 1.0,+ "bin": "0.80-0.87",+ "gap": -0.1727,+ "mean_confidence": 0.8273,+ "n": 12+ },+ {+ "accuracy": 1.0,+ "bin": "0.87-0.93",+ "gap": -0.0901,+ "mean_confidence": 0.9099,+ "n": 28+ },+ {+ "accuracy": 1.0,+ "bin": "0.93-1.00",+ "gap": -0.011,+ "mean_confidence": 0.989,+ "n": 129+ }+ ]+ },+ "invariants_checked": false,+ "n": 192+ },+ "laya": {+ "accuracy": 0.3125,+ "answer_confidence": {+ "acc_at_50_coverage": 0.375,+ "auroc": 0.6193,+ "ece": 0.2614,+ "gate_at_0.85": {+ "accuracy_of_automated": 0.4444,+ "automated": 27,+ "automated_share": 0.1406,+ "threshold": 0.85+ },+ "mean_confidence": 0.5739,+ "reliability": [+ {+ "accuracy": 0.0,+ "bin": "0.27-0.33",+ "gap": 0.3117,+ "mean_confidence": 0.3117,+ "n": 21+ },+ {+ "accuracy": 0.2727,+ "bin": "0.33-0.40",+ "gap": 0.096,+ "mean_confidence": 0.3687,+ "n": 33+ },+ {+ "accuracy": 0.2857,+ "bin": "0.40-0.47",+ "gap": 0.1513,+ "mean_confidence": 0.437,+ "n": 21+ },+ {+ "accuracy": 0.4545,+ "bin": "0.47-0.53",+ "gap": 0.0436,+ "mean_confidence": 0.4981,+ "n": 33+ },+ {+ "accuracy": 0.3333,+ "bin": "0.53-0.60",+ "gap": 0.2467,+ "mean_confidence": 0.58,+ "n": 9+ },+ {+ "accuracy": 0.3333,+ "bin": "0.60-0.67",+ "gap": 0.2868,+ "mean_confidence": 0.6202,+ "n": 9+ },+ {+ "accuracy": 0.0,+ "bin": "0.67-0.73",+ "gap": 0.7171,+ "mean_confidence": 0.7171,+ "n": 6+ },+ {+ "accuracy": 0.375,+ "bin": "0.73-0.80",+ "gap": 0.3899,+ "mean_confidence": 0.7649,+ "n": 24+ },+ {+ "accuracy": 0.3333,+ "bin": "0.80-0.87",+ "gap": 0.4807,+ "mean_confidence": 0.8141,+ "n": 9+ },+ {+ "accuracy": 0.6,+ "bin": "0.87-0.93",+ "gap": 0.3008,+ "mean_confidence": 0.9008,+ "n": 15+ },+ {+ "accuracy": 0.25,+ "bin": "0.93-1.00",+ "gap": 0.7135,+ "mean_confidence": 0.9635,+ "n": 12+ }+ ]+ },+ "choice_confidence_max_drift": 0.000223,+ "entropy_confidence": {+ "acc_at_50_coverage": 0.3438,+ "auroc": 0.5966,+ "ece": 0.2202,+ "gate_at_0.85": {+ "accuracy_of_automated": 0.5,+ "automated": 6,+ "automated_share": 0.0312,+ "threshold": 0.85+ },+ "mean_confidence": 0.2889,+ "reliability": [+ {+ "accuracy": 0.125,+ "bin": "0.00-0.07",+ "gap": -0.0895,+ "mean_confidence": 0.0355,+ "n": 24+ },+ {+ "accuracy": 0.25,+ "bin": "0.07-0.13",+ "gap": -0.1545,+ "mean_confidence": 0.0955,+ "n": 48+ },+ {+ "accuracy": 0.4545,+ "bin": "0.13-0.20",+ "gap": -0.2872,+ "mean_confidence": 0.1674,+ "n": 33+ },+ {+ "accuracy": 0.0,+ "bin": "0.20-0.27",+ "gap": 0.246,+ "mean_confidence": 0.246,+ "n": 6+ },+ {+ "accuracy": 0.5,+ "bin": "0.27-0.33",+ "gap": -0.2094,+ "mean_confidence": 0.2906,+ "n": 12+ },+ {+ "accuracy": 0.25,+ "bin": "0.33-0.40",+ "gap": 0.1227,+ "mean_confidence": 0.3727,+ "n": 12+ },+ {+ "accuracy": 0.25,+ "bin": "0.40-0.47",+ "gap": 0.179,+ "mean_confidence": 0.429,+ "n": 12+ },+ {+ "accuracy": 0.25,+ "bin": "0.47-0.53",+ "gap": 0.2287,+ "mean_confidence": 0.4787,+ "n": 12+ },+ {+ "accuracy": 0.5,+ "bin": "0.53-0.60",+ "gap": 0.0461,+ "mean_confidence": 0.5461,+ "n": 6+ },+ {+ "accuracy": 1.0,+ "bin": "0.60-0.67",+ "gap": -0.3658,+ "mean_confidence": 0.6342,+ "n": 3+ },+ {+ "accuracy": 0.3333,+ "bin": "0.67-0.73",+ "gap": 0.3761,+ "mean_confidence": 0.7094,+ "n": 9+ },+ {+ "accuracy": 1.0,+ "bin": "0.73-0.80",+ "gap": -0.2512,+ "mean_confidence": 0.7488,+ "n": 3+ },+ {+ "accuracy": 0.0,+ "bin": "0.80-0.87",+ "gap": 0.8253,+ "mean_confidence": 0.8253,+ "n": 6+ },+ {+ "accuracy": 0.5,+ "bin": "0.87-0.93",+ "gap": 0.4028,+ "mean_confidence": 0.9028,+ "n": 6+ }+ ]+ },+ "invariants_checked": true,+ "n": 192,+ "noul_answers": 768,+ "noul_confidence_max_drift": 0.0,+ "shipped_choice_confidence_is_entropy": true,+ "shipped_noul_confidence_is_max_p": true+ }+ },+ "source": "research/benchmarks/feishu_zh/results/v1"+}diff --git a/research/scripts/confidence_definitions.py b/research/scripts/confidence_definitions.pynew file mode 100644index 00000000..26d976f7--- /dev/null+++ b/research/scripts/confidence_definitions.py@@ -0,0 +1,260 @@+"""Recompute ECE under both definitions of `confidence`, from results already in the repo.++ USE_TF=0 python3 research/scripts/confidence_definitions.py+ USE_TF=0 python3 research/scripts/confidence_definitions.py --write++`confidence` carries two definitions. `choice` and `score` answers report normalized entropy,+`1 - H(p) / log(k)`; `noul` answers report `max(p_true, 1 - p_true)`, which over two options is+`max(p)`. Both benchmark harnesses in this directory take `conf = max(probs)` before calling+`ece_score`, so every published ECE figure describes `max(p)` -- not the number a `choice` or+`score` answer actually returns.++This script measures the gap on shipped output rather than on a synthetic distribution. It+reads the per-question probabilities recorded by the Chinese workflow benchmark and reports,+for each definition, the accuracy-weighted calibration error and the reliability bins behind+it. Nothing is re-run: the probabilities are read from the checked-in JSONL.++ECE alone does not settle it, and the output says so. ECE is sensitive to the scale of the+number, not only to how well it orders answers: a confidence that is systematically compressed+can land near a low accuracy and score a low ECE without tracking correctness at all. So the+report also carries two scale-free readings -- AUROC of confidence against correctness, and the+accuracy you get by keeping only the most confident half -- plus what actually happens at the+0.85 gate the README's "Automated Confidence Gating" section uses.++It also checks the two invariants the comparison rests on, and fails loudly if either breaks:+the shipped `choice` confidence really is `confidence_from_probs`, and the shipped `noul`+confidence really is `max(p)` -- so `answer_confidence` is a no-op on every `noul` answer here.++Caveat worth stating up front: the only per-question probabilities checked into this repository+are this benchmark's, and Laya scores near chance on it (4 options, accuracy around 0.31). Every+calibration number below inherits that. The comparison between the two definitions is still+meaningful -- both are computed on the same predictions -- but neither is a calibration figure+for the checkpoint in general.+"""+import argparse+import json+import os+import sys++import numpy as np++REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))+sys.path.insert(0, REPO)++from laya.common import answer_confidence, confidence_from_probs, ece_score # noqa: E402++RESULTS = os.path.join(REPO, "research", "benchmarks", "feishu_zh", "results", "v1")+OUT = os.path.join(REPO, "research", "results", "confidence_definitions.json")+BINS = 15+# The stored probabilities are rounded to four decimals, so entropy recomputed from them lands+# a little away from the value the run reported off full-precision probabilities.+ROUNDING_SLACK = 2e-3+++def load(model):+ path = os.path.join(RESULTS, model, "raw.jsonl")+ with open(path, encoding="utf-8") as f:+ return [json.loads(line) for line in f if line.strip()]+++def is_laya(rows):+ """The invariants below describe Laya's own answer shape, so they are only checked there."""+ return any(str((r.get("response") or {}).get("model", "")).startswith("laya") for r in rows)+++def choice_rows(rows):+ """(probability vector, gold index, shipped confidence) for every answered choice row."""+ out = []+ for r in rows:+ if r.get("mode") != "choice" or r.get("status") != "ok" or not r.get("probabilities"):+ continue+ labels = list(r["probabilities"])+ if r["expected"] not in labels:+ continue+ p = np.array([float(r["probabilities"][k]) for k in labels], dtype=float)+ out.append((p, labels.index(r["expected"]), r.get("confidence")))+ return out+++def noul_answers(rows):+ """Every individual noul answer across the four-question rows."""+ out = []+ for r in rows:+ if r.get("status") != "ok":+ continue+ for ans in (r.get("response") or {}).get("answers", {}).values():+ if ans.get("type") == "noul" and "noul" in ans and "confidence" in ans:+ out.append((float(ans["noul"]), float(ans["confidence"])))+ return out+++def auroc(conf, correct):+ """Probability that a correct answer outranks an incorrect one. Scale-free, so it isolates+ ordering from the units the confidence happens to be reported in."""+ pos, neg = conf[correct > 0.5], conf[correct <= 0.5]+ if not len(pos) or not len(neg):+ return float("nan")+ order = np.argsort(np.concatenate([pos, neg]), kind="mergesort")+ ranks = np.empty(len(order), dtype=float)+ ranks[order] = np.arange(1, len(order) + 1)+ values = np.concatenate([pos, neg])+ for v in np.unique(values): # average ranks over ties+ tie = values == v+ if tie.sum() > 1:+ ranks[tie] = ranks[tie].mean()+ return float((ranks[:len(pos)].sum() - len(pos) * (len(pos) + 1) / 2) / (len(pos) * len(neg)))+++def acc_at_coverage(conf, correct, coverage):+ """Accuracy over the most confident `coverage` share -- what selective gating actually buys."""+ k = max(1, int(round(len(conf) * coverage)))+ return float(correct[np.argsort(-conf, kind="mergesort")[:k]].mean())+++def gate(conf, correct, threshold=0.85):+ """What the README's gating example does at this threshold: how much is automated, and how+ much of that is right."""+ sel = conf >= threshold+ return {"threshold": threshold, "automated": int(sel.sum()),+ "automated_share": round(float(sel.mean()), 4),+ "accuracy_of_automated": round(float(correct[sel].mean()), 4) if sel.any() else None}+++def reliability(conf, correct, bins=BINS):+ """The per-bin table ECE sums over, so the single number can be read rather than trusted."""+ edges = np.linspace(0, 1, bins + 1)+ table = []+ for i, (lo, hi) in enumerate(zip(edges[:-1], edges[1:])):+ sel = (conf >= lo if i == 0 else conf > lo) & (conf <= hi)+ if sel.any():+ table.append({"bin": "%.2f-%.2f" % (lo, hi), "n": int(sel.sum()),+ "mean_confidence": round(float(conf[sel].mean()), 4),+ "accuracy": round(float(correct[sel].mean()), 4),+ "gap": round(float(conf[sel].mean() - correct[sel].mean()), 4)})+ return table+++def measure(rows):+ probs = [p for p, _, _ in rows]+ gold = np.array([g for _, g, _ in rows])+ pred = np.array([int(np.argmax(p)) for p in probs])+ correct = (pred == gold).astype(float)++ top = np.array([answer_confidence(p, len(p)) for p in probs])+ ent = np.array([confidence_from_probs(p, len(p)) for p in probs])++ def block(conf):+ return {"mean_confidence": round(float(conf.mean()), 4),+ "ece": round(ece_score(conf, correct), 4),+ "auroc": round(auroc(conf, correct), 4),+ "acc_at_50_coverage": round(acc_at_coverage(conf, correct, 0.5), 4),+ "gate_at_0.85": gate(conf, correct),+ "reliability": reliability(conf, correct)}++ return {"n": len(rows), "accuracy": round(float(correct.mean()), 4),+ "answer_confidence": block(top), "entropy_confidence": block(ent)}+++def main():+ ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ ap.add_argument("--write", action="store_true", help="also write %s" % os.path.relpath(OUT, REPO))+ ap.add_argument("--models", nargs="+", default=["laya", "jev"])+ args = ap.parse_args()++ report = {"source": os.path.relpath(RESULTS, REPO), "bins": BINS, "models": {}}+ problems = []++ for model in args.models:+ rows = load(model)+ ch = choice_rows(rows)+ if not ch:+ problems.append("%s: no answered choice rows" % model)+ continue+ report["models"][model] = m = measure(ch)++ if not is_laya(rows):+ # Another engine's `confidence` is its own business; only the comparison table above+ # is meaningful for it.+ m["invariants_checked"] = False+ continue+ m["invariants_checked"] = True++ # Invariant 1: the shipped choice confidence is the entropy definition.+ drift = max((abs(c - confidence_from_probs(p, len(p))) for p, _, c in ch if c is not None),+ default=None)+ m["shipped_choice_confidence_is_entropy"] = drift is not None and drift <= ROUNDING_SLACK+ m["choice_confidence_max_drift"] = None if drift is None else round(drift, 6)+ if drift is None or drift > ROUNDING_SLACK:+ problems.append("%s: shipped choice confidence is not confidence_from_probs "+ "(max drift %s)" % (model, drift))++ # Invariant 2: the shipped noul confidence is max(p), so `answer_confidence` adds nothing+ # there. That is the whole reason this change can be additive.+ nouls = noul_answers(rows)+ noul_drift = max((abs(c - answer_confidence(np.array([1.0 - t, t]), 2)) for t, c in nouls),+ default=None)+ m["noul_answers"] = len(nouls)+ m["shipped_noul_confidence_is_max_p"] = noul_drift is not None and noul_drift <= ROUNDING_SLACK+ m["noul_confidence_max_drift"] = None if noul_drift is None else round(noul_drift, 6)+ if noul_drift is None or noul_drift > ROUNDING_SLACK:+ problems.append("%s: shipped noul confidence is not max(p) (max drift %s)"+ % (model, noul_drift))++ print("Source : %s" % report["source"])+ print("Bins : %d\n" % BINS)+ for model, m in report["models"].items():+ print("%s n=%d accuracy=%.4f" % (model, m["n"], m["accuracy"]))+ print(" %-28s %8s %8s %8s %8s" % ("confidence definition", "mean", "ECE", "AUROC", "acc@50%"))+ for key, label in (("answer_confidence", "max(p)"),+ ("entropy_confidence", "1 - H/log(k)")):+ b = m[key]+ print(" %-28s %8.4f %8.4f %8.4f %8.4f"+ % (label, b["mean_confidence"], b["ece"], b["auroc"], b["acc_at_50_coverage"]))+ print(" gate at 0.85 (the README's example):")+ for key, label in (("answer_confidence", "max(p)"),+ ("entropy_confidence", "1 - H/log(k)")):+ g = m[key]["gate_at_0.85"]+ acc = "n/a" if g["accuracy_of_automated"] is None else "%.4f" % g["accuracy_of_automated"]+ print(" %-28s automated %3d/%d (%.1f%%), of which correct: %s"+ % (label, g["automated"], m["n"], 100 * g["automated_share"], acc))+ print()++ for model, m in report["models"].items():+ print("\n%s: reliability bins, confidence = max(p)" % model)+ for b in m["answer_confidence"]["reliability"]:+ print(" %-12s n=%-4d confidence=%.4f accuracy=%.4f gap=%+.4f"+ % (b["bin"], b["n"], b["mean_confidence"], b["accuracy"], b["gap"]))+ print("%s: reliability bins, confidence = 1 - H/log(k)" % model)+ for b in m["entropy_confidence"]["reliability"]:+ print(" %-12s n=%-4d confidence=%.4f accuracy=%.4f gap=%+.4f"+ % (b["bin"], b["n"], b["mean_confidence"], b["accuracy"], b["gap"]))++ print("Invariants (Laya answers only; another engine's `confidence` is its own definition):")+ for model, m in report["models"].items():+ if not m.get("invariants_checked"):+ print(" %-6s skipped, not a Laya response shape" % model)+ continue+ print(" %-6s shipped choice confidence is 1 - H/log(k) : %-5s (max drift %.2g over %d)"+ % (model, m["shipped_choice_confidence_is_entropy"],+ m["choice_confidence_max_drift"], m["n"]))+ print(" %-6s shipped noul confidence is max(p) : %-5s (max drift %.2g over %d)"+ % (model, m["shipped_noul_confidence_is_max_p"],+ m["noul_confidence_max_drift"], m["noul_answers"]))+ print(" %-6s -> `answer_confidence` changes nothing on a noul answer." % model)++ if args.write:+ with open(OUT, "w", encoding="utf-8") as f:+ json.dump(report, f, ensure_ascii=False, indent=2, sort_keys=True)+ f.write("\n")+ print("\nwrote %s" % os.path.relpath(OUT, REPO))++ if problems:+ print("\nFAILED:")+ for p in problems:+ print(" " + p)+ return 1+ return 0+++if __name__ == "__main__":+ sys.exit(main())