Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
research(step4): Gate D panel analyser with a stated verdict rule
Bruce-Yiiab281a479772
diff --git a/research/scripts/step4_benchmark/README.md b/research/scripts/step4_benchmark/README.mdindex ebcc2f92..27fc8000 100644--- a/research/scripts/step4_benchmark/README.md+++ b/research/scripts/step4_benchmark/README.md@@ -21,7 +21,9 @@ matrix below, not by a single seed.| `evaluate.py` | Accuracy / ECE / Brier on the test split, under shipped and raw temperatures || `metrics.py` | The notebook's metric definitions (shared by `evaluate.py` and the smoke test) || `order_flip.py` | Inference-only 20-option flip rate on `massive_intent.en` |+| `analyze_gate_d.py` | Summarise `step4_gate_d_results.json` into the 3-seed panel + verdict || `smoke_test.py` | CPU, weight-free: metric plumbing + CLI wiring + tiny end-to-end |+| `kaggle/` | Generators for the self-contained single-T4 Kaggle kernels (Gate R, Gate D) |## Exact benchmark contractdiff --git a/research/scripts/step4_benchmark/analyze_gate_d.py b/research/scripts/step4_benchmark/analyze_gate_d.pynew file mode 100644index 00000000..eeb2b1ce--- /dev/null+++ b/research/scripts/step4_benchmark/analyze_gate_d.py@@ -0,0 +1,174 @@+"""Summarise `step4_gate_d_results.json` into the canonical 3-seed x 2-loss panel and a verdict.++Reads the Gate D results file (as emitted by the Kaggle Gate D kernel) and prints/writes:++- the per-cell table (accuracy / ECE / Brier / per-type accuracy / wall clock);+- `soft-ce - rlcd` paired accuracy deltas at seeds 0/1/2, plus mean/min/max and sign consistency;+- mean and sample std of accuracy per loss, and mean/sample std of the paired deltas;+- the same summaries for ECE and Brier, explicitly labelled as calibration-scope numbers;+- one evidence verdict from the fixed rule below.++The verdict thresholds are stated in the output, not hidden. No p-values: n=3 training seeds is+not enough for significance, and the Coordinator's rule asks for direction, magnitude and sign+consistency instead.++ python research/scripts/step4_benchmark/analyze_gate_d.py --results step4_gate_d_results.json+"""+from __future__ import annotations++import argparse+import json+import statistics+import sys+from typing import Any, Dict, List, Optional++SEEDS = (0, 1, 2)+LOSSES = ("rlcd", "soft-ce")+# A soft-CE edge only counts as "useful" if it clears this in the mean and on every seed. Stated+# here so the verdict is reproducible rather than eyeballed.+MATERIAL_ACCURACY_DELTA = 0.005+# A per-type or calibration regression this large or larger counts as "material" for the verdict.+MATERIAL_REGRESSION = 0.010+++def cell(results: Dict[str, Any], loss: str, seed: int) -> Optional[Dict[str, Any]]:+ entry = results.get("%s_seed%d" % (loss, seed))+ if not isinstance(entry, dict) or "accuracy" not in entry:+ return None+ return entry+++def paired(results: Dict[str, Any], metric: str) -> Dict[int, Optional[float]]:+ out: Dict[int, Optional[float]] = {}+ for seed in SEEDS:+ a, b = cell(results, "soft-ce", seed), cell(results, "rlcd", seed)+ if a is None or b is None:+ out[seed] = None+ else:+ out[seed] = round(float(a[metric]) - float(b[metric]), 4)+ return out+++def summarise(values: List[float]) -> Dict[str, Any]:+ if not values:+ return {"n": 0}+ return {+ "n": len(values),+ "mean": round(statistics.fmean(values), 4),+ "min": round(min(values), 4),+ "max": round(max(values), 4),+ "std": round(statistics.stdev(values), 4) if len(values) > 1 else None,+ }+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--results", default="step4_gate_d_results.json")+ parser.add_argument("--out", default=None)+ args = parser.parse_args()++ with open(args.results, encoding="utf-8") as f:+ results = json.load(f)++ print("== per-cell ==")+ header = "%-8s %-4s %8s %8s %8s %8s %10s" % ("loss", "seed", "acc", "ece", "brier", "wall_s", "finite")+ print(header)+ for loss in LOSSES:+ for seed in SEEDS:+ c = cell(results, loss, seed)+ if c is None:+ print("%-8s %-4d %8s" % (loss, seed, "MISSING"))+ continue+ print("%-8s %-4d %8.4f %8.4f %8.4f %10s %10s"+ % (loss, seed, c["accuracy"], c["ece"], c["brier"],+ c.get("wall_clock_s", "-"), c.get("finite", "-")))++ acc_deltas = paired(results, "accuracy")+ ece_deltas = paired(results, "ece")+ brier_deltas = paired(results, "brier")+ acc_values = {loss: [cell(results, loss, s)["accuracy"] for s in SEEDS if cell(results, loss, s)]+ for loss in LOSSES}+ complete = all(acc_deltas[s] is not None for s in SEEDS)+ present = [acc_deltas[s] for s in SEEDS if acc_deltas[s] is not None]++ print("\n== paired soft-ce - rlcd ==")+ print("accuracy delta by seed:", acc_deltas)+ print("accuracy delta summary:", summarise(present))+ print("sign consistency (all 3 > 0):", complete and all(d > 0 for d in present))+ print("sign consistency (all 3 < 0):", complete and all(d < 0 for d in present))+ print("rlcd accuracy:", summarise(acc_values["rlcd"]))+ print("soft-ce accuracy:", summarise(acc_values["soft-ce"]))+ ece_present = [ece_deltas[s] for s in SEEDS if ece_deltas[s] is not None]+ brier_present = [brier_deltas[s] for s in SEEDS if brier_deltas[s] is not None]+ print("ECE delta by seed (calibration scope only):", ece_deltas, summarise(ece_present))+ print("Brier delta by seed (calibration scope only):", brier_deltas, summarise(brier_present))++ per_type = {}+ for loss in LOSSES:+ for seed in SEEDS:+ c = cell(results, loss, seed)+ if c and c.get("per_type_accuracy"):+ for k, v in c["per_type_accuracy"].items():+ per_type.setdefault(k, {}).setdefault(loss, []).append(float(v))+ print("\n== per-type accuracy (mean over seeds) ==")+ for k, by_loss in sorted(per_type.items()):+ print(" %-8s rlcd %s soft-ce %s"+ % (k, summarise(by_loss.get("rlcd", [])), summarise(by_loss.get("soft-ce", []))))++ verdict = "INSUFFICIENT"+ reason = "panel incomplete (missing a loss/seed cell)"+ if complete:+ consistent_up = all(d > 0 for d in present)+ mean_delta = statistics.fmean(present)+ worst_type_regress = 0.0+ for k, by_loss in per_type.items():+ if by_loss.get("rlcd") and by_loss.get("soft-ce"):+ worst_type_regress = min(worst_type_regress,+ statistics.fmean(by_loss["soft-ce"]) - statistics.fmean(by_loss["rlcd"]))+ if consistent_up and mean_delta >= MATERIAL_ACCURACY_DELTA and worst_type_regress > -MATERIAL_REGRESSION:+ verdict = "CHANGE_DEFAULT_SUPPORTED"+ reason = ("soft-ce ahead on all 3 seeds, mean delta %.4f >= %.3f, no per-type regression beyond %.3f"+ % (mean_delta, MATERIAL_ACCURACY_DELTA, MATERIAL_REGRESSION))+ elif consistent_up and mean_delta >= MATERIAL_ACCURACY_DELTA:+ verdict = "INSUFFICIENT"+ reason = ("accuracy favours soft-ce but a per-type regression of %.4f exceeds %.3f"+ % (worst_type_regress, MATERIAL_REGRESSION))+ elif all(d < 0 for d in present):+ verdict = "KEEP_RLCD_SUPPORTED"+ reason = "soft-ce behind on all 3 seeds"+ else:+ verdict = "INSUFFICIENT"+ reason = "sign not consistent or mean delta %.4f below the %.3f material bar" % (mean_delta, MATERIAL_ACCURACY_DELTA)++ summary = {+ "accuracy_delta_by_seed": acc_deltas,+ "accuracy_delta_summary": summarise(present),+ "sign_all_positive": complete and all(d > 0 for d in present),+ "rlcd_accuracy": summarise(acc_values["rlcd"]),+ "soft_ce_accuracy": summarise(acc_values["soft-ce"]),+ "ece_delta_by_seed": ece_deltas,+ "brier_delta_by_seed": brier_deltas,+ "per_type_mean": {k: {l: summarise(v) for l, v in by.items()} for k, by in per_type.items()},+ "verdict": verdict,+ "reason": reason,+ "thresholds": {"material_accuracy_delta": MATERIAL_ACCURACY_DELTA,+ "material_regression": MATERIAL_REGRESSION},+ "caveats": [+ "n=3 training seeds: direction/magnitude, not significance.",+ "ECE/Brier reflect the current evaluator's calibration path (notebook-aligned scope), "+ "not a universal all-question metric.",+ ],+ }+ print("\n== verdict ==")+ print(verdict, "--", reason)+ print("thresholds:", summary["thresholds"])++ if args.out:+ with open(args.out, "w", encoding="utf-8") as f:+ json.dump(summary, f, indent=2)+ print("wrote", args.out)+ return 0+++if __name__ == "__main__":+ sys.exit(main())