Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

research(step4): measure order stability on massive_intent.en at 20 options

Bruce-Yiia46ed10573a7
diff --git a/research/scripts/step4_benchmark/order_flip.py b/research/scripts/step4_benchmark/order_flip.py
new file mode 100644
index 00000000..c417cf22
--- /dev/null
+++ b/research/scripts/step4_benchmark/order_flip.py
@@ -0,0 +1,113 @@
+"""Measure the 20-option order-flip rate on `massive_intent.en` for a checkpoint.
+
+This is the *other* evidence surface of Step 4, and it is measurement-only on purpose. The
+original issue names `massive_intent.en` at 20 options as the order-stability surface; the
+typed-decisions benchmark is 4-option `choice` and must not be reused for a 20-option claim.
+
+GuilhermeFusari already posted seed-0/1 numbers for this surface, produced by `laya.train` at
+`8a6e132` (soft-CE, 4 epochs, effective batch 64, `max_len` 512 / `head_max_len` 256, shuffle vs
+no shuffle). The option-order code path has not changed semantically since then (only the
+`parallel` layout argument was threaded through, and `shuffle_options` still defaults to off), so
+Step 4 **references those numbers instead of starting a second training campaign**. This script
+exists so the referenced number can be reproduced inference-only on the exact checkpoint a
+reviewer hands over, without retraining.
+
+Construction matches `research/scripts/build_benchmark_nb.py`: each question is the gold intent
+plus 19 distractors drawn from the label space, in random order (seed 13). A case is answered
+with its options in the original order and in `--permutations` random permutations, in one call
+each; the flip rate is the share of permutations whose answer differs from the original order.
+
+Usage:
+ python order_flip.py --checkpoint <dir-or-hub-id> --out flip.json
+ python order_flip.py --checkpoint convaiinnovations/laya --per-lang 200 --device cuda
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import random
+import sys
+from typing import Any, Dict, List, Tuple
+
+
+def build_cases(n: int, n_opts: int, seed: int) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+ from datasets import load_dataset
+
+ d = load_dataset("mteb/amazon_massive_intent", "en", split="test")
+ labels = sorted(set(d["label_text"]))
+ rng = random.Random(seed)
+ cases, golds = [], []
+ for r in list(d)[:n]:
+ gold = r["label_text"]
+ pool = [x for x in labels if x != gold]
+ keys = [gold] + rng.sample(pool, min(n_opts - 1, len(pool)))
+ rng.shuffle(keys)
+ crit = {k: k.replace("_", " ").replace(".", ": ") for k in keys}
+ cases.append({"utterance": r["text"]})
+ golds.append({"label": {"type": "choice",
+ "instructions": "What is the user asking for in `utterance`?",
+ "criteria": crit}, "gold_label": gold})
+ return cases, golds
+
+
+def permute(qdef: Dict[str, Any], rng: random.Random) -> Dict[str, Any]:
+ keys = list(qdef["criteria"].keys())
+ order = list(range(len(keys)))
+ rng.shuffle(order)
+ return {**qdef, "criteria": {keys[i]: qdef["criteria"][keys[i]] for i in order}}
+
+
+def measure(checkpoint: str, n: int, n_opts: int, permutations: int, seed: int,
+ device: str) -> Dict[str, Any]:
+ import laya
+
+ agent = laya.load(checkpoint, device=device)
+ cases, golds = build_cases(n, n_opts, seed)
+ rng = random.Random(seed + 1)
+ flips = compared = 0
+ for case, gold in zip(cases, golds):
+ base_q = gold["label"]
+ base_answer = agent.predict(case, {"label": base_q})["answers"]["label"]["choice"]
+ for _ in range(permutations):
+ perm_q = permute(base_q, rng)
+ perm_answer = agent.predict(case, {"label": perm_q})["answers"]["label"]["choice"]
+ compared += 1
+ if perm_answer != base_answer:
+ flips += 1
+ return {
+ "checkpoint": checkpoint,
+ "suite": "massive_intent.en",
+ "n_options": n_opts,
+ "n_cases": len(cases),
+ "permutations_per_case": permutations,
+ "n_comparisons": compared,
+ "flip_count": flips,
+ "flip_rate": round(flips / max(1, compared), 4),
+ }
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--checkpoint", required=True)
+ parser.add_argument("--per-lang", type=int, default=300, help="cases to sample")
+ parser.add_argument("--n-options", type=int, default=20)
+ parser.add_argument("--permutations", type=int, default=3)
+ parser.add_argument("--seed", type=int, default=13)
+ parser.add_argument("--device", default="auto")
+ parser.add_argument("--out", default=None)
+ args = parser.parse_args()
+
+ result = measure(args.checkpoint, args.per_lang, args.n_options, args.permutations,
+ args.seed, args.device)
+ print(json.dumps(result, indent=2))
+ if args.out:
+ os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
+ with open(args.out, "w", encoding="utf-8") as f:
+ json.dump(result, f, indent=2)
+ print(f"wrote {args.out}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())