Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(calibration): clear inherited buckets from per-type exports (#139)

Alphin Tombdce407b75af
diff --git a/README.md b/README.md
index 4d2ca9eb..e1aed22a 100644
--- a/README.md
+++ b/README.md
@@ -474,6 +474,18 @@ policy gradient), fit calibration temperatures, evaluate, and push the result to
* **[`notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb`](notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb)**
+The notebook fits one `temperature` per type (`choice`, `score`, `noul`) and removes inherited
+`temperature_by_options` from the exported config. Otherwise those old bucket values take
+precedence at inference and silently mask the new fit. Existing checkpoints still honor
+intentional bucket-specific temperatures, falling back to the corresponding per-type value
+when a bucket is absent; the runtime's temperature clamp is unchanged.
+
+This fixes configuration persistence, not measured model accuracy or calibration quality.
+The notebook's calibration samples come from its training items; evaluate on separate held-out
+data before claiming an improvement. Already published checkpoints are not rewritten.
+Run the CPU-only regression checks with `python tests/test_calibration_persistence.py`
+(synthetic configs and tiny local fixtures; no pretrained downloads or training).
+
Fine-tuning is where most of the value is. On the typed-decisions benchmark the base
checkpoints score near chance zero-shot (0.36 and 0.35 against a 0.318 random baseline),
while the fine-tuned checkpoint reaches **0.766** on the same 2,000 decisions -- above
diff --git a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb
index c9c51fe8..33ee17d8 100644
--- a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb
+++ b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb
@@ -422,6 +422,8 @@
" cfg[\"fine_tuned\"] = True\n",
" cfg[\"model_name\"] = \"laya-typed-decisions\"\n",
" cfg[\"temperature\"] = fitted_temps\n",
+ " # This fit is per type; inherited bucket overrides would hide the new values.\n",
+ " cfg.pop(\"temperature_by_options\", None)\n",
" with open(os.path.join(output_dir, \"rl_agent_config.json\"), \"w\") as f:\n",
" json.dump(cfg, f, indent=2)\n",
" print(f\"Model successfully saved to {output_dir}!\")\n",
diff --git a/tests/test_calibration_persistence.py b/tests/test_calibration_persistence.py
new file mode 100644
index 00000000..0b6292ac
--- /dev/null
+++ b/tests/test_calibration_persistence.py
@@ -0,0 +1,170 @@
+"""Notebook export and inference regression tests; CPU only, no training or downloads.
+
+Run: python tests/test_calibration_persistence.py
+"""
+import ast
+import copy
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+import textwrap
+import unittest
+from unittest.mock import patch
+
+os.environ.setdefault("USE_TF", "0")
+os.environ.setdefault("USE_TORCH", "1")
+os.environ.setdefault("HF_HUB_OFFLINE", "1")
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import torch # noqa: E402
+from safetensors.torch import save_file # noqa: E402
+from tokenizers import Tokenizer # noqa: E402
+from tokenizers.models import WordLevel # noqa: E402
+from transformers import BertConfig, BertModel, PreTrainedTokenizerFast # noqa: E402
+
+from laya import load # noqa: E402
+from laya.common import DecisionModel, QTYPES # noqa: E402
+
+
+def export_notebook_config(cfg, fitted_temps, output_dir):
+ """Execute the notebook's actual config export, without its GPU/training code."""
+ notebook = Path(__file__).resolve().parents[1] / "notebooks" / (
+ "laya_finetune_typed_decisions_2xT4_kaggle.ipynb"
+ )
+ cells = json.loads(notebook.read_text(encoding="utf-8"))["cells"]
+ script, = ["".join(c["source"]) for c in cells
+ if "".join(c["source"]).startswith("%%writefile ")]
+ # This contiguous tail includes the temperature update AND the JSON write.
+ # Do not reproduce the export logic here: that would miss notebook regressions.
+ start = script.index(' cfg["fine_tuned"] = True')
+ end = script.index("\n dist.destroy_process_group()", start)
+ export = ast.parse(textwrap.dedent(script[start:end]))
+ exec(compile(export, str(notebook), "exec"), {
+ "cfg": cfg, "fitted_temps": fitted_temps, "output_dir": str(output_dir),
+ "os": os, "json": json,
+ })
+ return json.loads((output_dir / "rl_agent_config.json").read_text())
+
+
+class CalibrationPersistenceTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tmp = tempfile.TemporaryDirectory()
+ cls.addClassCleanup(cls.tmp.cleanup)
+ cls.checkpoint = Path(cls.tmp.name)
+ config = BertConfig(vocab_size=6, hidden_size=16, num_hidden_layers=1,
+ num_attention_heads=1, intermediate_size=32)
+ config.save_pretrained(cls.checkpoint / "encoder")
+ tokenizer = PreTrainedTokenizerFast(
+ tokenizer_object=Tokenizer(WordLevel(
+ {"[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4, "hello": 5},
+ unk_token="[UNK]")),
+ pad_token="[PAD]", unk_token="[UNK]", cls_token="[CLS]",
+ sep_token="[SEP]", mask_token="[MASK]",
+ )
+ tokenizer.save_pretrained(cls.checkpoint / "tokenizer")
+ model = DecisionModel(BertModel(config), head_layers=0)
+ save_file(model.state_dict(), cls.checkpoint / "model.safetensors")
+
+ def setUp(self):
+ self.cfg = {
+ "encoder": "unused/offline", "head_layers": 0, "act_costs": {"act": 0},
+ "max_len": 128, "head_max_len": 96,
+ "temperature": [1.0, 1.0, 1.0], "training": {"updates": 7},
+ }
+ self.fitted = [2.0, 3.0, 4.0]
+
+ def assert_inference_temperatures(self, expected):
+ # Use actual local config/tokenizer/weight loading; replace only logits so
+ # the expected scaling is deterministic and independent of random weights.
+ with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):
+ agent = load(str(self.checkpoint), device="cpu")
+ self.assertEqual(agent.device.type, "cpu")
+ questions = {}
+ for qtype, k in expected:
+ q = {"type": qtype, "instructions": "Pick one"}
+ if qtype != "noul":
+ q["criteria"] = [str(i) for i in range(k)]
+ questions[f"{qtype}_{k}"] = q
+ kmax = max(k for _, k in expected)
+ logits = torch.arange(kmax, dtype=torch.float32).repeat(len(questions), 1)
+ with patch.object(agent.model, "forward", return_value=(logits, torch.zeros(len(questions), 2))):
+ answers = agent.predict("hello", questions)["answers"]
+ for (qtype, k), temperature in expected.items():
+ with self.subTest(qtype=qtype, k=k, temperature=temperature):
+ p = torch.softmax(torch.arange(k, dtype=torch.float32) / temperature, -1)
+ answer = answers[f"{qtype}_{k}"]
+ if qtype == "noul":
+ self.assertAlmostEqual(answer["noul"], p[1].item(), delta=0.0001)
+ else:
+ for i in range(k):
+ self.assertAlmostEqual(answer["probabilities"][str(i)], p[i].item(), delta=0.0001)
+ if qtype == "score":
+ self.assertAlmostEqual(answer["score"], (torch.arange(k) * p).sum().item(), delta=0.0001)
+
+ def write_config(self):
+ (self.checkpoint / "rl_agent_config.json").write_text(json.dumps(self.cfg))
+
+ def test_new_export_removes_inherited_buckets_and_uses_fitted_types(self):
+ self.cfg["temperature_by_options"] = {
+ f"{qtype}:{bucket}": 1.5
+ for qtype in QTYPES for bucket in ("2", "3-5", "6-10", "11+")
+ }
+ original = copy.deepcopy(self.cfg)
+ saved = export_notebook_config(self.cfg, self.fitted, self.checkpoint)
+ self.assert_inference_temperatures({
+ (qtype, k): self.fitted[qt]
+ for qtype, qt in QTYPES.items()
+ for k in ((2,) if qtype == "noul" else (1, 2, 3, 5, 6, 10, 11))
+ })
+ self.assertFalse(saved.get("temperature_by_options"))
+ self.assertEqual(saved["temperature"], self.fitted)
+ self.assertTrue(saved["fine_tuned"])
+ self.assertEqual(saved["model_name"], "laya-typed-decisions")
+ for key in original.keys() - {"temperature", "temperature_by_options"}:
+ self.assertEqual(saved[key], original[key])
+
+ def test_export_accepts_absent_or_empty_bucket_map(self):
+ for buckets in (None, {}):
+ with self.subTest(buckets=buckets):
+ if buckets is not None:
+ self.cfg["temperature_by_options"] = buckets
+ saved = export_notebook_config(self.cfg, self.fitted, self.checkpoint)
+ self.assertFalse(saved.get("temperature_by_options"))
+ self.assertEqual(saved["temperature"], self.fitted)
+ self.assert_inference_temperatures({(t, 2): self.fitted[qt] for t, qt in QTYPES.items()})
+
+ def test_existing_bucket_calibration_keeps_precedence(self):
+ self.cfg["temperature"] = self.fitted
+ buckets = {"2": 0.75, "3-5": 1.25, "6-10": 1.5, "11+": 1.75}
+ self.cfg["temperature_by_options"] = {
+ f"{qtype}:{bucket}": value for qtype in QTYPES for bucket, value in buckets.items()
+ }
+ self.write_config()
+ original = (self.checkpoint / "rl_agent_config.json").read_bytes()
+ self.assert_inference_temperatures({
+ (qtype, k): value for qtype in QTYPES
+ for k, value in (((2, 0.75),) if qtype == "noul" else
+ ((1, 0.75), (2, 0.75), (3, 1.25), (5, 1.25),
+ (6, 1.5), (10, 1.5), (11, 1.75)))
+ })
+ self.assertEqual((self.checkpoint / "rl_agent_config.json").read_bytes(), original)
+
+ def test_missing_bucket_falls_back_to_corresponding_type(self):
+ self.cfg["temperature"] = self.fitted
+ self.cfg["temperature_by_options"] = {"choice:2": 0.75}
+ self.write_config()
+ self.assert_inference_temperatures({("choice", 2): 0.75, ("choice", 3): 2.0,
+ ("score", 2): 3.0, ("noul", 2): 4.0})
+
+ def test_missing_temperatures_keep_unit_defaults(self):
+ self.cfg.pop("temperature")
+ self.write_config()
+ self.assert_inference_temperatures({(t, 2): 1.0 for t in QTYPES})
+
+
+if __name__ == "__main__":
+ torch.set_num_threads(1)
+ unittest.main()