Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
"""Notebook export and inference regression tests; CPU only, no training or downloads.
Run: python tests/test_calibration_persistence.py
"""
import ast
import copy
import json
import os
from pathlib import Path
import re
import sys
import tempfile
import textwrap
import unittest
from unittest.mock import patch
os.environ.setdefault("USE_TF", "0")
os.environ.setdefault("USE_TORCH", "1")
os.environ.setdefault("HF_HUB_OFFLINE", "1")
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import torch # noqa: E402
from safetensors.torch import save_file # noqa: E402
from tokenizers import Tokenizer # noqa: E402
from tokenizers.models import WordLevel # noqa: E402
from transformers import BertConfig, BertModel, PreTrainedTokenizerFast # noqa: E402
from laya import load # noqa: E402
from laya.calibrate import fit_temperature_map, records_from_labeled # noqa: E402
from laya.common import DecisionModel, QTYPES, TEMP_MIN, TEMP_MAX # noqa: E402
def _config_only_save_checkpoint(model, tok, cfg, path):
"""`save_checkpoint`'s config write, without the weights it also owns.
This suite owns the shape of `rl_agent_config.json`, not `model.safetensors`. Binding the
notebook's own call to `save_checkpoint` is what still fails when it stops persisting the
config; reproducing how the dict is built here is what would miss a notebook regression.
"""
os.makedirs(path, exist_ok=True)
with open(os.path.join(path, "rl_agent_config.json"), "w", encoding="utf-8") as fh:
json.dump(cfg, fh, indent=2)
def export_notebook_config(cfg, fitted_temps, output_dir):
"""Execute the notebook's actual config export, without its GPU/training code."""
notebook = Path(__file__).resolve().parents[1] / "notebooks" / (
"laya_finetune_typed_decisions_2xT4_kaggle.ipynb"
)
cells = json.loads(notebook.read_text(encoding="utf-8"))["cells"]
script, = ["".join(c["source"]) for c in cells
if "".join(c["source"]).startswith("%%writefile ")]
# This contiguous tail includes the temperature update AND the config write.
# Do not reproduce the export logic here: that would miss notebook regressions.
start = script.index(' out_cfg = dict(cfg,')
end = script.index("\n dist.destroy_process_group()", start)
export = ast.parse(textwrap.dedent(script[start:end]))
exec(compile(export, str(notebook), "exec"), {
"cfg": cfg, "fitted_temps": fitted_temps,
# `fit_temperature_map` returns this shape; the caller passes only the scalar list.
"fitted": {"temperature": list(fitted_temps), "temperature_by_options": {}},
"max_len": cfg.get("max_len"), "head_max_len": cfg.get("head_max_len"),
"output_dir": str(output_dir), "os": os, "json": json,
"save_checkpoint": _config_only_save_checkpoint,
"model": None, "tok": None,
})
return json.loads((output_dir / "rl_agent_config.json").read_text())
ROOT = Path(__file__).resolve().parents[1]
def _runnable_source(rel):
"""What a fine-tuning entry point actually executes: its %%writefile cells for a
notebook, the file itself for a script."""
if rel.endswith(".ipynb"):
cells = json.loads((ROOT / rel).read_text(encoding="utf-8"))["cells"]
return "".join("".join(c["source"]) for c in cells
if "".join(c["source"]).startswith("%%writefile "))
return (ROOT / rel).read_text(encoding="utf-8")
def _temperature_fitters():
"""Every notebook or training script that still owns a fit clamping `exp()` of a
log-temperature.
Walks the tree rather than listing the known entry points, so a new script that copies
the pattern is held by the same check. Matches on the clamp of an `.exp()` because that
is the one shape these share; a fit that stops clamping is reported by the non-empty
assertion in the test rather than passing unnoticed.
Entry points that call `laya.train.finetune` are absent by design: their fit is
`laya.calibrate.fit_one_temperature`, which ends in `clamp_temperature(fitted, TEMP_MIN,
TEMP_MAX)` -- the same bounds, held by tests/test_calibrate.py. Their delegation is held
by tests/test_finetune_entrypoints.py.
"""
found = []
for base in ("notebooks", "research/scripts"):
for path in sorted((ROOT / base).rglob("*")):
if path.suffix not in (".py", ".ipynb") or not path.is_file():
continue
rel = path.relative_to(ROOT).as_posix()
text = _runnable_source(rel)
for match in re.finditer(r"^def (\w+)\(.*?(?=^\S|\Z)", text, re.M | re.S):
if re.search(r"torch\.clamp\(\s*\w+\.exp\(\)", match.group(0)):
found.append((rel, match.group(1), match.group(0)))
return found
class CalibrationPersistenceTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmp = tempfile.TemporaryDirectory()
cls.addClassCleanup(cls.tmp.cleanup)
cls.checkpoint = Path(cls.tmp.name)
config = BertConfig(vocab_size=6, hidden_size=16, num_hidden_layers=1,
num_attention_heads=1, intermediate_size=32)
config.save_pretrained(cls.checkpoint / "encoder")
tokenizer = PreTrainedTokenizerFast(
tokenizer_object=Tokenizer(WordLevel(
{"[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4, "hello": 5},
unk_token="[UNK]")),
pad_token="[PAD]", unk_token="[UNK]", cls_token="[CLS]",
sep_token="[SEP]", mask_token="[MASK]",
)
tokenizer.save_pretrained(cls.checkpoint / "tokenizer")
model = DecisionModel(BertModel(config), head_layers=0)
save_file(model.state_dict(), cls.checkpoint / "model.safetensors")
def setUp(self):
self.cfg = {
"encoder": "unused/offline", "head_layers": 0, "act_costs": {"act": 0},
"max_len": 128, "head_max_len": 96,
"temperature": [1.0, 1.0, 1.0], "training": {"updates": 7},
}
self.fitted = [2.0, 3.0, 4.0]
def assert_inference_temperatures(self, expected):
# Use actual local config/tokenizer/weight loading; replace only logits so
# the expected scaling is deterministic and independent of random weights.
with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):
agent = load(str(self.checkpoint), device="cpu")
self.assertEqual(agent.device.type, "cpu")
questions = {}
for qtype, k in expected:
q = {"type": qtype, "instructions": "Pick one"}
if qtype != "noul":
q["criteria"] = [str(i) for i in range(k)]
questions[f"{qtype}_{k}"] = q
kmax = max(k for _, k in expected)
logits = torch.arange(kmax, dtype=torch.float32).repeat(len(questions), 1)
with patch.object(agent.model, "forward", return_value=(logits, torch.zeros(len(questions), 2))):
answers = agent.predict("hello", questions)["answers"]
for (qtype, k), temperature in expected.items():
with self.subTest(qtype=qtype, k=k, temperature=temperature):
p = torch.softmax(torch.arange(k, dtype=torch.float32) / temperature, -1)
answer = answers[f"{qtype}_{k}"]
if qtype == "noul":
self.assertAlmostEqual(answer["noul"], p[1].item(), delta=0.0001)
else:
for i in range(k):
self.assertAlmostEqual(answer["probabilities"][str(i)], p[i].item(), delta=0.0001)
if qtype == "score":
self.assertAlmostEqual(answer["score"], (torch.arange(k) * p).sum().item(), delta=0.0001)
def write_config(self):
(self.checkpoint / "rl_agent_config.json").write_text(json.dumps(self.cfg))
def test_new_export_removes_inherited_buckets_and_uses_fitted_types(self):
self.cfg["temperature_by_options"] = {
f"{qtype}:{bucket}": 1.5
for qtype in QTYPES for bucket in ("2", "3-5", "6-10", "11+")
}
original = copy.deepcopy(self.cfg)
saved = export_notebook_config(self.cfg, self.fitted, self.checkpoint)
self.assert_inference_temperatures({
(qtype, k): self.fitted[qt]
for qtype, qt in QTYPES.items()
for k in ((2,) if qtype == "noul" else (1, 2, 3, 5, 6, 10, 11))
})
self.assertFalse(saved.get("temperature_by_options"))
self.assertEqual(saved["temperature"], self.fitted)
self.assertTrue(saved["fine_tuned"])
self.assertEqual(saved["model_name"], "laya-typed-decisions")
for key in original.keys() - {"temperature", "temperature_by_options"}:
self.assertEqual(saved[key], original[key])
def test_export_accepts_absent_or_empty_bucket_map(self):
for buckets in (None, {}):
with self.subTest(buckets=buckets):
if buckets is not None:
self.cfg["temperature_by_options"] = buckets
saved = export_notebook_config(self.cfg, self.fitted, self.checkpoint)
self.assertFalse(saved.get("temperature_by_options"))
self.assertEqual(saved["temperature"], self.fitted)
self.assert_inference_temperatures({(t, 2): self.fitted[qt] for t, qt in QTYPES.items()})
def test_existing_bucket_calibration_keeps_precedence(self):
self.cfg["temperature"] = self.fitted
buckets = {"2": 0.75, "3-5": 1.25, "6-10": 1.5, "11+": 1.75}
self.cfg["temperature_by_options"] = {
f"{qtype}:{bucket}": value for qtype in QTYPES for bucket, value in buckets.items()
}
self.write_config()
original = (self.checkpoint / "rl_agent_config.json").read_bytes()
self.assert_inference_temperatures({
(qtype, k): value for qtype in QTYPES
for k, value in (((2, 0.75),) if qtype == "noul" else
((1, 0.75), (2, 0.75), (3, 1.25), (5, 1.25),
(6, 1.5), (10, 1.5), (11, 1.75)))
})
self.assertEqual((self.checkpoint / "rl_agent_config.json").read_bytes(), original)
def test_missing_bucket_falls_back_to_corresponding_type(self):
self.cfg["temperature"] = self.fitted
self.cfg["temperature_by_options"] = {"choice:2": 0.75}
self.write_config()
self.assert_inference_temperatures({("choice", 2): 0.75, ("choice", 3): 2.0,
("score", 2): 3.0, ("noul", 2): 4.0})
def test_missing_temperatures_keep_unit_defaults(self):
self.cfg.pop("temperature")
self.write_config()
self.assert_inference_temperatures({(t, 2): 1.0 for t in QTYPES})
def test_records_from_labeled_runs_on_a_loaded_agent(self):
# `Agent._forward` turns the logits straight into numpy, which fails on a tensor that
# tracks gradients. The stand-in agents in test_calibrate.py return numpy and cannot see it.
self.write_config()
with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):
agent = load(str(self.checkpoint), device="cpu")
questions = {"flag": {"type": "noul", "instructions": "hello"}}
records = records_from_labeled(agent, [("hello", questions, {"flag": [0.0, 1.0]})])
self.assertEqual(len(records), 1)
qtype, logits, target, k = records[0]
self.assertEqual((qtype, k), (QTYPES["noul"], 2))
self.assertEqual(logits.shape, (2,))
self.assertEqual(target.tolist(), [0.0, 1.0])
def test_every_fine_tuning_fit_clamps_to_common_bounds(self):
# #642 fixed this for the Kaggle notebook, which owned a fit of its own. The
# Apple Silicon script and research/scripts/finetune_single_device.py used to keep
# `torch.clamp(..., 0.1, 10.0)` beside it, so a fit either of them persisted could
# land outside `[TEMP_MIN, TEMP_MAX]` and be re-clamped when the checkpoint is loaded
# -- the calibration measured during training is then not the one that gets served.
# All three now reach `laya.calibrate`, so what the tree sweep below still walks is a
# fit that copies the old pattern. Derived from the tree rather than listed, so a
# script that copies it is held too.
fitters = _temperature_fitters()
# Non-vacuity, redirected exactly as the sweep's own message asks for once no entry
# point owns a hand-rolled fit: pin that the notebook still reaches the canonical
# fitter and owns no fit of its own, so taking one back is a failure to read rather
# than a silent pass.
notebook_src = _runnable_source(
"notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb")
self.assertIn("fit_temperature_map", notebook_src,
"the notebook no longer reaches laya.calibrate")
self.assertNotIn("fit_one_temp", notebook_src,
"the notebook owns a private fit again")
# These two drive the LBFGS solution past TEMP_MAX and below TEMP_MIN respectively,
# so a range wider than the runtime's is caught in both directions.
peaked = [([10.0, 0.0], [0.5, 0.5]) for _ in range(20)]
dipped = [([0.0, 4.0], [0.0, 1.0]) for _ in range(20)]
# With no entry point owning a hand-rolled fit, the tree sweep below can match
# nothing, so drive the fit every entry point now reaches with the same two inputs:
# the bound this test exists for is asserted either way.
for label, sel in (("peaked", peaked), ("dipped", dipped)):
t_fit = fit_temperature_map(
[(QTYPES["choice"], z, t, len(z)) for z, t in sel]
)["temperature"][QTYPES["choice"]]
self.assertGreaterEqual(
t_fit, TEMP_MIN, "fit_temperature_map(%s) fitted %.4g" % (label, t_fit))
self.assertLessEqual(
t_fit, TEMP_MAX, "fit_temperature_map(%s) fitted %.4g" % (label, t_fit))
for rel, name, body in fitters:
with self.subTest(source=rel, function=name):
scope = {"torch": torch, "TEMP_MIN": TEMP_MIN, "TEMP_MAX": TEMP_MAX}
exec(compile(ast.parse(textwrap.dedent(body)), rel, "exec"), scope)
fit = scope[name]
for label, sel in (("peaked", peaked), ("dipped", dipped)):
fitted = fit(sel)
self.assertGreaterEqual(
fitted, TEMP_MIN, "%s: %s(%s) fitted %.4g" % (rel, name, label, fitted))
self.assertLessEqual(
fitted, TEMP_MAX, "%s: %s(%s) fitted %.4g" % (rel, name, label, fitted))
if __name__ == "__main__":
torch.set_num_threads(1)
unittest.main()