Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
test(train): guard the Kaggle DDP notebook against returning private training copies
Bruce-Yii61c1455b4953
diff --git a/scripts/test_suites.py b/scripts/test_suites.pyindex 546c0764..9c845b34 100644--- a/scripts/test_suites.py+++ b/scripts/test_suites.py@@ -88,6 +88,7 @@ SCRIPT_SUITES = ["tests/test_portability.py","tests/test_training.py","tests/test_train.py",+ "tests/test_kaggle_ddp_convergence.py","tests/test_example_server_limits.py","tests/test_blank_lang_routing.py","tests/test_export_onnx_safety.py",diff --git a/tests/test_calibration_persistence.py b/tests/test_calibration_persistence.pyindex 311bb12e..833774e4 100644--- a/tests/test_calibration_persistence.py+++ b/tests/test_calibration_persistence.py@@ -26,10 +26,22 @@ from tokenizers.models import WordLevel # noqa: E402from transformers import BertConfig, BertModel, PreTrainedTokenizerFast # noqa: E402from laya import load # noqa: E402-from laya.calibrate import records_from_labeled # noqa: E402+from laya.calibrate import fit_temperature_map, records_from_labeled # noqa: E402from laya.common import DecisionModel, QTYPES, TEMP_MIN, TEMP_MAX # noqa: E402+def _config_only_save_checkpoint(model, tok, cfg, path):+ """`save_checkpoint`'s config write, without the weights it also owns.++ This suite owns the shape of `rl_agent_config.json`, not `model.safetensors`. Binding the+ notebook's own call to `save_checkpoint` is what still fails when it stops persisting the+ config; reproducing how the dict is built here is what would miss a notebook regression.+ """+ os.makedirs(path, exist_ok=True)+ with open(os.path.join(path, "rl_agent_config.json"), "w", encoding="utf-8") as fh:+ json.dump(cfg, fh, indent=2)++def export_notebook_config(cfg, fitted_temps, output_dir):"""Execute the notebook's actual config export, without its GPU/training code."""notebook = Path(__file__).resolve().parents[1] / "notebooks" / (@@ -38,14 +50,19 @@ def export_notebook_config(cfg, fitted_temps, output_dir):cells = json.loads(notebook.read_text(encoding="utf-8"))["cells"]script, = ["".join(c["source"]) for c in cellsif "".join(c["source"]).startswith("%%writefile ")]- # This contiguous tail includes the temperature update AND the JSON write.+ # This contiguous tail includes the temperature update AND the config write.# Do not reproduce the export logic here: that would miss notebook regressions.- start = script.index(' cfg["fine_tuned"] = True')+ start = script.index(' out_cfg = dict(cfg,')end = script.index("\n dist.destroy_process_group()", start)export = ast.parse(textwrap.dedent(script[start:end]))exec(compile(export, str(notebook), "exec"), {- "cfg": cfg, "fitted_temps": fitted_temps, "output_dir": str(output_dir),- "os": os, "json": json,+ "cfg": cfg, "fitted_temps": fitted_temps,+ # `fit_temperature_map` returns this shape; the caller passes only the scalar list.+ "fitted": {"temperature": list(fitted_temps), "temperature_by_options": {}},+ "max_len": cfg.get("max_len"), "head_max_len": cfg.get("head_max_len"),+ "output_dir": str(output_dir), "os": os, "json": json,+ "save_checkpoint": _config_only_save_checkpoint,+ "model": None, "tok": None,})return json.loads((output_dir / "rl_agent_config.json").read_text())@@ -221,31 +238,43 @@ class CalibrationPersistenceTests(unittest.TestCase):self.assertEqual(target.tolist(), [0.0, 1.0])def test_every_fine_tuning_fit_clamps_to_common_bounds(self):- # #642 fixed this for the Kaggle notebook, which still owns a fit of its own. The+ # #642 fixed this for the Kaggle notebook, which owned a fit of its own. The# Apple Silicon script and research/scripts/finetune_single_device.py used to keep# `torch.clamp(..., 0.1, 10.0)` beside it, so a fit either of them persisted could# land outside `[TEMP_MIN, TEMP_MAX]` and be re-clamped when the checkpoint is loaded# -- the calibration measured during training is then not the one that gets served.- # Both now call `laya.train.finetune` and take those bounds from- # `laya.calibrate.fit_one_temperature`, so what is left to sweep is the fitter that- # is still hand-rolled. Derived from the tree rather than listed, so a script that- # copies the pattern is held too.+ # All three now reach `laya.calibrate`, so what the tree sweep below still walks is a+ # fit that copies the old pattern. Derived from the tree rather than listed, so a+ # script that copies it is held too.fitters = _temperature_fitters()- # Non-vacuity: the loop below asserts nothing when the sweep matches nothing, and the- # sweep matching nothing is exactly the state a converging tree walks into. Name the- # one entry point that still owns a fit, so losing its clamp -- or superseding this- # guard outright -- is a failure to read, not a silent pass.- self.assertIn(- "notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb",- [rel for rel, _, _ in fitters],- "no hand-rolled clamped fit left to sweep; if every entry point now goes through "- "laya.calibrate, retire or redirect this sweep rather than leaving it matching nothing")+ # Non-vacuity, redirected exactly as the sweep's own message asks for once no entry+ # point owns a hand-rolled fit: pin that the notebook still reaches the canonical+ # fitter and owns no fit of its own, so taking one back is a failure to read rather+ # than a silent pass.+ notebook_src = _runnable_source(+ "notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb")+ self.assertIn("fit_temperature_map", notebook_src,+ "the notebook no longer reaches laya.calibrate")+ self.assertNotIn("fit_one_temp", notebook_src,+ "the notebook owns a private fit again")# These two drive the LBFGS solution past TEMP_MAX and below TEMP_MIN respectively,# so a range wider than the runtime's is caught in both directions.peaked = [([10.0, 0.0], [0.5, 0.5]) for _ in range(20)]dipped = [([0.0, 4.0], [0.0, 1.0]) for _ in range(20)]+ # With no entry point owning a hand-rolled fit, the tree sweep below can match+ # nothing, so drive the fit every entry point now reaches with the same two inputs:+ # the bound this test exists for is asserted either way.+ for label, sel in (("peaked", peaked), ("dipped", dipped)):+ t_fit = fit_temperature_map(+ [(QTYPES["choice"], z, t, len(z)) for z, t in sel]+ )["temperature"][QTYPES["choice"]]+ self.assertGreaterEqual(+ t_fit, TEMP_MIN, "fit_temperature_map(%s) fitted %.4g" % (label, t_fit))+ self.assertLessEqual(+ t_fit, TEMP_MAX, "fit_temperature_map(%s) fitted %.4g" % (label, t_fit))+for rel, name, body in fitters:with self.subTest(source=rel, function=name):scope = {"torch": torch, "TEMP_MIN": TEMP_MIN, "TEMP_MAX": TEMP_MAX}diff --git a/tests/test_kaggle_ddp_convergence.py b/tests/test_kaggle_ddp_convergence.pynew file mode 100644index 00000000..1f2ad9f6--- /dev/null+++ b/tests/test_kaggle_ddp_convergence.py@@ -0,0 +1,127 @@+"""YU-344: the Kaggle 2xT4 DDP notebook converged onto the canonical training primitives.++Weight-free checks: the distributed script keeps its process-group / shard+orchestration but delegates item construction, encoding, collation, the RLCD+loss, the calibration holdout, record building and temperature fitting to+`laya.train` / `laya.common` / `laya.calibrate`.++Run: python tests/test_kaggle_ddp_convergence.py+"""+import json+import os+import random+import sys+import unittest+from pathlib import Path++os.environ.setdefault("USE_TF", "0")+os.environ.setdefault("USE_TORCH", "1")+os.environ.setdefault("HF_HUB_OFFLINE", "1")+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++import numpy as np # noqa: E402+import torch # noqa: E402++from laya.calibrate import fit_temperature_map # noqa: E402+from laya.common import TEMP_MAX, TEMP_MIN, collate_items # noqa: E402+from laya.train import sigma_at, split_calibration # noqa: E402++NOTEBOOK = Path(__file__).resolve().parents[1] / "notebooks" / "laya_finetune_typed_decisions_2xT4_kaggle.ipynb"+++def _cell(nb, i):+ return "".join(nb["cells"][i]["source"])+++class NotebookDelegationTest(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ cls.nb = json.loads(NOTEBOOK.read_text(encoding="utf-8"))+ cls.ddp = _cell(cls.nb, 8)+ cls.prep = _cell(cls.nb, 6)++ def test_ddp_script_imports_canonical_primitives(self):+ for name in ("encode_item", "rlcd_loss", "save_checkpoint", "sigma_at",+ "split_calibration", "calibration_records"):+ self.assertIn(name, self.ddp)+ self.assertIn("collate_items", self.ddp)+ self.assertIn("fit_temperature_map", self.ddp)++ def test_ddp_script_has_no_private_training_semantics(self):+ for gone in ("def collate_train_batch", "def fit_one_temp",+ "def build_training_item", "def fit_one_temperature"):+ self.assertNotIn(gone, self.ddp)++ def test_prep_cell_uses_canonical_item_construction(self):+ self.assertIn("items_from_rows", self.prep)+ self.assertNotIn("def build_training_item", self.prep)++ def test_calibration_held_out_before_sharding(self):+ split_at = self.ddp.index("split_calibration")+ shard_at = self.ddp.index("train_items[rank::world_size]")+ self.assertLess(split_at, shard_at)+++class SplitCalibrationTest(unittest.TestCase):+ def test_disjoint_deterministic_and_seeded(self):+ items = list(range(1000))+ tr1, ca1 = split_calibration(items, calib_max=400, calib_frac=0.1, seed=20260922)+ tr2, ca2 = split_calibration(items, calib_max=400, calib_frac=0.1, seed=20260922)+ self.assertEqual(ca1, ca2)+ self.assertEqual(tr1, tr2)+ self.assertEqual(len(ca1), 100)+ self.assertFalse(set(tr1) & set(ca1))+ self.assertEqual(len(tr1) + len(ca1), 1000)++ def test_calib_max_caps_count(self):+ items = list(range(500))+ _, ca = split_calibration(items, calib_max=10, calib_frac=0.5, seed=0)+ self.assertEqual(len(ca), 10)+++class CollateTest(unittest.TestCase):+ def test_collate_items_matches_notebook_shape(self):+ encoded = [+ {"ids": [1, 2, 3], "markers": [0, 2], "qtype": 0, "target": [0.7, 0.3]},+ {"ids": [4, 5], "markers": [1], "qtype": 2, "target": [1.0]},+ ]+ batch = collate_items([encoded], pad_id=0)+ self.assertEqual(batch["input_ids"].shape, (2, 3))+ self.assertEqual(batch["attention_mask"].tolist(), [[1, 1, 1], [1, 1, 0]])+ self.assertEqual(batch["marker_pos"].tolist(), [[0, 2], [1, 0]])+ self.assertEqual(batch["marker_mask"].tolist(), [[True, True], [True, False]])+ got = batch["target"].tolist()+ self.assertEqual(len(got), 2)+ self.assertAlmostEqual(got[0][0], 0.7, places=6)+ self.assertAlmostEqual(got[0][1], 0.3, places=6)+ self.assertEqual(got[1], [1.0, 0.0])+ self.assertEqual(batch["qtype"].tolist(), [0, 2])+++class SigmaScheduleTest(unittest.TestCase):+ def test_linear_endpoints(self):+ self.assertAlmostEqual(sigma_at(0, 4, 0.4, 0.1), 0.4)+ self.assertAlmostEqual(sigma_at(3, 4, 0.4, 0.1), 0.1)+ self.assertAlmostEqual(sigma_at(1, 4, 0.4, 0.1), 0.3)+++class TemperatureFitTest(unittest.TestCase):+ def test_fit_temperature_map_respects_runtime_bounds(self):+ rng = np.random.default_rng(0)+ records = []+ for i in range(40):+ k = int(rng.integers(2, 5))+ logits = rng.normal(0, 1, k)+ target = rng.dirichlet(np.ones(k))+ records.append((i % 3, logits, target, k))+ fitted = fit_temperature_map(records)+ for t in fitted["temperature"]:+ self.assertGreaterEqual(t, TEMP_MIN)+ self.assertLessEqual(t, TEMP_MAX)+ for v in fitted["temperature_by_options"].values():+ self.assertGreaterEqual(v, TEMP_MIN)+ self.assertLessEqual(v, TEMP_MAX)+++if __name__ == "__main__":+ unittest.main()