Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

test(train): guard the Kaggle DDP notebook against returning private training copies

Bruce-Yii61c1455b4953
diff --git a/scripts/test_suites.py b/scripts/test_suites.py
index 546c0764..9c845b34 100644
--- a/scripts/test_suites.py
+++ b/scripts/test_suites.py
@@ -88,6 +88,7 @@ SCRIPT_SUITES = [
"tests/test_portability.py",
"tests/test_training.py",
"tests/test_train.py",
+ "tests/test_kaggle_ddp_convergence.py",
"tests/test_example_server_limits.py",
"tests/test_blank_lang_routing.py",
"tests/test_export_onnx_safety.py",
diff --git a/tests/test_calibration_persistence.py b/tests/test_calibration_persistence.py
index 311bb12e..833774e4 100644
--- a/tests/test_calibration_persistence.py
+++ b/tests/test_calibration_persistence.py
@@ -26,10 +26,22 @@ from tokenizers.models import WordLevel # noqa: E402
from transformers import BertConfig, BertModel, PreTrainedTokenizerFast # noqa: E402
from laya import load # noqa: E402
-from laya.calibrate import records_from_labeled # noqa: E402
+from laya.calibrate import fit_temperature_map, records_from_labeled # noqa: E402
from laya.common import DecisionModel, QTYPES, TEMP_MIN, TEMP_MAX # noqa: E402
+def _config_only_save_checkpoint(model, tok, cfg, path):
+ """`save_checkpoint`'s config write, without the weights it also owns.
+
+ This suite owns the shape of `rl_agent_config.json`, not `model.safetensors`. Binding the
+ notebook's own call to `save_checkpoint` is what still fails when it stops persisting the
+ config; reproducing how the dict is built here is what would miss a notebook regression.
+ """
+ os.makedirs(path, exist_ok=True)
+ with open(os.path.join(path, "rl_agent_config.json"), "w", encoding="utf-8") as fh:
+ json.dump(cfg, fh, indent=2)
+
+
def export_notebook_config(cfg, fitted_temps, output_dir):
"""Execute the notebook's actual config export, without its GPU/training code."""
notebook = Path(__file__).resolve().parents[1] / "notebooks" / (
@@ -38,14 +50,19 @@ def export_notebook_config(cfg, fitted_temps, output_dir):
cells = json.loads(notebook.read_text(encoding="utf-8"))["cells"]
script, = ["".join(c["source"]) for c in cells
if "".join(c["source"]).startswith("%%writefile ")]
- # This contiguous tail includes the temperature update AND the JSON write.
+ # This contiguous tail includes the temperature update AND the config write.
# Do not reproduce the export logic here: that would miss notebook regressions.
- start = script.index(' cfg["fine_tuned"] = True')
+ start = script.index(' out_cfg = dict(cfg,')
end = script.index("\n dist.destroy_process_group()", start)
export = ast.parse(textwrap.dedent(script[start:end]))
exec(compile(export, str(notebook), "exec"), {
- "cfg": cfg, "fitted_temps": fitted_temps, "output_dir": str(output_dir),
- "os": os, "json": json,
+ "cfg": cfg, "fitted_temps": fitted_temps,
+ # `fit_temperature_map` returns this shape; the caller passes only the scalar list.
+ "fitted": {"temperature": list(fitted_temps), "temperature_by_options": {}},
+ "max_len": cfg.get("max_len"), "head_max_len": cfg.get("head_max_len"),
+ "output_dir": str(output_dir), "os": os, "json": json,
+ "save_checkpoint": _config_only_save_checkpoint,
+ "model": None, "tok": None,
})
return json.loads((output_dir / "rl_agent_config.json").read_text())
@@ -221,31 +238,43 @@ class CalibrationPersistenceTests(unittest.TestCase):
self.assertEqual(target.tolist(), [0.0, 1.0])
def test_every_fine_tuning_fit_clamps_to_common_bounds(self):
- # #642 fixed this for the Kaggle notebook, which still owns a fit of its own. The
+ # #642 fixed this for the Kaggle notebook, which owned a fit of its own. The
# Apple Silicon script and research/scripts/finetune_single_device.py used to keep
# `torch.clamp(..., 0.1, 10.0)` beside it, so a fit either of them persisted could
# land outside `[TEMP_MIN, TEMP_MAX]` and be re-clamped when the checkpoint is loaded
# -- the calibration measured during training is then not the one that gets served.
- # Both now call `laya.train.finetune` and take those bounds from
- # `laya.calibrate.fit_one_temperature`, so what is left to sweep is the fitter that
- # is still hand-rolled. Derived from the tree rather than listed, so a script that
- # copies the pattern is held too.
+ # All three now reach `laya.calibrate`, so what the tree sweep below still walks is a
+ # fit that copies the old pattern. Derived from the tree rather than listed, so a
+ # script that copies it is held too.
fitters = _temperature_fitters()
- # Non-vacuity: the loop below asserts nothing when the sweep matches nothing, and the
- # sweep matching nothing is exactly the state a converging tree walks into. Name the
- # one entry point that still owns a fit, so losing its clamp -- or superseding this
- # guard outright -- is a failure to read, not a silent pass.
- self.assertIn(
- "notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb",
- [rel for rel, _, _ in fitters],
- "no hand-rolled clamped fit left to sweep; if every entry point now goes through "
- "laya.calibrate, retire or redirect this sweep rather than leaving it matching nothing")
+ # Non-vacuity, redirected exactly as the sweep's own message asks for once no entry
+ # point owns a hand-rolled fit: pin that the notebook still reaches the canonical
+ # fitter and owns no fit of its own, so taking one back is a failure to read rather
+ # than a silent pass.
+ notebook_src = _runnable_source(
+ "notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb")
+ self.assertIn("fit_temperature_map", notebook_src,
+ "the notebook no longer reaches laya.calibrate")
+ self.assertNotIn("fit_one_temp", notebook_src,
+ "the notebook owns a private fit again")
# These two drive the LBFGS solution past TEMP_MAX and below TEMP_MIN respectively,
# so a range wider than the runtime's is caught in both directions.
peaked = [([10.0, 0.0], [0.5, 0.5]) for _ in range(20)]
dipped = [([0.0, 4.0], [0.0, 1.0]) for _ in range(20)]
+ # With no entry point owning a hand-rolled fit, the tree sweep below can match
+ # nothing, so drive the fit every entry point now reaches with the same two inputs:
+ # the bound this test exists for is asserted either way.
+ for label, sel in (("peaked", peaked), ("dipped", dipped)):
+ t_fit = fit_temperature_map(
+ [(QTYPES["choice"], z, t, len(z)) for z, t in sel]
+ )["temperature"][QTYPES["choice"]]
+ self.assertGreaterEqual(
+ t_fit, TEMP_MIN, "fit_temperature_map(%s) fitted %.4g" % (label, t_fit))
+ self.assertLessEqual(
+ t_fit, TEMP_MAX, "fit_temperature_map(%s) fitted %.4g" % (label, t_fit))
+
for rel, name, body in fitters:
with self.subTest(source=rel, function=name):
scope = {"torch": torch, "TEMP_MIN": TEMP_MIN, "TEMP_MAX": TEMP_MAX}
diff --git a/tests/test_kaggle_ddp_convergence.py b/tests/test_kaggle_ddp_convergence.py
new file mode 100644
index 00000000..1f2ad9f6
--- /dev/null
+++ b/tests/test_kaggle_ddp_convergence.py
@@ -0,0 +1,127 @@
+"""YU-344: the Kaggle 2xT4 DDP notebook converged onto the canonical training primitives.
+
+Weight-free checks: the distributed script keeps its process-group / shard
+orchestration but delegates item construction, encoding, collation, the RLCD
+loss, the calibration holdout, record building and temperature fitting to
+`laya.train` / `laya.common` / `laya.calibrate`.
+
+Run: python tests/test_kaggle_ddp_convergence.py
+"""
+import json
+import os
+import random
+import sys
+import unittest
+from pathlib import Path
+
+os.environ.setdefault("USE_TF", "0")
+os.environ.setdefault("USE_TORCH", "1")
+os.environ.setdefault("HF_HUB_OFFLINE", "1")
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import numpy as np # noqa: E402
+import torch # noqa: E402
+
+from laya.calibrate import fit_temperature_map # noqa: E402
+from laya.common import TEMP_MAX, TEMP_MIN, collate_items # noqa: E402
+from laya.train import sigma_at, split_calibration # noqa: E402
+
+NOTEBOOK = Path(__file__).resolve().parents[1] / "notebooks" / "laya_finetune_typed_decisions_2xT4_kaggle.ipynb"
+
+
+def _cell(nb, i):
+ return "".join(nb["cells"][i]["source"])
+
+
+class NotebookDelegationTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.nb = json.loads(NOTEBOOK.read_text(encoding="utf-8"))
+ cls.ddp = _cell(cls.nb, 8)
+ cls.prep = _cell(cls.nb, 6)
+
+ def test_ddp_script_imports_canonical_primitives(self):
+ for name in ("encode_item", "rlcd_loss", "save_checkpoint", "sigma_at",
+ "split_calibration", "calibration_records"):
+ self.assertIn(name, self.ddp)
+ self.assertIn("collate_items", self.ddp)
+ self.assertIn("fit_temperature_map", self.ddp)
+
+ def test_ddp_script_has_no_private_training_semantics(self):
+ for gone in ("def collate_train_batch", "def fit_one_temp",
+ "def build_training_item", "def fit_one_temperature"):
+ self.assertNotIn(gone, self.ddp)
+
+ def test_prep_cell_uses_canonical_item_construction(self):
+ self.assertIn("items_from_rows", self.prep)
+ self.assertNotIn("def build_training_item", self.prep)
+
+ def test_calibration_held_out_before_sharding(self):
+ split_at = self.ddp.index("split_calibration")
+ shard_at = self.ddp.index("train_items[rank::world_size]")
+ self.assertLess(split_at, shard_at)
+
+
+class SplitCalibrationTest(unittest.TestCase):
+ def test_disjoint_deterministic_and_seeded(self):
+ items = list(range(1000))
+ tr1, ca1 = split_calibration(items, calib_max=400, calib_frac=0.1, seed=20260922)
+ tr2, ca2 = split_calibration(items, calib_max=400, calib_frac=0.1, seed=20260922)
+ self.assertEqual(ca1, ca2)
+ self.assertEqual(tr1, tr2)
+ self.assertEqual(len(ca1), 100)
+ self.assertFalse(set(tr1) & set(ca1))
+ self.assertEqual(len(tr1) + len(ca1), 1000)
+
+ def test_calib_max_caps_count(self):
+ items = list(range(500))
+ _, ca = split_calibration(items, calib_max=10, calib_frac=0.5, seed=0)
+ self.assertEqual(len(ca), 10)
+
+
+class CollateTest(unittest.TestCase):
+ def test_collate_items_matches_notebook_shape(self):
+ encoded = [
+ {"ids": [1, 2, 3], "markers": [0, 2], "qtype": 0, "target": [0.7, 0.3]},
+ {"ids": [4, 5], "markers": [1], "qtype": 2, "target": [1.0]},
+ ]
+ batch = collate_items([encoded], pad_id=0)
+ self.assertEqual(batch["input_ids"].shape, (2, 3))
+ self.assertEqual(batch["attention_mask"].tolist(), [[1, 1, 1], [1, 1, 0]])
+ self.assertEqual(batch["marker_pos"].tolist(), [[0, 2], [1, 0]])
+ self.assertEqual(batch["marker_mask"].tolist(), [[True, True], [True, False]])
+ got = batch["target"].tolist()
+ self.assertEqual(len(got), 2)
+ self.assertAlmostEqual(got[0][0], 0.7, places=6)
+ self.assertAlmostEqual(got[0][1], 0.3, places=6)
+ self.assertEqual(got[1], [1.0, 0.0])
+ self.assertEqual(batch["qtype"].tolist(), [0, 2])
+
+
+class SigmaScheduleTest(unittest.TestCase):
+ def test_linear_endpoints(self):
+ self.assertAlmostEqual(sigma_at(0, 4, 0.4, 0.1), 0.4)
+ self.assertAlmostEqual(sigma_at(3, 4, 0.4, 0.1), 0.1)
+ self.assertAlmostEqual(sigma_at(1, 4, 0.4, 0.1), 0.3)
+
+
+class TemperatureFitTest(unittest.TestCase):
+ def test_fit_temperature_map_respects_runtime_bounds(self):
+ rng = np.random.default_rng(0)
+ records = []
+ for i in range(40):
+ k = int(rng.integers(2, 5))
+ logits = rng.normal(0, 1, k)
+ target = rng.dirichlet(np.ones(k))
+ records.append((i % 3, logits, target, k))
+ fitted = fit_temperature_map(records)
+ for t in fitted["temperature"]:
+ self.assertGreaterEqual(t, TEMP_MIN)
+ self.assertLessEqual(t, TEMP_MAX)
+ for v in fitted["temperature_by_options"].values():
+ self.assertGreaterEqual(v, TEMP_MIN)
+ self.assertLessEqual(v, TEMP_MAX)
+
+
+if __name__ == "__main__":
+ unittest.main()