Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

14.7 KB
Raw Blame
"""Notebook export and inference regression tests; CPU only, no training or downloads.

Run: python tests/test_calibration_persistence.py
"""
import ast
import copy
import json
import os
from pathlib import Path
import re
import sys
import tempfile
import textwrap
import unittest
from unittest.mock import patch

os.environ.setdefault("USE_TF", "0")
os.environ.setdefault("USE_TORCH", "1")
os.environ.setdefault("HF_HUB_OFFLINE", "1")
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

import torch  # noqa: E402
from safetensors.torch import save_file  # noqa: E402
from tokenizers import Tokenizer  # noqa: E402
from tokenizers.models import WordLevel  # noqa: E402
from transformers import BertConfig, BertModel, PreTrainedTokenizerFast  # noqa: E402

from laya import load  # noqa: E402
from laya.calibrate import fit_temperature_map, records_from_labeled  # noqa: E402
from laya.common import DecisionModel, QTYPES, TEMP_MIN, TEMP_MAX  # noqa: E402


def _config_only_save_checkpoint(model, tok, cfg, path):
    """`save_checkpoint`'s config write, without the weights it also owns.

    This suite owns the shape of `rl_agent_config.json`, not `model.safetensors`. Binding the
    notebook's own call to `save_checkpoint` is what still fails when it stops persisting the
    config; reproducing how the dict is built here is what would miss a notebook regression.
    """
    os.makedirs(path, exist_ok=True)
    with open(os.path.join(path, "rl_agent_config.json"), "w", encoding="utf-8") as fh:
        json.dump(cfg, fh, indent=2)


def export_notebook_config(cfg, fitted_temps, output_dir):
    """Execute the notebook's actual config export, without its GPU/training code."""
    notebook = Path(__file__).resolve().parents[1] / "notebooks" / (
        "laya_finetune_typed_decisions_2xT4_kaggle.ipynb"
    )
    cells = json.loads(notebook.read_text(encoding="utf-8"))["cells"]
    script, = ["".join(c["source"]) for c in cells
               if "".join(c["source"]).startswith("%%writefile ")]
    # This contiguous tail includes the temperature update AND the config write.
    # Do not reproduce the export logic here: that would miss notebook regressions.
    start = script.index('        out_cfg = dict(cfg,')
    end = script.index("\n    dist.destroy_process_group()", start)
    export = ast.parse(textwrap.dedent(script[start:end]))
    exec(compile(export, str(notebook), "exec"), {
        "cfg": cfg, "fitted_temps": fitted_temps,
        # `fit_temperature_map` returns this shape; the caller passes only the scalar list.
        "fitted": {"temperature": list(fitted_temps), "temperature_by_options": {}},
        "max_len": cfg.get("max_len"), "head_max_len": cfg.get("head_max_len"),
        "output_dir": str(output_dir), "os": os, "json": json,
        "save_checkpoint": _config_only_save_checkpoint,
        "model": None, "tok": None,
    })
    return json.loads((output_dir / "rl_agent_config.json").read_text())


ROOT = Path(__file__).resolve().parents[1]


def _runnable_source(rel):
    """What a fine-tuning entry point actually executes: its %%writefile cells for a
    notebook, the file itself for a script."""
    if rel.endswith(".ipynb"):
        cells = json.loads((ROOT / rel).read_text(encoding="utf-8"))["cells"]
        return "".join("".join(c["source"]) for c in cells
                       if "".join(c["source"]).startswith("%%writefile "))
    return (ROOT / rel).read_text(encoding="utf-8")


def _temperature_fitters():
    """Every notebook or training script that still owns a fit clamping `exp()` of a
    log-temperature.

    Walks the tree rather than listing the known entry points, so a new script that copies
    the pattern is held by the same check. Matches on the clamp of an `.exp()` because that
    is the one shape these share; a fit that stops clamping is reported by the non-empty
    assertion in the test rather than passing unnoticed.

    Entry points that call `laya.train.finetune` are absent by design: their fit is
    `laya.calibrate.fit_one_temperature`, which ends in `clamp_temperature(fitted, TEMP_MIN,
    TEMP_MAX)` -- the same bounds, held by tests/test_calibrate.py. Their delegation is held
    by tests/test_finetune_entrypoints.py.
    """
    found = []
    for base in ("notebooks", "research/scripts"):
        for path in sorted((ROOT / base).rglob("*")):
            if path.suffix not in (".py", ".ipynb") or not path.is_file():
                continue
            rel = path.relative_to(ROOT).as_posix()
            text = _runnable_source(rel)
            for match in re.finditer(r"^def (\w+)\(.*?(?=^\S|\Z)", text, re.M | re.S):
                if re.search(r"torch\.clamp\(\s*\w+\.exp\(\)", match.group(0)):
                    found.append((rel, match.group(1), match.group(0)))
    return found


class CalibrationPersistenceTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.tmp = tempfile.TemporaryDirectory()
        cls.addClassCleanup(cls.tmp.cleanup)
        cls.checkpoint = Path(cls.tmp.name)
        config = BertConfig(vocab_size=6, hidden_size=16, num_hidden_layers=1,
                            num_attention_heads=1, intermediate_size=32)
        config.save_pretrained(cls.checkpoint / "encoder")
        tokenizer = PreTrainedTokenizerFast(
            tokenizer_object=Tokenizer(WordLevel(
                {"[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4, "hello": 5},
                unk_token="[UNK]")),
            pad_token="[PAD]", unk_token="[UNK]", cls_token="[CLS]",
            sep_token="[SEP]", mask_token="[MASK]",
        )
        tokenizer.save_pretrained(cls.checkpoint / "tokenizer")
        model = DecisionModel(BertModel(config), head_layers=0)
        save_file(model.state_dict(), cls.checkpoint / "model.safetensors")

    def setUp(self):
        self.cfg = {
            "encoder": "unused/offline", "head_layers": 0, "act_costs": {"act": 0},
            "max_len": 128, "head_max_len": 96,
            "temperature": [1.0, 1.0, 1.0], "training": {"updates": 7},
        }
        self.fitted = [2.0, 3.0, 4.0]

    def assert_inference_temperatures(self, expected):
        # Use actual local config/tokenizer/weight loading; replace only logits so
        # the expected scaling is deterministic and independent of random weights.
        with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):
            agent = load(str(self.checkpoint), device="cpu")
        self.assertEqual(agent.device.type, "cpu")
        questions = {}
        for qtype, k in expected:
            q = {"type": qtype, "instructions": "Pick one"}
            if qtype != "noul":
                q["criteria"] = [str(i) for i in range(k)]
            questions[f"{qtype}_{k}"] = q
        kmax = max(k for _, k in expected)
        logits = torch.arange(kmax, dtype=torch.float32).repeat(len(questions), 1)
        with patch.object(agent.model, "forward", return_value=(logits, torch.zeros(len(questions), 2))):
            answers = agent.predict("hello", questions)["answers"]
        for (qtype, k), temperature in expected.items():
            with self.subTest(qtype=qtype, k=k, temperature=temperature):
                p = torch.softmax(torch.arange(k, dtype=torch.float32) / temperature, -1)
                answer = answers[f"{qtype}_{k}"]
                if qtype == "noul":
                    self.assertAlmostEqual(answer["noul"], p[1].item(), delta=0.0001)
                else:
                    for i in range(k):
                        self.assertAlmostEqual(answer["probabilities"][str(i)], p[i].item(), delta=0.0001)
                    if qtype == "score":
                        self.assertAlmostEqual(answer["score"], (torch.arange(k) * p).sum().item(), delta=0.0001)

    def write_config(self):
        (self.checkpoint / "rl_agent_config.json").write_text(json.dumps(self.cfg))

    def test_new_export_removes_inherited_buckets_and_uses_fitted_types(self):
        self.cfg["temperature_by_options"] = {
            f"{qtype}:{bucket}": 1.5
            for qtype in QTYPES for bucket in ("2", "3-5", "6-10", "11+")
        }
        original = copy.deepcopy(self.cfg)
        saved = export_notebook_config(self.cfg, self.fitted, self.checkpoint)
        self.assert_inference_temperatures({
            (qtype, k): self.fitted[qt]
            for qtype, qt in QTYPES.items()
            for k in ((2,) if qtype == "noul" else (1, 2, 3, 5, 6, 10, 11))
        })
        self.assertFalse(saved.get("temperature_by_options"))
        self.assertEqual(saved["temperature"], self.fitted)
        self.assertTrue(saved["fine_tuned"])
        self.assertEqual(saved["model_name"], "laya-typed-decisions")
        for key in original.keys() - {"temperature", "temperature_by_options"}:
            self.assertEqual(saved[key], original[key])

    def test_export_accepts_absent_or_empty_bucket_map(self):
        for buckets in (None, {}):
            with self.subTest(buckets=buckets):
                if buckets is not None:
                    self.cfg["temperature_by_options"] = buckets
                saved = export_notebook_config(self.cfg, self.fitted, self.checkpoint)
                self.assertFalse(saved.get("temperature_by_options"))
                self.assertEqual(saved["temperature"], self.fitted)
                self.assert_inference_temperatures({(t, 2): self.fitted[qt] for t, qt in QTYPES.items()})

    def test_existing_bucket_calibration_keeps_precedence(self):
        self.cfg["temperature"] = self.fitted
        buckets = {"2": 0.75, "3-5": 1.25, "6-10": 1.5, "11+": 1.75}
        self.cfg["temperature_by_options"] = {
            f"{qtype}:{bucket}": value for qtype in QTYPES for bucket, value in buckets.items()
        }
        self.write_config()
        original = (self.checkpoint / "rl_agent_config.json").read_bytes()
        self.assert_inference_temperatures({
            (qtype, k): value for qtype in QTYPES
            for k, value in (((2, 0.75),) if qtype == "noul" else
                             ((1, 0.75), (2, 0.75), (3, 1.25), (5, 1.25),
                              (6, 1.5), (10, 1.5), (11, 1.75)))
        })
        self.assertEqual((self.checkpoint / "rl_agent_config.json").read_bytes(), original)

    def test_missing_bucket_falls_back_to_corresponding_type(self):
        self.cfg["temperature"] = self.fitted
        self.cfg["temperature_by_options"] = {"choice:2": 0.75}
        self.write_config()
        self.assert_inference_temperatures({("choice", 2): 0.75, ("choice", 3): 2.0,
                                            ("score", 2): 3.0, ("noul", 2): 4.0})

    def test_missing_temperatures_keep_unit_defaults(self):
        self.cfg.pop("temperature")
        self.write_config()
        self.assert_inference_temperatures({(t, 2): 1.0 for t in QTYPES})

    def test_records_from_labeled_runs_on_a_loaded_agent(self):
        # `Agent._forward` turns the logits straight into numpy, which fails on a tensor that
        # tracks gradients. The stand-in agents in test_calibrate.py return numpy and cannot see it.
        self.write_config()
        with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):
            agent = load(str(self.checkpoint), device="cpu")
        questions = {"flag": {"type": "noul", "instructions": "hello"}}
        records = records_from_labeled(agent, [("hello", questions, {"flag": [0.0, 1.0]})])
        self.assertEqual(len(records), 1)
        qtype, logits, target, k = records[0]
        self.assertEqual((qtype, k), (QTYPES["noul"], 2))
        self.assertEqual(logits.shape, (2,))
        self.assertEqual(target.tolist(), [0.0, 1.0])

    def test_every_fine_tuning_fit_clamps_to_common_bounds(self):
        # #642 fixed this for the Kaggle notebook, which owned a fit of its own. The
        # Apple Silicon script and research/scripts/finetune_single_device.py used to keep
        # `torch.clamp(..., 0.1, 10.0)` beside it, so a fit either of them persisted could
        # land outside `[TEMP_MIN, TEMP_MAX]` and be re-clamped when the checkpoint is loaded
        # -- the calibration measured during training is then not the one that gets served.
        # All three now reach `laya.calibrate`, so what the tree sweep below still walks is a
        # fit that copies the old pattern. Derived from the tree rather than listed, so a
        # script that copies it is held too.
        fitters = _temperature_fitters()
        # Non-vacuity, redirected exactly as the sweep's own message asks for once no entry
        # point owns a hand-rolled fit: pin that the notebook still reaches the canonical
        # fitter and owns no fit of its own, so taking one back is a failure to read rather
        # than a silent pass.
        notebook_src = _runnable_source(
            "notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb")
        self.assertIn("fit_temperature_map", notebook_src,
                      "the notebook no longer reaches laya.calibrate")
        self.assertNotIn("fit_one_temp", notebook_src,
                         "the notebook owns a private fit again")

        # These two drive the LBFGS solution past TEMP_MAX and below TEMP_MIN respectively,
        # so a range wider than the runtime's is caught in both directions.
        peaked = [([10.0, 0.0], [0.5, 0.5]) for _ in range(20)]
        dipped = [([0.0, 4.0], [0.0, 1.0]) for _ in range(20)]

        # With no entry point owning a hand-rolled fit, the tree sweep below can match
        # nothing, so drive the fit every entry point now reaches with the same two inputs:
        # the bound this test exists for is asserted either way.
        for label, sel in (("peaked", peaked), ("dipped", dipped)):
            t_fit = fit_temperature_map(
                [(QTYPES["choice"], z, t, len(z)) for z, t in sel]
            )["temperature"][QTYPES["choice"]]
            self.assertGreaterEqual(
                t_fit, TEMP_MIN, "fit_temperature_map(%s) fitted %.4g" % (label, t_fit))
            self.assertLessEqual(
                t_fit, TEMP_MAX, "fit_temperature_map(%s) fitted %.4g" % (label, t_fit))

        for rel, name, body in fitters:
            with self.subTest(source=rel, function=name):
                scope = {"torch": torch, "TEMP_MIN": TEMP_MIN, "TEMP_MAX": TEMP_MAX}
                exec(compile(ast.parse(textwrap.dedent(body)), rel, "exec"), scope)
                fit = scope[name]
                for label, sel in (("peaked", peaked), ("dipped", dipped)):
                    fitted = fit(sel)
                    self.assertGreaterEqual(
                        fitted, TEMP_MIN, "%s: %s(%s) fitted %.4g" % (rel, name, label, fitted))
                    self.assertLessEqual(
                        fitted, TEMP_MAX, "%s: %s(%s) fitted %.4g" % (rel, name, label, fitted))


if __name__ == "__main__":
    torch.set_num_threads(1)
    unittest.main()