Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(training): honor decision-head activation checkpointing (#149)
xinpingzhang0cca7cc0f085
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex c15af608..069952d2 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -60,6 +60,7 @@ jobs:python tests/test_download.pypython tests/test_shortlist.pypython tests/test_decision_model.py+ python tests/test_head_checkpointing.pypython tests/test_packaging.pypython tests/test_lang_guess.pypython tests/test_calibration_persistence.pydiff --git a/README.md b/README.mdindex 6b25f7a5..58f11db5 100644--- a/README.md+++ b/README.md@@ -649,6 +649,13 @@ policy gradient), fit calibration temperatures, evaluate, and push the result to* **[`notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb`](notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb)**+The notebook enables gradient checkpointing on both the encoder and the decision head.+For custom training loops, `model.head_checkpointing = True` enables activation+checkpointing for the decision-head layers; enable the encoder's gradient checkpointing+separately. During gradient-enabled training, this reduces stored intermediate activations+by recomputing them during backward, trading extra computation for lower activation memory.+The head flag defaults to `False` and is bypassed in evaluation and under `torch.no_grad()`.+The notebook fits one `temperature` per type (`choice`, `score`, `noul`) and removes inherited`temperature_by_options` from the exported config. Otherwise those old bucket values takeprecedence at inference and silently mask the new fit. Existing checkpoints still honordiff --git a/laya/common.py b/laya/common.pyindex 950c41df..9003e08f 100644--- a/laya/common.py+++ b/laya/common.py@@ -7,6 +7,7 @@ from typing import Dict, List, Optional, Unionimport numpy as npimport torchimport torch.nn as nn+from torch.utils.checkpoint import checkpointQTYPES = {"choice": 0, "score": 1, "noul": 2}QTYPE_NAMES = {v: k for k, v in QTYPES.items()}@@ -110,7 +111,12 @@ class DecisionModel(nn.Module):if self.head is not None:pad = ~attention_mask.bool()for layer in self.head.layers:- h = layer(h, src_key_padding_mask=pad)+ if self.head_checkpointing and self.training and torch.is_grad_enabled():+ # Non-reentrant checkpointing also trains the head when its input+ # is frozen. Default RNG preservation keeps dropout consistent.+ h = checkpoint(layer, h, src_key_padding_mask=pad, use_reentrant=False)+ else:+ h = layer(h, src_key_padding_mask=pad)idx = marker_pos.clamp(min=0)[:, :, None].expand(-1, -1, h.size(-1))m = torch.gather(h, 1, idx)logits = self.scorer(m).squeeze(-1).float()diff --git a/tests/test_head_checkpointing.py b/tests/test_head_checkpointing.pynew file mode 100644index 00000000..2c850201--- /dev/null+++ b/tests/test_head_checkpointing.py@@ -0,0 +1,152 @@+"""Decision-head checkpointing regressions with a tiny BERT; no downloads.++Run: python tests/test_head_checkpointing.py+"""+import copy+import os+from pathlib import Path+import sys+import unittest+from unittest.mock import patch++os.environ.setdefault("USE_TF", "0")+os.environ.setdefault("USE_TORCH", "1")+os.environ.setdefault("HF_HUB_OFFLINE", "1")+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++import torch # noqa: E402+from transformers import BertConfig, BertModel # noqa: E402++from laya.common import DecisionModel # noqa: E402+++def tiny_model(head_layers=2):+ torch.manual_seed(41)+ config = BertConfig(vocab_size=64, hidden_size=16, num_hidden_layers=1,+ num_attention_heads=1, intermediate_size=32)+ model = DecisionModel(BertModel(config), head_layers=head_layers, dropout=0.2)+ if model.head is not None:+ # TransformerEncoder clones its layers. Distinct weights catch accidentally+ # recomputing the last layer for every iteration of the forward loop.+ with torch.no_grad():+ for i, layer in enumerate(model.head.layers):+ layer.linear1.weight.add_(0.1 * i)+ return model+++def inputs():+ torch.manual_seed(42)+ ids = torch.randint(0, 64, (3, 12))+ attention = torch.ones_like(ids)+ attention[0, 9:] = 0+ attention[1, 11:] = 0+ positions = torch.tensor([[2, 0, 0], [2, 5, 0], [2, 5, 8]])+ markers = torch.tensor([[1, 0, 0], [1, 1, 0], [1, 1, 1]], dtype=torch.bool)+ return ids, attention, positions, markers, torch.tensor([0, 2, 1])+++def training_step(model, *, detach_encoder=False):+ batch = inputs()+ optimizer = torch.optim.SGD(model.parameters(), lr=0.01)+ optimizer.zero_grad(set_to_none=True)+ torch.manual_seed(123)+ logits, act = model(*batch, detach_encoder=detach_encoder)+ loss = logits.masked_select(batch[3]).square().mean() + act.square().mean()+ loss.backward()+ gradients = {name: p.grad.detach().clone() if p.grad is not None else None+ for name, p in model.named_parameters()}+ optimizer.step()+ return logits.detach(), act.detach(), loss.detach(), gradients, torch.get_rng_state()+++class HeadCheckpointingTests(unittest.TestCase):+ def assert_same_step(self, model, **kwargs):+ ordinary = copy.deepcopy(model).train()+ checkpointed = copy.deepcopy(model).train()+ ordinary.head_checkpointing = False+ checkpointed.head_checkpointing = True+ expected = training_step(ordinary, **kwargs)+ actual = training_step(checkpointed, **kwargs)+ for before, after in zip(expected[:3], actual[:3]):+ torch.testing.assert_close(before, after, rtol=1e-5, atol=1e-6)+ for name, gradient in expected[3].items():+ other = actual[3][name]+ with self.subTest(parameter=name):+ if gradient is None:+ self.assertIsNone(other)+ else:+ self.assertIsNotNone(other)+ torch.testing.assert_close(gradient, other, rtol=1e-5, atol=1e-6)+ self.assertTrue(torch.equal(expected[4], actual[4]), "dropout RNG state changed")+ for name, value in ordinary.state_dict().items():+ torch.testing.assert_close(value, checkpointed.state_dict()[name], rtol=1e-5, atol=1e-6)+ return actual[3]++ def test_enabled_head_recomputes_each_layer_during_backward(self):+ for enabled in (False, True):+ with self.subTest(enabled=enabled):+ model = tiny_model().train()+ model.head_checkpointing = enabled+ calls = [0, 0]+ handles = []+ for i, layer in enumerate(model.head.layers):+ def count(_module, _args, index=i):+ calls[index] += 1+ handles.append(layer.register_forward_pre_hook(count))+ try:+ training_step(model)+ finally:+ for handle in handles:+ handle.remove()+ self.assertEqual(calls, [2, 2] if enabled else [1, 1])++ def test_training_outputs_gradients_and_updates_match_with_dropout(self):+ self.assert_same_step(tiny_model())++ def test_encoder_and_head_checkpointing_work_together(self):+ model = tiny_model()+ model.encoder.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})+ self.assert_same_step(model)++ def test_detached_encoder_keeps_head_gradients(self):+ gradients = self.assert_same_step(tiny_model(), detach_encoder=True)+ self.assertTrue(all(g is None for n, g in gradients.items() if n.startswith("encoder.")))+ self.assertIsNotNone(gradients["head.layers.0.linear1.weight"])++ def test_frozen_head_inputs_still_train_head_parameters(self):+ model = tiny_model()+ for parameter in list(model.encoder.parameters()) + list(model.type_emb.parameters()):+ parameter.requires_grad_(False)+ gradients = self.assert_same_step(model)+ for i in range(2):+ gradient = gradients["head.layers.%d.linear1.weight" % i]+ self.assertIsNotNone(gradient)+ self.assertGreater(gradient.abs().sum().item(), 0.0)++ def test_eval_and_no_grad_bypass_checkpointing(self):+ for training, grad_enabled in ((False, True), (False, False), (True, False)):+ with self.subTest(training=training, grad_enabled=grad_enabled):+ model = tiny_model().train(training)+ batch = inputs()+ with torch.set_grad_enabled(grad_enabled):+ torch.manual_seed(123)+ expected = model(*batch)+ model.head_checkpointing = True+ with patch("laya.common.checkpoint", side_effect=AssertionError("unexpected checkpoint")):+ torch.manual_seed(123)+ actual = model(*batch)+ torch.testing.assert_close(expected, actual, rtol=0, atol=0)++ def test_zero_head_layers_remain_supported(self):+ self.assert_same_step(tiny_model(head_layers=0))++ def test_default_is_disabled(self):+ model = tiny_model().train()+ self.assertFalse(model.head_checkpointing)+ with patch("laya.common.checkpoint", side_effect=AssertionError("unexpected checkpoint")):+ training_step(model)+++if __name__ == "__main__":+ torch.set_num_threads(1)+ unittest.main()