Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(training): honor decision-head activation checkpointing (#149)

xinpingzhang0cca7cc0f085
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c15af608..069952d2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -60,6 +60,7 @@ jobs:
python tests/test_download.py
python tests/test_shortlist.py
python tests/test_decision_model.py
+ python tests/test_head_checkpointing.py
python tests/test_packaging.py
python tests/test_lang_guess.py
python tests/test_calibration_persistence.py
diff --git a/README.md b/README.md
index 6b25f7a5..58f11db5 100644
--- a/README.md
+++ b/README.md
@@ -649,6 +649,13 @@ policy gradient), fit calibration temperatures, evaluate, and push the result to
* **[`notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb`](notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb)**
+The notebook enables gradient checkpointing on both the encoder and the decision head.
+For custom training loops, `model.head_checkpointing = True` enables activation
+checkpointing for the decision-head layers; enable the encoder's gradient checkpointing
+separately. During gradient-enabled training, this reduces stored intermediate activations
+by recomputing them during backward, trading extra computation for lower activation memory.
+The head flag defaults to `False` and is bypassed in evaluation and under `torch.no_grad()`.
+
The notebook fits one `temperature` per type (`choice`, `score`, `noul`) and removes inherited
`temperature_by_options` from the exported config. Otherwise those old bucket values take
precedence at inference and silently mask the new fit. Existing checkpoints still honor
diff --git a/laya/common.py b/laya/common.py
index 950c41df..9003e08f 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -7,6 +7,7 @@ from typing import Dict, List, Optional, Union
import numpy as np
import torch
import torch.nn as nn
+from torch.utils.checkpoint import checkpoint
QTYPES = {"choice": 0, "score": 1, "noul": 2}
QTYPE_NAMES = {v: k for k, v in QTYPES.items()}
@@ -110,7 +111,12 @@ class DecisionModel(nn.Module):
if self.head is not None:
pad = ~attention_mask.bool()
for layer in self.head.layers:
- h = layer(h, src_key_padding_mask=pad)
+ if self.head_checkpointing and self.training and torch.is_grad_enabled():
+ # Non-reentrant checkpointing also trains the head when its input
+ # is frozen. Default RNG preservation keeps dropout consistent.
+ h = checkpoint(layer, h, src_key_padding_mask=pad, use_reentrant=False)
+ else:
+ h = layer(h, src_key_padding_mask=pad)
idx = marker_pos.clamp(min=0)[:, :, None].expand(-1, -1, h.size(-1))
m = torch.gather(h, 1, idx)
logits = self.scorer(m).squeeze(-1).float()
diff --git a/tests/test_head_checkpointing.py b/tests/test_head_checkpointing.py
new file mode 100644
index 00000000..2c850201
--- /dev/null
+++ b/tests/test_head_checkpointing.py
@@ -0,0 +1,152 @@
+"""Decision-head checkpointing regressions with a tiny BERT; no downloads.
+
+Run: python tests/test_head_checkpointing.py
+"""
+import copy
+import os
+from pathlib import Path
+import sys
+import unittest
+from unittest.mock import patch
+
+os.environ.setdefault("USE_TF", "0")
+os.environ.setdefault("USE_TORCH", "1")
+os.environ.setdefault("HF_HUB_OFFLINE", "1")
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import torch # noqa: E402
+from transformers import BertConfig, BertModel # noqa: E402
+
+from laya.common import DecisionModel # noqa: E402
+
+
+def tiny_model(head_layers=2):
+ torch.manual_seed(41)
+ config = BertConfig(vocab_size=64, hidden_size=16, num_hidden_layers=1,
+ num_attention_heads=1, intermediate_size=32)
+ model = DecisionModel(BertModel(config), head_layers=head_layers, dropout=0.2)
+ if model.head is not None:
+ # TransformerEncoder clones its layers. Distinct weights catch accidentally
+ # recomputing the last layer for every iteration of the forward loop.
+ with torch.no_grad():
+ for i, layer in enumerate(model.head.layers):
+ layer.linear1.weight.add_(0.1 * i)
+ return model
+
+
+def inputs():
+ torch.manual_seed(42)
+ ids = torch.randint(0, 64, (3, 12))
+ attention = torch.ones_like(ids)
+ attention[0, 9:] = 0
+ attention[1, 11:] = 0
+ positions = torch.tensor([[2, 0, 0], [2, 5, 0], [2, 5, 8]])
+ markers = torch.tensor([[1, 0, 0], [1, 1, 0], [1, 1, 1]], dtype=torch.bool)
+ return ids, attention, positions, markers, torch.tensor([0, 2, 1])
+
+
+def training_step(model, *, detach_encoder=False):
+ batch = inputs()
+ optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
+ optimizer.zero_grad(set_to_none=True)
+ torch.manual_seed(123)
+ logits, act = model(*batch, detach_encoder=detach_encoder)
+ loss = logits.masked_select(batch[3]).square().mean() + act.square().mean()
+ loss.backward()
+ gradients = {name: p.grad.detach().clone() if p.grad is not None else None
+ for name, p in model.named_parameters()}
+ optimizer.step()
+ return logits.detach(), act.detach(), loss.detach(), gradients, torch.get_rng_state()
+
+
+class HeadCheckpointingTests(unittest.TestCase):
+ def assert_same_step(self, model, **kwargs):
+ ordinary = copy.deepcopy(model).train()
+ checkpointed = copy.deepcopy(model).train()
+ ordinary.head_checkpointing = False
+ checkpointed.head_checkpointing = True
+ expected = training_step(ordinary, **kwargs)
+ actual = training_step(checkpointed, **kwargs)
+ for before, after in zip(expected[:3], actual[:3]):
+ torch.testing.assert_close(before, after, rtol=1e-5, atol=1e-6)
+ for name, gradient in expected[3].items():
+ other = actual[3][name]
+ with self.subTest(parameter=name):
+ if gradient is None:
+ self.assertIsNone(other)
+ else:
+ self.assertIsNotNone(other)
+ torch.testing.assert_close(gradient, other, rtol=1e-5, atol=1e-6)
+ self.assertTrue(torch.equal(expected[4], actual[4]), "dropout RNG state changed")
+ for name, value in ordinary.state_dict().items():
+ torch.testing.assert_close(value, checkpointed.state_dict()[name], rtol=1e-5, atol=1e-6)
+ return actual[3]
+
+ def test_enabled_head_recomputes_each_layer_during_backward(self):
+ for enabled in (False, True):
+ with self.subTest(enabled=enabled):
+ model = tiny_model().train()
+ model.head_checkpointing = enabled
+ calls = [0, 0]
+ handles = []
+ for i, layer in enumerate(model.head.layers):
+ def count(_module, _args, index=i):
+ calls[index] += 1
+ handles.append(layer.register_forward_pre_hook(count))
+ try:
+ training_step(model)
+ finally:
+ for handle in handles:
+ handle.remove()
+ self.assertEqual(calls, [2, 2] if enabled else [1, 1])
+
+ def test_training_outputs_gradients_and_updates_match_with_dropout(self):
+ self.assert_same_step(tiny_model())
+
+ def test_encoder_and_head_checkpointing_work_together(self):
+ model = tiny_model()
+ model.encoder.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
+ self.assert_same_step(model)
+
+ def test_detached_encoder_keeps_head_gradients(self):
+ gradients = self.assert_same_step(tiny_model(), detach_encoder=True)
+ self.assertTrue(all(g is None for n, g in gradients.items() if n.startswith("encoder.")))
+ self.assertIsNotNone(gradients["head.layers.0.linear1.weight"])
+
+ def test_frozen_head_inputs_still_train_head_parameters(self):
+ model = tiny_model()
+ for parameter in list(model.encoder.parameters()) + list(model.type_emb.parameters()):
+ parameter.requires_grad_(False)
+ gradients = self.assert_same_step(model)
+ for i in range(2):
+ gradient = gradients["head.layers.%d.linear1.weight" % i]
+ self.assertIsNotNone(gradient)
+ self.assertGreater(gradient.abs().sum().item(), 0.0)
+
+ def test_eval_and_no_grad_bypass_checkpointing(self):
+ for training, grad_enabled in ((False, True), (False, False), (True, False)):
+ with self.subTest(training=training, grad_enabled=grad_enabled):
+ model = tiny_model().train(training)
+ batch = inputs()
+ with torch.set_grad_enabled(grad_enabled):
+ torch.manual_seed(123)
+ expected = model(*batch)
+ model.head_checkpointing = True
+ with patch("laya.common.checkpoint", side_effect=AssertionError("unexpected checkpoint")):
+ torch.manual_seed(123)
+ actual = model(*batch)
+ torch.testing.assert_close(expected, actual, rtol=0, atol=0)
+
+ def test_zero_head_layers_remain_supported(self):
+ self.assert_same_step(tiny_model(head_layers=0))
+
+ def test_default_is_disabled(self):
+ model = tiny_model().train()
+ self.assertFalse(model.head_checkpointing)
+ with patch("laya.common.checkpoint", side_effect=AssertionError("unexpected checkpoint")):
+ training_step(model)
+
+
+if __name__ == "__main__":
+ torch.set_num_threads(1)
+ unittest.main()