Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

6.4 KB
Raw Blame
"""Decision-head checkpointing regressions with a tiny BERT; no downloads.

Run: python tests/test_head_checkpointing.py
"""
import copy
import os
from pathlib import Path
import sys
import unittest
from unittest.mock import patch

os.environ.setdefault("USE_TF", "0")
os.environ.setdefault("USE_TORCH", "1")
os.environ.setdefault("HF_HUB_OFFLINE", "1")
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

import torch  # noqa: E402
from transformers import BertConfig, BertModel  # noqa: E402

from laya.common import DecisionModel  # noqa: E402


def tiny_model(head_layers=2):
    torch.manual_seed(41)
    config = BertConfig(vocab_size=64, hidden_size=16, num_hidden_layers=1,
                        num_attention_heads=1, intermediate_size=32)
    model = DecisionModel(BertModel(config), head_layers=head_layers, dropout=0.2)
    if model.head is not None:
        # TransformerEncoder clones its layers. Distinct weights catch accidentally
        # recomputing the last layer for every iteration of the forward loop.
        with torch.no_grad():
            for i, layer in enumerate(model.head.layers):
                layer.linear1.weight.add_(0.1 * i)
    return model


def inputs():
    torch.manual_seed(42)
    ids = torch.randint(0, 64, (3, 12))
    attention = torch.ones_like(ids)
    attention[0, 9:] = 0
    attention[1, 11:] = 0
    positions = torch.tensor([[2, 0, 0], [2, 5, 0], [2, 5, 8]])
    markers = torch.tensor([[1, 0, 0], [1, 1, 0], [1, 1, 1]], dtype=torch.bool)
    return ids, attention, positions, markers, torch.tensor([0, 2, 1])


def training_step(model, *, detach_encoder=False):
    batch = inputs()
    optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
    optimizer.zero_grad(set_to_none=True)
    torch.manual_seed(123)
    logits, act = model(*batch, detach_encoder=detach_encoder)
    loss = logits.masked_select(batch[3]).square().mean() + act.square().mean()
    loss.backward()
    gradients = {name: p.grad.detach().clone() if p.grad is not None else None
                 for name, p in model.named_parameters()}
    optimizer.step()
    return logits.detach(), act.detach(), loss.detach(), gradients, torch.get_rng_state()


class HeadCheckpointingTests(unittest.TestCase):
    def assert_same_step(self, model, **kwargs):
        ordinary = copy.deepcopy(model).train()
        checkpointed = copy.deepcopy(model).train()
        ordinary.head_checkpointing = False
        checkpointed.head_checkpointing = True
        expected = training_step(ordinary, **kwargs)
        actual = training_step(checkpointed, **kwargs)
        for before, after in zip(expected[:3], actual[:3]):
            torch.testing.assert_close(before, after, rtol=1e-5, atol=1e-6)
        for name, gradient in expected[3].items():
            other = actual[3][name]
            with self.subTest(parameter=name):
                if gradient is None:
                    self.assertIsNone(other)
                else:
                    self.assertIsNotNone(other)
                    torch.testing.assert_close(gradient, other, rtol=1e-5, atol=1e-6)
        self.assertTrue(torch.equal(expected[4], actual[4]), "dropout RNG state changed")
        for name, value in ordinary.state_dict().items():
            torch.testing.assert_close(value, checkpointed.state_dict()[name], rtol=1e-5, atol=1e-6)
        return actual[3]

    def test_enabled_head_recomputes_each_layer_during_backward(self):
        for enabled in (False, True):
            with self.subTest(enabled=enabled):
                model = tiny_model().train()
                model.head_checkpointing = enabled
                calls = [0, 0]
                handles = []
                for i, layer in enumerate(model.head.layers):
                    def count(_module, _args, index=i):
                        calls[index] += 1
                    handles.append(layer.register_forward_pre_hook(count))
                try:
                    training_step(model)
                finally:
                    for handle in handles:
                        handle.remove()
                self.assertEqual(calls, [2, 2] if enabled else [1, 1])

    def test_training_outputs_gradients_and_updates_match_with_dropout(self):
        self.assert_same_step(tiny_model())

    def test_encoder_and_head_checkpointing_work_together(self):
        model = tiny_model()
        model.encoder.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
        self.assert_same_step(model)

    def test_detached_encoder_keeps_head_gradients(self):
        gradients = self.assert_same_step(tiny_model(), detach_encoder=True)
        self.assertTrue(all(g is None for n, g in gradients.items() if n.startswith("encoder.")))
        self.assertIsNotNone(gradients["head.layers.0.linear1.weight"])

    def test_frozen_head_inputs_still_train_head_parameters(self):
        model = tiny_model()
        for parameter in list(model.encoder.parameters()) + list(model.type_emb.parameters()):
            parameter.requires_grad_(False)
        gradients = self.assert_same_step(model)
        for i in range(2):
            gradient = gradients["head.layers.%d.linear1.weight" % i]
            self.assertIsNotNone(gradient)
            self.assertGreater(gradient.abs().sum().item(), 0.0)

    def test_eval_and_no_grad_bypass_checkpointing(self):
        for training, grad_enabled in ((False, True), (False, False), (True, False)):
            with self.subTest(training=training, grad_enabled=grad_enabled):
                model = tiny_model().train(training)
                batch = inputs()
                with torch.set_grad_enabled(grad_enabled):
                    torch.manual_seed(123)
                    expected = model(*batch)
                    model.head_checkpointing = True
                    with patch("laya.common.checkpoint", side_effect=AssertionError("unexpected checkpoint")):
                        torch.manual_seed(123)
                        actual = model(*batch)
                torch.testing.assert_close(expected, actual, rtol=0, atol=0)

    def test_zero_head_layers_remain_supported(self):
        self.assert_same_step(tiny_model(head_layers=0))

    def test_default_is_disabled(self):
        model = tiny_model().train()
        self.assertFalse(model.head_checkpointing)
        with patch("laya.common.checkpoint", side_effect=AssertionError("unexpected checkpoint")):
            training_step(model)


if __name__ == "__main__":
    torch.set_num_threads(1)
    unittest.main()