Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

20.1 KB
Raw Blame
"""Layer streaming: ``training.stream_nf4_scale_cache`` keeps expanded NF4 scales resident.

A double-quantised NF4 weight stores its per-block absmax as uint8 codes plus a
nested absmax and offset. On bitsandbytes' dequantise + linear path every visit
first expands those to fp32 (``dequantize_blockwise`` + ``+= offset``) and only
then dequantises the 4-bit weight: two extra kernels and a temporary per projection,
on the forward and again on the checkpoint recompute. The base is frozen, so the
fp32 scales of a (layer, weight) never change. With the opt-in on, the streamed
layer expands them once and keeps them on the compute device, at a persistent
cost of 4 bytes per 64-element block (416 MiB for Mistral-7B) that the VRAM
pre-flight charges up front.

The fused custom-GEMM path is left alone: it dequantises the nested scales inside
its kernel, and feeding it pre-expanded scales changes the arithmetic.
"""

import pytest

from soup_cli.config.loader import load_config_from_string


def _cfg(**training):
    body = "\n".join(f"  {k}: {v}" for k, v in training.items())
    return (
        "base: HuggingFaceTB/SmolLM2-135M\n"
        "task: sft\n"
        "data:\n"
        "  train: d.jsonl\n"
        "  max_length: 512\n"
        "training:\n"
        "  batch_size: 1\n"
        "  lora:\n"
        "    r: 8\n" + body + "\n"
    )


class TestSchema:
    def test_defaults_off(self):
        cfg = load_config_from_string(_cfg(stream_layers="true", quantization="4bit"))
        assert cfg.training.stream_nf4_scale_cache is False

    def test_parses_with_streamed_nf4(self):
        cfg = load_config_from_string(
            _cfg(stream_layers="true", quantization="4bit", stream_nf4_scale_cache="true")
        )
        assert cfg.training.stream_nf4_scale_cache is True

    def test_refused_while_streaming_is_off(self):
        with pytest.raises(Exception) as exc:
            load_config_from_string(_cfg(stream_layers="false", stream_nf4_scale_cache="true"))
        assert "stream_nf4_scale_cache" in str(exc.value)
        assert "stream_layers" in str(exc.value)

    def test_refused_without_4bit(self):
        with pytest.raises(Exception) as exc:
            load_config_from_string(
                _cfg(stream_layers="true", quantization="none", stream_nf4_scale_cache="true")
            )
        message = str(exc.value)
        assert "stream_nf4_scale_cache" in message
        assert "quantization='4bit'" in message

    def test_refused_without_double_quant(self):
        with pytest.raises(Exception) as exc:
            load_config_from_string(
                _cfg(
                    stream_layers="true",
                    quantization="4bit",
                    bnb_4bit_use_double_quant="false",
                    stream_nf4_scale_cache="true",
                )
            )
        message = str(exc.value)
        assert "stream_nf4_scale_cache" in message
        assert "bnb_4bit_use_double_quant" in message


def _spec(shape, nested=True, blocksize=64):
    from soup_cli.utils.layer_shard import NF4WeightSpec

    return NF4WeightSpec(
        shape=tuple(shape),
        dtype="bfloat16",
        blocksize=blocksize,
        quant_type="nf4",
        nested=nested,
        nested_blocksize=256,
    )


class TestCacheBytes:
    def test_mistral_7b_is_416_mib(self):
        from soup_cli.utils.layer_stream import nf4_scale_cache_bytes

        hidden, kv, inter = 4096, 1024, 14336
        specs = {
            "self_attn.q_proj.weight": _spec((hidden, hidden)),
            "self_attn.k_proj.weight": _spec((kv, hidden)),
            "self_attn.v_proj.weight": _spec((kv, hidden)),
            "self_attn.o_proj.weight": _spec((hidden, hidden)),
            "mlp.gate_proj.weight": _spec((inter, hidden)),
            "mlp.up_proj.weight": _spec((inter, hidden)),
            "mlp.down_proj.weight": _spec((hidden, inter)),
        }
        assert nf4_scale_cache_bytes(specs, n_layers=32) == 436_207_616

    def test_a_partial_block_rounds_up(self):
        from soup_cli.utils.layer_stream import nf4_scale_cache_bytes

        # 65 elements -> 2 blocks of 64 -> 8 bytes per layer.
        assert nf4_scale_cache_bytes({"w": _spec((5, 13))}, n_layers=3) == 24

    def test_non_nested_weights_cost_nothing(self):
        from soup_cli.utils.layer_stream import nf4_scale_cache_bytes

        specs = {"a": _spec((64, 64), nested=False), "b": _spec((64, 64))}
        assert nf4_scale_cache_bytes(specs, n_layers=2) == 2 * 64 * 4

    def test_the_block_size_is_read_from_the_spec(self):
        from soup_cli.utils.layer_stream import nf4_scale_cache_bytes

        assert nf4_scale_cache_bytes({"w": _spec((128, 128), blocksize=128)}, n_layers=1) == 512


class TestPeakEstimate:
    def _estimate(self, **extra):
        from soup_cli.utils.layer_stream import estimate_stream_peak_vram

        return estimate_stream_peak_vram(
            layer_bytes=1000,
            buffers=2,
            extras_bytes=0,
            adapter_params=10,
            vocab_size=64,
            hidden_size=8,
            intermediate_size=16,
            n_layers=4,
            seq_len=4,
            **extra,
        )

    def test_persistent_bytes_add_exactly(self):
        assert self._estimate(persistent_bytes=12_345) == self._estimate() + 12_345

    def test_default_adds_nothing(self):
        assert self._estimate(persistent_bytes=0) == self._estimate()


class _Lora:
    r = 8
    target_modules = ["q_proj", "v_proj"]


class _ModelConfig:
    vocab_size = 64
    hidden_size = 8
    intermediate_size = 16
    num_hidden_layers = 4


class _Data:
    max_length = 4


class _Config:
    data = _Data()


class _TrainConfig:
    batch_size = 1
    stream_buffers = 2
    stream_vram_probe = False
    lora = _Lora()

    def __init__(self, cache):
        self.stream_nf4_scale_cache = cache


class _Index:
    n_layers = 4

    def __init__(self):
        self.quant_specs = {"self_attn.q_proj.weight": _spec((64, 64))}


class TestPreflightCharge:
    def _captured_persistent_bytes(self, monkeypatch, cache):
        from soup_cli.trainer.stream_setup import StreamingSetupMixin
        from soup_cli.utils import layer_stream

        real = layer_stream.estimate_stream_peak_vram
        seen = {}

        def capture(**kwargs):
            seen.update(kwargs)
            return real(**kwargs)

        monkeypatch.setattr(layer_stream, "estimate_stream_peak_vram", capture)
        lines, _plan = StreamingSetupMixin()._stream_budget_lines(
            _Config(),
            _TrainConfig(cache),
            model_config=_ModelConfig(),
            layer_bytes=1000,
            embed_bytes=0,
            index=_Index(),
            on_cuda=False,
        )
        return seen.get("persistent_bytes", 0), "\n".join(lines)

    def test_the_cache_is_charged_when_on(self, monkeypatch):
        charged, text = self._captured_persistent_bytes(monkeypatch, cache=True)
        # 4 layers x 64 blocks x 4 bytes.
        assert charged == 4 * 64 * 4
        assert "stream_nf4_scale_cache" in text

    def test_nothing_is_charged_when_off(self, monkeypatch):
        charged, text = self._captured_persistent_bytes(monkeypatch, cache=False)
        assert charged == 0
        assert "stream_nf4_scale_cache" not in text


def _nf4_stream(tmp_path, *, cache, n_layers=2, double_quant=True, device="cpu", dtype="float32"):
    pytest.importorskip("torch")
    pytest.importorskip("bitsandbytes")
    pytest.importorskip("peft")
    from soup_cli.utils.layer_shard import QUANT_NF4, shard_checkpoint
    from soup_cli.utils.layer_stream_runtime import (
        build_meta_skeleton,
        build_streamed_model,
        quantised_layer_suffixes,
    )
    from tests.test_v07202 import _tiny_llama_dir, _tiny_lora

    weights, _resident, _ = _tiny_llama_dir(tmp_path, n_layers=n_layers)
    probe = build_meta_skeleton(weights, dtype=dtype, quant=QUANT_NF4)
    suffixes = quantised_layer_suffixes(probe)
    del probe
    shards = str(tmp_path / "shards")
    index = shard_checkpoint(
        weights,
        shards,
        dtype=dtype,
        arch="llama",
        quant=QUANT_NF4,
        quant_suffixes=suffixes,
        quant_device=device,
        double_quant=double_quant,
    )
    model, runtime = build_streamed_model(
        model_id=weights,
        shard_dir=shards,
        index=index,
        lora_config=_tiny_lora(),
        device=device,
        dtype=dtype,
        buffers=2,
        pin=False,
        seed=3,
        quant=QUANT_NF4,
        double_quant=double_quant,
        nf4_scale_cache=cache,
    )
    return model, runtime, index


def _count_blockwise_dequant(monkeypatch):
    import bitsandbytes.functional as bnb_functional

    real = bnb_functional.dequantize_blockwise
    calls = {"n": 0}

    def counting(*args, **kwargs):
        calls["n"] += 1
        return real(*args, **kwargs)

    monkeypatch.setattr(bnb_functional, "dequantize_blockwise", counting)
    return calls


def _step(model, optimizer, ids):
    optimizer.zero_grad(set_to_none=True)
    out = model(input_ids=ids, labels=ids)
    out.loss.backward()
    grads = [p.grad.detach().clone() for p in model.parameters() if p.requires_grad]
    optimizer.step()
    return out.logits.detach().clone(), out.loss.detach().clone(), grads


class TestRuntime:
    def test_off_builds_no_cache(self, tmp_path):
        _model, runtime, _index = _nf4_stream(tmp_path, cache=False)
        assert runtime.scale_cache is None
        assert runtime.stats()["nf4_scale_cache_bytes"] == 0

    def test_one_step_fills_every_nested_weight_once(self, tmp_path):
        import torch

        from soup_cli.utils.layer_stream import nf4_scale_cache_bytes

        model, runtime, index = _nf4_stream(tmp_path, cache=True)
        nested = sum(1 for spec in index.quant_specs.values() if spec.nested)
        assert nested > 0
        assert len(runtime.scale_cache) == 0

        ids = torch.randint(0, 64, (1, 8), generator=torch.Generator().manual_seed(0))
        model(input_ids=ids, labels=ids).loss.backward()

        assert len(runtime.scale_cache) == index.n_layers * nested
        # Every layer is filled, so the planned bytes are exactly what is held.
        assert runtime.scale_cache.nbytes == nf4_scale_cache_bytes(
            index.quant_specs, n_layers=index.n_layers
        )
        assert runtime.stats()["nf4_scale_cache_bytes"] == runtime.scale_cache.nbytes

    def test_a_filled_cache_skips_the_nested_expansion(self, tmp_path, monkeypatch):
        import torch

        model, runtime, _index = _nf4_stream(tmp_path, cache=True)
        ids = torch.randint(0, 64, (1, 8), generator=torch.Generator().manual_seed(0))
        model(input_ids=ids, labels=ids).loss.backward()
        filled = len(runtime.scale_cache)

        calls = _count_blockwise_dequant(monkeypatch)
        model(input_ids=ids, labels=ids).loss.backward()
        assert calls["n"] == 0
        assert len(runtime.scale_cache) == filled

    def test_without_the_cache_every_visit_expands(self, tmp_path, monkeypatch):
        """The control for the test above: the counter does see the expansion."""
        import torch

        model, _runtime, _index = _nf4_stream(tmp_path, cache=False)
        ids = torch.randint(0, 64, (1, 8), generator=torch.Generator().manual_seed(0))
        calls = _count_blockwise_dequant(monkeypatch)
        model(input_ids=ids, labels=ids).loss.backward()
        assert calls["n"] > 0

    def test_training_is_bit_identical_with_and_without_the_cache(self, tmp_path):
        import torch

        runs = {}
        for cache in (False, True):
            torch.manual_seed(0)
            model, _runtime, _index = _nf4_stream(tmp_path / str(cache), cache=cache)
            optimizer = torch.optim.AdamW(
                [p for p in model.parameters() if p.requires_grad], lr=1e-2
            )
            generator = torch.Generator().manual_seed(1)
            steps = []
            for _ in range(3):
                ids = torch.randint(0, 64, (1, 8), generator=generator)
                steps.append(_step(model, optimizer, ids))
            runs[cache] = steps

        nonzero = False
        for (logits_a, loss_a, grads_a), (logits_b, loss_b, grads_b) in zip(
            runs[False], runs[True]
        ):
            assert torch.equal(logits_a, logits_b)
            assert torch.equal(loss_a, loss_b)
            assert len(grads_a) == len(grads_b)
            for grad_a, grad_b in zip(grads_a, grads_b):
                assert torch.equal(grad_a, grad_b)
                nonzero = nonzero or bool(grad_a.abs().sum() > 0)
        assert nonzero, "vacuous: every gradient was zero"

    def test_close_releases_the_cache(self, tmp_path):
        import torch

        model, runtime, _index = _nf4_stream(tmp_path, cache=True)
        ids = torch.randint(0, 64, (1, 8), generator=torch.Generator().manual_seed(0))
        model(input_ids=ids, labels=ids).loss.backward()
        assert len(runtime.scale_cache) > 0

        runtime.close()
        assert len(runtime.scale_cache) == 0
        assert runtime.scale_cache.nbytes == 0

    def test_the_fused_path_keeps_the_nested_state(self, tmp_path, monkeypatch):
        """The custom GEMM folds the nested dequantisation into its kernel; giving
        it pre-expanded scales changed the loss in the study this ports. The fused
        arm must receive the original nested state and must not fill the cache."""
        import torch
        from bitsandbytes.functional import dequantize_4bit

        import soup_cli.utils.layer_stream_runtime as runtime_module

        model, runtime, _index = _nf4_stream(tmp_path, cache=True)
        seen = []

        def fake_fused(x, packed, quant_state, bias=None):
            seen.append(bool(quant_state.nested))
            weight = dequantize_4bit(packed, quant_state)
            return torch.nn.functional.linear(x, weight.to(x.dtype), bias)

        monkeypatch.setattr(
            runtime_module, "_can_use_checkpoint_visible_nf4_gemm", lambda *_a, **_k: True
        )
        monkeypatch.setattr(runtime_module, "checkpoint_visible_nf4_linear", fake_fused)

        ids = torch.randint(0, 64, (1, 8), generator=torch.Generator().manual_seed(0))
        with torch.no_grad():
            model(input_ids=ids)
        assert seen and all(seen)
        assert len(runtime.scale_cache) == 0


class TestEdges:
    def test_shards_without_double_quant_are_refused(self, tmp_path):
        with pytest.raises(ValueError, match="no nested NF4 weights"):
            _nf4_stream(tmp_path, cache=True, double_quant=False)

    def test_a_key_reused_for_another_shape_is_refused(self):
        torch = pytest.importorskip("torch")
        pytest.importorskip("bitsandbytes")
        from bitsandbytes.functional import quantize_4bit

        from soup_cli.utils.layer_stream_runtime import NF4ScaleCache

        cache = NF4ScaleCache()
        _packed, first = quantize_4bit(
            torch.randn(64, 64), compress_statistics=True, quant_type="nf4"
        )
        _packed, second = quantize_4bit(
            torch.randn(32, 128), compress_statistics=True, quant_type="nf4"
        )
        cache.dense_state((0, "w"), first)
        with pytest.raises(RuntimeError, match="was built for shape"):
            cache.dense_state((0, "w"), second)

    def test_filled_under_inference_mode_then_trained(self, tmp_path):
        """An eval or generation pass may be the first to visit a layer; the
        scales it leaves behind must serve training unchanged."""
        import torch

        runs = {}
        for cache in (False, True):
            torch.manual_seed(0)
            model, _runtime, _index = _nf4_stream(tmp_path / str(cache), cache=cache)
            ids = torch.randint(0, 64, (1, 8), generator=torch.Generator().manual_seed(2))
            with torch.inference_mode():
                model(input_ids=ids)
            optimizer = torch.optim.AdamW(
                [p for p in model.parameters() if p.requires_grad], lr=1e-2
            )
            runs[cache] = [_step(model, optimizer, ids) for _ in range(2)]

        for (logits_a, loss_a, grads_a), (logits_b, loss_b, grads_b) in zip(
            runs[False], runs[True]
        ):
            assert torch.equal(logits_a, logits_b)
            assert torch.equal(loss_a, loss_b)
            for grad_a, grad_b in zip(grads_a, grads_b):
                assert torch.equal(grad_a, grad_b)


class TestSetupWiring:
    """The real setup path, from YAML to the runtime builder (spied), on CPU."""

    @pytest.mark.parametrize("flag", [True, False])
    def test_the_setting_reaches_the_runtime_builder(self, tmp_path, monkeypatch, flag):
        pytest.importorskip("torch")
        pytest.importorskip("bitsandbytes")
        from unittest.mock import MagicMock

        import soup_cli.utils.layer_stream as ls
        import soup_cli.utils.layer_stream_runtime as rt
        from soup_cli.trainer.sft import SFTTrainerWrapper
        from tests.test_issue623_stream_pin_allocation import (
            _FREE_BYTES,
            _cfg_yaml,
            _tiny_checkpoint,
        )

        weights = _tiny_checkpoint(tmp_path)
        monkeypatch.chdir(tmp_path)
        monkeypatch.setenv("SOUP_LAYER_STREAM_CACHE_DIR", str(tmp_path / "cache"))
        monkeypatch.setattr(
            "soup_cli.utils.spectrum_scan.resolve_model_weights", lambda *_a, **_k: weights
        )
        monkeypatch.setattr(ls, "free_ram_bytes", lambda: _FREE_BYTES)
        monkeypatch.setattr(
            ls, "classify_disk_kind", lambda *_a, **_k: ls.DiskClassification("nvme")
        )
        captured = {}

        def spy_build(**kwargs):
            captured.update(kwargs)
            runtime = MagicMock()
            runtime.tier = "ram"
            runtime.stats.return_value = {
                "tier": "ram",
                "store_bytes": 1_000_000,
                "disk_bytes": 0,
                "pinned": False,
                "read_ahead": None,
                "buffers": 2,
                "buffer_bytes": 4_000,
                "n_layers": 3,
            }
            return MagicMock(), runtime

        monkeypatch.setattr(rt, "build_streamed_model", spy_build)
        text = _cfg_yaml(
            weights, "", extra_training_yaml=f"  stream_nf4_scale_cache: {str(flag).lower()}\n"
        ).replace("quantization: none", "quantization: 4bit")
        cfg = load_config_from_string(text)
        wrapper = SFTTrainerWrapper(cfg)
        wrapper.device = "cpu"
        wrapper._setup_streaming_transformers(cfg, cfg.training)

        assert captured["nf4_scale_cache"] is flag


class TestCuda:
    """bitsandbytes' CUDA kernels on the dense path, with and without the cache."""

    @pytest.mark.gpu
    @pytest.mark.parametrize("force_dense", [True, False])
    def test_bit_identical_on_cuda(self, tmp_path, monkeypatch, force_dense):
        torch = pytest.importorskip("torch")
        import soup_cli.utils.layer_stream_runtime as runtime_module

        if force_dense:
            # Tiny shapes would otherwise go to the fused GEMM, which the cache
            # never touches; forcing the dense arm is what puts the cache to work.
            monkeypatch.setattr(
                runtime_module, "_can_use_checkpoint_visible_nf4_gemm", lambda *_a, **_k: False
            )
        runs = {}
        for cache in (False, True):
            torch.manual_seed(0)
            model, runtime, _index = _nf4_stream(
                tmp_path / str(cache), cache=cache, device="cuda", dtype="bfloat16"
            )
            optimizer = torch.optim.AdamW(
                [p for p in model.parameters() if p.requires_grad], lr=1e-2
            )
            generator = torch.Generator().manual_seed(1)
            steps = []
            for _ in range(3):
                ids = torch.randint(0, 64, (1, 8), generator=generator).cuda()
                steps.append(_step(model, optimizer, ids))
            runs[cache] = steps
            if cache:
                filled = len(runtime.scale_cache)
            runtime.close()

        assert (filled > 0) is force_dense
        for (logits_a, loss_a, grads_a), (logits_b, loss_b, grads_b) in zip(
            runs[False], runs[True]
        ):
            assert torch.equal(logits_a, logits_b)
            assert torch.equal(loss_a, loss_b)
            for grad_a, grad_b in zip(grads_a, grads_b):
                assert torch.equal(grad_a, grad_b)