Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

10.9 KB
Raw Blame
"""#1241 — `soup shrink` must keep per-layer config lists in step with the layers.

`prune_model_layers` drops a block of decoder layers and patches
`config.num_hidden_layers`. Qwen2, Qwen3, Qwen2-MoE and SmolLM3 configs also
carry per-layer lists (`layer_types`, and SmolLM3's `no_rope_layers`). Left at
the old length, transformers 5 refuses to save the pruned model, so the shrink
run dies after scoring with nothing written.

The lists must be sliced with the same kept indices as the layers, not cut at
the tail: a mixed full/sliding Qwen2 list and SmolLM3's `no_rope_layers` both
have a different entry in the dropped block than at the end, so a tail cut
fails here.
"""
import pytest

pytest.importorskip("torch")
transformers = pytest.importorskip("transformers")

_COMMON = dict(
    vocab_size=64,
    hidden_size=16,
    intermediate_size=32,
    num_hidden_layers=6,
    num_attention_heads=2,
    num_key_value_heads=2,
)

_LINEAR_ATTN = dict(
    linear_num_key_heads=2,
    linear_num_value_heads=2,
    linear_key_head_dim=8,
    linear_value_head_dim=8,
)

_CASES = [
    ("LlamaConfig", {}),
    ("Qwen2Config", {}),
    (
        "Qwen2Config",
        {"use_sliding_window": True, "max_window_layers": 3, "sliding_window": 8},
    ),
    ("Qwen3Config", {"head_dim": 8}),
    (
        "Qwen2MoeConfig",
        {
            "num_experts": 2,
            "num_experts_per_tok": 1,
            "moe_intermediate_size": 16,
            "shared_expert_intermediate_size": 16,
        },
    ),
    (
        "Qwen3MoeConfig",
        {"head_dim": 8, "num_experts": 2, "num_experts_per_tok": 1, "moe_intermediate_size": 16},
    ),
    ("SmolLM3Config", {"pad_token_id": 0, "bos_token_id": 1, "eos_token_id": 2}),
    # Hybrid: `layer_types` decides whether each layer is built with linear or
    # full attention, so a wrong slice changes the architecture, not a flag.
    ("Qwen3_5TextConfig", {"head_dim": 8, **_LINEAR_ATTN}),
]

# (start, block_size) on 6 layers: first allowed position, middle, ending at
# n - 1, and the largest block the first/last-layer guard allows.
_DROPS = [(1, 1), (2, 2), (4, 1), (1, 4)]


def _per_layer_lists(config, n_layers: int) -> dict:
    return {
        key: list(value)
        for key, value in vars(config).items()
        if isinstance(value, (list, tuple)) and len(value) == n_layers
    }


@pytest.mark.parametrize("start,block_size", _DROPS)
@pytest.mark.parametrize(
    "config_name,extra", _CASES, ids=[f"{name}-{i}" for i, (name, _) in enumerate(_CASES)]
)
def test_prune_slices_per_layer_lists_and_round_trips(
    tmp_path, config_name, extra, start, block_size
):
    import torch
    from transformers import AutoConfig, AutoModelForCausalLM

    from soup_cli.utils.shrink import prune_model_layers

    config = getattr(transformers, config_name)(**_COMMON, **extra)
    torch.manual_seed(0)
    model = AutoModelForCausalLM.from_config(config)
    before = _per_layer_lists(model.config, 6)
    kept = [i for i in range(6) if not (start <= i < start + block_size)]

    prune_model_layers(model, start, block_size)

    for key, old in before.items():
        assert list(getattr(model.config, key)) == [old[i] for i in kept], key

    model.save_pretrained(str(tmp_path))
    reloaded = AutoConfig.from_pretrained(str(tmp_path))
    assert reloaded.num_hidden_layers == len(kept)
    for key, old in before.items():
        assert list(getattr(reloaded, key)) == [old[i] for i in kept], key
    AutoModelForCausalLM.from_pretrained(str(tmp_path))


@pytest.mark.parametrize(
    "config_name,extra", _CASES, ids=[f"{name}-{i}" for i, (name, _) in enumerate(_CASES)]
)
def test_every_per_layer_list_is_in_the_declared_table(config_name, extra):
    """A per-layer list a supported config carries, missing from the table,
    would be left at the old length by the prune — the next #1241."""
    from soup_cli.utils.shrink import _PER_LAYER_CONFIG_ATTRS

    config = getattr(transformers, config_name)(**_COMMON, **extra)
    assert set(_per_layer_lists(config, 6)) <= set(_PER_LAYER_CONFIG_ATTRS)


_QWEN2_MOE = (
    "Qwen2MoeConfig",
    {
        "num_experts": 2,
        "num_experts_per_tok": 1,
        "moe_intermediate_size": 16,
        "shared_expert_intermediate_size": 16,
    },
)
_QWEN3_MOE = (
    "Qwen3MoeConfig",
    {"head_dim": 8, "num_experts": 2, "num_experts_per_tok": 1, "moe_intermediate_size": 16},
)
_LLAMA4_TEXT = (
    "Llama4TextConfig",
    {"head_dim": 8, "num_local_experts": 2, "num_experts_per_tok": 1, "intermediate_size_mlp": 32},
)

# Index-valued MoE layout attributes: a non-default value names layers by
# number, which a slice cannot remap, so the prune must refuse it.
_LAYER_INDEX_CASES = [
    (*_QWEN2_MOE, "mlp_only_layers", [3]),
    (*_QWEN3_MOE, "decoder_sparse_step", 2),
    (*_LLAMA4_TEXT, "moe_layers", [1, 3]),
    (*_LLAMA4_TEXT, "interleave_moe_layer_step", 2),
]
_LAYER_INDEX_IDS = [attr for _, _, attr, _ in _LAYER_INDEX_CASES]


@pytest.mark.parametrize(
    "config_name,extra,attr,value", _LAYER_INDEX_CASES, ids=_LAYER_INDEX_IDS
)
def test_prune_refuses_a_non_default_layer_index_attr(config_name, extra, attr, value):
    import torch
    from transformers import AutoModelForCausalLM

    from soup_cli.utils.shrink import prune_model_layers

    config = getattr(transformers, config_name)(**_COMMON, **extra, **{attr: value})
    torch.manual_seed(0)
    model = AutoModelForCausalLM.from_config(config)

    with pytest.raises(ValueError, match=attr):
        prune_model_layers(model, 1, 1)
    assert len(model.model.layers) == 6
    assert model.config.num_hidden_layers == 6


@pytest.mark.parametrize(
    "config_name,extra,attr,value", _LAYER_INDEX_CASES, ids=_LAYER_INDEX_IDS
)
def test_prune_accepts_the_default_layer_index_attr(tmp_path, config_name, extra, attr, value):
    import torch
    from transformers import AutoModelForCausalLM

    from soup_cli.utils.shrink import prune_model_layers

    config = getattr(transformers, config_name)(**_COMMON, **extra)
    torch.manual_seed(0)
    model = AutoModelForCausalLM.from_config(config)

    prune_model_layers(model, 1, 1)

    assert len(model.model.layers) == 5
    model.save_pretrained(str(tmp_path))
    _, info = AutoModelForCausalLM.from_pretrained(str(tmp_path), output_loading_info=True)
    assert not info["missing_keys"] and not info["unexpected_keys"], info


def _write_tiny_qwen2(dir_path, lines):
    """Tiny random-init Qwen2 + a byte-level BPE tokenizer trained on ``lines``
    (no download)."""
    from tokenizers import ByteLevelBPETokenizer
    from transformers import AutoModelForCausalLM, PreTrainedTokenizerFast

    bpe = ByteLevelBPETokenizer()
    bpe.train_from_iterator(lines, vocab_size=300, min_frequency=1, special_tokens=["<eos>"])
    tok = PreTrainedTokenizerFast(tokenizer_object=bpe, eos_token="<eos>", pad_token="<eos>")
    config = transformers.Qwen2Config(
        **{**_COMMON, "vocab_size": len(tok)},
        use_sliding_window=True,
        max_window_layers=3,
        sliding_window=8,
    )
    AutoModelForCausalLM.from_config(config).save_pretrained(str(dir_path))
    tok.save_pretrained(str(dir_path))
    return str(dir_path)


def test_shrink_cli_saves_a_pruned_qwen2(tmp_path, monkeypatch):
    import json

    from typer.testing import CliRunner

    from soup_cli.cli import app
    from tests.conftest import strip_ansi

    monkeypatch.chdir(tmp_path)
    lines = [
        "the quick brown fox jumps over the lazy dog",
        "a small model keeps the first and last layer",
        "pruning drops a contiguous block of layers",
        "the calibration set scores each block",
    ]
    model_dir = _write_tiny_qwen2(tmp_path / "tiny-qwen2", lines)
    (tmp_path / "calib.jsonl").write_text(
        "\n".join(json.dumps({"text": line}) for line in lines), encoding="utf-8"
    )
    out_dir = tmp_path / "shrunk"
    r = CliRunner().invoke(
        app,
        ["shrink", "--model", model_dir, "--drop-layers", "2", "--calib", "calib.jsonl",
         "--device", "cpu", "--output-dir", str(out_dir), "--tolerance", "5.0"],
    )
    plain = " ".join(strip_ansi(r.output).split())
    assert r.exit_code == 0, (plain, repr(r.exception))
    cfg = json.loads((out_dir / "model" / "config.json").read_text(encoding="utf-8"))
    assert cfg["num_hidden_layers"] == 4
    assert len(cfg["layer_types"]) == 4
    report = json.loads((out_dir / "shrink_report.json").read_text(encoding="utf-8"))
    assert report["layers_before"] == 6 and report["layers_after"] == 4
    assert "Layers: 6 -> 4" in plain, plain


def test_a_default_moe_layout_stays_default_after_the_prune(tmp_path):
    """Llama4-text fills ``moe_layers`` with every layer by default. After the
    prune it must name every KEPT layer, or the saved config carries an index
    past ``num_hidden_layers`` and the shrunk model cannot be shrunk again."""
    import torch
    from transformers import AutoModelForCausalLM

    from soup_cli.utils.shrink import prune_model_layers

    config_name, extra = _LLAMA4_TEXT
    config = getattr(transformers, config_name)(**_COMMON, **extra)
    torch.manual_seed(0)
    model = AutoModelForCausalLM.from_config(config)

    prune_model_layers(model, 1, 1)
    assert model.config.moe_layers == list(range(5))

    model.save_pretrained(str(tmp_path))
    again = AutoModelForCausalLM.from_pretrained(str(tmp_path))
    prune_model_layers(again, 1, 1)
    assert again.config.num_hidden_layers == 4


def test_shrink_cli_refuses_an_index_valued_layout_before_loading(tmp_path, monkeypatch):
    """The refusal needs only config.json, so it comes before the weights are
    loaded and scored, and ``--plan-only`` does not print a plan the real run
    would refuse."""
    import json

    from tokenizers import ByteLevelBPETokenizer
    from transformers import AutoModelForCausalLM, PreTrainedTokenizerFast
    from typer.testing import CliRunner

    from soup_cli.cli import app
    from tests.conftest import strip_ansi

    monkeypatch.chdir(tmp_path)
    lines = ["the quick brown fox jumps over the lazy dog", "pruning drops a block of layers"]
    bpe = ByteLevelBPETokenizer()
    bpe.train_from_iterator(lines, vocab_size=300, min_frequency=1, special_tokens=["<eos>"])
    tok = PreTrainedTokenizerFast(tokenizer_object=bpe, eos_token="<eos>", pad_token="<eos>")
    config_name, extra = _QWEN2_MOE
    config = getattr(transformers, config_name)(
        **{**_COMMON, "vocab_size": len(tok)}, **extra, mlp_only_layers=[3]
    )
    AutoModelForCausalLM.from_config(config).save_pretrained("moe")
    tok.save_pretrained("moe")
    (tmp_path / "calib.jsonl").write_text(
        "\n".join(json.dumps({"text": line}) for line in lines), encoding="utf-8"
    )

    r = CliRunner().invoke(
        app,
        ["shrink", "--model", "moe", "--drop-layers", "1", "--calib", "calib.jsonl",
         "--device", "cpu", "--plan-only"],
    )
    plain = " ".join(strip_ansi(r.output).split())
    assert r.exit_code == 1, (plain, repr(r.exception))
    assert "mlp_only_layers" in plain, plain
    assert "Scoring importance" not in plain, plain