Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

13.5 KB
Raw Blame
"""Tests for freeze training: freeze_layers, freeze_ratio config and layer freezing."""

from __future__ import annotations

from unittest.mock import MagicMock

import pytest

from soup_cli.config.schema import SoupConfig, TrainingConfig

# ---------------------------------------------------------------------------
# Config validation
# ---------------------------------------------------------------------------


class TestFreezeConfig:
    """Tests for freeze training config fields."""

    def test_freeze_layers_default_none(self):
        """freeze_layers defaults to None."""
        cfg = TrainingConfig()
        assert cfg.freeze_layers is None

    def test_freeze_ratio_default_none(self):
        """freeze_ratio defaults to None."""
        cfg = TrainingConfig()
        assert cfg.freeze_ratio is None

    def test_freeze_layers_valid(self):
        """freeze_layers accepts positive int."""
        cfg = TrainingConfig(freeze_layers=24)
        assert cfg.freeze_layers == 24

    def test_freeze_layers_zero_rejected(self):
        """freeze_layers must be >= 1."""
        with pytest.raises(Exception):
            TrainingConfig(freeze_layers=0)

    def test_freeze_layers_negative_rejected(self):
        """freeze_layers must be positive."""
        with pytest.raises(Exception):
            TrainingConfig(freeze_layers=-5)

    def test_freeze_ratio_valid(self):
        """freeze_ratio accepts float in (0, 1)."""
        cfg = TrainingConfig(freeze_ratio=0.75)
        assert cfg.freeze_ratio == 0.75

    def test_freeze_ratio_zero_rejected(self):
        """freeze_ratio must be > 0."""
        with pytest.raises(Exception):
            TrainingConfig(freeze_ratio=0.0)

    def test_freeze_ratio_one_rejected(self):
        """freeze_ratio must be < 1 (can't freeze everything)."""
        with pytest.raises(Exception):
            TrainingConfig(freeze_ratio=1.0)

    def test_freeze_ratio_over_one_rejected(self):
        """freeze_ratio must be < 1."""
        with pytest.raises(Exception):
            TrainingConfig(freeze_ratio=1.5)

    def test_freeze_layers_and_ratio_both_set(self):
        """Both freeze_layers and freeze_ratio can be set (layers takes priority)."""
        cfg = TrainingConfig(freeze_layers=10, freeze_ratio=0.5)
        assert cfg.freeze_layers == 10
        assert cfg.freeze_ratio == 0.5

    def test_freeze_in_yaml_roundtrip(self):
        """freeze fields survive YAML round-trip via SoupConfig."""
        cfg = SoupConfig(
            base="test/model",
            data={"train": "data.jsonl"},
            training={"freeze_layers": 16},
        )
        assert cfg.training.freeze_layers == 16

    def test_freeze_ratio_in_yaml_roundtrip(self):
        """freeze_ratio survives YAML round-trip."""
        cfg = SoupConfig(
            base="test/model",
            data={"train": "data.jsonl"},
            training={"freeze_ratio": 0.5},
        )
        assert cfg.training.freeze_ratio == 0.5


# ---------------------------------------------------------------------------
# Layer freezing logic
# ---------------------------------------------------------------------------


class TestFreezeModelLayers:
    """Tests for freeze_model_layers utility function."""

    def _make_mock_model(self, num_layers: int = 32):
        """Create a mock model with named_parameters."""
        model = MagicMock()
        params = []
        for layer_idx in range(num_layers):
            parts = [
                "self_attn.q_proj.weight",
                "self_attn.v_proj.weight",
                "mlp.up_proj.weight",
            ]
            for part in parts:
                param = MagicMock()
                param.requires_grad = True
                name = f"model.layers.{layer_idx}.{part}"
                params.append((name, param))
        # Add non-layer params (embed, lm_head)
        embed_param = MagicMock()
        embed_param.requires_grad = True
        params.append(("model.embed_tokens.weight", embed_param))

        head_param = MagicMock()
        head_param.requires_grad = True
        params.append(("lm_head.weight", head_param))

        model.named_parameters.return_value = params
        return model, params

    def test_freeze_by_layer_count(self):
        """freeze_model_layers freezes first N layers."""
        from soup_cli.utils.freeze import freeze_model_layers

        model, params = self._make_mock_model(32)
        frozen_count = freeze_model_layers(model, freeze_layers=24)

        # First 24 layers' params should have requires_grad = False
        for name, param in params:
            if "layers." in name:
                layer_idx = int(name.split("layers.")[1].split(".")[0])
                if layer_idx < 24:
                    assert param.requires_grad is False
                else:
                    assert param.requires_grad is True

        assert frozen_count > 0

    def test_freeze_by_ratio(self):
        """freeze_model_layers freezes by ratio."""
        from soup_cli.utils.freeze import freeze_model_layers

        model, params = self._make_mock_model(32)
        frozen_count = freeze_model_layers(model, freeze_ratio=0.75)

        # 75% of 32 = 24 layers frozen
        for name, param in params:
            if "layers." in name:
                layer_idx = int(name.split("layers.")[1].split(".")[0])
                if layer_idx < 24:
                    assert param.requires_grad is False

        assert frozen_count > 0

    def test_freeze_layers_priority_over_ratio(self):
        """freeze_layers takes priority when both specified."""
        from soup_cli.utils.freeze import freeze_model_layers

        model, params = self._make_mock_model(32)
        freeze_model_layers(model, freeze_layers=10, freeze_ratio=0.75)

        # Should freeze 10 layers, not 24
        for name, param in params:
            if "layers." in name:
                layer_idx = int(name.split("layers.")[1].split(".")[0])
                if layer_idx < 10:
                    assert param.requires_grad is False
                else:
                    assert param.requires_grad is True

    def test_freeze_does_not_freeze_embeddings(self):
        """Embeddings are not frozen (they're not layer params)."""
        from soup_cli.utils.freeze import freeze_model_layers

        model, params = self._make_mock_model(32)
        freeze_model_layers(model, freeze_layers=24)

        # embed_tokens and lm_head should remain trainable
        for name, param in params:
            if "embed_tokens" in name or "lm_head" in name:
                assert param.requires_grad is True

    def test_freeze_more_than_total_layers(self):
        """Freezing more layers than model has freezes all layers."""
        from soup_cli.utils.freeze import freeze_model_layers

        model, params = self._make_mock_model(8)
        freeze_model_layers(model, freeze_layers=100)

        # All 8 layers frozen
        for name, param in params:
            if "layers." in name:
                assert param.requires_grad is False

    def test_freeze_returns_count(self):
        """freeze_model_layers returns number of frozen parameters."""
        from soup_cli.utils.freeze import freeze_model_layers

        model, params = self._make_mock_model(32)
        frozen = freeze_model_layers(model, freeze_layers=16)
        # 16 layers × 3 params each = 48
        assert frozen == 48

    def test_no_freeze_when_none(self):
        """No freezing when both are None."""
        from soup_cli.utils.freeze import freeze_model_layers

        model, params = self._make_mock_model(8)
        frozen = freeze_model_layers(model, freeze_layers=None, freeze_ratio=None)
        assert frozen == 0

    def test_freeze_warns_when_no_numbered_layers_detected(self, caplog):
        """A freeze request against a model with no 'layers.N.'/'h.N.' naming
        is a no-op, and must say so rather than silently doing nothing."""
        from soup_cli.utils.freeze import freeze_model_layers

        model = MagicMock()
        model.named_parameters.return_value = [("embed.weight", MagicMock())]
        with caplog.at_level("WARNING"):
            frozen = freeze_model_layers(model, freeze_layers=4)
        assert frozen == 0
        assert "could not detect numbered layers" in caplog.text

    def test_detect_num_layers(self):
        """_detect_num_layers extracts layer count from model params."""
        from soup_cli.utils.freeze import _detect_num_layers

        model = MagicMock()
        params = [
            (f"model.layers.{idx}.self_attn.weight", MagicMock())
            for idx in range(32)
        ]
        model.named_parameters.return_value = params
        assert _detect_num_layers(model) == 32

    def test_detect_num_layers_no_layers(self):
        """_detect_num_layers returns 0 for models without numbered layers."""
        from soup_cli.utils.freeze import _detect_num_layers

        model = MagicMock()
        model.named_parameters.return_value = [
            ("embed.weight", MagicMock()),
        ]
        assert _detect_num_layers(model) == 0

    def test_detect_num_layers_gpt2_style(self):
        """_detect_num_layers handles GPT-2 style 'transformer.h.N.' naming."""
        from soup_cli.utils.freeze import _detect_num_layers

        model = MagicMock()
        params = [
            (f"transformer.h.{idx}.attn.weight", MagicMock())
            for idx in range(12)
        ]
        model.named_parameters.return_value = params
        assert _detect_num_layers(model) == 12


# ---------------------------------------------------------------------------
# #1432: freeze_layer_cutoff (the range LoRA must respect too)
# ---------------------------------------------------------------------------


class TestFreezeLayerCutoff:
    """freeze_layer_cutoff resolves the same (cutoff, total_layers) pair
    freeze_model_layers acts on, for a caller that needs the boundary
    without mutating requires_grad (#1432: the LoRA branch reads this to
    build ``layers_to_transform`` for the range freeze_model_layers left
    trainable)."""

    def _make_model(self, num_layers: int):
        model = MagicMock()
        model.named_parameters.return_value = [
            (f"model.layers.{idx}.self_attn.q_proj.weight", MagicMock())
            for idx in range(num_layers)
        ]
        return model

    def test_both_none_returns_none(self):
        from soup_cli.utils.freeze import freeze_layer_cutoff

        model = self._make_model(32)
        assert freeze_layer_cutoff(model, freeze_layers=None, freeze_ratio=None) is None

    def test_no_numbered_layers_returns_none(self):
        from soup_cli.utils.freeze import freeze_layer_cutoff

        model = MagicMock()
        model.named_parameters.return_value = [("embed.weight", MagicMock())]
        assert freeze_layer_cutoff(model, freeze_layers=4) is None

    def test_freeze_layers_cutoff(self):
        from soup_cli.utils.freeze import freeze_layer_cutoff

        model = self._make_model(32)
        assert freeze_layer_cutoff(model, freeze_layers=24) == (24, 32)

    def test_freeze_layers_clamped_to_total(self):
        """Mirrors freeze_model_layers' own clamp: freezing more layers than
        exist freezes all of them, cutoff == total_layers."""
        from soup_cli.utils.freeze import freeze_layer_cutoff

        model = self._make_model(8)
        assert freeze_layer_cutoff(model, freeze_layers=100) == (8, 8)

    def test_freeze_ratio_cutoff(self):
        from soup_cli.utils.freeze import freeze_layer_cutoff

        model = self._make_model(32)
        assert freeze_layer_cutoff(model, freeze_ratio=0.75) == (24, 32)

    def test_freeze_layers_takes_priority_over_ratio(self):
        from soup_cli.utils.freeze import freeze_layer_cutoff

        model = self._make_model(32)
        assert freeze_layer_cutoff(model, freeze_layers=10, freeze_ratio=0.75) == (10, 32)

    def test_matches_freeze_model_layers_frozen_set(self):
        """CONTROL: the boundary this returns must be the exact same one
        freeze_model_layers used to set requires_grad=False, or the two
        would silently drift apart the next time either is edited."""
        from soup_cli.utils.freeze import freeze_layer_cutoff, freeze_model_layers

        model = self._make_model(32)
        params = dict(model.named_parameters())
        for _name, param in params.items():
            param.requires_grad = True
        freeze_model_layers(model, freeze_layers=24)
        cutoff, total_layers = freeze_layer_cutoff(model, freeze_layers=24)
        assert total_layers == 32
        for name, param in params.items():
            layer_idx = int(name.split("layers.")[1].split(".")[0])
            assert param.requires_grad == (layer_idx >= cutoff)


# ---------------------------------------------------------------------------
# Sweep integration
# ---------------------------------------------------------------------------


class TestFreezeSweep:
    """Tests for freeze fields in sweep param support."""

    def test_freeze_layers_in_sweep(self):
        """freeze_layers is a valid sweep param."""
        from soup_cli.commands.sweep import _parse_sweep_params

        params = _parse_sweep_params(["training.freeze_layers=8,16,24"])
        assert "training.freeze_layers" in params
        assert len(params["training.freeze_layers"]) == 3

    def test_freeze_ratio_in_sweep(self):
        """freeze_ratio is a valid sweep param."""
        from soup_cli.commands.sweep import _parse_sweep_params

        params = _parse_sweep_params(["training.freeze_ratio=0.25,0.5,0.75"])
        assert "training.freeze_ratio" in params
        assert len(params["training.freeze_ratio"]) == 3