Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

20.2 KB
Raw Blame
"""#379: publish the STEP 14 control-versus-repair measurement harness.

The historical gate compared a streamed NF4 control and the repaired streamed
path against one resident NF4 reference. Its first run reported ``0/0`` exact
gradients because the parameter-name intersection was empty. These tests pin
the non-vacuous contract on CPU; real 32B/72B numbers still require CUDA.
"""

from __future__ import annotations

import importlib.util
import subprocess
import sys
from pathlib import Path
from types import SimpleNamespace
from typing import Any

import pytest

_REPO_ROOT = Path(__file__).resolve().parents[1]
_HARNESS_DIR = _REPO_ROOT / "benchmarks" / "harness"
_HARNESS = _HARNESS_DIR / "variant2_gate.py"


def _load_harness():
    sys.path.insert(0, str(_HARNESS_DIR))
    try:
        spec = importlib.util.spec_from_file_location("variant2_gate", _HARNESS)
        assert spec is not None and spec.loader is not None
        module = importlib.util.module_from_spec(spec)
        sys.modules[spec.name] = module
        spec.loader.exec_module(module)
        return module
    finally:
        sys.path.remove(str(_HARNESS_DIR))


_harness = _load_harness()


class _FakeParameter:
    def __init__(self, grad=object()) -> None:
        self.grad = grad


class _FakeModel:
    def __init__(self, names: list[str], *, grad=object()) -> None:
        self._parameters = [(name, _FakeParameter(grad)) for name in names]

    def named_parameters(self):
        return iter(self._parameters)


class TestArgumentValidation:
    @pytest.mark.parametrize(
        ("flag", "value"),
        [("--seq", "0"), ("--repeats", "0"), ("--buffers", "1")],
    )
    def test_bad_protocol_values_fail_before_cuda_skip(
        self, monkeypatch, capsys, flag: str, value: str
    ) -> None:
        monkeypatch.setattr(_harness, "cuda_available", lambda: False)
        monkeypatch.setattr(
            sys,
            "argv",
            ["variant2_gate.py", "--weights", "x", "--shards", "y", flag, value],
        )

        assert _harness.main() == 2
        assert flag in capsys.readouterr().out

    def test_valid_protocol_is_an_intentional_no_cuda_skip(self, monkeypatch, capsys) -> None:
        monkeypatch.setattr(_harness, "cuda_available", lambda: False)
        monkeypatch.setattr(
            sys,
            "argv",
            ["variant2_gate.py", "--weights", "x", "--shards", "y"],
        )

        assert _harness.main() == 0
        assert "intentional skip" in capsys.readouterr().out.lower()


class TestCanonicalGradientIntersection:
    def test_empty_model_intersection_is_a_hard_failure(self) -> None:
        left = _FakeModel(["model.layers.0.q_proj.lora_A.default.weight"])
        right = _FakeModel(["model.layers.1.q_proj.lora_A.default.weight"])

        with pytest.raises(ValueError, match="no canonical parameter name"):
            _harness.shared_lora_gradient_names(left, right)

    def test_shared_parameters_without_shared_lora_gradients_fail(self) -> None:
        left = _FakeModel(["model.embed_tokens.weight"])
        right = _FakeModel(["model.embed_tokens.weight"])

        with pytest.raises(ValueError, match="no shared LoRA gradient"):
            _harness.shared_lora_gradient_names(left, right)

    def test_shared_lora_gradient_names_are_returned(self) -> None:
        name = "model.layers.0.q_proj.lora_A.default.weight"
        assert _harness.shared_lora_gradient_names(
            _FakeModel([name]), _FakeModel([name])
        ) == frozenset({name})


class TestGradientVerdict:
    def test_gate_requires_the_control_to_break_and_repair_to_be_exact(self) -> None:
        broken = _harness.GradientSummary(compared=4, exact=1, wrong_layers=2, worst_abs=0.5)
        exact = _harness.GradientSummary(compared=4, exact=4, wrong_layers=0, worst_abs=0.0)

        assert _harness.gate_passes(control=broken, repaired=exact)
        assert not _harness.gate_passes(control=exact, repaired=exact)
        assert not _harness.gate_passes(control=broken, repaired=broken)

    def test_every_repetition_must_have_a_matching_control_and_repair(self) -> None:
        broken = _harness.GradientSummary(compared=4, exact=1, wrong_layers=2, worst_abs=0.5)
        exact = _harness.GradientSummary(compared=4, exact=4, wrong_layers=0, worst_abs=0.0)

        assert _harness._paired_gradient_gate(
            {"gradients": [vars(broken), vars(broken)]},
            {"gradients": [vars(exact), vars(exact)]},
        )
        assert not _harness._paired_gradient_gate(
            {"gradients": [vars(broken), vars(broken)]},
            {"gradients": [vars(exact)]},
        )

    def test_real_tensors_distinguish_exact_and_broken_gradients(self) -> None:
        torch = pytest.importorskip("torch")
        name = "model.layers.3.q_proj.lora_A.default.weight"
        left = _FakeModel([name], grad=torch.tensor([1.0, 2.0]))
        same = _FakeModel([name], grad=torch.tensor([1.0, 2.0]))
        different = _FakeModel([name], grad=torch.tensor([1.0, 4.0]))

        exact = _harness.compare_lora_gradients(left, same)
        broken = _harness.compare_lora_gradients(left, different)

        assert exact == _harness.GradientSummary(1, 1, 0, 0.0)
        assert broken == _harness.GradientSummary(1, 0, 1, 2.0)


def _gate_result(
    *, exact: int, wrong_layers: int, rewired_modules: int, losses_exact: bool = True
) -> dict[str, Any]:
    return {
        "gradients": [vars(_harness.GradientSummary(256, exact, wrong_layers, 0.0))],
        "losses_exact": losses_exact,
        "rewired_modules": rewired_modules,
    }


class TestWholeRunVerdict:
    @pytest.mark.parametrize(
        ("control", "repaired", "expected"),
        [
            ((8, 62, 0, True), (256, 0, 448, True), (True, True, True, True, True)),
            ((256, 0, 0, True), (256, 0, 0, True), (False, False, True, True, False)),
            ((8, 62, 0, True), (256, 0, 0, True), (False, True, True, True, False)),
            ((8, 62, 7, True), (256, 0, 448, True), (False, True, True, True, False)),
            ((8, 62, 0, False), (256, 0, 448, True), (False, False, True, False, True)),
            ((8, 62, 0, True), (256, 0, 448, False), (False, True, False, False, True)),
        ],
    )
    def test_verdict_pins_gradients_losses_and_wiring(
        self,
        control: tuple[int, int, int, bool],
        repaired: tuple[int, int, int, bool],
        expected: tuple[bool, bool, bool, bool, bool],
    ) -> None:
        control_result = _gate_result(
            exact=control[0],
            wrong_layers=control[1],
            rewired_modules=control[2],
            losses_exact=control[3],
        )
        repaired_result = _gate_result(
            exact=repaired[0],
            wrong_layers=repaired[1],
            rewired_modules=repaired[2],
            losses_exact=repaired[3],
        )

        assert _harness.verdict(control_result, repaired_result) == expected

    @pytest.mark.parametrize(
        ("control", "repaired", "exit_code"),
        [
            ((8, 62, 0, True), (256, 0, 448, True), 0),
            ((256, 0, 0, True), (256, 0, 0, True), 1),
            ((8, 62, 0, True), (256, 0, 0, True), 1),
            ((8, 62, 7, True), (256, 0, 448, True), 1),
            ((8, 62, 0, False), (256, 0, 448, True), 1),
            ((8, 62, 0, True), (256, 0, 448, False), 1),
        ],
    )
    def test_main_builds_a_real_control_and_reports_the_verdict(
        self,
        monkeypatch: pytest.MonkeyPatch,
        tmp_path: Path,
        capsys: pytest.CaptureFixture[str],
        control: tuple[int, int, int, bool],
        repaired: tuple[int, int, int, bool],
        exit_code: int,
    ) -> None:
        torch = pytest.importorskip("torch")
        from soup_cli.utils import layer_shard, layer_stream_runtime, spectrum_scan

        monkeypatch.setattr(
            _harness,
            "parse_args",
            lambda: SimpleNamespace(
                weights="fake",
                shards=str(tmp_path / "shards"),
                seq=128,
                batch=1,
                buffers=2,
                repeats=1,
                json=None,
            ),
        )
        monkeypatch.setattr(_harness, "cuda_available", lambda: True)
        monkeypatch.setattr(_harness, "_source_sha", lambda: "a" * 40)
        monkeypatch.setattr(_harness, "_versions", lambda: {"commit": "a" * 40})
        monkeypatch.setattr(layer_stream_runtime, "build_meta_skeleton", lambda *a, **k: object())
        monkeypatch.setattr(layer_stream_runtime, "quantised_layer_suffixes", lambda *_: ())
        monkeypatch.setattr(layer_shard, "shard_checkpoint", lambda *a, **k: object())
        monkeypatch.setattr(spectrum_scan, "resolve_model_weights", lambda *_: "fake-weights")
        monkeypatch.setattr(_harness.shared, "model_arch_name", lambda *_: "fake-arch")
        monkeypatch.setattr(
            _harness.shared, "load_resident_reference", lambda *a, **k: _load_reference(seed_calls)
        )
        monkeypatch.setattr(_harness.shared, "make_non_vacuous_lora", lambda *_: None)
        monkeypatch.setattr(_harness.shared, "copy_lora", lambda *a: None)
        seed_calls: list[tuple[str, int]] = []
        monkeypatch.setattr(torch, "manual_seed", lambda seed: seed_calls.append(("cpu", seed)))
        monkeypatch.setattr(
            torch.cuda, "manual_seed_all", lambda seed: seed_calls.append(("cuda", seed))
        )

        def _load_reference(calls: list[tuple[str, int]]) -> Any:
            assert calls == [
                ("cpu", _harness.DEFAULT_SEED),
                ("cuda", _harness.DEFAULT_SEED),
            ]
            return SimpleNamespace(config=SimpleNamespace(vocab_size=16))

        monkeypatch.setattr(
            torch, "Generator", lambda **k: SimpleNamespace(manual_seed=lambda *_: object())
        )
        monkeypatch.setattr(torch, "randint", lambda *a, **k: object())
        monkeypatch.setattr(torch.cuda, "get_device_name", lambda *_: "simulated CUDA")
        monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None)

        build_flags: list[bool] = []

        def fake_build_arm(*, repair_enabled: bool, **kwargs: object) -> tuple[object, Any]:
            build_flags.append(repair_enabled)
            return object(), SimpleNamespace(close=lambda: None)

        results = {
            "control": _gate_result(
                exact=control[0],
                wrong_layers=control[1],
                rewired_modules=control[2],
                losses_exact=control[3],
            ),
            "repaired": _gate_result(
                exact=repaired[0],
                wrong_layers=repaired[1],
                rewired_modules=repaired[2],
                losses_exact=repaired[3],
            ),
        }

        def fake_run_arm(*, label: str, **kwargs: object) -> dict[str, Any]:
            return results[label]

        monkeypatch.setattr(_harness, "_build_arm", fake_build_arm)
        monkeypatch.setattr(_harness, "_run_arm", fake_run_arm)

        assert _harness.main() == exit_code
        assert build_flags == [False, True]
        assert f"RESULT: {'passed' if exit_code == 0 else 'failed'}" in capsys.readouterr().out


class TestSourceCommit:
    @pytest.mark.parametrize("status", ["", " M src/soup_cli/changed.py\n"])
    def test_source_sha_uses_imported_package_tree_and_marks_dirty(
        self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, status: str
    ) -> None:
        import soup_cli

        package_file = soup_cli.__file__
        assert package_file is not None
        package_dir = Path(package_file).resolve().parent
        expected_sha = "a" * 40
        calls: list[tuple[list[str], Path]] = []

        def fake_run(command: list[str], *, cwd: Path, **kwargs: object) -> Any:
            calls.append((command, cwd))
            if command == ["git", "rev-parse", "--show-toplevel"]:
                return SimpleNamespace(stdout=f"{tmp_path}\n")
            if command == ["git", "rev-parse", "HEAD"]:
                return SimpleNamespace(stdout=f"{expected_sha}\n")
            return SimpleNamespace(stdout=status)

        monkeypatch.chdir(tmp_path)
        monkeypatch.setattr(subprocess, "run", fake_run)

        assert _harness._source_sha() == expected_sha + ("-dirty" if status else "")
        assert calls == [
            (["git", "rev-parse", "--show-toplevel"], package_dir),
            (["git", "ls-files", "--error-unmatch", str(Path(package_file).resolve())], tmp_path),
            (["git", "rev-parse", "HEAD"], tmp_path),
            (["git", "status", "--porcelain", "--untracked-files=normal"], tmp_path),
        ]


class TestRepairToggle:
    def test_control_disables_repair_only_while_building(self) -> None:
        calls: list[str] = []

        def install(_module) -> int:
            calls.append("repair")
            return 7

        runtime_module = SimpleNamespace(install_dequant_forward=install)

        def build_model():
            return runtime_module.install_dequant_forward(object())

        assert (
            _harness.build_streamed_arm(
                repair_enabled=False,
                runtime_module=runtime_module,
                build_model=build_model,
            )
            == 0
        )
        assert calls == []
        assert runtime_module.install_dequant_forward is install

        assert (
            _harness.build_streamed_arm(
                repair_enabled=True,
                runtime_module=runtime_module,
                build_model=build_model,
            )
            == 7
        )
        assert calls == ["repair"]


class TestRewiringCounter:
    def test_real_torch_wrapper_hidden_by_named_modules_is_counted(self) -> None:
        torch = pytest.importorskip("torch")

        class HiddenWrapper(torch.nn.Module):
            def __init__(self) -> None:
                super().__init__()
                self.inner = torch.nn.Identity()
                self.n_dequant_forward = 7

            def named_modules(self, memo=None, prefix="", remove_duplicate=True):
                yield from self.inner.named_modules(memo, prefix, remove_duplicate)

        wrapper = HiddenWrapper()
        model = torch.nn.Sequential(wrapper)
        assert all(module is not wrapper for module in model.modules())
        assert _harness._rewired_modules(model) == 7

    def test_walks_structural_children_hidden_by_named_modules(self) -> None:
        inner = SimpleNamespace(_modules={})
        wrapper = SimpleNamespace(_modules={"inner": inner}, n_dequant_forward=7)
        root = SimpleNamespace(_modules={"first": wrapper, "alias": wrapper})
        # Model.modules() delegates to the overridden named_modules() and
        # hides StreamedDecoderLayer itself after #1010.
        root.modules = lambda: iter((root, inner))

        assert _harness._rewired_modules(root) == 7

    def test_repaired_arm_fails_loudly_before_measurement_when_counter_is_dead(self) -> None:
        model = SimpleNamespace(_modules={}, train=lambda: pytest.fail("entered training"))

        with pytest.raises(RuntimeError, match="repaired arm.*no rewired modules"):
            _harness._run_arm(
                label="repaired",
                model=model,
                reference=object(),
                input_ids=object(),
                repeats=1,
            )

    def test_both_tiny_nf4_arms_report_their_real_wrapper_counts(self, monkeypatch) -> None:
        torch = pytest.importorskip("torch")
        from soup_cli.utils import layer_stream_runtime as runtime_module

        class FakeNF4Linear(torch.nn.Module):
            def __init__(self) -> None:
                super().__init__()
                self.is_fake_nf4 = True

        def fake_install_dequant_forward(inner: Any) -> int:
            return sum(1 for module in inner.modules() if getattr(module, "is_fake_nf4", False))

        monkeypatch.setattr(runtime_module, "install_dequant_forward", fake_install_dequant_forward)
        streamed_layer = runtime_module._build_streamed_layer_class()

        def build_tiny_nf4_model() -> Any:
            return torch.nn.Sequential(
                *[
                    streamed_layer(
                        inner=torch.nn.Sequential(*(FakeNF4Linear() for _ in range(7))),
                        idx=layer_index,
                        pool=None,
                        prefetcher=None,
                        quant_specs={"nf4": object()},
                        codes={},
                    )
                    for layer_index in range(2)
                ]
            )

        control = _harness.build_streamed_arm(
            repair_enabled=False,
            runtime_module=runtime_module,
            build_model=build_tiny_nf4_model,
        )
        repaired = _harness.build_streamed_arm(
            repair_enabled=True,
            runtime_module=runtime_module,
            build_model=build_tiny_nf4_model,
        )

        assert _harness._rewired_modules(control) == 0
        assert _harness._rewired_modules(repaired) == 14
        assert runtime_module.install_dequant_forward is fake_install_dequant_forward


class TestRunArm:
    def test_measures_identical_control_and_perturbed_repair_on_cpu(self, monkeypatch) -> None:
        torch = pytest.importorskip("torch")

        class TinyLoraArm(torch.nn.Module):
            def __init__(self, weight: tuple[tuple[float, float], ...], rewired: int) -> None:
                super().__init__()
                self.lora_A = torch.nn.Parameter(torch.tensor(weight, dtype=torch.float32))
                self.n_dequant_forward = rewired

            def forward(self, *, input_ids: Any, labels: Any) -> Any:
                prediction = input_ids @ self.lora_A
                return SimpleNamespace(loss=torch.nn.functional.mse_loss(prediction, labels))

        monkeypatch.setattr(torch.cuda, "reset_peak_memory_stats", lambda: None)
        monkeypatch.setattr(torch.cuda, "synchronize", lambda: None)
        monkeypatch.setattr(torch.cuda, "max_memory_allocated", lambda: 0)

        reference = TinyLoraArm(((1.0, 0.0), (0.0, 1.0)), rewired=0)
        control = TinyLoraArm(((1.0, 0.0), (0.0, 1.0)), rewired=0)
        repaired = TinyLoraArm(((1.5, 0.0), (0.0, 1.0)), rewired=14)
        input_ids = torch.tensor([[1.0, 2.0], [-1.0, 0.5]])
        control_result = _harness._run_arm(
            label="control",
            model=control,
            reference=reference,
            input_ids=input_ids,
            repeats=1,
        )
        repaired_result = _harness._run_arm(
            label="repaired",
            model=repaired,
            reference=reference,
            input_ids=input_ids,
            repeats=1,
        )

        assert control_result["losses_exact"] is True
        assert control_result["gradients"][0]["exact"] == 1
        assert control_result["rewired_modules"] == 0
        assert repaired_result["losses_exact"] is False
        assert repaired_result["gradients"][0]["exact"] == 0
        assert repaired_result["rewired_modules"] == 14


def test_versions_record_the_runtime_used_for_the_verdict(monkeypatch) -> None:
    fake_torch = SimpleNamespace(
        __version__="2.14.0",
        version=SimpleNamespace(cuda="13.0"),
        cuda=SimpleNamespace(get_device_capability=lambda device=0: (12, 0)),
    )
    monkeypatch.setitem(sys.modules, "torch", fake_torch)
    monkeypatch.setitem(sys.modules, "bitsandbytes", SimpleNamespace(__version__="0.50.2"))
    monkeypatch.setitem(sys.modules, "peft", SimpleNamespace(__version__="0.20.0"))
    monkeypatch.setitem(sys.modules, "transformers", SimpleNamespace(__version__="5.17.0"))
    monkeypatch.setattr(_harness, "_source_sha", lambda: "a" * 40)
    monkeypatch.setattr(_harness, "distribution_version", lambda name: "0.26.2")

    versions = _harness._versions()

    assert versions["cuda_runtime"] == "13.0"
    assert versions["cuda_compute_capability"] == "12.0"
    assert versions["trl"] == "0.26.2"


def test_optional_distribution_version_marks_trl_missing(monkeypatch) -> None:
    from importlib.metadata import PackageNotFoundError

    def missing_version(distribution: str) -> str:
        raise PackageNotFoundError(distribution)

    monkeypatch.setattr(_harness, "distribution_version", missing_version)

    assert _harness._optional_distribution_version("trl") == "not-installed"


def test_harness_is_indexed() -> None:
    readme = (_REPO_ROOT / "benchmarks" / "README.md").read_text(encoding="utf-8")
    assert "harness/variant2_gate.py" in readme
    assert "failed closed, as expected" in readme
    assert "sm_120" in readme