Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

11.9 KB
Raw Blame
"""Drive ``pincost.main()`` through its real CPU-testable decisions."""
from __future__ import annotations

import sys
from types import SimpleNamespace

import pytest
import torch
from torch import nn

from tests.conftest import strip_ansi
from tests.test_pincost import module as pincost

HIDDEN, VOCAB, LAYERS = 8, 32, 2
FAST, SLOW = 1.0, 4.0

def _plain(text: str) -> str:
    return " ".join(strip_ansi(text).split())

class _GradScale(torch.autograd.Function):
    @staticmethod
    def forward(ctx, x, factor):
        ctx.factor = factor
        return x.view_as(x)
    @staticmethod
    def backward(ctx, grad):
        return grad * ctx.factor, None

class _Layer(nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.base = nn.Linear(HIDDEN, HIDDEN, bias=False).requires_grad_(False)
        self.lora_A = nn.Linear(HIDDEN, 2, bias=False)
        self.lora_B = nn.Linear(2, HIDDEN, bias=False)
    def forward(self, x, factor=None):
        delta = self.lora_B(self.lora_A(x))
        if hasattr(self, "lora_extra"):
            delta = delta + self.lora_extra(x).sum() * 0.0
        if factor is not None:
            delta = _GradScale.apply(delta, factor)
        return torch.tanh(self.base(x) + delta)

class _Inner(nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.layers = nn.ModuleList(_Layer() for _ in range(LAYERS))

class _Toy(nn.Module):
    def __init__(self, scenario: str, clock: SimpleNamespace, *, pin=None) -> None:
        super().__init__()
        torch.manual_seed(0)
        self.config = SimpleNamespace(vocab_size=VOCAB, model_type="llama")
        self.embed = nn.Embedding(VOCAB, HIDDEN).requires_grad_(False)
        self.model = _Inner()
        self.head = nn.Linear(HIDDEN, VOCAB, bias=False).requires_grad_(False)
        self.scenario, self.clock, self.pin = scenario, clock, pin
        self.streamed_calls = 0
        streamed = pin is not None
        if scenario == "extra_lora" and streamed:
            self.model.layers[0].lora_extra = nn.Linear(HIDDEN, 2, bias=False)
        if scenario == "inf_grads" or (
            scenario == "inf_grads_streamed" and streamed
        ) or (scenario == "inf_grads_reference" and not streamed):
            for name, parameter in self.named_parameters():
                if "lora_" in name:
                    parameter.register_hook(lambda g: torch.full_like(g, float("inf")))
    def forward(self, input_ids, labels):
        streamed = self.pin is not None
        factor = None
        if streamed:
            self.streamed_calls += 1
            slow = not self.pin
            if self.scenario == "inverted":
                slow = self.pin
            elif self.scenario == "no_cost":
                slow = False
            step = SLOW if slow else FAST
            self.clock.now += -step if self.scenario == "clock_backwards" else step
            if self.scenario == "wrong_first":
                factor = 3.0
            elif self.scenario == "pinned_wrong_all" and self.pin:
                factor = 3.0
            elif (
                self.scenario in {"wrong_after_first", "pinned_wrong_after_first"}
                and self.streamed_calls > 1
            ):
                factor = 3.0
            if self.scenario == "pinned_wrong_after_first" and not self.pin:
                factor = None
        x = self.embed(input_ids)
        for layer in self.model.layers:
            x = layer(x, factor)
        loss = nn.functional.cross_entropy(self.head(x).view(-1, VOCAB), labels.view(-1))
        if streamed and self.scenario == "loss_offset":
            loss = loss + 1e-3
        if self.scenario == "inf_loss" or (
            self.scenario == "inf_loss_streamed" and streamed
        ) or (self.scenario == "inf_loss_reference" and not streamed):
            loss = loss + torch.tensor(float("inf"))
        return SimpleNamespace(loss=loss)

class _Runtime:
    def __init__(self, pinned: bool) -> None:
        self.pinned = pinned
        self.source = SimpleNamespace(nbytes=1)
    def close(self) -> None:
        pass

def _run(monkeypatch, capsys, tmp_path, scenario: str) -> tuple[int, str]:
    import soup_cli.utils.layer_shard as layer_shard
    import soup_cli.utils.layer_stream_runtime as layer_runtime
    import soup_cli.utils.spectrum_scan as spectrum_scan
    clock = SimpleNamespace(now=0.0)
    def build_streamed(*args, **kwargs):
        pin = kwargs["pin"]
        reported = True if scenario == "lying_pinned" else pin
        runtime = _Runtime(reported)
        if scenario == "empty_store":
            runtime.source.nbytes = 0
        return _Toy(scenario, clock, pin=pin), runtime
    def cpu_non_vacuous(model) -> None:
        generator = torch.Generator(device="cpu").manual_seed(23)
        with torch.no_grad():
            for name, parameter in layer_runtime.canonical_named_parameters(model):
                if "lora_B" in name:
                    parameter.copy_(torch.randn(parameter.shape, generator=generator) * 0.02)
    monkeypatch.setattr(pincost, "DEVICE", "cpu")
    monkeypatch.setattr(pincost, "cuda_available", lambda: True)
    monkeypatch.setattr(pincost, "time", SimpleNamespace(perf_counter=lambda: clock.now))
    monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)
    monkeypatch.setattr(torch.cuda, "empty_cache", lambda *a, **k: None)
    monkeypatch.setattr(torch.cuda, "get_device_name", lambda *a, **k: "cpu-stub")
    monkeypatch.setattr(spectrum_scan, "resolve_model_weights", lambda weights: str(weights))
    monkeypatch.setattr(layer_runtime, "build_meta_skeleton", lambda *a, **k: _Toy(scenario, clock))
    monkeypatch.setattr(layer_runtime, "quantised_layer_suffixes", lambda probe: ())
    monkeypatch.setattr(layer_runtime, "build_streamed_model", build_streamed)
    monkeypatch.setattr(layer_shard, "shard_checkpoint", lambda *a, **k: {})
    monkeypatch.setattr(
        pincost.bitexact,
        "load_resident_reference",
        lambda *a, **k: _Toy(scenario, clock),
    )
    monkeypatch.setattr(pincost.bitexact, "make_non_vacuous_lora", cpu_non_vacuous)
    monkeypatch.setattr(
        sys,
        "argv",
        [
            "pincost.py", "--weights", str(tmp_path), "--shards",
            str(tmp_path / "shards"), "--seq", "4", "--warmup", "1",
            "--steps", "3", "--repeats", "2",
        ],
    )
    return pincost.main(), _plain(capsys.readouterr().out)

def test_a_real_cost_passes_with_the_arms_the_right_way_round(monkeypatch, capsys, tmp_path):
    code, out = _run(monkeypatch, capsys, tmp_path, "cost")
    assert code == 0, out
    assert "median pin=True 4.00 tok/s" in out
    assert "median pin=False 1.00 tok/s" in out
    assert "pin=True/pin=False ratio 4.00x" in out

@pytest.mark.parametrize("scenario, ratio", [("no_cost", "1.00x"), ("inverted", "0.25x")])
def test_no_cost_or_an_inverted_cost_fails(monkeypatch, capsys, tmp_path, scenario, ratio):
    code, out = _run(monkeypatch, capsys, tmp_path, scenario)
    assert code == 1, out
    assert f"pin=True/pin=False ratio {ratio}" in out
    assert "measured pinning advantage is absent or too small" in out

@pytest.mark.parametrize(
    "scenario, message",
    [
        ("loss_offset", "loss is not bit-exact before timing"),
        ("inf_loss", "produced a non-finite loss before timing"),
        ("inf_grads", "gradient is non-finite"),
        ("lying_pinned", "requested pin=False, but runtime reports pinned=True"),
    ],
)
def test_an_arm_that_fails_its_gate_is_never_reported(
    monkeypatch, capsys, tmp_path, scenario, message
):
    code, out = _run(monkeypatch, capsys, tmp_path, scenario)
    assert code == 1, out
    assert message in out
    assert "RESULT:" not in out

def test_wrong_pinned_arm_is_reported_but_can_still_be_timed(
    monkeypatch, capsys, tmp_path
):
    code, out = _run(monkeypatch, capsys, tmp_path, "pinned_wrong_after_first")
    assert code == 0, out
    assert "gradients pin=True 4/4,0/4,0/4 WRONG" in out
    assert "RESULT: historical-control pinning cost relationship reproduced" in out

def test_an_invalid_timing_exits_3(monkeypatch, capsys, tmp_path):
    code, out = _run(monkeypatch, capsys, tmp_path, "clock_backwards")
    assert code == 3, out
    assert "invalid measurement" in out

def test_verdict_refuses_a_ratio_inside_the_arms_own_spread():
    reproduced, _, _, ratio, spread = pincost.pinning_verdict([400.0, 100.0, 200.0], [100.0] * 3)
    assert ratio == pytest.approx(2.0)
    assert spread == pytest.approx(1.5)
    assert reproduced is False

@pytest.mark.parametrize("scenario", ["cost", "loss_offset"])
def test_historical_control_is_restored_after_measurement(monkeypatch, capsys, tmp_path, scenario):
    import soup_cli.utils.layer_stream_runtime as layer_runtime
    original = layer_runtime.install_dequant_forward
    _run(monkeypatch, capsys, tmp_path, scenario)
    assert layer_runtime.install_dequant_forward is original


@pytest.mark.parametrize("scenario", ["wrong_first", "wrong_after_first"])
def test_wrong_pageable_arm_refuses_timing_result(monkeypatch, capsys, tmp_path, scenario):
    code, out = _run(monkeypatch, capsys, tmp_path, scenario)
    assert code == 1, out
    assert "pin=False failed gradient correctness across all repetitions:" in out
    assert "median pin=True" not in out
    assert "RESULT:" not in out


def test_historical_pinned_arm_wrong_on_every_backward_can_be_timed(monkeypatch, capsys, tmp_path):
    code, out = _run(monkeypatch, capsys, tmp_path, "pinned_wrong_all")
    assert code == 0, out
    assert "gradients pin=True 0/4,0/4,0/4 WRONG" in out
    assert "gradients pin=False 4/4,4/4,4/4 OK" in out
    assert "median pin=True 4.00 tok/s" in out
    assert "pin=True/pin=False ratio 4.00x" in out
    assert "timing pin=True 4.00 tok/s, median_step=1.0000 s, gradients=WRONG" in out


@pytest.mark.parametrize(
    "scenario, message",
    [
        ("inf_loss_streamed", "produced a non-finite loss before timing"),
        ("inf_loss_reference", "produced a non-finite loss before timing"),
        ("inf_grads_streamed", "streamed gradient is non-finite"),
        ("inf_grads_reference", "reference gradient is non-finite"),
        ("empty_store", "reported an empty streaming store"),
        ("extra_lora", "gradient sets differ"),
    ],
)
def test_independent_measurement_guards(monkeypatch, capsys, tmp_path, scenario, message):
    code, out = _run(monkeypatch, capsys, tmp_path, scenario)
    assert code == 1, out
    assert message in out
    assert "RESULT:" not in out


def test_historical_control_is_installed_for_every_arm(monkeypatch, capsys, tmp_path):
    import soup_cli.utils.layer_stream_runtime as layer_runtime

    installed = []
    monkeypatch.setattr(pincost, "install_historical_control", installed.append)
    code, out = _run(monkeypatch, capsys, tmp_path, "cost")
    assert code == 0, out
    assert installed == [layer_runtime] * 4


def test_self_test_runs():
    assert pincost.run_self_test() == 0


def test_correctness_rejects_empty_canonical_intersection():
    streamed, reference = nn.Module(), nn.Module()
    streamed.register_parameter("streamed_only", nn.Parameter(torch.ones(1)))
    reference.register_parameter("reference_only", nn.Parameter(torch.ones(1)))
    with pytest.raises(ValueError, match="no canonical parameter name is shared"):
        pincost.run_correctness(streamed, reference, torch.ones(1), pin=True)


def test_no_lora_gradient_on_either_side_is_refused(monkeypatch, capsys, tmp_path):
    """0/0 compared gradients is not a correct arm: nothing was checked."""
    original_init = _Toy.__init__

    def init(self, scenario, clock, *, pin=None):
        original_init(self, scenario, clock, pin=pin)
        for name, parameter in self.named_parameters():
            if "lora_" in name:
                parameter.requires_grad_(False)
        self.head.requires_grad_(True)

    monkeypatch.setattr(_Toy, "__init__", init)
    code, out = _run(monkeypatch, capsys, tmp_path, "cost")
    assert code == 1, out
    assert "produced zero LoRA gradients" in out
    assert "RESULT:" not in out