Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

6.8 KB
Raw Blame
from argparse import Namespace

import pytest

from benchmarks.harness.depth_vs_bytes import (
    MeasurementInvalidError,
    depth_vs_bytes_verdict,
    measure_synthetic_sweep,
    run_self_test,
)


def recorded_rows():
    return [
        {
            "depth": 48,
            "quant": "nf4",
            "layer_mib": 163.8,
            "exact_counts": [192] * 3,
            "total": 192,
        },
        {
            "depth": 48,
            "quant": "nf4",
            "layer_mib": 171.5,
            "exact_counts": [192, 8, 8],
            "total": 192,
        },
        {
            "depth": 32,
            "quant": "bf16",
            "layer_mib": 935.0,
            "exact_counts": [128] * 3,
            "total": 128,
        },
    ]


def test_recorded_threshold_pattern_passes():
    assert depth_vs_bytes_verdict(recorded_rows())


def test_every_nf4_point_below_boundary_must_be_exact():
    rows = recorded_rows()
    rows.append(
        {
            "depth": 48,
            "quant": "nf4",
            "layer_mib": 105.0,
            "exact_counts": [192, 8, 8],
            "total": 192,
        }
    )
    assert not depth_vs_bytes_verdict(rows)


def test_every_nf4_point_above_boundary_must_be_wrong_after_first_read():
    rows = recorded_rows()
    rows.append(
        {
            "depth": 48,
            "quant": "nf4",
            "layer_mib": 432.0,
            "exact_counts": [192] * 3,
            "total": 192,
        }
    )
    assert not depth_vs_bytes_verdict(rows)


def test_bf16_control_must_be_exact_and_above_boundary():
    rows = recorded_rows()
    rows[-1]["exact_counts"] = [128, 8, 8]
    assert not depth_vs_bytes_verdict(rows)

    rows = recorded_rows()
    rows[-1]["layer_mib"] = 1.0
    assert not depth_vs_bytes_verdict(rows)


def test_empty_and_nonfinite_measurements_fail():
    assert not depth_vs_bytes_verdict([])
    rows = recorded_rows()
    rows[0]["layer_mib"] = float("inf")
    with pytest.raises(MeasurementInvalidError, match="finite"):
        depth_vs_bytes_verdict(rows)


def test_self_test_pins_recorded_boundaries():
    assert run_self_test() == 0


def test_measurement_sweep_requires_distinct_layer_sizes():
    with pytest.raises(MeasurementInvalidError, match="two layer sizes"):
        measure_synthetic_sweep(intermediate_sizes=[13824, 13824], measure=lambda **_: {})


def test_measurement_sweep_holds_nf4_depth_fixed_and_includes_recorded_bf16_control():
    calls = []

    def measure(**kwargs):
        calls.append(kwargs)
        return {"layer_mib": 1.0, "exact_counts": [192] * 3, "total": 192}

    rows = measure_synthetic_sweep(intermediate_sizes=[13824, 18432], measure=measure)
    assert {row["quant"] for row in rows} == {"nf4", "bf16"}
    assert [call["quant"] for call in calls] == ["nf4", "nf4", "bf16"]
    assert all(call["depth"] == 48 for call in calls[:2])
    assert calls[-1] == {"depth": 32, "intermediate_size": 18432, "quant": "bf16"}
    assert len(rows) == 3


def test_live_sweep_builds_recorded_shapes(tmp_path, monkeypatch):
    from benchmarks.harness import depth_vs_bytes

    generated = []
    measured = []

    def fake_run(command, *, check):
        assert check
        checkpoint = command[command.index("--out") + 1]
        generated.append(command)
        __import__("pathlib").Path(checkpoint).mkdir(parents=True)

    def fake_measure(weights, shards, quant):
        measured.append((weights, shards, quant))
        return {"layer_mib": 200.0, "exact_counts": [1, 0, 0], "total": 1}

    monkeypatch.setattr("subprocess.run", fake_run)
    monkeypatch.setattr(depth_vs_bytes, "_measure_checkpoint", fake_measure)

    rows = depth_vs_bytes.run_live_sweep(tmp_path)

    assert len(generated) == 8
    assert len(rows) == 8
    assert [row["quant"] for row in rows] == ["nf4"] * 7 + ["bf16"]
    assert all(row["depth"] == 48 for row in rows[:7])
    assert rows[-1]["depth"] == 32
    assert rows[-1]["intermediate_size"] == 27648
    assert any("17408" in command for command in generated)
    assert any("18432" in command for command in generated)
    assert len(measured) == 8


def _row(mib, counts, quant="nf4", depth=48, total=192):
    return {
        "depth": depth,
        "quant": quant,
        "layer_mib": mib,
        "exact_counts": counts,
        "total": total,
    }


def test_bracket_needs_a_point_on_each_side():
    bf16 = _row(935.0, [128] * 3, quant="bf16", depth=32, total=128)
    no_below = [_row(171.5, [192, 8, 8]), _row(179.3, [8, 8, 8]), bf16]
    assert not depth_vs_bytes_verdict(no_below)
    no_above = [_row(105.0, [192] * 3), _row(163.8, [192] * 3), bf16]
    assert not depth_vs_bytes_verdict(no_above)


def test_above_bracket_point_must_be_wrong_after_the_first_read():
    rows = recorded_rows()
    rows[1]["exact_counts"] = [8, 192, 192]
    assert not depth_vs_bytes_verdict(rows)


@pytest.mark.parametrize(
    ("field", "value", "message"),
    [
        ("depth", 0, "positive integer"),
        ("quant", "fp8", "nf4 or bf16"),
        ("total", 0, "positive integer"),
        ("exact_counts", [192, 192], "three reads"),
        ("exact_counts", [300, 300, 300], "out of range"),
    ],
)
def test_malformed_rows_fail_closed(field, value, message):
    rows = recorded_rows()
    rows[0][field] = value
    with pytest.raises(MeasurementInvalidError, match=message):
        depth_vs_bytes_verdict(rows)


def test_live_sweep_uses_the_recorded_layer_shape(tmp_path, monkeypatch):
    """The recorded sweep uses hidden 5120 with 40 heads and 10 KV heads."""
    from benchmarks.harness import depth_vs_bytes

    commands = []

    def fake_run(command, *, check):
        assert check
        commands.append(command)
        __import__("pathlib").Path(command[command.index("--out") + 1]).mkdir(parents=True)

    monkeypatch.setattr("subprocess.run", fake_run)
    monkeypatch.setattr(
        depth_vs_bytes,
        "_measure_checkpoint",
        lambda weights, shards, quant: {
            "layer_mib": 200.0,
            "exact_counts": [1, 0, 0],
            "total": 1,
        },
    )
    depth_vs_bytes.run_live_sweep(tmp_path)
    for command in commands:
        assert command[command.index("--heads") + 1] == "40"
        assert command[command.index("--kv-heads") + 1] == "10"


@pytest.mark.parametrize("error", [RuntimeError("runtime"), ValueError("setup")])
def test_live_measurement_setup_errors_are_invalid(monkeypatch, tmp_path, error):
    from benchmarks.harness import depth_vs_bytes

    monkeypatch.setattr(
        depth_vs_bytes,
        "parse_args",
        lambda: Namespace(self_test=False, measure=True, work_dir=tmp_path, records=None),
    )
    monkeypatch.setattr("torch.cuda.is_available", lambda: True)
    monkeypatch.setattr(
        depth_vs_bytes,
        "run_live_sweep",
        lambda work_dir: (_ for _ in ()).throw(error),
    )
    assert depth_vs_bytes.main() == 3