Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

32.8 KB
Raw Blame
"""CPU tests for the L2L step-0 probe harness (benchmarks/harness/l2l_*.py).

The harness is benchmark code, not shipped; these tests pin the parts a GPU run
cannot check by itself: the schedule's bit-exactness against gradient
accumulation, the activation store's byte round trip and spill ordering, the
suspend watch, and the rule's verdict logic.
"""

from __future__ import annotations

import importlib.util
import json
import sys
import threading
import time
from pathlib import Path

import pytest


def _psutil_has_io_counters() -> bool:
    """macOS psutil has no per-process io_counters; the two tests that read it skip there."""
    try:
        import psutil
    except ImportError:
        return False
    return hasattr(psutil.Process(), "io_counters")


_NEEDS_IO_COUNTERS = pytest.mark.skipif(
    not _psutil_has_io_counters(),
    reason="psutil exposes no per-process io_counters on this platform",
)

HARNESS = Path(__file__).resolve().parent.parent / "benchmarks" / "harness"


def _load(name: str):
    if str(HARNESS) not in sys.path:
        sys.path.insert(0, str(HARNESS))
    spec = importlib.util.spec_from_file_location(name, HARNESS / f"{name}.py")
    module = importlib.util.module_from_spec(spec)
    sys.modules[name] = module
    spec.loader.exec_module(module)
    return module


# --------------------------------------------------------------------------
# l2l_activations
# --------------------------------------------------------------------------
@pytest.fixture()
def acts():
    return _load("l2l_activations")


def _chunk(seed: int, shape=(1, 8, 128)):
    import torch

    gen = torch.Generator().manual_seed(seed)
    return torch.randn(shape, generator=gen)


def _store(acts, tmp_path, *, n_layers=4, k=3, spill_layers=0, backend=None, ring=8):
    import torch

    factory = None
    if spill_layers:
        made = backend or acts.BufferedSpillFile

        def factory(size):
            return made(str(tmp_path / "spill.bin"), size)

    return acts.ActivationStore(
        n_layers=n_layers,
        k=k,
        chunk_shape=(1, 8, 128),
        dtype=torch.float32,
        device="cpu",
        pin=False,
        spill_layers=spill_layers,
        spill_factory=factory,
        ring=ring,
    )


def _forward_puts(store, n_layers, k):
    for layer in range(n_layers):
        for mb in range(k):
            store.put(layer, mb, _chunk(layer * 100 + mb))


def test_aligned_buffer_is_sector_aligned_and_exact(acts):
    buf = acts.aligned_buffer(3 * 4096, pin=False)
    assert buf.data_ptr() % 4096 == 0
    assert buf.numel() == 3 * 4096


def test_resident_round_trip_returns_a_copy(acts, tmp_path):
    import torch

    store = _store(acts, tmp_path)
    _forward_puts(store, 4, 3)
    store.begin_backward()
    for layer in range(3, -1, -1):
        for mb in range(3):
            got = store.get(layer, mb)
            assert torch.equal(got, _chunk(layer * 100 + mb))
            got.zero_()  # a caller scribbling on its tensor must not reach the store
            assert torch.equal(store.get(layer, mb), _chunk(layer * 100 + mb))
    store.end_step()
    store.close()


def test_spill_round_trip_picks_the_lowest_layers(acts, tmp_path):
    import torch

    store = _store(acts, tmp_path, spill_layers=2)
    assert [store.is_spilled(layer) for layer in range(4)] == [True, True, False, False]
    assert store.stats.resident_chunks == 2 * 3
    assert store.stats.spilled_chunks == 2 * 3
    _forward_puts(store, 4, 3)
    store.begin_backward()
    for layer in range(3, -1, -1):
        for mb in range(3):
            assert torch.equal(store.get(layer, mb), _chunk(layer * 100 + mb))
    store.end_step()
    assert store.stats.bytes_written == 2 * 3 * store.chunk_bytes
    assert store.stats.bytes_read == 2 * 3 * store.chunk_bytes
    store.close()
    assert not (tmp_path / "spill.bin").exists()


def test_spill_survives_two_steps_with_new_values(acts, tmp_path):
    import torch

    store = _store(acts, tmp_path, spill_layers=2)
    for step in range(2):
        for layer in range(4):
            for mb in range(3):
                store.put(layer, mb, _chunk(step * 1000 + layer * 100 + mb))
        store.begin_backward()
        for layer in range(3, -1, -1):
            for mb in range(3):
                assert torch.equal(store.get(layer, mb), _chunk(step * 1000 + layer * 100 + mb))
        store.end_step()
    store.close()


def test_get_without_begin_backward_reads_on_demand(acts, tmp_path):
    import torch

    store = _store(acts, tmp_path, spill_layers=1)
    _forward_puts(store, 4, 3)
    assert torch.equal(store.get(0, 2), _chunk(2))
    store.close()


def test_a_ring_of_one_back_pressures_instead_of_overwriting(acts, tmp_path):
    import torch

    class SlowFile(acts.BufferedSpillFile):
        def write(self, offset, buf):
            time.sleep(0.02)  # the staging buffer must not be refilled meanwhile
            super().write(offset, buf)

    store = _store(acts, tmp_path, spill_layers=3, backend=SlowFile, ring=1)
    _forward_puts(store, 4, 3)
    store.begin_backward()
    for layer in range(3, -1, -1):
        for mb in range(3):
            assert torch.equal(store.get(layer, mb), _chunk(layer * 100 + mb))
    store.end_step()
    assert store.stats.put_wait_s > 0.0
    store.close()


def test_spill_write_failure_surfaces_on_the_next_call(acts, tmp_path):
    class BrokenFile(acts.BufferedSpillFile):
        def write(self, offset, buf):
            raise OSError("disk full")

    store = _store(acts, tmp_path, spill_layers=2, backend=BrokenFile)
    store.put(0, 0, _chunk(0))
    with pytest.raises(RuntimeError, match="spill.bin"):
        deadline = time.time() + 5
        while time.time() < deadline:
            store.put(0, 1, _chunk(1))
            time.sleep(0.01)
    store.close()


def test_end_step_refuses_unconsumed_spilled_chunks(acts, tmp_path):
    store = _store(acts, tmp_path, spill_layers=2)
    _forward_puts(store, 4, 3)
    store.begin_backward()
    store.get(3, 0)
    with pytest.raises(RuntimeError, match="not consumed"):
        store.end_step()
    store.close()


def test_spill_layers_without_a_factory_is_refused(acts):
    import torch

    with pytest.raises(ValueError, match="spill_factory"):
        acts.ActivationStore(
            n_layers=4,
            k=2,
            chunk_shape=(1, 8, 128),
            dtype=torch.float32,
            device="cpu",
            pin=False,
            spill_layers=1,
        )


def test_direct_spill_refuses_an_unaligned_size(acts, tmp_path):
    with pytest.raises(ValueError, match="multiple of 4096"):
        acts.DirectSpillFile(str(tmp_path / "x.bin"), 4096 + 512)


# --------------------------------------------------------------------------
# l2l_box
# --------------------------------------------------------------------------
@pytest.fixture()
def box():
    return _load("l2l_box")


class _FakeClock:
    def __init__(self, times):
        self.times = list(times)

    def __call__(self):
        return self.times.pop(0)


def test_steady_samples_are_not_void(box):
    watch = box.SuspendWatch(clock=_FakeClock([0, 2, 4, 6]), power=lambda: True)
    for _ in range(4):
        watch.sample()
    report = watch.report()
    assert report["samples"] == 4
    assert report["max_gap_s"] == pytest.approx(2.0)
    assert report["void"] is False


def test_gap_over_limit_voids(box):
    watch = box.SuspendWatch(clock=_FakeClock([0, 2, 33]), power=lambda: True)
    for _ in range(3):
        watch.sample()
    assert watch.report()["max_gap_s"] == pytest.approx(31.0)
    assert watch.report()["void"] is True


def test_battery_sample_voids(box):
    power = iter([True, False, True])
    watch = box.SuspendWatch(clock=_FakeClock([0, 2, 4]), power=lambda: next(power))
    for _ in range(3):
        watch.sample()
    assert watch.report()["battery_samples"] == 1
    assert watch.report()["void"] is True


def test_unknown_power_is_recorded_not_void(box):
    watch = box.SuspendWatch(clock=_FakeClock([0, 2]), power=lambda: None)
    watch.sample()
    watch.sample()
    assert watch.report()["unknown_power_samples"] == 2
    assert watch.report()["void"] is False


def test_thread_start_stop_takes_samples(box):
    watch = box.SuspendWatch(interval=0.01, power=lambda: True)
    watch.start()
    time.sleep(0.1)
    report = watch.stop()
    assert report["samples"] >= 3
    assert not any(t.name == "l2l-suspend-watch" for t in threading.enumerate())


def test_box_stamp_has_the_keys(box):
    stamp = box.box_stamp()
    for key in (
        "unix",
        "avail_phys_gb",
        "commit_avail_gb",
        "ac",
        "gpu_pids",
        "gpu_mem_used_mib",
        "python_processes",
    ):
        assert key in stamp


def test_foreign_readers_flags_heavy_other_processes(box):
    before = {
        10: {"name": "a.exe", "read": 0},
        11: {"name": "b.exe", "read": 5},
        99: {"name": "python.exe", "read": 0},
    }
    after = {
        10: {"name": "a.exe", "read": 2_000_000_000},
        11: {"name": "b.exe", "read": 6},
        99: {"name": "python.exe", "read": 3_000_000_000},
        12: {"name": "SearchIndexer.exe", "read": 1_500_000_000},
    }
    heavy = box.foreign_readers(before, after, own_pids={99}, threshold=1_000_000_000)
    assert [(row["pid"], row["name"]) for row in heavy] == [
        (10, "a.exe"),
        (12, "SearchIndexer.exe"),
    ]
    assert heavy[0]["read_bytes"] == 2_000_000_000


def test_foreign_readers_is_none_without_snapshots(box):
    assert box.foreign_readers(None, {}, own_pids=set()) is None


@_NEEDS_IO_COUNTERS
def test_process_read_bytes_sees_this_process(box):
    import os

    snapshot = box.process_read_bytes()
    if snapshot is None:
        pytest.skip("psutil is not installed")
    assert os.getpid() in snapshot
    assert snapshot[os.getpid()]["read"] >= 0


# --------------------------------------------------------------------------
# l2l_schedule — bit-exact against gradient accumulation, on the SHIPPED runtime
# --------------------------------------------------------------------------
SEQ = 8


@pytest.fixture()
def sched():
    return _load("l2l_schedule")


@pytest.fixture()
def streamed(tmp_path):
    """A tiny untied Llama through shard_checkpoint + build_streamed_model on CPU.

    Untied, so the embed/head share the streamed large slot exactly as the 70B
    fixture's do; three layers, so the zig-zag turnaround and the backward
    prefetch both run.
    """
    pytest.importorskip("peft")
    from peft import LoraConfig, TaskType

    from soup_cli.utils.layer_shard import shard_checkpoint
    from soup_cli.utils.layer_stream_runtime import build_streamed_model
    from tests.test_v07204 import _tiny_llama_dir

    weights, _model, _config = _tiny_llama_dir(tmp_path, n_layers=3, tie=False)
    shards = str(tmp_path / "shards")
    index = shard_checkpoint(weights, shards, dtype="float32", arch="llama")
    lora = LoraConfig(
        r=4,
        lora_alpha=8,
        lora_dropout=0.0,
        bias="none",
        target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
        task_type=TaskType.CAUSAL_LM,
    )
    model, runtime = build_streamed_model(
        model_id=weights,
        shard_dir=shards,
        index=index,
        lora_config=lora,
        device="cpu",
        dtype="float32",
        buffers=2,
        pin=False,
        seed=3,
    )
    yield model, runtime
    runtime.close()


def _micro_batches(k, vocab=64, seed=17):
    import torch

    gen = torch.Generator().manual_seed(seed)
    return [torch.randint(0, vocab, (1, SEQ), generator=gen) for _ in range(k)]


def _cpu_store(acts, k, n_layers=3, hidden=64):
    import torch

    return acts.ActivationStore(
        n_layers=n_layers,
        k=k,
        chunk_shape=(1, SEQ, hidden),
        dtype=torch.float32,
        device="cpu",
        pin=False,
    )


@pytest.mark.parametrize("k", [1, 2, 3])
def test_l2l_equals_gradient_accumulation_bit_for_bit(sched, acts, streamed, k):
    model, runtime = streamed
    sched.make_non_vacuous(model)
    mbs = _micro_batches(k)
    sched.zero_grads(model)
    ga_losses = sched.ga_step(model, mbs)
    ga = sched.collect_grads(model)

    call = sched.capture_layer_call(model, mbs[0])
    step = sched.L2LStep(model, runtime, call)
    sched.zero_grads(model)
    l2l_losses = step.run(mbs, _cpu_store(acts, k))
    l2l = sched.collect_grads(model)

    result = sched.compare(ga, l2l)
    assert result["tensors"] == 3 * 4 * 2  # 3 layers x q/k/v/o x A/B
    assert result["all_zero"] == []
    assert result["equal"], result
    assert sched.losses_equal(ga_losses, l2l_losses)


def test_the_divisor_matters(sched, acts, streamed):
    """Mutation guard: dividing by one micro-batch's tokens instead of all k must differ."""
    model, runtime = streamed
    sched.make_non_vacuous(model)
    mbs = _micro_batches(2)
    sched.zero_grads(model)
    sched.ga_step(model, mbs)
    ga = sched.collect_grads(model)
    step = sched.L2LStep(model, runtime, sched.capture_layer_call(model, mbs[0]))
    sched.zero_grads(model)
    step.run(mbs, _cpu_store(acts, 2), n_items=SEQ - 1)
    assert not sched.compare(ga, sched.collect_grads(model))["equal"]


def test_losses_are_compared_per_position(sched, acts, streamed):
    """Mutation guard: the same micro-batches in another order must not pass."""
    model, runtime = streamed
    mbs = _micro_batches(2)
    sched.zero_grads(model)
    ga_losses = sched.ga_step(model, mbs)
    step = sched.L2LStep(model, runtime, sched.capture_layer_call(model, mbs[0]))
    sched.zero_grads(model)
    swapped = step.run(list(reversed(mbs)), _cpu_store(acts, 2))
    assert not sched.losses_equal(ga_losses, swapped)


def test_one_prime_and_the_loads_of_one_micro_batch(sched, acts, streamed):
    """`prime` loads layer 0 unconditionally and `advance` skips owned slots, so a
    step's load count depends on what the previous walk left in the slots. Both
    measured steps therefore follow the same forward-only walk."""
    model, runtime = streamed
    mbs = _micro_batches(3)
    step = sched.L2LStep(model, runtime, sched.capture_layer_call(model, mbs[0]))
    sched.zero_grads(model)
    loads = runtime.pool.loads
    sched.ga_step(model, mbs[:1])
    ga_loads = runtime.pool.loads - loads
    sched.capture_layer_call(model, mbs[0])
    sched.zero_grads(model)
    loads, primes = runtime.pool.loads, runtime.prefetcher.primes
    step.run(mbs, _cpu_store(acts, 3))
    assert runtime.prefetcher.primes - primes == 1
    assert runtime.pool.loads - loads == ga_loads


def test_checkpoint_flags_are_restored(sched, acts, streamed):
    model, runtime = streamed
    mbs = _micro_batches(2)
    step = sched.L2LStep(model, runtime, sched.capture_layer_call(model, mbs[0]))
    step.run(mbs, _cpu_store(acts, 2))
    assert all(layer.use_checkpoint for layer in step.layers)


def test_capture_drops_num_items_and_keeps_rotary(sched, streamed):
    model, _runtime = streamed
    call = sched.capture_layer_call(model, _micro_batches(1)[0])
    assert "num_items_in_batch" not in call.kwargs
    assert call.kwargs.get("position_embeddings") is not None


def test_run_refuses_more_micro_batches_than_the_store_holds(sched, acts, streamed):
    model, runtime = streamed
    mbs = _micro_batches(3)
    step = sched.L2LStep(model, runtime, sched.capture_layer_call(model, mbs[0]))
    with pytest.raises(ValueError, match="3 micro-batches.*holds 2"):
        step.run(mbs, _cpu_store(acts, 2))
    with pytest.raises(ValueError, match="at least one"):
        step.run([], _cpu_store(acts, 2))


def test_missing_prime_hook_is_refused(sched, streamed):
    model, runtime = streamed
    runtime.hook.remove()
    with pytest.raises(RuntimeError, match="prime hook"):
        sched.L2LStep(model, runtime, sched.LayerCall(args=(), kwargs={}))


def test_snapshot_restore_round_trip(sched, streamed):
    import torch

    model, _runtime = streamed
    snap = sched.snapshot_adapters(model)
    sched.make_non_vacuous(model, seed=99)
    sched.restore_adapters(model, snap)
    for name, param in sched.lora_parameters(model).items():
        assert torch.equal(param.detach(), snap[name])
        assert param.grad is None


# --------------------------------------------------------------------------
# l2l_rule
# --------------------------------------------------------------------------
@pytest.fixture()
def rule():
    return _load("l2l_rule")


def _cmp(equal=True, zero=()):
    return {
        "tensors": 4,
        "unequal": [] if equal else ["x"],
        "max_abs": 0.0 if equal else 1e-3,
        "worst": [],
        "all_zero": list(zero),
        "equal": equal,
    }


def _corr(fixture, *, a_vs_a=True, l2l=True, det=False, error=None):
    rec = {
        "kind": "correctness",
        "fixture": fixture,
        "k": 2,
        "deterministic": det,
        "a_vs_a": _cmp(a_vs_a),
        "a_vs_a_losses_equal": a_vs_a,
        "l2l": _cmp(l2l),
        "l2l_losses_equal": l2l,
    }
    if error:
        rec["deterministic_error"] = error
    return {"meta": {}, "records": [rec]}


def _arm(
    mode, arm, k, rnd, step_s, *, peak=4.4, void=False, variant=None, skipped=None, loads=157.0
):
    return {
        "kind": "arm",
        "mode": mode,
        "arm": arm,
        "k": k,
        "round": rnd,
        "variant": variant,
        "step_s_mean": step_s,
        "tokens_per_step": 512 * k,
        "peak_alloc_gb": peak,
        "layer_loads_per_step": loads,
        "large_loads_per_step": 2.0,
        "bytes_moved_per_step": loads * 441e6 + 2 * 525e6,
        "suspend": {"void": void},
        "store": None,
        "skipped": skipped,
    }


def _timing(**over):
    """Predicted-shape numbers that pass every gate."""
    times = {
        ("plain", "A", 1): 17.1,
        ("plain", "B", 1): 6.43,
        ("l2l", "A", 1): 17.2,
        ("l2l", "B", 1): 6.5,
        ("l2l", "A", 16): 104.0,
        ("l2l", "B", 16): 103.0,
        ("l2l", "C", 4): 26.0,
        ("l2l", "D", 4): 20.0,
        ("l2l", "A", 4): 26.5,
        ("l2l", "B", 4): 26.0,
    }
    times.update(over.pop("times", {}))
    records = []
    for (mode, arm, k), step_s in times.items():
        for rnd in (0, 1):
            records.append(
                _arm(mode, arm, k, rnd, step_s, peak=over.get("peak16", 4.6) if k == 16 else 4.4)
            )
    for patch in over.pop("patch", []):
        patch(records)
    meta = {"direct_io": over.get("direct_io", True), "v3": {"ok": over.get("v3", True)}}
    return {"meta": meta, "records": records}


def _write(tmp_path, timing=None, f1=None, f2=None, f1_det=None, spill=None):
    files = {
        "timing.json": timing or _timing(),
        "correctness_f1.json": f1 or _corr("F1"),
        "correctness_f2.json": f2 or _corr("F2"),
    }
    if f1_det:
        files["correctness_f1_det.json"] = f1_det
    if spill:
        files["spill.json"] = spill
    for name, payload in files.items():
        (tmp_path / name).write_text(json.dumps(payload), encoding="utf-8")
    return str(tmp_path)


def test_all_gates_pass(rule, tmp_path):
    out = rule.evaluate(_write(tmp_path))
    assert [out["gates"][g]["status"] for g in ("G1", "G2", "G3", "G4")] == ["pass"] * 4
    assert out["verdict"] == "BUILD STEP 1"


def test_g2_fails_when_the_read_does_not_hide(rule, tmp_path):
    timing = _timing(times={("l2l", "A", 16): 140.0})  # T_A = 0.74 x T_B
    out = rule.evaluate(_write(tmp_path, timing=timing))
    assert out["gates"]["G2"]["status"] == "fail"
    assert out["verdict"].startswith("STOP")


def test_g4_fails_when_vram_grows(rule, tmp_path):
    out = rule.evaluate(_write(tmp_path, timing=_timing(peak16=5.0)))
    assert out["gates"]["G4"]["status"] == "fail"


def test_round_spread_over_5pct_removes_the_gates_using_that_arm(rule, tmp_path):
    def widen(records):
        for rec in records:
            if (rec["mode"], rec["arm"], rec["k"], rec["round"]) == ("l2l", "A", 16, 1):
                rec["step_s_mean"] = 104.0 * 1.12

    out = rule.evaluate(_write(tmp_path, timing=_timing(patch=[widen])))
    for gate in ("G2", "G3", "G4"):
        assert out["gates"][gate]["status"] == "no verdict", gate
    assert out["verdict"].startswith("NO VERDICT")


def test_suspend_void_on_b16_removes_g2_only(rule, tmp_path):
    def void(records):
        for rec in records:
            if (rec["mode"], rec["arm"], rec["k"]) == ("l2l", "B", 16):
                rec["suspend"]["void"] = True

    out = rule.evaluate(_write(tmp_path, timing=_timing(patch=[void])))
    assert out["gates"]["G2"]["status"] == "no verdict"
    assert out["gates"]["G3"]["status"] == "pass"
    assert out["gates"]["G4"]["status"] == "pass"


def test_v2_judges_the_work_not_the_time(rule, tmp_path):
    """L2L at k=1 skips the checkpoint machinery, so it may be faster than the
    shipped step with the same reads and the same arithmetic (smoke, SmolLM2:
    10-20%). V2 compares loads and bytes per step; the time ratio is I6."""
    out = rule.evaluate(_write(tmp_path, timing=_timing(times={("l2l", "B", 1): 5.5})))
    assert out["validity"]["V2"]["ok"] is True
    assert out["info"]["I6"]["b"] == pytest.approx(5.5 / 6.43)
    assert out["verdict"] == "BUILD STEP 1"


def test_v2_fails_when_l2l_k1_loads_differ_from_the_shipped_step(rule, tmp_path):
    def fewer(records):
        for rec in records:
            if (rec["mode"], rec["k"]) == ("l2l", 1):
                rec["layer_loads_per_step"] = 80.0

    out = rule.evaluate(_write(tmp_path, timing=_timing(patch=[fewer])))
    assert out["validity"]["V2"]["ok"] is False
    for gate in ("G2", "G3", "G4"):
        assert out["gates"][gate]["status"] == "no verdict"


def test_direct_io_off_removes_every_timing_gate(rule, tmp_path):
    out = rule.evaluate(_write(tmp_path, timing=_timing(direct_io=False)))
    assert out["validity"]["V1"]["ok"] is False
    assert out["gates"]["G2"]["status"] == "no verdict"


def test_skipped_k16_removes_g2_to_g4(rule, tmp_path):
    def skip(records):
        for rec in records:
            if rec["k"] == 16:
                rec["skipped"] = "V3: free 12.0 GB < need 10.7 + 4.0 GB"
                rec["step_s_mean"] = None

    out = rule.evaluate(_write(tmp_path, timing=_timing(patch=[skip])))
    for gate in ("G2", "G3", "G4"):
        assert out["gates"][gate]["status"] == "no verdict"


def test_g1_uses_the_deterministic_rerun_when_a_vs_a_is_inexact(rule, tmp_path):
    out = rule.evaluate(
        _write(tmp_path, f1=_corr("F1", a_vs_a=False, l2l=False), f1_det=_corr("F1", det=True))
    )
    assert out["gates"]["G1"]["status"] == "pass"


def test_g1_no_verdict_when_even_deterministic_is_inexact(rule, tmp_path):
    out = rule.evaluate(
        _write(tmp_path, f1=_corr("F1", a_vs_a=False), f1_det=_corr("F1", a_vs_a=False, det=True))
    )
    assert out["gates"]["G1"]["status"] == "no verdict"


def test_g1_fails_when_l2l_differs_with_exact_a_vs_a(rule, tmp_path):
    out = rule.evaluate(_write(tmp_path, f2=_corr("F2", l2l=False)))
    assert out["gates"]["G1"]["status"] == "fail"


def test_g1_fails_on_an_all_zero_gradient(rule, tmp_path):
    f1 = _corr("F1")
    f1["records"][0]["a_vs_a"]["all_zero"] = ["lora_A.x"]
    out = rule.evaluate(_write(tmp_path, f1=f1))
    assert out["gates"]["G1"]["status"] == "fail"


def test_spill_ratio_is_informational(rule, tmp_path):
    spill = {
        "meta": {"direct_io": True},
        "records": [
            _arm("l2l", "A", 16, 0, 104.0, variant="pinned"),
            _arm("l2l", "A", 16, 1, 104.0, variant="pinned"),
            _arm("l2l", "A", 16, 0, 106.0, variant="spill"),
            _arm("l2l", "A", 16, 1, 106.0, variant="spill"),
        ],
    }
    out = rule.evaluate(_write(tmp_path, spill=spill))
    assert out["info"]["I4"]["ratio"] == pytest.approx(104.0 / 106.0)
    assert out["verdict"] == "BUILD STEP 1"


def test_foreign_reader_on_a16_removes_g2_to_g4(rule, tmp_path):
    def heavy(records):
        for rec in records:
            if (rec["mode"], rec["arm"], rec["k"], rec["round"]) == ("l2l", "A", 16, 0):
                rec["foreign_readers"] = [
                    {"pid": 4, "name": "SearchIndexer.exe", "read_bytes": 4_940_000_000}
                ]

    out = rule.evaluate(_write(tmp_path, timing=_timing(patch=[heavy])))
    for gate in ("G2", "G3", "G4"):
        assert out["gates"][gate]["status"] == "no verdict", gate
    assert "SearchIndexer.exe" in out["gates"]["G2"]["detail"]


def test_unknown_foreign_readers_do_not_void(rule, tmp_path):
    def unknown(records):
        for rec in records:
            rec["foreign_readers"] = None

    out = rule.evaluate(_write(tmp_path, timing=_timing(patch=[unknown])))
    assert out["verdict"] == "BUILD STEP 1"


# --------------------------------------------------------------------------
# l2l_probe — pure planning helpers (the GPU paths run in Task 7)
# --------------------------------------------------------------------------
@pytest.fixture()
def probe():
    return _load("l2l_probe")


def test_timing_plan_interleaves_and_alternates(probe):
    plan = probe.plan_timing_arms([1, 2, 16], rounds=2, cd_k=4)
    r0 = [row for row in plan if row[0] == 0]
    r1 = [row for row in plan if row[0] == 1]
    assert r0[:2] == [(0, "plain", "A", 1), (0, "plain", "B", 1)]
    assert r1[:2] == [(1, "plain", "B", 1), (1, "plain", "A", 1)]
    assert [row[3] for row in r0 if row[1] == "l2l" and row[2] in "AB"] == [1, 1, 2, 2, 16, 16]
    assert [row[3] for row in r1 if row[1] == "l2l" and row[2] in "AB"] == [16, 16, 2, 2, 1, 1]
    assert [row[2] for row in r0 if row[1] == "l2l" and row[3] == 2] == ["A", "B"]
    assert [row[2] for row in r1 if row[1] == "l2l" and row[3] == 2] == ["B", "A"]
    assert r0[-2:] == [(0, "l2l", "C", 4), (0, "l2l", "D", 4)]
    assert r1[-2:] == [(1, "l2l", "D", 4), (1, "l2l", "C", 4)]


def test_spill_plan_alternates(probe):
    assert probe.plan_spill_arms(2) == [(0, "pinned"), (0, "spill"), (1, "spill"), (1, "pinned")]


def test_spill_layers_for_the_70b_shape(probe):
    assert probe.spill_layers_for(80, 16, 8) == 40
    assert probe.spill_layers_for(80, 8, 8) == 0
    with pytest.raises(ValueError):
        probe.spill_layers_for(80, 16, 0)


def test_store_allocation_failure_becomes_a_skipped_arm(probe, monkeypatch):
    import torch

    calls = []

    def boom(**_kwargs):
        raise RuntimeError("CUDA error: out of memory")

    monkeypatch.setattr(probe.l2l_activations, "ActivationStore", boom)
    monkeypatch.setattr(probe, "_recover", lambda: calls.append("recovered"))
    store, reason = probe.allocate_store(
        n_layers=4, k=2, chunk_shape=(1, 8, 64), dtype=torch.float32, device="cpu", pin=False
    )
    assert store is None
    assert "out of memory" in reason
    assert calls == ["recovered"]  # drain the stale error AND release the cached pinned blocks


def test_observe_attaches_stamps_watch_and_foreign_readers(probe, monkeypatch):
    snaps = iter(
        [
            {1: {"name": "a.exe", "read": 0}},
            {1: {"name": "a.exe", "read": 2_000_000_000}},
        ]
    )
    monkeypatch.setattr(probe.l2l_box, "process_read_bytes", lambda: next(snaps))
    monkeypatch.setattr(probe.l2l_box, "box_stamp", lambda: {"unix": 0.0})
    result = probe.observe(lambda: {"step_s_mean": 1.0}, interval=0.01)
    assert result["step_s_mean"] == 1.0
    assert result["suspend"]["samples"] >= 2
    assert result["box_before"] == {"unix": 0.0} and result["box_after"] == {"unix": 0.0}
    assert result["foreign_readers"] == [{"pid": 1, "name": "a.exe", "read_bytes": 2_000_000_000}]


@_NEEDS_IO_COUNTERS
def test_observe_stops_the_watch_when_the_body_raises(probe):
    with pytest.raises(ValueError, match="boom"):
        probe.observe(lambda: (_ for _ in ()).throw(ValueError("boom")), interval=0.01)
    assert not any(t.name == "l2l-suspend-watch" for t in threading.enumerate())


def test_fits_in_ram_keeps_the_margin(probe):
    gib8 = 8 * 1024**3
    assert probe.fits_in_ram(gib8, free_gb=8.59 + 4.0 + 0.01) is True
    assert probe.fits_in_ram(gib8, free_gb=8.59 + 3.9) is False
    assert probe.fits_in_ram(gib8, free_gb=None) is False


# --------------------------------------------------------------------------
# final-review fix pass (2026-10-02)
# --------------------------------------------------------------------------
def test_v2_ignores_timing_validity_of_the_k1_arms(rule, tmp_path):
    """I1: V2 is a loads/bytes identity. A 10% round spread on the shipped step
    voids that arm's time (V4), but no gate reads P1's time, so G2-G4 stand."""

    def spread(records):
        for rec in records:
            if (rec["mode"], rec["arm"], rec["round"]) == ("plain", "A", 1):
                rec["step_s_mean"] = 17.1 * 1.10

    out = rule.evaluate(_write(tmp_path, timing=_timing(patch=[spread])))
    assert out["validity"]["V2"]["ok"] is True
    assert out["verdict"] == "BUILD STEP 1"


def test_v2_fails_when_the_shipped_step_never_ran(rule, tmp_path):
    def skip(records):
        for rec in records:
            if rec["mode"] == "plain":
                rec["skipped"] = "never ran"
                rec["step_s_mean"] = None

    out = rule.evaluate(_write(tmp_path, timing=_timing(patch=[skip])))
    assert out["validity"]["V2"]["ok"] is False
    assert out["gates"]["G2"]["status"] == "no verdict"


def _rec_file(*records):
    return {"meta": {}, "records": list(records)}


def test_g1_no_verdict_when_the_deterministic_rerun_raised(rule, tmp_path):
    """I2: the probe writes no a_vs_a when deterministic mode raises."""
    det = _rec_file(
        {
            "kind": "correctness",
            "fixture": "F1",
            "k": 4,
            "deterministic": True,
            "deterministic_error": "an op does not have a deterministic implementation",
        }
    )
    out = rule.evaluate(_write(tmp_path, f1=_corr("F1", a_vs_a=False), f1_det=det))
    assert out["gates"]["G1"]["status"] == "no verdict"
    assert "deterministic" in out["gates"]["G1"]["detail"]


def test_g1_no_verdict_on_an_empty_correctness_file(rule, tmp_path):
    """I2: a block that crashed leaves Sink's first flush, records == []."""
    out = rule.evaluate(_write(tmp_path, f2=_rec_file()))
    assert out["gates"]["G1"]["status"] == "no verdict"


def test_g1_no_verdict_on_a_skipped_correctness_record(rule, tmp_path):
    reason = "activation store allocation failed: CUDA error: out of memory"
    skipped = _rec_file(
        {"kind": "correctness", "fixture": "F2", "k": 2, "deterministic": False, "skipped": reason}
    )
    out = rule.evaluate(_write(tmp_path, f2=skipped))
    assert out["gates"]["G1"]["status"] == "no verdict"
    assert "skipped" in out["gates"]["G1"]["detail"]


def test_start_refusal_names_the_failing_row(probe):
    """I4: a block whose V1/V3 already fail at its start cannot reach a verdict."""
    v3_bad = {"ok": False, "ram_ok": False, "gpu_ok": True, "free_gb": 9.0, "need_gb": 10.7}
    assert probe.start_refusal("timing", v3_bad, True, "disk").startswith("V3")
    assert "V1" in probe.start_refusal("spill", {"ok": True}, False, "disk")
    assert probe.start_refusal("timing", {"ok": True}, True, "disk") is None
    assert probe.start_refusal("timing", {"ok": True}, None, "ram") is None
    assert probe.start_refusal("correctness", {"ok": False}, False, "disk") is None


def test_close_survives_a_spill_file_held_by_another_process(acts, tmp_path, monkeypatch):
    """M2: an indexer or scanner holding the file must not crash the block."""
    store = _store(acts, tmp_path, spill_layers=1)

    def held(path):
        raise PermissionError(13, "The process cannot access the file", path)

    monkeypatch.setattr(acts.os, "remove", held)
    store.close()
    assert store.stats.spill_left_behind == str(tmp_path / "spill.bin")


def test_release_arm_state_drops_the_zero_weight_cache_after_c_and_d(probe):
    """M3: the nodequant arms cache a zero weight per shape on the GPU (~1.1 GB on
    the 70B shape); later arms must not carry it."""

    class Inst:
        def __init__(self):
            self._zero_cache = {(1, 2): object()}

    for arm, emptied in (("A", False), ("B", False), ("C", True), ("D", True)):
        inst = Inst()
        probe.release_arm_state(inst, arm)
        assert (not inst._zero_cache) is emptied, arm


def test_fits_in_ram_also_needs_commit(probe):
    """M4: pinned memory is charged against Windows commit as well."""
    gib8 = 8 * 1024**3
    assert probe.fits_in_ram(gib8, free_gb=20.0, commit_gb=10.0) is False
    assert probe.fits_in_ram(gib8, free_gb=20.0, commit_gb=13.0) is True
    assert probe.fits_in_ram(gib8, free_gb=20.0) is True


def test_the_l2l_step_calls_each_layer_twice_per_micro_batch(sched, acts, streamed):
    """M9: one no-grad forward and one forward under grad per (layer, micro-batch).
    The wrapper's checkpoint left on would add a recompute: three calls, not two."""
    model, runtime = streamed
    mbs = _micro_batches(2)
    step = sched.L2LStep(model, runtime, sched.capture_layer_call(model, mbs[0]))
    calls = []
    original = runtime.prefetcher.advance

    def counting(idx):
        calls.append(idx)
        return original(idx)

    runtime.prefetcher.advance = counting
    try:
        step.run(mbs, _cpu_store(acts, 2))
    finally:
        runtime.prefetcher.advance = original
    assert len(calls) == 2 * 2 * 3