Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""v0.72.0 — Layer Streaming (BETA).
Pure-side truth tables + the four silent-failure regression classes the
correctness/throughput gates surfaced on a real RTX 3050 4 GB box.
The CUDA-gated tests are skipped without a GPU and say so.
"""
import ast
import os
import pytest
from tests.conftest import accelerator_device, mps_is_the_accelerator
# ==========================================================================
# C1 — the pure planner (utils/layer_stream.py)
# ==========================================================================
class _Cfg:
"""Minimal AutoConfig stand-in: the planner only reads model_type."""
def __init__(self, model_type=None, **kw):
if model_type is not None:
self.model_type = model_type
for key, val in kw.items():
setattr(self, key, val)
class TestStreamArchAllowlist:
def test_llama_family_supported(self):
from soup_cli.utils.layer_stream import stream_arch_of
assert stream_arch_of(_Cfg("llama")) == "llama"
def test_qwen2_and_qwen3_supported(self):
from soup_cli.utils.layer_stream import stream_arch_of
assert stream_arch_of(_Cfg("qwen2")) == "qwen2"
assert stream_arch_of(_Cfg("qwen3")) == "qwen3"
def test_case_insensitive(self):
from soup_cli.utils.layer_stream import stream_arch_of
assert stream_arch_of(_Cfg("LLaMA")) == "llama"
def test_unsupported_arch_names_the_allowlist(self):
"""v0.72.0 scope is Llama/Qwen. The refusal must say what IS allowed."""
from soup_cli.utils.layer_stream import stream_arch_of
with pytest.raises(ValueError, match="llama"):
stream_arch_of(_Cfg("gpt2"))
def test_multimodal_gemma3_is_rejected_while_gemma3_text_is_not(self):
"""v0.72.3 added the Gemma family, but ``gemma3`` and ``gemma3_text`` are
NOT the same thing: a real ``google/gemma-3-*`` reports ``gemma3`` for its
vision-capable wrapper. Streaming that as a causal LM is exactly the
silent mis-train the allowlist exists to prevent, so the pair is pinned
together — the accepted half is the control that makes the refusal
meaningful rather than a spelling accident."""
from soup_cli.utils.layer_stream import stream_arch_of
assert stream_arch_of(_Cfg("gemma3_text")) == "gemma3_text"
with pytest.raises(ValueError, match="gemma3"):
stream_arch_of(_Cfg("gemma3"))
def test_missing_model_type_raises(self):
from soup_cli.utils.layer_stream import stream_arch_of
with pytest.raises(ValueError, match="model_type"):
stream_arch_of(_Cfg())
class TestChooseTier:
def test_fits_in_ram_returns_ram(self):
from soup_cli.utils.layer_stream import TIER_RAM, choose_tier
assert choose_tier(1_000, 10_000, "nvme") == TIER_RAM
def test_headroom_is_strict_at_the_boundary(self):
"""model_bytes < free*0.7 — exactly at 0.7 is NOT ram (plan 5.1)."""
from soup_cli.utils.layer_stream import TIER_DISK, choose_tier
assert choose_tier(7_000, 10_000, "nvme") == TIER_DISK
assert choose_tier(6_999, 10_000, "nvme") == "ram"
def test_too_big_for_ram_falls_to_nvme(self):
from soup_cli.utils.layer_stream import TIER_DISK, choose_tier
assert choose_tier(9_000, 10_000, "nvme") == TIER_DISK
def test_hdd_is_refused_loudly(self):
"""plan P11: 80 shards x 2 reads = 160 seeks per step."""
from soup_cli.utils.layer_stream import choose_tier
with pytest.raises(ValueError, match="NVMe"):
choose_tier(9_000, 10_000, "hdd")
def test_sata_ssd_is_refused(self):
from soup_cli.utils.layer_stream import choose_tier
with pytest.raises(ValueError, match="NVMe"):
choose_tier(9_000, 10_000, "sata")
def test_unknown_disk_kind_is_refused(self):
from soup_cli.utils.layer_stream import choose_tier
with pytest.raises(ValueError, match="NVMe"):
choose_tier(9_000, 10_000, "wat")
class TestDecidePinning:
"""User requirement: the 7.12 GB pinned ceiling measured on the dev box is
real behaviour, not a caveat. Exceeding it must fall back to a pageable
store AND say the utilisation cost out loud."""
def test_store_within_limit_is_pinned(self):
from soup_cli.utils.layer_stream import decide_pinning
decision = decide_pinning(2 * 10**9, 7 * 10**9)
assert decision.pinned is True
def test_store_above_limit_falls_back_to_pageable(self):
from soup_cli.utils.layer_stream import decide_pinning
decision = decide_pinning(8 * 10**9, 7 * 10**9)
assert decision.pinned is False
def test_fallback_reason_states_the_utilisation_cost(self):
"""Silently taking the 97% -> 79% hit is the failure mode."""
from soup_cli.utils.layer_stream import decide_pinning
decision = decide_pinning(8 * 10**9, 7 * 10**9)
assert "utilisation" in decision.reason.lower()
assert "pageable" in decision.reason.lower()
def test_unknown_limit_attempts_pinning(self):
"""No probe result -> try pinned; the runtime falls back on failure."""
from soup_cli.utils.layer_stream import decide_pinning
assert decide_pinning(8 * 10**9, None).pinned is True
def test_boundary_equal_is_pinned(self):
from soup_cli.utils.layer_stream import decide_pinning
assert decide_pinning(7 * 10**9, 7 * 10**9).pinned is True
def test_stream_pin_false_forces_pageable_and_states_the_cost(self):
"""#366: pinning could not be turned off from config. stream_pin=false
forces the pageable store even where the box could page-lock it, and the
reason states the throughput it costs rather than absorbing it.
#366 re-review: pin the ACTUAL figures, not just the word 'throughput' —
a bare-word assertion let the measured gain be changed to 1.01 with the
suite green. The forced-off reason must carry the same GPU-utilisation
sentence the automatic-fallback branch does, and both benchmark figures.
"""
from soup_cli.utils.layer_stream import (
PIN_THROUGHPUT_GAIN_REAL,
PIN_THROUGHPUT_GAIN_SYNTHETIC,
decide_pinning,
)
decision = decide_pinning(2 * 10**9, 7 * 10**9, stream_pin=False)
assert decision.pinned is False
reason = decision.reason
# The exact measured figures (benchmarks/gate-h100-validation.md), so a
# silent change to either constant fails here.
assert f"{PIN_THROUGHPUT_GAIN_REAL:.2f}x" in reason
assert f"{PIN_THROUGHPUT_GAIN_SYNTHETIC:.2f}x" in reason
assert "6.56x" in reason and "7.41x" in reason
# The same GPU-utilisation sentence the automatic fallback carries.
assert "~97% to ~79%" in reason
def test_stream_pin_true_forces_pinned_even_above_ceiling(self):
"""The override pins where the automatic path would have fallen back."""
from soup_cli.utils.layer_stream import decide_pinning
decision = decide_pinning(50 * 10**9, 7 * 10**9, stream_pin=True)
assert decision.pinned is True
assert "refuses" in decision.reason.lower()
def test_stream_pin_none_keeps_automatic_behaviour(self):
from soup_cli.utils.layer_stream import decide_pinning
assert decide_pinning(8 * 10**9, 7 * 10**9, stream_pin=None).pinned is False
assert decide_pinning(2 * 10**9, 7 * 10**9, stream_pin=None).pinned is True
class TestBufferValidation:
def test_default_is_two(self):
from soup_cli.utils.layer_stream import DEFAULT_STREAM_BUFFERS
assert DEFAULT_STREAM_BUFFERS == 2
def test_valid_range_accepted(self):
from soup_cli.utils.layer_stream import validate_stream_buffers
for n in (2, 3, 8):
assert validate_stream_buffers(n) == n
def test_one_buffer_is_a_scheduler_bug_not_a_config(self):
"""plan 8: '1 is a scheduler bug, not a config'."""
from soup_cli.utils.layer_stream import validate_stream_buffers
with pytest.raises(ValueError, match="2"):
validate_stream_buffers(1)
def test_above_max_rejected(self):
from soup_cli.utils.layer_stream import validate_stream_buffers
with pytest.raises(ValueError, match="8"):
validate_stream_buffers(9)
def test_bool_rejected(self):
from soup_cli.utils.layer_stream import validate_stream_buffers
with pytest.raises(ValueError, match="bool"):
validate_stream_buffers(True)
class TestThroughputEstimate:
def test_uses_c_equals_six_for_gradient_checkpointing(self):
"""plan 4.3: C=6 WITH checkpointing (2 fwd + 2 recompute + 2 dL/dx).
The source docs used C=4 while also mandating checkpointing."""
from soup_cli.utils.layer_stream import FLOPS_PER_PARAM_PER_TOKEN
assert FLOPS_PER_PARAM_PER_TOKEN == 6
def test_hand_computed_rate(self):
from soup_cli.utils.layer_stream import estimate_stream_tokens_per_sec
# 4.9 TFLOPS effective / (6 * 1.5437e9) = 529.1 tok/s
got = estimate_stream_tokens_per_sec(1_543_714_304, 4.9)
assert got == pytest.approx(529.1, rel=0.01)
def test_matches_the_measured_15b_number_within_10pct(self):
"""Calibration: the dev box measured 525.0 tok/s on Qwen2.5-1.5B at
4.86 TFLOPS effective. The model must reproduce it."""
from soup_cli.utils.layer_stream import estimate_stream_tokens_per_sec
got = estimate_stream_tokens_per_sec(1_543_714_304, 4.86)
assert 0.9 * 525.0 <= got <= 1.1 * 525.0
def test_zero_params_rejected(self):
from soup_cli.utils.layer_stream import estimate_stream_tokens_per_sec
with pytest.raises(ValueError):
estimate_stream_tokens_per_sec(0, 4.9)
def test_epoch_seconds(self):
from soup_cli.utils.layer_stream import estimate_epoch_seconds
assert estimate_epoch_seconds(1_000_000, 143.1) == pytest.approx(6987.0, rel=0.01)
def test_epoch_seconds_rejects_non_positive_rate(self):
from soup_cli.utils.layer_stream import estimate_epoch_seconds
with pytest.raises(ValueError):
estimate_epoch_seconds(1000, 0.0)
class TestVramEstimate:
def test_sums_the_plan_41_terms(self):
from soup_cli.utils.layer_stream import estimate_stream_vram
got = estimate_stream_vram(
layer_bytes=100,
buffers=2,
embed_bytes=50,
adapter_bytes=10,
activation_bytes=5,
logits_bytes=20,
workspace_bytes=1000,
)
assert got == 2 * 100 + 50 + 10 + 5 + 20 + 1000
def test_buffer_count_scales_the_layer_term(self):
from soup_cli.utils.layer_stream import estimate_stream_vram
two = estimate_stream_vram(layer_bytes=100, buffers=2, embed_bytes=0, workspace_bytes=0)
three = estimate_stream_vram(layer_bytes=100, buffers=3, embed_bytes=0, workspace_bytes=0)
assert three - two == 100
def test_large_layer_slot_is_charged_once(self):
from soup_cli.utils.layer_stream import estimate_stream_vram
got = estimate_stream_vram(
layer_bytes=100,
buffers=2,
embed_bytes=25,
large_layer_bytes=400,
workspace_bytes=0,
)
assert got == 2 * 100 + 25 + 400
def test_logits_budget_includes_the_whole_loss_path(self):
"""plan P5: logits, not weights, OOM you first on a small card.
v0.72.0 charged 2 + 4 here (bf16 logits + fp32 upcast) from first
principles. v0.72.3 GATE 2 measured the real peak: 14 bytes per element,
because ``ForCausalLMLoss`` also holds log-softmax's fp32 output and the
fp32 gradient live at the same time. The old figure under-predicted this
term by 2.33x — ~5 GB on a 152k-vocab model at batch 8."""
from soup_cli.utils.layer_stream import estimate_logits_bytes
got = estimate_logits_bytes(vocab_size=151936, seq_len=512, batch_size=1)
assert got == 512 * 151936 * 14
def test_logits_budget_without_upcast(self):
from soup_cli.utils.layer_stream import estimate_logits_bytes
got = estimate_logits_bytes(
vocab_size=1000, seq_len=10, batch_size=1, upcast_fp32=False
)
assert got == 10 * 1000 * 2
class TestLayerSpec:
def test_nbytes_from_shape_and_dtype(self):
from soup_cli.utils.layer_stream import LayerSpec
spec = LayerSpec(name="self_attn.q_proj.weight", shape=(2048, 2048), dtype="bfloat16")
assert spec.nbytes == 2048 * 2048 * 2
def test_is_frozen(self):
import dataclasses
from soup_cli.utils.layer_stream import LayerSpec
spec = LayerSpec(name="w", shape=(2, 2), dtype="bfloat16")
with pytest.raises(dataclasses.FrozenInstanceError):
spec.name = "other"
def test_unsupported_dtype_rejected(self):
from soup_cli.utils.layer_stream import LayerSpec
with pytest.raises(ValueError, match="dtype"):
LayerSpec(name="w", shape=(2, 2), dtype="int4").nbytes
#: Distinguishing fragment of the forced-ON pin note, so the assertions below
#: select that note rather than any line that happens to mention the key.
_FORCED_PIN_TEXT = "training.stream_pin=true"
#: Free RAM too small for the fixture base, so the plan lands on the disk tier.
_NO_RAM_BYTES = 1000
class TestStreamPlan:
def test_build_plan_reports_tier_and_pinning(self):
from soup_cli.utils.layer_stream import build_stream_plan
plan = build_stream_plan(
arch="qwen2",
n_layers=36,
layer_bytes=154 * 10**6,
embed_bytes=622 * 10**6,
available_ram_bytes=12 * 10**9,
pinned_limit_bytes=7 * 10**9,
buffers=2,
disk_kind="nvme",
)
assert plan.tier == "ram"
assert plan.n_layers == 36
assert plan.store_bytes == 36 * 154 * 10**6
# 5.5 GB store fits under a 7 GB pinned ceiling
assert plan.pinned is True
def test_large_layers_count_in_host_store_but_only_one_device_slot(self):
from soup_cli.utils.layer_stream import build_stream_plan
plan = build_stream_plan(
arch="llama",
n_layers=2,
layer_bytes=100,
embed_bytes=25,
large_store_bytes=800,
large_buffer_bytes=400,
available_ram_bytes=10_000,
pinned_limit_bytes=10_000,
buffers=2,
disk_kind="nvme",
)
assert plan.store_bytes == 2 * 100 + 800
assert plan.large_store_bytes == 800
assert plan.buffer_bytes == 2 * 100 + 400
def test_plan_falls_back_to_pageable_and_records_a_note(self):
from soup_cli.utils.layer_stream import build_stream_plan
plan = build_stream_plan(
arch="qwen2",
n_layers=36,
layer_bytes=300 * 10**6,
embed_bytes=622 * 10**6,
available_ram_bytes=40 * 10**9,
pinned_limit_bytes=7 * 10**9,
buffers=2,
disk_kind="nvme",
)
assert plan.pinned is False
assert any("pageable" in note.lower() for note in plan.notes)
def test_stream_pin_false_reaches_the_plan_as_pageable_with_cost(self):
"""#366 criteria 1+2: stream_pin=false proceeds pageable AND the
pre-flight note states the throughput cost — the store here fits under
the pinned ceiling, so the automatic path would otherwise pin it."""
from soup_cli.utils.layer_stream import build_stream_plan
plan = build_stream_plan(
arch="qwen2",
n_layers=36,
layer_bytes=154 * 10**6,
embed_bytes=622 * 10**6,
available_ram_bytes=12 * 10**9,
pinned_limit_bytes=7 * 10**9,
buffers=2,
disk_kind="nvme",
stream_pin=False,
)
assert plan.pinned is False
assert any("throughput" in note.lower() for note in plan.notes)
def _plan_with_stream_pin(self, stream_pin, *, available_ram_bytes=12 * 10**9):
from soup_cli.utils.layer_stream import build_stream_plan
return build_stream_plan(
arch="qwen2",
n_layers=36,
layer_bytes=154 * 10**6,
embed_bytes=622 * 10**6,
available_ram_bytes=available_ram_bytes,
pinned_limit_bytes=7 * 10**9,
buffers=2,
disk_kind="nvme",
stream_pin=stream_pin,
)
@staticmethod
def _forced_on_notes(plan):
return [note for note in plan.notes if _FORCED_PIN_TEXT in note]
def test_stream_pin_true_is_recorded_in_the_plan_notes(self):
"""#366 round-3 nit: `decide_pinning`'s docstring says True "only records
the intent so the pre-flight reflects it", but the plan appended the
reason only when the decision came back UNpinned — so a forced-on pin was
the one branch that decided something and said nothing. This PR's whole
framing is record, never silence."""
plan = self._plan_with_stream_pin(True)
assert plan.pinned is True
notes = self._forced_on_notes(plan)
assert notes, plan.notes
# The SEMANTICS, not just the key name: a note that merely mentioned
# stream_pin would otherwise pass.
assert any("page-locked host memory" in note for note in notes), notes
assert any("the run refuses" in note for note in notes), notes
def test_automatic_pinning_stays_quiet(self):
"""Control: only an EXPLICIT request is worth a note. Without this a
mutant that always appended the reason would satisfy the case above, and
every automatic run would grow a line of noise."""
plan = self._plan_with_stream_pin(None)
assert plan.pinned is True
assert self._forced_on_notes(plan) == [], plan.notes
def test_the_disk_tier_records_the_forced_pin_too_and_names_its_staging(self):
"""This assertion used to be its exact inverse, and the inversion is the
finding. The note was withheld on the disk tier because it claimed a RAM
store that tier did not have, and the runtime announced the
inapplicability instead — but #971 gave the disk tier host staging that
`stream_pin` honours or refuses, and DELETED that announcement. Withheld,
the explicit request was recorded nowhere at all on the tier where it now
decides something. It must print, and it must name what this tier
actually page-locks."""
plan = self._plan_with_stream_pin(True, available_ram_bytes=_NO_RAM_BYTES)
assert plan.tier == "disk"
notes = self._forced_on_notes(plan)
assert notes, plan.notes
assert any("staging" in note for note in notes), notes
# And it must not have become a disk-only note: the RAM tier keeps it.
assert any("base store on the RAM tier" in note for note in notes), notes
def test_both_spellings_of_the_disk_tier_print_the_same_pin_prose(self):
"""One tier, two behaviours chosen by the spelling — the defect R2 fixed
for the pin itself, still present in the prose that explains it.
`build_stream_plan` computes its notes from `choose_tier`'s answer,
BEFORE `stream_setup` forces the tier, so gating the note on
`tier == TIER_RAM` meant `stream_source: disk` on a RAM-sized box printed
a RAM-tier promise under a panel headed `tier disk`, while `auto` on a
RAM-poor box printed nothing. With the gate gone the note is identical
either way — which is only checkable because it now names both tiers.
"""
fell_into = self._plan_with_stream_pin(True, available_ram_bytes=_NO_RAM_BYTES)
forced_from = self._plan_with_stream_pin(True)
assert fell_into.tier == "disk" and forced_from.tier == "ram"
assert self._forced_on_notes(fell_into) == self._forced_on_notes(forced_from)
def test_plan_is_frozen(self):
import dataclasses
from soup_cli.utils.layer_stream import build_stream_plan
plan = build_stream_plan(
arch="llama", n_layers=2, layer_bytes=10, embed_bytes=10,
available_ram_bytes=10**9, pinned_limit_bytes=None, buffers=2, disk_kind="nvme",
)
with pytest.raises(dataclasses.FrozenInstanceError):
plan.tier = "disk"
def test_render_panel_names_the_beta_status_and_tier(self):
from rich.console import Console
from rich.panel import Panel
from soup_cli.utils.layer_stream import build_stream_plan, render_stream_panel
plan = build_stream_plan(
arch="llama", n_layers=2, layer_bytes=10, embed_bytes=10,
available_ram_bytes=10**9, pinned_limit_bytes=None, buffers=2, disk_kind="nvme",
)
panel = render_stream_panel(plan)
assert isinstance(panel, Panel)
import io
buf = io.StringIO()
Console(file=buf, width=100).print(panel)
out = buf.getvalue()
assert "BETA" in out
assert "ram" in out.lower()
class TestNoTopLevelTorch:
"""utils/layer_stream.py is the pure half — importing torch at module level
would put torch on the light CLI's import path."""
def test_layer_stream_has_no_top_level_torch(self):
import soup_cli.utils.layer_stream as mod
source = open(mod.__file__, encoding="utf-8").read()
tree = ast.parse(source)
for node in tree.body:
if isinstance(node, ast.Import):
for alias in node.names:
assert not alias.name.startswith("torch"), alias.name
elif isinstance(node, ast.ImportFrom):
assert not (node.module or "").startswith("torch"), node.module
def test_importable_without_torch_installed(self):
"""The light install has no torch; the planner must still import."""
import importlib
import sys
saved = {k: v for k, v in sys.modules.items() if k.startswith("torch")}
for key in list(saved):
sys.modules[key] = None
try:
importlib.reload(importlib.import_module("soup_cli.utils.layer_stream"))
finally:
for key, val in saved.items():
sys.modules[key] = val
importlib.reload(importlib.import_module("soup_cli.utils.layer_stream"))
# ==========================================================================
# C2 — the checkpoint sharder (utils/layer_shard.py)
# ==========================================================================
def _write_safetensors(path, tensors):
"""Write a fake HF weights shard (torch is a [train]-extra dep)."""
from safetensors.torch import save_file
save_file({k: v.clone() for k, v in tensors.items()}, path)
return path
def _fake_weights_dir(tmp_path, n_layers=3, split=False, untied=False):
import torch
torch.manual_seed(0)
layer = {}
for idx in range(n_layers):
pre = f"model.layers.{idx}."
layer[pre + "self_attn.q_proj.weight"] = torch.randn(8, 8, dtype=torch.float32)
layer[pre + "mlp.down_proj.weight"] = torch.randn(8, 16, dtype=torch.float32)
layer[pre + "input_layernorm.weight"] = torch.randn(8, dtype=torch.float32)
extras = {
"model.embed_tokens.weight": torch.randn(32, 8, dtype=torch.float32),
"model.norm.weight": torch.randn(8, dtype=torch.float32),
}
if untied:
extras["lm_head.weight"] = torch.randn(32, 8, dtype=torch.float32)
src = tmp_path / "weights"
src.mkdir()
if split:
# a layer's tensors deliberately straddle two files
keys = sorted(layer)
half = len(keys) // 2
_write_safetensors(
str(src / "model-00001.safetensors"), {k: layer[k] for k in keys[:half]}
)
_write_safetensors(
str(src / "model-00002.safetensors"),
{**{k: layer[k] for k in keys[half:]}, **extras},
)
else:
_write_safetensors(str(src / "model.safetensors"), {**layer, **extras})
return str(src), layer, extras
class TestShardRoundTrip:
def test_layer_tensors_survive_byte_identical(self, tmp_path):
"""A sharder that silently corrupts a tensor still trains — badly."""
import torch
from safetensors.torch import load_file
from soup_cli.utils.layer_shard import layer_shard_path, shard_checkpoint
src, layer, _ = _fake_weights_dir(tmp_path)
out = str(tmp_path / "shards")
index = shard_checkpoint(src, out, dtype="float32")
assert index.n_layers == 3
blob = load_file(layer_shard_path(out, 1))
for key, tensor in layer.items():
if key.startswith("model.layers.1."):
short = key[len("model.layers.1.") :]
assert torch.equal(blob[short], tensor), short
def test_tied_embedding_stays_resident_and_unaffected(self, tmp_path):
import torch
from safetensors.torch import load_file
from soup_cli.utils.layer_shard import extras_shard_path, shard_checkpoint
src, _, extras = _fake_weights_dir(tmp_path)
out = str(tmp_path / "shards")
index = shard_checkpoint(src, out, dtype="float32")
resident = load_file(extras_shard_path(out))
assert set(resident) == {"model.embed_tokens.weight", "model.norm.weight"}
assert torch.equal(
resident["model.embed_tokens.weight"], extras["model.embed_tokens.weight"]
)
assert index.large_keys == ()
def test_untied_vocabulary_pair_is_split_from_resident_extras(self, tmp_path):
import torch
from safetensors.torch import load_file
from soup_cli.utils.layer_shard import (
extras_shard_path,
large_shard_path,
shard_checkpoint,
)
src, _, weights = _fake_weights_dir(tmp_path, untied=True)
out = str(tmp_path / "shards")
index = shard_checkpoint(src, out, dtype="float32")
resident = load_file(extras_shard_path(out))
embedding = load_file(large_shard_path(out, "model.embed_tokens.weight"))
head = load_file(large_shard_path(out, "lm_head.weight"))
assert set(resident) == {"model.norm.weight"}
assert torch.equal(
embedding["model.embed_tokens.weight"], weights["model.embed_tokens.weight"]
)
assert torch.equal(head["lm_head.weight"], weights["lm_head.weight"])
assert index.large_keys == ("lm_head.weight", "model.embed_tokens.weight")
def test_layers_split_across_source_files_are_gathered(self, tmp_path):
"""Real checkpoints shard by size, not by layer boundary."""
import torch
from safetensors.torch import load_file
from soup_cli.utils.layer_shard import layer_shard_path, shard_checkpoint
src, layer, _ = _fake_weights_dir(tmp_path, split=True)
out = str(tmp_path / "shards")
index = shard_checkpoint(src, out, dtype="float32")
assert index.n_layers == 3
for idx in range(3):
blob = load_file(layer_shard_path(out, idx))
assert set(blob) == {
"self_attn.q_proj.weight",
"mlp.down_proj.weight",
"input_layernorm.weight",
}
pre = f"model.layers.{idx}."
assert torch.equal(
blob["mlp.down_proj.weight"], layer[pre + "mlp.down_proj.weight"]
)
def test_dtype_is_converted(self, tmp_path):
import torch
from safetensors.torch import load_file
from soup_cli.utils.layer_shard import layer_shard_path, shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path)
out = str(tmp_path / "shards")
index = shard_checkpoint(src, out, dtype="bfloat16")
assert index.dtype == "bfloat16"
blob = load_file(layer_shard_path(out, 0))
assert blob["self_attn.q_proj.weight"].dtype == torch.bfloat16
def test_index_round_trips(self, tmp_path):
from soup_cli.utils.layer_shard import read_shard_index, shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path)
out = str(tmp_path / "shards")
written = shard_checkpoint(src, out, dtype="float32", arch="llama")
read = read_shard_index(out)
assert read == written
assert read.arch == "llama"
assert read.total_params > 0
assert "self_attn.q_proj.weight" in read.layer_keys
def test_second_call_reuses_the_cache(self, tmp_path):
from soup_cli.utils.layer_shard import layer_shard_path, shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path)
out = str(tmp_path / "shards")
shard_checkpoint(src, out, dtype="float32")
before = os.path.getmtime(layer_shard_path(out, 0))
shard_checkpoint(src, out, dtype="float32")
assert os.path.getmtime(layer_shard_path(out, 0)) == before
def test_force_reshards(self, tmp_path):
from soup_cli.utils.layer_shard import read_shard_index, shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path)
out = str(tmp_path / "shards")
shard_checkpoint(src, out, dtype="float32")
index = shard_checkpoint(src, out, dtype="bfloat16", force=True)
assert index.dtype == "bfloat16"
assert read_shard_index(out).dtype == "bfloat16"
def test_dtype_change_invalidates_a_stale_cache(self, tmp_path):
"""Reusing float32 shards for a bfloat16 request would stream the wrong
dtype into the pool and silently mis-train."""
from soup_cli.utils.layer_shard import shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path)
out = str(tmp_path / "shards")
shard_checkpoint(src, out, dtype="float32")
index = shard_checkpoint(src, out, dtype="bfloat16")
assert index.dtype == "bfloat16"
class TestShardGuards:
def test_no_safetensors_refuses(self, tmp_path):
from soup_cli.utils.layer_shard import shard_checkpoint
empty = tmp_path / "empty"
empty.mkdir()
with pytest.raises(FileNotFoundError, match="safetensors"):
shard_checkpoint(str(empty), str(tmp_path / "out"))
def test_no_decoder_layers_refuses(self, tmp_path):
import torch
from soup_cli.utils.layer_shard import shard_checkpoint
src = tmp_path / "weights"
src.mkdir()
_write_safetensors(
str(src / "model.safetensors"), {"model.embed_tokens.weight": torch.randn(4, 4)}
)
with pytest.raises(ValueError, match="decoder layer"):
shard_checkpoint(str(src), str(tmp_path / "out"), dtype="float32")
@pytest.mark.requires_symlink
def test_symlinked_source_shard_is_skipped(self, tmp_path):
"""Mirrors spectrum_scan._discover_safetensors."""
import torch
from soup_cli.utils.layer_shard import shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path)
outside = tmp_path / "outside.safetensors"
_write_safetensors(
str(outside), {"model.layers.9.mlp.down_proj.weight": torch.randn(4, 4)}
)
os.symlink(str(outside), os.path.join(src, "model-evil.safetensors"))
index = shard_checkpoint(src, str(tmp_path / "out"), dtype="float32")
assert index.n_layers == 3 # the symlinked layer 9 was not picked up
def test_too_many_layers_refused(self, tmp_path, monkeypatch):
import soup_cli.utils.layer_shard as mod
src, _, _ = _fake_weights_dir(tmp_path)
monkeypatch.setattr(mod, "_MAX_LAYERS", 2)
with pytest.raises(ValueError, match="layers"):
mod.shard_checkpoint(src, str(tmp_path / "out"), dtype="float32")
def test_oversized_tensor_refused(self, tmp_path, monkeypatch):
import soup_cli.utils.layer_shard as mod
src, _, _ = _fake_weights_dir(tmp_path)
monkeypatch.setattr(mod, "_MAX_TENSOR_ELEMENTS", 4)
with pytest.raises(ValueError, match="too large"):
mod.shard_checkpoint(src, str(tmp_path / "out"), dtype="float32")
def test_unsupported_dtype_refused(self, tmp_path):
from soup_cli.utils.layer_shard import shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path)
with pytest.raises(ValueError, match="dtype"):
shard_checkpoint(src, str(tmp_path / "out"), dtype="int4")
def test_corrupt_index_reshards_instead_of_crashing(self, tmp_path):
from soup_cli.utils.layer_shard import shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path)
out = str(tmp_path / "shards")
shard_checkpoint(src, out, dtype="float32")
with open(os.path.join(out, "index.json"), "w", encoding="utf-8") as fh:
fh.write("{not json")
index = shard_checkpoint(src, out, dtype="float32")
assert index.n_layers == 3
class TestShardCacheDir:
def test_slug_is_traversal_safe(self):
from soup_cli.utils.layer_shard import model_slug
slug = model_slug("../../etc/passwd")
assert "/" not in slug and "\\" not in slug and ".." not in slug
def test_slug_maps_org_slash_name(self):
from soup_cli.utils.layer_shard import model_slug
assert model_slug("Qwen/Qwen2.5-1.5B") == "Qwen__Qwen2.5-1.5B"
def test_slug_rejects_empty(self):
from soup_cli.utils.layer_shard import model_slug
with pytest.raises(ValueError):
model_slug(" ")
def test_env_override_inside_tmp_is_honoured(self, tmp_path, monkeypatch):
from soup_cli.utils.layer_shard import resolve_shard_dir
monkeypatch.setenv("SOUP_LAYER_STREAM_CACHE_DIR", str(tmp_path))
got = resolve_shard_dir("Qwen/Qwen2.5-1.5B")
assert os.path.realpath(str(tmp_path)) in os.path.realpath(got)
def test_env_override_outside_bounds_falls_back(self, monkeypatch):
"""An env var is operator config, not API input — silent fall-through."""
from soup_cli.utils.layer_shard import (
default_layer_stream_cache_dir,
resolve_shard_dir,
)
bad = "C:\\Windows\\System32" if os.name == "nt" else "/etc/soup-evil"
monkeypatch.setenv("SOUP_LAYER_STREAM_CACHE_DIR", bad)
got = os.path.realpath(resolve_shard_dir("m"))
assert got.startswith(os.path.realpath(default_layer_stream_cache_dir()))
def test_control_chars_in_override_rejected(self, tmp_path, monkeypatch):
"""ESC, not NUL: a NUL byte cannot live in an environment variable at
all (POSIX putenv and CPython >= 3.11 both reject it outright), so a
NUL-based test only ever passed on one platform/interpreter pair.
The override here is otherwise VALID — a real path under $TMPDIR — so
the control character is the ONLY reason it can be refused. Drop the
`ord(ch) < 0x20` guard and this test goes red.
"""
from soup_cli.utils.layer_shard import (
default_layer_stream_cache_dir,
resolve_shard_dir,
)
monkeypatch.setenv("SOUP_LAYER_STREAM_CACHE_DIR", str(tmp_path / "cache") + "\x1b")
got = os.path.realpath(resolve_shard_dir("m"))
assert got.startswith(os.path.realpath(default_layer_stream_cache_dir()))
class TestShardNoTopLevelTorch:
def test_layer_shard_has_no_top_level_torch(self):
import soup_cli.utils.layer_shard as mod
tree = ast.parse(open(mod.__file__, encoding="utf-8").read())
for node in tree.body:
if isinstance(node, ast.Import):
for alias in node.names:
assert not alias.name.startswith(("torch", "safetensors")), alias.name
elif isinstance(node, ast.ImportFrom):
assert not (node.module or "").startswith(("torch", "safetensors"))
class TestShardUniformity:
"""Heterogeneous layers are allowed, but a shared key must keep one layout."""
def test_non_uniform_layer_parameter_set_is_allowed(self, tmp_path):
import torch
from safetensors.torch import load_file
from soup_cli.utils.layer_shard import layer_shard_path, shard_checkpoint
src = tmp_path / "weights"
src.mkdir()
_write_safetensors(
str(src / "model.safetensors"),
{
"model.layers.0.self_attn.q_proj.weight": torch.randn(4, 4),
"model.layers.1.self_attn.q_proj.weight": torch.randn(4, 4),
"model.layers.1.mlp.extra.weight": torch.randn(4, 4),
"model.embed_tokens.weight": torch.randn(4, 4),
},
)
index = shard_checkpoint(str(src), str(tmp_path / "out"), dtype="float32")
assert index.n_layers == 2
assert "mlp.extra.weight" not in load_file(layer_shard_path(str(tmp_path / "out"), 0))
assert "mlp.extra.weight" in load_file(layer_shard_path(str(tmp_path / "out"), 1))
def test_shared_key_with_conflicting_shape_is_still_refused(self, tmp_path):
import torch
from soup_cli.utils.layer_shard import shard_checkpoint
src = tmp_path / "weights"
src.mkdir()
_write_safetensors(
str(src / "model.safetensors"),
{
"model.layers.0.mlp.extra.weight": torch.randn(4, 4),
"model.layers.1.mlp.extra.weight": torch.randn(4, 8),
"model.embed_tokens.weight": torch.randn(4, 4),
},
)
with pytest.raises(ValueError, match="stored shapes or dtypes"):
shard_checkpoint(str(src), str(tmp_path / "out"), dtype="float32")
def test_non_contiguous_layer_indices_refused(self, tmp_path):
import torch
from soup_cli.utils.layer_shard import shard_checkpoint
src = tmp_path / "weights"
src.mkdir()
_write_safetensors(
str(src / "model.safetensors"),
{
"model.layers.0.self_attn.q_proj.weight": torch.randn(4, 4),
"model.layers.2.self_attn.q_proj.weight": torch.randn(4, 4),
"model.embed_tokens.weight": torch.randn(4, 4),
},
)
with pytest.raises(ValueError, match="contiguous"):
shard_checkpoint(str(src), str(tmp_path / "out"), dtype="float32")
# ==========================================================================
# C3 — schema fields + cross-validators (config/schema.py)
# ==========================================================================
def _stream_yaml(training=None, **top):
"""Minimal valid streaming config; `training` overlays the training block."""
import yaml
cfg = {
"base": "HuggingFaceTB/SmolLM2-135M",
"task": "sft",
"backend": "transformers",
"modality": "text",
"data": {"train": "train.jsonl"},
"training": {
"batch_size": 1,
"gradient_accumulation_steps": 1,
"quantization": "none",
"stream_layers": True,
},
}
cfg.update(top)
if training:
cfg["training"].update(training)
return yaml.safe_dump(cfg)
def _load(yaml_text):
from soup_cli.config.loader import load_config_from_string
return load_config_from_string(yaml_text)
class TestStreamSchemaDefaults:
def test_defaults_are_off(self):
cfg = _load(_stream_yaml(training={"stream_layers": False}))
assert cfg.training.stream_layers is False
assert cfg.training.stream_source == "auto"
assert cfg.training.stream_ngram_source == "auto"
assert cfg.training.stream_buffers == 2
assert cfg.training.stream_vram_override is None
def test_happy_path_parses(self):
cfg = _load(_stream_yaml(training={"stream_source": "ram", "stream_buffers": 3}))
assert cfg.training.stream_layers is True
assert cfg.training.stream_source == "ram"
assert cfg.training.stream_buffers == 3
class TestStreamVramOverride:
def test_defaults_to_none(self):
assert _load(_stream_yaml()).training.stream_vram_override is None
def test_accepts_an_explicit_byte_count(self):
cfg = _load(_stream_yaml(training={"stream_vram_override": 4_000_000_000}))
assert cfg.training.stream_vram_override == 4_000_000_000
def test_zero_is_accepted(self):
"""A degenerate but legal value: 'assume nothing is free'."""
cfg = _load(_stream_yaml(training={"stream_vram_override": 0}))
assert cfg.training.stream_vram_override == 0
def test_negative_rejected(self):
with pytest.raises(ValueError, match="stream_vram_override"):
_load(_stream_yaml(training={"stream_vram_override": -1}))
def test_bool_rejected(self):
with pytest.raises(ValueError, match="bool"):
_load(_stream_yaml(training={"stream_vram_override": True}))
def test_without_stream_layers_rejected(self):
with pytest.raises(ValueError, match="stream_layers"):
_load(
_stream_yaml(
training={"stream_layers": False, "stream_vram_override": 4_000_000_000}
)
)
class TestStreamTaskAndBackendGates:
def test_unsupported_task_rejected(self):
"""v0.72.4 opened the four preference losses, so `dpo` is no longer the
example here. `reward_model` still has no streaming path, and `grpo` is
excluded permanently — rollouts re-read every layer per generated
token."""
with pytest.raises(ValueError, match="stream_layers"):
_load(_stream_yaml(task="reward_model"))
with pytest.raises(ValueError, match="generation"):
_load(_stream_yaml(task="grpo"))
def test_preference_tasks_are_accepted_since_v0724(self):
assert _load(_stream_yaml(task="dpo")).task == "dpo"
def test_unsloth_backend_rejected(self):
with pytest.raises(ValueError, match="transformers"):
_load(_stream_yaml(backend="unsloth"))
def test_mlx_backend_rejected(self):
with pytest.raises(ValueError, match="transformers"):
_load(_stream_yaml(backend="mlx"))
def test_vision_modality_rejected(self):
with pytest.raises(ValueError, match="text"):
_load(_stream_yaml(modality="vision"))
class TestStreamScopeGates:
def test_nf4_is_accepted_since_v0_72_2(self):
"""v0.72.0 refused 4-bit and named v0.72.2; v0.72.2 delivered it."""
cfg = _load(_stream_yaml(training={"quantization": "4bit"}))
assert cfg.training.quantization == "4bit"
def test_other_quantisations_still_rejected(self):
"""The gate did not simply disappear — only NF4 was added to it."""
with pytest.raises(ValueError, match="stream_layers"):
_load(_stream_yaml(training={"quantization": "8bit"}))
def test_disk_source_accepted_in_v0723(self):
"""The disk overflow tier shipped in v0.72.3. It stays NVMe-only —
`choose_tier` refuses spinning disks (plan P11) — but that is a runtime
decision about the machine, not a schema refusal about the config."""
cfg = _load(_stream_yaml(training={"stream_source": "disk"}))
assert cfg.training.stream_source == "disk"
def test_batch_size_above_one_accepted_in_v0723(self):
"""Lifted in v0.72.3. Bigger batches are where streaming pays off — one
weight read amortised over more tokens — and the v0.72.2 refusal already
named this slot, so leaving it would have shipped a lie."""
assert _load(_stream_yaml(training={"batch_size": 4})).training.batch_size == 4
def test_auto_batch_size_rejected(self):
"""Still refused, and for a reason that does not expire: "auto" sizes a
RESIDENT model by OOM-probing it, and a streaming run never loads one."""
with pytest.raises(ValueError, match="batch_size"):
_load(_stream_yaml(training={"batch_size": "auto"}))
def test_gradient_accumulation_accepted_in_v0723(self):
"""Lifted in v0.72.3, and plan P9's premise turned out to be half wrong.
Accumulation does re-read the base per micro-batch, but it also processes
that many more tokens, so layer reads per 1k tokens are UNCHANGED
(measured constant at 175.78 across accum 1/2/4). The real cost is
opportunity cost against raising batch_size — 2.52x at matched effective
batch — which is advice for the pre-flight, not grounds for a refusal:
accumulation is the only way to raise effective batch once VRAM is
spent."""
cfg = _load(_stream_yaml(training={"gradient_accumulation_steps": 4}))
assert cfg.training.gradient_accumulation_steps == 4
def test_lora_disabled_is_a_no_op_and_rejected(self):
"""Streaming a frozen base with nothing trainable trains nothing."""
with pytest.raises(ValueError, match="LoRA"):
_load(_stream_yaml(training={"lora": {"r": 0}}))
def test_stream_pin_is_committable_to_config(self):
"""#366: the pinning override joins the streaming config keys. Default
is None so today's automatic behaviour is unchanged."""
assert _load(_stream_yaml(training={"stream_pin": False})).training.stream_pin is False
assert _load(_stream_yaml(training={"stream_pin": True})).training.stream_pin is True
assert _load(_stream_yaml()).training.stream_pin is None
class TestStreamMutualExclusions:
def test_unfrozen_parameters_conflict(self):
with pytest.raises(ValueError, match="unfrozen_parameters"):
_load(_stream_yaml(training={"unfrozen_parameters": ["model.layers.0.mlp"]}))
def test_lisa_conflict(self):
with pytest.raises(ValueError, match="lisa_enabled"):
_load(_stream_yaml(training={"lisa_enabled": True}))
def test_packing_conflict(self):
with pytest.raises(ValueError, match="packing"):
_load(_stream_yaml(training={"packing": True}))
def test_multipack_conflict(self):
with pytest.raises(ValueError, match="multipack"):
_load(_stream_yaml(training={"multipack": True}))
def test_fsdp2_compile_conflict(self):
with pytest.raises(ValueError, match="use_fsdp2_compile"):
_load(_stream_yaml(training={"use_fsdp2_compile": True}))
def test_train_router_only_conflict(self):
"""moe_lora is set so the PRE-EXISTING train_router_only validator is
satisfied — only the streaming gate can reject this, and the message
must say so (otherwise the test passes for the wrong reason)."""
with pytest.raises(ValueError, match="stream_layers"):
_load(_stream_yaml(training={"train_router_only": True, "moe_lora": True}))
def test_moe_expert_quant_conflict_names_the_silent_noop(self):
"""Expert quantization is wired only into the resident setup path.
``moe_lora`` satisfies the older feature validator, so this assertion
can pass only if the streaming cross-validator refuses the reachable
silent no-op explicitly.
"""
with pytest.raises(
ValueError,
match=r"stream_layers.*moe_expert_quant.*silently ignored",
):
_load(
_stream_yaml(
training={"moe_lora": True, "moe_expert_quant": "nf4"}
)
)
def test_expand_layers_conflict(self):
"""Likewise: freeze_trainable_layers satisfies the LLaMA-Pro validator."""
with pytest.raises(ValueError, match="stream_layers"):
_load(
_stream_yaml(
training={"expand_layers": 2, "freeze_trainable_layers": 2}
)
)
class TestStreamFootgunRejection:
def test_stream_buffers_without_stream_layers_rejected(self):
with pytest.raises(ValueError, match="stream_layers"):
_load(_stream_yaml(training={"stream_layers": False, "stream_buffers": 4}))
def test_stream_source_without_stream_layers_rejected(self):
with pytest.raises(ValueError, match="stream_layers"):
_load(_stream_yaml(training={"stream_layers": False, "stream_source": "ram"}))
def test_stream_ngram_source_without_stream_layers_rejected(self):
with pytest.raises(ValueError, match="stream_layers"):
_load(
_stream_yaml(
training={
"stream_layers": False,
"stream_ngram_source": "disk",
}
)
)
@pytest.mark.parametrize("stream_pin", [True, False])
def test_stream_pin_without_stream_layers_rejected(self, stream_pin):
"""#366 criterion 4: stream_pin set while streaming is off is a footgun.
#366 re-review: cover BOTH values. The guard is ``stream_pin is not
None``; testing only ``False`` let it be narrowed to ``is False`` with the
suite green, leaving ``true`` + ``stream_layers: false`` unguarded."""
with pytest.raises(ValueError, match="stream_layers"):
_load(_stream_yaml(training={"stream_layers": False, "stream_pin": stream_pin}))
class TestStreamBufferBounds:
def test_one_buffer_rejected(self):
with pytest.raises(ValueError, match="stream_buffers"):
_load(_stream_yaml(training={"stream_buffers": 1}))
def test_nine_buffers_rejected(self):
with pytest.raises(ValueError, match="stream_buffers"):
_load(_stream_yaml(training={"stream_buffers": 9}))
def test_bool_buffers_rejected(self):
with pytest.raises(ValueError, match="bool"):
_load(_stream_yaml(training={"stream_buffers": True}))
def test_bad_stream_source_rejected(self):
with pytest.raises(ValueError, match="stream_source"):
_load(_stream_yaml(training={"stream_source": "network"}))
def test_bad_stream_ngram_source_rejected(self):
with pytest.raises(ValueError, match="stream_ngram_source"):
_load(_stream_yaml(training={"stream_ngram_source": "network"}))
# ==========================================================================
# C4 — the streaming runtime (utils/layer_stream_runtime.py)
#
# The four classes below are SILENT failures: if any regresses, training still
# runs and still converges. That is exactly why they are tests, not comments.
# ==========================================================================
def _tiny_llama_dir(tmp_path, n_layers=2, tie=True):
"""A real (tiny) Llama checkpoint on disk: config.json + model.safetensors."""
import torch
from safetensors.torch import save_file
from transformers import LlamaConfig, LlamaForCausalLM
torch.manual_seed(7)
config = LlamaConfig(
vocab_size=64,
hidden_size=32,
intermediate_size=64,
num_hidden_layers=n_layers,
num_attention_heads=4,
num_key_value_heads=2,
tie_word_embeddings=tie,
max_position_embeddings=128,
)
model = LlamaForCausalLM(config).to(torch.float32).eval()
weights = tmp_path / "model"
weights.mkdir(parents=True, exist_ok=True)
state = {k: v.contiguous() for k, v in model.state_dict().items()}
if tie:
state.pop("lm_head.weight", None)
save_file(state, str(weights / "model.safetensors"))
config.save_pretrained(str(weights))
return str(weights), model, config
def _tiny_lora():
from peft import LoraConfig, TaskType
return LoraConfig(
r=4, lora_alpha=8, lora_dropout=0.0, bias="none",
target_modules=["q_proj", "v_proj"], task_type=TaskType.CAUSAL_LM,
)
def _build_streamed_cpu(tmp_path, n_layers=2, tie=True, buffers=2, device="cpu"):
from soup_cli.utils.layer_shard import shard_checkpoint
from soup_cli.utils.layer_stream_runtime import build_streamed_model
weights, resident, _ = _tiny_llama_dir(tmp_path, n_layers=n_layers, tie=tie)
shards = str(tmp_path / "shards")
index = shard_checkpoint(weights, shards, dtype="float32", arch="llama")
model, runtime = build_streamed_model(
model_id=weights, shard_dir=shards, index=index,
lora_config=_tiny_lora(), device=device, dtype="float32",
buffers=buffers, pin=False, seed=3,
)
return model, runtime, resident, weights
class TestDecoderOwner:
def test_finds_the_module_that_owns_layers(self, tmp_path):
from peft import get_peft_model
from soup_cli.utils.layer_stream_runtime import decoder_owner
_, resident, _ = _tiny_llama_dir(tmp_path)
peft_model = get_peft_model(resident, _tiny_lora())
owner = decoder_owner(peft_model)
assert hasattr(owner, "layers")
assert len(owner.layers) == 2
def test_raises_when_no_layer_container(self):
import torch.nn as nn
from soup_cli.utils.layer_stream_runtime import decoder_owner
with pytest.raises(ValueError, match="layers"):
decoder_owner(nn.Linear(2, 2))
class TestRegressionPrefetchHookAttachPoint:
"""SILENT FAILURE 1 — PEFT's LoraModel calls ``self.model.forward(...)``
DIRECTLY, which bypasses __call__ and therefore every forward hook on the
CausalLM wrapper. Attaching the prefetch there means layer 0 is never
primed; the buffer-ownership assert then fires (or, worse, a stale buffer
is used). The hook must go on the module that owns ``.layers``."""
def test_prefetch_primes_through_a_real_peft_wrapper(self, tmp_path):
import torch
model, runtime, _, _ = _build_streamed_cpu(tmp_path)
assert runtime.prefetcher.primes == 0
model(input_ids=torch.randint(0, 64, (1, 8)))
assert runtime.prefetcher.primes >= 1, (
"prefetch never primed — the hook was attached to a module PEFT "
"bypasses with a direct .forward() call"
)
def test_hook_is_registered_on_the_layer_owner(self, tmp_path):
from soup_cli.utils.layer_stream_runtime import decoder_owner
model, runtime, _, _ = _build_streamed_cpu(tmp_path)
owner = decoder_owner(model)
assert len(owner._forward_pre_hooks) >= 1
class TestRegressionAttributeTransparency:
"""SILENT FAILURE 2 — transformers reads contract attributes straight off
the layer object (this version reads ``decoder_layer.attention_type`` in
Qwen2Model.forward). A wrapper that is not attribute-transparent raises
AttributeError at forward time; worse, a wrapper that returns a DEFAULT
would silently select the wrong attention path."""
def test_wrapper_forwards_unknown_attributes_to_the_wrapped_layer(self, tmp_path):
model, _, _, _ = _build_streamed_cpu(tmp_path)
from soup_cli.utils.layer_stream_runtime import decoder_owner
layer = decoder_owner(model).layers[0]
assert layer.__class__.__name__ == "StreamedDecoderLayer"
# a real attribute of the wrapped LlamaDecoderLayer
assert layer.self_attn is layer.inner.self_attn
def test_attention_type_is_visible_when_the_inner_layer_has_it(self, tmp_path):
import torch.nn as nn
from soup_cli.utils.layer_stream_runtime import StreamedDecoderLayer
class _Inner(nn.Module):
attention_type = "full_attention"
def forward(self, x, **kw):
return x
wrapper = StreamedDecoderLayer(_Inner(), 0, pool=None, prefetcher=None)
assert wrapper.attention_type == "full_attention"
def test_genuinely_missing_attribute_still_raises(self, tmp_path):
import torch.nn as nn
from soup_cli.utils.layer_stream_runtime import StreamedDecoderLayer
class _Inner(nn.Module):
def forward(self, x, **kw):
return x
wrapper = StreamedDecoderLayer(_Inner(), 0, pool=None, prefetcher=None)
with pytest.raises(AttributeError):
wrapper.definitely_not_a_real_attribute
class TestRegressionMetaAdapterMaterialisation:
"""SILENT FAILURE 3 — PEFT initialises adapter weights on the BASE layer's
device, which is ``meta`` in a streaming build. Meta parameters have no
storage: the optimizer sees them, the run proceeds, and nothing trains."""
def test_no_adapter_parameter_is_left_on_meta(self, tmp_path):
model, _, _, _ = _build_streamed_cpu(tmp_path)
stranded = [
name for name, param in model.named_parameters()
if "lora_" in name and param.is_meta
]
assert stranded == [], stranded
def test_adapters_are_trainable_and_base_is_not(self, tmp_path):
model, _, _, _ = _build_streamed_cpu(tmp_path)
trainable = [n for n, p in model.named_parameters() if p.requires_grad]
assert trainable, "nothing is trainable"
assert all("lora_" in n for n in trainable), trainable
def test_lora_b_is_zero_and_lora_a_is_not(self, tmp_path):
"""PEFT's init scheme. B=0 makes the adapter a no-op at step 0; an
all-zero A would instead freeze the adapter forever."""
model, _, _, _ = _build_streamed_cpu(tmp_path)
a_seen = b_seen = False
for name, param in model.named_parameters():
if "lora_A" in name:
a_seen = True
assert param.abs().sum().item() > 0, name
if "lora_B" in name:
b_seen = True
assert param.abs().sum().item() == 0, name
assert a_seen and b_seen
def test_base_decoder_weights_never_materialise(self, tmp_path):
"""The whole point: the resident load must not happen."""
model, runtime, _, _ = _build_streamed_cpu(tmp_path)
meta_layer_params = [
name for name, param in model.named_parameters()
if param.is_meta and ".layers." in name
]
assert meta_layer_params, "decoder weights were materialised — not streaming"
assert runtime.n_layers == 2
class TestRegressionAllocateOnceThenCopy:
"""SILENT FAILURE 4 — the obvious ``load_file -> .to(dtype) -> .pin_memory()``
costs three transient copies of every layer. On the dev box that transient,
not the store, is what pushed a 5.55 GB base over the 7.12 GB page-locked
ceiling and made a 3B run impossible. The store must be allocated once and
filled with copy_."""
def test_ram_source_never_calls_tensor_pin_memory(self, tmp_path, monkeypatch):
import torch
from soup_cli.utils.layer_shard import shard_checkpoint
from soup_cli.utils.layer_stream_runtime import RamSource
weights, _, _ = _tiny_llama_dir(tmp_path)
shards = str(tmp_path / "shards")
index = shard_checkpoint(weights, shards, dtype="float32")
calls = []
monkeypatch.setattr(
torch.Tensor, "pin_memory",
lambda self, *a, **k: calls.append(1) or self,
)
spec = RamSource.spec_from_shard(shards)
source = RamSource(shards, index.n_layers, spec, pin=False)
assert source.nbytes > 0
assert calls == [], "RamSource used .pin_memory() instead of allocating once"
def test_store_matches_the_shard_bytes(self, tmp_path):
import torch
from safetensors.torch import load_file
from soup_cli.utils.layer_shard import layer_shard_path, shard_checkpoint
from soup_cli.utils.layer_stream_runtime import RamSource
weights, _, _ = _tiny_llama_dir(tmp_path)
shards = str(tmp_path / "shards")
index = shard_checkpoint(weights, shards, dtype="float32")
spec = RamSource.spec_from_shard(shards)
source = RamSource(shards, index.n_layers, spec, pin=False)
blob = load_file(layer_shard_path(shards, 1))
for name in spec:
assert torch.equal(source.get(1, name), blob[name]), name
class TestBufferPool:
def test_slot_round_robin(self):
from soup_cli.utils.layer_stream_runtime import LayerBufferPool
pool = LayerBufferPool({"w": ((2, 2), "float32")}, n_buffers=2, device="cpu")
assert [pool.slot_for(i) for i in range(5)] == [0, 1, 0, 1, 0]
def test_wait_on_an_unloaded_slot_is_the_p1_tripwire(self):
"""plan P1: a buffer overwritten early yields silently wrong gradients,
not a crash. The ownership assert is what makes it loud."""
from soup_cli.utils.layer_stream_runtime import LayerBufferPool
pool = LayerBufferPool({"w": ((2, 2), "float32")}, n_buffers=2, device="cpu")
with pytest.raises(RuntimeError, match="scheduler"):
pool.wait(0)
def test_wait_after_load_returns_the_buffer(self, tmp_path):
from soup_cli.utils.layer_shard import shard_checkpoint
from soup_cli.utils.layer_stream_runtime import LayerBufferPool, RamSource
weights, _, _ = _tiny_llama_dir(tmp_path)
shards = str(tmp_path / "shards")
index = shard_checkpoint(weights, shards, dtype="float32")
spec = RamSource.spec_from_shard(shards)
source = RamSource(shards, index.n_layers, spec, pin=False)
pool = LayerBufferPool(spec, n_buffers=2, device="cpu")
pool.load_async(0, source)
got = pool.wait(0)
assert set(got) == set(spec)
def test_early_overwrite_is_detected(self, tmp_path):
"""Layer 2 recycles slot 0 (2 % 2 == 0) while layer 0 still owns it."""
from soup_cli.utils.layer_shard import shard_checkpoint
from soup_cli.utils.layer_stream_runtime import LayerBufferPool, RamSource
weights, _, _ = _tiny_llama_dir(tmp_path, n_layers=3)
shards = str(tmp_path / "shards")
index = shard_checkpoint(weights, shards, dtype="float32")
spec = RamSource.spec_from_shard(shards)
source = RamSource(shards, index.n_layers, spec, pin=False)
pool = LayerBufferPool(spec, n_buffers=2, device="cpu")
pool.load_async(0, source)
pool.load_async(2, source) # slot 0 again — layer 0 is gone
with pytest.raises(RuntimeError, match="scheduler"):
pool.wait(0)
class TestPrefetcherDirection:
"""Forward walks 0..L-1; backward recompute walks L-1..0. The prefetcher
must reverse with it or every backward layer is a buffer miss."""
class _FakePool:
def __init__(self, n=2):
self.n = n
self.owner = [None] * n
self.loaded = []
def slot_for(self, idx):
return idx % self.n
def load_async(self, idx, source, stream=None):
self.owner[self.slot_for(idx)] = idx
self.loaded.append(idx)
def test_forward_prefetches_ascending(self):
from soup_cli.utils.layer_stream_runtime import StreamPrefetcher
pool = self._FakePool()
pre = StreamPrefetcher(pool, source=None, n_layers=4)
pre.prime()
for idx in range(4):
pre.advance(idx)
assert pool.loaded == [0, 1, 2, 3]
def test_backward_prefetches_descending(self):
from soup_cli.utils.layer_stream_runtime import StreamPrefetcher
pool = self._FakePool()
pre = StreamPrefetcher(pool, source=None, n_layers=4)
pre.prime()
for idx in range(4):
pre.advance(idx)
for idx in (3, 2, 1, 0): # backward recompute order
pre.advance(idx)
assert pool.loaded[-2:] == [1, 0]
def test_does_not_reload_a_layer_already_owned(self):
from soup_cli.utils.layer_stream_runtime import StreamPrefetcher
pool = self._FakePool()
pre = StreamPrefetcher(pool, source=None, n_layers=4)
pre.prime()
pre.advance(0)
before = list(pool.loaded)
pre.advance(0)
assert pool.loaded == before, "re-prefetched a layer the pool already owns"
class TestExpandableSegments:
"""User requirement 4: the brief says to auto-set
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True. It is silently IGNORED on
Windows (torch warns 'not supported on this platform'). Attempt, detect,
and never claim it is active when it is not."""
def test_returns_a_bool(self):
from soup_cli.utils.layer_stream_runtime import probe_expandable_segments
assert isinstance(probe_expandable_segments(), bool)
def test_reports_false_on_windows(self):
import sys
from soup_cli.utils.layer_stream_runtime import probe_expandable_segments
if sys.platform.startswith("win"):
assert probe_expandable_segments() is False
class TestStreamedForwardParityCpu:
"""The Gate-1 property, as a CI test: streaming substitutes the identical
weight bytes into the identical kernels, so the logits must be bit-exact."""
def test_streamed_logits_match_resident_bit_exactly(self, tmp_path):
import torch
from peft import get_peft_model
model, _, resident, _ = _build_streamed_cpu(tmp_path)
ref = get_peft_model(resident, _tiny_lora())
_copy_lora(model, ref)
ids = torch.randint(0, 64, (1, 12))
with torch.no_grad():
got = model(input_ids=ids).logits
want = ref(input_ids=ids).logits
assert torch.equal(got, want), (got - want).abs().max().item()
def test_untied_large_layers_share_one_slot_and_match_resident(self, tmp_path):
"""#324: two distinct vocabulary matrices must not mean two VRAM copies."""
import torch
from peft import get_peft_model
model, runtime, resident, _ = _build_streamed_cpu(tmp_path, tie=False)
ref = get_peft_model(resident, _tiny_lora())
_copy_lora(model, ref)
ids = torch.randint(0, 64, (1, 12))
with torch.no_grad():
got = model(input_ids=ids).logits
want = ref(input_ids=ids).logits
assert torch.equal(got, want), (got - want).abs().max().item()
matrix_bytes = 64 * 32 * 4
assert runtime.large_pool is not None
assert set(runtime.large_pool.specs) == {
"model.embed_tokens.weight",
"lm_head.weight",
}
assert runtime.large_pool.nbytes == matrix_bytes
assert runtime.large_pool.loads == 2
assert runtime.large_pool.owner == "lm_head.weight"
def test_untied_large_layers_match_resident_backward_gradients(self, tmp_path):
"""The untied slot must preserve training numerics, not only logits."""
import torch
from peft import get_peft_model
from soup_cli.utils.layer_stream_runtime import canonical_named_parameters
model, _runtime, resident, _ = _build_streamed_cpu(tmp_path, tie=False)
ref = get_peft_model(resident, _tiny_lora())
generator = torch.Generator().manual_seed(17)
with torch.no_grad():
for name, parameter in model.named_parameters():
if "lora_B" in name:
parameter.copy_(
torch.randn(parameter.shape, generator=generator) * 0.02
)
_copy_lora(model, ref)
model.eval()
ref.eval()
model.zero_grad(set_to_none=True)
ref.zero_grad(set_to_none=True)
ids = torch.randint(0, 64, (2, 12), generator=generator)
streamed_loss = model(input_ids=ids, labels=ids).loss
resident_loss = ref(input_ids=ids, labels=ids).loss
assert torch.equal(streamed_loss, resident_loss), (
streamed_loss - resident_loss
).abs().item()
streamed_loss.backward()
resident_loss.backward()
streamed_grads = {
name: parameter.grad
for name, parameter in canonical_named_parameters(model)
if "lora_" in name and parameter.grad is not None
}
resident_grads = {
name: parameter.grad
for name, parameter in canonical_named_parameters(ref)
if "lora_" in name and parameter.grad is not None
}
assert streamed_grads and set(streamed_grads) == set(resident_grads)
assert any(float(gradient.abs().sum()) > 0.0 for gradient in streamed_grads.values())
for name, gradient in streamed_grads.items():
assert torch.equal(gradient, resident_grads[name]), name
def test_tied_embedding_control_uses_the_unchanged_resident_path(self, tmp_path):
import torch
model, runtime, _resident, _ = _build_streamed_cpu(tmp_path, tie=True)
model(input_ids=torch.randint(0, 64, (1, 8)))
assert runtime.large_pool is None
assert not model.get_input_embeddings().weight.is_meta
assert model.get_input_embeddings().weight is model.get_output_embeddings().weight
def test_all_aliases_of_a_tied_boundary_module_are_replaced(self):
import torch.nn as nn
from soup_cli.utils.layer_stream_runtime import _replace_module_references
root = nn.Module()
shared = nn.Linear(4, 4, bias=False)
replacement = nn.Identity()
root.input = shared
root.output = shared
assert _replace_module_references(root, shared, replacement) == 2
assert root.input is replacement
assert root.output is replacement
def test_layer_zero_adapter_gradient_is_non_zero(self, tmp_path):
"""plan P2: a detach()/no_grad() around the base severs the graph.
Loss still falls (upper layers learn), so only this catches it."""
import torch
model, _, _, _ = _build_streamed_cpu(tmp_path, n_layers=3)
ids = torch.randint(0, 64, (1, 12))
model(input_ids=ids, labels=ids).loss.backward()
grads = {
name: param.grad
for name, param in model.named_parameters()
if "lora_" in name and param.grad is not None
}
layer0 = [g for n, g in grads.items() if ".layers.0." in n]
assert layer0, "layer 0 has no adapter gradient at all"
assert sum(float(g.abs().sum()) for g in layer0) > 0.0
def test_buffer_count_does_not_change_the_result(self, tmp_path):
import torch
two, _, _, _ = _build_streamed_cpu(tmp_path / "a", n_layers=3, buffers=2)
three, _, _, _ = _build_streamed_cpu(tmp_path / "b", n_layers=3, buffers=3)
ids = torch.randint(0, 64, (1, 10))
with torch.no_grad():
assert torch.equal(two(input_ids=ids).logits, three(input_ids=ids).logits)
@pytest.mark.skipif(not mps_is_the_accelerator(), reason="needs Apple Silicon MPS")
class TestStreamedLargeLayerParityMps:
def test_untied_logits_match_resident(self, tmp_path):
"""Supplementary #324 hardware control; the required CI oracle is CPU."""
import torch
from peft import get_peft_model
model, runtime, resident, _ = _build_streamed_cpu(
tmp_path, tie=False, device="mps"
)
ref = get_peft_model(resident.to("mps"), _tiny_lora())
_copy_lora(model, ref)
ids = torch.randint(0, 64, (1, 12), device="mps")
with torch.no_grad():
got = model(input_ids=ids).logits
want = ref(input_ids=ids).logits
assert torch.equal(got, want), (got - want).abs().max().item()
assert runtime.large_pool.loads == 2
def _copy_lora(src, dst):
"""StreamedDecoderLayer inserts an `.inner.` segment into state-dict keys."""
def norm(key):
return key.replace(".inner.", ".")
src_lora = {norm(k): v for k, v in src.state_dict().items() if "lora_" in k}
dst_lora = {norm(k): v for k, v in dst.state_dict().items() if "lora_" in k}
assert src_lora and set(src_lora) == set(dst_lora)
for key, val in src_lora.items():
dst_lora[key].copy_(val)
CUDA = pytest.mark.gpu(reason="layer streaming is a GPU feature")
@CUDA
class TestStreamedTrainingParityCuda:
def test_five_steps_match_resident(self, tmp_path):
import torch
from peft import get_peft_model
from soup_cli.utils.layer_shard import shard_checkpoint
from soup_cli.utils.layer_stream_runtime import build_streamed_model
weights, resident, _ = _tiny_llama_dir(tmp_path, n_layers=3)
shards = str(tmp_path / "shards")
index = shard_checkpoint(weights, shards, dtype="float32", arch="llama")
streamed, _ = build_streamed_model(
model_id=weights, shard_dir=shards, index=index,
lora_config=_tiny_lora(), device="cuda", dtype="float32",
buffers=2, pin=True, seed=3,
)
ref = get_peft_model(resident, _tiny_lora()).to("cuda")
_copy_lora(streamed, ref)
torch.manual_seed(0)
batches = [torch.randint(0, 64, (1, 12), device="cuda") for _ in range(5)]
losses = []
for model in (streamed, ref):
opt = torch.optim.AdamW([p for p in model.parameters() if p.requires_grad], lr=1e-3)
run = []
for ids in batches:
out = model(input_ids=ids, labels=ids)
out.loss.backward()
opt.step()
opt.zero_grad(set_to_none=True)
run.append(float(out.loss))
losses.append(run)
for got, want in zip(*losses):
assert abs(got - want) < 1e-4, losses
def test_same_seed_twice_is_identical(self, tmp_path):
"""Catches stream races: a racy prefetch is non-deterministic."""
import torch
from soup_cli.utils.layer_shard import shard_checkpoint
from soup_cli.utils.layer_stream_runtime import build_streamed_model
weights, _, _ = _tiny_llama_dir(tmp_path, n_layers=3)
shards = str(tmp_path / "shards")
index = shard_checkpoint(weights, shards, dtype="float32", arch="llama")
runs = []
for _ in range(2):
model, _ = build_streamed_model(
model_id=weights, shard_dir=shards, index=index,
lora_config=_tiny_lora(), device="cuda", dtype="float32",
buffers=2, pin=True, seed=3,
)
torch.manual_seed(0)
batches = [torch.randint(0, 64, (1, 12), device="cuda") for _ in range(4)]
opt = torch.optim.AdamW(
[p for p in model.parameters() if p.requires_grad], lr=1e-3
)
run = []
for ids in batches:
out = model(input_ids=ids, labels=ids)
out.loss.backward()
opt.step()
opt.zero_grad(set_to_none=True)
run.append(float(out.loss))
runs.append(run)
del model
torch.cuda.empty_cache()
assert runs[0] == runs[1], runs
class TestRuntimeNoTopLevelTorch:
def test_runtime_is_torch_lazy(self):
import soup_cli.utils.layer_stream_runtime as mod
tree = ast.parse(open(mod.__file__, encoding="utf-8").read())
for node in tree.body:
if isinstance(node, ast.Import):
for alias in node.names:
assert not alias.name.startswith(("torch", "peft", "transformers"))
elif isinstance(node, ast.ImportFrom):
assert not (node.module or "").startswith(
("torch", "peft", "transformers")
)
# ==========================================================================
# C5 — trainer/sft.py streaming branch
# ==========================================================================
class _StubTokenizer:
pad_token = "<pad>"
eos_token = "</s>"
pad_token_id = 0
eos_token_id = 1
def __call__(self, *a, **k): # pragma: no cover - not exercised here
raise NotImplementedError
def _stream_config(base_dir, **training):
import yaml
from soup_cli.config.loader import load_config_from_string
cfg = {
"base": base_dir,
"task": "sft",
"backend": "transformers",
"modality": "text",
"data": {"train": "train.jsonl"},
"training": {
"batch_size": 1,
"gradient_accumulation_steps": 1,
"quantization": "none",
"stream_layers": True,
"lora": {"r": 4, "alpha": 8, "target_modules": ["q_proj", "v_proj"]},
**training,
},
}
return load_config_from_string(yaml.safe_dump(cfg))
class TestHfGradientCheckpointingInteraction:
"""StreamedDecoderLayer already wraps every layer in checkpoint(). Letting
HF ALSO enable gradient checkpointing double-recomputes every layer: the
run still converges, it is just silently ~1.5x slower."""
def test_streaming_disables_hf_gradient_checkpointing(self):
from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
assert should_enable_hf_gradient_checkpointing(True, stream_layers=True) is False
def test_non_streaming_keeps_hf_gradient_checkpointing(self):
from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
assert should_enable_hf_gradient_checkpointing(True, stream_layers=False) is True
def test_tier_string_also_disabled_under_streaming(self):
from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
assert should_enable_hf_gradient_checkpointing("auto", stream_layers=True) is False
def test_off_stays_off(self):
from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
assert should_enable_hf_gradient_checkpointing(False, stream_layers=False) is False
def test_sft_guards_the_hf_flag_with_the_helper(self):
"""Pin the call site: the trainer must consult the helper, not the raw
tcfg.gradient_checkpointing."""
import inspect
from soup_cli.trainer import sft
source = inspect.getsource(sft)
assert "should_enable_hf_gradient_checkpointing" in source
class TestStreamingDispatch:
def test_setup_routes_to_the_streaming_branch(self, tmp_path, monkeypatch):
from soup_cli.trainer.sft import SFTTrainerWrapper
weights, _, _ = _tiny_llama_dir(tmp_path)
cfg = _stream_config(weights)
calls = []
class _StopError(Exception):
pass
def _stream(self, c, t):
calls.append("streaming")
raise _StopError()
def _plain(self, c, t):
calls.append("plain")
raise _StopError()
monkeypatch.setattr(SFTTrainerWrapper, "_setup_streaming_transformers", _stream)
monkeypatch.setattr(SFTTrainerWrapper, "_setup_transformers", _plain)
wrapper = SFTTrainerWrapper(cfg, device="cpu")
with pytest.raises(_StopError):
wrapper.setup({"train": [{"messages": []}]})
assert calls == ["streaming"]
def test_setup_uses_the_plain_branch_when_streaming_is_off(self, tmp_path, monkeypatch):
from soup_cli.trainer.sft import SFTTrainerWrapper
weights, _, _ = _tiny_llama_dir(tmp_path)
cfg = _stream_config(weights, stream_layers=False)
calls = []
class _StopError(Exception):
pass
def _stream(self, c, t):
calls.append("streaming")
raise _StopError()
def _plain(self, c, t):
calls.append("plain")
raise _StopError()
monkeypatch.setattr(SFTTrainerWrapper, "_setup_streaming_transformers", _stream)
monkeypatch.setattr(SFTTrainerWrapper, "_setup_transformers", _plain)
wrapper = SFTTrainerWrapper(cfg, device="cpu")
with pytest.raises(_StopError):
wrapper.setup({"train": [{"messages": []}]})
assert calls == ["plain"]
class TestStreamingSetupIntegration:
"""The real construction path on CPU: meta skeleton -> extras -> LoRA ->
streaming, with NO resident base load at any point."""
def _run(self, tmp_path, monkeypatch, n_layers=2):
import transformers
from soup_cli.trainer.sft import SFTTrainerWrapper
weights, _, _ = _tiny_llama_dir(tmp_path, n_layers=n_layers)
monkeypatch.setenv("SOUP_LAYER_STREAM_CACHE_DIR", str(tmp_path / "cache"))
monkeypatch.setattr(
transformers.AutoTokenizer, "from_pretrained",
classmethod(lambda cls, *a, **k: _StubTokenizer()),
)
cfg = _stream_config(weights)
wrapper = SFTTrainerWrapper(cfg, device="cpu")
wrapper._setup_streaming_transformers(cfg, cfg.training)
return wrapper
def test_decoder_weights_stay_on_meta(self, tmp_path, monkeypatch):
wrapper = self._run(tmp_path, monkeypatch)
meta_layer = [
n for n, p in wrapper.model.named_parameters()
if p.is_meta and ".layers." in n
]
assert meta_layer, "the base was materialised — that defeats streaming"
def test_only_adapters_are_trainable(self, tmp_path, monkeypatch):
wrapper = self._run(tmp_path, monkeypatch)
trainable = [n for n, p in wrapper.model.named_parameters() if p.requires_grad]
assert trainable and all("lora_" in n for n in trainable)
def test_runtime_is_attached_with_stats(self, tmp_path, monkeypatch):
wrapper = self._run(tmp_path, monkeypatch)
stats = wrapper._stream_runtime.stats()
assert stats["n_layers"] == 2
assert stats["buffers"] == 2
assert stats["store_bytes"] > 0
def test_a_forward_pass_runs(self, tmp_path, monkeypatch):
import torch
wrapper = self._run(tmp_path, monkeypatch, n_layers=3)
out = wrapper.model(input_ids=torch.randint(0, 64, (1, 8)))
assert out.logits.shape == (1, 8, 64)
def test_unsupported_architecture_is_refused(self, tmp_path, monkeypatch):
import torch
import transformers
from safetensors.torch import save_file
from transformers import GPT2Config, GPT2LMHeadModel
from soup_cli.trainer.sft import SFTTrainerWrapper
config = GPT2Config(n_layer=2, n_embd=32, n_head=4, vocab_size=64)
model = GPT2LMHeadModel(config)
weights = tmp_path / "gpt2"
weights.mkdir()
state = {
k: v.contiguous()
for k, v in model.state_dict().items()
if not k.endswith(".attn.bias") and not k.endswith(".attn.masked_bias")
}
state.pop("lm_head.weight", None) # tied to wte; safetensors refuses aliases
save_file(state, str(weights / "model.safetensors"))
config.save_pretrained(str(weights))
monkeypatch.setenv("SOUP_LAYER_STREAM_CACHE_DIR", str(tmp_path / "cache"))
monkeypatch.setattr(
transformers.AutoTokenizer, "from_pretrained",
classmethod(lambda cls, *a, **k: _StubTokenizer()),
)
cfg = _stream_config(str(weights))
wrapper = SFTTrainerWrapper(cfg, device="cpu")
with pytest.raises(ValueError, match="gpt2"):
wrapper._setup_streaming_transformers(cfg, cfg.training)
assert torch # keep the import meaningful
class TestStreamingResumeAccepted:
"""v0.72.0-.2 refused --resume: a streamed model's named_parameters() carry
an `.inner.` segment that load_state_dict narrows away, so a canonical
checkpoint matched NOTHING and training silently continued from scratch.
v0.72.3 redirects canonical keys at load time; the refusal is gone and the
round-trip is pinned in test_v07203.py."""
def test_train_accepts_resume_with_streaming(self, tmp_path, monkeypatch):
import json as _json
import yaml
from typer.testing import CliRunner
from soup_cli.cli import app
monkeypatch.chdir(tmp_path)
row = {
"messages": [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "yo"},
]
}
with open("train.jsonl", "w", encoding="utf-8") as fh:
fh.write(_json.dumps(row) + "\n")
cfg = {
"base": "HuggingFaceTB/SmolLM2-135M",
"task": "sft",
"backend": "transformers",
"modality": "text",
"data": {"train": "train.jsonl"},
"training": {
"batch_size": 1,
"gradient_accumulation_steps": 1,
"quantization": "none",
"stream_layers": True,
},
}
with open("soup.yaml", "w", encoding="utf-8") as fh:
yaml.safe_dump(cfg, fh)
result = CliRunner().invoke(
app, ["train", "--config", "soup.yaml", "--resume", "latest", "--yes"]
)
assert "are not supported with" not in result.output, result.output
assert "lands in v0.72.3" not in result.output, result.output
class TestRegressionTrainerConstructsWithoutMovingMetaWeights:
"""SILENT-UNTIL-FIRST-RUN FAILURE — transformers' Trainer.__init__ calls
_move_model_to_device -> model.to(args.device), and .to() on a module that
still holds meta parameters raises NotImplementedError. Streaming keeps the
decoder weights on meta ON PURPOSE, so without a device-map marker EVERY
streaming run dies at trainer construction, right after printing
'Layer streaming ready'. Unit tests that stop at model(input_ids=...) do not
see it — only building a real Trainer does."""
def test_to_device_is_safe_and_leaves_the_base_on_meta(self, tmp_path):
"""A bare nn.Module holding meta params raises on .to(); the streamed
layer must pass meta tensors through so Trainer/accelerate can move the
model, while the base placeholders STAY meta (or streaming is over)."""
model, _, _, _ = _build_streamed_cpu(tmp_path)
model.to("cpu") # must not raise
still_meta = [
n for n, p in model.named_parameters() if p.is_meta and ".layers." in n
]
assert still_meta, "the base was materialised by .to() — streaming defeated"
assert all(
not p.is_meta for n, p in model.named_parameters() if "lora_" in n
), "adapters must remain real after a device move"
def test_streamed_model_declares_a_device_map(self, tmp_path):
model, runtime, _, _ = _build_streamed_cpu(tmp_path)
assert getattr(model, "hf_device_map", None) is not None, (
"no hf_device_map — Trainer will try to .to() the meta weights"
)
assert runtime.device in str(model.hf_device_map)
def test_real_trainer_constructs_from_a_streamed_model(self, tmp_path):
from transformers import Trainer, TrainingArguments
model, _, _, _ = _build_streamed_cpu(tmp_path)
args = TrainingArguments(
output_dir=str(tmp_path / "out"),
per_device_train_batch_size=1,
report_to=[],
use_cpu=True,
)
Trainer(model=model, args=args) # must not raise
class TestStreamLoraVariantGates:
"""DoRA / VeRA / PiSSA / OLoRA all READ the real base weight at
get_peft_model() time to initialise. Under streaming the base is still on
meta then, so they crash with an opaque torch error ("Fan in and fan out
can not be computed...") instead of a refusal naming the incompatibility."""
def test_dora_conflict(self):
with pytest.raises(ValueError, match="use_dora"):
_load(_stream_yaml(training={"lora": {"r": 4, "use_dora": True}}))
def test_rslora_is_allowed(self):
"""rsLoRA only rescales alpha — it never reads the base weight."""
cfg = _load(_stream_yaml(training={"lora": {"r": 4, "use_rslora": True}}))
assert cfg.training.lora.use_rslora is True
def test_pissa_init_conflict(self):
with pytest.raises(ValueError, match="init_strategy"):
_load(_stream_yaml(training={"lora": {"r": 4, "init_strategy": "pissa"}}))
def test_olora_init_conflict(self):
with pytest.raises(ValueError, match="init_strategy"):
_load(_stream_yaml(training={"lora": {"r": 4, "init_strategy": "olora"}}))
class TestStreamLoraHeadTargetGate:
"""#1012 follow-up (#1019 review): the forward pass for a LoRA target on
lm_head/embed_tokens under streaming is correct, but saving and resuming
that adapter is not implemented yet. The refusal moved from parse time to
the streamed setup (#1048), which knows whether the checkpoint streams
those modules and which ones a target spelling matched; it is tested in
tests/test_layer_stream_boundary_lora.py. What stays here is that ordinary
targets load."""
def test_non_boundary_targets_still_accepted(self):
"""Negative control: q_proj/v_proj/o_proj/k_proj are unaffected. This
is the exact config #1019's own forward-fix tests exercise and it must
keep training."""
cfg = _load(
_stream_yaml(
training={
"lora": {"r": 4, "target_modules": ["q_proj", "v_proj", "k_proj", "o_proj"]}
}
)
)
assert cfg.training.lora.target_modules == ["q_proj", "v_proj", "k_proj", "o_proj"]
def test_auto_target_modules_still_accepted(self):
"""Negative control: the default 'auto' sentinel is untouched."""
cfg = _load(_stream_yaml(training={"lora": {"r": 4}}))
assert cfg.training.lora.target_modules == "auto"
class TestPrefetchDirectionIsExplicit:
"""Direction was inferred from call order, which is correct today only
because the turnaround index happens to be the last layer. Make it explicit
so a future lookahead change cannot silently turn a missed prefetch into a
stall or a hazard."""
def test_turnaround_resets_state(self):
from soup_cli.utils.layer_stream_runtime import StreamPrefetcher
class _Pool:
n = 2
def __init__(self):
self.owner = [None, None]
self.loaded = []
def slot_for(self, idx):
return idx % self.n
def load_async(self, idx, source, stream=None):
self.owner[self.slot_for(idx)] = idx
self.loaded.append(idx)
pool = _Pool()
pre = StreamPrefetcher(pool, source=None, n_layers=4)
pre.prime()
assert pre.direction == 1
for idx in range(4):
pre.advance(idx)
pre.advance(3) # backward recompute starts at the SAME index
pre.advance(2)
assert pre.direction == -1
pre.prime() # next step's forward
assert pre.direction == 1
def test_output_prefetch_runs_once_per_forward_not_during_backward(self):
from soup_cli.utils.layer_stream_runtime import StreamPrefetcher
class _Pool:
n = 2
def __init__(self):
self.owner = [None, None]
def slot_for(self, idx):
return idx % self.n
def load_async(self, idx, source, stream=None):
self.owner[self.slot_for(idx)] = idx
tails = []
pre = StreamPrefetcher(
_Pool(), source=None, n_layers=3, tail_prefetch=lambda: tails.append("head")
)
pre.prime()
for idx in (0, 1, 2, 2, 1, 0):
pre.advance(idx)
assert tails == ["head"]
pre.prime()
for idx in (0, 1, 2):
pre.advance(idx)
assert tails == ["head", "head"]
class TestShardCacheIdentityBinding:
"""SILENT FAILURE — the shard cache is keyed by a model SLUG. If the source
checkpoint changes (a local dir retrained in place) or two ids collide onto
one slug, a dtype-only cache check happily reuses stale shards and streams
the WRONG WEIGHTS into training. Nothing errors; the loss curve just
describes a different model."""
def test_index_records_a_source_fingerprint(self, tmp_path):
from soup_cli.utils.layer_shard import shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path)
index = shard_checkpoint(src, str(tmp_path / "shards"), dtype="float32")
assert index.source_fingerprint
def test_changed_source_weights_invalidate_the_cache(self, tmp_path):
import torch
from safetensors.torch import load_file
from soup_cli.utils.layer_shard import layer_shard_path, shard_checkpoint
src, layer, extras = _fake_weights_dir(tmp_path)
out = str(tmp_path / "shards")
shard_checkpoint(src, out, dtype="float32")
# retrain in place: same filenames, different weights
changed = {k: torch.full_like(v, 0.5) for k, v in layer.items()}
_write_safetensors(
str(tmp_path / "weights" / "model.safetensors"), {**changed, **extras}
)
index = shard_checkpoint(src, out, dtype="float32")
blob = load_file(layer_shard_path(out, 0))
assert torch.allclose(blob["mlp.down_proj.weight"], torch.tensor(0.5)), (
"stale shards reused after the source checkpoint changed"
)
assert index.source_fingerprint
def test_identical_source_still_hits_the_cache(self, tmp_path):
from soup_cli.utils.layer_shard import layer_shard_path, shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path)
out = str(tmp_path / "shards")
shard_checkpoint(src, out, dtype="float32")
before = os.path.getmtime(layer_shard_path(out, 0))
shard_checkpoint(src, out, dtype="float32")
assert os.path.getmtime(layer_shard_path(out, 0)) == before
class TestShardWriteContainment:
def test_out_dir_outside_the_allowed_roots_is_refused(self, tmp_path):
"""shard_checkpoint must bound its OWN writes, not rely on its caller."""
from soup_cli.utils.layer_shard import shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path)
outside = (
os.path.join("C:" + os.sep, "Windows", "System32", "soup-evil")
if os.name == "nt"
else "/etc/soup-evil"
)
with pytest.raises(ValueError, match="under"):
shard_checkpoint(src, outside, dtype="float32")
@pytest.mark.requires_symlink
def test_symlinked_ancestor_is_resolved_not_followed_blindly(self, tmp_path):
from soup_cli.utils.layer_shard import shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path)
real = tmp_path / "real"
real.mkdir()
link = tmp_path / "link"
os.symlink(str(real), str(link))
index = shard_checkpoint(src, str(link / "shards"), dtype="float32")
# resolved through the symlink to its real location, and recorded there
assert (real / "shards" / "index.json").exists()
assert index.n_layers == 3
def test_total_tensor_count_is_capped(self, tmp_path, monkeypatch):
import soup_cli.utils.layer_shard as mod
src, _, _ = _fake_weights_dir(tmp_path)
monkeypatch.setattr(mod, "_MAX_TOTAL_TENSORS", 3)
with pytest.raises(ValueError, match="tensors"):
mod.shard_checkpoint(src, str(tmp_path / "out"), dtype="float32")
class TestSourceSizeProbe:
"""Refuse an obviously-too-big base BEFORE spending minutes sharding it."""
def test_reports_the_source_byte_size(self, tmp_path):
from soup_cli.utils.layer_shard import source_weight_bytes
src, _, _ = _fake_weights_dir(tmp_path)
assert source_weight_bytes(src) > 0
def test_missing_dir_raises(self, tmp_path):
from soup_cli.utils.layer_shard import source_weight_bytes
with pytest.raises(FileNotFoundError):
source_weight_bytes(str(tmp_path / "nope"))
# ==========================================================================
# Review-round hardening (ECC round 4 — test-quality audit)
# ==========================================================================
def _write_tiny_tokenizer(directory):
"""A real, offline PreTrainedTokenizerFast so `setup()` can tokenize.
The end-to-end streaming test must not depend on the HF cache or network.
"""
import json as _json
from tokenizers import Tokenizer, models, pre_tokenizers
vocab = {"<unk>": 0, "<s>": 1, "</s>": 2, "<pad>": 3}
for word in ("hello", "world", "hi", "yo", "the", "cat", "sat", "on", "mat"):
vocab[word] = len(vocab)
tokenizer = Tokenizer(models.WordLevel(vocab=vocab, unk_token="<unk>"))
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
tokenizer.save(os.path.join(directory, "tokenizer.json"))
with open(os.path.join(directory, "tokenizer_config.json"), "w", encoding="utf-8") as fh:
_json.dump(
{
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "<unk>",
"bos_token": "<s>",
"eos_token": "</s>",
"pad_token": "<pad>",
"model_max_length": 128,
"clean_up_tokenization_spaces": False,
},
fh,
)
class TestStreamingEndToEndSetup:
"""THE integration gap. Unit tests stop at `model(input_ids=...)`; the real
path is `wrapper.setup(dataset)` -> tokenize -> TRL SFTTrainer -> train().
A crash-on-every-run bug already hid in exactly this gap once."""
def _wrapper(self, tmp_path, monkeypatch):
import yaml
from soup_cli.config.loader import load_config_from_string
from soup_cli.trainer.sft import SFTTrainerWrapper
weights, _, _ = _tiny_llama_dir(tmp_path, n_layers=2)
_write_tiny_tokenizer(weights)
monkeypatch.setenv("SOUP_LAYER_STREAM_CACHE_DIR", str(tmp_path / "cache"))
monkeypatch.chdir(tmp_path)
cfg = load_config_from_string(
yaml.safe_dump(
{
"base": weights,
"task": "sft",
"backend": "transformers",
"modality": "text",
"data": {
"train": "train.jsonl",
"max_length": 64,
"chat_template": "chatml",
},
"training": {
"batch_size": 1,
"gradient_accumulation_steps": 1,
"quantization": "none",
"stream_layers": True,
"epochs": 1,
"logging_steps": 1,
"save_steps": 1000,
"gradient_checkpointing": True,
"lora": {"r": 4, "alpha": 8, "target_modules": ["q_proj", "v_proj"]},
},
"output": str(tmp_path / "out"),
}
)
)
dataset = {
"train": [
{
"messages": [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "hello world"},
]
}
for _ in range(4)
]
}
# Use the REAL device: TrainingArguments picks cuda or mps when one is
# available, so forcing the model to cpu here would only produce a
# device mismatch that no user would ever hit.
device = accelerator_device()
return SFTTrainerWrapper(cfg, device=device), dataset
def test_setup_builds_a_real_trl_trainer(self, tmp_path, monkeypatch):
wrapper, dataset = self._wrapper(tmp_path, monkeypatch)
wrapper.setup(dataset)
assert wrapper.trainer is not None
assert wrapper.model is not None
def test_hf_gradient_checkpointing_is_off_under_streaming(self, tmp_path, monkeypatch):
"""The behavioural version of the helper test: streaming checkpoints
per layer itself, so TrainingArguments must NOT also enable it."""
wrapper, dataset = self._wrapper(tmp_path, monkeypatch)
wrapper.setup(dataset)
assert wrapper.trainer.args.gradient_checkpointing is False
def test_one_training_step_actually_runs(self, tmp_path, monkeypatch):
"""Forward + backward + optimizer step through the streamed layers."""
wrapper, dataset = self._wrapper(tmp_path, monkeypatch)
wrapper.setup(dataset)
wrapper.trainer.args.max_steps = 1
wrapper.trainer.train()
loads = wrapper._stream_runtime.pool.loads
assert loads > 0, "no layer was ever streamed during training"
class TestInstallStreamingGuards:
def test_layer_count_mismatch_is_refused(self, tmp_path):
import dataclasses
from soup_cli.utils.layer_shard import shard_checkpoint
from soup_cli.utils.layer_stream_runtime import build_streamed_model
weights, _, _ = _tiny_llama_dir(tmp_path, n_layers=2)
shards = str(tmp_path / "shards")
index = shard_checkpoint(weights, shards, dtype="float32", arch="llama")
lying = dataclasses.replace(index, n_layers=5)
with pytest.raises(ValueError, match="decoder layers"):
build_streamed_model(
model_id=weights, shard_dir=shards, index=lying,
lora_config=_tiny_lora(), device="cpu", dtype="float32",
buffers=2, pin=False, seed=1,
)
def test_shard_missing_a_decoder_weight_is_refused(self, tmp_path):
from safetensors.torch import load_file, save_file
from soup_cli.utils.layer_shard import layer_shard_path, shard_checkpoint
from soup_cli.utils.layer_stream_runtime import build_streamed_model
weights, _, _ = _tiny_llama_dir(tmp_path, n_layers=2)
shards = str(tmp_path / "shards")
index = shard_checkpoint(weights, shards, dtype="float32", arch="llama")
import gc
for idx in range(index.n_layers):
blob = {k: v.clone() for k, v in load_file(layer_shard_path(shards, idx)).items()}
gc.collect() # Windows refuses to rewrite a still-mmapped file (err 1224)
blob.pop("self_attn.q_proj.weight")
save_file(blob, layer_shard_path(shards, idx))
with pytest.raises(ValueError, match="missing decoder weights"):
build_streamed_model(
model_id=weights, shard_dir=shards, index=index,
lora_config=_tiny_lora(), device="cpu", dtype="float32",
buffers=2, pin=False, seed=1,
)
class TestPinnedFallbackRuntime:
"""decide_pinning() is byte math; this is the RUNTIME retry that actually
catches a failed page-lock and downgrades to a pageable store."""
def test_pin_failure_falls_back_and_warns(self, tmp_path, monkeypatch):
import soup_cli.utils.layer_stream_runtime as rt
from soup_cli.utils.layer_shard import shard_checkpoint
weights, _, _ = _tiny_llama_dir(tmp_path)
shards = str(tmp_path / "shards")
index = shard_checkpoint(weights, shards, dtype="float32")
spec = rt.RamSource.spec_from_shard(shards)
real = rt.RamSource
attempts = []
class _FailsWhenPinned(real):
def __init__(self, shard_dir, n_layers, spec, *, pin=True):
attempts.append(pin)
if pin:
raise RuntimeError("CUDA error: out of memory")
super().__init__(shard_dir, n_layers, spec, pin=False)
monkeypatch.setattr(rt, "RamSource", _FailsWhenPinned)
printed = []
class _Console:
def print(self, msg):
printed.append(str(msg))
source, pinned = rt._build_source(shards, index.n_layers, spec, True, _Console())
assert attempts == [True, False]
assert pinned is False
assert source.nbytes > 0
assert any("pageable" in msg.lower() for msg in printed)
assert any("utilisation" in msg.lower() for msg in printed)
#: One decoder layer's weight for the AC3 refusal message, sized realistically so
#: the rendered figure is a real number and not `0.00 GB`.
_REFUSAL_HIDDEN = 4096
_REFUSAL_SPEC = {"weight": ((_REFUSAL_HIDDEN, _REFUSAL_HIDDEN), "bfloat16")}
#: (n_layers, rendered GB) pairs. TWO of them on purpose: a single fixed
#: expectation would be satisfied by a message that hardcoded that string, which
#: is the presence-not-value defect this PR's review round is about. Each case
#: also asserts the OTHER size is absent, so only a figure computed from the
#: actual store can satisfy both. 4096^2 bf16 = 32 MiB per layer, so 32 layers is
#: 1.073 GB and 64 layers is 2.147 GB.
_REFUSAL_STORE_SIZES = ((32, "1.07 GB"), (64, "2.15 GB"))
#: The distinguishing phrase of the disk-tier "pinning is inapplicable here"
#: announcement, DELETED in #971. It was true while the disk tier allocated a
#: fresh tensor per call and had nothing to page-lock; the async reader stages
#: into reusable host buffers, which pin exactly as the RAM store does. Kept so
#: the rewritten cases can assert it is ABSENT — a re-introduced explanation
#: would otherwise read as a harmless extra line.
_DISK_TIER_INAPPLICABLE_TEXT = "Pinning does not apply"
#: Same convention as `_REFUSAL_STORE_SIZES`, for the depth the disk tier's pin
#: refusal must quote: two values, each case asserting the OTHER is absent, so a
#: message hardcoding one of them cannot pass. Both inside
#: [MIN_STREAM_READ_AHEAD, MAX_STREAM_READ_AHEAD], and neither is the default —
#: a depth equal to the default would also be satisfied by a message that fell
#: back to it.
_REFUSAL_READ_AHEAD_DEPTHS = (3, 4)
class TestStreamPinRuntimeRefusal:
"""#366 criterion 3: training.stream_pin=true (require_pin) must refuse
loudly when the box cannot page-lock the store, instead of silently
degrading to a pageable one and spending the whole throughput margin."""
_SPEC = {"weight": ((2, 2), "float32")}
def _patch_ramsource_to_fail_pinning(self, monkeypatch):
import soup_cli.utils.layer_stream_runtime as rt
class _FailsWhenPinned:
def __init__(self, shard_dir, n_layers, spec, *, pin=True):
if pin:
raise RuntimeError("CUDA error: cannot allocate pinned memory")
self.nbytes = 1
# The real `RamSource` reports what it actually got, and since
# the RAM branch of `_build_source` returns `source.pinned`
# rather than a literal `False`, a double that omits it is no
# longer a faithful stand-in.
self.pinned = pin
monkeypatch.setattr(rt, "RamSource", _FailsWhenPinned)
return rt
def test_auto_falls_back_to_pageable(self, monkeypatch):
"""Without the override, a page-lock failure still falls back (default)."""
rt = self._patch_ramsource_to_fail_pinning(monkeypatch)
source, pinned = rt._build_source("d", 1, self._SPEC, True, None)
assert pinned is False
assert source.nbytes > 0
def test_the_returned_flag_is_read_off_the_source_on_the_ram_tier_too(
self, monkeypatch
):
"""`_build_source`'s docstring promises the second element "means the
same thing on both tiers". The disk branch returns `source.pinned`; the
RAM branch returned the literal it had just asked for, which is a claim
made at the CALL SITE about what the constructor did.
Value-identical with the real `RamSource`, which raises rather than
returning a pageable store under `pin=True` — so this stub is
counterfactual on purpose. It is the only way to tell "what the source
says" from "what the caller assumed", and that distinction is what the
docstring is asserting.
"""
import soup_cli.utils.layer_stream_runtime as rt
class _SaysPageableAnyway:
def __init__(self, shard_dir, n_layers, spec, *, pin=True):
self.nbytes = 1
self.pinned = False
monkeypatch.setattr(rt, "RamSource", _SaysPageableAnyway)
_, pinned = rt._build_source("d", 1, self._SPEC, True, None)
assert pinned is False
def test_forced_pin_refuses_instead_of_falling_back(self, monkeypatch):
rt = self._patch_ramsource_to_fail_pinning(monkeypatch)
with pytest.raises(RuntimeError, match="stream_pin"):
rt._build_source("d", 1, self._SPEC, True, None, require_pin=True)
@pytest.mark.parametrize(("n_layers", "expected"), _REFUSAL_STORE_SIZES)
def test_the_refusal_names_the_store_size(self, monkeypatch, n_layers, expected):
"""#366 AC3: the refusal must cite the STORE SIZE — not the page-locked
ceiling, which is deliberately left unprobed (`pinned_limit_bytes=None`)
and so cannot be quoted honestly. Dropping the figure from the message
left the suite green, which made AC3 unenforced.
Two sizes, each asserting the other is ABSENT: a message that hardcoded
one figure would satisfy a single-case test identically, which is exactly
the presence-not-value failure this review round is about."""
rt = self._patch_ramsource_to_fail_pinning(monkeypatch)
others = [gb for layers, gb in _REFUSAL_STORE_SIZES if layers != n_layers]
with pytest.raises(RuntimeError) as excinfo:
rt._build_source("d", n_layers, _REFUSAL_SPEC, True, None, require_pin=True)
message = str(excinfo.value)
assert expected in message, message
for other in others:
assert other not in message, message
def test_the_refusal_sums_heterogeneous_layer_specs(self, monkeypatch):
"""The #426 per-layer spec must survive #416's refusal path.
The two layers deliberately have different, disjoint keys and sizes.
Summing them gives 0.05 GB; multiplying the largest layer by two gives
0.07 GB, while multiplying the merged union by two gives 0.10 GB.
"""
rt = self._patch_ramsource_to_fail_pinning(monkeypatch)
layer_specs = [
{"self_attn.q_proj.weight": ((4096, 4096), "bfloat16")},
{"linear_attn.in_proj_qkv.weight": ((2048, 4096), "bfloat16")},
]
with pytest.raises(RuntimeError) as excinfo:
rt._build_source(
"d", 2, layer_specs, True, None, require_pin=True
)
message = str(excinfo.value)
assert "0.05 GB" in message, message
assert "0.07 GB" not in message, message
assert "0.10 GB" not in message, message
class TestDiskTierPinsItsStagingOrRefuses:
"""#971: the disk tier now pins the same way the RAM tier does.
It used to ANNOUNCE that pinning was inapplicable, which was true while the
tier allocated a fresh tensor per call — there was no host buffer to
page-lock. ``AsyncDiskSource`` reads ahead into REUSABLE host staging, and
pinning that staging is what lets the host-to-device copy overlap compute
at all. So an explicit ``training.stream_pin`` is honoured or refused here,
never explained away; and because the depth decides how much is page-locked,
the refusal offers a remedy the RAM tier cannot.
``require_pin`` is gated on a real CUDA device upstream, so this drives
`_build_source` directly to reach the branch without a GPU.
"""
_SPEC = {"weight": ((2, 2), "float32")}
class _RecordingConsole:
def __init__(self):
self.messages = []
def print(self, *args, **_kwargs):
self.messages.append(" ".join(str(a) for a in args))
def _patch_async_source(self, monkeypatch, *, fails_when_pinned: bool):
"""Mirrors `_patch_ramsource_to_fail_pinning` above, one tier down.
Patched on `soup_cli.utils.async_disk_source` because `_build_source`
imports the class lazily — that module attribute is the name the import
resolves, so patching the runtime module would not be seen.
"""
import soup_cli.utils.async_disk_source as ads
opened = []
class _StubAsyncDisk:
def __init__(self, shard_dir, n_layers, spec, **kwargs):
opened.append(kwargs)
if kwargs.get("pin") and fails_when_pinned:
raise RuntimeError("CUDA error: cannot allocate pinned memory")
self.pinned = bool(kwargs.get("pin"))
self.read_ahead = kwargs.get("read_ahead")
self.nbytes = 16
self.disk_bytes = 64
def close(self):
pass
monkeypatch.setattr(ads, "AsyncDiskSource", _StubAsyncDisk)
return opened
def _build_on_disk(self, monkeypatch, *, require_pin, pin=True, fails=True,
read_ahead=4):
import soup_cli.utils.layer_stream_runtime as rt
opened = self._patch_async_source(monkeypatch, fails_when_pinned=fails)
console = self._RecordingConsole()
source, pinned = rt._build_source(
"d", 1, self._SPEC, pin, console, "disk",
require_pin=require_pin, read_ahead=read_ahead,
)
return source, pinned, console.messages, opened
@pytest.mark.parametrize("read_ahead", _REFUSAL_READ_AHEAD_DEPTHS)
def test_an_explicit_request_that_cannot_be_met_refuses(
self, monkeypatch, read_ahead
):
"""The RAM tier's contract, now the disk tier's: stream_pin=true means
refuse, not degrade. The remedy list must include the one that is
specific to this tier — the read-ahead depth IS the multiplier on how
much host memory gets page-locked.
Two depths, each asserting the other is ABSENT. `assert "4" in message`
was a one-character check that a message hardcoding `read_ahead=4`
satisfied identically — the presence-not-value defect this file's own
`_REFUSAL_STORE_SIZES` comment spends six lines explaining.
"""
others = [d for d in _REFUSAL_READ_AHEAD_DEPTHS if d != read_ahead]
with pytest.raises(RuntimeError) as excinfo:
self._build_on_disk(monkeypatch, require_pin=True, read_ahead=read_ahead)
message = str(excinfo.value)
assert "training.stream_pin" in message, message
assert f"training.stream_read_ahead={read_ahead}" in message, (
"the refusal must quote the depth it could not pin, not just name "
f"the field: {message}"
)
for other in others:
assert f"stream_read_ahead={other}" not in message, message
def test_without_the_flag_it_falls_back_loudly_and_retries_pageable(
self, monkeypatch
):
"""Refusing here would brick the very runs the disk tier exists for, so
the default is a fallback — but a silent one would spend the whole
overlap margin without saying so."""
source, pinned, messages, opened = self._build_on_disk(
monkeypatch, require_pin=False
)
assert pinned is False
assert source.pinned is False
assert [kwargs["pin"] for kwargs in opened] == [True, False], (
"the fallback must RETRY with pin=False, not hand back the object "
f"that failed: {opened}"
)
assert len(messages) == 1, messages
# The measured figures, not the punctuation between them: the RAM
# branch's fallback spells it "drops from ~97% to ~79%" and its refusal
# "~97% -> ~79%", and this must stay readable beside both.
assert "~97%" in messages[0] and "~79%" in messages[0], messages[0]
assert "training.stream_read_ahead" in messages[0], messages[0]
def test_a_successful_pin_says_nothing(self, monkeypatch):
"""Control: the RAM tier keeps silent when its default works, and so
must this one. Without this half, a mutant that always printed the
fallback warning would pass the case above."""
source, pinned, messages, opened = self._build_on_disk(
monkeypatch, require_pin=False, fails=False
)
assert pinned is True
assert source.pinned is True
assert [kwargs["pin"] for kwargs in opened] == [True]
assert messages == []
def test_an_honoured_request_never_says_pinning_does_not_apply(
self, monkeypatch
):
"""The deleted announcement was FALSE once staging existed. A run that
asked for pinning and got it must not be told it was inapplicable."""
_source, pinned, messages, _opened = self._build_on_disk(
monkeypatch, require_pin=True, fails=False
)
assert pinned is True
assert not any(_DISK_TIER_INAPPLICABLE_TEXT in m for m in messages), messages
assert messages == []
def test_the_depth_reaches_the_source(self, monkeypatch):
"""#748: `training.stream_read_ahead` is read by nothing unless it lands
on the constructor. Asserted on the value, not on presence."""
source, _pinned, _messages, opened = self._build_on_disk(
monkeypatch, require_pin=False, fails=False, read_ahead=7
)
assert opened[0]["read_ahead"] == 7
assert source.read_ahead == 7
def test_the_fallback_still_reaches_the_log_without_a_console(
self, monkeypatch, caplog
):
"""`_build_source` is also called with `console=None` (the runtime does
not always have one). The decision must still be recorded there, or the
degradation is silent on exactly the path with no screen to print to."""
import logging
import soup_cli.utils.layer_stream_runtime as rt
self._patch_async_source(monkeypatch, fails_when_pinned=True)
with caplog.at_level(logging.WARNING):
_source, pinned = rt._build_source(
"d", 1, self._SPEC, True, None, "disk", require_pin=False
)
assert pinned is False
assert "~97%" in caplog.text and "~79%" in caplog.text, caplog.text
class TestCachedIndexInvalidation:
def _prepare(self, tmp_path):
from soup_cli.utils.layer_shard import shard_checkpoint
src, _, _ = _fake_weights_dir(tmp_path, untied=True)
out = str(tmp_path / "shards")
shard_checkpoint(src, out, dtype="float32")
return src, out
def test_missing_extras_shard_reshards(self, tmp_path):
from soup_cli.utils.layer_shard import extras_shard_path, shard_checkpoint
src, out = self._prepare(tmp_path)
os.unlink(extras_shard_path(out))
shard_checkpoint(src, out, dtype="float32")
assert os.path.exists(extras_shard_path(out))
def test_missing_layer_shard_reshards(self, tmp_path):
from soup_cli.utils.layer_shard import layer_shard_path, shard_checkpoint
src, out = self._prepare(tmp_path)
os.unlink(layer_shard_path(out, 1))
shard_checkpoint(src, out, dtype="float32")
assert os.path.exists(layer_shard_path(out, 1))
def test_missing_large_layer_shard_reshards(self, tmp_path):
from soup_cli.utils.layer_shard import large_shard_path, shard_checkpoint
src, out = self._prepare(tmp_path)
path = large_shard_path(out, "model.embed_tokens.weight")
os.unlink(path)
shard_checkpoint(src, out, dtype="float32")
assert os.path.exists(path)
def test_index_with_missing_keys_reshards(self, tmp_path):
import json as _json
from soup_cli.utils.layer_shard import shard_checkpoint
src, out = self._prepare(tmp_path)
with open(os.path.join(out, "index.json"), "w", encoding="utf-8") as fh:
_json.dump({"n_layers": 3}, fh) # well-formed JSON, missing fields
index = shard_checkpoint(src, out, dtype="float32")
assert index.n_layers == 3
assert index.layer_keys
class TestMaterializeExtrasGuard:
def test_incomplete_extras_shard_is_refused(self, tmp_path):
"""A corrupt extras shard must fail loudly, not leave resident weights meta."""
from safetensors.torch import load_file, save_file
from soup_cli.utils.layer_shard import extras_shard_path, shard_checkpoint
from soup_cli.utils.layer_stream_runtime import build_meta_skeleton, materialize_extras
weights, _, _ = _tiny_llama_dir(tmp_path)
shards = str(tmp_path / "shards")
index = shard_checkpoint(weights, shards, dtype="float32")
import gc
blob = {k: v.clone() for k, v in load_file(extras_shard_path(shards)).items()}
gc.collect() # Windows refuses to rewrite a still-mmapped file (err 1224)
blob.pop("model.norm.weight")
save_file(blob, extras_shard_path(shards))
model = build_meta_skeleton(weights, dtype="float32")
with pytest.raises(RuntimeError, match="unmaterialised"):
materialize_extras(model, shards, index, device="cpu", dtype="float32")
class TestHardwareFitGateIsStreamingAware:
"""The pre-flight VRAM gate models a RESIDENT run: full weights + optimizer
+ grads on the card. Under streaming, peak VRAM is bounded by ONE layer, so
the resident prediction refuses precisely the runs layer streaming exists to
enable (a 3B base on a 4 GB card predicts >4 GB and is blocked)."""
def _cfg(self, stream: bool):
import yaml
from soup_cli.config.loader import load_config_from_string
body = {
"base": "Qwen/Qwen2.5-3B",
"task": "sft",
"backend": "transformers",
"modality": "text",
"data": {"train": "train.jsonl", "max_length": 512},
"training": {
"batch_size": 1,
"gradient_accumulation_steps": 1,
"quantization": "none",
"stream_layers": stream,
"lora": {"r": 8, "alpha": 16},
},
}
return load_config_from_string(yaml.safe_dump(body))
def test_resident_run_is_still_gated(self):
"""Control: without streaming the gate must still fire on a 4 GB card,
otherwise this test proves nothing about the streaming branch."""
import typer
from soup_cli.commands.train import _hardware_fit_preflight
gpu = {"memory_total_bytes": 4 * 10**9}
with pytest.raises(typer.Exit):
_hardware_fit_preflight(self._cfg(False), gpu, allow_oom_attempt=False)
def test_streaming_run_is_not_blocked_by_the_resident_prediction(self):
from soup_cli.commands.train import _hardware_fit_preflight
gpu = {"memory_total_bytes": 4 * 10**9}
_hardware_fit_preflight(self._cfg(True), gpu, allow_oom_attempt=False)