Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Expert-granularity streaming (``utils/expert_stream.py``).
The gate is bit-exactness against transformers' eager experts loop: the
streamed module must equal the resident one forward AND backward, on the
block alone and inside a streamed DeepSeek-V3 model.
"""
from __future__ import annotations
import pytest
from tests._windows_ci import skip_on_windows_ci
CUDA = pytest.mark.gpu(reason="NF4 dequantisation needs bitsandbytes on CUDA")
# --------------------------------------------------------------------------
# block level
# --------------------------------------------------------------------------
def _deepseek_v3_experts(seed):
import torch
from transformers import DeepseekV3Config
from transformers.models.deepseek_v3.modeling_deepseek_v3 import DeepseekV3Experts
config = DeepseekV3Config(
hidden_size=32, moe_intermediate_size=16, n_routed_experts=8, n_group=2,
topk_group=1, num_experts_per_tok=2,
)
config._experts_implementation = "eager"
torch.manual_seed(seed)
module = DeepseekV3Experts(config)
with torch.no_grad():
module.gate_up_proj.normal_(0.0, 0.2)
module.down_proj.normal_(0.0, 0.2)
module.requires_grad_(False)
return module, 8, 2
def _olmoe_experts(seed):
import torch
from transformers import OlmoeConfig
from transformers.models.olmoe.modeling_olmoe import OlmoeExperts
config = OlmoeConfig(hidden_size=32, intermediate_size=16, num_experts=8, num_experts_per_tok=2)
config._experts_implementation = "eager"
torch.manual_seed(seed)
module = OlmoeExperts(config)
with torch.no_grad():
module.gate_up_proj.normal_(0.0, 0.2)
module.down_proj.normal_(0.0, 0.2)
module.requires_grad_(False)
return module, 8, 2
def _streamed_copy(tmp_path, module, n_experts):
from soup_cli.utils.expert_stream import (
ExpertFetcher,
ExpertShardReader,
streamed_experts_class,
write_expert_shard,
)
path = str(tmp_path / "experts_001.safetensors")
pairs = (
(module.gate_up_proj[e].detach(), module.down_proj[e].detach())
for e in range(n_experts)
)
info = write_expert_shard(path, pairs, quantise=False)
reader = ExpertShardReader(
path, count=info.count, shapes=info.shapes, quant_specs={}, dtype="float32"
)
streamed = streamed_experts_class()(
reader, ExpertFetcher(device="cpu", dtype="float32"), module.act_fn, 1
)
return streamed, reader
def _routing(n_tokens, n_experts, top_k, hidden, seed):
import torch
generator = torch.Generator().manual_seed(seed)
scores = torch.rand(n_tokens, n_experts, generator=generator)
top_k_index = scores.topk(top_k, dim=-1).indices
top_k_weights = torch.rand(n_tokens, top_k, generator=generator).requires_grad_(True)
states = torch.randn(n_tokens, hidden, generator=generator).requires_grad_(True)
grad_out = torch.randn(n_tokens, hidden, generator=generator)
return states, top_k_index, top_k_weights, grad_out
@pytest.mark.parametrize(
"build", [_deepseek_v3_experts, _olmoe_experts], ids=["deepseek_v3", "olmoe"]
)
def test_streamed_experts_forward_equals_eager_bit_exactly(tmp_path, build):
import torch
module, n_experts, top_k = build(5)
streamed, reader = _streamed_copy(tmp_path, module, n_experts)
states, index, weights, _ = _routing(24, n_experts, top_k, 32, 9)
with torch.no_grad():
want = module(states, index, weights)
got = streamed(states, index, weights)
assert torch.equal(got, want), (got - want).abs().max().item()
reader.close()
@pytest.mark.parametrize(
"build", [_deepseek_v3_experts, _olmoe_experts], ids=["deepseek_v3", "olmoe"]
)
def test_streamed_experts_backward_equals_eager_bit_exactly(tmp_path, build):
"""Input and routing-weight gradients, the two a frozen expert passes on."""
import torch
module, n_experts, top_k = build(5)
streamed, reader = _streamed_copy(tmp_path, module, n_experts)
states, index, weights, grad_out = _routing(24, n_experts, top_k, 32, 9)
module(states, index, weights).backward(grad_out)
want_states, want_weights = states.grad.clone(), weights.grad.clone()
states.grad = None
weights.grad = None
streamed(states, index, weights).backward(grad_out)
assert float(want_states.abs().sum()) > 0.0
assert torch.equal(states.grad, want_states), (states.grad - want_states).abs().max().item()
assert torch.equal(weights.grad, want_weights)
reader.close()
# The bf16 case runs a CPU bf16 matmul under autocast, and part of GitHub's windows-latest
# fleet dies on it with 0xc000001d (illegal instruction) in the eager reference module's
# forward; the crash kills the xdist worker and the cell later times out (#382, #1314).
# Only that case is skipped, and only on a Windows CI runner: the fp32 control, a developer's
# Windows box, ubuntu and macos all still run it.
@pytest.mark.parametrize(
"autocast",
[
pytest.param(False, id="fp32-control"),
pytest.param(True, id="cpu-bf16-autocast", marks=skip_on_windows_ci),
],
)
def test_streamed_experts_backward_preserves_forward_autocast(tmp_path, autocast):
"""Recomputation must differentiate the forward precision after its AMP region exits."""
import torch
module, n_experts, top_k = _deepseek_v3_experts(5)
streamed, reader = _streamed_copy(tmp_path, module, n_experts)
states, index, weights, grad_out = _routing(24, n_experts, top_k, 32, 9)
try:
with torch.autocast("cpu", dtype=torch.bfloat16, enabled=autocast):
want = module(states, index, weights)
want.backward(grad_out)
want_states, want_weights = states.grad.clone(), weights.grad.clone()
states.grad = None
weights.grad = None
with torch.autocast("cpu", dtype=torch.bfloat16, enabled=autocast):
got = streamed(states, index, weights)
got.backward(grad_out)
assert torch.equal(got, want)
assert float(want_states.abs().sum()) > 0.0
assert torch.equal(states.grad, want_states), (states.grad - want_states).abs().max().item()
assert torch.equal(weights.grad, want_weights)
finally:
reader.close()
def test_each_routed_expert_is_read_once_forward_and_once_backward(tmp_path):
"""The cost the docstring states: an expert nobody routed to is never read."""
import torch
module, n_experts, _ = _deepseek_v3_experts(5)
streamed, reader = _streamed_copy(tmp_path, module, n_experts)
states = torch.randn(6, 32, requires_grad=True)
index = torch.tensor([[0, 3]] * 6)
weights = torch.full((6, 2), 0.5, requires_grad=True)
out = streamed(states, index, weights)
assert reader.fetches == 2
out.sum().backward()
assert reader.fetches == 4
reader.close()
def test_fetched_weights_are_owned_not_views_of_the_staging_buffer(tmp_path):
"""Every read lands in one shared staging buffer; a fetch must hand back
tensors the next read cannot overwrite (on a CPU target `.to` would not copy)."""
import torch
module, n_experts, _ = _deepseek_v3_experts(5)
streamed, reader = _streamed_copy(tmp_path, module, n_experts)
first = [tensor.clone() for tensor in streamed.fetch(0)]
held = streamed.fetch(0)
streamed.fetch(1)
assert torch.equal(held[0], first[0]) and torch.equal(held[1], first[1])
assert not torch.equal(first[0], module.gate_up_proj[1])
reader.close()
def test_expert_weights_are_released_after_each_expert(tmp_path):
"""The VRAM bound: nothing keeps a fetched expert alive once its MLP ran, not
the autograd graph after forward, not anything after backward."""
import gc
import weakref
module, n_experts, top_k = _deepseek_v3_experts(5)
streamed, reader = _streamed_copy(tmp_path, module, n_experts)
refs = []
real_fetch = streamed.fetch
def tracking_fetch(expert):
gate_up, down = real_fetch(expert)
refs.extend((weakref.ref(gate_up), weakref.ref(down)))
return gate_up, down
streamed.fetch = tracking_fetch
states, index, weights, grad_out = _routing(24, n_experts, top_k, 32, 9)
out = streamed(states, index, weights)
gc.collect()
assert refs and all(ref() is None for ref in refs), "forward kept expert weights alive"
refs.clear()
out.backward(grad_out)
gc.collect()
assert refs and all(ref() is None for ref in refs), "backward kept expert weights alive"
reader.close()
def test_reader_direct_and_buffered_reads_agree(tmp_path):
import torch
from soup_cli.utils.expert_stream import (
ExpertShardReader,
aligned_staging,
write_expert_shard,
)
path = str(tmp_path / "experts_001.safetensors")
pairs = [(torch.randn(6, 4), torch.randn(4, 3)) for _ in range(3)]
info = write_expert_shard(path, pairs, quantise=False)
readers = [
ExpertShardReader(
path, count=3, shapes=info.shapes, quant_specs={}, dtype="float32", direct=direct
)
for direct in (True, False)
]
assert readers[1].direct is False
for expert in range(3):
views = []
for reader in readers:
_owner, staging = aligned_staging(reader.max_span)
views.append({k: v.clone() for k, v in reader.read_host(expert, staging).items()})
assert views[0].keys() == views[1].keys()
for key in views[0]:
assert torch.equal(views[0][key], views[1][key])
assert torch.equal(views[0][f"{expert}.gate_up"], pairs[expert][0])
with pytest.raises(IndexError):
readers[0].read_host(3, aligned_staging(readers[0].max_span)[1])
with pytest.raises(ValueError, match="staging holds"):
readers[0].read_host(0, aligned_staging(16)[1])
for reader in readers:
reader.close()
@pytest.mark.parametrize("expert", [-1, 3, 4])
def test_read_host_refuses_an_expert_outside_the_file(tmp_path, expert):
"""-1 would otherwise read the last expert through Python's negative index."""
import torch
from soup_cli.utils.expert_stream import (
ExpertShardReader,
aligned_staging,
write_expert_shard,
)
path = str(tmp_path / "experts_001.safetensors")
pairs = [(torch.randn(6, 4), torch.randn(4, 3)) for _ in range(3)]
info = write_expert_shard(path, pairs, quantise=False)
reader = ExpertShardReader(path, count=3, shapes=info.shapes, quant_specs={}, dtype="float32")
with pytest.raises(IndexError, match=r"outside \[0, 3\)"):
reader.read_host(expert, aligned_staging(reader.max_span)[1])
reader.close()
def test_a_reader_holds_no_handle_until_its_first_fetch(tmp_path):
"""An install that fails after the readers were built must not leave the
experts files open (Windows keeps an open file locked until collection)."""
import torch
from soup_cli.utils.expert_stream import (
ExpertShardReader,
aligned_staging,
write_expert_shard,
)
path = str(tmp_path / "experts_001.safetensors")
info = write_expert_shard(path, [(torch.zeros(6, 4), torch.zeros(4, 3))], quantise=False)
reader = ExpertShardReader(path, count=1, shapes=info.shapes, quant_specs={}, dtype="float32")
assert reader._handle is None
staging = aligned_staging(reader.max_span)[1]
reader.read_host(0, staging)
first = reader._handle
assert first is not None
reader.read_host(0, staging)
# One handle for the reader's life: a reopen per fetch would leak the last one.
assert reader._handle is first
reader.close()
assert reader._handle is None
def test_a_file_replaced_after_the_header_was_read_is_refused_at_first_fetch(tmp_path):
import os
import torch
from soup_cli.utils.expert_stream import (
ExpertShardReader,
aligned_staging,
write_expert_shard,
)
path = str(tmp_path / "experts_001.safetensors")
info = write_expert_shard(path, [(torch.zeros(6, 4), torch.zeros(4, 3))], quantise=False)
reader = ExpertShardReader(path, count=1, shapes=info.shapes, quant_specs={}, dtype="float32")
replacement = str(tmp_path / "other.safetensors")
write_expert_shard(replacement, [(torch.ones(6, 4), torch.ones(4, 3))], quantise=False)
os.replace(replacement, path)
with pytest.raises(ValueError, match="changed on disk since its header was read"):
reader.read_host(0, aligned_staging(reader.max_span)[1])
assert reader._handle is None
def test_install_refuses_mismatched_layers_and_expert_adapters(tmp_path):
import torch.nn as nn
from soup_cli.utils.expert_stream import ExpertFetcher, install_expert_streaming
module, n_experts, _ = _deepseek_v3_experts(5)
_, reader = _streamed_copy(tmp_path, module, n_experts)
fetcher = ExpertFetcher(device="cpu", dtype="float32")
with pytest.raises(ValueError, match="no experts file for it"):
install_expert_streaming([_holder(module)], {}, fetcher)
with pytest.raises(ValueError, match="which has no routed-experts module"):
install_expert_streaming([nn.Module()], {0: reader}, fetcher)
adapted = _holder(module)
adapted.mlp.experts.lora_A = nn.Parameter(module.gate_up_proj.new_zeros(2, 2))
with pytest.raises(ValueError, match="adapters on the routed experts"):
install_expert_streaming([adapted], {0: reader}, fetcher)
reader.close()
@pytest.mark.parametrize(
("field", "value", "match"),
[
("expert_layout", "experts", "unknown expert_layout"),
("expert_layers", [2, 1], "distinct ascending"),
("expert_layers", [9], "distinct ascending"),
("expert_count", 5000, "do not describe one layout"),
("expert_shapes", {"gate_up": [64, 0], "down": [64, 32]}, "do not describe one layout"),
],
)
def test_read_shard_index_refuses_a_malformed_expert_layout(tmp_path, field, value, match):
import json
from soup_cli.utils.layer_shard import read_shard_index
weights, _ = _tiny_deepseek_v3(tmp_path)
_shard(tmp_path, weights, expert_layout="per_expert")
path = tmp_path / "shards" / "index.json"
payload = json.loads(path.read_text(encoding="utf-8"))
payload[field] = value
path.write_text(json.dumps(payload), encoding="utf-8")
with pytest.raises(ValueError, match=match):
read_shard_index(str(tmp_path / "shards"))
def test_missing_experts_file_is_a_cache_miss(tmp_path):
import os
from soup_cli.utils.layer_shard import expert_paths, inspect_shard_cache
weights, _ = _tiny_deepseek_v3(tmp_path)
index = _shard(tmp_path, weights, expert_layout="per_expert")
os.remove(expert_paths(str(tmp_path / "shards"), index)[2])
cached, reason = inspect_shard_cache(
str(tmp_path / "shards"), "float32", index.source_fingerprint, index.source_files,
"none", False, "", "", stripe_roots=(), expert_layout="per_expert",
)
assert cached is None and "experts_002.safetensors" in reason
def test_resharding_to_the_fused_layout_removes_the_experts_files(tmp_path):
import os
from soup_cli.utils.layer_shard import shard_cache_disk_bytes
weights, _ = _tiny_deepseek_v3(tmp_path)
per_expert = _shard(tmp_path, weights, expert_layout="per_expert")
shards = tmp_path / "shards"
assert shard_cache_disk_bytes(str(shards), per_expert) > sum(
os.path.getsize(path) for path in shards.glob("layer_*.safetensors")
)
fused = _shard(tmp_path, weights)
assert fused.expert_layout == ""
assert not list(shards.glob("experts_*.safetensors"))
def test_per_expert_layout_refuses_a_checkpoint_without_per_expert_keys_before_writing(tmp_path):
import torch
from safetensors.torch import save_file
from soup_cli.utils.layer_shard import shard_checkpoint
weights = tmp_path / "w"
weights.mkdir()
save_file(
{
"model.embed_tokens.weight": torch.zeros(8, 4),
"model.layers.0.mlp.experts.gate_up_proj": torch.zeros(2, 6, 4),
"model.layers.0.mlp.experts.down_proj": torch.zeros(2, 4, 3),
},
str(weights / "model.safetensors"),
)
with pytest.raises(ValueError, match="found no per-expert routed weights"):
shard_checkpoint(
str(weights), str(tmp_path / "s"), dtype="float32", arch="deepseek_v3",
expert_layout="per_expert",
)
assert not list((tmp_path / "s").glob("layer_*.safetensors"))
# --------------------------------------------------------------------------
# experts file + reader
# --------------------------------------------------------------------------
def test_expert_shard_stores_gate_rows_before_up_rows(tmp_path):
import torch
from safetensors.torch import load_file
from soup_cli.utils.expert_stream import write_expert_shard
gate_up = torch.cat((torch.full((3, 4), 1.0), torch.full((3, 4), 2.0)))
down = torch.full((4, 3), 3.0)
path = str(tmp_path / "experts_001.safetensors")
info = write_expert_shard(path, [(gate_up, down), (gate_up + 10, down + 10)], quantise=False)
stored = load_file(path)
assert info.count == 2 and dict(info.shapes) == {"gate_up": (6, 4), "down": (4, 3)}
assert torch.equal(stored["1.gate_up"][:3], torch.full((3, 4), 11.0))
assert torch.equal(stored["1.down"], torch.full((4, 3), 13.0))
def test_expert_shard_refuses_an_inconsistent_expert(tmp_path):
import torch
from soup_cli.utils.expert_stream import write_expert_shard
good = (torch.zeros(6, 4), torch.zeros(4, 3))
bad = (torch.zeros(8, 4), torch.zeros(4, 3))
with pytest.raises(ValueError, match="expert 1 gate_up has shape"):
write_expert_shard(str(tmp_path / "x.safetensors"), [good, bad], quantise=False)
def test_reader_refuses_a_file_missing_an_expert(tmp_path):
import torch
from soup_cli.utils.expert_stream import ExpertShardReader, write_expert_shard
path = str(tmp_path / "experts_001.safetensors")
info = write_expert_shard(path, [(torch.zeros(6, 4), torch.zeros(4, 3))], quantise=False)
with pytest.raises(ValueError, match="has no tensor '1.gate_up'"):
ExpertShardReader(path, count=2, shapes=info.shapes, quant_specs={}, dtype="float32")
def _nf4_spec(shape, nested=False):
from soup_cli.utils.layer_shard import NF4WeightSpec
return NF4WeightSpec(
shape=shape, dtype="bfloat16", blocksize=64, quant_type="nf4",
nested=nested, nested_blocksize=256 if nested else 0,
)
def _nf4_file(path, *, packed_bytes, absmax_blocks):
"""A one-expert NF4 experts file with chosen sidecar sizes (no bitsandbytes)."""
import torch
from safetensors.torch import save_file
blob = {}
for part, weights in (("gate_up", 6 * 64), ("down", 64 * 3)):
n_packed = packed_bytes if part == "gate_up" else weights // 2
n_blocks = absmax_blocks if part == "gate_up" else weights // 64
blob[f"0.{part}"] = torch.zeros((n_packed, 1), dtype=torch.uint8)
blob[f"0.{part}::absmax"] = torch.ones(n_blocks)
save_file(blob, str(path))
return str(path)
def test_reader_accepts_nf4_sidecars_of_exactly_the_spec_size(tmp_path):
from soup_cli.utils.expert_stream import ExpertShardReader
path = _nf4_file(tmp_path / "e.safetensors", packed_bytes=6 * 64 // 2, absmax_blocks=6)
specs = {"gate_up": _nf4_spec((6, 64)), "down": _nf4_spec((64, 3))}
reader = ExpertShardReader(
path, count=1, shapes={"gate_up": (6, 64), "down": (64, 3)},
quant_specs=specs, dtype="bfloat16",
)
reader.close()
@pytest.mark.parametrize(
("packed_bytes", "absmax_blocks"), [(16, 6), (6 * 64 // 2, 2)], ids=["packed", "absmax"]
)
def test_reader_refuses_nf4_sidecars_smaller_than_the_spec(tmp_path, packed_bytes, absmax_blocks):
"""The dequantisation reads prod(spec.shape) weights natively without a bounds
check, so a header-valid file holding fewer bytes than the spec must be refused
before any read (soup-c2's security read of this reader)."""
from soup_cli.utils.expert_stream import ExpertShardReader
path = _nf4_file(
tmp_path / "e.safetensors", packed_bytes=packed_bytes, absmax_blocks=absmax_blocks
)
specs = {"gate_up": _nf4_spec((6, 64)), "down": _nf4_spec((64, 3))}
with pytest.raises(ValueError, match="the index expects"):
ExpertShardReader(
path, count=1, shapes={"gate_up": (6, 64), "down": (64, 3)},
quant_specs=specs, dtype="bfloat16",
)
def test_reader_refuses_an_nf4_spec_that_disagrees_with_the_index_shape(tmp_path):
from soup_cli.utils.expert_stream import ExpertShardReader
path = _nf4_file(tmp_path / "e.safetensors", packed_bytes=6 * 64 // 2, absmax_blocks=6)
specs = {"gate_up": _nf4_spec((12, 64)), "down": _nf4_spec((64, 3))}
with pytest.raises(ValueError, match="is not the index's"):
ExpertShardReader(
path, count=1, shapes={"gate_up": (6, 64), "down": (64, 3)},
quant_specs=specs, dtype="bfloat16",
)
def test_reader_refuses_a_dense_dtype_other_than_the_index_dtype(tmp_path):
import torch
from soup_cli.utils.expert_stream import ExpertShardReader, write_expert_shard
path = str(tmp_path / "experts_001.safetensors")
info = write_expert_shard(path, [(torch.zeros(6, 4), torch.zeros(4, 3))], quantise=False)
with pytest.raises(ValueError, match="the index expects torch.bfloat16"):
ExpertShardReader(path, count=1, shapes=info.shapes, quant_specs={}, dtype="bfloat16")
def test_reader_refuses_an_expert_over_the_byte_limit(tmp_path, monkeypatch):
import torch
import soup_cli.utils.expert_stream as expert_stream
path = str(tmp_path / "experts_001.safetensors")
info = expert_stream.write_expert_shard(
path, [(torch.zeros(6, 4), torch.zeros(4, 3))], quantise=False
)
monkeypatch.setattr(expert_stream, "_MAX_EXPERT_BYTES", 1024)
with pytest.raises(ValueError, match="over the 1024-byte limit"):
expert_stream.ExpertShardReader(
path, count=1, shapes=info.shapes, quant_specs={}, dtype="float32"
)
def test_reader_refuses_a_file_swapped_while_it_opens(tmp_path, monkeypatch):
"""The header is read through the identity of the handle the reads use."""
import dataclasses
import torch
import soup_cli.utils.safetensors_reader as safetensors_reader
from soup_cli.utils.expert_stream import ExpertShardReader, write_expert_shard
path = str(tmp_path / "experts_001.safetensors")
info = write_expert_shard(path, [(torch.zeros(6, 4), torch.zeros(4, 3))], quantise=False)
real = safetensors_reader.read_header_with_identity
def swapped(target):
header, identity = real(target)
return header, dataclasses.replace(identity, mtime_ns=identity.mtime_ns + 1)
monkeypatch.setattr(safetensors_reader, "read_header_with_identity", swapped)
with pytest.raises(ValueError, match="changed while it was being opened"):
ExpertShardReader(path, count=1, shapes=info.shapes, quant_specs={}, dtype="float32")
def test_reader_refuses_a_file_with_more_experts_than_the_index(tmp_path):
import torch
from soup_cli.utils.expert_stream import ExpertShardReader, write_expert_shard
path = str(tmp_path / "experts_001.safetensors")
pair = (torch.zeros(6, 4), torch.zeros(4, 3))
info = write_expert_shard(path, [pair, (pair[0] + 1, pair[1] + 1)], quantise=False)
with pytest.raises(ValueError, match="holds more than 1 experts"):
ExpertShardReader(path, count=1, shapes=info.shapes, quant_specs={}, dtype="float32")
# --------------------------------------------------------------------------
# a whole streamed DeepSeek-V3 with per-expert layout vs the resident model
# --------------------------------------------------------------------------
def _tiny_deepseek_v3(tmp_path):
import torch
from transformers import DeepseekV3Config, DeepseekV3ForCausalLM
torch.manual_seed(11)
config = DeepseekV3Config(
vocab_size=128, hidden_size=64, intermediate_size=128, moe_intermediate_size=32,
num_hidden_layers=3, first_k_dense_replace=1, num_attention_heads=4,
num_key_value_heads=4, n_shared_experts=1, n_routed_experts=8, n_group=2,
topk_group=1, num_experts_per_tok=2, kv_lora_rank=16, q_lora_rank=24,
qk_nope_head_dim=16, qk_rope_head_dim=8, v_head_dim=16, max_position_embeddings=128,
)
model = DeepseekV3ForCausalLM(config).to(torch.float32).eval()
generator = torch.Generator().manual_seed(23)
with torch.no_grad():
for layer in model.model.layers[1:]:
bias = layer.mlp.gate.e_score_correction_bias
bias.copy_(torch.randn(bias.shape, generator=generator))
weights = tmp_path / "model"
model.save_pretrained(str(weights))
# The streamed experts follow the eager loop; the resident control must too.
model.set_experts_implementation("eager")
return str(weights), model
def _lora():
from peft import LoraConfig, TaskType
return LoraConfig(
r=4, lora_alpha=8, lora_dropout=0.0, bias="none",
target_modules=["q_b_proj", "kv_b_proj", "o_proj"], task_type=TaskType.CAUSAL_LM,
)
def _shard(tmp_path, weights, **kwargs):
from soup_cli.utils.layer_shard import shard_checkpoint
return shard_checkpoint(
weights, str(tmp_path / "shards"), dtype="float32", arch="deepseek_v3",
num_hidden_layers=3, **kwargs,
)
def _paired(tmp_path):
import torch
from peft import get_peft_model
from soup_cli.utils.layer_stream_runtime import build_streamed_model
weights, resident = _tiny_deepseek_v3(tmp_path)
index = _shard(tmp_path, weights, expert_layout="per_expert")
model, runtime = build_streamed_model(
model_id=weights, shard_dir=str(tmp_path / "shards"), index=index,
lora_config=_lora(), device="cpu", dtype="float32", buffers=2, pin=False, seed=3,
)
ref = get_peft_model(resident, _lora())
generator = torch.Generator().manual_seed(17)
with torch.no_grad():
for name, parameter in model.named_parameters():
if "lora_B" in name:
parameter.copy_(torch.randn(parameter.shape, generator=generator) * 0.02)
src = {k.replace(".inner.", "."): v for k, v in model.state_dict().items() if "lora_" in k}
dst = {k: v for k, v in ref.state_dict().items() if "lora_" in k}
assert src and set(src) == set(dst)
for key, value in src.items():
dst[key].copy_(value)
model.eval()
ref.eval()
ids = torch.randint(0, 128, (2, 12), generator=generator)
return model, runtime, ref, ids, index
def test_per_expert_layout_moves_routed_experts_out_of_the_layer_file(tmp_path):
from safetensors.torch import load_file
from soup_cli.utils.layer_shard import expert_paths, layer_paths, read_shard_index
weights, _ = _tiny_deepseek_v3(tmp_path)
index = _shard(tmp_path, weights, expert_layout="per_expert")
assert index.expert_layout == "per_expert"
assert index.expert_layers == (1, 2)
assert index.expert_count == 8
assert dict(index.expert_shapes) == {"gate_up": (64, 64), "down": (64, 32)}
assert read_shard_index(str(tmp_path / "shards")) == index
layer_one = load_file(layer_paths(str(tmp_path / "shards"), index)[1])
assert not any(key.startswith("mlp.experts") for key in layer_one)
assert "mlp.shared_experts.gate_proj.weight" in layer_one
assert set(expert_paths(str(tmp_path / "shards"), index)) == {1, 2}
def test_streamed_per_expert_forward_matches_resident_eager_bit_exactly(tmp_path):
import torch
model, runtime, ref, ids, _ = _paired(tmp_path)
with torch.no_grad():
got = model(input_ids=ids, labels=ids)
want = ref(input_ids=ids, labels=ids)
assert torch.equal(got.logits, want.logits), (got.logits - want.logits).abs().max().item()
assert torch.equal(got.loss, want.loss)
assert runtime.stats()["expert_fetches"] > 0
assert runtime.expert_readers
assert all(reader._handle is not None for reader in runtime.expert_readers)
runtime.close()
# Closing the runtime releases every experts file (Windows keeps an open file locked).
assert all(reader._handle is None for reader in runtime.expert_readers)
def test_streamed_per_expert_backward_matches_resident_lora_gradients(tmp_path):
import torch
from soup_cli.utils.layer_stream_runtime import canonical_named_parameters
model, runtime, ref, ids, _ = _paired(tmp_path)
loss = model(input_ids=ids, labels=ids).loss
forward_reads = runtime.stats()["expert_fetches"]
loss.backward()
# Under the checkpointed decoder layer a routed expert is read three times:
# forward, the checkpoint's recompute, and the expert's own backward.
assert runtime.stats()["expert_fetches"] == 3 * forward_reads > 0
ref(input_ids=ids, labels=ids).loss.backward()
def grads(net):
return {
name: parameter.grad
for name, parameter in canonical_named_parameters(net)
if "lora_" in name and parameter.grad is not None
}
streamed, resident = grads(model), grads(ref)
assert streamed and set(streamed) == set(resident)
# Layer 0's adapters sit below both MoE layers: their gradient crossed
# every streamed expert's backward.
assert any(".layers.0." in name and float(g.abs().sum()) > 0 for name, g in streamed.items())
for name, gradient in streamed.items():
assert torch.equal(gradient, resident[name]), name
runtime.close()
def test_layout_change_is_a_cache_miss_naming_the_layout(tmp_path):
from soup_cli.utils.layer_shard import inspect_shard_cache
weights, _ = _tiny_deepseek_v3(tmp_path)
index = _shard(tmp_path, weights, expert_layout="per_expert")
cached, reason = inspect_shard_cache(
str(tmp_path / "shards"), "float32", index.source_fingerprint, index.source_files,
"none", False, "", "", stripe_roots=(), expert_layout="",
)
assert cached is None
assert "expert layout changed" in reason
reused = _shard(tmp_path, weights, expert_layout="per_expert")
assert reused == index
def test_per_expert_layout_is_refused_for_an_unadmitted_arch(tmp_path):
from soup_cli.utils.layer_shard import shard_checkpoint
with pytest.raises(ValueError, match="admitted: deepseek_v3"):
shard_checkpoint(
str(tmp_path), str(tmp_path / "s"), dtype="float32", arch="qwen4_exp",
expert_layout="per_expert",
)
def test_unknown_expert_layout_is_refused(tmp_path):
from soup_cli.utils.layer_shard import shard_checkpoint
with pytest.raises(ValueError, match="unsupported expert_layout"):
shard_checkpoint(
str(tmp_path), str(tmp_path / "s"), dtype="float32", arch="deepseek_v3",
expert_layout="experts",
)
def test_install_refuses_a_model_whose_experts_disagree_with_the_file(tmp_path):
from soup_cli.utils.expert_stream import ExpertFetcher, install_expert_streaming
module, n_experts, _ = _deepseek_v3_experts(5)
_, reader = _streamed_copy(tmp_path, module, n_experts)
reader.count = 4 # what a stale index would claim
holder = _holder(module)
with pytest.raises(ValueError, match="reshard the checkpoint"):
install_expert_streaming(
[holder], {0: reader}, ExpertFetcher(device="cpu", dtype="float32")
)
reader.close()
def _holder(experts):
import torch.nn as nn
class Layer(nn.Module):
def __init__(self):
super().__init__()
self.mlp = nn.Module()
self.mlp.experts = experts
return Layer()
# --------------------------------------------------------------------------
# NF4 experts (CUDA)
# --------------------------------------------------------------------------
@CUDA
def test_sharder_writes_nf4_per_expert_files_a_streamed_model_runs_on(tmp_path):
"""The NF4 path of shard_checkpoint(expert_layout="per_expert") end to end."""
pytest.importorskip("bitsandbytes")
import torch
from soup_cli.utils.layer_shard import shard_checkpoint
from soup_cli.utils.layer_stream_runtime import (
build_meta_skeleton,
build_streamed_model,
quantised_layer_suffixes,
)
weights, _ = _tiny_deepseek_v3(tmp_path)
probe = build_meta_skeleton(weights, dtype="bfloat16", quant="nf4")
suffixes = quantised_layer_suffixes(probe)
del probe
index = shard_checkpoint(
weights, str(tmp_path / "shards"), dtype="bfloat16", arch="deepseek_v3", quant="nf4",
quant_suffixes=suffixes, quant_device="cuda", num_hidden_layers=3,
expert_layout="per_expert",
)
assert index.expert_quant_specs["gate_up"].nested is True
model, runtime = build_streamed_model(
model_id=weights, shard_dir=str(tmp_path / "shards"), index=index,
lora_config=_lora(), device="cuda", dtype="bfloat16", buffers=2, pin=False,
quant="nf4", double_quant=True,
)
ids = torch.randint(0, 128, (1, 12), generator=torch.Generator().manual_seed(1)).cuda()
loss = model(input_ids=ids, labels=ids).loss
loss.backward()
assert torch.isfinite(loss)
assert runtime.stats()["expert_fetches"] > 0
runtime.close()
@CUDA
def test_nf4_streamed_experts_equal_the_dequantised_resident_block(tmp_path):
pytest.importorskip("bitsandbytes")
import torch
from bitsandbytes.functional import dequantize_4bit, quantize_4bit
from soup_cli.utils.expert_stream import (
ExpertFetcher,
ExpertShardReader,
streamed_experts_class,
write_expert_shard,
)
from soup_cli.utils.layer_shard import NF4_CODE_KEY, NF4_NESTED_CODE_KEY
module, n_experts, top_k = _deepseek_v3_experts(5)
module = module.to("cuda", torch.bfloat16)
path = str(tmp_path / "experts_001.safetensors")
pairs = [
(module.gate_up_proj[e].detach(), module.down_proj[e].detach())
for e in range(n_experts)
]
info = write_expert_shard(path, iter(pairs), quantise=True, double_quant=True, device="cuda")
with torch.no_grad():
for e, (gate_up, down) in enumerate(pairs):
for target, tensor in ((module.gate_up_proj, gate_up), (module.down_proj, down)):
packed, state = quantize_4bit(
tensor, blocksize=64, compress_statistics=True, quant_type="nf4"
)
target[e].copy_(dequantize_4bit(packed, quant_state=state))
codes = {NF4_CODE_KEY: info.codes[0].cuda(), NF4_NESTED_CODE_KEY: info.codes[1].cuda()}
reader = ExpertShardReader(
path, count=info.count, shapes=info.shapes, quant_specs=info.quant_specs, dtype="bfloat16"
)
streamed = streamed_experts_class()(
reader, ExpertFetcher(device="cuda", dtype="bfloat16", codes=codes), module.act_fn, 1
)
states, index, weights, _ = _routing(24, n_experts, top_k, 32, 9)
states = states.detach().to("cuda", torch.bfloat16)
index = index.cuda()
weights = weights.detach().to("cuda", torch.bfloat16)
with torch.no_grad():
assert torch.equal(streamed(states, index, weights), module(states, index, weights))
reader.close()
# --------------------------------------------------------------------------
# the synthetic store writer and the one-step harness (benchmarks/harness)
# --------------------------------------------------------------------------
def _harness(name):
import importlib.util
import sys
from pathlib import Path
harness = Path(__file__).resolve().parent.parent / "benchmarks" / "harness"
if str(harness) not in sys.path:
sys.path.insert(0, str(harness))
spec = importlib.util.spec_from_file_location(name, harness / f"{name}.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_synthetic_tiny_store_is_a_cache_the_runtime_accepts(tmp_path):
"""The writer skips the sharder, so its index and files must be exactly
what the sharder would write: the index reads back, every experts file
opens with the index's shapes and NF4 specs, and every linear has its
sidecars."""
from safetensors import safe_open
from soup_cli.utils.expert_stream import ExpertShardReader
from soup_cli.utils.layer_shard import expert_paths, layer_paths, read_shard_index
store = str(tmp_path / "store")
summary = _harness("synth_moe_store").write_store(store, "tiny", log=lambda _line: None)
assert summary["label"] == "random weights, shape only"
index = read_shard_index(store)
assert index.arch == "deepseek_v3" and index.quant == "nf4"
assert index.expert_layers == (1, 2) and index.expert_count == 8
for layer, path in expert_paths(store, index).items():
reader = ExpertShardReader(
path, count=index.expert_count, shapes=index.expert_shapes,
quant_specs=index.expert_quant_specs, dtype=index.dtype,
)
reader.close()
with safe_open(layer_paths(store, index)[1], framework="pt") as handle:
keys = set(handle.keys())
assert "self_attn.q_a_proj.weight::absmax" in keys
assert "self_attn.q_a_proj.weight::nested_offset" in keys
assert index.double_quant is True
assert index.expert_quant_specs["down"].nested is True
assert "mlp.shared_experts.down_proj.weight" in keys
assert "mlp.gate.e_score_correction_bias" in keys
def test_synthetic_writer_refuses_an_existing_cache(tmp_path, capsys):
store = tmp_path / "store"
store.mkdir()
(store / "index.json").write_text("{}", encoding="utf-8")
assert _harness("synth_moe_store").main(["--out", str(store), "--shape", "tiny"]) == 2
assert "already holds a shard cache" in capsys.readouterr().out
@CUDA
def test_one_step_on_the_synthetic_tiny_store(tmp_path):
import json
pytest.importorskip("bitsandbytes")
store = str(tmp_path / "store")
_harness("synth_moe_store").write_store(store, "tiny", log=lambda _line: None)
out = tmp_path / "step.json"
assert _harness("moe_stream_step").main(
["--store", store, "--out", str(out), "--seq", "16", "--steps", "2", "--no-pin"]
) == 0
report = json.loads(out.read_text(encoding="utf-8"))
assert report["label"] == "random weights, shape only"
assert len(report["steps"]) == 2
for step in report["steps"]:
assert step["loss_finite"] and step["grads_finite"]
assert step["expert_fetches"] > 0