Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

64.1 KB
Raw Blame
"""Layer streaming shards a DeepSeek-V3 FP8 block-quantised checkpoint by dequantising it.

DeepSeek-V3 (and Kimi K2) are published with each decoder linear as ``float8_e4m3fn``
plus a float32 ``<name>.weight_scale_inv`` per 128 x 128 block, and ``config.json`` names
the format. The sharder used to refuse any FP8 tensor; it now dequantises each one at
shard time (``w = w_fp8.float() * scale_inv`` per block, cropped at a ragged edge), casts
to the stream dtype and stores it by the ordinary rules. Every other FP8 spelling stays
refused, naming what is supported.

Gate record: ``benchmarks/gate-fp8-shard.md``. G1 (dequant exactness) and G2 (streamed vs
resident parity, byte-identical caches) are the tests below.
"""

from __future__ import annotations

import json
import os
import re
import shutil
import types

import pytest

#: DeepSeek-V3's own ``quantization_config`` (Kimi K2's is the same).
DEEPSEEK_FP8 = {
    "activation_scheme": "dynamic",
    "fmt": "e4m3",
    "quant_method": "fp8",
    "weight_block_size": [128, 128],
}
_SUPPORTED_TAIL = "DeepSeek-V3's block FP8"
_LORA_TARGETS = ["q_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"]


def _harness():
    import importlib.util
    import sys
    from pathlib import Path

    harness = Path(__file__).resolve().parent.parent / "benchmarks" / "harness"
    if str(harness) not in sys.path:
        sys.path.insert(0, str(harness))
    spec = importlib.util.spec_from_file_location(
        "fp8_block_quantize", harness / "fp8_block_quantize.py"
    )
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def _reference_dequant(weight_fp8, scale_inv, block):
    """Written here, independently of Soup and the harness: one block at a time."""
    import torch

    rows, cols = weight_fp8.shape
    block_rows, block_cols = block
    out = torch.full((rows, cols), float("nan"), dtype=torch.float32)
    for i in range((rows + block_rows - 1) // block_rows):
        for j in range((cols + block_cols - 1) // block_cols):
            rs = slice(i * block_rows, min((i + 1) * block_rows, rows))
            cs = slice(j * block_cols, min((j + 1) * block_cols, cols))
            out[rs, cs] = weight_fp8[rs, cs].to(torch.float32) * scale_inv[i, j]
    assert not torch.isnan(out).any()  # every element was written exactly once
    return out


def _blockwise_weight(shape, block, seed, *, zero_block=True):
    """bf16 normals with a different magnitude per block (and one all-zero block)."""
    import torch

    generator = torch.Generator().manual_seed(seed)
    rows, cols = shape
    weight = torch.randn(rows, cols, generator=generator)
    grid_rows = (rows + block[0] - 1) // block[0]
    grid_cols = (cols + block[1] - 1) // block[1]
    magnitudes = torch.exp(torch.randn(grid_rows, grid_cols, generator=generator) * 2.0)
    expanded = magnitudes.repeat_interleave(block[0], 0)[:rows]
    weight = weight * expanded.repeat_interleave(block[1], 1)[:, :cols]
    if zero_block and grid_rows * grid_cols > 1:
        weight[: block[0], : block[1]] = 0.0
    return weight.to(torch.bfloat16)


# ==========================================================================
# G1 — dequantisation exactness
# ==========================================================================
G1_CASES = [
    ((576, 2048), (128, 128)),
    ((129, 257), (128, 128)),
    ((127, 1), (128, 128)),
    ((1, 1), (128, 128)),
    ((256, 384), (128, 128)),
    ((2048, 1408), (128, 128)),
    ((24, 64), (16, 32)),
    ((100, 70), (16, 32)),
    ((64, 96), (16, 32)),
]
G1_IDS = [f"{s[0]}x{s[1]}-b{b[0]}x{b[1]}" for s, b in G1_CASES]
_TILEABLE = [
    (shape, block) for shape, block in G1_CASES
    if shape[0] % block[0] == 0 and shape[1] % block[1] == 0
]


@pytest.mark.parametrize(("shape", "block"), G1_CASES, ids=G1_IDS)
def test_dequant_equals_the_reference_block_loop(shape, block):
    import torch

    from soup_cli.utils.fp8_block import dequantize_fp8_block

    weight, scale = _harness().quantize_fp8_block(_blockwise_weight(shape, block, 7), block)
    assert weight.dtype == torch.float8_e4m3fn
    assert tuple(scale.shape) == (-(-shape[0] // block[0]), -(-shape[1] // block[1]))
    got = dequantize_fp8_block(weight, scale, block)
    want = _reference_dequant(weight, scale, block)
    assert got.dtype == torch.float32
    assert torch.equal(got, want), (got - want).abs().max().item()
    for dtype in (torch.bfloat16, torch.float16):
        assert torch.equal(got.to(dtype), want.to(dtype))


def test_every_finite_e4m3_code_point_dequantises_exactly():
    """+-448, every subnormal and both zeros, under a different scale per 8x8 block."""
    import torch

    from soup_cli.utils.fp8_block import dequantize_fp8_block

    codes = torch.arange(256, dtype=torch.int32).to(torch.uint8)
    values = codes.view(torch.float8_e4m3fn)
    finite = values[torch.isfinite(values.to(torch.float32))]
    assert finite.numel() == 254  # 0x7F and 0xFF are the e4m3fn NaNs
    weight = torch.cat((finite, finite[:2])).reshape(16, 16)
    scale = torch.rand(2, 2, generator=torch.Generator().manual_seed(3)) * 5.0 + 1e-3
    got = dequantize_fp8_block(weight, scale, (8, 8))
    assert torch.equal(got, _reference_dequant(weight, scale, (8, 8)))
    assert float(got.abs().max()) > 0.0


@pytest.mark.parametrize(
    ("shape", "block"), _TILEABLE, ids=[f"{s[0]}x{s[1]}" for s, _ in _TILEABLE]
)
def test_transformers_dequant_agrees_on_tileable_shapes(shape, block):
    """The second oracle: transformers' own CPU-callable FP8 block dequant."""
    import torch

    finegrained = pytest.importorskip("transformers.integrations.finegrained_fp8")
    from soup_cli.utils.fp8_block import dequantize_fp8_block

    weight, scale = _harness().quantize_fp8_block(_blockwise_weight(shape, block, 11), block)
    theirs = finegrained.Fp8Dequantize(None)._dequantize_one(
        weight, scale, output_dtype=torch.float32
    )
    assert torch.equal(dequantize_fp8_block(weight, scale, block), theirs)


@pytest.mark.parametrize(
    ("shape", "block"), _TILEABLE, ids=[f"{s[0]}x{s[1]}" for s, _ in _TILEABLE]
)
def test_the_harness_quantiser_writes_what_transformers_writes(shape, block):
    """The converter used by G2 and G3 produces transformers' own FP8 bytes and scales."""
    import torch

    transformers = pytest.importorskip("transformers")
    finegrained = pytest.importorskip("transformers.integrations.finegrained_fp8")

    source = _blockwise_weight(shape, block, 13)
    config = transformers.FineGrainedFP8Config(weight_block_size=block)
    quantizer = types.SimpleNamespace(quantization_config=config)
    theirs = finegrained.Fp8Quantize(quantizer).convert({"x.weight": source})
    weight, scale = _harness().quantize_fp8_block(source, block)
    assert torch.equal(weight.view(torch.uint8), theirs["x.weight"].view(torch.uint8))
    assert torch.equal(scale, theirs["x.weight_scale_inv"])


def test_dequant_refuses_a_non_finite_scale():
    import torch

    from soup_cli.utils.fp8_block import UnsupportedFP8Error, dequantize_fp8_block

    weight = torch.zeros(4, 4).to(torch.float8_e4m3fn)
    for bad in (float("inf"), float("nan")):
        scale = torch.ones(1, 1)
        scale[0, 0] = bad
        with pytest.raises(UnsupportedFP8Error, match="non-finite"):
            dequantize_fp8_block(weight, scale, (4, 4), name="w")


@pytest.mark.parametrize(
    ("weight_dtype", "scale_shape", "scale_dtype", "needle"),
    [
        ("float8_e5m2", (1, 1), "float32", "not a 2-D float8_e4m3fn"),
        ("bfloat16", (1, 1), "float32", "not a 2-D float8_e4m3fn"),
        ("float8_e4m3fn", (2, 1), "float32", "a float32 [1, 1] is needed"),
        ("float8_e4m3fn", (1, 1), "bfloat16", "a float32 [1, 1] is needed"),
    ],
    ids=["e5m2", "bf16", "grid", "scale-dtype"],
)
def test_dequant_refuses_a_tensor_outside_the_format(
    weight_dtype, scale_shape, scale_dtype, needle
):
    import torch

    from soup_cli.utils.fp8_block import UnsupportedFP8Error, dequantize_fp8_block

    weight = torch.zeros(4, 4).to(getattr(torch, weight_dtype))
    scale = torch.ones(scale_shape, dtype=getattr(torch, scale_dtype))
    with pytest.raises(UnsupportedFP8Error, match=re.escape(needle)):
        dequantize_fp8_block(weight, scale, (4, 4))


# ==========================================================================
# the config: the published spellings are accepted, every other is refused by name
# ==========================================================================
def test_the_published_config_spellings_are_accepted():
    transformers = pytest.importorskip("transformers")
    from soup_cli.utils.fp8_block import fp8_block_size

    assert fp8_block_size(DEEPSEEK_FP8, arch="deepseek_v3") == (128, 128)
    # transformers' own to_dict: an enum quant_method, a tuple block, scale_fmt "float"
    exported = transformers.FineGrainedFP8Config().to_dict()
    assert fp8_block_size(exported, arch="deepseek_v3") == (128, 128)
    assert fp8_block_size(transformers.FineGrainedFP8Config(), arch="deepseek_v3") == (128, 128)
    minimal = {"quant_method": "fp8", "weight_block_size": [64, 32]}
    assert fp8_block_size(minimal, arch="deepseek_v3") == (64, 32)


@pytest.mark.parametrize(
    "config", [None, {}, {"quant_method": "bitsandbytes", "load_in_4bit": True}, "fp8"],
    ids=["none", "empty", "bnb", "not-a-mapping"],
)
def test_a_config_without_fp8_is_not_an_fp8_source(config):
    from soup_cli.utils.fp8_block import fp8_block_size

    assert fp8_block_size(config, arch="llama") is None


_BAD_CONFIGS = {
    "per-tensor-missing": ({"quant_method": "fp8"}, "per-tensor FP8 scales"),
    "per-tensor-null": (
        {"quant_method": "fp8", "weight_block_size": None},
        "per-tensor FP8 scales",
    ),
    "block-one-int": ({**DEEPSEEK_FP8, "weight_block_size": [128]}, "two positive ints"),
    "block-zero": ({**DEEPSEEK_FP8, "weight_block_size": [0, 128]}, "two positive ints"),
    "block-bool": ({**DEEPSEEK_FP8, "weight_block_size": [True, 128]}, "two positive ints"),
    "block-float": ({**DEEPSEEK_FP8, "weight_block_size": [128.0, 128]}, "two positive ints"),
    "block-string": ({**DEEPSEEK_FP8, "weight_block_size": "128"}, "two positive ints"),
    "e5m2": ({**DEEPSEEK_FP8, "fmt": "e5m2"}, "e5m2 is not"),
    "ue8m0": ({**DEEPSEEK_FP8, "scale_fmt": "ue8m0"}, "ue8m0 power-of-two scales"),
    "static": ({**DEEPSEEK_FP8, "activation_scheme": "static"}, "static activation scales"),
}


@pytest.mark.parametrize("spelling", sorted(_BAD_CONFIGS))
def test_an_unsupported_fp8_config_is_refused_naming_the_format(spelling):
    from soup_cli.utils.fp8_block import UnsupportedFP8Error, fp8_block_size

    config, needle = _BAD_CONFIGS[spelling]
    with pytest.raises(UnsupportedFP8Error) as caught:
        fp8_block_size(config, arch="deepseek_v3")
    assert needle in str(caught.value)
    assert _SUPPORTED_TAIL in str(caught.value)


@pytest.mark.parametrize(
    ("block", "accepted"),
    [
        ([2**20, 2**20], True),
        ([2**20 + 1, 128], False),
        ([128, 2**20 + 1], False),
        ([2**63, 128], False),
        ([2**63, 2**63], False),
        ([2**80, 2**80], False),
    ],
    ids=["at-cap", "rows-over", "cols-over", "rows-2^63", "both-2^63", "both-2^80"],
)
def test_weight_block_size_is_refused_past_its_range(block, accepted):
    from soup_cli.utils.fp8_block import UnsupportedFP8Error, fp8_block_size

    config = {**DEEPSEEK_FP8, "weight_block_size": block}
    if accepted:
        assert fp8_block_size(config, arch="deepseek_v3") == tuple(block)
        return
    with pytest.raises(UnsupportedFP8Error) as caught:
        fp8_block_size(config, arch="deepseek_v3")
    assert "two positive ints of at most 1048576" in str(caught.value)
    assert _SUPPORTED_TAIL in str(caught.value)


def test_an_out_of_range_block_is_refused_alike_by_plan_estimator_sharder_and_setup(
    tmp_path, monkeypatch
):
    """One check: every entry point refuses it in the same words, before a header is read."""
    from soup_cli.utils.fp8_block import UnsupportedFP8Error, fp8_block_size, plan_fp8_block
    from soup_cli.utils.layer_shard import estimate_shard_cache_bytes, shard_checkpoint
    from tests.test_issue602_qwen4_ple_streaming import _drive_qwen4_streaming_setup

    config = {**DEEPSEEK_FP8, "weight_block_size": [2**63, 2**63]}
    with pytest.raises(UnsupportedFP8Error) as direct:
        fp8_block_size(config, arch="deepseek_v3")
    with pytest.raises(UnsupportedFP8Error) as planned:
        plan_fp8_block(config, _header_tensors(), arch="deepseek_v3")
    with pytest.raises(UnsupportedFP8Error) as estimated:
        estimate_shard_cache_bytes(
            [str(tmp_path / "gone.safetensors")], dtype="bfloat16", arch="deepseek_v3",
            quantization_config=config,
        )
    weights = _corrupt_checkpoint_with_config(tmp_path / "w", config)
    with pytest.raises(UnsupportedFP8Error) as sharded:
        shard_checkpoint(weights, str(tmp_path / "s"), dtype="bfloat16", arch="deepseek_v3")

    def _must_not_resolve(*_args, **_kwargs):
        raise AssertionError("weights resolved for a base that cannot stream")

    with pytest.raises(UnsupportedFP8Error) as setup:
        _drive_qwen4_streaming_setup(
            tmp_path, monkeypatch, resolve_weights=_must_not_resolve, arch="deepseek_v3",
            model_cfg=_fp8_setup_cfg(config),
        )
    messages = {str(caught.value) for caught in (direct, planned, estimated, sharded, setup)}
    assert len(messages) == 1


@pytest.mark.parametrize("arch", ["llama", "qwen3", "qwen4_exp", ""])
def test_an_fp8_base_on_another_architecture_is_refused(arch):
    from soup_cli.utils.fp8_block import UnsupportedFP8Error, fp8_block_size

    with pytest.raises(UnsupportedFP8Error, match="admitted: deepseek_v3"):
        fp8_block_size(DEEPSEEK_FP8, arch=arch)


# ==========================================================================
# the tensors: every FP8 tensor needs its scale, every scale its FP8 tensor
# ==========================================================================
def _header_tensors(**overrides):
    """key -> (safetensors dtype, shape) for one FP8 block layer (block 128 x 128)."""
    tensors = {
        "model.embed_tokens.weight": ("BF16", (8, 300)),
        "model.layers.0.input_layernorm.weight": ("BF16", (300,)),
        "model.layers.0.self_attn.o_proj.weight": ("F8_E4M3", (300, 300)),
        "model.layers.0.self_attn.o_proj.weight_scale_inv": ("F32", (3, 3)),
        "model.layers.0.mlp.gate.weight": ("BF16", (4, 300)),
        "model.layers.0.mlp.experts.0.down_proj.weight": ("F8_E4M3", (300, 129)),
        "model.layers.0.mlp.experts.0.down_proj.weight_scale_inv": ("F32", (3, 2)),
    }
    for key, value in overrides.items():
        if value is None:
            tensors.pop(key)
        else:
            tensors[key] = value
    return tensors


_O = "model.layers.0.self_attn.o_proj.weight"
_BAD_TENSORS = {
    "fp8-without-scale": ({_O + "_scale_inv": None}, "has no block scale"),
    "scale-without-tensor": ({_O: None}, "has no tensor"),
    "scale-beside-bf16": ({_O: ("BF16", (300, 300))}, "not an FP8 tensor"),
    "misshapen-scale": ({_O + "_scale_inv": ("F32", (3, 2))}, "needs [3, 3]"),
    "transposed-grid": (
        {"model.layers.0.mlp.experts.0.down_proj.weight_scale_inv": ("F32", (2, 3))},
        "needs [3, 2]",
    ),
    "per-tensor-scale": ({_O + "_scale_inv": ("F32", ())}, "a per-tensor scale is not"),
    "per-tensor-scale-1": ({_O + "_scale_inv": ("F32", (1,))}, "a per-tensor scale is not"),
    "bf16-scale": ({_O + "_scale_inv": ("BF16", (3, 3))}, "is 'BF16', not F32"),
    "fp8-scale": ({_O + "_scale_inv": ("F8_E4M3", (3, 3))}, "is 'F8_E4M3', not F32"),
    "e5m2-weight": ({_O: ("F8_E5M2", (300, 300))}, "is 'F8_E5M2', not F8_E4M3"),
    "fnuz-weight": ({_O: ("F8_E4M3FNUZ", (300, 300))}, "not F8_E4M3"),
    "fp8-vector": (
        {_O: ("F8_E4M3", (300,)), _O + "_scale_inv": ("F32", (3,))},
        "covers 2-D weights only",
    ),
    "fp8-3d": (
        {_O: ("F8_E4M3", (2, 300, 300)), _O + "_scale_inv": ("F32", (2, 3, 3))},
        "covers 2-D weights only",
    ),
}


@pytest.mark.parametrize("spelling", sorted(_BAD_TENSORS))
def test_an_fp8_tensor_layout_outside_the_format_is_refused_by_name(spelling):
    from soup_cli.utils.fp8_block import UnsupportedFP8Error, plan_fp8_block

    overrides, needle = _BAD_TENSORS[spelling]
    with pytest.raises(UnsupportedFP8Error) as caught:
        plan_fp8_block(DEEPSEEK_FP8, _header_tensors(**overrides), arch="deepseek_v3")
    assert needle in str(caught.value)
    assert _SUPPORTED_TAIL in str(caught.value)


@pytest.mark.parametrize(
    ("config", "needle"),
    [
        (None, "declares no FP8 quantization_config"),
        ({"quant_method": "compressed-tensors", "format": "float-quantized"},
         "'compressed-tensors', whose scales are not 'weight_scale_inv' block scales"),
        ({"quant_method": "bitsandbytes"}, "declares quant_method 'bitsandbytes'"),
    ],
    ids=["no-config", "compressed-tensors", "other-method"],
)
def test_fp8_tensors_without_the_fp8_block_config_are_refused(config, needle):
    from soup_cli.utils.fp8_block import UnsupportedFP8Error, plan_fp8_block

    with pytest.raises(UnsupportedFP8Error) as caught:
        plan_fp8_block(config, _header_tensors(), arch="deepseek_v3")
    assert needle in str(caught.value)


def test_an_enum_quant_method_is_named_by_its_value():
    """transformers' config objects carry ``quant_method`` as an enum member."""
    transformers = pytest.importorskip("transformers")
    from soup_cli.utils.fp8_block import UnsupportedFP8Error, plan_fp8_block

    bnb = transformers.BitsAndBytesConfig(load_in_4bit=True)
    with pytest.raises(UnsupportedFP8Error) as caught:
        plan_fp8_block(bnb, _header_tensors(), arch="deepseek_v3")
    assert "declares quant_method 'bitsandbytes'" in str(caught.value)


def test_a_compressed_tensors_checkpoint_with_weight_scale_is_refused():
    """compressed-tensors FP8 spells its scale ``weight_scale``, not ``weight_scale_inv``."""
    from soup_cli.utils.fp8_block import UnsupportedFP8Error, plan_fp8_block

    tensors = _header_tensors(**{_O + "_scale_inv": None})
    tensors[_O.removesuffix(".weight") + ".weight_scale"] = ("F32", (300, 1))
    with pytest.raises(UnsupportedFP8Error, match="whose scales are not 'weight_scale_inv'"):
        plan_fp8_block({"quant_method": "compressed-tensors"}, tensors, arch="deepseek_v3")


def test_a_complete_fp8_block_layout_is_planned():
    from soup_cli.utils.fp8_block import plan_fp8_block

    plan = plan_fp8_block(DEEPSEEK_FP8, _header_tensors(), arch="deepseek_v3")
    assert plan.block == (128, 128)
    assert plan.weights == {_O, "model.layers.0.mlp.experts.0.down_proj.weight"}
    assert plan.scales == {key + "_scale_inv" for key in plan.weights}


def test_no_fp8_tensor_means_no_plan_even_under_an_fp8_config():
    from soup_cli.utils.fp8_block import plan_fp8_block

    plain = {"model.layers.0.self_attn.o_proj.weight": ("BF16", (4, 4))}
    assert plan_fp8_block(DEEPSEEK_FP8, plain, arch="deepseek_v3") is None
    assert plan_fp8_block(None, plain, arch="llama") is None


def test_a_refusal_never_quotes_raw_control_bytes():
    """Header keys are untrusted and the message reaches a terminal."""
    from soup_cli.utils.fp8_block import UnsupportedFP8Error, plan_fp8_block

    tensors = {"model.layers.0.\x1b[31mx.weight": ("F8_E4M3", (4, 4))}
    with pytest.raises(UnsupportedFP8Error) as caught:
        plan_fp8_block(DEEPSEEK_FP8, tensors, arch="deepseek_v3")
    assert "\x1b" not in str(caught.value)


# ==========================================================================
# the sharder on synthetic checkpoints
# ==========================================================================
HIDDEN, INTER, VOCAB, EXPERTS, LAYERS = 6, 5, 8, 2, 2
BLOCK = (4, 4)  # every matrix below is ragged against it


def _synthetic_state(seed=1, *, fp8_embeddings=False):
    """A DeepSeek-V3-shaped bf16 state, the linears as the harness converts them."""
    import torch

    generator = torch.Generator().manual_seed(seed)

    def normal(*shape):
        return torch.randn(*shape, generator=generator).to(torch.bfloat16)

    state = {
        "model.embed_tokens.weight": normal(VOCAB, HIDDEN),
        "model.norm.weight": normal(HIDDEN),
        "lm_head.weight": normal(VOCAB, HIDDEN),
    }
    linears = []
    for layer in range(LAYERS):
        prefix = f"model.layers.{layer}."
        state[prefix + "input_layernorm.weight"] = normal(HIDDEN)
        linears.append(prefix + "self_attn.o_proj.weight")
        state[linears[-1]] = normal(HIDDEN, HIDDEN)
        state[prefix + "mlp.gate.weight"] = normal(EXPERTS, HIDDEN)
        state[prefix + "mlp.gate.e_score_correction_bias"] = torch.randn(
            EXPERTS, generator=generator
        )
        for expert in range(EXPERTS):
            for name, shape in (
                ("gate_proj", (INTER, HIDDEN)),
                ("up_proj", (INTER, HIDDEN)),
                ("down_proj", (HIDDEN, INTER)),
            ):
                linears.append(prefix + f"mlp.experts.{expert}.{name}.weight")
                state[linears[-1]] = normal(*shape)
    if fp8_embeddings:  # transformers' modules_to_convert: FP8 embedding tables
        linears += ["model.embed_tokens.weight", "lm_head.weight"]
    return state, linears


def _to_fp8(state, linears, block=BLOCK):
    quantize = _harness().quantize_fp8_block
    out = dict(state)
    for key in linears:
        out[key], out[key + "_scale_inv"] = quantize(state[key], block)
    return out


def _dequantised(fp8_state, block=BLOCK):
    """The reference checkpoint: every FP8 weight replaced by its fp32 reference dequant."""
    out = {}
    for key, value in fp8_state.items():
        if key.endswith("_scale_inv"):
            continue
        scale = fp8_state.get(key + "_scale_inv")
        out[key] = value if scale is None else _reference_dequant(value, scale, block)
    return out


def _write(directory, state, *, quant=DEEPSEEK_FP8, block=BLOCK, layers=LAYERS):
    from safetensors.torch import save_file

    directory.mkdir(parents=True, exist_ok=True)
    save_file(state, str(directory / "model.safetensors"))
    config = {"model_type": "deepseek_v3", "num_hidden_layers": layers}
    if quant is not None:
        config["quantization_config"] = {**quant, "weight_block_size": list(block)}
    (directory / "config.json").write_text(json.dumps(config), encoding="utf-8")
    return str(directory)


def _synthetic_pair(tmp_path, **kwargs):
    """(FP8 checkpoint dir, reference-dequantised checkpoint dir)."""
    state, linears = _synthetic_state(**kwargs)
    fp8_state = _to_fp8(state, linears)
    fp8 = _write(tmp_path / "fp8", fp8_state)
    ref = _write(tmp_path / "ref", _dequantised(fp8_state), quant=None)
    return fp8, ref


def _cache_files(directory):
    return sorted(
        name for name in os.listdir(directory)
        if name.endswith(".safetensors") and not name.startswith(".soup.")
    )


def _assert_same_cache(left, right, left_index, right_index):
    import torch
    from safetensors.torch import load_file

    assert _cache_files(left) == _cache_files(right)
    for name in _cache_files(left):
        mine, theirs = load_file(os.path.join(left, name)), load_file(os.path.join(right, name))
        assert sorted(mine) == sorted(theirs), name
        assert not any(key.endswith("_scale_inv") for key in mine), name
        for key in mine:
            assert mine[key].dtype == theirs[key].dtype, (name, key)
            assert torch.equal(mine[key], theirs[key]), (name, key)
    ignored = {"source_fingerprint", "source_files"}
    for field_name in left_index.__dataclass_fields__:
        if field_name not in ignored:
            assert getattr(left_index, field_name) == getattr(right_index, field_name), field_name


def _shard(weights, out, **kwargs):
    from soup_cli.utils.layer_shard import shard_checkpoint

    kwargs.setdefault("dtype", "float32")
    kwargs.setdefault("arch", "deepseek_v3")
    kwargs.setdefault("num_hidden_layers", LAYERS)
    return shard_checkpoint(weights, str(out), **kwargs)


@pytest.mark.parametrize("layout", ["", "per_expert"], ids=["fused", "per_expert"])
@pytest.mark.parametrize("dtype", ["float32", "bfloat16"])
def test_the_fp8_source_shards_to_the_cache_of_its_dequantised_weights(tmp_path, layout, dtype):
    fp8, ref = _synthetic_pair(tmp_path)
    mine = _shard(fp8, tmp_path / "a", dtype=dtype, expert_layout=layout)
    theirs = _shard(ref, tmp_path / "b", dtype=dtype, expert_layout=layout)
    _assert_same_cache(str(tmp_path / "a"), str(tmp_path / "b"), mine, theirs)
    assert mine.total_params == theirs.total_params


def test_fp8_embeddings_and_head_are_dequantised_into_their_shards(tmp_path):
    """transformers' modules_to_convert can make the embedding tables FP8 too."""
    fp8, ref = _synthetic_pair(tmp_path, fp8_embeddings=True)
    mine = _shard(fp8, tmp_path / "a")
    theirs = _shard(ref, tmp_path / "b")
    assert mine.large_keys == ("lm_head.weight", "model.embed_tokens.weight")
    _assert_same_cache(str(tmp_path / "a"), str(tmp_path / "b"), mine, theirs)


def test_the_router_and_its_bias_keep_their_stored_dtypes(tmp_path):
    import torch
    from safetensors.torch import load_file

    from soup_cli.utils.layer_shard import layer_paths

    fp8, _ref = _synthetic_pair(tmp_path)
    index = _shard(fp8, tmp_path / "s", dtype="bfloat16")
    tensors = load_file(layer_paths(str(tmp_path / "s"), index)[1])
    assert tensors["mlp.gate.e_score_correction_bias"].dtype == torch.float32
    assert tensors["mlp.gate.weight"].dtype == torch.bfloat16
    assert tensors["mlp.experts.gate_up_proj"].dtype == torch.bfloat16
    assert not any(key.endswith("_scale_inv") for key in tensors)


def test_the_plain_read_backstop_still_refuses_an_fp8_tensor(tmp_path):
    """A cast that bypassed the plan would keep the raw e4m3 values: never allowed."""
    from safetensors import safe_open

    from soup_cli.utils.fp8_block import UnsupportedFP8Error
    from soup_cli.utils.layer_shard import _read_tensor

    fp8, _ref = _synthetic_pair(tmp_path)
    with safe_open(os.path.join(fp8, "model.safetensors"), framework="pt") as handle:
        for key in (
            "model.layers.0.self_attn.o_proj.weight",
            "model.layers.0.self_attn.o_proj.weight_scale_inv",
        ):
            with pytest.raises(UnsupportedFP8Error, match="cast without its scales"):
                _read_tensor(handle, key, "bfloat16")


def _mutated_checkpoint(tmp_path, spelling):
    """An FP8 checkpoint in one refused spelling (key, dtype, config or architecture)."""
    import torch

    state, linears = _synthetic_state()
    fp8_state = _to_fp8(state, linears)
    o_proj = "model.layers.0.self_attn.o_proj.weight"
    quant, arch = DEEPSEEK_FP8, "deepseek_v3"
    if spelling == "fp8-without-scale":
        del fp8_state[o_proj + "_scale_inv"]
    elif spelling == "scale-without-tensor":
        fp8_state[o_proj] = state[o_proj]
    elif spelling == "misshapen-scale":
        fp8_state[o_proj + "_scale_inv"] = torch.ones(1, 2)
    elif spelling == "e5m2-weight":
        fp8_state[o_proj] = fp8_state[o_proj].to(torch.float32).to(torch.float8_e5m2)
    elif spelling == "per-tensor-config":
        quant = {"quant_method": "fp8", "weight_block_size": None}
    elif spelling == "compressed-tensors":
        # compressed-tensors' own FP8 spelling: a bare quant_method no longer says FP8,
        # and is refused as a quantised source of unknown format.
        quant = {"quant_method": "compressed-tensors", "format": "float-quantized"}
    elif spelling == "no-config":
        quant = None
    elif spelling == "llama":
        arch = "llama"
    else:  # pragma: no cover — a typo in the parametrize list
        raise AssertionError(spelling)
    if quant is not None and quant.get("weight_block_size", 1) is None:
        weights = _write(tmp_path / "w", fp8_state, quant=None)
        config = json.loads((tmp_path / "w" / "config.json").read_text(encoding="utf-8"))
        config["quantization_config"] = quant
        (tmp_path / "w" / "config.json").write_text(json.dumps(config), encoding="utf-8")
    else:
        weights = _write(tmp_path / "w", fp8_state, quant=quant)
    return weights, arch


_SHARDER_SPELLINGS = [
    "fp8-without-scale", "scale-without-tensor", "misshapen-scale", "e5m2-weight",
    "per-tensor-config", "compressed-tensors", "no-config", "llama",
]


@pytest.mark.parametrize("spelling", _SHARDER_SPELLINGS)
def test_the_sharder_refuses_before_anything_is_written(tmp_path, spelling):
    from soup_cli.utils.fp8_block import UnsupportedFP8Error
    from soup_cli.utils.layer_shard import shard_checkpoint

    weights, arch = _mutated_checkpoint(tmp_path, spelling)
    out = tmp_path / "s"
    with pytest.raises(UnsupportedFP8Error, match="Dequantise"):
        shard_checkpoint(weights, str(out), dtype="bfloat16", arch=arch, num_hidden_layers=2)
    assert not out.exists() or not os.listdir(out)


@pytest.mark.parametrize("spelling", _SHARDER_SPELLINGS)
def test_the_disk_preflight_refuses_in_the_sharders_words(tmp_path, spelling):
    from soup_cli.utils.fp8_block import read_checkpoint_quantization_config
    from soup_cli.utils.layer_shard import (
        _discover_safetensors,
        estimate_shard_cache_bytes,
        shard_checkpoint,
    )

    weights, arch = _mutated_checkpoint(tmp_path, spelling)
    with pytest.raises(ValueError) as sharder:
        shard_checkpoint(
            weights, str(tmp_path / "s"), dtype="bfloat16", arch=arch, num_hidden_layers=2
        )
    with pytest.raises(ValueError) as estimator:
        estimate_shard_cache_bytes(
            _discover_safetensors(weights), dtype="bfloat16", arch=arch, num_hidden_layers=2,
            quantization_config=read_checkpoint_quantization_config(weights),
        )
    assert str(estimator.value) == str(sharder.value)


def _corrupt_checkpoint_with_config(directory, quantization_config):
    """A config that names a refused FP8 spelling beside weights no reader could open."""
    directory.mkdir(parents=True)
    (directory / "model.safetensors").write_bytes(b"\xff" * 64)
    config = {"model_type": "deepseek_v3", "quantization_config": quantization_config}
    (directory / "config.json").write_text(json.dumps(config), encoding="utf-8")
    return str(directory)


def test_the_sharder_refuses_an_fp8_config_spelling_before_reading_a_header(tmp_path):
    from soup_cli.utils.fp8_block import UnsupportedFP8Error
    from soup_cli.utils.layer_shard import shard_checkpoint

    weights = _corrupt_checkpoint_with_config(tmp_path / "w", {**DEEPSEEK_FP8, "fmt": "e5m2"})
    with pytest.raises(UnsupportedFP8Error, match="e5m2 is not"):
        shard_checkpoint(weights, str(tmp_path / "s"), dtype="bfloat16", arch="deepseek_v3")


def test_the_estimate_refuses_an_fp8_config_spelling_before_reading_a_header(tmp_path):
    from soup_cli.utils.fp8_block import UnsupportedFP8Error
    from soup_cli.utils.layer_shard import estimate_shard_cache_bytes

    with pytest.raises(UnsupportedFP8Error, match="per-tensor FP8 scales"):
        estimate_shard_cache_bytes(
            [str(tmp_path / "gone.safetensors")], dtype="bfloat16", arch="deepseek_v3",
            quantization_config={"quant_method": "fp8"},
        )


# ==========================================================================
# config.json, read as untrusted input
# ==========================================================================
@pytest.mark.parametrize(
    ("content", "expected"),
    [
        (None, None),
        (b"", None),
        (b"not json", None),
        (b"[1, 2]", None),
        # Returned as found since the review round of the packed-quant refusal: the
        # gate refuses a declaration that is not a mapping as malformed.
        (b'{"quantization_config": "fp8"}', "fp8"),
        (b'{"model_type": "deepseek_v3"}', None),
        (b'{"quantization_config": {"quant_method": "fp8"}}', {"quant_method": "fp8"}),
    ],
    ids=["missing", "empty", "not-json", "not-object", "not-mapping", "absent", "fp8"],
)
def test_the_checkpoint_config_is_read_bounded(tmp_path, content, expected):
    from soup_cli.utils.fp8_block import read_checkpoint_quantization_config

    if content is not None:
        (tmp_path / "config.json").write_bytes(content)
    assert read_checkpoint_quantization_config(str(tmp_path)) == expected


def test_a_config_too_deep_to_parse_is_not_read(tmp_path, monkeypatch):
    """``json`` raises RecursionError past the interpreter's nesting limit, and where that
    limit falls differs between builds (5,000 levels raise on Windows CPython 3.12.10 and
    parse on Linux 3.12.15), so the error is raised here directly rather than by depth."""
    import soup_cli.utils.fp8_block as fp8_block

    (tmp_path / "config.json").write_bytes(b'{"quantization_config": {"quant_method": "fp8"}}')

    def _too_deep(_handle):
        raise RecursionError("maximum recursion depth exceeded while decoding a JSON array")

    monkeypatch.setattr(fp8_block, "json", types.SimpleNamespace(load=_too_deep))
    assert fp8_block.read_checkpoint_quantization_config(str(tmp_path)) is None


def test_an_oversized_config_is_not_read(tmp_path, monkeypatch):
    import soup_cli.utils.fp8_block as fp8_block

    payload = b'{"quantization_config": {"quant_method": "fp8"}}'
    (tmp_path / "config.json").write_bytes(payload)
    monkeypatch.setattr(fp8_block, "_CONFIG_LIMIT", len(payload) - 1)
    assert fp8_block.read_checkpoint_quantization_config(str(tmp_path)) is None
    monkeypatch.setattr(fp8_block, "_CONFIG_LIMIT", len(payload))
    assert fp8_block.read_checkpoint_quantization_config(str(tmp_path)) == {
        "quant_method": "fp8"
    }


@pytest.mark.requires_symlink
def test_a_symlinked_config_is_not_read(tmp_path):
    from soup_cli.utils.fp8_block import read_checkpoint_quantization_config

    real = tmp_path / "elsewhere.json"
    real.write_text(json.dumps({"quantization_config": DEEPSEEK_FP8}), encoding="utf-8")
    os.symlink(str(real), str(tmp_path / "config.json"))
    assert read_checkpoint_quantization_config(str(tmp_path)) is None


# ==========================================================================
# the disk pre-flight: the estimate is the cache the sharder writes
# ==========================================================================
def _estimate_for(weights, **kwargs):
    from soup_cli.utils.fp8_block import read_checkpoint_quantization_config
    from soup_cli.utils.layer_shard import _discover_safetensors, estimate_shard_cache_bytes

    kwargs.setdefault("dtype", "float32")
    kwargs.setdefault("arch", "deepseek_v3")
    kwargs.setdefault("num_hidden_layers", LAYERS)
    return estimate_shard_cache_bytes(
        _discover_safetensors(weights),
        quantization_config=read_checkpoint_quantization_config(weights),
        **kwargs,
    )


def _payload(path):
    import struct

    with open(path, "rb") as handle:
        (length,) = struct.unpack("<Q", handle.read(8))
    return os.path.getsize(path) - 8 - length


@pytest.mark.parametrize("layout", ["", "per_expert"], ids=["fused", "per_expert"])
@pytest.mark.parametrize("dtype", ["float32", "bfloat16"])
def test_the_estimate_is_exact_for_an_fp8_source(tmp_path, layout, dtype):
    from soup_cli.utils.expert_stream import expert_shard_name
    from soup_cli.utils.layer_shard import layer_paths, shard_cache_disk_bytes

    fp8, ref = _synthetic_pair(tmp_path)
    estimate = _estimate_for(fp8, dtype=dtype, expert_layout=layout)
    # the scales are charged nothing: the same figure as the dequantised checkpoint
    assert estimate == _estimate_for(ref, dtype=dtype, expert_layout=layout)
    out = tmp_path / "s"
    index = _shard(fp8, out, dtype=dtype, expert_layout=layout)
    for idx, path in enumerate(layer_paths(str(out), index)):
        experts = out / expert_shard_name(idx)
        payload = _payload(path) + (_payload(str(experts)) if experts.exists() else 0)
        assert estimate.layer_bytes[idx] == payload, idx
    other = sum(
        _payload(str(out / name)) for name in os.listdir(out)
        if name == "extras.safetensors" or name.startswith("large_")
    )
    assert estimate.other_bytes == other
    assert estimate.total_bytes >= shard_cache_disk_bytes(str(out), index)


def test_the_estimate_without_the_config_refuses_an_fp8_source(tmp_path):
    from soup_cli.utils.fp8_block import UnsupportedFP8Error
    from soup_cli.utils.layer_shard import _discover_safetensors, estimate_shard_cache_bytes

    fp8, _ref = _synthetic_pair(tmp_path)
    with pytest.raises(UnsupportedFP8Error, match="declares no FP8 quantization_config"):
        estimate_shard_cache_bytes(
            _discover_safetensors(fp8), dtype="bfloat16", arch="deepseek_v3",
            num_hidden_layers=LAYERS,
        )


def test_the_header_reader_reads_fp8_dtypes_at_one_byte(tmp_path):
    import torch
    from safetensors.torch import save_file

    from soup_cli.utils.safetensors_reader import read_header

    path = str(tmp_path / "f.safetensors")
    save_file(
        {
            "a": torch.zeros(3, 5).to(torch.float8_e4m3fn),
            "b": torch.zeros(7).to(torch.float8_e5m2),
        },
        path,
    )
    header = read_header(path)
    assert (header["a"].dtype, header["a"].nbytes) == ("float8_e4m3fn", 15)
    assert (header["b"].dtype, header["b"].nbytes) == ("float8_e5m2", 7)


def test_the_preflight_estimate_is_not_approximate_for_an_fp8_source(tmp_path):
    from soup_cli.trainer.stream_setup import _preflight_cache_estimate
    from soup_cli.utils.fp8_block import read_checkpoint_quantization_config

    fp8, _ref = _synthetic_pair(tmp_path)
    plan = types.SimpleNamespace(weights_dir=fp8, needs_materialization=False)
    got = _preflight_cache_estimate(
        plan, dtype="float32", arch="deepseek_v3", quant="none", quant_suffixes=(),
        double_quant=True, expert_layout="", num_hidden_layers=LAYERS,
        quantization_config=read_checkpoint_quantization_config(fp8),
    )
    assert got == _estimate_for(fp8)


# ==========================================================================
# the cache identity: a bf16 cache is never reused for the FP8 source, nor the reverse
# ==========================================================================
def _replace_checkpoint(target, source):
    """Put ``source``'s files at ``target`` keeping the old files' mtimes: the size and the
    config must invalidate the cache on their own."""
    for name in ("model.safetensors", "config.json"):
        old = os.stat(os.path.join(target, name)).st_mtime_ns
        shutil.copyfile(os.path.join(source, name), os.path.join(target, name))
        os.utime(os.path.join(target, name), ns=(old, old))


@pytest.mark.parametrize("direction", ["bf16-then-fp8", "fp8-then-bf16"])
def test_a_cache_of_one_format_is_never_reused_for_the_other(tmp_path, direction):
    import torch
    from safetensors.torch import load_file

    from soup_cli.utils.layer_shard import layer_paths

    state, linears = _synthetic_state()
    fp8_state = _to_fp8(state, linears)
    plain = _write(tmp_path / "plain", state, quant=None)
    fp8 = _write(tmp_path / "fp8", fp8_state)
    first, second = (plain, fp8) if direction == "bf16-then-fp8" else (fp8, plain)
    weights = tmp_path / "w"
    shutil.copytree(first, weights)
    cache = tmp_path / "cache"
    _shard(str(weights), cache)
    _replace_checkpoint(str(weights), second)
    notes = []
    index = _shard(str(weights), cache, notify=notes.append)
    assert any("Re-sharding layer cache" in note for note in notes), notes
    want_state = _dequantised(fp8_state) if second == fp8 else state
    layer = load_file(layer_paths(str(cache), index)[0])
    assert torch.equal(
        layer["self_attn.o_proj.weight"],
        want_state["model.layers.0.self_attn.o_proj.weight"].to(torch.float32),
    )


def test_every_fp8_admitted_architecture_keys_its_cache_on_config_json():
    from soup_cli.utils.fp8_block import FP8_BLOCK_ARCHS
    from soup_cli.utils.layer_shard import source_identity_includes_config

    assert FP8_BLOCK_ARCHS
    assert all(source_identity_includes_config(arch) for arch in FP8_BLOCK_ARCHS)


# ==========================================================================
# the setup: refusals from the config alone, and the pre-flight is fed the config
# ==========================================================================
def _fp8_setup_cfg(quantization_config):
    return types.SimpleNamespace(
        model_type="deepseek_v3", hidden_size=HIDDEN, num_hidden_layers=LAYERS,
        vocab_size=VOCAB, intermediate_size=INTER, quantization_config=quantization_config,
    )


def test_the_setup_refuses_an_unsupported_fp8_base_before_resolving_weights(
    tmp_path, monkeypatch
):
    from tests.test_issue602_qwen4_ple_streaming import _drive_qwen4_streaming_setup

    def _must_not_resolve(*_args, **_kwargs):
        raise AssertionError("weights resolved for a base that cannot stream")

    with pytest.raises(ValueError, match="per-tensor FP8 scales"):
        _drive_qwen4_streaming_setup(
            tmp_path, monkeypatch, resolve_weights=_must_not_resolve, arch="deepseek_v3",
            model_cfg=_fp8_setup_cfg({"quant_method": "fp8"}),
        )


def test_the_setup_refuses_an_fp8_base_on_another_architecture(tmp_path, monkeypatch):
    from tests.test_issue602_qwen4_ple_streaming import _drive_qwen4_streaming_setup

    def _must_not_resolve(*_args, **_kwargs):
        raise AssertionError("weights resolved for a base that cannot stream")

    cfg = _fp8_setup_cfg(DEEPSEEK_FP8)
    cfg.model_type = "llama"
    with pytest.raises(ValueError, match="admitted: deepseek_v3"):
        _drive_qwen4_streaming_setup(
            tmp_path, monkeypatch, resolve_weights=_must_not_resolve, arch="llama",
            model_cfg=cfg,
        )


def test_the_setup_hands_the_fp8_config_to_the_disk_preflight(tmp_path, monkeypatch):
    import soup_cli.trainer.stream_setup as stream_setup
    from tests.test_issue602_qwen4_ple_streaming import _drive_qwen4_streaming_setup

    fp8, _ref = _synthetic_pair(tmp_path)
    size = os.path.getsize(os.path.join(fp8, "model.safetensors"))
    plan = types.SimpleNamespace(
        weights_dir=fp8, source_dir=fp8, source_bytes=size, materialized_copy_bytes=0,
        materialize_bytes=0, needs_materialization=False,
        source_files=(("model.safetensors", size, 0),), source_revision=None,
    )

    def _resolve(_base, *, before_materialize=None):
        before_materialize(plan)
        return fp8

    panels = []
    real = stream_setup._render_stream_disk_preflight
    monkeypatch.setattr(
        stream_setup, "_render_stream_disk_preflight",
        lambda **kwargs: (panels.append(kwargs), real(**kwargs))[1],
    )
    monkeypatch.setattr(stream_setup, "_disk_volume", lambda _path: (1, 10**15))
    with open(os.path.join(fp8, "config.json"), encoding="utf-8") as handle:
        quantization_config = json.load(handle)["quantization_config"]
    _drive_qwen4_streaming_setup(
        tmp_path, monkeypatch, resolve_weights=_resolve, arch="deepseek_v3",
        model_cfg=_fp8_setup_cfg(quantization_config),
    )
    from soup_cli.utils.layer_stream import resolve_stream_dtype

    expected = _estimate_for(fp8, dtype=resolve_stream_dtype("cpu"))
    assert len(panels) == 1
    assert panels[0]["approximate"] is False
    assert panels[0]["shard_bytes"] == expected.total_bytes


# ==========================================================================
# G2 — a tiny DeepseekV3ForCausalLM: FP8-sourced stream vs the dequantised resident model
# ==========================================================================
def _tiny_config(q_lora_rank):
    from transformers import DeepseekV3Config

    return DeepseekV3Config(
        vocab_size=128, hidden_size=64, intermediate_size=128, moe_intermediate_size=32,
        num_hidden_layers=3, first_k_dense_replace=1, num_attention_heads=4,
        num_key_value_heads=4, n_shared_experts=1, n_routed_experts=8, n_group=2,
        topk_group=1, num_experts_per_tok=2, kv_lora_rank=16, q_lora_rank=q_lora_rank,
        qk_nope_head_dim=16, qk_rope_head_dim=8, v_head_dim=16, max_position_embeddings=128,
    )


def _build_tiny(root, q_lora_rank, block):
    """(bf16 original, FP8 block copy, reference-dequantised copy) of one tiny model."""
    import torch
    from safetensors.torch import load_file, save_file
    from transformers import DeepseekV3ForCausalLM

    torch.manual_seed(11)
    model = DeepseekV3ForCausalLM(_tiny_config(q_lora_rank)).to(torch.float32).eval()
    generator = torch.Generator().manual_seed(23)
    with torch.no_grad():
        for layer in model.model.layers[1:]:
            bias = layer.mlp.gate.e_score_correction_bias
            bias.copy_(torch.randn(bias.shape, generator=generator) * 2.0)
        for parameter in model.parameters():
            parameter.data = parameter.data.to(torch.bfloat16)
    original = root / "bf16"
    model.save_pretrained(str(original))
    fp8 = root / "fp8"
    _harness().convert_checkpoint(str(original), str(fp8), block=block, log=lambda _l: None)
    fp8_state = load_file(str(fp8 / "model.safetensors"))
    assert any(value.dtype == torch.float8_e4m3fn for value in fp8_state.values())
    ref = root / "ref"
    ref.mkdir()
    save_file(_dequantised(fp8_state, block), str(ref / "model.safetensors"))
    shutil.copyfile(original / "config.json", ref / "config.json")
    return str(original), str(fp8), str(ref)


@pytest.fixture(scope="module")
def tiny_models(tmp_path_factory):
    pytest.importorskip("transformers")
    built = {}

    def get(q_lora_rank, block):
        key = (q_lora_rank, tuple(block))
        if key not in built:
            root = tmp_path_factory.mktemp(f"tiny-q{q_lora_rank}-b{block[0]}x{block[1]}")
            built[key] = _build_tiny(root, q_lora_rank, tuple(block))
        return built[key]

    return get


def _tiny_lora():
    from peft import LoraConfig, TaskType

    return LoraConfig(
        r=4, lora_alpha=8, lora_dropout=0.0, bias="none",
        target_modules=_LORA_TARGETS, task_type=TaskType.CAUSAL_LM,
    )


def _resident(weights, layout):
    import torch
    from transformers import DeepseekV3ForCausalLM

    model = DeepseekV3ForCausalLM.from_pretrained(weights, dtype=torch.float32).eval()
    if layout == "per_expert":
        # The streamed experts follow the eager loop; the resident control must too.
        model.set_experts_implementation("eager")
    return model


def _streamed_vs_resident(tmp_path, fp8, ref, layout):
    import torch
    from peft import get_peft_model

    from soup_cli.utils.layer_stream_runtime import build_streamed_model

    shards = str(tmp_path / "shards")
    index = _shard(fp8, shards, num_hidden_layers=3, expert_layout=layout)
    model, runtime = build_streamed_model(
        model_id=fp8, shard_dir=shards, index=index, lora_config=_tiny_lora(),
        device="cpu", dtype="float32", buffers=2, pin=False, seed=3,
    )
    resident = get_peft_model(_resident(ref, layout), _tiny_lora())
    generator = torch.Generator().manual_seed(17)
    with torch.no_grad():
        for name, parameter in model.named_parameters():
            if "lora_B" in name:
                parameter.copy_(torch.randn(parameter.shape, generator=generator) * 0.02)
    src = {k.replace(".inner.", "."): v for k, v in model.state_dict().items() if "lora_" in k}
    dst = {k: v for k, v in resident.state_dict().items() if "lora_" in k}
    assert src and set(src) == set(dst)
    for key, value in src.items():
        dst[key].copy_(value)
    model.eval()
    resident.eval()
    ids = torch.randint(0, 128, (2, 12), generator=generator)
    return model, runtime, resident, ids


G2_CELLS = [
    ("", 24, (16, 32)),
    ("", None, (16, 32)),
    ("per_expert", 24, (16, 32)),
    ("per_expert", None, (16, 32)),
    ("", 24, (128, 128)),
]


@pytest.mark.parametrize(
    ("layout", "q_lora_rank", "block"), G2_CELLS,
    ids=[f"{lay or 'fused'}-q{q}-b{b[0]}x{b[1]}" for lay, q, b in G2_CELLS],
)
def test_streamed_from_fp8_matches_the_dequantised_resident_bit_exactly(
    tmp_path, tiny_models, layout, q_lora_rank, block
):
    import torch

    from soup_cli.utils.layer_stream_runtime import canonical_named_parameters

    _original, fp8, ref = tiny_models(q_lora_rank, block)
    model, runtime, resident, ids = _streamed_vs_resident(tmp_path, fp8, ref, layout)
    try:
        with torch.no_grad():
            got = model(input_ids=ids, labels=ids)
            want = resident(input_ids=ids, labels=ids)
        assert torch.equal(got.logits, want.logits), (got.logits - want.logits).abs().max()
        assert torch.equal(got.loss, want.loss)

        model.zero_grad(set_to_none=True)
        resident.zero_grad(set_to_none=True)
        model(input_ids=ids, labels=ids).loss.backward()
        resident(input_ids=ids, labels=ids).loss.backward()

        def grads(net):
            return {
                name: parameter.grad
                for name, parameter in canonical_named_parameters(net)
                if "lora_" in name and parameter.grad is not None
            }

        streamed, resident_grads = grads(model), grads(resident)
        assert streamed and set(streamed) == set(resident_grads)
        assert any(
            ".layers.0." in name and float(gradient.abs().sum()) > 0
            for name, gradient in streamed.items()
        )
        for name, gradient in streamed.items():
            assert torch.equal(gradient, resident_grads[name]), name
    finally:
        runtime.close()


def test_fp8_is_lossy_so_the_parity_pins_the_dequant(tiny_models):
    """The control: the dequantised model is not the bf16 original."""
    import torch

    original, _fp8, ref = tiny_models(24, (16, 32))
    ids = torch.randint(0, 128, (2, 12), generator=torch.Generator().manual_seed(5))
    with torch.no_grad():
        dequantised = _resident(ref, "").forward(input_ids=ids).logits
        bf16 = _resident(original, "").forward(input_ids=ids).logits
    assert not torch.equal(dequantised, bf16)


def test_the_streamed_model_is_built_bf16_not_with_transformers_fp8_layers(
    tmp_path, tiny_models
):
    import torch.nn as nn

    _original, fp8, ref = tiny_models(24, (16, 32))
    model, runtime, _resident_model, _ids = _streamed_vs_resident(tmp_path, fp8, ref, "")
    try:
        base = model.get_base_model()
        names = {type(module).__name__ for module in base.modules()}
        assert not any("FP8" in name or "Fp8" in name for name in names), names
        attention = base.model.layers[0].inner.self_attn
        assert type(attention.o_proj.base_layer) is nn.Linear
        assert getattr(base.config, "quantization_config", None) is None
    finally:
        runtime.close()


def test_the_meta_skeleton_never_hands_an_fp8_config_to_from_config(tiny_models, monkeypatch):
    """Built bf16 whatever a transformers version would do with the FP8 config."""
    import transformers

    from soup_cli.utils.layer_stream_runtime import build_meta_skeleton

    _original, fp8, _ref = tiny_models(24, (16, 32))
    seen = []
    real = transformers.AutoModelForCausalLM.from_config

    def _spy(config, **kwargs):
        seen.append(getattr(config, "quantization_config", None))
        return real(config, **kwargs)

    monkeypatch.setattr(transformers.AutoModelForCausalLM, "from_config", _spy)
    build_meta_skeleton(fp8, dtype="bfloat16")
    assert seen == [None]


def test_the_resident_view_of_the_checkpoint_is_untouched(tiny_models):
    """Only the skeleton's own config object loses the FP8 config: config.json and a
    fresh resident config still carry it."""
    from transformers import AutoConfig

    from soup_cli.utils.layer_stream_runtime import build_meta_skeleton

    _original, fp8, _ref = tiny_models(24, (16, 32))
    path = os.path.join(fp8, "config.json")
    before = open(path, "rb").read()
    build_meta_skeleton(fp8, dtype="bfloat16")
    assert open(path, "rb").read() == before
    fresh = AutoConfig.from_pretrained(fp8).quantization_config
    assert fresh["quant_method"] == "fp8" and fresh["weight_block_size"] == [16, 32]


@pytest.mark.parametrize("layout", ["", "per_expert"], ids=["fused", "per_expert"])
@pytest.mark.parametrize(
    ("quant", "dtype"),
    [("none", "float32"), ("none", "bfloat16"), ("nf4", "bfloat16")],
    ids=["none-fp32", "none-bf16", "nf4-bf16"],
)
def test_the_fp8_cache_is_the_dequantised_cache_byte_for_byte(
    tmp_path, tiny_models, layout, quant, dtype
):
    """Where NF4 is gated on the CPU: the CUDA NF4 path gets exactly these bytes."""
    _original, fp8, ref = tiny_models(24, (16, 32))
    kwargs = dict(dtype=dtype, num_hidden_layers=3, expert_layout=layout, quant=quant)
    if quant == "nf4":
        pytest.importorskip("bitsandbytes")
        from soup_cli.utils.layer_stream_runtime import (
            build_meta_skeleton,
            quantised_layer_suffixes,
        )

        suffixes = quantised_layer_suffixes(build_meta_skeleton(fp8, dtype=dtype, quant="nf4"))
        assert "self_attn.kv_a_proj_with_mqa.weight" in suffixes
        kwargs.update(quant_suffixes=suffixes, quant_device="cpu", double_quant=True)
    mine = _shard(fp8, tmp_path / "a", **kwargs)
    theirs = _shard(ref, tmp_path / "b", **kwargs)
    _assert_same_cache(str(tmp_path / "a"), str(tmp_path / "b"), mine, theirs)
    assert _estimate_for(fp8, **{k: v for k, v in kwargs.items() if k != "quant_device"}) == (
        _estimate_for(ref, **{k: v for k, v in kwargs.items() if k != "quant_device"})
    )


# ==========================================================================
# bounded work: a hostile block size, and a big tensor, cost one band of fp32
# ==========================================================================
class _TensorSizes:
    """Records the element count and dtype of every tensor a torch call returns."""

    def __init__(self):
        from torch.overrides import TorchFunctionMode

        sizes = self.sizes = []

        class _Mode(TorchFunctionMode):
            def __torch_function__(self, func, types, args=(), kwargs=None):
                import torch

                out = func(*args, **(kwargs or {}))
                if isinstance(out, torch.Tensor):
                    sizes.append((out.numel(), out.dtype))
                return out

        self.mode = _Mode()

    def __enter__(self):
        self.mode.__enter__()
        return self

    def __exit__(self, *exc):
        self.mode.__exit__(*exc)

    def largest(self, dtype):
        return max((numel for numel, kind in self.sizes if kind == dtype), default=0)


@pytest.mark.parametrize("block", [(2**24, 2**24), (2**24, 4), (4, 2**24)],
                         ids=["both", "rows", "cols"])
def test_a_huge_block_size_stays_cheap_and_exact(block):
    """A ``weight_block_size`` far past the matrix must not expand the scale to its size:
    a 4 x 4 weight with a [1, 1] scale used to allocate a 4 x 2**24 fp32 grid."""
    import torch

    from soup_cli.utils.fp8_block import dequantize_fp8_block

    generator = torch.Generator().manual_seed(2)
    weight = (torch.randn(4, 4, generator=generator) * 30).to(torch.float8_e4m3fn)
    scale = torch.tensor([[0.375]])
    with _TensorSizes() as seen:
        got = dequantize_fp8_block(weight, scale, block, dtype=torch.bfloat16)
    assert torch.equal(got, (weight.to(torch.float32) * 0.375).to(torch.bfloat16))
    assert max(numel for numel, _kind in seen.sizes) <= 16


def test_an_fp8_source_with_a_huge_block_size_shards_exactly(tmp_path):
    import torch
    from safetensors.torch import load_file

    from soup_cli.utils.layer_shard import layer_paths

    state, linears = _synthetic_state()
    huge = (2**20, 2**20)  # the largest weight_block_size accepted
    fp8_state = _to_fp8(state, linears, block=huge)
    assert all(tuple(fp8_state[key + "_scale_inv"].shape) == (1, 1) for key in linears)
    fp8 = _write(tmp_path / "fp8", fp8_state, block=huge)
    index = _shard(fp8, tmp_path / "s")
    tensors = load_file(layer_paths(str(tmp_path / "s"), index)[0])
    key = "model.layers.0.self_attn.o_proj.weight"
    want = fp8_state[key].to(torch.float32) * fp8_state[key + "_scale_inv"][0, 0]
    assert torch.equal(tensors["self_attn.o_proj.weight"], want)


def test_dequant_works_in_bands_into_the_stream_dtype(monkeypatch):
    """The fp32 product is formed one band of rows at a time and cast into a preallocated
    output: the peak is the output plus one band, not 9-11 bytes per element."""
    import torch

    import soup_cli.utils.fp8_block as fp8_block

    block = (128, 128)
    weight, scale = _harness().quantize_fp8_block(_blockwise_weight((300, 200), block, 9), block)
    monkeypatch.setattr(fp8_block, "_DEQUANT_BAND_ELEMENTS", 1000)
    with _TensorSizes() as seen:
        got = fp8_block.dequantize_fp8_block(weight, scale, block, dtype=torch.bfloat16)
    want = _reference_dequant(weight, scale, block).to(torch.bfloat16)
    assert got.dtype == torch.bfloat16
    assert torch.equal(got, want)
    # one band is 1000 // 200 = 5 rows; its scale grid before the crop is 5 x 256
    assert seen.largest(torch.float32) <= 5 * 256
    assert [numel for numel, kind in seen.sizes if kind == torch.bfloat16 and numel == 60000]


@pytest.mark.parametrize("dtype", ["float32", "bfloat16", "float16"])
def test_the_banded_result_is_the_unbanded_cast(dtype):
    import torch

    from soup_cli.utils.fp8_block import dequantize_fp8_block

    block = (128, 128)
    weight, scale = _harness().quantize_fp8_block(
        _blockwise_weight((2048, 1408), block, 21), block
    )
    target = getattr(torch, dtype)
    want = _reference_dequant(weight, scale, block).to(target)
    assert torch.equal(dequantize_fp8_block(weight, scale, block, dtype=target), want)
    # bands that do not line up with the 128-row blocks (100,000 // 1408 = 71 rows)
    import soup_cli.utils.fp8_block as fp8_block

    original = fp8_block._DEQUANT_BAND_ELEMENTS
    fp8_block._DEQUANT_BAND_ELEMENTS = 100_000
    try:
        assert torch.equal(dequantize_fp8_block(weight, scale, block, dtype=target), want)
    finally:
        fp8_block._DEQUANT_BAND_ELEMENTS = original


# ==========================================================================
# values the format cannot hold are refused, naming the tensor
# ==========================================================================
@pytest.mark.parametrize("code", [0x7F, 0xFF], ids=["nan", "neg-nan"])
@pytest.mark.parametrize("dtype", ["float32", "bfloat16"])
def test_a_nan_fp8_byte_is_refused_after_dequant(code, dtype):
    import torch

    from soup_cli.utils.fp8_block import UnsupportedFP8Error, dequantize_fp8_block

    raw = torch.zeros(4, 4, dtype=torch.uint8)
    raw[2, 1] = code
    weight = raw.view(torch.float8_e4m3fn)
    with pytest.raises(UnsupportedFP8Error) as caught:
        dequantize_fp8_block(
            weight, torch.ones(1, 1), (128, 128), name="layers.0.q", dtype=getattr(torch, dtype)
        )
    text = str(caught.value)
    assert "'layers.0.q'" in text and "NaN" in text


def test_a_float16_overflow_is_refused_suggesting_bfloat16():
    import torch

    from soup_cli.utils.fp8_block import UnsupportedFP8Error, dequantize_fp8_block

    weight = torch.full((2, 2), 448.0).to(torch.float8_e4m3fn)
    scale = torch.full((1, 1), 200.0)  # 448 x 200 = 89,600 > 65,504
    with pytest.raises(UnsupportedFP8Error) as caught:
        dequantize_fp8_block(weight, scale, (128, 128), name="w", dtype=torch.float16)
    assert "overflows float16" in str(caught.value) and "bfloat16" in str(caught.value)
    for dtype in (torch.bfloat16, torch.float32):
        out = dequantize_fp8_block(weight, scale, (128, 128), name="w", dtype=dtype)
        assert torch.equal(out, torch.full((2, 2), 89600.0).to(dtype))


def test_a_negative_scale_is_refused():
    import torch

    from soup_cli.utils.fp8_block import UnsupportedFP8Error, dequantize_fp8_block

    weight = torch.ones(4, 4).to(torch.float8_e4m3fn)
    scale = torch.tensor([[0.5, -0.25]])
    with pytest.raises(UnsupportedFP8Error, match="negative"):
        dequantize_fp8_block(weight, scale, (4, 2), name="w")


def test_a_zero_scale_on_an_all_zero_block_is_kept():
    import torch

    from soup_cli.utils.fp8_block import dequantize_fp8_block

    weight = torch.zeros(4, 4).to(torch.float8_e4m3fn)
    weight[:, 2:] = torch.ones(4, 2).to(torch.float8_e4m3fn)
    scale = torch.tensor([[0.0, 0.5], [-0.0, 0.5]])
    got = dequantize_fp8_block(weight, scale, (2, 2), name="w")
    assert torch.equal(got, _reference_dequant(weight, scale, (2, 2)))


def test_the_sharder_refuses_a_nan_weight_naming_it_before_the_index(tmp_path):
    import torch

    from soup_cli.utils.fp8_block import UnsupportedFP8Error

    state, linears = _synthetic_state()
    fp8_state = _to_fp8(state, linears)
    key = "model.layers.1.mlp.experts.0.down_proj.weight"
    raw = fp8_state[key].view(torch.uint8).clone()
    raw[0, 0] = 0x7F
    fp8_state[key] = raw.view(torch.float8_e4m3fn)
    fp8 = _write(tmp_path / "fp8", fp8_state)
    with pytest.raises(UnsupportedFP8Error) as caught:
        _shard(fp8, tmp_path / "s", dtype="bfloat16")
    assert key in str(caught.value)
    assert not (tmp_path / "s" / "index.json").exists()


# ==========================================================================
# a mixed checkpoint: a bf16 linear beside FP8 siblings is stored as it is
# ==========================================================================
def test_a_bf16_linear_beside_fp8_siblings_is_cast_as_stored(tmp_path):
    """Real DeepSeek-V3 checkpoints mix the two: only the FP8 tensors carry a scale."""
    import torch
    from safetensors.torch import load_file

    from soup_cli.utils.layer_shard import layer_paths

    state, linears = _synthetic_state()
    kept = "model.layers.1.self_attn.o_proj.weight"
    fp8_state = _to_fp8(state, [key for key in linears if key != kept])
    assert fp8_state[kept].dtype == torch.bfloat16 and kept + "_scale_inv" not in fp8_state
    fp8 = _write(tmp_path / "fp8", fp8_state)
    ref = _write(tmp_path / "ref", _dequantised(fp8_state), quant=None)
    mine = _shard(fp8, tmp_path / "a", dtype="bfloat16")
    theirs = _shard(ref, tmp_path / "b", dtype="bfloat16")
    _assert_same_cache(str(tmp_path / "a"), str(tmp_path / "b"), mine, theirs)
    layer = load_file(layer_paths(str(tmp_path / "a"), mine)[1])
    assert torch.equal(layer["self_attn.o_proj.weight"], state[kept])


# ==========================================================================
# the backstop and the skeleton, directly
# ==========================================================================
@pytest.mark.parametrize(
    ("key", "dtype"),
    [
        ("model.layers.0.q_proj.weight", "F8_E4M3"),
        ("model.layers.0.q_proj.weight", "F8_E5M2"),
        ("model.layers.0.q_proj.weight", "F8_E4M3FNUZ"),
        ("model.layers.0.q_proj.weight_scale_inv", "F32"),
    ],
    ids=["e4m3", "e5m2", "e4m3fnuz", "stray-scale"],
)
def test_the_plain_read_backstop_refuses_every_fp8_spelling(key, dtype):
    from soup_cli.utils.fp8_block import UnsupportedFP8Error
    from soup_cli.utils.layer_shard import _refuse_fp8

    with pytest.raises(UnsupportedFP8Error, match="cast without its scales"):
        _refuse_fp8(key, dtype)


# U8 used to pass here too: the backstop now refuses every integer or packed type
# (tests/test_stream_refuses_packed_quant.py), so a plain tensor is a floating one.
@pytest.mark.parametrize("dtype", ["BF16", "F16", "F32", "F64"])
def test_the_plain_read_backstop_passes_a_plain_tensor(dtype):
    from soup_cli.utils.layer_shard import _refuse_fp8

    _refuse_fp8("model.layers.0.q_proj.weight", dtype)


def test_the_skeleton_keeps_a_quantization_config_that_is_not_fp8(tiny_models, monkeypatch):
    """Only an FP8 config is dropped. Any other one reaches ``from_config`` and the built
    model exactly as before this change (pinned against the pre-change code: the config's
    dict is handed through untouched)."""
    import transformers

    from soup_cli.utils.layer_stream_runtime import build_meta_skeleton

    original, _fp8, _ref = tiny_models(24, (16, 32))
    weights = os.path.join(os.path.dirname(original), "bnb-config")
    if not os.path.isdir(weights):
        shutil.copytree(original, weights)
        config_path = os.path.join(weights, "config.json")
        with open(config_path, encoding="utf-8") as handle:
            config = json.load(handle)
        config["quantization_config"] = {"quant_method": "bitsandbytes", "load_in_4bit": True}
        with open(config_path, "w", encoding="utf-8") as handle:
            json.dump(config, handle)
    seen = []
    real = transformers.AutoModelForCausalLM.from_config

    def _spy(config, **kwargs):
        seen.append(getattr(config, "quantization_config", None))
        return real(config, **kwargs)

    monkeypatch.setattr(transformers.AutoModelForCausalLM, "from_config", _spy)
    model = build_meta_skeleton(weights, dtype="bfloat16")
    expected = {"quant_method": "bitsandbytes", "load_in_4bit": True}
    assert seen == [expected]
    assert model.config.quantization_config == expected