Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

36.6 KB
Raw Blame
"""R4 — setup: the roots reach the sharder, the depth reaches every consumer."""

import ast
import inspect
import io
import json
import os
import re
import struct
from types import SimpleNamespace

import pytest
from rich.console import Console

from soup_cli.trainer import stream_setup
from soup_cli.utils import layer_shard
from soup_cli.utils.async_disk_source import DEFAULT_STREAM_READ_AHEAD, MAX_STREAM_READ_AHEAD


def _tcfg(value):
    return SimpleNamespace(stream_read_ahead=value)


def _console():
    buffer = io.StringIO()
    return Console(file=buffer, width=200, color_system=None), buffer


def _depth(value, n_roots):
    console, out = _console()
    decision = stream_setup._effective_read_ahead(_tcfg(value), n_roots, console)
    return decision.depth, out.getvalue()


class TestTheDepth:
    def test_one_root_leaves_the_setting_alone(self):
        assert _depth(2, 1)[0] == 2
        assert _depth(1, 1)[0] == 1

    def test_the_default_becomes_one_more_than_the_drives(self):
        depth, said = _depth(DEFAULT_STREAM_READ_AHEAD, 2)
        assert depth == 3 and "stream_read_ahead" in said
        assert _depth(DEFAULT_STREAM_READ_AHEAD, 3)[0] == 4

    def test_a_value_already_deep_enough_is_kept(self):
        """Review Focus 5: never bump what the operator already set high enough."""
        assert _depth(5, 2) == (5, "")

    def test_a_shallower_value_is_kept_with_a_warning_naming_the_idle_drives(self):
        depth, said = _depth(1, 2)
        assert depth == 1 and "idle" in said and "3" in said

    def test_the_default_never_exceeds_the_schema_ceiling(self):
        """Review Focus 5: seven roots want 8, the most the setting allows."""
        assert _depth(DEFAULT_STREAM_READ_AHEAD, 7)[0] == 8 == MAX_STREAM_READ_AHEAD

    def test_the_clamp_holds_past_the_most_roots_the_setting_can_serve(self):
        """Seven roots reach the ceiling exactly, so only a bigger count shows the clamp."""
        assert _depth(DEFAULT_STREAM_READ_AHEAD, 9)[0] == MAX_STREAM_READ_AHEAD


class TestTheReadyLineNamesDriveModes:
    @staticmethod
    def _stats(drive_count, direct_io_drives):
        return {
            "tier": "disk",
            "pinned": False,
            "pinned_bytes": 0,
            "disk_bytes": 1_000_000_000,
            "read_ahead": 3,
            "store_bytes": 20_000_000,
            "drive_count": drive_count,
            "direct_io_drives": direct_io_drives,
        }

    def test_two_direct_drives_are_counted(self):
        line = stream_setup._stream_source_line(self._stats(2, 2))
        assert "2 NVMe volumes, direct I/O 2/2" in line

    def test_a_mixed_pair_is_counted(self):
        line = stream_setup._stream_source_line(self._stats(2, 1))
        assert "2 NVMe volumes, direct I/O 1/2" in line

    def test_one_drive_keeps_the_existing_wording(self):
        line = stream_setup._stream_source_line(self._stats(1, 1))
        assert "on an NVMe volume" in line
        assert "direct I/O" not in line


def test_every_depth_consumer_in_the_setup_reads_the_effective_value():
    """The default N+1 is worth nothing if one consumer still reads the raw setting."""
    source = inspect.getsource(stream_setup)
    tree = ast.parse(source)
    raw_reads = [
        node.lineno
        for node in ast.walk(tree)
        if isinstance(node, ast.Attribute)
        and node.attr == "stream_read_ahead"
        and isinstance(node.value, ast.Name)
        and node.value.id == "tcfg"
    ]
    helper = next(
        node for node in ast.walk(tree)
        if isinstance(node, ast.FunctionDef) and node.name == "_effective_read_ahead"
    )
    inside = range(helper.lineno, helper.end_lineno + 1)
    assert [line for line in raw_reads if line not in inside] == []


class TestTheDiskPreflight:
    def test_shares_split_the_estimate_across_roots_and_sum_to_it(self):
        assert stream_setup._stripe_write_shares(10, 1) == [10]
        shares = stream_setup._stripe_write_shares(10, 3)
        assert sum(shares) == 10 and max(shares) - min(shares) <= 1

    def test_a_full_stripe_volume_is_refused_by_name(self, tmp_path, monkeypatch):
        # `console` is the module-level Rich console: keep the panel off the real terminal.
        monkeypatch.setattr(stream_setup, "console", _console()[0])
        full = str(tmp_path / "stripe")
        monkeypatch.setattr(
            stream_setup, "_disk_volume",
            lambda path: (2, 0) if path == full else (1, 10**15),
        )
        with pytest.raises(ValueError, match="layer-shard stripe"):
            stream_setup._render_stream_disk_preflight(
                source_bytes=1, materialized_copy_bytes=0, materialize_bytes=0,
                materialized_path=str(tmp_path), shard_bytes=10, shard_write_bytes=5,
                shard_path=str(tmp_path / "cache"), stripe_writes=((full, 5),),
            )

    def test_a_full_stripe_volume_names_the_stripe_variable(self, tmp_path, monkeypatch):
        """Final review Minor 6: the remedy named only the primary cache variables."""
        monkeypatch.setattr(stream_setup, "console", _console()[0])
        full = str(tmp_path / "stripe")
        monkeypatch.setattr(
            stream_setup, "_disk_volume",
            lambda path: (2, 0) if path == full else (1, 10**15),
        )
        with pytest.raises(ValueError, match="SOUP_LAYER_STREAM_STRIPE_DIRS"):
            stream_setup._render_stream_disk_preflight(
                source_bytes=1, materialized_copy_bytes=0, materialize_bytes=0,
                materialized_path=str(tmp_path), shard_bytes=10, shard_write_bytes=5,
                shard_path=str(tmp_path / "cache"), stripe_writes=((full, 5),),
            )

    def test_the_additional_writes_line_counts_every_stripe(self, tmp_path, monkeypatch):
        """A striped cache writes to several drives; the headline figure must add them up."""
        recorder = io.StringIO()
        monkeypatch.setattr(
            stream_setup, "console", Console(file=recorder, width=200, color_system=None)
        )
        monkeypatch.setattr(stream_setup, "_disk_volume", lambda path: (hash(path), 10**15))
        stream_setup._render_stream_disk_preflight(
            source_bytes=0, materialized_copy_bytes=0, materialize_bytes=0,
            materialized_path=str(tmp_path), shard_bytes=int(3e9), shard_write_bytes=int(1e9),
            shard_path=str(tmp_path / "cache"),
            stripe_writes=((str(tmp_path / "a"), int(1e9)), (str(tmp_path / "b"), int(1e9))),
        )
        assert "Additional writes before training: 3.00 GB" in recorder.getvalue()

    def test_exact_shares_charge_primary_drive_more_than_even_split_two_roots(self, tmp_path):
        """#1612: 5 layers over 2 roots puts 3 layers + non-decoder weights on root 0."""
        weights = _make_synthetic_untied_checkpoint(tmp_path, n_layers=5)
        file_size = os.path.getsize(os.path.join(weights, "model.safetensors"))
        shares, approx = stream_setup.estimate_stripe_write_shares(
            weights, total_bytes=file_size, n_roots=2, dtype="bfloat16"
        )
        assert not approx
        assert sum(shares) == file_size
        even_shares = stream_setup._stripe_write_shares(file_size, 2)
        # Primary drive receives 3 of 5 layers plus embedding/head; must exceed even split
        assert shares[0] > even_shares[0]
        assert shares[1] < even_shares[1]

    def test_exact_shares_three_roots_odd_layer_count(self, tmp_path):
        """#1612: 5 layers over 3 roots gives (2, 2, 1) layer placement."""
        weights = _make_synthetic_untied_checkpoint(tmp_path, n_layers=5)
        file_size = os.path.getsize(os.path.join(weights, "model.safetensors"))
        shares, approx = stream_setup.estimate_stripe_write_shares(
            weights, total_bytes=file_size, n_roots=3, dtype="bfloat16"
        )
        assert not approx
        assert sum(shares) == file_size
        even_shares = stream_setup._stripe_write_shares(file_size, 3)
        assert shares[0] > even_shares[0]
        # Root 0 and 1 each have 2 layers; root 0 also has non-decoder weights
        assert shares[0] > shares[1]
        # Root 2 has only 1 layer
        assert shares[1] > shares[2]

    def test_preflight_refusal_when_primary_volume_short_two_roots(self, tmp_path, monkeypatch):
        """#1612: pre-flight refuses when primary drive fits even split but not real bytes."""
        monkeypatch.setattr(stream_setup, "console", _console()[0])
        weights = _make_synthetic_untied_checkpoint(tmp_path, n_layers=5)
        file_size = os.path.getsize(os.path.join(weights, "model.safetensors"))
        shares, approx = stream_setup.estimate_stripe_write_shares(
            weights, total_bytes=file_size, n_roots=2, dtype="bfloat16"
        )
        assert not approx
        even_shares = stream_setup._stripe_write_shares(file_size, 2)
        assert shares[0] > even_shares[0]

        primary_dir = str(tmp_path / "cache")
        stripe_dir = str(tmp_path / "stripe")
        # Primary volume has room for even share, but NOT for real share
        primary_free = even_shares[0] + 1
        monkeypatch.setattr(
            stream_setup,
            "_disk_volume",
            lambda path: (1, primary_free) if path == primary_dir else (2, 10**15),
        )

        # Real shares: refusal fires and names layer-shard cache
        with pytest.raises(ValueError, match="layer-shard cache"):
            stream_setup._render_stream_disk_preflight(
                source_bytes=file_size, materialized_copy_bytes=0, materialize_bytes=0,
                materialized_path=weights, shard_bytes=file_size, shard_write_bytes=shares[0],
                shard_path=primary_dir, stripe_writes=((stripe_dir, shares[1]),),
            )

        # Mutation check: with even split, pre-flight would have falsely passed
        recorder = io.StringIO()
        monkeypatch.setattr(
            stream_setup, "console", Console(file=recorder, width=200, color_system=None)
        )
        stream_setup._render_stream_disk_preflight(
            source_bytes=file_size, materialized_copy_bytes=0, materialize_bytes=0,
            materialized_path=weights, shard_bytes=file_size, shard_write_bytes=even_shares[0],
            shard_path=primary_dir, stripe_writes=((stripe_dir, even_shares[1]),),
        )
        assert "Layer-shard cache" in recorder.getvalue()

    def test_preflight_refusal_when_primary_volume_short_three_roots(self, tmp_path, monkeypatch):
        """#1612: 3-root case refuses when primary volume is short."""
        monkeypatch.setattr(stream_setup, "console", _console()[0])
        weights = _make_synthetic_untied_checkpoint(tmp_path, n_layers=5)
        file_size = os.path.getsize(os.path.join(weights, "model.safetensors"))
        shares, _approx = stream_setup.estimate_stripe_write_shares(
            weights, total_bytes=file_size, n_roots=3, dtype="bfloat16"
        )
        even_shares = stream_setup._stripe_write_shares(file_size, 3)
        assert shares[0] > even_shares[0]

        primary_dir = str(tmp_path / "cache")
        stripe_a = str(tmp_path / "stripe_a")
        stripe_b = str(tmp_path / "stripe_b")
        primary_free = even_shares[0] + 1
        monkeypatch.setattr(
            stream_setup,
            "_disk_volume",
            lambda path: (1, primary_free) if path == primary_dir else (hash(path), 10**15),
        )

        with pytest.raises(ValueError, match="layer-shard cache"):
            stream_setup._render_stream_disk_preflight(
                source_bytes=file_size, materialized_copy_bytes=0, materialize_bytes=0,
                materialized_path=weights, shard_bytes=file_size, shard_write_bytes=shares[0],
                shard_path=primary_dir,
                stripe_writes=((stripe_a, shares[1]), (stripe_b, shares[2])),
            )

    def test_preflight_panel_line_approximate_when_unreadable(self, tmp_path, monkeypatch):
        """#1612: when headers cannot be read, even split is kept and panel says approximately."""
        recorder = io.StringIO()
        monkeypatch.setattr(
            stream_setup, "console", Console(file=recorder, width=200, color_system=None)
        )
        monkeypatch.setattr(stream_setup, "_disk_volume", lambda path: (hash(path), 10**15))

        shares, approx = stream_setup.estimate_stripe_write_shares(
            str(tmp_path / "nonexistent"), total_bytes=1000, n_roots=2
        )
        assert approx is True
        assert shares == stream_setup._stripe_write_shares(1000, 2)

        stream_setup._render_stream_disk_preflight(
            source_bytes=0, materialized_copy_bytes=0, materialize_bytes=0,
            materialized_path=str(tmp_path), shard_bytes=1000, shard_write_bytes=shares[0],
            shard_path=str(tmp_path / "cache"), stripe_writes=((str(tmp_path / "a"), shares[1]),),
            approximate=approx,
        )
        assert "Layer-shard cache: approximately 0.00 GB (write required)" in recorder.getvalue()

    def test_exact_shares_match_actual_sharded_files_on_disk(self, tmp_path):
        """#1612: compare charged shares with actual sharded bytes on disk."""
        pytest.importorskip("torch")
        pytest.importorskip("safetensors")
        pytest.importorskip("transformers")
        import torch
        from safetensors.torch import save_file
        from transformers import AutoModelForCausalLM, LlamaConfig

        from soup_cli.utils.layer_shard import shard_checkpoint

        weights = tmp_path / "model"
        weights.mkdir(parents=True, exist_ok=True)
        config = LlamaConfig(
            vocab_size=64, hidden_size=64, intermediate_size=64,
            num_hidden_layers=5, num_attention_heads=4,
            num_key_value_heads=2, tie_word_embeddings=False,
            max_position_embeddings=128,
        )
        torch.manual_seed(42)
        model = AutoModelForCausalLM.from_config(config).to(torch.bfloat16).eval()
        save_file(
            {k: v.contiguous() for k, v in model.state_dict().items()},
            str(weights / "model.safetensors"),
        )
        config.save_pretrained(str(weights))

        file_size = os.path.getsize(str(weights / "model.safetensors"))
        shares, approx = stream_setup.estimate_stripe_write_shares(
            str(weights), total_bytes=file_size, n_roots=2, dtype="bfloat16"
        )
        assert not approx

        cache_dir = tmp_path / "cache" / "model"
        stripe_dir = tmp_path / "stripe"
        stripe_dir.mkdir(parents=True, exist_ok=True)
        shard_checkpoint(
            str(weights), str(cache_dir), dtype="bfloat16", stripe_roots=(str(stripe_dir),)
        )

        actual_root0 = sum(f.stat().st_size for f in cache_dir.glob("*.safetensors"))
        subdirs = [d for d in stripe_dir.iterdir() if d.is_dir()]
        stripe_sub = subdirs[0] if subdirs else stripe_dir
        actual_root1 = sum(f.stat().st_size for f in stripe_sub.glob("*.safetensors"))

        assert shares[0] >= actual_root0
        assert shares[1] >= actual_root1
        # Within 10% tolerance
        assert abs(shares[0] - actual_root0) / actual_root0 < 0.10
        assert abs(shares[1] - actual_root1) / actual_root1 < 0.10

        # Mutation check: with even split, root 0's share is below its actual bytes
        even_shares = stream_setup._stripe_write_shares(file_size, 2)
        assert even_shares[0] < actual_root0

    def test_exact_shares_nf4_charges_non_decoder_at_streaming_dtype(self, tmp_path):
        """#1612: NF4 applies to linears, non-decoder charged at streaming dtype."""
        weights = _make_synthetic_untied_checkpoint(tmp_path, n_layers=5)
        file_size = os.path.getsize(os.path.join(weights, "model.safetensors"))
        shares_bf16, _ = stream_setup.estimate_stripe_write_shares(
            weights, total_bytes=file_size, n_roots=2, dtype="bfloat16", quant="none"
        )
        shares_nf4, approx = stream_setup.estimate_stripe_write_shares(
            weights, total_bytes=file_size, n_roots=2, dtype="bfloat16", quant="nf4"
        )
        assert not approx
        assert sum(shares_nf4) == file_size
        # Root 0 holds non-decoder weights charged at full itemsize, so its share ratio
        # under NF4 is higher than under unquantised bf16
        assert shares_nf4[0] / sum(shares_nf4) > shares_bf16[0] / sum(shares_bf16)

    def test_preflight_unparseable_header_falls_back_to_even_split(self, tmp_path):
        """#1612 mutation check: garbage safetensors header falls back to even split."""
        corrupt = tmp_path / "corrupt"
        corrupt.mkdir(parents=True, exist_ok=True)
        with open(corrupt / "model.safetensors", "wb") as f:
            f.write(struct.pack("<Q", 100))
            f.write(b"NOT_VALID_JSON_GARBAGE_HEADER_DATA")
        shares, approx = stream_setup.estimate_stripe_write_shares(
            str(corrupt), total_bytes=1000, n_roots=2
        )
        assert approx is True
        assert shares == stream_setup._stripe_write_shares(1000, 2)

    def test_exact_shares_qwen4_exp_ple_skipped_and_even_split_used(self, tmp_path):
        """#1612: qwen4_exp arch falls back to even split so external PLE tables are not charged."""
        qwen_dir = tmp_path / "qwen4_model"
        qwen_dir.mkdir(parents=True, exist_ok=True)
        cur = 0
        tensors = {
            "model.embed_tokens.weight": {
                "dtype": "BF16", "shape": [128, 64], "data_offsets": [cur, cur + 16384]
            },
            "model.layers.0.ple.ple_embedding.ngram_embedding.weight": {
                "dtype": "BF16", "shape": [64, 64], "data_offsets": [cur + 16384, cur + 24576]
            },
            "model.layers.0.self_attn.q_proj.weight": {
                "dtype": "BF16", "shape": [64, 64], "data_offsets": [cur + 24576, cur + 32768]
            },
        }
        cur += 32768
        data = json.dumps(tensors).encode("utf-8")
        with open(qwen_dir / "model.safetensors", "wb") as f:
            f.write(struct.pack("<Q", len(data)))
            f.write(data)
            f.write(b"\x00" * cur)

        shares, approx = stream_setup.estimate_stripe_write_shares(
            str(qwen_dir), total_bytes=10000, n_roots=2, arch="qwen4_exp"
        )
        assert approx is True
        assert shares == stream_setup._stripe_write_shares(10000, 2)

    def test_exact_shares_never_charge_less_than_the_header_bytes(self, tmp_path):
        """#1612: an estimate below the real header bytes is raised to them, not trusted."""
        weights = _make_synthetic_untied_checkpoint(tmp_path, n_layers=5)
        shares, approx = stream_setup.estimate_stripe_write_shares(
            weights, total_bytes=1000, n_roots=2, dtype="float32"
        )
        assert not approx
        assert sum(shares) > 1000


def _make_synthetic_untied_checkpoint(tmp_path, n_layers: int = 5) -> str:
    """A synthetic safetensors checkpoint with n_layers and untied lm_head (no torch needed)."""
    weights = tmp_path / "model"
    weights.mkdir(parents=True, exist_ok=True)
    tensors = {}
    cur = 0
    tensors["model.embed_tokens.weight"] = {
        "dtype": "BF16", "shape": [128, 64], "data_offsets": [cur, cur + 16384]
    }
    cur += 16384
    tensors["lm_head.weight"] = {
        "dtype": "BF16", "shape": [128, 64], "data_offsets": [cur, cur + 16384]
    }
    cur += 16384
    tensors["model.norm.weight"] = {
        "dtype": "BF16", "shape": [64], "data_offsets": [cur, cur + 128]
    }
    cur += 128
    for i in range(n_layers):
        for proj in ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]:
            name = (
                f"model.layers.{i}.self_attn.{proj}.weight"
                if "proj" in proj and "gate" not in proj and "up" not in proj and "down" not in proj
                else f"model.layers.{i}.mlp.{proj}.weight"
            )
            tensors[name] = {"dtype": "BF16", "shape": [64, 64], "data_offsets": [cur, cur + 8192]}
            cur += 8192
        tensors[f"model.layers.{i}.input_layernorm.weight"] = {
            "dtype": "BF16", "shape": [64], "data_offsets": [cur, cur + 128]
        }
        cur += 128
        tensors[f"model.layers.{i}.post_attention_layernorm.weight"] = {
            "dtype": "BF16", "shape": [64], "data_offsets": [cur, cur + 128]
        }
        cur += 128

    data = json.dumps(tensors).encode("utf-8")
    file_path = weights / "model.safetensors"
    with open(file_path, "wb") as f:
        f.write(struct.pack("<Q", len(data)))
        f.write(data)
        f.write(b"\x00" * cur)
    return str(weights)


def _drive_setup(tmp_path, monkeypatch, *, striped, stream_source="ram"):
    """The real `_setup_streaming_transformers` on a tiny checkpoint, runtime boundary spied.

    Same harness as test_issue623 (simulated CUDA, ``build_streamed_model`` spied). Two temp
    folders share one volume, so ``volume_of`` is faked to tell them apart; everything else —
    the variable, the validation, the disk-kind callable, the sharder, the depth — is real.
    """
    import torch

    import soup_cli.utils.layer_stream as layer_stream
    import soup_cli.utils.layer_stream_runtime as runtime_module
    import soup_cli.utils.stripe_roots as stripe_module
    from soup_cli.config.loader import load_config_from_string
    from soup_cli.trainer.sft import SFTTrainerWrapper
    from tests.test_issue623_stream_pin_allocation import _cfg_yaml, _tiny_checkpoint

    free = 10_000_000_000
    # A second call in the same folder reuses the checkpoint: rebuilding it would move its
    # mtimes, and the cache (keyed on them) would rightly be rewritten.
    existing = tmp_path / "model"
    weights = str(existing) if existing.is_dir() else _tiny_checkpoint(tmp_path)
    second_drive = tmp_path / "second-drive"
    second_drive.mkdir(exist_ok=True)
    monkeypatch.chdir(tmp_path)
    monkeypatch.setenv("SOUP_LAYER_STREAM_CACHE_DIR", str(tmp_path / "cache"))
    if striped:
        monkeypatch.setenv("SOUP_LAYER_STREAM_STRIPE_DIRS", str(second_drive))
    else:
        monkeypatch.delenv("SOUP_LAYER_STREAM_STRIPE_DIRS", raising=False)
    monkeypatch.setattr(
        stripe_module, "volume_of", lambda path: 2 if "second-drive" in str(path) else 1
    )
    monkeypatch.setattr(layer_stream, "free_ram_bytes", lambda: free)
    monkeypatch.setattr(
        layer_stream, "classify_disk_kind",
        lambda *_a, **_k: layer_stream.DiskClassification("nvme"),
    )
    monkeypatch.setattr(torch.cuda, "mem_get_info", lambda *_a, **_k: (free, free))
    monkeypatch.setattr(runtime_module, "expandable_segments_status", lambda *_a, **_k: (True, ""))

    preflight_calls = []
    orig_preflight = stream_setup._render_stream_disk_preflight

    def spy_preflight(**kwargs):
        preflight_calls.append(kwargs)
        return orig_preflight(**kwargs)

    monkeypatch.setattr(stream_setup, "_render_stream_disk_preflight", spy_preflight)

    captured = {}

    def spy_build(**kwargs):
        from unittest.mock import MagicMock

        captured.update(kwargs)
        runtime = MagicMock()
        runtime.stats.return_value = {
            "tier": stream_source, "store_bytes": 1_000_000, "disk_bytes": 1_000_000,
            "pinned": False, "read_ahead": kwargs.get("read_ahead"), "buffers": 2,
            "buffer_bytes": 4_000, "n_layers": 3,
        }
        return MagicMock(), runtime

    monkeypatch.setattr(runtime_module, "build_streamed_model", spy_build)
    buffer = io.StringIO()
    monkeypatch.setattr(
        stream_setup, "console", Console(file=buffer, width=400, color_system=None)
    )

    cfg = load_config_from_string(_cfg_yaml(weights, "", stream_source=stream_source))
    wrapper = SFTTrainerWrapper(cfg)
    wrapper.device = "cuda"
    wrapper._setup_streaming_transformers(cfg, cfg.training)
    return captured, " ".join(buffer.getvalue().split()), second_drive, preflight_calls


class TestTheSetupPathUsesTheRoots:
    """R4 end to end: the variable reaches the sharder, the pre-flight and the reader."""

    def test_two_drives_stripe_the_cache_and_deepen_the_reader(self, tmp_path, monkeypatch):
        captured, said, second_drive, preflight_calls = _drive_setup(
            tmp_path, monkeypatch, striped=True
        )
        index = captured["index"]
        assert index.stripe_roots == (os.path.realpath(second_drive),)
        assert index.layer_roots == (0, 1, 0)
        assert captured["read_ahead"] == DEFAULT_STREAM_READ_AHEAD + 1
        # Finding A: the page-lock advice downstream needs to know the depth was raised.
        assert captured["read_ahead_decision"].raised
        assert "Layer cache striped over" in said
        assert "layer-shard stripe 1" in said  # the second drive is charged on its own
        assert f"training.stream_read_ahead {DEFAULT_STREAM_READ_AHEAD} -> 3" not in said
        stripe_folder = second_drive / os.listdir(second_drive)[0]
        assert any(name.startswith("layer_") for name in os.listdir(stripe_folder)), (
            "no layer file landed on the second drive"
        )
        # #1612: exact shares charge root 0 more than root 1 (2 of 3 layers plus extras on root 0)
        assert len(preflight_calls) >= 1
        assert preflight_calls[0]["shard_write_bytes"] > preflight_calls[0]["stripe_writes"][0][1]

    def test_disk_tier_announces_the_effective_stripe_depth(self, tmp_path, monkeypatch):
        _captured, said, _second_drive, _preflight = _drive_setup(
            tmp_path, monkeypatch, striped=True, stream_source="disk"
        )
        assert f"training.stream_read_ahead {DEFAULT_STREAM_READ_AHEAD} -> 3" in said

    def test_one_drive_is_left_exactly_as_it_was(self, tmp_path, monkeypatch):
        captured, said, _second_drive, _preflight = _drive_setup(
            tmp_path, monkeypatch, striped=False
        )
        index = captured["index"]
        assert index.stripe_roots == () and index.layer_roots == ()
        assert captured["read_ahead"] == DEFAULT_STREAM_READ_AHEAD
        assert "striped" not in said and "stripe 1" not in said

    def test_a_striped_cache_is_reused_not_rewritten_on_the_next_run(self, tmp_path, monkeypatch):
        """The pre-flight asks the cache check about the SAME roots the sharder will use."""
        _drive_setup(tmp_path, monkeypatch, striped=True)
        _captured, said, _second_drive, _preflight = _drive_setup(
            tmp_path, monkeypatch, striped=True
        )
        assert "(reusable)" in said
        assert "layer-shard stripe 1" not in said  # no write is planned for a cache we keep

    def test_the_setup_hands_the_estimate_its_dtype_quant_and_arch(self, tmp_path, monkeypatch):
        calls = []
        real = stream_setup.estimate_stripe_write_shares

        def spy(*args, **kwargs):
            calls.append(kwargs)
            return real(*args, **kwargs)

        monkeypatch.setattr(stream_setup, "estimate_stripe_write_shares", spy)
        _drive_setup(tmp_path, monkeypatch, striped=True)
        assert calls, "the striped setup never asked for the exact shares"
        assert {"dtype", "quant", "double_quant", "arch"} <= set(calls[0])
        assert calls[0]["arch"] == "llama"


# The tiny checkpoint ties its embeddings, so its cache has no large shard: a `large_*` file in
# a cache folder is a stale one, and is left out of the oracle on purpose.
_CACHE_FILE = re.compile(r"^(index\.json|extras\.safetensors|layer_\d+\.safetensors)$")


def _cache_folders(tmp_path):
    """Every folder a cache lives in: the primary one and each stripe folder."""
    found = []
    for top in (tmp_path / "cache", tmp_path / "second-drive"):
        for folder, _dirs, names in os.walk(top):
            if any(_CACHE_FILE.match(name) for name in names):
                found.append(folder)
    return found


def _primary_folder(tmp_path):
    """The one folder holding ``index.json``: the cache's primary folder."""
    (folder,) = [f for f in _cache_folders(tmp_path) if "index.json" in os.listdir(f)]
    return folder


def _cache_files_bytes(tmp_path):
    """The files the cache is made of, picked by name, so a stray file is never in the sum."""
    return sum(
        os.path.getsize(os.path.join(folder, name))
        for folder in _cache_folders(tmp_path)
        for name in os.listdir(folder)
        if _CACHE_FILE.match(name)
    )


def _drop_strays(tmp_path):
    """Junk a cache folder can hold: a half-written temp file, a note, a stale large shard."""
    assert not any(n.startswith("large_") for f in _cache_folders(tmp_path) for n in os.listdir(f))
    for folder in _cache_folders(tmp_path):
        for name in (".soup.x.tmp", "notes.bin"):
            with open(os.path.join(folder, name), "wb") as handle:
                handle.write(b"x" * 100_000)
        with open(os.path.join(folder, "large_embed.safetensors"), "wb") as handle:
            handle.write(b"x" * 100_000)


class TestAReusableCacheIsReportedAtItsRealSize:
    @pytest.mark.parametrize("striped", [False, True])
    def test_reusable_cache_reports_its_files_not_the_estimate(
        self, tmp_path, monkeypatch, striped
    ):
        _drive_setup(tmp_path, monkeypatch, striped=striped)  # builds the cache
        _drop_strays(tmp_path)
        _captured, said, _second_drive, seen = _drive_setup(
            tmp_path, monkeypatch, striped=striped
        )
        assert "(reusable)" in said
        assert seen[-1]["shard_write_bytes"] == 0
        assert all(share == 0 for _dir, share in seen[-1]["stripe_writes"])
        assert seen[-1]["shard_bytes"] == _cache_files_bytes(tmp_path)

    def test_a_measured_cache_is_not_called_approximate(self, tmp_path, monkeypatch):
        """qwen4_exp's even-split fallback says "approximately"; a measured size is exact."""
        _drive_setup(tmp_path, monkeypatch, striped=True)  # builds the cache
        real = stream_setup.estimate_stripe_write_shares

        def as_qwen4_exp(*args, **kwargs):
            return real(*args, **{**kwargs, "arch": "qwen4_exp"})

        monkeypatch.setattr(stream_setup, "estimate_stripe_write_shares", as_qwen4_exp)
        assert real(None, total_bytes=0, n_roots=2, arch="qwen4_exp")[1] is True
        _captured, said, _second_drive, seen = _drive_setup(tmp_path, monkeypatch, striped=True)
        assert "(reusable)" in said
        assert seen[-1]["shard_bytes"] == _cache_files_bytes(tmp_path)
        assert seen[-1]["approximate"] is False
        assert "approximately" not in said

    def test_a_file_that_cannot_be_measured_keeps_the_estimate(self, tmp_path, monkeypatch):
        captured, _said, _drive, seen = _drive_setup(tmp_path, monkeypatch, striped=False)
        estimate = seen[-1]["shard_bytes"]  # the first run writes the cache: a plain estimate
        index = captured["index"]
        shard_dir = _primary_folder(tmp_path)
        assert layer_shard.shard_cache_disk_bytes(shard_dir, index) is not None
        monkeypatch.setattr(layer_shard, "shard_cache_disk_bytes", lambda *_a: None)
        _captured, _said, _drive, seen = _drive_setup(tmp_path, monkeypatch, striped=False)
        assert seen[-1]["shard_write_bytes"] == 0
        assert seen[-1]["shard_bytes"] == estimate

    def test_a_deleted_layer_file_is_not_measured(self, tmp_path, monkeypatch):
        captured, _said, _drive, _preflight = _drive_setup(tmp_path, monkeypatch, striped=False)
        index = captured["index"]
        shard_dir = _primary_folder(tmp_path)
        os.remove(layer_shard.layer_paths(shard_dir, index)[0])
        assert layer_shard.shard_cache_disk_bytes(shard_dir, index) is None

    def test_a_cache_file_lost_after_the_check_does_not_fail_the_preflight(
        self, tmp_path, monkeypatch
    ):
        """The file goes between `inspect_shard_cache` and the sizing; the estimate stays."""
        captured, _said, _drive, seen = _drive_setup(tmp_path, monkeypatch, striped=True)
        estimate = seen[-1]["shard_bytes"]  # the first run writes the cache: a plain estimate
        assert _cache_files_bytes(tmp_path) != estimate  # else the last assert proves nothing
        primary = _primary_folder(tmp_path)
        victim = layer_shard.layer_paths(primary, captured["index"])[1]  # on the second drive
        assert os.path.exists(victim) and os.path.dirname(victim) != primary
        real = layer_shard.inspect_shard_cache
        lost = []

        def inspect_then_lose_a_file(*args, **kwargs):
            result = real(*args, **kwargs)
            if not lost and result[0] is not None:
                os.remove(victim)
                lost.append(victim)
            return result

        monkeypatch.setattr(layer_shard, "inspect_shard_cache", inspect_then_lose_a_file)
        _captured, _said, _drive, seen = _drive_setup(  # must not raise
            tmp_path, monkeypatch, striped=True
        )
        assert lost == [victim]
        assert seen[-1]["shard_write_bytes"] == 0
        assert seen[-1]["shard_bytes"] == estimate

    def test_an_untied_striped_cache_counts_its_vocabulary_shards(self, tmp_path):
        """The tied fixture above has no `large_*` file; an untied checkpoint writes two."""
        from tests.test_disk_tier_stripe_runtime import _tiny_llama

        weights = _tiny_llama(tmp_path, tie=False)
        stripe = tmp_path / "stripe"
        stripe.mkdir()
        cache = str(tmp_path / "cache" / "tiny")
        index = layer_shard.shard_checkpoint(
            weights, cache, dtype="float32", arch="llama", stripe_roots=(str(stripe),)
        )
        assert index.large_keys and index.layer_roots == (0, 1, 0)
        folders = layer_shard.stripe_dirs(cache, index.stripe_roots)
        files = [
            os.path.join(folder, name)
            for folder in folders
            for name in os.listdir(folder)
            if name != layer_shard.STRIPE_MARKER_NAME
        ]
        assert any(os.path.basename(path).startswith("large_") for path in files)
        assert layer_shard.shard_cache_disk_bytes(cache, index) == sum(
            os.path.getsize(path) for path in files
        )


class TestRefusalIsUnchanged:
    """Reporting the real size must not move what the pre-flight refuses."""

    @staticmethod
    def _free(monkeypatch, free):
        monkeypatch.setattr(stream_setup, "_disk_volume", lambda _path: (1, free))

    @pytest.mark.parametrize("free", [1, 10_000_000_000])
    def test_a_reusable_cache_needs_no_space(self, tmp_path, monkeypatch, free):
        _drive_setup(tmp_path, monkeypatch, striped=False)
        self._free(monkeypatch, free)
        _captured, said, _drive, _preflight = _drive_setup(tmp_path, monkeypatch, striped=False)
        assert "(reusable)" in said

    @pytest.mark.parametrize("free, refused", [(1, True), (10_000_000_000, False)])
    def test_a_missing_cache_is_refused_only_without_room(
        self, tmp_path, monkeypatch, free, refused
    ):
        self._free(monkeypatch, free)
        if refused:
            with pytest.raises(ValueError, match="additional disk space"):
                _drive_setup(tmp_path, monkeypatch, striped=False)
        else:
            _drive_setup(tmp_path, monkeypatch, striped=False)

    @pytest.mark.parametrize("free, refused", [(1, True), (10_000_000_000, False)])
    def test_a_materialization_is_refused_only_without_room(
        self, tmp_path, monkeypatch, free, refused
    ):
        monkeypatch.setattr(stream_setup, "console", Console(file=io.StringIO()))
        monkeypatch.setattr(stream_setup, "_disk_volume", lambda _path: (1, free))
        args = dict(
            source_bytes=1000,
            materialized_copy_bytes=500,
            materialize_bytes=500,
            materialized_path=str(tmp_path / "weights"),
            shard_bytes=300,  # the real size of a cache we keep
            shard_write_bytes=0,
            shard_path=str(tmp_path / "shards"),
        )
        if refused:
            with pytest.raises(ValueError, match="additional disk space"):
                stream_setup._render_stream_disk_preflight(**args)
        else:
            stream_setup._render_stream_disk_preflight(**args)

    @pytest.mark.parametrize("copy_bytes, refused", [(2_000_000_000, True), (100_000_000, False)])
    @pytest.mark.parametrize("cache_exists", [False, True])
    def test_a_materialization_through_the_setup_is_refused_only_without_room(
        self, tmp_path, monkeypatch, copy_bytes, refused, cache_exists
    ):
        """The copy is charged through the setup's own call, with or without a cache on disk."""
        import dataclasses

        import soup_cli.utils.spectrum_scan as spectrum_scan

        if cache_exists:
            _drive_setup(tmp_path, monkeypatch, striped=False)

        def needs_a_copy(model, *, before_materialize=None):
            plan = spectrum_scan.plan_model_weights(model)
            plan = dataclasses.replace(
                plan, materialize_bytes=copy_bytes, materialized_copy_bytes=copy_bytes
            )
            before_materialize(plan)
            return plan.weights_dir

        monkeypatch.setattr(spectrum_scan, "resolve_model_weights", needs_a_copy)
        self._free(monkeypatch, 1_000_000_000)  # the tiny cache fits; the copy decides
        if refused:
            with pytest.raises(ValueError, match="materialized weight copy"):
                _drive_setup(tmp_path, monkeypatch, striped=False)
        else:
            _drive_setup(tmp_path, monkeypatch, striped=False)