Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

36.1 KB
Raw Blame
from __future__ import annotations

import math
import types
from pathlib import Path

import pytest
import torch
from safetensors.torch import load_file, save_file

from tests.test_issue502_qwen35_transformers5 import _REQUIRES_TRAINING_FLOOR


def _toy_qwen35_classes():
    """Register a tiny heterogeneous MoE under the real Qwen3.5 model type.

    The alternating attention modules are the property that matters here: a
    layer-0-derived stream spec cannot execute layer 1.  The small MoE is real
    computation rather than naming decoration, so resident-vs-streamed logits
    exercise the complete decoder graph on CPU.
    """
    from torch import nn
    from transformers import AutoModelForCausalLM, PretrainedConfig, PreTrainedModel
    from transformers.modeling_outputs import CausalLMOutput

    cached = getattr(_toy_qwen35_classes, "_cached", None)
    if cached is not None:
        return cached

    class ToyQwen35Config(PretrainedConfig):
        model_type = "qwen3_5_moe"

        def __init__(
            self,
            vocab_size=32,
            hidden_size=8,
            num_hidden_layers=2,
            num_experts=2,
            layer_types=None,
            **kwargs,
        ):
            tie_word_embeddings = kwargs.pop("tie_word_embeddings", False)
            super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
            self.vocab_size = vocab_size
            self.hidden_size = hidden_size
            self.num_hidden_layers = num_hidden_layers
            self.num_experts = num_experts
            self.layer_types = layer_types or ["full_attention", "linear_attention"]

    class ToyMoe(nn.Module):
        def __init__(self, config):
            super().__init__()
            self.gate = nn.Linear(config.hidden_size, config.num_experts, bias=False)
            self.experts = nn.ModuleList(
                nn.Linear(config.hidden_size, config.hidden_size, bias=False)
                for _ in range(config.num_experts)
            )

        def forward(self, hidden_states):
            routing = self.gate(hidden_states).softmax(dim=-1)
            expert_outputs = torch.stack(
                [expert(hidden_states) for expert in self.experts], dim=-2
            )
            return (expert_outputs * routing.unsqueeze(-1)).sum(dim=-2)

    class ToyFullAttentionLayer(nn.Module):
        def __init__(self, config):
            super().__init__()
            self.self_attn = nn.Module()
            self.self_attn.q_proj = nn.Linear(
                config.hidden_size, config.hidden_size, bias=False
            )
            self.mlp = ToyMoe(config)

        def forward(self, hidden_states, *args, **kwargs):
            del args, kwargs
            return hidden_states + torch.tanh(
                self.self_attn.q_proj(hidden_states)
            ) + self.mlp(hidden_states)

    class ToyLinearAttentionLayer(nn.Module):
        def __init__(self, config):
            super().__init__()
            self.linear_attn = nn.Module()
            self.linear_attn.in_proj_qkv = nn.Linear(
                config.hidden_size, config.hidden_size, bias=False
            )
            self.mlp = ToyMoe(config)

        def forward(self, hidden_states, *args, **kwargs):
            del args, kwargs
            return hidden_states + torch.sigmoid(
                self.linear_attn.in_proj_qkv(hidden_states)
            ) + self.mlp(hidden_states)

    class ToyDecoder(nn.Module):
        def __init__(self, config):
            super().__init__()
            self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
            self.layers = nn.ModuleList(
                ToyFullAttentionLayer(config)
                if layer_type == "full_attention"
                else ToyLinearAttentionLayer(config)
                for layer_type in config.layer_types
            )
            self.norm = nn.LayerNorm(config.hidden_size)

        def forward(self, input_ids):
            hidden_states = self.embed_tokens(input_ids)
            for layer in self.layers:
                hidden_states = layer(hidden_states)
            return self.norm(hidden_states)

    class ToyQwen35ForCausalLM(PreTrainedModel):
        config_class = ToyQwen35Config
        base_model_prefix = "model"

        def __init__(self, config):
            super().__init__(config)
            self.model = ToyDecoder(config)
            self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
            self.post_init()

        def _init_weights(self, module):
            if isinstance(module, (nn.Linear, nn.Embedding)):
                nn.init.normal_(module.weight, mean=0.0, std=0.02)
            elif isinstance(module, nn.LayerNorm):
                nn.init.ones_(module.weight)
                nn.init.zeros_(module.bias)

        def get_input_embeddings(self):
            return self.model.embed_tokens

        def set_input_embeddings(self, value):
            self.model.embed_tokens = value

        def get_output_embeddings(self):
            return self.lm_head

        def set_output_embeddings(self, value):
            self.lm_head = value

        def prepare_inputs_for_generation(self, input_ids, **kwargs):
            del kwargs
            return {"input_ids": input_ids}

        def forward(self, input_ids=None, inputs_embeds=None, **kwargs):
            del kwargs
            if input_ids is None:
                if inputs_embeds is None:
                    raise ValueError("input_ids or inputs_embeds is required")
                hidden_states = inputs_embeds
                for layer in self.model.layers:
                    hidden_states = layer(hidden_states)
                hidden_states = self.model.norm(hidden_states)
            else:
                hidden_states = self.model(input_ids)
            return CausalLMOutput(logits=self.lm_head(hidden_states))

    AutoModelForCausalLM.register(
        ToyQwen35Config, ToyQwen35ForCausalLM, exist_ok=True
    )
    result = ToyQwen35Config, ToyQwen35ForCausalLM
    _toy_qwen35_classes._cached = result
    return result


def _tiny_qwen35_lora():
    from peft import LoraConfig, TaskType

    return LoraConfig(
        r=2,
        lora_alpha=4,
        lora_dropout=0.0,
        bias="none",
        target_modules=["q_proj", "in_proj_qkv"],
        task_type=TaskType.CAUSAL_LM,
    )


def _copy_toy_lora(src, dst):
    def canonical(key):
        return key.replace(".inner.", ".")

    src_lora = {canonical(k): v for k, v in src.state_dict().items() if "lora_" in k}
    dst_lora = {canonical(k): v for k, v in dst.state_dict().items() if "lora_" in k}
    assert src_lora and set(src_lora) == set(dst_lora)
    with torch.no_grad():
        for key, value in src_lora.items():
            dst_lora[key].copy_(value)


def _heterogeneous_weights_dir(tmp_path: Path, *, vlm_prefix: bool = False) -> str:
    weights = tmp_path / "weights"
    weights.mkdir()
    prefix = "model.language_model." if vlm_prefix else "model."
    save_file(
        {
            prefix + "layers.0.self_attn.q_proj.weight": torch.randn(4, 4),
            prefix + "layers.0.mlp.gate_proj.weight": torch.randn(4, 4),
            prefix + "layers.1.linear_attn.in_proj_qkv.weight": torch.randn(4, 4),
            prefix + "layers.1.mlp.gate_proj.weight": torch.randn(4, 4),
            prefix + "embed_tokens.weight": torch.randn(8, 4),
        },
        str(weights / "model.safetensors"),
    )
    return str(weights)


def _heterogeneous_meta_model():
    import torch.nn as nn

    class _MetaLinear(nn.Module):
        def __init__(self):
            super().__init__()
            self.weight = nn.Parameter(torch.empty(4, 4, device="meta"))

    class _Layer0(nn.Module):
        def __init__(self):
            super().__init__()
            self.self_attn = nn.Module()
            self.self_attn.q_proj = _MetaLinear()
            self.mlp = nn.Module()
            self.mlp.gate_proj = _MetaLinear()

    class _Layer1(nn.Module):
        def __init__(self):
            super().__init__()
            self.linear_attn = nn.Module()
            self.linear_attn.in_proj_qkv = _MetaLinear()
            self.mlp = nn.Module()
            self.mlp.gate_proj = _MetaLinear()

    class _Decoder(nn.Module):
        def __init__(self):
            super().__init__()
            self.embed_tokens = nn.Embedding(8, 4, device="meta")
            self.layers = nn.ModuleList([_Layer0(), _Layer1()])

    class _Model(nn.Module):
        def __init__(self):
            super().__init__()
            self.model = _Decoder()
            self.lm_head = nn.Linear(4, 8, bias=False, device="meta")
            self.lm_head.weight = self.model.embed_tokens.weight

        def get_input_embeddings(self):
            return self.model.embed_tokens

        def get_output_embeddings(self):
            return self.lm_head

    return _Model()


class TestQwen35StreamingAliases:
    def test_stream_arch_of_accepts_qwen35_dense_text_aliases(self):
        from soup_cli.utils.layer_stream import stream_arch_of

        text = types.SimpleNamespace(model_type="qwen3_5_text")
        assert stream_arch_of(text) == "qwen3"

        wrapped = types.SimpleNamespace(
            model_type="qwen3_5",
            text_config=text,
        )
        assert stream_arch_of(wrapped) == "qwen3"

    def test_stream_arch_of_accepts_qwen35_moe_aliases(self):
        from soup_cli.utils.layer_stream import stream_arch_of

        cfg = types.SimpleNamespace(model_type="qwen3_5_moe")
        assert stream_arch_of(cfg) == "qwen3"

        wrapped = types.SimpleNamespace(
            model_type="qwen2_vl",
            text_config=types.SimpleNamespace(model_type="qwen3_5_moe_text"),
        )
        assert stream_arch_of(wrapped) == "qwen3"

    def test_multimodal_gemma3_wrapper_is_still_rejected(self):
        from soup_cli.utils.layer_stream import stream_arch_of

        wrapped = types.SimpleNamespace(
            model_type="gemma3",
            text_config=types.SimpleNamespace(model_type="gemma3_text"),
        )
        with pytest.raises(ValueError, match="gemma3"):
            stream_arch_of(wrapped)

    def test_stream_setup_reads_text_subconfig_and_moe_intermediate_size(self):
        from soup_cli.trainer.stream_setup import StreamingSetupMixin

        class _Wrapper(StreamingSetupMixin):
            pass

        wrapper = _Wrapper()
        cfg = types.SimpleNamespace(
            text_config=types.SimpleNamespace(
                hidden_size=4096,
                num_hidden_layers=48,
                vocab_size=151936,
                moe_intermediate_size=1536,
                num_experts_per_tok=8,
                shared_expert_intermediate_size=2048,
            )
        )
        assert wrapper._stream_shape_config(cfg) is cfg.text_config
        assert wrapper._stream_intermediate_size(cfg.text_config) == 1536 * 8 + 2048


class TestQwen35StreamingParity:
    @_REQUIRES_TRAINING_FLOOR
    def test_dense_text_decoder_logits_match_resident_bit_exactly(self, tmp_path):
        """Admit the real dense Qwen3.5 decoder graph, not just its model-type name.

        Qwen3.8-27B alternates linear-attention and full-attention blocks. This
        native Transformers fixture preserves both layer kinds while shrinking
        every dimension enough for a CPU parity control.
        """
        from peft import get_peft_model
        from transformers import Qwen3_5ForCausalLM, Qwen3_5TextConfig

        from soup_cli.utils.layer_shard import shard_checkpoint
        from soup_cli.utils.layer_stream import stream_arch_of
        from soup_cli.utils.layer_stream_runtime import build_streamed_model

        config = Qwen3_5TextConfig(
            vocab_size=64,
            hidden_size=16,
            intermediate_size=32,
            num_hidden_layers=2,
            num_attention_heads=2,
            num_key_value_heads=1,
            head_dim=8,
            linear_key_head_dim=4,
            linear_value_head_dim=4,
            linear_num_key_heads=2,
            linear_num_value_heads=2,
            layer_types=["linear_attention", "full_attention"],
            max_position_embeddings=128,
        )
        torch.manual_seed(31)
        resident = Qwen3_5ForCausalLM(config).to(torch.float32).eval()
        weights = tmp_path / "weights"
        resident.save_pretrained(weights, safe_serialization=True)

        arch = stream_arch_of(config)
        assert arch == "qwen3"
        shards = str(tmp_path / "shards")
        index = shard_checkpoint(str(weights), shards, dtype="float32", arch=arch)

        streamed, runtime = build_streamed_model(
            model_id=str(weights),
            shard_dir=shards,
            index=index,
            lora_config=_tiny_qwen35_lora(),
            device="cpu",
            dtype="float32",
            buffers=2,
            pin=False,
            seed=37,
        )
        try:
            reference = get_peft_model(resident, _tiny_qwen35_lora())
            _copy_toy_lora(streamed, reference)
            streamed.eval()
            reference.eval()
            input_ids = torch.tensor([[1, 7, 3, 11, 5, 2]])
            with torch.no_grad():
                got = streamed(input_ids=input_ids).logits
                want = reference(input_ids=input_ids).logits
            assert torch.equal(got, want), (got - want).abs().max().item()
        finally:
            runtime.close()

    def test_heterogeneous_moe_logits_match_resident_bit_exactly(
        self, tmp_path, monkeypatch
    ):
        """Admission gate: exercise the alias through the complete CPU runtime.

        Layer 0 owns ``self_attn.q_proj`` while layer 1 owns
        ``linear_attn.in_proj_qkv``.  Reusing layer 0's stream spec therefore
        cannot pass this test, and equal logits prove more than buffer-copy
        fidelity: the substituted weights execute in the same decoder kernels
        as the resident reference.
        """
        from peft import get_peft_model
        from transformers import AutoConfig

        from soup_cli.utils.layer_shard import layer_shard_path, shard_checkpoint
        from soup_cli.utils.layer_stream import stream_arch_of
        from soup_cli.utils.layer_stream_runtime import build_streamed_model

        config_cls, model_cls = _toy_qwen35_classes()
        torch.manual_seed(23)
        config = config_cls()
        resident = model_cls(config).to(torch.float32).eval()
        weights = tmp_path / "weights"
        resident.save_pretrained(weights, safe_serialization=True)

        # Avoid registering the official Qwen3.5 model type globally.  That
        # would replace Transformers' native config in newer environments and
        # make this test order-dependent.  The runtime still receives a config
        # whose real model_type exercises the production alias.
        monkeypatch.setattr(
            AutoConfig,
            "from_pretrained",
            staticmethod(
                lambda model_id, **kwargs: config_cls.from_pretrained(model_id)
            ),
        )

        arch = stream_arch_of(config)
        assert arch == "qwen3"
        shards = str(tmp_path / "shards")
        index = shard_checkpoint(str(weights), shards, dtype="float32", arch=arch)
        layer0 = load_file(layer_shard_path(shards, 0))
        layer1 = load_file(layer_shard_path(shards, 1))
        assert "self_attn.q_proj.weight" in layer0
        assert "self_attn.q_proj.weight" not in layer1
        assert "linear_attn.in_proj_qkv.weight" in layer1
        assert "linear_attn.in_proj_qkv.weight" not in layer0

        streamed, runtime = build_streamed_model(
            model_id=str(weights),
            shard_dir=shards,
            index=index,
            lora_config=_tiny_qwen35_lora(),
            device="cpu",
            dtype="float32",
            buffers=2,
            pin=False,
            seed=29,
        )
        try:
            reference = get_peft_model(resident, _tiny_qwen35_lora())
            _copy_toy_lora(streamed, reference)
            streamed.eval()
            reference.eval()
            input_ids = torch.tensor([[1, 7, 3, 11, 5, 2]])
            with torch.no_grad():
                got = streamed(input_ids=input_ids).logits
                want = reference(input_ids=input_ids).logits
            assert torch.equal(got, want), (got - want).abs().max().item()
        finally:
            runtime.close()


class TestQwen35StreamingSharder:
    def test_sharder_accepts_heterogeneous_layers_and_canonicalises_vlm_prefix(self, tmp_path):
        from soup_cli.utils.layer_shard import (
            extras_shard_path,
            layer_shard_path,
            shard_checkpoint,
        )

        weights = _heterogeneous_weights_dir(tmp_path, vlm_prefix=True)
        out = str(tmp_path / "shards")
        index = shard_checkpoint(weights, out, dtype="float32", arch="qwen3")

        assert "self_attn.q_proj.weight" in index.layer_keys
        assert "linear_attn.in_proj_qkv.weight" in index.layer_keys

        layer0 = load_file(layer_shard_path(out, 0))
        layer1 = load_file(layer_shard_path(out, 1))
        extras = load_file(extras_shard_path(out))

        assert "self_attn.q_proj.weight" in layer0
        assert "linear_attn.in_proj_qkv.weight" not in layer0
        assert "linear_attn.in_proj_qkv.weight" in layer1
        assert "self_attn.q_proj.weight" not in layer1
        assert "model.embed_tokens.weight" in extras
        assert index.large_keys == ()

    def test_sharder_refuses_canonical_key_collisions(self, tmp_path):
        from soup_cli.utils.layer_shard import shard_checkpoint

        weights = tmp_path / "weights"
        weights.mkdir()
        save_file(
            {
                "model.layers.0.self_attn.q_proj.weight": torch.randn(4, 4),
                "model.language_model.layers.0.self_attn.q_proj.weight": torch.randn(4, 4),
                "model.embed_tokens.weight": torch.randn(8, 4),
            },
            str(weights / "model.safetensors"),
        )

        with pytest.raises(ValueError, match="canonicalise"):
            shard_checkpoint(str(weights), str(tmp_path / "shards"), dtype="float32")

    def test_sharder_refuses_conflicting_layouts_for_a_shared_key(self, tmp_path):
        from soup_cli.utils.layer_shard import shard_checkpoint

        weights = tmp_path / "weights"
        weights.mkdir()
        save_file(
            {
                "model.layers.0.mlp.gate_proj.weight": torch.randn(4, 4),
                "model.layers.1.mlp.gate_proj.weight": torch.randn(4, 8),
                "model.embed_tokens.weight": torch.randn(8, 4),
            },
            str(weights / "model.safetensors"),
        )

        with pytest.raises(ValueError, match="stored shapes or dtypes"):
            shard_checkpoint(str(weights), str(tmp_path / "shards"), dtype="float32")


class TestQwen35StreamingRuntime:
    def test_quantised_layer_suffixes_union_all_decoder_variants(self, monkeypatch):
        import sys

        import torch.nn as nn

        bnb = types.ModuleType("bitsandbytes")
        bnb.nn = types.SimpleNamespace(Params4bit=nn.Parameter)
        monkeypatch.setitem(sys.modules, "bitsandbytes", bnb)

        class _QuantLinear(nn.Module):
            def __init__(self):
                super().__init__()
                self.weight = nn.Parameter(torch.empty(4, 4))

        class _Layer0(nn.Module):
            def __init__(self):
                super().__init__()
                self.self_attn = nn.Module()
                self.self_attn.q_proj = _QuantLinear()

        class _Layer1(nn.Module):
            def __init__(self):
                super().__init__()
                self.linear_attn = nn.Module()
                self.linear_attn.in_proj_qkv = _QuantLinear()

        class _Decoder(nn.Module):
            def __init__(self):
                super().__init__()
                self.layers = nn.ModuleList([_Layer0(), _Layer1()])

        class _Model(nn.Module):
            def __init__(self):
                super().__init__()
                self.model = _Decoder()

        from soup_cli.utils.layer_stream_runtime import quantised_layer_suffixes

        assert quantised_layer_suffixes(_Model()) == frozenset(
            {"self_attn.q_proj.weight", "linear_attn.in_proj_qkv.weight"}
        )

    def test_ram_source_explicitly_requests_cpu_allocations(self, tmp_path, monkeypatch):
        from soup_cli.utils.layer_shard import shard_checkpoint
        from soup_cli.utils.layer_stream_runtime import RamSource

        weights = _heterogeneous_weights_dir(tmp_path)
        out = str(tmp_path / "shards")
        index = shard_checkpoint(weights, out, dtype="float32", arch="qwen3")
        layer_specs = RamSource.layer_specs_from_shards(out, index.n_layers)

        requested_devices = []
        real_empty = torch.empty

        def _cpu_empty(*args, **kwargs):
            if "pin_memory" not in kwargs:
                return real_empty(*args, **kwargs)
            requested_devices.append(kwargs.get("device"))
            return torch.zeros(
                *args,
                dtype=kwargs.get("dtype"),
                device="cpu",
                pin_memory=kwargs.get("pin_memory", False),
            )

        monkeypatch.setattr(torch, "empty", _cpu_empty)
        source = RamSource(out, index.n_layers, layer_specs, pin=False)

        assert requested_devices
        assert all(str(device) == "cpu" for device in requested_devices)
        assert all(
            tensor.device.type == "cpu"
            for layer in source.store
            for tensor in layer.values()
        )

    @pytest.mark.parametrize("device_type", ["meta", "mps"])
    def test_ram_source_refuses_a_non_cpu_allocation(
        self, tmp_path, monkeypatch, device_type
    ):
        from soup_cli.utils.layer_shard import shard_checkpoint
        from soup_cli.utils.layer_stream_runtime import RamSource

        weights = _heterogeneous_weights_dir(tmp_path)
        out = str(tmp_path / "shards")
        index = shard_checkpoint(weights, out, dtype="float32", arch="qwen3")
        layer_specs = RamSource.layer_specs_from_shards(out, index.n_layers)
        def _misplaced_empty(*args, **kwargs):
            del args
            if kwargs.get("pin_memory"):
                return types.SimpleNamespace(device=torch.device(device_type))
            raise AssertionError("RamSource allocations should request pin_memory")

        monkeypatch.setattr(torch, "empty", _misplaced_empty)
        with pytest.raises(RuntimeError, match=rf"requested a CPU tensor.*{device_type}"):
            RamSource(out, index.n_layers, layer_specs, pin=True)

    def test_ram_source_refuses_pageable_memory_when_pinning_was_requested(
        self, tmp_path, monkeypatch
    ):
        from soup_cli.utils.layer_shard import shard_checkpoint
        from soup_cli.utils.layer_stream_runtime import RamSource

        weights = _heterogeneous_weights_dir(tmp_path)
        out = str(tmp_path / "shards")
        index = shard_checkpoint(weights, out, dtype="float32", arch="qwen3")
        layer_specs = RamSource.layer_specs_from_shards(out, index.n_layers)
        real_empty = torch.empty

        def _pageable_empty(*args, **kwargs):
            allocation = dict(kwargs)
            allocation["pin_memory"] = False
            return real_empty(*args, **allocation)

        monkeypatch.setattr(torch, "empty", _pageable_empty)
        with pytest.raises(RuntimeError, match="returned pageable memory"):
            RamSource(out, index.n_layers, layer_specs, pin=True)

    def test_mps_gate_disables_pinning_for_direct_callers(
        self, tmp_path, monkeypatch
    ):
        import soup_cli.utils.layer_stream_runtime as runtime_module
        from soup_cli.utils.layer_shard import shard_checkpoint

        class _SourceCapturedError(Exception):
            pass

        captured = {}
        messages = []

        class _RecordingConsole:
            def print(self, *args, **_kwargs):
                messages.append(" ".join(str(arg) for arg in args))

        def _capture_source(*args, **kwargs):
            captured["pin"] = args[3]
            captured["require_pin"] = kwargs["require_pin"]
            raise _SourceCapturedError

        weights = _heterogeneous_weights_dir(tmp_path)
        out = str(tmp_path / "shards")
        index = shard_checkpoint(weights, out, dtype="float32", arch="qwen3")
        monkeypatch.setattr(runtime_module, "_build_source", _capture_source)

        with pytest.raises(_SourceCapturedError):
            runtime_module.install_streaming(
                _heterogeneous_meta_model(),
                shard_dir=out,
                index=index,
                device="mps",
                pin=True,
                require_pin=True,
                console=_RecordingConsole(),
            )

        assert captured == {"pin": False, "require_pin": False}
        assert any("require_pin=True" in message for message in messages)
        assert any("pageable CPU source" in message for message in messages)

    @pytest.mark.skipif(
        not (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()),
        reason="needs Apple Silicon MPS",
    )
    def test_mps_target_keeps_the_host_store_on_cpu(self, tmp_path):
        from soup_cli.utils.layer_shard import shard_checkpoint
        from soup_cli.utils.layer_stream_runtime import install_streaming

        weights = _heterogeneous_weights_dir(tmp_path)
        out = str(tmp_path / "shards")
        index = shard_checkpoint(weights, out, dtype="float32", arch="qwen3")
        runtime = install_streaming(
            _heterogeneous_meta_model(),
            shard_dir=out,
            index=index,
            device="mps",
            pin=True,
            require_pin=True,
        )
        try:
            assert runtime.pinned is False
            assert all(
                tensor.device.type == "cpu"
                for layer in runtime.source.store
                for tensor in layer.values()
            )
            assert all(
                tensor.device.type == "mps"
                for slot in runtime.pool.buffers
                for tensor in slot.values()
            )
            runtime.pool.load_async(0, runtime.source)
            copied = runtime.pool.wait(0)["self_attn.q_proj.weight"].cpu()
            source = runtime.source.get(0, "self_attn.q_proj.weight")
            assert torch.equal(copied, source)
        finally:
            runtime.close()

    def test_install_streaming_accepts_heterogeneous_layer_sets(self, tmp_path):
        from soup_cli.utils.layer_shard import shard_checkpoint
        from soup_cli.utils.layer_stream_runtime import install_streaming

        weights = _heterogeneous_weights_dir(tmp_path)
        out = str(tmp_path / "shards")
        index = shard_checkpoint(weights, out, dtype="float32", arch="qwen3")
        model = _heterogeneous_meta_model()

        runtime = install_streaming(
            model, shard_dir=out, index=index, device="cpu", pin=False
        )
        try:
            for layer in runtime.source.store:
                for tensor in layer.values():
                    assert tensor.device.type == "cpu"
            runtime.pool.load_async(0, runtime.source)
            buffers0 = runtime.pool.wait(0)
            assert runtime.source.get(0, "self_attn.q_proj.weight").device.type == "cpu"
            assert buffers0["self_attn.q_proj.weight"].device.type == "cpu"
            assert torch.equal(
                buffers0["self_attn.q_proj.weight"],
                runtime.source.get(0, "self_attn.q_proj.weight"),
            )

            runtime.pool.load_async(1, runtime.source)
            buffers1 = runtime.pool.wait(1)
            assert runtime.source.get(1, "linear_attn.in_proj_qkv.weight").device.type == "cpu"
            assert buffers1["linear_attn.in_proj_qkv.weight"].device.type == "cpu"
            assert torch.equal(
                buffers1["linear_attn.in_proj_qkv.weight"],
                runtime.source.get(1, "linear_attn.in_proj_qkv.weight"),
            )
        finally:
            runtime.close()

    def test_build_stream_plan_accepts_actual_store_bytes_override(self):
        from soup_cli.utils.layer_stream import TIER_RAM, build_stream_plan

        plan = build_stream_plan(
            arch="qwen3",
            n_layers=2,
            layer_bytes=10,
            store_bytes=13,
            embed_bytes=0,
            available_ram_bytes=100,
            pinned_limit_bytes=None,
            buffers=2,
        )

        assert plan.tier == TIER_RAM
        assert plan.store_bytes == 13

    def test_merge_layer_specs_refuses_conflicting_shared_layouts(self):
        from soup_cli.utils.layer_stream_runtime import RamSource

        with pytest.raises(ValueError, match="stored shapes or dtypes"):
            RamSource.merge_layer_specs(
                [
                    {"mlp.gate_proj.weight": ((4, 4), "float32")},
                    {"mlp.gate_proj.weight": ((4, 8), "float32")},
                ]
            )

    def test_stream_layer_budget_bytes_uses_the_union_pool_spec(self, tmp_path):
        from soup_cli.trainer.stream_setup import StreamingSetupMixin
        from soup_cli.utils.layer_shard import shard_checkpoint
        from soup_cli.utils.layer_stream import dtype_bytes
        from soup_cli.utils.layer_stream_runtime import RamSource

        weights = _heterogeneous_weights_dir(tmp_path)
        out = str(tmp_path / "shards")
        index = shard_checkpoint(weights, out, dtype="float32", arch="qwen3")
        layer_specs = RamSource.layer_specs_from_shards(out, index.n_layers)

        budget = StreamingSetupMixin._stream_layer_budget_bytes(layer_specs)
        union = RamSource.merge_layer_specs(layer_specs)
        actual = sum(
            math.prod(shape) * dtype_bytes(stored) for shape, stored in union.values()
        )
        per_layer = [
            sum(math.prod(shape) * dtype_bytes(stored) for shape, stored in spec.values())
            for spec in layer_specs
        ]

        assert budget == actual
        assert budget > max(per_layer)


class TestQwen35StreamingSetup:
    def test_stream_setup_threads_moe_targets_and_union_budget(self, monkeypatch, tmp_path):
        """moe_lora's targets reach the LoRA config, and the budget is the union
        of the heterogeneous layers' specs.

        The target assertion used to be the seven MoE names alone. That pinned a
        replace nobody chose: the test was written in #426, when ``auto`` still
        resolved to ``None`` on this path, so the MoE list was all there was. Once
        #507 resolved ``auto`` to Qwen3.5's list, the replace silently dropped
        ``in_proj_qkv`` / ``out_proj``. Streaming now extends the resolved list as
        the resident trainers do (#1590), so the expected value is the union: the
        ``auto`` list first, then the MoE names it does not already have.
        """
        from soup_cli.trainer.stream_setup import StreamingSetupMixin

        class _Wrapper(StreamingSetupMixin):
            def __init__(self):
                self.device = "cpu"
                self._trust_remote_code = False

            def _stream_budget_lines(self, *_args, **_kwargs):
                return (), None

        cfg = types.SimpleNamespace(
            base="Qwen/Qwen3.5-35B-A3B",
            data=types.SimpleNamespace(max_length=64),
        )
        tcfg = types.SimpleNamespace(
            quantization="none",
            double_quant_on=True,
            stream_source="auto",
            stream_buffers=2,
            stream_read_ahead=2,
            stream_disk_kind=None,
            stream_pin=None,
            seed=7,
            moe_lora=True,
            batch_size=1,
            gradient_accumulation_steps=1,
            stream_vram_probe=False,
            stream_vram_override=None,
            lora=types.SimpleNamespace(
                r=8,
                alpha=16,
                dropout=0.0,
                target_modules="auto",
                use_dora=False,
                use_rslora=False,
                rank_pattern=None,
                alpha_pattern=None,
            ),
        )
        model_cfg = types.SimpleNamespace(
            model_type="qwen2_vl",
            text_config=types.SimpleNamespace(
                model_type="qwen3_5_moe_text",
                hidden_size=64,
                num_hidden_layers=2,
                vocab_size=128,
                moe_intermediate_size=32,
                num_experts_per_tok=2,
                num_experts=16,
            ),
        )
        index = types.SimpleNamespace(n_layers=2, total_params=100, quant="none", quant_specs={})
        runtime = types.SimpleNamespace(
            stats=lambda: {
                "tier": "ram",
                "store_bytes": 0,
                "pinned": False,
                "n_layers": 2,
                "buffers": 2,
                "buffer_bytes": 8,
            }
        )
        tokenizer = types.SimpleNamespace(pad_token=None, eos_token="</s>")
        captured = {}
        layer_specs = [
            {"self_attn.q_proj.weight": ((4, 4), "float32")},
            {"linear_attn.in_proj_qkv.weight": ((4, 4), "float32")},
        ]

        def fake_lora_config(**kwargs):
            captured["target_modules"] = kwargs["target_modules"]
            return types.SimpleNamespace(**kwargs)

        from soup_cli.utils.layer_stream import build_stream_plan as real_build_stream_plan

        def capture_stream_plan(**kwargs):
            captured["layer_bytes"] = kwargs["layer_bytes"]
            return real_build_stream_plan(**kwargs)

        monkeypatch.setattr(
            "transformers.AutoTokenizer.from_pretrained",
            lambda *_a, **_k: tokenizer,
        )
        monkeypatch.setattr(
            "transformers.AutoConfig.from_pretrained",
            lambda *_a, **_k: model_cfg,
        )
        monkeypatch.setattr("peft.LoraConfig", fake_lora_config)
        monkeypatch.setattr(
            "soup_cli.utils.layer_shard.resolve_shard_dir",
            lambda *_a, **_k: str(tmp_path / "shards"),
        )
        monkeypatch.setattr(
            "soup_cli.utils.layer_shard.shard_checkpoint",
            lambda *_a, **_k: index,
        )
        monkeypatch.setattr(
            "soup_cli.utils.layer_shard.source_weight_bytes",
            lambda *_a, **_k: 1024,
        )
        monkeypatch.setattr(
            "soup_cli.utils.spectrum_scan.resolve_model_weights",
            lambda *_a, **_k: str(tmp_path / "weights"),
        )
        monkeypatch.setattr("soup_cli.utils.layer_stream.free_ram_bytes", lambda: 1_000_000)
        monkeypatch.setattr(
            "soup_cli.utils.layer_stream_runtime.build_meta_skeleton",
            lambda *_a, **_k: types.SimpleNamespace(),
        )
        monkeypatch.setattr(
            "soup_cli.utils.layer_stream_runtime.RamSource.layer_specs_from_paths",
            lambda *_a, **_k: layer_specs,
        )
        monkeypatch.setattr(
            "soup_cli.utils.layer_stream.build_stream_plan", capture_stream_plan
        )
        monkeypatch.setattr(
            "soup_cli.utils.layer_stream_runtime.extras_resident_bytes",
            lambda *_a, **_k: 0,
        )
        monkeypatch.setattr(
            "soup_cli.utils.layer_stream_runtime.build_streamed_model",
            lambda **_kwargs: (types.SimpleNamespace(), runtime),
        )
        monkeypatch.setattr(
            "soup_cli.utils.moe.detect_moe_model",
            lambda *_a, **_k: True,
        )
        monkeypatch.setattr(
            "soup_cli.utils.moe.get_moe_target_modules",
            lambda *_a, **_k: [
                "q_proj",
                "k_proj",
                "v_proj",
                "o_proj",
                "gate_proj",
                "up_proj",
                "down_proj",
            ],
        )
        monkeypatch.setattr(
            "soup_cli.utils.layer_stream.render_stream_panel",
            lambda *_a, **_k: "panel",
        )
        monkeypatch.setattr(
            "soup_cli.utils.layer_stream_runtime.expandable_segments_status",
            lambda: (True, ""),
        )

        wrapper = _Wrapper()
        wrapper._setup_streaming_transformers(cfg, tcfg)

        assert captured["target_modules"] == [
            # target_modules: auto on qwen3_5_moe_text (QWEN35_TEXT_LORA_TARGETS)
            "q_proj",
            "v_proj",
            "in_proj_qkv",
            "out_proj",
            # added by moe_lora (the stubbed get_moe_target_modules, minus repeats)
            "k_proj",
            "o_proj",
            "gate_proj",
            "up_proj",
            "down_proj",
        ]
        assert captured["layer_bytes"] == 128