Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

23.2 KB
Raw Blame
"""Layer streaming for the DeepSeek-V3 architecture (MLA + DeepSeekMoE).

The gate for admitting an architecture is resident-vs-streamed parity on a
real (tiny) model, forward and backward stated separately, the same rule every
other entry of ``SUPPORTED_STREAM_ARCHS`` passed.
"""

from __future__ import annotations

from types import SimpleNamespace

import pytest

_LORA_TARGETS = ["q_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"]


# --------------------------------------------------------------------------
# allowlist
# --------------------------------------------------------------------------
def test_deepseek_v3_is_an_admitted_stream_arch():
    from soup_cli.utils.layer_stream import SUPPORTED_STREAM_ARCHS, stream_arch_of

    assert "deepseek_v3" in SUPPORTED_STREAM_ARCHS
    assert stream_arch_of(SimpleNamespace(model_type="deepseek_v3")) == "deepseek_v3"


def test_kimi_k2_is_admitted_through_the_guarded_config_loader(tmp_path):
    """Kimi-K2-Base / -Instruct declare ``model_type: kimi_k2``, which AutoConfig refuses as
    custom code. The streamed model's config loader (``utils/kimi_k2.py``) reads it into the
    native ``DeepseekV3Config`` after a guard, so a K2 that passes streams as this family. The
    gate is ``tests/test_kimi_k2_streaming.py`` (benchmarks/gate-kimi-k2-admission.md)."""
    import json

    pytest.importorskip("transformers")
    from soup_cli.utils.kimi_k2 import load_stream_model_config
    from soup_cli.utils.layer_stream import stream_arch_of
    from tests.test_kimi_k2_streaming import K2_BASE_CONFIG

    (tmp_path / "config.json").write_text(json.dumps(K2_BASE_CONFIG), encoding="utf-8")
    assert stream_arch_of(load_stream_model_config(str(tmp_path))) == "deepseek_v3"


def test_kimi_k2_that_fails_the_guard_is_refused_by_name(tmp_path):
    """Kimi-K2-Thinking (compressed-tensors INT4) among them: the field and value are named."""
    import json

    pytest.importorskip("transformers")
    from soup_cli.utils.kimi_k2 import UnsupportedKimiK2Error, load_stream_model_config
    from tests.test_kimi_k2_streaming import K2_BASE_CONFIG, K2_THINKING_CHANGES

    thinking = dict(K2_BASE_CONFIG, **K2_THINKING_CHANGES)
    (tmp_path / "config.json").write_text(json.dumps(thinking), encoding="utf-8")
    with pytest.raises(UnsupportedKimiK2Error) as caught:
        load_stream_model_config(str(tmp_path))
    assert "quantization_config.quant_method='compressed-tensors'" in str(caught.value)
    assert "kimi_k2" in str(caught.value)


def test_a_kimi_k2_config_that_bypassed_the_loader_is_still_refused_naming_the_allowlist():
    """``kimi_k2`` itself is not an admitted name: a config still carrying it (K2's own remote
    config class keeps it) did not pass the guard, and the message points at the family."""
    from soup_cli.utils.layer_stream import stream_arch_of

    with pytest.raises(ValueError, match="deepseek_v3") as caught:
        stream_arch_of(SimpleNamespace(model_type="kimi_k2"))
    assert "kimi_k2" in str(caught.value)


# --------------------------------------------------------------------------
# sharder: synthetic safetensors, no model
# --------------------------------------------------------------------------
def _write_layer_checkpoint(path, *, n_layers, n_experts, extra_layer_keys=()):
    import torch
    from safetensors.torch import save_file

    hidden, inter = 4, 3
    state = {
        "model.embed_tokens.weight": torch.zeros(8, hidden),
        "model.norm.weight": torch.ones(hidden),
        "lm_head.weight": torch.zeros(8, hidden),
    }
    for layer in range(n_layers):
        prefix = f"model.layers.{layer}."
        state[prefix + "input_layernorm.weight"] = torch.ones(hidden)
        state[prefix + "mlp.gate.weight"] = torch.full((n_experts, hidden), float(layer))
        state[prefix + "mlp.gate.e_score_correction_bias"] = torch.tensor(
            [0.1 + 1e-3 * expert for expert in range(n_experts)], dtype=torch.float32
        )
        for expert in range(n_experts):
            base = 1000.0 * layer + 10.0 * expert
            state[prefix + f"mlp.experts.{expert}.gate_proj.weight"] = torch.full(
                (inter, hidden), base + 1.0
            )
            state[prefix + f"mlp.experts.{expert}.up_proj.weight"] = torch.full(
                (inter, hidden), base + 2.0
            )
            state[prefix + f"mlp.experts.{expert}.down_proj.weight"] = torch.full(
                (hidden, inter), base + 3.0
            )
    for key in extra_layer_keys:
        state[key] = torch.ones(hidden)
    path.mkdir(parents=True, exist_ok=True)
    save_file(state, str(path / "model.safetensors"))
    (path / "config.json").write_text('{"model_type": "deepseek_v3"}', encoding="utf-8")
    return str(path)


def _layer_file(shards, index, layer):
    from soup_cli.utils.layer_shard import layer_paths

    return layer_paths(str(shards), index)[layer]


def test_per_expert_keys_stack_gate_first_in_numeric_expert_order(tmp_path):
    """11 experts: a lexical sort would put expert 10 before expert 2."""
    import torch
    from safetensors.torch import load_file

    from soup_cli.utils.layer_shard import shard_checkpoint

    weights = _write_layer_checkpoint(tmp_path / "w", n_layers=2, n_experts=11)
    index = shard_checkpoint(
        weights, str(tmp_path / "s"), dtype="float32", arch="deepseek_v3", num_hidden_layers=2
    )
    for layer in range(2):
        tensors = load_file(_layer_file(tmp_path / "s", index, layer))
        assert not any(".experts.0." in key for key in tensors)
        gate_up = tensors["mlp.experts.gate_up_proj"]
        down = tensors["mlp.experts.down_proj"]
        assert tuple(gate_up.shape) == (11, 6, 4)
        assert tuple(down.shape) == (11, 4, 3)
        for expert in range(11):
            base = 1000.0 * layer + 10.0 * expert
            assert torch.equal(gate_up[expert, :3], torch.full((3, 4), base + 1.0))
            assert torch.equal(gate_up[expert, 3:], torch.full((3, 4), base + 2.0))
            assert torch.equal(down[expert], torch.full((4, 3), base + 3.0))


def test_mtp_layers_past_num_hidden_layers_are_dropped(tmp_path):
    from soup_cli.utils.layer_shard import shard_checkpoint

    mtp = ("model.layers.2.enorm.weight", "model.layers.2.hnorm.weight")
    weights = _write_layer_checkpoint(
        tmp_path / "w", n_layers=2, n_experts=2, extra_layer_keys=mtp
    )
    index = shard_checkpoint(
        weights, str(tmp_path / "s"), dtype="float32", arch="deepseek_v3", num_hidden_layers=2
    )
    assert index.n_layers == 2


def test_mtp_control_without_the_layer_count_keeps_the_extra_layer(tmp_path):
    """The control for the test above: the extra layer is real input."""
    from soup_cli.utils.layer_shard import shard_checkpoint

    mtp = ("model.layers.2.enorm.weight", "model.layers.2.hnorm.weight")
    weights = _write_layer_checkpoint(
        tmp_path / "w", n_layers=2, n_experts=2, extra_layer_keys=mtp
    )
    index = shard_checkpoint(weights, str(tmp_path / "s"), dtype="float32", arch="deepseek_v3")
    assert index.n_layers == 3


@pytest.mark.parametrize("bad", [True, 0, -1, 2.0, "2", 100000])
def test_num_hidden_layers_is_validated(tmp_path, bad):
    from soup_cli.utils.layer_shard import shard_checkpoint

    weights = _write_layer_checkpoint(tmp_path / "w", n_layers=2, n_experts=2)
    with pytest.raises(ValueError, match="num_hidden_layers must be an int"):
        shard_checkpoint(
            weights, str(tmp_path / "s"), dtype="float32", arch="deepseek_v3",
            num_hidden_layers=bad,
        )


def test_router_correction_bias_stays_fp32_under_a_bf16_stream(tmp_path):
    """transformers keeps ``e_score_correction_bias`` in fp32 on load; a bf16
    shard would route with rounded values the resident model never sees."""
    import torch
    from safetensors.torch import load_file

    from soup_cli.utils.layer_shard import shard_checkpoint

    weights = _write_layer_checkpoint(tmp_path / "w", n_layers=2, n_experts=4)
    index = shard_checkpoint(
        weights, str(tmp_path / "s"), dtype="bfloat16", arch="deepseek_v3", num_hidden_layers=2
    )
    tensors = load_file(_layer_file(tmp_path / "s", index, 1))
    bias = tensors["mlp.gate.e_score_correction_bias"]
    assert bias.dtype == torch.float32
    assert torch.equal(bias, torch.tensor([0.1, 0.101, 0.102, 0.103], dtype=torch.float32))
    assert tensors["mlp.gate.weight"].dtype == torch.bfloat16
    assert tensors["mlp.experts.gate_up_proj"].dtype == torch.bfloat16


@pytest.mark.parametrize("fp8_key", ["weight", "scale"])
def test_fp8_checkpoint_is_refused_before_anything_is_written(tmp_path, fp8_key):
    """DeepSeek-V3 / Kimi K2 publish FP8 weights plus ``weight_scale_inv`` block
    scales. A cast would keep the raw e4m3 values and drop the scales, so an FP8
    tensor or a block scale is accepted only as that block format, declared by
    ``config.json`` (and then dequantised at shard time, see
    ``tests/test_fp8_block_streaming.py``). This checkpoint has no config, so either
    spelling is refused naming the supported format."""
    import torch
    from safetensors.torch import save_file

    from soup_cli.utils.layer_shard import shard_checkpoint

    weights = tmp_path / "w"
    weights.mkdir()
    state = {
        "model.embed_tokens.weight": torch.zeros(8, 4),
        "model.layers.0.input_layernorm.weight": torch.ones(4),
        "model.layers.0.self_attn.o_proj.weight": torch.zeros(4, 4),
    }
    if fp8_key == "weight":
        state["model.layers.0.self_attn.o_proj.weight"] = torch.zeros(4, 4).to(
            torch.float8_e4m3fn
        )
    else:
        state["model.layers.0.self_attn.o_proj.weight_scale_inv"] = torch.ones(1, 1)
    save_file(state, str(weights / "model.safetensors"))
    with pytest.raises(ValueError, match="Dequantise this checkpoint to bf16 first") as caught:
        shard_checkpoint(str(weights), str(tmp_path / "s"), dtype="bfloat16", arch="deepseek_v3")
    assert "declares no FP8 quantization_config" in str(caught.value)
    assert not (tmp_path / "s" / "index.json").exists()


def test_layer_buffers_load_from_the_shard_at_the_stored_dtype(tmp_path):
    """Covers each branch of `_load_layer_buffers` on a hand-built layer: a
    persistent buffer is replaced by the shard's (fp32 over a bf16 skeleton);
    a non-persistent one and one the shard lacks keep their init values; a
    LoRA-style ``.base_layer.`` segment maps to the checkpoint name."""
    import torch
    import torch.nn as nn
    from safetensors.torch import save_file

    from soup_cli.utils.layer_stream_runtime import _load_layer_buffers

    layer = nn.Module()
    layer.gate = nn.Module()
    layer.gate.register_buffer("bias", torch.zeros(3, dtype=torch.bfloat16))
    layer.gate.register_buffer("scratch", torch.zeros(3), persistent=False)
    layer.gate.register_buffer("absent", torch.full((2,), 5.0))
    layer.proj = nn.Module()
    layer.proj.base_layer = nn.Module()
    layer.proj.base_layer.register_buffer("table", torch.zeros(2))
    layer.proj.register_buffer("counts", torch.arange(4))
    shard = {
        "gate.bias": torch.tensor([0.1, 0.2, 0.3], dtype=torch.float32),
        "gate.scratch": torch.ones(3),
        "proj.table": torch.tensor([7.0, 8.0]),
        # An integer buffer the sharder cast to the stream dtype (qwen4_exp has
        # one): not listed, so it must keep the skeleton's own value and dtype.
        "proj.counts": torch.zeros(4),
    }
    path = str(tmp_path / "layer_000.safetensors")
    save_file(shard, path)
    specs = [{key: (tuple(value.shape), "x") for key, value in shard.items()}]
    keys = ("gate.bias", "gate.scratch", "gate.absent", "proj.table")

    assert _load_layer_buffers([layer], [path], specs, keys) == 2
    assert layer.gate.bias.dtype == torch.float32
    assert torch.equal(layer.gate.bias, shard["gate.bias"])
    assert torch.equal(layer.gate.scratch, torch.zeros(3))
    assert torch.equal(layer.gate.absent, torch.full((2,), 5.0))
    assert torch.equal(layer.proj.base_layer.table, shard["proj.table"])
    assert torch.equal(layer.proj.counts, torch.arange(4))


def test_only_deepseek_v3_streams_a_layer_buffer():
    from soup_cli.utils.layer_shard import layer_buffer_keys

    assert layer_buffer_keys("deepseek_v3") == ("mlp.gate.e_score_correction_bias",)
    for arch in ("llama", "qwen4_exp", "qwen3", ""):
        assert layer_buffer_keys(arch) == ()


def test_layer_buffer_shape_mismatch_names_the_buffer(tmp_path):
    import torch
    import torch.nn as nn
    from safetensors.torch import save_file

    from soup_cli.utils.layer_stream_runtime import _load_layer_buffers

    layer = nn.Module()
    layer.register_buffer("bias", torch.zeros(4))
    path = str(tmp_path / "layer_000.safetensors")
    save_file({"bias": torch.zeros(3)}, path)
    with pytest.raises(ValueError, match="buffer 'bias' has shape"):
        _load_layer_buffers([layer], [path], [{"bias": ((3,), "F32")}], ("bias",))


def _deepseek_setup_cfg():
    return SimpleNamespace(
        model_type="deepseek_v3",
        hidden_size=4,
        num_hidden_layers=1,
        vocab_size=8,
        intermediate_size=8,
    )


@pytest.mark.parametrize(
    ("arch", "expected"), [("deepseek_v3", 1), ("llama", None)], ids=["deepseek_v3", "llama"]
)
def test_setup_hands_the_config_layer_count_to_the_sharder_for_deepseek_v3_only(
    tmp_path, monkeypatch, arch, expected
):
    """Without it a real DeepSeek-V3 (61 layers + the MTP layer 61) shards 62
    layers and the install refuses with "model has 61 decoder layers"."""
    from tests.test_issue602_qwen4_ple_streaming import _drive_qwen4_streaming_setup

    calls = []
    model_cfg = _deepseek_setup_cfg()
    model_cfg.model_type = arch
    _drive_qwen4_streaming_setup(
        tmp_path, monkeypatch, arch=arch, model_cfg=model_cfg, shard_calls=calls
    )
    assert len(calls) == 1
    assert calls[0]["num_hidden_layers"] == expected


def test_streamed_model_is_built_from_native_code_while_the_tokenizer_trusts_remote(
    tmp_path, monkeypatch
):
    """Moonlight-16B (deepseek_v3) ships its tokenizer only as remote code, and its
    remote modelling code keeps one module per expert, which the stacked shards do
    not fit. So the tokenizer gets the user's trust_remote_code and the model side
    (config, skeleton, install) never does."""
    from tests.test_issue602_qwen4_ple_streaming import _drive_qwen4_streaming_setup

    calls = {}
    _drive_qwen4_streaming_setup(
        tmp_path, monkeypatch, arch="deepseek_v3", model_cfg=_deepseek_setup_cfg(),
        load_calls=calls, trust_remote_code=True,
    )
    assert [call["trust_remote_code"] for call in calls["tokenizer"]] == [True]
    assert [call["trust_remote_code"] for call in calls["config"]] == [False]
    assert [call["trust_remote_code"] for call in calls["build"]] == [False]


def test_setup_meta_probe_is_built_from_native_code(monkeypatch):
    """The NF4 / moe_lora probe in the setup: the same rule as the model itself."""
    import soup_cli.utils.layer_stream_runtime as runtime
    from soup_cli.trainer.stream_setup import StreamingSetupMixin

    seen = []
    monkeypatch.setattr(runtime, "build_meta_skeleton", lambda *a, **k: seen.append(k) or "probe")
    probe = StreamingSetupMixin._stream_meta_skeleton("base", dtype="bfloat16", quant="nf4")
    assert probe == "probe"
    assert seen == [{"dtype": "bfloat16", "quant": "nf4", "trust_remote_code": False}]


def test_planner_fingerprints_config_json_for_deepseek_v3(tmp_path, monkeypatch):
    """The planner must agree with the sharder (`source_identity_includes_config`),
    or it reports a stale cache as reusable and the disk pre-flight under-counts."""
    import types

    import torch
    from safetensors.torch import save_file

    import soup_cli.utils.layer_shard as layer_shard
    from tests.test_issue602_qwen4_ple_streaming import _drive_qwen4_streaming_setup

    seen = []
    real = layer_shard.checkpoint_source_components

    def _recording(weights_dir, source_files, *, include_config):
        seen.append(include_config)
        return real(weights_dir, source_files, include_config=include_config)

    monkeypatch.setattr(layer_shard, "checkpoint_source_components", _recording)
    weights = tmp_path / "weights"
    weights.mkdir()
    blob = weights / "model.safetensors"
    save_file({"model.layers.0.proj.weight": torch.ones(2, 2)}, str(blob))
    (weights / "config.json").write_text('{"model_type": "deepseek_v3"}', encoding="utf-8")
    stat = blob.stat()
    plan = types.SimpleNamespace(
        weights_dir=str(weights),
        source_bytes=stat.st_size,
        materialized_copy_bytes=0,
        materialize_bytes=0,
        needs_materialization=False,
        source_files=((str(blob), stat.st_size, int(stat.st_mtime)),),
    )

    def _resolve(_base, *, before_materialize=None):
        if before_materialize is not None:
            before_materialize(plan)
        return str(weights)

    _drive_qwen4_streaming_setup(
        tmp_path, monkeypatch, resolve_weights=_resolve, arch="deepseek_v3",
        model_cfg=_deepseek_setup_cfg(),
    )
    assert seen == [True], seen


def test_config_is_part_of_the_cache_identity_for_deepseek_v3():
    from soup_cli.utils.layer_shard import source_identity_includes_config

    assert source_identity_includes_config("deepseek_v3") is True
    assert source_identity_includes_config("qwen4_exp") is True
    assert source_identity_includes_config("llama") is False


# --------------------------------------------------------------------------
# resident vs streamed parity on a real tiny DeepseekV3ForCausalLM (CPU fp32)
# --------------------------------------------------------------------------
def _tiny_deepseek_v3_dir(tmp_path, *, q_lora_rank):
    import torch
    from transformers import DeepseekV3Config, DeepseekV3ForCausalLM

    torch.manual_seed(11)
    config = DeepseekV3Config(
        vocab_size=128,
        hidden_size=64,
        intermediate_size=128,
        moe_intermediate_size=32,
        num_hidden_layers=3,
        first_k_dense_replace=1,
        num_attention_heads=4,
        num_key_value_heads=4,
        n_shared_experts=1,
        n_routed_experts=8,
        n_group=2,
        topk_group=1,
        num_experts_per_tok=2,
        kv_lora_rank=16,
        q_lora_rank=q_lora_rank,
        qk_nope_head_dim=16,
        qk_rope_head_dim=8,
        v_head_dim=16,
        max_position_embeddings=128,
    )
    model = DeepseekV3ForCausalLM(config).to(torch.float32).eval()
    generator = torch.Generator().manual_seed(23)
    with torch.no_grad():
        for layer in model.model.layers[config.first_k_dense_replace :]:
            # Large enough to change which experts win: the streamed run must
            # load it from the shard, not keep the skeleton's zeros.
            bias = layer.mlp.gate.e_score_correction_bias
            bias.copy_(torch.randn(bias.shape, generator=generator) * 2.0)
    weights = tmp_path / "model"
    model.save_pretrained(str(weights))
    return str(weights), model


def _tiny_lora():
    from peft import LoraConfig, TaskType

    return LoraConfig(
        r=4, lora_alpha=8, lora_dropout=0.0, bias="none",
        target_modules=_LORA_TARGETS, task_type=TaskType.CAUSAL_LM,
    )


def _build(tmp_path, *, q_lora_rank):
    from soup_cli.utils.layer_shard import shard_checkpoint
    from soup_cli.utils.layer_stream_runtime import build_streamed_model

    weights, resident = _tiny_deepseek_v3_dir(tmp_path, q_lora_rank=q_lora_rank)
    shards = str(tmp_path / "shards")
    index = shard_checkpoint(
        weights, shards, dtype="float32", arch="deepseek_v3", num_hidden_layers=3
    )
    model, runtime = build_streamed_model(
        model_id=weights, shard_dir=shards, index=index,
        lora_config=_tiny_lora(), device="cpu", dtype="float32",
        buffers=2, pin=False, seed=3,
    )
    return model, runtime, resident


def _copy_lora(src, dst):
    def norm(key):
        return key.replace(".inner.", ".")

    src_lora = {norm(k): v for k, v in src.state_dict().items() if "lora_" in k}
    dst_lora = {norm(k): v for k, v in dst.state_dict().items() if "lora_" in k}
    assert src_lora and set(src_lora) == set(dst_lora)
    for key, val in src_lora.items():
        dst_lora[key].copy_(val)


def _paired(tmp_path, q_lora_rank):
    import torch
    from peft import get_peft_model

    model, runtime, resident = _build(tmp_path, q_lora_rank=q_lora_rank)
    ref = get_peft_model(resident, _tiny_lora())
    generator = torch.Generator().manual_seed(17)
    with torch.no_grad():
        for name, parameter in model.named_parameters():
            if "lora_B" in name:
                parameter.copy_(torch.randn(parameter.shape, generator=generator) * 0.02)
    _copy_lora(model, ref)
    model.eval()
    ref.eval()
    ids = torch.randint(0, 128, (2, 12), generator=generator)
    return model, runtime, ref, ids


@pytest.mark.parametrize("q_lora_rank", [24, None], ids=["q_lora", "q_proj"])
def test_streamed_forward_matches_resident_bit_exactly(tmp_path, q_lora_rank):
    import torch

    model, runtime, ref, ids = _paired(tmp_path, q_lora_rank)
    with torch.no_grad():
        got = model(input_ids=ids, labels=ids)
        want = ref(input_ids=ids, labels=ids)
    assert torch.equal(got.logits, want.logits), (got.logits - want.logits).abs().max().item()
    assert torch.equal(got.loss, want.loss)
    runtime.close()


@pytest.mark.parametrize("q_lora_rank", [24, None], ids=["q_lora", "q_proj"])
def test_streamed_backward_matches_resident_lora_gradients_bit_exactly(tmp_path, q_lora_rank):
    import torch

    from soup_cli.utils.layer_stream_runtime import canonical_named_parameters

    model, runtime, ref, ids = _paired(tmp_path, q_lora_rank)
    model.zero_grad(set_to_none=True)
    ref.zero_grad(set_to_none=True)
    model(input_ids=ids, labels=ids).loss.backward()
    ref(input_ids=ids, labels=ids).loss.backward()

    def grads(net):
        return {
            name: parameter.grad
            for name, parameter in canonical_named_parameters(net)
            if "lora_" in name and parameter.grad is not None
        }

    streamed, resident = grads(model), grads(ref)
    assert streamed and set(streamed) == set(resident)
    # Every MoE layer's attention adapters receive gradient through the
    # streamed experts of the layers above them.
    assert any(".layers.0." in name for name in streamed)
    assert any(float(gradient.abs().sum()) > 0.0 for gradient in streamed.values())
    for name, gradient in streamed.items():
        assert torch.equal(gradient, resident[name]), name
    runtime.close()


def test_router_bias_is_loaded_and_the_tiny_model_depends_on_it(tmp_path):
    """Makes the parity tests above sensitive to the bias: a zero bias changes
    the logits of this model, and the streamed layers hold the checkpoint's."""
    import torch

    model, runtime, ref, ids = _paired(tmp_path, 24)
    base = ref.get_base_model()
    streamed_layers = model.get_base_model().model.layers
    for idx in range(1, 3):
        want = base.model.layers[idx].mlp.gate.e_score_correction_bias
        got = streamed_layers[idx].inner.mlp.gate.e_score_correction_bias
        assert float(want.abs().sum()) > 0.0
        assert got.dtype == want.dtype
        assert torch.equal(got, want)
    with torch.no_grad():
        before = ref(input_ids=ids).logits
        for idx in range(1, 3):
            base.model.layers[idx].mlp.gate.e_score_correction_bias.zero_()
        after = ref(input_ids=ids).logits
    assert not torch.equal(before, after)
    runtime.close()