Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix beam search on the fast decode path (#13053)

Daniel Han5ff0e3487b2f
diff --git a/tests/test_beam_search_reorder_cache.py b/tests/test_beam_search_reorder_cache.py
new file mode 100644
index 00000000000..d84156eb8d9
--- /dev/null
+++ b/tests/test_beam_search_reorder_cache.py
@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""CPU checks for beam search on the fast decode path (issue #1099), against unsloth/models/llama.py."""
+
+import types
+
+import torch
+from transformers.cache_utils import DynamicCache
+
+BEAMS, HEADS, DIM, SLACK = 3, 2, 4, 5
+
+
+def _llama():
+ from unsloth.models import llama
+ return llama
+
+
+def _attention_with_buffer(seq_len):
+ # Same layout LlamaAttention_fast_forward_inference allocates: (seq, 2, bsz, kv_heads, head_dim).
+ attn = types.SimpleNamespace()
+ attn.paged_attention = torch.randn(seq_len + SLACK, 2, BEAMS, HEADS, DIM)
+ attn.paged_attention_K = attn.paged_attention[:, 0]
+ attn.paged_attention_V = attn.paged_attention[:, 1]
+ K = attn.paged_attention_K[:seq_len].permute(1, 2, 0, 3)
+ V = attn.paged_attention_V[:seq_len].permute(1, 2, 0, 3)
+ return attn, (K, V)
+
+
+def _model(attentions):
+ layers = [types.SimpleNamespace(self_attn = attn) for attn in attentions]
+ return types.SimpleNamespace(model = types.SimpleNamespace(layers = layers))
+
+
+def test_fast_forward_classes_get_reorder_cache():
+ llama = _llama()
+
+ class Fake:
+ def prepare_inputs_for_generation(self):
+ pass
+
+ llama.fix_prepare_inputs_for_generation(Fake)
+ assert Fake._reorder_cache is llama._fast_reorder_cache
+
+
+def test_decode_buffers_are_reordered_in_place():
+ seq_len = 6
+ attentions, past = zip(*(_attention_with_buffer(seq_len) for _ in range(2)))
+ before = [a.paged_attention.clone() for a in attentions]
+ beam_idx = torch.tensor([2, 2, 0])
+
+ out = _llama()._fast_reorder_cache(_model(attentions), list(past), beam_idx)
+
+ for attn, old, (K, V), (K_out, V_out) in zip(attentions, before, past, out):
+ # The next decode step reads the buffer, not the returned tuple.
+ assert torch.equal(attn.paged_attention[:seq_len], old[:seq_len, :, beam_idx])
+ assert torch.equal(attn.paged_attention[seq_len:], old[seq_len:])
+ assert K_out is K and V_out is V
+ assert torch.equal(K_out, old[:seq_len, 0, beam_idx].permute(1, 2, 0, 3))
+
+
+def test_prefill_tuples_without_buffers_are_index_selected():
+ past = [(torch.randn(BEAMS, HEADS, 4, DIM), torch.randn(BEAMS, HEADS, 4, DIM))]
+ attn = types.SimpleNamespace() # Buffers are dropped by the prefill forward.
+ beam_idx = torch.tensor([1, 0, 0])
+
+ out = _llama()._fast_reorder_cache(_model([attn]), past, beam_idx)
+
+ assert torch.equal(out[0][0], past[0][0][beam_idx])
+ assert torch.equal(out[0][1], past[0][1][beam_idx])
+
+
+def test_cache_objects_use_their_own_reorder():
+ cache = DynamicCache()
+ keys = torch.randn(BEAMS, HEADS, 4, DIM)
+ cache.update(keys, keys.clone(), 0)
+ beam_idx = torch.tensor([2, 1, 1])
+
+ out = _llama()._fast_reorder_cache(_model([]), cache, beam_idx)
+
+ assert out is cache
+ assert torch.equal(cache.layers[0].keys, keys[beam_idx])
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index ba07f05caf8..4005d7400bc 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -466,9 +466,36 @@ def _fast_prepare_inputs_for_generation(
return result
+def _fast_reorder_cache(self, past_key_values, beam_idx):
+ # Beam search (#1099). The fast decode path reads K/V from each attention's paged buffer, so
+ # reordering only the returned tuples would silently mix beams: reorder the buffer in place.
+ if isinstance(past_key_values, Cache):
+ past_key_values.reorder_cache(beam_idx)
+ return past_key_values
+ layers = getattr(getattr(self, "model", None), "layers", None) or ()
+ attentions = [getattr(layer, "self_attn", None) for layer in layers]
+ reordered = []
+ for idx, (K, V) in enumerate(past_key_values):
+ attn = attentions[idx] if idx < len(attentions) else None
+ buffer = getattr(attn, "paged_attention", None)
+ if (
+ buffer is not None
+ and K.untyped_storage().data_ptr() == buffer.untyped_storage().data_ptr()
+ ):
+ n = K.shape[-2]
+ buffer[:n] = buffer[:n].index_select(2, beam_idx.to(buffer.device))
+ reordered.append((K, V))
+ else:
+ reordered.append(
+ (K.index_select(0, beam_idx.to(K.device)), V.index_select(0, beam_idx.to(V.device)))
+ )
+ return reordered
+
+
def fix_prepare_inputs_for_generation(module):
if hasattr(module, "prepare_inputs_for_generation"):
module.prepare_inputs_for_generation = _fast_prepare_inputs_for_generation
+ module._reorder_cache = _fast_reorder_cache
torch_matmul = torch.matmul