Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix: Support past_key_values in model.generate for multi-turn conversations (#4232)

Daniel Han0dae825e6963
diff --git a/scripts/kv_cache_multiturn_benchmark.py b/scripts/kv_cache_multiturn_benchmark.py
new file mode 100644
index 00000000000..4d767830822
--- /dev/null
+++ b/scripts/kv_cache_multiturn_benchmark.py
@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Multi-turn generate(): re-encode the whole conversation vs continue from a KV cache of the history.
+
+python scripts/kv_cache_multiturn_benchmark.py --model unsloth/Llama-3.2-1B-Instruct --turns 4 8 16 32
+"""
+
+import argparse
+import time
+
+from unsloth import FastLanguageModel
+import torch
+
+
+def timed(fn, runs):
+ times = []
+ for _ in range(runs):
+ torch.cuda.synchronize()
+ t0 = time.perf_counter()
+ out = fn()
+ torch.cuda.synchronize()
+ times.append(time.perf_counter() - t0)
+ return out, sorted(times)[len(times) // 2]
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--model", default = "unsloth/Llama-3.2-1B-Instruct")
+ parser.add_argument("--turns", type = int, nargs = "+", default = [4, 8, 16, 32])
+ parser.add_argument("--max-new-tokens", type = int, default = 32)
+ parser.add_argument("--runs", type = int, default = 5)
+ parser.add_argument("--load-in-4bit", action = "store_true")
+ args = parser.parse_args()
+
+ model, tokenizer = FastLanguageModel.from_pretrained(
+ args.model, max_seq_length = 8192, load_in_4bit = args.load_in_4bit
+ )
+ FastLanguageModel.for_inference(model)
+ gen = dict(
+ max_new_tokens = args.max_new_tokens, min_new_tokens = args.max_new_tokens, do_sample = False
+ )
+
+ print(
+ "| turns | history tok | new tok | full re-encode s | from cache s | speedup | tokens match |"
+ )
+ print("|---:|---:|---:|---:|---:|---:|:---:|")
+ for turns in args.turns:
+ history = []
+ for i in range(turns):
+ history.append(
+ {
+ "role": "user",
+ "content": f"Fact {i}: item {i} costs {3 * i + 7} dollars. Please remember it.",
+ }
+ )
+ history.append(
+ {"role": "assistant", "content": f"Noted: item {i} costs {3 * i + 7} dollars."}
+ )
+ question = [{"role": "user", "content": "What does item 2 cost?"}]
+ text_h = tokenizer.apply_chat_template(history, tokenize = False)
+ text_f = tokenizer.apply_chat_template(
+ history + question, tokenize = False, add_generation_prompt = True
+ )
+ enc = lambda t: tokenizer(t, return_tensors = "pt", add_special_tokens = False).to("cuda")
+ hist, full = enc(text_h), enc(text_f)
+ n, L = hist.input_ids.shape[1], full.input_ids.shape[1]
+ if not torch.equal(full.input_ids[:, :n], hist.input_ids):
+ raise SystemExit("chat template does not keep the history as a token prefix")
+ with torch.no_grad():
+ cache = model(**hist, use_cache = True).past_key_values
+
+ model.generate(**full, max_new_tokens = 2)
+ model.generate(**full, past_key_values = cache, max_new_tokens = 2)
+ base, t_base = timed(lambda: model.generate(**full, **gen), args.runs)
+ kv, t_kv = timed(lambda: model.generate(**full, past_key_values = cache, **gen), args.runs)
+ match = torch.equal(base[0, L:], kv[0, L:])
+ print(
+ f"| {turns} | {n} | {L - n} | {t_base:.4f} | {t_kv:.4f} | {t_base / t_kv:.2f}x | {'YES' if match else 'NO'} |"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_past_kv_models.py b/tests/test_past_kv_models.py
new file mode 100644
index 00000000000..a2c20a8e8be
--- /dev/null
+++ b/tests/test_past_kv_models.py
@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""GPU check: generate() continuing from a user KV cache of the conversation history (issue #497)
+must emit the same greedy tokens as re-encoding the whole conversation."""
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA GPU")
+
+MODELS = [
+ "trl-internal-testing/tiny-LlamaForCausalLM-3.2",
+ "trl-internal-testing/tiny-Qwen3ForCausalLM",
+ "hf-internal-testing/tiny-random-MistralForCausalLM",
+ "trl-internal-testing/tiny-Gemma2ForCausalLM",
+]
+HISTORY = (
+ "user: My name is Zorblat and my favourite colour is teal.\nassistant: Nice to meet you!\n"
+)
+OTHER = "user: I live near the mountains and I own two small cats.\nassistant: How lovely!\n" * 3
+QUESTION = "user: What is my name?\nassistant:"
+GEN = dict(max_new_tokens = 8, min_new_tokens = 8, do_sample = False)
+
+
[email protected](scope = "module", params = MODELS)
+def model_and_tokenizer(request):
+ from unsloth import FastLanguageModel
+
+ model, tokenizer = FastLanguageModel.from_pretrained(
+ request.param, max_seq_length = 512, load_in_4bit = False, dtype = torch.bfloat16
+ )
+ FastLanguageModel.for_inference(model)
+ yield model, tokenizer
+ del model
+ torch.cuda.empty_cache()
+
+
+def _ids(tokenizer, text):
+ return tokenizer(text, return_tensors = "pt", add_special_tokens = False).to("cuda")
+
+
+def _cache(model, ids):
+ with torch.no_grad():
+ return model(**ids, use_cache = True).past_key_values
+
+
+def _as_tuple(cache):
+ if hasattr(cache, "layers"):
+ return tuple((layer.keys, layer.values) for layer in cache.layers)
+ return tuple((k, v) for k, v in cache)
+
+
+def _generate(model, full, **kwargs):
+ out = model.generate(**full, **GEN, output_logits = True, return_dict_in_generate = True, **kwargs)
+ return out.sequences[0, full.input_ids.shape[1] :], out.logits[0][0].float()
+
+
+def test_generate_from_history_cache_matches_full_prompt(model_and_tokenizer):
+ model, tokenizer = model_and_tokenizer
+ history, full = _ids(tokenizer, HISTORY), _ids(tokenizer, HISTORY + QUESTION)
+ n = history.input_ids.shape[1]
+ assert torch.equal(full.input_ids[:, :n], history.input_ids)
+
+ expected_tokens, expected_logits = _generate(model, full)
+ tolerance = 0.02 * expected_logits.abs().max()
+ cache = _cache(model, history)
+ runs = [
+ _generate(model, full, past_key_values = past) for past in (cache, _as_tuple(cache), cache)
+ ]
+ tokens, logits = runs[0]
+ # Chunked vs full prefill differs only by bf16 rounding; tiny random models have 1-ulp logit ties.
+ assert (logits - expected_logits).abs().max() <= tolerance
+ assert tokens[0] == expected_tokens[0]
+ for other_tokens, other_logits in runs[1:]:
+ assert torch.equal(other_tokens, tokens) and torch.equal(other_logits, logits)
+
+ other = _ids(tokenizer, OTHER)
+ wrong = {k: v[:, :n] for k, v in other.items()}
+ assert wrong["input_ids"].shape[1] == n
+ _, wrong_logits = _generate(model, full, past_key_values = _cache(model, wrong))
+ assert (wrong_logits - expected_logits).abs().max() > 5 * tolerance
+
+
[email protected]("attention_mask", [False, True])
+def test_direct_forward_onto_cache_continues_after_it(model_and_tokenizer, attention_mask):
+ model, tokenizer = model_and_tokenizer
+ full = _ids(tokenizer, HISTORY + QUESTION)
+ n = _ids(tokenizer, HISTORY).input_ids.shape[1]
+ with torch.no_grad():
+ expected = model(full.input_ids).logits[0, n:].float()
+ cache = _cache(model, {"input_ids": full.input_ids[:, :n]})
+ kwargs = {"attention_mask": full.attention_mask} if attention_mask else {}
+ got = model(full.input_ids[:, n:], past_key_values = cache, **kwargs).logits[0].float()
+ assert (got - expected).abs().max() <= 0.02 * expected.abs().max()
+
+
+def test_saved_cache_reused_after_another_generate(model_and_tokenizer):
+ model, tokenizer = model_and_tokenizer
+ full = _ids(tokenizer, HISTORY + QUESTION)
+ cache = _cache(model, {"input_ids": full.input_ids[:, :-1]})
+ expected_tokens, expected_logits = _generate(model, full, past_key_values = cache)
+ model.generate(**_ids(tokenizer, OTHER), max_new_tokens = 8)
+ tokens, logits = _generate(model, full, past_key_values = cache)
+ assert torch.equal(tokens, expected_tokens) and torch.equal(logits, expected_logits)
diff --git a/tests/test_past_kv_utils.py b/tests/test_past_kv_utils.py
new file mode 100644
index 00000000000..6c146c90032
--- /dev/null
+++ b/tests/test_past_kv_utils.py
@@ -0,0 +1,147 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""CPU checks for user-supplied past_key_values in generate() (issue #497), against unsloth/models/llama.py."""
+
+import pytest
+import torch
+from transformers.cache_utils import Cache, DynamicCache
+
+BS, PAST_LEN, SEQ = 2, 3, 5
+
+
+class FakeModel:
+ dtype = torch.float32
+ config = None
+
+
+def _llama():
+ from unsloth.models import llama
+ return llama
+
+
+def _kv(length, layers = 2):
+ return tuple(
+ (torch.randn(BS, 1, length, 4), torch.randn(BS, 1, length, 4)) for _ in range(layers)
+ )
+
+
+def _prepare(input_ids, **kwargs):
+ return _llama()._fast_prepare_inputs_for_generation(FakeModel(), input_ids, **kwargs)
+
+
+def test_tuple_cache_becomes_dynamic_cache_for_generate():
+ legacy = _kv(PAST_LEN)
+ cache = _llama()._ensure_cache_is_dynamic(legacy)
+ assert isinstance(cache, Cache)
+ assert cache.get_seq_length() == PAST_LEN
+ assert _llama()._ensure_cache_is_dynamic(cache) is cache
+ assert _llama()._ensure_cache_is_dynamic(None) is None
+
+
[email protected]("as_dynamic", [False, True])
+def test_prefill_onto_partial_cache_feeds_every_uncached_token(as_dynamic):
+ legacy = _kv(PAST_LEN)
+ past = _llama()._ensure_cache_is_dynamic(legacy) if as_dynamic else legacy
+ input_ids = torch.arange(BS * SEQ).reshape(BS, SEQ)
+ mask = torch.ones(BS, SEQ, dtype = torch.long)
+ result = _prepare(input_ids, attention_mask = mask, past_key_values = past)
+
+ assert torch.equal(result["input_ids"], input_ids[:, PAST_LEN:])
+ assert result["position_ids"].tolist() == [[3, 4]] * BS
+ assert result["cache_position"].tolist() == [3, 4]
+ out = result["past_key_values"]
+ assert isinstance(out, tuple) and len(out) == len(legacy)
+ for (k, v), (k0, v0) in zip(out, legacy):
+ assert torch.equal(k, k0) and torch.equal(v, v0)
+
+
+def test_decode_step_still_feeds_only_the_last_token():
+ input_ids = torch.arange(BS * SEQ).reshape(BS, SEQ)
+ result = _prepare(input_ids, past_key_values = _kv(SEQ - 1))
+ assert torch.equal(result["input_ids"], input_ids[:, [-1]])
+
+
+def test_full_length_user_position_ids_are_sliced_to_the_fed_tokens():
+ input_ids = torch.arange(BS * SEQ).reshape(BS, SEQ)
+ pos = torch.arange(SEQ).expand(BS, -1) + 10
+ result = _prepare(input_ids, past_key_values = _kv(PAST_LEN), position_ids = pos)
+ assert result["position_ids"].tolist() == [[13, 14]] * BS
+
+ result = _prepare(input_ids, position_ids = pos[0])
+ assert torch.equal(result["position_ids"], pos[0])
+
+
+def test_user_position_ids_with_inputs_embeds_and_no_input_ids():
+ embeds = torch.randn(BS, SEQ, 4)
+ pos = torch.arange(SEQ).expand(BS, -1)
+ result = _prepare(None, inputs_embeds = embeds, position_ids = pos)
+ assert torch.equal(result["position_ids"], pos)
+ assert result["inputs_embeds"] is embeds
+
+
+def test_empty_dynamic_cache_is_dropped_before_prefill():
+ input_ids = torch.arange(BS * SEQ).reshape(BS, SEQ)
+ result = _prepare(input_ids, past_key_values = DynamicCache())
+ assert result["past_key_values"] is None
+ assert result["input_ids"].shape == (BS, SEQ)
+
+
+def test_suffix_only_input_with_full_mask_feeds_the_whole_new_turn():
+ # transformers 5 generate() accepts only the new tokens when attention_mask spans cache + new.
+ new = torch.arange(BS * 2).reshape(BS, 2)
+ mask = torch.ones(BS, PAST_LEN + 2, dtype = torch.long)
+ result = _prepare(new, attention_mask = mask, past_key_values = _kv(PAST_LEN))
+ assert torch.equal(result["input_ids"], new)
+ assert result["position_ids"].tolist() == [[3, 4]] * BS
+
+
+def test_static_cache_is_flattened_to_its_filled_length():
+ from transformers import LlamaConfig
+ from transformers.cache_utils import StaticCache
+
+ config = LlamaConfig(
+ num_hidden_layers = 2, num_attention_heads = 1, num_key_value_heads = 1, hidden_size = 4
+ )
+ try:
+ cache = StaticCache(config = config, max_cache_len = 8)
+ except TypeError:
+ pytest.skip("StaticCache signature differs on this transformers")
+ legacy = _kv(PAST_LEN)
+ for i, (k, v) in enumerate(legacy):
+ cache.update(k, v, i, {"cache_position": torch.arange(PAST_LEN)})
+ out = _llama()._cache_as_legacy_tuple(cache)
+ for (k, v), (k0, v0) in zip(out, legacy):
+ assert torch.equal(k, k0) and torch.equal(v, v0)
+
+
+def test_cache_layers_missing_positions_are_rejected():
+ cache = _llama()._ensure_cache_is_dynamic(_kv(PAST_LEN))
+ # A QuantizedLayer counts every token but keeps only the unquantized tail in .keys.
+ cache.layers[0].keys = cache.layers[0].keys[..., 1:, :]
+ cache.get_seq_length = lambda layer_idx = 0: PAST_LEN
+ with pytest.raises(ValueError, match = "does not keep every cached position"):
+ _llama()._cache_as_legacy_tuple(cache)
+
+
+def test_partial_static_cache_mask_spans_the_flattened_length():
+ from transformers import LlamaConfig
+ from transformers.cache_utils import StaticCache
+ from tests.utils.test_prepare_inputs_leftpad import FakeModelWith4DMask
+
+ config = LlamaConfig(
+ num_hidden_layers = 2, num_attention_heads = 1, num_key_value_heads = 1, hidden_size = 4
+ )
+ try:
+ cache = StaticCache(config = config, max_cache_len = 16)
+ except TypeError:
+ pytest.skip("StaticCache signature differs on this transformers")
+ for i, (k, v) in enumerate(_kv(PAST_LEN)):
+ cache.update(k, v, i, {"cache_position": torch.arange(PAST_LEN)})
+ model = FakeModelWith4DMask()
+ input_ids = torch.arange(BS * SEQ).reshape(BS, SEQ)
+ mask = torch.ones(BS, SEQ, dtype = torch.long)
+ _llama()._fast_prepare_inputs_for_generation(
+ model, input_ids, attention_mask = mask, past_key_values = cache
+ )
+ assert model.mask_calls[0]["target_length"] == SEQ # past_len + fed tokens, not max_cache_len
diff --git a/unsloth/kernels/flex_attention.py b/unsloth/kernels/flex_attention.py
index 0384ec55512..cf7fe8f20db 100644
--- a/unsloth/kernels/flex_attention.py
+++ b/unsloth/kernels/flex_attention.py
@@ -37,18 +37,18 @@ except:
if not HAS_FLEX_ATTENTION:
# Logit softcapping
@torch.compile(fullgraph = True, dynamic = True, options = torch_compile_options)
- def _compiled_slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len):
+ def _compiled_slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, kv_len):
n_heads = self.config.num_attention_heads
head_dim = self.head_dim
n_kv_heads = self.config.num_key_value_heads
n_groups = self.num_key_value_groups
+ actual_q_len = Q.shape[-2]
# Grouped query attention
- # Grouped query attention
- K = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)
- V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)
- K = K.reshape(bsz, n_heads, q_len, head_dim)
- V = V.reshape(bsz, n_heads, q_len, head_dim)
+ K = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, kv_len, head_dim)
+ V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, kv_len, head_dim)
+ K = K.reshape(bsz, n_heads, kv_len, head_dim)
+ V = V.reshape(bsz, n_heads, kv_len, head_dim)
# Gemma 9b should use 256, not hidden_size // num_attention_heads (224); 27b uses the derived value,
# so default to the config. See google/gemma_pytorch commit 03e6575.
@@ -58,12 +58,15 @@ if not HAS_FLEX_ATTENTION:
Q = Q * torch.tensor(s**-0.5, dtype = Q.dtype)
A = torch.matmul(Q, K.transpose(2, 3))
A = t * torch.tanh(A / t)
- A += causal_mask[..., :q_len, :q_len]
- # Much slower under torch compile than the masked_fill_ it replaces.
+ # 2D static masks index absolute positions; 4D masks are already per query row.
+ if causal_mask.dim() == 2:
+ A += causal_mask[kv_len - actual_q_len : kv_len, :kv_len]
+ else:
+ A += causal_mask[..., :actual_q_len, :kv_len]
A = torch.nn.functional.softmax(A, dim = -1, dtype = torch.float32).to(Q.dtype)
A = torch.matmul(A, V)
A = A.transpose(1, 2).contiguous()
- A = A.reshape(bsz, q_len, n_heads * head_dim)
+ A = A.reshape(bsz, actual_q_len, n_heads * head_dim)
return A
_SOFTCAP_EAGER = {}
@@ -180,16 +183,17 @@ torch_tanh = torch.tanh
torch_nn_functional_softmax = torch.nn.functional.softmax
-def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len):
+def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, kv_len):
n_heads = self.config.num_attention_heads
head_dim = self.head_dim
n_kv_heads = self.config.num_key_value_heads
n_groups = self.num_key_value_groups
+ actual_q_len = Q.shape[-2]
- K = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)
- V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)
- K = K.reshape(bsz, n_heads, q_len, head_dim)
- V = V.reshape(bsz, n_heads, q_len, head_dim)
+ K = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, kv_len, head_dim)
+ V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, kv_len, head_dim)
+ K = K.reshape(bsz, n_heads, kv_len, head_dim)
+ V = V.reshape(bsz, n_heads, kv_len, head_dim)
# Gemma 9b should use 256, not hidden_size // num_attention_heads (224); 27b uses the derived value,
# so default to the config. See google/gemma_pytorch commit 03e6575.
@@ -203,10 +207,12 @@ def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len)
A /= t
torch_tanh(A, out = A)
A *= t
- A += causal_mask[..., :q_len, :q_len]
- # Much slower under torch compile than the masked_fill_ it replaces.
+ if causal_mask.dim() == 2:
+ A += causal_mask[kv_len - actual_q_len : kv_len, :kv_len]
+ else:
+ A += causal_mask[..., :actual_q_len, :kv_len]
A = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32).to(Q.dtype)
A = torch_matmul(A, V)
A = A.transpose(1, 2).contiguous()
- A = A.reshape(bsz, q_len, n_heads * head_dim)
+ A = A.reshape(bsz, actual_q_len, n_heads * head_dim)
return A
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 70e355d5616..02e73f90c94 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -108,6 +108,7 @@ from unsloth.models._attn_mask_compat import (
AttentionMaskConverter,
_prepare_4d_causal_attention_mask_for_sdpa,
)
+from transformers.cache_utils import DynamicCache, Cache
from ..kernels import *
from ..kernels.utils import has_mxfp4_base
from ..tokenizer_utils import *
@@ -236,6 +237,63 @@ def _offload_frozen_module_for_training(
module.original_module.requires_grad_(False)
+def _ensure_cache_is_dynamic(past_key_values):
+ # transformers 5 rejects legacy tuple caches in generate().
+ if past_key_values is None:
+ return None
+ if isinstance(past_key_values, Cache):
+ return past_key_values
+ if isinstance(past_key_values, (tuple, list)) and len(past_key_values) > 0:
+ cache = DynamicCache()
+ for layer_idx, layer_kv in enumerate(past_key_values):
+ cache.update(layer_kv[0], layer_kv[1], layer_idx)
+ return cache
+ return past_key_values
+
+
+def _cache_as_legacy_tuple(past_key_values):
+ # Unsloth's forwards index past_key_values[layer][0|1]; transformers 5 caches are not subscriptable.
+ if not isinstance(past_key_values, Cache):
+ return past_key_values
+ past_len = past_key_values.get_seq_length()
+ if past_len == 0:
+ return None
+ layers = getattr(past_key_values, "layers", None)
+ if layers is None:
+ return past_key_values.to_legacy_cache()
+ # Static layers allocate max_cache_len; quantized / sliding layers hold fewer than past_len.
+ for layer in layers:
+ if layer.keys is None or layer.keys.shape[-2] < past_len:
+ raise ValueError(
+ f"Unsloth: {type(layer).__name__} does not keep every cached position, so it cannot be "
+ "used as past_key_values. Pass a DynamicCache or the model's own past_key_values."
+ )
+ return tuple(
+ (layer.keys[..., :past_len, :], layer.values[..., :past_len, :]) for layer in layers
+ )
+
+
+def _cached_prefill_defaults(
+ past_key_values, input_ids, inputs_embeds, position_ids, attention_mask
+):
+ # Continue after the cache as transformers does; else RoPE restarts at 0 and xFormers runs unmasked.
+ past_len = past_key_values[0][0].shape[-2]
+ ref = input_ids if input_ids is not None else inputs_embeds
+ bsz, q_len = ref.shape[:2]
+ if position_ids is None:
+ position_ids = torch.arange(past_len, past_len + q_len, device = ref.device)
+ position_ids = position_ids.unsqueeze(0).expand(bsz, -1)
+ if attention_mask is None:
+ attention_mask = torch.ones((bsz, past_len + q_len), dtype = torch.long, device = ref.device)
+ return position_ids, attention_mask
+
+
+def _slice_position_ids(position_ids, seq_length):
+ if position_ids is not None and 0 < seq_length < position_ids.shape[-1]:
+ position_ids = position_ids[..., -seq_length:]
+ return position_ids
+
+
def _fast_prepare_inputs_for_generation(
self,
input_ids,
@@ -269,11 +327,24 @@ def _fast_prepare_inputs_for_generation(
kwargs["past_key_values"] = None
use_inputs_embeds = inputs_embeds is not None
else:
+ if hasattr(past_key_values, "get_seq_length"):
+ past_len = int(past_key_values.get_seq_length())
+ else:
+ # Legacy tuple cache: (layer, (K, V)).
+ past_len = int(past_key_values[0][0].shape[-2])
+ kwargs["past_key_values"] = _cache_as_legacy_tuple(past_key_values)
+
if input_ids is not None and input_ids.numel() > 0:
bs = input_ids.shape[0]
- input_ids = input_ids[:, [-1]]
device = input_ids.device
- seq_length = 1
+ # The 2D mask spans cache + new tokens, so it also counts a suffix-only turn (transformers 5).
+ n_new = 0
+ if original_attention_mask is not None and original_attention_mask.dim() == 2:
+ n_new = original_attention_mask.shape[-1] - past_len
+ if not 0 < n_new <= input_ids.shape[1]:
+ n_new = max(input_ids.shape[1] - past_len, 1)
+ input_ids = input_ids[:, -n_new:]
+ seq_length = input_ids.shape[1]
elif inputs_embeds is not None:
bs, seq_length, _ = inputs_embeds.shape
device = inputs_embeds.device
@@ -281,14 +352,11 @@ def _fast_prepare_inputs_for_generation(
bs, seq_length = 1, 0
device = "cuda" if torch.cuda.is_available() else "cpu"
- if hasattr(past_key_values, "get_seq_length"):
- past_len = int(past_key_values.get_seq_length())
- else:
- # Legacy tuple cache: (layer, (K, V)).
- past_len = int(past_key_values[0][0].shape[-2])
-
+ # A flattened cache holds exactly past_len positions, whatever the Cache allocated.
max_cache_len = None
- if hasattr(past_key_values, "get_max_cache_shape"):
+ if kwargs["past_key_values"] is not past_key_values:
+ pass
+ elif hasattr(past_key_values, "get_max_cache_shape"):
m = past_key_values.get_max_cache_shape()
max_cache_len = int(m) if m is not None and m > 0 else None
elif hasattr(past_key_values, "get_max_length"):
@@ -369,6 +437,8 @@ def _fast_prepare_inputs_for_generation(
if cp.dim() == 1:
cp = cp.unsqueeze(0).expand(bs, -1)
kwargs["position_ids"] = cp
+ else:
+ kwargs["position_ids"] = _slice_position_ids(kwargs["position_ids"], seq_length)
result = {
"attention_mask": attention_mask,
@@ -1458,7 +1528,10 @@ def CausalLM_fast_forward(fast_forward_inference):
*args,
**kwargs,
) -> Union[Tuple, CausalLMOutputWithPast]:
- if past_key_values is not None:
+ past_key_values = _cache_as_legacy_tuple(past_key_values)
+ if past_key_values is not None and len(past_key_values) == 0:
+ past_key_values = None
+ if past_key_values is not None and input_ids is not None and input_ids.shape[1] == 1:
outputs = fast_forward_inference(
self,
input_ids,
@@ -1468,7 +1541,13 @@ def CausalLM_fast_forward(fast_forward_inference):
**kwargs,
)
else:
- causal_mask = xformers.attn_bias.LowerTriangularMask() if HAS_XFORMERS else None
+ causal_mask = None
+ if past_key_values is not None:
+ position_ids, attention_mask = _cached_prefill_defaults(
+ past_key_values, input_ids, inputs_embeds, position_ids, attention_mask
+ )
+ elif HAS_XFORMERS:
+ causal_mask = xformers.attn_bias.LowerTriangularMask()
output_attentions = (
output_attentions
@@ -2228,7 +2307,23 @@ def unsloth_fast_generate(self, *args, **kwargs):
# Must patch accelerate for Xformers.
- kwargs["cache_implementation"] = "dynamic"
+ # transformers raises if cache_implementation is set beside a user-supplied cache.
+ if kwargs.get("past_key_values", None) is not None:
+ # FalconH1 keeps its own input preparation for its hybrid Mamba/attention cache.
+ if getattr(getattr(self, "config", None), "model_type", None) == "falcon_h1":
+ raise NotImplementedError(
+ "Unsloth: passing past_key_values to generate() is not supported for FalconH1 yet."
+ )
+ kwargs["past_key_values"] = _ensure_cache_is_dynamic(kwargs["past_key_values"])
+ # The fast decode path only seeds missing KV buffers; drop the previous generate()'s.
+ for module in self.modules():
+ if hasattr(module, "paged_attention"):
+ del module.paged_attention_K, module.paged_attention_V, module.paged_attention
+ # A user StaticCache triggers CUDA-graph auto-compile, which overwrites the decode buffers.
+ if hasattr(getattr(self, "generation_config", None), "disable_compile"):
+ kwargs.setdefault("disable_compile", True)
+ else:
+ kwargs["cache_implementation"] = "dynamic"
# transformers 4.50 renamed num_logits_to_keep to logits_to_keep; pop both and re-emit under the
# spelling forward() accepts.
_provided_num = kwargs.pop("num_logits_to_keep", None)
diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py
index fbc67823dbc..156bbac7a8e 100644
--- a/unsloth/models/mistral.py
+++ b/unsloth/models/mistral.py
@@ -36,6 +36,8 @@ from .llama import (
apply_logit_transforms,
resolve_logit_scaling,
resolve_logit_transforms,
+ _cache_as_legacy_tuple,
+ _cached_prefill_defaults,
)
from transformers.models.mistral.modeling_mistral import (
MistralAttention,
@@ -179,6 +181,9 @@ def MistralForCausalLM_fast_forward(
*args,
**kwargs,
) -> Union[Tuple, CausalLMOutputWithPast]:
+ past_key_values = _cache_as_legacy_tuple(past_key_values)
+ if past_key_values is not None and len(past_key_values) == 0:
+ past_key_values = None
# PrefixGrouper brings its own mask: a synthesized causal attention_mask would trip
# resolve_prefix_seg_info on the no-xFormers path and force a fallback.
# Not using xformers - need to create attention masks
@@ -261,7 +266,12 @@ def MistralForCausalLM_fast_forward(
self.model._has_no_labels = labels is None
- if past_key_values is not None:
+ if past_key_values is not None and not (input_ids is not None and input_ids.shape[1] == 1):
+ position_ids, attention_mask = _cached_prefill_defaults(
+ past_key_values, input_ids, inputs_embeds, position_ids, attention_mask
+ )
+
+ if past_key_values is not None and input_ids is not None and input_ids.shape[1] == 1:
outputs = LlamaModel_fast_forward_inference(
self,
input_ids,