Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix: Support past_key_values in model.generate for multi-turn conversations (#4232)
Daniel Han0dae825e6963
diff --git a/scripts/kv_cache_multiturn_benchmark.py b/scripts/kv_cache_multiturn_benchmark.pynew file mode 100644index 00000000000..4d767830822--- /dev/null+++ b/scripts/kv_cache_multiturn_benchmark.py@@ -0,0 +1,84 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Multi-turn generate(): re-encode the whole conversation vs continue from a KV cache of the history.++python scripts/kv_cache_multiturn_benchmark.py --model unsloth/Llama-3.2-1B-Instruct --turns 4 8 16 32+"""++import argparse+import time++from unsloth import FastLanguageModel+import torch+++def timed(fn, runs):+ times = []+ for _ in range(runs):+ torch.cuda.synchronize()+ t0 = time.perf_counter()+ out = fn()+ torch.cuda.synchronize()+ times.append(time.perf_counter() - t0)+ return out, sorted(times)[len(times) // 2]+++def main():+ parser = argparse.ArgumentParser()+ parser.add_argument("--model", default = "unsloth/Llama-3.2-1B-Instruct")+ parser.add_argument("--turns", type = int, nargs = "+", default = [4, 8, 16, 32])+ parser.add_argument("--max-new-tokens", type = int, default = 32)+ parser.add_argument("--runs", type = int, default = 5)+ parser.add_argument("--load-in-4bit", action = "store_true")+ args = parser.parse_args()++ model, tokenizer = FastLanguageModel.from_pretrained(+ args.model, max_seq_length = 8192, load_in_4bit = args.load_in_4bit+ )+ FastLanguageModel.for_inference(model)+ gen = dict(+ max_new_tokens = args.max_new_tokens, min_new_tokens = args.max_new_tokens, do_sample = False+ )++ print(+ "| turns | history tok | new tok | full re-encode s | from cache s | speedup | tokens match |"+ )+ print("|---:|---:|---:|---:|---:|---:|:---:|")+ for turns in args.turns:+ history = []+ for i in range(turns):+ history.append(+ {+ "role": "user",+ "content": f"Fact {i}: item {i} costs {3 * i + 7} dollars. Please remember it.",+ }+ )+ history.append(+ {"role": "assistant", "content": f"Noted: item {i} costs {3 * i + 7} dollars."}+ )+ question = [{"role": "user", "content": "What does item 2 cost?"}]+ text_h = tokenizer.apply_chat_template(history, tokenize = False)+ text_f = tokenizer.apply_chat_template(+ history + question, tokenize = False, add_generation_prompt = True+ )+ enc = lambda t: tokenizer(t, return_tensors = "pt", add_special_tokens = False).to("cuda")+ hist, full = enc(text_h), enc(text_f)+ n, L = hist.input_ids.shape[1], full.input_ids.shape[1]+ if not torch.equal(full.input_ids[:, :n], hist.input_ids):+ raise SystemExit("chat template does not keep the history as a token prefix")+ with torch.no_grad():+ cache = model(**hist, use_cache = True).past_key_values++ model.generate(**full, max_new_tokens = 2)+ model.generate(**full, past_key_values = cache, max_new_tokens = 2)+ base, t_base = timed(lambda: model.generate(**full, **gen), args.runs)+ kv, t_kv = timed(lambda: model.generate(**full, past_key_values = cache, **gen), args.runs)+ match = torch.equal(base[0, L:], kv[0, L:])+ print(+ f"| {turns} | {n} | {L - n} | {t_base:.4f} | {t_kv:.4f} | {t_base / t_kv:.2f}x | {'YES' if match else 'NO'} |"+ )+++if __name__ == "__main__":+ main()diff --git a/tests/test_past_kv_models.py b/tests/test_past_kv_models.pynew file mode 100644index 00000000000..a2c20a8e8be--- /dev/null+++ b/tests/test_past_kv_models.py@@ -0,0 +1,105 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""GPU check: generate() continuing from a user KV cache of the conversation history (issue #497)+must emit the same greedy tokens as re-encoding the whole conversation."""++import pytest+import torch++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA GPU")++MODELS = [+ "trl-internal-testing/tiny-LlamaForCausalLM-3.2",+ "trl-internal-testing/tiny-Qwen3ForCausalLM",+ "hf-internal-testing/tiny-random-MistralForCausalLM",+ "trl-internal-testing/tiny-Gemma2ForCausalLM",+]+HISTORY = (+ "user: My name is Zorblat and my favourite colour is teal.\nassistant: Nice to meet you!\n"+)+OTHER = "user: I live near the mountains and I own two small cats.\nassistant: How lovely!\n" * 3+QUESTION = "user: What is my name?\nassistant:"+GEN = dict(max_new_tokens = 8, min_new_tokens = 8, do_sample = False)++[email protected](scope = "module", params = MODELS)+def model_and_tokenizer(request):+ from unsloth import FastLanguageModel++ model, tokenizer = FastLanguageModel.from_pretrained(+ request.param, max_seq_length = 512, load_in_4bit = False, dtype = torch.bfloat16+ )+ FastLanguageModel.for_inference(model)+ yield model, tokenizer+ del model+ torch.cuda.empty_cache()+++def _ids(tokenizer, text):+ return tokenizer(text, return_tensors = "pt", add_special_tokens = False).to("cuda")+++def _cache(model, ids):+ with torch.no_grad():+ return model(**ids, use_cache = True).past_key_values+++def _as_tuple(cache):+ if hasattr(cache, "layers"):+ return tuple((layer.keys, layer.values) for layer in cache.layers)+ return tuple((k, v) for k, v in cache)+++def _generate(model, full, **kwargs):+ out = model.generate(**full, **GEN, output_logits = True, return_dict_in_generate = True, **kwargs)+ return out.sequences[0, full.input_ids.shape[1] :], out.logits[0][0].float()+++def test_generate_from_history_cache_matches_full_prompt(model_and_tokenizer):+ model, tokenizer = model_and_tokenizer+ history, full = _ids(tokenizer, HISTORY), _ids(tokenizer, HISTORY + QUESTION)+ n = history.input_ids.shape[1]+ assert torch.equal(full.input_ids[:, :n], history.input_ids)++ expected_tokens, expected_logits = _generate(model, full)+ tolerance = 0.02 * expected_logits.abs().max()+ cache = _cache(model, history)+ runs = [+ _generate(model, full, past_key_values = past) for past in (cache, _as_tuple(cache), cache)+ ]+ tokens, logits = runs[0]+ # Chunked vs full prefill differs only by bf16 rounding; tiny random models have 1-ulp logit ties.+ assert (logits - expected_logits).abs().max() <= tolerance+ assert tokens[0] == expected_tokens[0]+ for other_tokens, other_logits in runs[1:]:+ assert torch.equal(other_tokens, tokens) and torch.equal(other_logits, logits)++ other = _ids(tokenizer, OTHER)+ wrong = {k: v[:, :n] for k, v in other.items()}+ assert wrong["input_ids"].shape[1] == n+ _, wrong_logits = _generate(model, full, past_key_values = _cache(model, wrong))+ assert (wrong_logits - expected_logits).abs().max() > 5 * tolerance++[email protected]("attention_mask", [False, True])+def test_direct_forward_onto_cache_continues_after_it(model_and_tokenizer, attention_mask):+ model, tokenizer = model_and_tokenizer+ full = _ids(tokenizer, HISTORY + QUESTION)+ n = _ids(tokenizer, HISTORY).input_ids.shape[1]+ with torch.no_grad():+ expected = model(full.input_ids).logits[0, n:].float()+ cache = _cache(model, {"input_ids": full.input_ids[:, :n]})+ kwargs = {"attention_mask": full.attention_mask} if attention_mask else {}+ got = model(full.input_ids[:, n:], past_key_values = cache, **kwargs).logits[0].float()+ assert (got - expected).abs().max() <= 0.02 * expected.abs().max()+++def test_saved_cache_reused_after_another_generate(model_and_tokenizer):+ model, tokenizer = model_and_tokenizer+ full = _ids(tokenizer, HISTORY + QUESTION)+ cache = _cache(model, {"input_ids": full.input_ids[:, :-1]})+ expected_tokens, expected_logits = _generate(model, full, past_key_values = cache)+ model.generate(**_ids(tokenizer, OTHER), max_new_tokens = 8)+ tokens, logits = _generate(model, full, past_key_values = cache)+ assert torch.equal(tokens, expected_tokens) and torch.equal(logits, expected_logits)diff --git a/tests/test_past_kv_utils.py b/tests/test_past_kv_utils.pynew file mode 100644index 00000000000..6c146c90032--- /dev/null+++ b/tests/test_past_kv_utils.py@@ -0,0 +1,147 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""CPU checks for user-supplied past_key_values in generate() (issue #497), against unsloth/models/llama.py."""++import pytest+import torch+from transformers.cache_utils import Cache, DynamicCache++BS, PAST_LEN, SEQ = 2, 3, 5+++class FakeModel:+ dtype = torch.float32+ config = None+++def _llama():+ from unsloth.models import llama+ return llama+++def _kv(length, layers = 2):+ return tuple(+ (torch.randn(BS, 1, length, 4), torch.randn(BS, 1, length, 4)) for _ in range(layers)+ )+++def _prepare(input_ids, **kwargs):+ return _llama()._fast_prepare_inputs_for_generation(FakeModel(), input_ids, **kwargs)+++def test_tuple_cache_becomes_dynamic_cache_for_generate():+ legacy = _kv(PAST_LEN)+ cache = _llama()._ensure_cache_is_dynamic(legacy)+ assert isinstance(cache, Cache)+ assert cache.get_seq_length() == PAST_LEN+ assert _llama()._ensure_cache_is_dynamic(cache) is cache+ assert _llama()._ensure_cache_is_dynamic(None) is None++[email protected]("as_dynamic", [False, True])+def test_prefill_onto_partial_cache_feeds_every_uncached_token(as_dynamic):+ legacy = _kv(PAST_LEN)+ past = _llama()._ensure_cache_is_dynamic(legacy) if as_dynamic else legacy+ input_ids = torch.arange(BS * SEQ).reshape(BS, SEQ)+ mask = torch.ones(BS, SEQ, dtype = torch.long)+ result = _prepare(input_ids, attention_mask = mask, past_key_values = past)++ assert torch.equal(result["input_ids"], input_ids[:, PAST_LEN:])+ assert result["position_ids"].tolist() == [[3, 4]] * BS+ assert result["cache_position"].tolist() == [3, 4]+ out = result["past_key_values"]+ assert isinstance(out, tuple) and len(out) == len(legacy)+ for (k, v), (k0, v0) in zip(out, legacy):+ assert torch.equal(k, k0) and torch.equal(v, v0)+++def test_decode_step_still_feeds_only_the_last_token():+ input_ids = torch.arange(BS * SEQ).reshape(BS, SEQ)+ result = _prepare(input_ids, past_key_values = _kv(SEQ - 1))+ assert torch.equal(result["input_ids"], input_ids[:, [-1]])+++def test_full_length_user_position_ids_are_sliced_to_the_fed_tokens():+ input_ids = torch.arange(BS * SEQ).reshape(BS, SEQ)+ pos = torch.arange(SEQ).expand(BS, -1) + 10+ result = _prepare(input_ids, past_key_values = _kv(PAST_LEN), position_ids = pos)+ assert result["position_ids"].tolist() == [[13, 14]] * BS++ result = _prepare(input_ids, position_ids = pos[0])+ assert torch.equal(result["position_ids"], pos[0])+++def test_user_position_ids_with_inputs_embeds_and_no_input_ids():+ embeds = torch.randn(BS, SEQ, 4)+ pos = torch.arange(SEQ).expand(BS, -1)+ result = _prepare(None, inputs_embeds = embeds, position_ids = pos)+ assert torch.equal(result["position_ids"], pos)+ assert result["inputs_embeds"] is embeds+++def test_empty_dynamic_cache_is_dropped_before_prefill():+ input_ids = torch.arange(BS * SEQ).reshape(BS, SEQ)+ result = _prepare(input_ids, past_key_values = DynamicCache())+ assert result["past_key_values"] is None+ assert result["input_ids"].shape == (BS, SEQ)+++def test_suffix_only_input_with_full_mask_feeds_the_whole_new_turn():+ # transformers 5 generate() accepts only the new tokens when attention_mask spans cache + new.+ new = torch.arange(BS * 2).reshape(BS, 2)+ mask = torch.ones(BS, PAST_LEN + 2, dtype = torch.long)+ result = _prepare(new, attention_mask = mask, past_key_values = _kv(PAST_LEN))+ assert torch.equal(result["input_ids"], new)+ assert result["position_ids"].tolist() == [[3, 4]] * BS+++def test_static_cache_is_flattened_to_its_filled_length():+ from transformers import LlamaConfig+ from transformers.cache_utils import StaticCache++ config = LlamaConfig(+ num_hidden_layers = 2, num_attention_heads = 1, num_key_value_heads = 1, hidden_size = 4+ )+ try:+ cache = StaticCache(config = config, max_cache_len = 8)+ except TypeError:+ pytest.skip("StaticCache signature differs on this transformers")+ legacy = _kv(PAST_LEN)+ for i, (k, v) in enumerate(legacy):+ cache.update(k, v, i, {"cache_position": torch.arange(PAST_LEN)})+ out = _llama()._cache_as_legacy_tuple(cache)+ for (k, v), (k0, v0) in zip(out, legacy):+ assert torch.equal(k, k0) and torch.equal(v, v0)+++def test_cache_layers_missing_positions_are_rejected():+ cache = _llama()._ensure_cache_is_dynamic(_kv(PAST_LEN))+ # A QuantizedLayer counts every token but keeps only the unquantized tail in .keys.+ cache.layers[0].keys = cache.layers[0].keys[..., 1:, :]+ cache.get_seq_length = lambda layer_idx = 0: PAST_LEN+ with pytest.raises(ValueError, match = "does not keep every cached position"):+ _llama()._cache_as_legacy_tuple(cache)+++def test_partial_static_cache_mask_spans_the_flattened_length():+ from transformers import LlamaConfig+ from transformers.cache_utils import StaticCache+ from tests.utils.test_prepare_inputs_leftpad import FakeModelWith4DMask++ config = LlamaConfig(+ num_hidden_layers = 2, num_attention_heads = 1, num_key_value_heads = 1, hidden_size = 4+ )+ try:+ cache = StaticCache(config = config, max_cache_len = 16)+ except TypeError:+ pytest.skip("StaticCache signature differs on this transformers")+ for i, (k, v) in enumerate(_kv(PAST_LEN)):+ cache.update(k, v, i, {"cache_position": torch.arange(PAST_LEN)})+ model = FakeModelWith4DMask()+ input_ids = torch.arange(BS * SEQ).reshape(BS, SEQ)+ mask = torch.ones(BS, SEQ, dtype = torch.long)+ _llama()._fast_prepare_inputs_for_generation(+ model, input_ids, attention_mask = mask, past_key_values = cache+ )+ assert model.mask_calls[0]["target_length"] == SEQ # past_len + fed tokens, not max_cache_lendiff --git a/unsloth/kernels/flex_attention.py b/unsloth/kernels/flex_attention.pyindex 0384ec55512..cf7fe8f20db 100644--- a/unsloth/kernels/flex_attention.py+++ b/unsloth/kernels/flex_attention.py@@ -37,18 +37,18 @@ except:if not HAS_FLEX_ATTENTION:# Logit softcapping@torch.compile(fullgraph = True, dynamic = True, options = torch_compile_options)- def _compiled_slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len):+ def _compiled_slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, kv_len):n_heads = self.config.num_attention_headshead_dim = self.head_dimn_kv_heads = self.config.num_key_value_headsn_groups = self.num_key_value_groups+ actual_q_len = Q.shape[-2]# Grouped query attention- # Grouped query attention- K = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)- V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)- K = K.reshape(bsz, n_heads, q_len, head_dim)- V = V.reshape(bsz, n_heads, q_len, head_dim)+ K = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, kv_len, head_dim)+ V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, kv_len, head_dim)+ K = K.reshape(bsz, n_heads, kv_len, head_dim)+ V = V.reshape(bsz, n_heads, kv_len, head_dim)# Gemma 9b should use 256, not hidden_size // num_attention_heads (224); 27b uses the derived value,# so default to the config. See google/gemma_pytorch commit 03e6575.@@ -58,12 +58,15 @@ if not HAS_FLEX_ATTENTION:Q = Q * torch.tensor(s**-0.5, dtype = Q.dtype)A = torch.matmul(Q, K.transpose(2, 3))A = t * torch.tanh(A / t)- A += causal_mask[..., :q_len, :q_len]- # Much slower under torch compile than the masked_fill_ it replaces.+ # 2D static masks index absolute positions; 4D masks are already per query row.+ if causal_mask.dim() == 2:+ A += causal_mask[kv_len - actual_q_len : kv_len, :kv_len]+ else:+ A += causal_mask[..., :actual_q_len, :kv_len]A = torch.nn.functional.softmax(A, dim = -1, dtype = torch.float32).to(Q.dtype)A = torch.matmul(A, V)A = A.transpose(1, 2).contiguous()- A = A.reshape(bsz, q_len, n_heads * head_dim)+ A = A.reshape(bsz, actual_q_len, n_heads * head_dim)return A_SOFTCAP_EAGER = {}@@ -180,16 +183,17 @@ torch_tanh = torch.tanhtorch_nn_functional_softmax = torch.nn.functional.softmax-def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len):+def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, kv_len):n_heads = self.config.num_attention_headshead_dim = self.head_dimn_kv_heads = self.config.num_key_value_headsn_groups = self.num_key_value_groups+ actual_q_len = Q.shape[-2]- K = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)- V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)- K = K.reshape(bsz, n_heads, q_len, head_dim)- V = V.reshape(bsz, n_heads, q_len, head_dim)+ K = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, kv_len, head_dim)+ V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, kv_len, head_dim)+ K = K.reshape(bsz, n_heads, kv_len, head_dim)+ V = V.reshape(bsz, n_heads, kv_len, head_dim)# Gemma 9b should use 256, not hidden_size // num_attention_heads (224); 27b uses the derived value,# so default to the config. See google/gemma_pytorch commit 03e6575.@@ -203,10 +207,12 @@ def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len)A /= ttorch_tanh(A, out = A)A *= t- A += causal_mask[..., :q_len, :q_len]- # Much slower under torch compile than the masked_fill_ it replaces.+ if causal_mask.dim() == 2:+ A += causal_mask[kv_len - actual_q_len : kv_len, :kv_len]+ else:+ A += causal_mask[..., :actual_q_len, :kv_len]A = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32).to(Q.dtype)A = torch_matmul(A, V)A = A.transpose(1, 2).contiguous()- A = A.reshape(bsz, q_len, n_heads * head_dim)+ A = A.reshape(bsz, actual_q_len, n_heads * head_dim)return Adiff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 70e355d5616..02e73f90c94 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -108,6 +108,7 @@ from unsloth.models._attn_mask_compat import (AttentionMaskConverter,_prepare_4d_causal_attention_mask_for_sdpa,)+from transformers.cache_utils import DynamicCache, Cachefrom ..kernels import *from ..kernels.utils import has_mxfp4_basefrom ..tokenizer_utils import *@@ -236,6 +237,63 @@ def _offload_frozen_module_for_training(module.original_module.requires_grad_(False)+def _ensure_cache_is_dynamic(past_key_values):+ # transformers 5 rejects legacy tuple caches in generate().+ if past_key_values is None:+ return None+ if isinstance(past_key_values, Cache):+ return past_key_values+ if isinstance(past_key_values, (tuple, list)) and len(past_key_values) > 0:+ cache = DynamicCache()+ for layer_idx, layer_kv in enumerate(past_key_values):+ cache.update(layer_kv[0], layer_kv[1], layer_idx)+ return cache+ return past_key_values+++def _cache_as_legacy_tuple(past_key_values):+ # Unsloth's forwards index past_key_values[layer][0|1]; transformers 5 caches are not subscriptable.+ if not isinstance(past_key_values, Cache):+ return past_key_values+ past_len = past_key_values.get_seq_length()+ if past_len == 0:+ return None+ layers = getattr(past_key_values, "layers", None)+ if layers is None:+ return past_key_values.to_legacy_cache()+ # Static layers allocate max_cache_len; quantized / sliding layers hold fewer than past_len.+ for layer in layers:+ if layer.keys is None or layer.keys.shape[-2] < past_len:+ raise ValueError(+ f"Unsloth: {type(layer).__name__} does not keep every cached position, so it cannot be "+ "used as past_key_values. Pass a DynamicCache or the model's own past_key_values."+ )+ return tuple(+ (layer.keys[..., :past_len, :], layer.values[..., :past_len, :]) for layer in layers+ )+++def _cached_prefill_defaults(+ past_key_values, input_ids, inputs_embeds, position_ids, attention_mask+):+ # Continue after the cache as transformers does; else RoPE restarts at 0 and xFormers runs unmasked.+ past_len = past_key_values[0][0].shape[-2]+ ref = input_ids if input_ids is not None else inputs_embeds+ bsz, q_len = ref.shape[:2]+ if position_ids is None:+ position_ids = torch.arange(past_len, past_len + q_len, device = ref.device)+ position_ids = position_ids.unsqueeze(0).expand(bsz, -1)+ if attention_mask is None:+ attention_mask = torch.ones((bsz, past_len + q_len), dtype = torch.long, device = ref.device)+ return position_ids, attention_mask+++def _slice_position_ids(position_ids, seq_length):+ if position_ids is not None and 0 < seq_length < position_ids.shape[-1]:+ position_ids = position_ids[..., -seq_length:]+ return position_ids++def _fast_prepare_inputs_for_generation(self,input_ids,@@ -269,11 +327,24 @@ def _fast_prepare_inputs_for_generation(kwargs["past_key_values"] = Noneuse_inputs_embeds = inputs_embeds is not Noneelse:+ if hasattr(past_key_values, "get_seq_length"):+ past_len = int(past_key_values.get_seq_length())+ else:+ # Legacy tuple cache: (layer, (K, V)).+ past_len = int(past_key_values[0][0].shape[-2])+ kwargs["past_key_values"] = _cache_as_legacy_tuple(past_key_values)+if input_ids is not None and input_ids.numel() > 0:bs = input_ids.shape[0]- input_ids = input_ids[:, [-1]]device = input_ids.device- seq_length = 1+ # The 2D mask spans cache + new tokens, so it also counts a suffix-only turn (transformers 5).+ n_new = 0+ if original_attention_mask is not None and original_attention_mask.dim() == 2:+ n_new = original_attention_mask.shape[-1] - past_len+ if not 0 < n_new <= input_ids.shape[1]:+ n_new = max(input_ids.shape[1] - past_len, 1)+ input_ids = input_ids[:, -n_new:]+ seq_length = input_ids.shape[1]elif inputs_embeds is not None:bs, seq_length, _ = inputs_embeds.shapedevice = inputs_embeds.device@@ -281,14 +352,11 @@ def _fast_prepare_inputs_for_generation(bs, seq_length = 1, 0device = "cuda" if torch.cuda.is_available() else "cpu"- if hasattr(past_key_values, "get_seq_length"):- past_len = int(past_key_values.get_seq_length())- else:- # Legacy tuple cache: (layer, (K, V)).- past_len = int(past_key_values[0][0].shape[-2])-+ # A flattened cache holds exactly past_len positions, whatever the Cache allocated.max_cache_len = None- if hasattr(past_key_values, "get_max_cache_shape"):+ if kwargs["past_key_values"] is not past_key_values:+ pass+ elif hasattr(past_key_values, "get_max_cache_shape"):m = past_key_values.get_max_cache_shape()max_cache_len = int(m) if m is not None and m > 0 else Noneelif hasattr(past_key_values, "get_max_length"):@@ -369,6 +437,8 @@ def _fast_prepare_inputs_for_generation(if cp.dim() == 1:cp = cp.unsqueeze(0).expand(bs, -1)kwargs["position_ids"] = cp+ else:+ kwargs["position_ids"] = _slice_position_ids(kwargs["position_ids"], seq_length)result = {"attention_mask": attention_mask,@@ -1458,7 +1528,10 @@ def CausalLM_fast_forward(fast_forward_inference):*args,**kwargs,) -> Union[Tuple, CausalLMOutputWithPast]:- if past_key_values is not None:+ past_key_values = _cache_as_legacy_tuple(past_key_values)+ if past_key_values is not None and len(past_key_values) == 0:+ past_key_values = None+ if past_key_values is not None and input_ids is not None and input_ids.shape[1] == 1:outputs = fast_forward_inference(self,input_ids,@@ -1468,7 +1541,13 @@ def CausalLM_fast_forward(fast_forward_inference):**kwargs,)else:- causal_mask = xformers.attn_bias.LowerTriangularMask() if HAS_XFORMERS else None+ causal_mask = None+ if past_key_values is not None:+ position_ids, attention_mask = _cached_prefill_defaults(+ past_key_values, input_ids, inputs_embeds, position_ids, attention_mask+ )+ elif HAS_XFORMERS:+ causal_mask = xformers.attn_bias.LowerTriangularMask()output_attentions = (output_attentions@@ -2228,7 +2307,23 @@ def unsloth_fast_generate(self, *args, **kwargs):# Must patch accelerate for Xformers.- kwargs["cache_implementation"] = "dynamic"+ # transformers raises if cache_implementation is set beside a user-supplied cache.+ if kwargs.get("past_key_values", None) is not None:+ # FalconH1 keeps its own input preparation for its hybrid Mamba/attention cache.+ if getattr(getattr(self, "config", None), "model_type", None) == "falcon_h1":+ raise NotImplementedError(+ "Unsloth: passing past_key_values to generate() is not supported for FalconH1 yet."+ )+ kwargs["past_key_values"] = _ensure_cache_is_dynamic(kwargs["past_key_values"])+ # The fast decode path only seeds missing KV buffers; drop the previous generate()'s.+ for module in self.modules():+ if hasattr(module, "paged_attention"):+ del module.paged_attention_K, module.paged_attention_V, module.paged_attention+ # A user StaticCache triggers CUDA-graph auto-compile, which overwrites the decode buffers.+ if hasattr(getattr(self, "generation_config", None), "disable_compile"):+ kwargs.setdefault("disable_compile", True)+ else:+ kwargs["cache_implementation"] = "dynamic"# transformers 4.50 renamed num_logits_to_keep to logits_to_keep; pop both and re-emit under the# spelling forward() accepts._provided_num = kwargs.pop("num_logits_to_keep", None)diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.pyindex fbc67823dbc..156bbac7a8e 100644--- a/unsloth/models/mistral.py+++ b/unsloth/models/mistral.py@@ -36,6 +36,8 @@ from .llama import (apply_logit_transforms,resolve_logit_scaling,resolve_logit_transforms,+ _cache_as_legacy_tuple,+ _cached_prefill_defaults,)from transformers.models.mistral.modeling_mistral import (MistralAttention,@@ -179,6 +181,9 @@ def MistralForCausalLM_fast_forward(*args,**kwargs,) -> Union[Tuple, CausalLMOutputWithPast]:+ past_key_values = _cache_as_legacy_tuple(past_key_values)+ if past_key_values is not None and len(past_key_values) == 0:+ past_key_values = None# PrefixGrouper brings its own mask: a synthesized causal attention_mask would trip# resolve_prefix_seg_info on the no-xFormers path and force a fallback.# Not using xformers - need to create attention masks@@ -261,7 +266,12 @@ def MistralForCausalLM_fast_forward(self.model._has_no_labels = labels is None- if past_key_values is not None:+ if past_key_values is not None and not (input_ids is not None and input_ids.shape[1] == 1):+ position_ids, attention_mask = _cached_prefill_defaults(+ past_key_values, input_ids, inputs_embeds, position_ids, attention_mask+ )++ if past_key_values is not None and input_ids is not None and input_ids.shape[1] == 1:outputs = LlamaModel_fast_forward_inference(self,input_ids,