The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Qwen3/3.5/3.8 cudagraphs and w4a8 gemv support (CORE-390) (#15623)
Jukka Seppänen9a77c1db9eff
diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.pyindex 418faa51c..2b5cd042a 100644--- a/comfy/model_prefetch.py+++ b/comfy/model_prefetch.py@@ -46,6 +46,22 @@ class _PauseMallocGraph:def pause_malloc_graph(sync=False):return _PauseMallocGraph(sync)+class _MallocGraphScope:+ def __init__(self, device):+ self.device = device++ def __enter__(self):+ malloc_graph_begin(self.device)++ def __exit__(self, exc_type, *args):+ if exc_type is None:+ malloc_graph_end()+ else:+ cleanup_malloc_graph()++def malloc_graph_scope(device):+ return _MallocGraphScope(device)+def malloc_graph_begin(device):global MALLOC_GRAPH_USEDif not malloc_graph_enabled(device):@@ -86,6 +102,24 @@ def cleanup_malloc_graph():MALLOC_GRAPH_ROGUES += graph.rogue_countdel graph+def pin_modules(comfy_modules, device, dtype=None):+ registerable_size = 0+ for s in comfy_modules:+ registerable_size += comfy.memory_management.vram_aligned_size([s.weight, s.bias])+ for param_key in ("weight", "bias"):+ lowvram_fn = getattr(s, param_key + "_lowvram_function", None)+ if lowvram_fn is not None:+ registerable_size += lowvram_fn.memory_required()++ offload_stream, fully_faulted = comfy.ops.cast_modules_with_vbar(comfy_modules, None, device, None, True, return_faulted=True)+ if not (comfy_modules and comfy_modules[0]._pin_state["fast_disk"]):+ comfy.model_management.ensure_pin_registerable(registerable_size)+ comfy.model_management.sync_stream(device, offload_stream)+ if fully_faulted and dtype is not None:+ for comfy_module in comfy_modules:+ comfy.ops.resolve_cast_module_with_vbar(comfy_module, dtype, device, dtype, None, False, return_weights=False)+ return offload_stream, fully_faulted+def cleanup_prefetched_modules(module, comfy_modules):for s in comfy_modules:prefetch = getattr(s, "_prefetch", None)@@ -201,21 +235,7 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grapif hasattr(s, "_v"):comfy_modules.append(s)- registerable_size = 0- for s in comfy_modules:- registerable_size += comfy.memory_management.vram_aligned_size([s.weight, s.bias])- for param_key in ("weight", "bias"):- lowvram_fn = getattr(s, param_key + "_lowvram_function", None)- if lowvram_fn is not None:- registerable_size += lowvram_fn.memory_required()-- offload_stream, fully_faulted = comfy.ops.cast_modules_with_vbar(comfy_modules, None, device, None, True, return_faulted=True)- if not (comfy_modules and comfy_modules[0]._pin_state["fast_disk"]):- comfy.model_management.ensure_pin_registerable(registerable_size)- comfy.model_management.sync_stream(device, offload_stream)- if fully_faulted and dtype is not None:- for comfy_module in comfy_modules:- comfy.ops.resolve_cast_module_with_vbar(comfy_module, dtype, device, dtype, None, False, return_weights=False)+ offload_stream, fully_faulted = pin_modules(comfy_modules, device, dtype)queue[0] = (offload_stream, (module, comfy_modules))if core is not None:diff --git a/comfy/sd.py b/comfy/sd.pyindex 640d59352..230d4cf82 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -469,7 +469,7 @@ class CLIP:def get_key_patches(self):return self.patcher.get_key_patches()- def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0):+ def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0, mtp=True):self.cond_stage_model.reset_clip_options()self.load_model(tokens)@@ -478,7 +478,7 @@ class CLIP:self.cond_stage_model.set_clip_options({"execution_device": device})with model_management.cuda_device_context(device), comfy.ops.use_quantized_matmul(self.cond_stage_model, device):- return self.cond_stage_model.generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty)+ return self.cond_stage_model.generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty, mtp=mtp)def decode(self, token_ids, skip_special_tokens=True):return self.tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)@@ -1910,7 +1910,7 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipelif te_model in (TEModel.QWEN35_08B, TEModel.QWEN35_2B, TEModel.QWEN35_4B, TEModel.QWEN35_9B, TEModel.QWEN35_27B):clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})qwen35_type = {TEModel.QWEN35_08B: "qwen35_08b", TEModel.QWEN35_2B: "qwen35_2b", TEModel.QWEN35_4B: "qwen35_4b", TEModel.QWEN35_9B: "qwen35_9b", TEModel.QWEN35_27B: "qwen35_27b"}[te_model]- clip_target.clip = comfy.text_encoders.qwen35.te(**llama_detect(clip_data), model_type=qwen35_type)+ clip_target.clip = comfy.text_encoders.qwen35.te(**llama_detect(clip_data), model_type=qwen35_type, mtp="mtp.fc.weight" in clip_data[0])clip_target.tokenizer = comfy.text_encoders.qwen35.tokenizer(model_type=qwen35_type)elif te_model in (TEModel.QWEN3VL_4B, TEModel.QWEN3VL_8B):if clip_type == CLIPType.IDEOGRAM4 and te_model == TEModel.QWEN3VL_8B: # Ideogram4 reuses the full Qwen3-VL-8B (13-layer tap for conditioning + multimodal generate).diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.pyindex f0fdf1aa5..01651bd23 100644--- a/comfy/sd1_clip.py+++ b/comfy/sd1_clip.py@@ -308,7 +308,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):def load_sd(self, sd):return self.transformer.load_state_dict(sd, strict=False, assign=getattr(self, "can_assign_sd", False))- def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0):+ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0, mtp=True):if isinstance(tokens, dict):tokens_only = next(iter(tokens.values())) # todo: get this better?else:@@ -746,5 +746,11 @@ class SD1ClipModel(torch.nn.Module):def load_sd(self, sd):return getattr(self, self.clip).load_sd(sd)- def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0):- return getattr(self, self.clip).generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty)+ def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0, mtp=True):+ return getattr(self, self.clip).generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty, mtp=mtp)++ def get_dynamic_vram__units(self):+ # forward to the inner transformer so ModelPatcher can register vbar units (graph decode)+ model = getattr(getattr(getattr(self, self.clip), "transformer", None), "model", None)+ get_units = getattr(model, "get_dynamic_vram__units", None)+ return get_units() if get_units is not None else ([], [])diff --git a/comfy/text_encoders/gemma4.py b/comfy/text_encoders/gemma4.pyindex ee035d522..0dcd226cb 100644--- a/comfy/text_encoders/gemma4.py+++ b/comfy/text_encoders/gemma4.py@@ -1664,7 +1664,7 @@ class Gemma4Model(sd1_clip.SDClipModel):self.dtypes.add(dtype)super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={}, dtype=dtype, special_tokens={"start": 2, "pad": 0}, layer_norm_hidden_state=False, model_class=self.model_class, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, model_options=model_options)- def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0):+ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0, mtp=True):if isinstance(tokens, dict):tokens = next(iter(tokens.values()))tokens_only = [[t[0] for t in b] for b in tokens]@@ -1708,9 +1708,6 @@ def gemma4_te(dtype_llama=None, llama_quantization_metadata=None, model_class=Noif dtype_llama is not None:dtype = dtype_llamasuper().__init__(device=device, dtype=dtype, name="gemma4", clip_model=clip_model, model_options=model_options)-- def get_dynamic_vram__units(self):- return getattr(self, self.clip).transformer.model.get_dynamic_vram__units()return Gemma4TEModel_diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex d71daf3a3..6a778e4f0 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -32,6 +32,72 @@ class FixedKV:def advance(self, num_tokens):self.index += num_tokens+ def rollback(self, discard=1):+ # drop the rejected verify tail; stale slots and bias are overwritten by the next write/prepare+ self.index -= discard++@dataclass+class FixedKVBias(FixedKV):+ # full-capacity decode bias [1, 1, rows, capacity], last `seq` rows serve the queries; shared across layers+ bias: torch.Tensor = None+ tracker: dict = None++ def prepare(self, num_tokens):+ if self.tracker["step"] == (self.index, num_tokens):+ return+ self.tracker["step"] = (self.index, num_tokens)+ i = self.index+ rows = self.bias.shape[-2]+ if num_tokens <= rows:+ torch.arange(i, i + rows, out=self.position)+ window = self.tracker[num_tokens]+ start = i - rows + 1+ skip = max(-start, 0)+ end = min(i + rows, self.bias.shape[-1])+ self.bias[..., :, start + skip:end] = window[:, skip:end - start]+ else:+ self.bias[..., :, i:i + num_tokens] = 0++ @staticmethod+ def shared(capacity, device, dtype):+ # all layers advance in lockstep, so the bias caches share one position/bias/tracker+ rows = 6+ position = torch.empty((rows,), device=device, dtype=torch.int64)+ bias = torch.full((1, 1, rows, capacity), torch.finfo(dtype).min, device=device, dtype=dtype)+ tracker = {"step": -1}+ # window templates per decode width: row r serves query j = r - (rows - n) and may see slots <= index + j+ for n in range(1, rows + 1):+ window = torch.full((rows, 2 * rows - 1), torch.finfo(dtype).min, device=device, dtype=dtype)+ for r in range(rows):+ window[r, :rows + max(r - (rows - n), 0)] = 0+ tracker[n] = window+ return position, bias, tracker++ @classmethod+ def zeros(cls, batch, kv_heads, capacity, head_dim, device, dtype, shared):+ # zero-init: decode attends full capacity with masked tails, 0*0 stays finite+ key = torch.zeros((batch, kv_heads, capacity, head_dim), device=device, dtype=dtype)+ return cls(key, torch.zeros_like(key), 0, shared[0], None, shared[1], shared[2])++ def append(self, xk, xv):+ seq = xk.shape[2]+ self.key[:, :, self.index:self.index + seq] = xk+ self.value[:, :, self.index:self.index + seq] = xv+ return self.key[:, :, :self.index + seq], self.value[:, :, :self.index + seq]++ def decode(self, xq, xk, xv, num_kv_heads):+ # CUDA-graphable: device-side write position, masked attention over the full capacity+ batch_size, num_heads, seq, head_dim = xq.shape+ self.key.index_copy_(2, self.position[:seq], xk)+ self.value.index_copy_(2, self.position[:seq], xv)+ groups = num_heads // num_kv_heads+ q = xq.reshape(batch_size, num_kv_heads, groups, seq, head_dim) * head_dim ** -0.5+ bias = self.bias[..., self.bias.shape[-2] - seq:, :].unsqueeze(1)+ scores = (q @ self.key.transpose(-1, -2).unsqueeze(2)).add_(bias)+ probs = torch.softmax(scores, dim=-1, dtype=torch.float32).to(xq.dtype)+ out = probs @ self.value.unsqueeze(2)+ return out.permute(0, 3, 1, 2, 4).reshape(batch_size, seq, num_heads * head_dim)+@dataclassclass Llama2Config:vocab_size: int = 128320@@ -278,6 +344,9 @@ class Qwen3VL_8BConfig(Qwen3_8BConfig):rope_theta: float = 5000000.0rope_dims = [24, 20, 20]interleaved_mrope = True+ fixed_kv: bool = True+ graph_dynamic_vbar_blocks = True+ prefetch_dynamic_vbars = True@dataclassclass Qwen3VL_4BConfig(Qwen3VL_8BConfig):@@ -776,7 +845,8 @@ class Llama2_(nn.Module):super().__init__()self.config = configself.fixed_kv = getattr(config, "fixed_kv", False)- self.graph_dynamic_vbar_blocks = False+ self.graph_dynamic_vbar_blocks = getattr(config, "graph_dynamic_vbar_blocks", False)+ self.prefetch_dynamic_vbars = getattr(config, "prefetch_dynamic_vbars", False)self.vocab_size = config.vocab_sizeif self.config.transformer_type == "gemma2" or self.config.transformer_type == "gemma3":@@ -808,14 +878,15 @@ class Llama2_(nn.Module):def init_kv_cache(self, batch, capacity, device, dtype):caches = []- fixed_kv = self.fixed_kv and comfy_kitchen.flash_attention_decode_is_available(device)+ flash = getattr(comfy_kitchen, "flash_attention_decode_is_available", None)+ flash_kv = self.fixed_kv and flash is not None and flash(device)for _ in range(self.config.num_hidden_layers):- if fixed_kv:+ if flash_kv:key = torch.empty((batch, capacity, self.config.num_key_value_heads, self.config.head_dim), device=device, dtype=dtype)value = torch.empty_like(key)- position = torch.empty((batch,), device=device, dtype=torch.int64)+ pos = torch.empty((batch,), device=device, dtype=torch.int64)seqlen = torch.zeros((batch,), device=device, dtype=torch.int32)- caches.append(FixedKV(key, value, 0, position, seqlen))+ caches.append(FixedKV(key, value, 0, pos, seqlen))else:key = torch.empty((batch, self.config.num_key_value_heads, capacity, self.config.head_dim), device=device, dtype=dtype)caches.append((key, torch.empty_like(key), 0))@@ -856,7 +927,8 @@ class Llama2_(nn.Module):mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])).expand(attention_mask.shape[0], 1, seq_len, attention_mask.shape[-1])mask = mask.masked_fill(mask.to(torch.bool), torch.finfo(x.dtype).min / 4)- if seq_len > 1:+ spec_decode = fixed_kv and any(isinstance(kv, FixedKVBias) for kv in past_key_values) and 2 <= seq_len <= 6 and past_len > 0 and attention_mask is None+ if seq_len > 1 and not spec_decode: # spec verify: the staircase decode bias is causalcausal_mask = torch.empty(past_len + seq_len, past_len + seq_len, dtype=x.dtype, device=x.device).fill_(torch.finfo(x.dtype).min / 4).triu_(1)if mask is not None:mask += causal_mask@@ -865,7 +937,7 @@ class Llama2_(nn.Module):optimized_attention = optimized_attention_for_device(x.device, mask=mask is not None, small_input=True)- enable_graph = self.graph_dynamic_vbar_blocks and fixed_kv_decode+ enable_graph = self.graph_dynamic_vbar_blocks and (fixed_kv_decode or spec_decode)if enable_graph:if decode_buffers is None:x = x.clone()@@ -889,7 +961,7 @@ class Llama2_(nn.Module):elif intermediate_output < 0:intermediate_output = len(self.layers) + intermediate_output- prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.layers), x.device, {"prefetch_dynamic_vbars": getattr(self, "prefetch_dynamic_vbars", False)})+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.layers), x.device, {"prefetch_dynamic_vbars": self.prefetch_dynamic_vbars and past_key_values is not None})next_key_values = list(past_key_values) if past_key_values is not None else []for i, layer in enumerate(self.layers):if all_intermediate is not None:@@ -1001,6 +1073,18 @@ class BaseLlama:def forward(self, input_ids, *args, **kwargs):return self.model(input_ids, *args, **kwargs)+def penalty_active(repetition_penalty, presence_penalty):+ return repetition_penalty != 1.0 or (presence_penalty is not None and presence_penalty != 0.0)+++def apply_penalty(logits, repetition_penalty, presence_penalty):+ if repetition_penalty != 1.0:+ logits = torch.where(logits < 0, logits * repetition_penalty, logits / repetition_penalty)+ if presence_penalty is not None and presence_penalty != 0.0:+ logits = logits - presence_penalty+ return logits++class BaseGenerate:def logits(self, x):input = x[:, -1:]@@ -1010,7 +1094,7 @@ class BaseGenerate:module = self.model.embed_tokensif not module.comfy_cast_weights:- return torch.nn.functional.linear(input, self.model.embed_tokens.weight.to(x), None)+ return torch.nn.functional.linear(input, module.weight.to(x), None)with comfy.ops.CastBiasWeightContext(module, input, offloadable=True) as (weight, _bias):return torch.nn.functional.linear(input, weight, None)@@ -1046,6 +1130,8 @@ class BaseGenerate:compile_allocations = self.model.graph_dynamic_vbar_blocks and comfy.model_prefetch.malloc_graph_enabled(device)decode_tokens = torch.empty((embeds.shape[0], 1), dtype=torch.long, device=device)+ penalize = penalty_active(repetition_penalty, presence_penalty)+ penalty_mask = None# Generation loopcurrent_input_ids = initial_input_ids@@ -1064,9 +1150,16 @@ class BaseGenerate:extra["visual_pos_masks"] = visual_pos_masksx, _, past_key_values = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=past_key_values, input_ids=current_input_ids, position_ids=position_ids, **extra, embeds_info=(embeds_info if step == 0 else None))logits = self.logits(x)[:, -1]- next_token = self.sample_token(logits, temperature, top_k, top_p, min_p, repetition_penalty, initial_tokens + generated_token_ids, generator, do_sample=do_sample, presence_penalty=presence_penalty)+ if penalty_mask is None and do_sample and penalize:+ # allocated on the (unbracketed) first step; later steps only index_fill_ it+ penalty_mask = torch.zeros((logits.shape[-1],), dtype=torch.bool, device=device)+ if len(initial_tokens) > 0:+ penalty_mask.index_fill_(0, torch.tensor(initial_tokens, device=device), True)+ next_token = self.sample_token(logits, temperature, top_k, top_p, min_p, repetition_penalty, [], generator, do_sample=do_sample, presence_penalty=presence_penalty, penalty_mask=penalty_mask)decode_tokens.copy_(next_token)+ if penalty_mask is not None:+ penalty_mask.index_fill_(0, decode_tokens[0], True)del next_token, logits, x, embeds, position_idsif step > 0 and compile_allocations:comfy.model_prefetch.malloc_graph_end()@@ -1083,48 +1176,23 @@ class BaseGenerate:return generated_token_ids- def sample_token(self, logits, temperature, top_k, top_p, min_p, repetition_penalty, token_history, generator, do_sample=True, presence_penalty=0.0):-- if not do_sample or temperature == 0.0:- return torch.argmax(logits, dim=-1, keepdim=True)-- # Sampling mode- if len(token_history) > 0 and (repetition_penalty != 1.0 or (presence_penalty is not None and presence_penalty != 0.0)):- token_ids = torch.tensor(list(set(token_history)), device=logits.device)- token_logits = logits[:, token_ids]- if repetition_penalty != 1.0:- token_logits = torch.where(token_logits < 0, token_logits * repetition_penalty, token_logits / repetition_penalty)- if presence_penalty is not None and presence_penalty != 0.0:- token_logits = token_logits - presence_penalty- logits[:, token_ids] = token_logits+ def processed_probs(self, logits, temperature, top_k, top_p, min_p, repetition_penalty, token_history, presence_penalty=0.0, penalty_mask=None):+ # returns (probs, indices); penalty_mask [vocab] bool stands in for token_history+ if penalty_active(repetition_penalty, presence_penalty):+ if penalty_mask is not None:+ logits = torch.where(penalty_mask.unsqueeze(0), apply_penalty(logits, repetition_penalty, presence_penalty), logits)+ elif len(token_history) > 0:+ token_ids = torch.tensor(list(set(token_history)), device=logits.device)+ logits[:, token_ids] = apply_penalty(logits[:, token_ids], repetition_penalty, presence_penalty)if temperature != 1.0:logits = logits / temperature+ top_indices = Noneif top_k > 0:top_k = min(top_k, logits.shape[-1])logits, top_indices = torch.topk(logits, top_k)- if min_p > 0.0:- probs_before_filter = torch.nn.functional.softmax(logits, dim=-1)- top_probs, _ = probs_before_filter.max(dim=-1, keepdim=True)- min_threshold = min_p * top_probs- indices_to_remove = probs_before_filter < min_threshold- logits[indices_to_remove] = torch.finfo(logits.dtype).min-- if top_p < 1.0:- sorted_logits, sorted_indices = torch.sort(logits, descending=True)- cumulative_probs = torch.cumsum(torch.nn.functional.softmax(sorted_logits, dim=-1), dim=-1)- sorted_indices_to_remove = cumulative_probs > top_p- sorted_indices_to_remove[..., 0] = False- indices_to_remove = torch.zeros_like(logits, dtype=torch.bool)- indices_to_remove.scatter_(1, sorted_indices, sorted_indices_to_remove)- logits[indices_to_remove] = torch.finfo(logits.dtype).min-- probs = torch.nn.functional.softmax(logits, dim=-1)- next_token = torch.multinomial(probs, num_samples=1, generator=generator)- return top_indices.gather(1, next_token)-if min_p > 0.0:probs_before_filter = torch.nn.functional.softmax(logits, dim=-1)top_probs, _ = probs_before_filter.max(dim=-1, keepdim=True)@@ -1141,9 +1209,18 @@ class BaseGenerate:indices_to_remove.scatter_(1, sorted_indices, sorted_indices_to_remove)logits[indices_to_remove] = torch.finfo(logits.dtype).min- probs = torch.nn.functional.softmax(logits, dim=-1)+ return torch.nn.functional.softmax(logits, dim=-1), top_indices- return torch.multinomial(probs, num_samples=1, generator=generator)+ def sample_token(self, logits, temperature, top_k, top_p, min_p, repetition_penalty, token_history, generator, do_sample=True, presence_penalty=0.0, penalty_mask=None):++ if not do_sample or temperature == 0.0:+ return torch.argmax(logits, dim=-1, keepdim=True)++ probs, top_indices = self.processed_probs(logits, temperature, top_k, top_p, min_p, repetition_penalty, token_history, presence_penalty=presence_penalty, penalty_mask=penalty_mask)+ next_token = torch.multinomial(probs, num_samples=1, generator=generator)+ if top_indices is not None:+ return top_indices.gather(1, next_token)+ return next_tokenclass BaseQwen3:def logits(self, x):diff --git a/comfy/text_encoders/lt.py b/comfy/text_encoders/lt.pyindex c512a7d48..2579e1b15 100644--- a/comfy/text_encoders/lt.py+++ b/comfy/text_encoders/lt.py@@ -102,7 +102,7 @@ class Gemma3_12BModel(sd1_clip.SDClipModel):self.dtypes.add(dtype)super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={}, dtype=dtype, special_tokens={"start": 2, "pad": 0}, layer_norm_hidden_state=False, model_class=comfy.text_encoders.llama.Gemma3_12B, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, model_options=model_options)- def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty):+ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty, mtp=True):tokens_only = [[t[0] for t in b] for b in tokens]embeds, _, _, _ = self.process_tokens(tokens_only, self.execution_device)return self.transformer.generate(embeds, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, stop_tokens=[106], presence_penalty=presence_penalty) # 106 is <end_of_turn>@@ -205,7 +205,7 @@ class LTXAVTEModel(torch.nn.Module):return out.to(device=out_device, dtype=torch.float), pooled, extra- def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty):+ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty, mtp=True):return self.gemma3_12b.generate(tokens[self.text_encoder_key], do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty)def load_sd(self, sd):diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.pyindex 304a4357f..29b5d469d 100644--- a/comfy/text_encoders/qwen35.py+++ b/comfy/text_encoders/qwen35.py@@ -2,14 +2,53 @@ import torchimport torch.nn as nnimport torch.nn.functional as Ffrom dataclasses import dataclass, field+from tqdm import tqdm+import contextlibimport os+import warningsimport comfy.model_management+import comfy.model_prefetch+import comfy.ops+import comfy_kitchen+from comfy.quant_ops import QuantizedTensorfrom comfy.ldm.modules.attention import optimized_attention_for_devicefrom comfy import sd1_clipimport comfy.text_encoders.qwen_vl-from .llama import BaseLlama, BaseGenerate, Llama2_, MLP, RMSNorm, apply_rope+from .llama import BaseLlama, BaseGenerate, FixedKV, FixedKVBias, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix+++@dataclass+class LinearKV(FixedKV):+ # DeltaNet state on the FixedKV interface: key=conv_state, value=recurrent_state (fp32)+ g_decay: torch.Tensor = None+ dt_bias: torch.Tensor = None+ snapshots: list = None # [(recurrent, conv)] taken after step 1, 2, ... of the last verify+ last_seq: int = 1+ snap_backing: torch.Tensor = None+ conv_snap_backing: torch.Tensor = None+ norm_weight: torch.Tensor = None++ def prepare(self, num_tokens):+ pass++ def rollback(self, discard=1):+ # discard the rejected tail: restore the snapshot taken after the last kept token+ rec, conv = self.snapshots[self.last_seq - discard - 1]+ self.recurrent_state.copy_(rec)+ self.conv_state.copy_(conv)+ self.index -= discard++ @property+ def conv_state(self):+ return self.key++ @property+ def recurrent_state(self):+ return self.value++def _qwen35_layer_types(n):@@ -48,6 +87,7 @@ class Qwen35Config:transformer_type: str = "qwen35_2b"rope_dims: list = Nonerope_scale: float = None+ mtp: bool = FalseQWEN35_VISION_DEFAULTS = dict(hidden_size=1024, num_heads=16, intermediate_size=4096, depth=24, patch_size=16, temporal_patch_size=2, in_channels=3, spatial_merge_size=2, num_position_embeddings=2304)@@ -135,18 +175,6 @@ def torch_chunk_gated_delta_rule(query, key, value, g, beta, chunk_size=64, initreturn core_attn_out, last_recurrent_state-def torch_causal_conv1d_update(x, conv_state, weight, bias=None):- # conv_state: [B, channels, kernel_size-1], x: [B, channels, 1]- # weight: [channels, kernel_size]- state_len = conv_state.shape[-1]- combined = torch.cat([conv_state, x], dim=-1).to(weight.dtype) # [B, channels, kernel_size]- conv_state.copy_(combined[:, :, -state_len:])- out = (combined * weight).sum(dim=-1, keepdim=True) # [B, channels, 1]- if bias is not None:- out = out + bias.unsqueeze(0).unsqueeze(-1)- return F.silu(out).to(x.dtype)--# GatedDeltaNet - Linear Attention Layerclass GatedDeltaNet(nn.Module):@@ -185,25 +213,55 @@ class GatedDeltaNet(nn.Module):use_recurrent = (past_key_value is not None- and past_key_value[2] > 0- and seq_len == 1+ and past_key_value.index > 0+ and seq_len <= 6)+ fused_available = getattr(comfy_kitchen, "gated_delta_decode_is_available", None)+ use_fused = (use_recurrent and fused_available is not None and fused_available(x.device, self.key_head_dim, self.value_head_dim)+ and (seq_len == 1 or past_key_value.snap_backing is not None))+# Projections (shared)- mixed_qkv = self.in_proj_qkv(x).transpose(1, 2) # [B, conv_dim, seq_len]+ proj = self.in_proj_qkv(x) # [B, seq_len, conv_dim]z = self.in_proj_z(x)++ if use_fused:+ # decode: kitchen conv step, then gates + delta rule + gated norm in one kernel+ if seq_len > 1:+ past_key_value.last_seq = seq_len+ with comfy.ops.CastBiasWeightContext(self.conv1d, proj, offloadable=True) as (conv_weight, conv_bias):+ conv_out = comfy_kitchen.deltanet_conv_step(proj, past_key_value.conv_state, conv_weight, conv_bias,+ past_key_value.conv_snap_backing[:seq_len - 1] if seq_len > 1 else None)+ with comfy.ops.CastBiasWeightContext(self.in_proj_a, x, offloadable=True) as (w_a, _), \+ comfy.ops.CastBiasWeightContext(self.in_proj_b, x, offloadable=True) as (w_b, _):+ if isinstance(w_a, QuantizedTensor):+ w_a = w_a.dequantize()+ if isinstance(w_b, QuantizedTensor):+ w_b = w_b.dequantize()+ core_attn_out = comfy_kitchen.gated_delta_decode_fused(+ conv_out, x, w_a, w_b, past_key_value.dt_bias, past_key_value.g_decay, past_key_value.recurrent_state,+ self.key_dim, self.num_key_heads, self.key_head_dim ** -0.5, z, past_key_value.norm_weight, self.norm.eps,+ past_key_value.snap_backing[:seq_len - 1] if seq_len > 1 else None)+ return self.out_proj(core_attn_out.reshape(batch_size, seq_len, -1)), past_key_value++ mixed_qkv = proj.transpose(1, 2) # [B, conv_dim, seq_len]b = self.in_proj_b(x)a = self.in_proj_a(x)# Conv1dif use_recurrent:- recurrent_state, conv_state, step_index = past_key_value- conv_weight = comfy.model_management.cast_to_device(self.conv1d.weight, mixed_qkv.device, mixed_qkv.dtype).squeeze(1)- conv_bias = comfy.model_management.cast_to_device(self.conv1d.bias, mixed_qkv.device, mixed_qkv.dtype) if self.conv1d.bias is not None else None- mixed_qkv = torch_causal_conv1d_update(mixed_qkv, conv_state, conv_weight, conv_bias)+ # decode: exact-width causal window, weight resolved via the vbar-aware context+ combined = torch.cat([past_key_value.conv_state, mixed_qkv], dim=-1)+ if seq_len > 1:+ past_key_value.last_seq = seq_len+ for s in range(seq_len - 1):+ past_key_value.snapshots[s][1].copy_(combined[:, :, 1 + s:1 + s + self.conv_kernel_size - 1])+ past_key_value.conv_state.copy_(combined[:, :, seq_len:])+ with comfy.ops.CastBiasWeightContext(self.conv1d, combined, offloadable=True) as (conv_weight, conv_bias):+ mixed_qkv = F.silu(F.conv1d(combined, conv_weight, conv_bias, groups=self.conv1d.groups))else:if past_key_value is not None:- recurrent_state, conv_state, step_index = past_key_value+ conv_state = past_key_value.conv_stateconv_state_init = F.pad(mixed_qkv, (self.conv_kernel_size - mixed_qkv.shape[-1], 0))conv_state.copy_(conv_state_init[:, :, -conv_state.shape[-1]:])mixed_qkv = F.silu(self.conv1d(mixed_qkv)[:, :, :seq_len])@@ -211,47 +269,51 @@ class GatedDeltaNet(nn.Module):# Split QKV and compute beta/gmixed_qkv = mixed_qkv.transpose(1, 2) # [B, seq_len, conv_dim]query, key, value = mixed_qkv.split([self.key_dim, self.key_dim, self.value_dim], dim=-1)+ if use_recurrent:+ g_decay, dt_bias = past_key_value.g_decay, past_key_value.dt_bias+ else:+ g_decay = -comfy.model_management.cast_to_device(self.A_log, x.device, torch.float32).exp()+ dt_bias = comfy.model_management.cast_to_device(self.dt_bias, x.device, torch.float32)+ if past_key_value is not None:+ past_key_value.g_decay = g_decay+ past_key_value.dt_bias = dt_bias+ past_key_value.norm_weight = comfy.model_management.cast_to(self.norm.weight, dtype=x.dtype, device=x.device)+beta = b.sigmoid()- g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias.float())+ g = g_decay * F.softplus(a.float() + dt_bias)+ query = query.reshape(batch_size, seq_len, self.num_key_heads, self.key_head_dim)+ key = key.reshape(batch_size, seq_len, self.num_key_heads, self.key_head_dim)+ value = value.reshape(batch_size, seq_len, self.num_value_heads, self.value_head_dim)+ if self.num_value_heads != self.num_key_heads:+ rep = self.num_value_heads // self.num_key_heads+ query = query.repeat_interleave(rep, dim=2)+ key = key.repeat_interleave(rep, dim=2)# Delta ruleif use_recurrent:- # single-token path: work in [B, heads, dim] without seq dim- query = query.reshape(batch_size, self.num_key_heads, self.key_head_dim)- key = key.reshape(batch_size, self.num_key_heads, self.key_head_dim)- value = value.reshape(batch_size, self.num_value_heads, self.value_head_dim)-- if self.num_value_heads != self.num_key_heads:- rep = self.num_value_heads // self.num_key_heads- query = query.repeat_interleave(rep, dim=1)- key = key.repeat_interleave(rep, dim=1)-scale = self.key_head_dim ** -0.5q = F.normalize(query.float(), dim=-1) * scalek = F.normalize(key.float(), dim=-1)v = value.float()- beta_t = beta.reshape(batch_size, -1)- g_t = g.reshape(batch_size, -1).exp()+ beta_t = beta.reshape(batch_size, seq_len, -1)+ g_t = g.reshape(batch_size, seq_len, -1).exp()# In-place state update: [B, heads, k_dim, v_dim]- recurrent_state.mul_(g_t[:, :, None, None])- kv_mem = torch.einsum('bhk,bhkv->bhv', k, recurrent_state)- delta = (v - kv_mem) * beta_t[:, :, None]- recurrent_state.add_(k.unsqueeze(-1) * delta.unsqueeze(-2))- core_attn_out = torch.einsum('bhk,bhkv->bhv', q, recurrent_state)-- core_attn_out = core_attn_out.to(x.dtype).unsqueeze(1)- present_key_value = (recurrent_state, conv_state, step_index + 1)+ recurrent_state = past_key_value.recurrent_state+ outs = []+ for s in range(seq_len):+ recurrent_state.mul_(g_t[:, s, :, None, None])+ kv_mem = torch.einsum('bhk,bhkv->bhv', k[:, s], recurrent_state)+ delta = (v[:, s] - kv_mem) * beta_t[:, s, :, None]+ # rank-1 update via baddbmm_: no materialized [B, H, D, D] outer-product temp+ recurrent_state.view(-1, self.key_head_dim, self.value_head_dim).baddbmm_(+ k[:, s].reshape(-1, self.key_head_dim, 1), delta.reshape(-1, 1, self.value_head_dim))+ outs.append(torch.einsum('bhk,bhkv->bhv', q[:, s], recurrent_state))+ if seq_len > 1 and s < seq_len - 1:+ past_key_value.snapshots[s][0].copy_(recurrent_state)+ core_attn_out = torch.stack(outs, dim=1).to(x.dtype)+ present_key_value = past_key_valueelse:- query = query.reshape(batch_size, seq_len, -1, self.key_head_dim)- key = key.reshape(batch_size, seq_len, -1, self.key_head_dim)- value = value.reshape(batch_size, seq_len, -1, self.value_head_dim)-- if self.num_value_heads != self.num_key_heads:- rep = self.num_value_heads // self.num_key_heads- query = query.repeat_interleave(rep, dim=2)- key = key.repeat_interleave(rep, dim=2)-core_attn_out, last_recurrent_state = torch_chunk_gated_delta_rule(query, key, value, g=g, beta=beta,initial_state=None,@@ -261,8 +323,8 @@ class GatedDeltaNet(nn.Module):present_key_value = Noneif past_key_value is not None:if last_recurrent_state is not None:- recurrent_state.copy_(last_recurrent_state.to(recurrent_state.dtype))- present_key_value = (recurrent_state, conv_state, step_index + seq_len)+ past_key_value.recurrent_state.copy_(last_recurrent_state.to(past_key_value.recurrent_state.dtype))+ present_key_value = past_key_value# Gated norm + output projection (shared)core_attn_out = self.norm(core_attn_out.reshape(-1, self.value_head_dim), z.reshape(-1, self.value_head_dim))@@ -271,29 +333,6 @@ class GatedDeltaNet(nn.Module):# GatedAttention - Full Attention with output gating-def precompute_partial_rope(head_dim, rotary_dim, position_ids, theta, device=None, mrope_section=None):- """Compute RoPE frequencies for partial rotary embeddings."""- theta_numerator = torch.arange(0, rotary_dim, 2, device=device).float()- inv_freq = 1.0 / (theta ** (theta_numerator / rotary_dim))-- inv_freq_expanded = inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)- position_ids_expanded = position_ids[:, None, :].float()- freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)- emb = torch.cat((freqs, freqs), dim=-1)- cos = emb.cos()- sin = emb.sin()-- if mrope_section is not None and position_ids.shape[0] == 3:- mrope_section_2 = [s * 2 for s in mrope_section]- cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section_2, dim=-1))], dim=-1).unsqueeze(0)- sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section_2, dim=-1))], dim=-1).unsqueeze(0)-- cos = cos.unsqueeze(1)- sin = sin.unsqueeze(1)- sin_split = sin.shape[-1] // 2- return (cos, sin[..., :sin_split], -sin[..., sin_split:])--def apply_partial_rope(xq, xk, freqs_cis, rotary_dim):"""Apply RoPE to only the first rotary_dim dimensions."""xq_rot = xq[..., :rotary_dim]@@ -350,26 +389,16 @@ class GatedAttention(nn.Module):xq, xk = apply_partial_rope(xq, xk, freqs_cis, self.rotary_dim)# KV cache- present_key_value = None- if past_key_value is not None:- past_key, past_value, index = past_key_value- num_tokens = xk.shape[2]- if past_key.shape[2] >= (index + num_tokens):- past_key[:, :, index:index + num_tokens] = xk- past_value[:, :, index:index + num_tokens] = xv- xk = past_key[:, :, :index + num_tokens]- xv = past_value[:, :, :index + num_tokens]- present_key_value = (past_key, past_value, index + num_tokens)- else:- if index > 0:- xk = torch.cat((past_key[:, :, :index], xk), dim=2)- xv = torch.cat((past_value[:, :, :index], xv), dim=2)- present_key_value = (xk, xv, index + num_tokens)+ present_key_value = past_key_value+ if past_key_value is not None and seq_length <= 6 and attention_mask is None:+ output = past_key_value.decode(xq, xk, xv, self.num_kv_heads)+ else:+ if past_key_value is not None:+ xk, xv = past_key_value.append(xk, xv)+ gqa_kwargs = {"enable_gqa": True} if self.num_heads != self.num_kv_heads else {}+ output = optimized_attention(xq, xk, xv, self.num_heads, mask=attention_mask, skip_reshape=True, **gqa_kwargs)- gqa_kwargs = {"enable_gqa": True} if self.num_heads != self.num_kv_heads else {}- output = optimized_attention(xq, xk, xv, self.num_heads, mask=attention_mask, skip_reshape=True, **gqa_kwargs)output = output * gate.sigmoid()-return self.o_proj(output), present_key_value@@ -387,13 +416,15 @@ class Qwen35TransformerBlock(nn.Module):self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)def forward(self, x, attention_mask=None, freqs_cis=None, optimized_attention=None, past_key_value=None):+ output = xif self.layer_type == "linear_attention":h, present_key_value = self.linear_attn(self.input_layernorm(x), attention_mask=attention_mask, past_key_value=past_key_value)else:h, present_key_value = self.self_attn(self.input_layernorm(x), attention_mask=attention_mask, freqs_cis=freqs_cis, optimized_attention=optimized_attention, past_key_value=past_key_value)- x = x + h- x = x + self.mlp(self.post_attention_layernorm(x))+ # in-place into the input buffer so CUDA-graph replays land in the static x+ x = torch.add(x, h, out=output)+ x = torch.add(x, self.mlp(self.post_attention_layernorm(x)), out=output)return x, present_key_value@@ -402,6 +433,8 @@ class Qwen35Transformer(Llama2_):def __init__(self, config, device=None, dtype=None, ops=None):nn.Module.__init__(self)self.config = config+ self.prefetch_dynamic_vbars = True+ self.graph_dynamic_vbar_blocks = Trueself.vocab_size = config.vocab_sizeself.embed_tokens = ops.Embedding(config.vocab_size, config.hidden_size, device=device, dtype=dtype)self.layers = nn.ModuleList([@@ -417,21 +450,10 @@ class Qwen35Transformer(Llama2_):if config.lm_head:self.lm_head = ops.Linear(config.hidden_size, config.vocab_size, bias=False, device=device, dtype=dtype)- def get_past_len(self, past_key_values):- for i, layer in enumerate(self.layers):- if layer.layer_type == "full_attention":- if len(past_key_values) > i:- return past_key_values[i][2]- break- return 0-def compute_freqs_cis(self, position_ids, device):rotary_dim = int(self.config.head_dim * self.config.partial_rotary_factor)- return precompute_partial_rope(- self.config.head_dim, rotary_dim, position_ids,- self.config.rope_theta, device=device,- mrope_section=self.config.mrope_section,- )+ return precompute_freqs_cis(rotary_dim, position_ids, self.config.rope_theta,+ rope_dims=self.config.mrope_section, interleaved_mrope=True, device=device)# Vision Encoder@@ -671,6 +693,23 @@ class Qwen35VisionModel(nn.Module):return merged, deepstack_featuresreturn merged+class MTPHead(nn.Module):+ # MTP draft head: fc(cat[norm(embed), norm(hidden)]) -> attention block (own KV) -> norm+ def __init__(self, config, device=None, dtype=None, ops=None):+ super().__init__()+ self.fc = ops.Linear(config.hidden_size * 2, config.hidden_size, bias=False, device=device, dtype=dtype)+ self.pre_fc_norm_embedding = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)+ self.pre_fc_norm_hidden = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)+ self.layers = nn.ModuleList([Qwen35TransformerBlock(config, index=3, device=device, dtype=dtype, ops=ops)])+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)++ def forward(self, embeds, hidden, freqs_cis, past_key_value):+ x = self.fc(torch.cat([self.pre_fc_norm_embedding(embeds), self.pre_fc_norm_hidden(hidden)], dim=-1))+ attention = optimized_attention_for_device(x.device, mask=False, small_input=True)+ x, _ = self.layers[0](x, attention_mask=None, freqs_cis=freqs_cis, optimized_attention=attention, past_key_value=past_key_value)+ return self.norm(x), x # (for logits, pre-norm hidden for recursive drafting)++# Model Wrapperclass Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):model_type = "qwen35_2b"@@ -680,6 +719,9 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):config = _make_config(self.model_type, config_dict)self.num_layers = config.num_hidden_layersself.model = Qwen35Transformer(config, device=device, dtype=dtype, ops=operations)+ self.mtp = None+ if config.mtp:+ self.mtp = MTPHead(config, device=device, dtype=dtype, ops=operations)vision_overrides = QWEN35_MODELS.get(self.model_type, {}).get("vision", {})vision_config = {**QWEN35_VISION_DEFAULTS, **vision_overrides, "out_hidden_size": config.hidden_size}self.visual = Qwen35VisionModel(vision_config, device=device, dtype=dtype, ops=operations)@@ -687,7 +729,8 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):def preprocess_embed(self, embed, device):if embed["type"] == "image":- image, grid = comfy.text_encoders.qwen_vl.process_qwen2vl_images(embed["data"], patch_size=16)+ # Qwen3.5 normalizes to [-1, 1] (mean/std 0.5), same as Qwen3-VL.+ image, grid = comfy.text_encoders.qwen_vl.process_qwen2vl_images(embed["data"], patch_size=16, image_mean=[0.5, 0.5, 0.5], image_std=[0.5, 0.5, 0.5])return self.visual(image.to(device, dtype=torch.float32), grid), gridreturn None, None@@ -695,9 +738,271 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, embeds.shape[1], embeds.device)return super().forward(x, attention_mask=attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=final_layer_norm_intermediate, dtype=dtype, position_ids=position_ids, past_key_values=past_key_values)+ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, **kwargs):+ mtp = kwargs.pop("mtp", True)+ if self.mtp is None or not mtp or kwargs.get("position_ids") is not None or kwargs.get("initial_input_ids") is not None:+ return super().generate(embeds=embeds, do_sample=do_sample, max_length=max_length, temperature=temperature,+ top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty,+ seed=seed, stop_tokens=stop_tokens, **kwargs)+ sampling = None+ if do_sample and temperature != 0.0:+ sampling = {"temperature": temperature, "top_k": top_k, "top_p": top_p, "min_p": min_p,+ "repetition_penalty": repetition_penalty,+ "presence_penalty": kwargs.get("presence_penalty", 0.0) or 0.0,+ "seed": seed if seed is not None else 42}+ fixed_depth = None if mtp is True else max(2, min(5, int(mtp)))+ return self._generate_mtp(embeds, max_length, stop_tokens, sampling=sampling, fixed_depth=fixed_depth)++ def _generate_mtp(self, embeds, max_length, stop_tokens, sampling=None, fixed_depth=None):+ device = embeds.device+ cfg = self.model.config+ if stop_tokens is None:+ stop_tokens = cfg.stop_tokens+ dt = torch.bfloat16 if comfy.model_management.should_use_bf16(device) else torch.float32+ embeds = embeds.to(dt)+ if embeds.ndim == 2:+ embeds = embeds.unsqueeze(0)+ # greedy drafts 3 deep (5 after the probe); sampled stays at 2+ depth = fixed_depth if fixed_depth is not None else (3 if sampling is None else 2)+ cap = embeds.shape[1] + max_length + 7+ pkv = self.init_kv_cache(embeds.shape[0], cap, device, dt)+ # repair window: drafting ahead plus a near-full rollback+ mtp_kv = FixedKVBias.zeros(embeds.shape[0], cfg.num_key_value_heads, cap, cfg.head_dim, device, dt,+ FixedKVBias.shared(cap, device, dt))+ head = self.model.lm_head if hasattr(self.model, "lm_head") else self.model.embed_tokens++ def verify_logits(x):+ if not head.comfy_cast_weights:+ return F.linear(x, head.weight.to(x), None)+ with comfy.ops.CastBiasWeightContext(head, x, offloadable=True) as (w, _bias):+ return F.linear(x, w)++ # the draft graph bakes these weights' addresses: keep them resident for the generate+ hot = list({id(m): m for m in [head, self.model.embed_tokens, *self.mtp.modules()]}.values())+ pinned = [m for m in hot if hasattr(m, "_v")]++ generator = None+ if sampling is not None:+ generator = torch.Generator(device=device).manual_seed(sampling["seed"])+ penalized = sampling is not None and penalty_active(sampling["repetition_penalty"], sampling["presence_penalty"])++ # rope table once per generate, sliced per draft+ ftab = rope_matrix(self.model.compute_freqs_cis(torch.arange(cap, device=device, dtype=torch.float).unsqueeze(0), device))++ def freqs_at(p, n=1):+ return ftab[:, :, p:p + n]++ # cross-step state lives in static carriers: nothing allocated inside a step may outlive it+ nt_buf = torch.empty((embeds.shape[0], 1), device=device, dtype=torch.long)+ h_buf = torch.empty((embeds.shape[0], 1, cfg.hidden_size), device=device, dtype=dt)+ x, _, _ = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=pkv)+ lg0 = self.logits(x)[:, -1]+ if sampling is None:+ nt_buf.copy_(lg0.argmax(dim=-1, keepdim=True))+ else:+ nt_buf.copy_(self.sample_token(lg0, sampling["temperature"], sampling["top_k"], sampling["top_p"], sampling["min_p"],+ sampling["repetition_penalty"], [], generator, presence_penalty=sampling["presence_penalty"]))+ pen_mask = torch.zeros((lg0.shape[-1],), device=device, dtype=torch.bool) if penalized else None+ h_buf.copy_(x[:, -1:, :])+ del x, lg0+ ids = [nt_buf[0].item()]+ if penalized:+ pen_mask.index_fill_(0, nt_buf.reshape(-1), True)+ pos = embeds.shape[1]+ progress = comfy.utils.ProgressBar(max_length)+ console = tqdm(total=max_length, desc="Generating tokens", initial=1)++ def update_progress(n):+ progress.update(n)+ console.update(n)++ verify_buffers = None+ snapshot_bytes = sum(kv.recurrent_state.numel() * 4 + kv.conv_state.numel() * kv.conv_state.element_size()+ for kv in pkv if isinstance(kv, LinearKV))++ def set_depth(d):+ nonlocal depth, verify_buffers+ depth = d+ for kv in pkv:+ if isinstance(kv, LinearKV):+ # snapshot views share one backing slab so the fused kernel can write them+ kv.snap_backing = torch.empty((d,) + tuple(kv.recurrent_state.shape), device=device, dtype=torch.float32)+ kv.conv_snap_backing = torch.empty((d,) + tuple(kv.conv_state.shape), device=device, dtype=kv.conv_state.dtype)+ kv.snapshots = [(kv.snap_backing[s], kv.conv_snap_backing[s]) for s in range(d)]+ # static hidden/rope buffers: graphed layers bake their input addresses+ verify_buffers = (torch.empty((embeds.shape[0], d + 1, cfg.hidden_size), device=device, dtype=dt), freqs_at(pos, d + 1).clone())++ set_depth(depth)+ use_graph = (device.type == "cuda"+ and comfy.model_management.NUM_STREAMS > 0+ and not comfy.model_management.args.disable_cuda_graphs)+ compile_allocations = use_graph and self.model.graph_dynamic_vbar_blocks and comfy.model_prefetch.malloc_graph_enabled(device)+ draft_state = {}++ def drop_draft_graph():+ # free inside torch API calls so the allocator's benign notices stay catchable+ with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ for t in (draft_state.get("d"), draft_state.get("r"), *draft_state.get("keep", ())):+ if t is not None:+ t.set_()+ g = draft_state.pop("graph", None)+ if g is not None:+ g.reset()+ draft_state.clear()++ def draft_capture():+ # captured outside the compiler bracket; its static buffers live for the generate+ ds = draft_state+ mtp_kv.prepare(1)+ ds["tok"] = nt_buf.clone()+ ds["hid"] = h_buf.clone()+ ds["f"] = freqs_at(pos).clone()+ side = torch.cuda.Stream()+ side.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(side):+ for _ in range(2):+ n1, r1 = self.mtp(self.model.embed_tokens(ds["tok"]).to(dt), ds["hid"], ds["f"], mtp_kv)+ self.logits(n1)[:, -1].argmax(dim=-1, keepdim=True)+ torch.cuda.current_stream().wait_stream(side)+ del n1, r1 # freed before the capture, not shadowed inside it+ g = torch.cuda.CUDAGraph()+ with torch.cuda.graph(g, capture_error_mode="thread_local"):+ n1, r1 = self.mtp(self.model.embed_tokens(ds["tok"]).to(dt), ds["hid"], ds["f"], mtp_kv)+ lg1 = self.logits(n1)+ ds["d"] = lg1[:, -1].argmax(dim=-1, keepdim=True)+ ds["r"] = r1+ ds["keep"] = (n1, lg1) # captured allocations must outlive the graph+ ds["graph"] = g++ def draft(token, hidden, p):+ # one drafted token: mtp head + lm_head argmax, graph-replayed on cuda+ mtp_kv.prepare(1)+ f = freqs_at(p)+ if not use_graph:+ n1, r1 = self.mtp(self.model.embed_tokens(token).to(dt), hidden, f, mtp_kv)+ mtp_kv.advance(1)+ return self.logits(n1)[:, -1].argmax(dim=-1, keepdim=True), r1+ ds = draft_state+ ds["tok"].copy_(token)+ ds["hid"].copy_(hidden)+ ds["f"].copy_(f)+ ds["graph"].replay()+ mtp_kv.advance(1)+ return ds["d"], ds["r"]++ def verify_sample(lg, drafts):+ # accept draft i w.p. p_i(draft), else sample the residual; all depth+1 columns as one batch+ s = sampling+ rows = lg[0].float()+ if penalized:+ # committed tokens penalize every column, each draft the columns after it+ mask = pen_mask.unsqueeze(0).repeat(depth + 1, 1)+ for c, d in enumerate(drafts):+ mask[c + 1:].index_fill_(1, d.reshape(-1), True)+ rows = torch.where(mask, apply_penalty(rows, s["repetition_penalty"], s["presence_penalty"]), rows)+ probs, idx = self.processed_probs(rows, s["temperature"], s["top_k"], s["top_p"], s["min_p"], 1.0, [])+ dr = torch.cat(drafts, dim=1).reshape(-1, 1)+ if idx is None:+ p_draft = probs[:depth].gather(1, dr).reshape(-1)+ w = probs.clone()+ w[:depth].scatter_(1, dr, 0.0)+ else:+ hit = idx[:depth] == dr+ p_draft = (probs[:depth] * hit).sum(1)+ w = probs.clone()+ w[:depth].masked_fill_(hit, 0.0)+ # residual: p with the draft zeroed (the rescue term only matters when discarded anyway)+ w = w + (w.sum(1, keepdim=True) == 0) * probs+ tok = torch.multinomial(w, num_samples=1, generator=generator)+ corr = tok if idx is None else idx.gather(1, tok)+ u = torch.rand(depth, device=device, generator=generator)+ accepted = (u < p_draft).long().cumprod(0).sum()+ return dr, corr, accepted++ def step():+ # scoped so every temporary dies before the compiler bracket closes+ nonlocal pos+ drafts = []+ tok_in, hid_in = nt_buf, h_buf+ for k in range(depth):+ dk, rk = draft(tok_in, hid_in, pos + k)+ if k < depth - 1:+ dk = dk.clone() # later replays overwrite the static output+ drafts.append(dk)+ tok_in, hid_in = dk, rk+ ev = self.model.embed_tokens(torch.cat([nt_buf] + drafts, dim=1)).to(dt)+ x, _, _ = self.model.forward(None, embeds=ev, attention_mask=None, past_key_values=pkv, decode_buffers=verify_buffers)+ # all verify positions in one lm_head GEMV, accept decided GPU-side, one sync+ lg = verify_logits(x)+ if sampling is None:+ toks = lg.argmax(dim=-1)+ vals = torch.cat([toks[0]] + [d[0] for d in drafts]).tolist()+ t, dr = vals[:depth + 1], vals[depth + 1:]+ accepts = 0+ while accepts < depth and t[accepts] == dr[accepts]:+ accepts += 1+ next_toks = tuple(toks[:, i:i + 1] for i in range(depth + 1))+ commit = tuple(dr[:accepts]) + (t[accepts],)+ else:+ dr, corr, accepted = verify_sample(lg, drafts)+ vals = torch.cat([dr[:, 0], corr[:, 0], accepted.reshape(1)]).tolist()+ dr, cv, accepts = vals[:depth], vals[depth:2 * depth + 1], vals[-1]+ next_toks = tuple(corr[i:i + 1] for i in range(depth + 1))+ commit = tuple(dr[:accepts]) + (cv[accepts],)+ if accepts < depth:+ for kv in pkv:+ kv.rollback(depth - accepts)+ if accepts < depth - 1:+ mtp_kv.rollback(depth - 1 - accepts) # mtp entries fed by a rejected draft token+ nt_buf.copy_(next_toks[accepts])+ h_buf.copy_(x[:, accepts:accepts + 1, :])+ if penalized:+ for d in drafts[:accepts]:+ pen_mask.index_fill_(0, d.reshape(-1), True)+ pen_mask.index_fill_(0, nt_buf.reshape(-1), True)+ pos += accepts + 1+ return accepts, commit++ probe = None if fixed_depth is not None else [0, 0] # steps, accepted drafts+ try:+ if pinned:+ comfy.model_prefetch.pin_modules(pinned, device, dt)+ if use_graph and len(ids) < max_length and ids[-1] not in stop_tokens:+ draft_capture()+ while len(ids) < max_length and ids[-1] not in stop_tokens:+ with (comfy.model_prefetch.malloc_graph_scope(device) if compile_allocations else contextlib.nullcontext()):+ accepts, commit = step()+ commit = list(commit[:max_length - len(ids)])+ stop = next((i for i, t in enumerate(commit) if t in stop_tokens), None)+ if stop is not None:+ del commit[stop + 1:]+ ids.extend(commit)+ update_progress(len(commit))+ if probe is not None:+ probe[0] += 1+ probe[1] += accepts+ if probe[0] == 32:+ # deepen once acceptance sustains it and a recapture round can amortize+ if (sampling is None and max_length - len(ids) > 512 and 1 + probe[1] / probe[0] >= 2.2+ and 5 * snapshot_bytes < comfy.model_management.get_free_memory(device)):+ comfy.model_prefetch.cleanup_prefetch_queues()+ drop_draft_graph()+ set_depth(5)+ if use_graph:+ draft_capture()+ probe = None+ finally:+ console.close()+ drop_draft_graph()+ if pinned:+ comfy.model_prefetch.cleanup_prefetched_modules(None, pinned)+ return ids+def init_kv_cache(self, batch, max_cache_len, device, execution_dtype):model_config = self.model.configpast_key_values = []+ shared = FixedKVBias.shared(max_cache_len, device, execution_dtype)for i in range(model_config.num_hidden_layers):if model_config.layer_types[i] == "linear_attention":recurrent_state = torch.zeros(@@ -709,13 +1014,9 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):[batch, conv_dim, model_config.conv_kernel_size - 1],device=device, dtype=execution_dtype)- past_key_values.append((recurrent_state, conv_state, 0))+ past_key_values.append(LinearKV(conv_state, recurrent_state, 0, None, None))else:- past_key_values.append((- torch.empty([batch, model_config.num_key_value_heads, max_cache_len, model_config.head_dim], device=device, dtype=execution_dtype),- torch.empty([batch, model_config.num_key_value_heads, max_cache_len, model_config.head_dim], device=device, dtype=execution_dtype),- 0- ))+ past_key_values.append(FixedKVBias.zeros(batch, model_config.num_key_value_heads, max_cache_len, model_config.head_dim, device, execution_dtype, shared))return past_key_values# Tokenizer and Text Encoder Wrappers@@ -777,19 +1078,28 @@ class Qwen35ImageTokenizer(sd1_clip.SD1Tokenizer):class Qwen35ClipModel(sd1_clip.SDClipModel):- def __init__(self, device="cpu", layer="hidden", layer_idx=-2, dtype=None, attention_mask=True, model_options={}, model_type="qwen35_2b"):+ def __init__(self, device="cpu", layer="hidden", layer_idx=-2, dtype=None, attention_mask=True, model_options={}, model_type="qwen35_2b", mtp=False):class Qwen35_(Qwen35):passQwen35_.model_type = model_type- super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={},+ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={"mtp": True} if mtp else {},dtype=dtype, special_tokens={"pad": 248044}, layer_norm_hidden_state=False,model_class=Qwen35_, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, model_options=model_options)+ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0, mtp=True):+ if isinstance(tokens, dict):+ tokens = next(iter(tokens.values()))+ tokens_only = [[t[0] for t in b] for b in tokens]+ embeds, _, _, embeds_info = self.process_tokens(tokens_only, self.execution_device)+ position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, embeds.shape[1], embeds.device)+ return self.transformer.generate(embeds, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed,+ presence_penalty=presence_penalty, position_ids=position_ids, mtp=mtp)+class Qwen35TEModel(sd1_clip.SD1ClipModel):- def __init__(self, device="cpu", dtype=None, model_options={}, model_type="qwen35_2b"):- clip_model = lambda **kw: Qwen35ClipModel(**kw, model_type=model_type)+ def __init__(self, device="cpu", dtype=None, model_options={}, model_type="qwen35_2b", mtp=False):+ clip_model = lambda **kw: Qwen35ClipModel(**kw, model_type=model_type, mtp=mtp)super().__init__(device=device, dtype=dtype, name=model_type, clip_model=clip_model, model_options=model_options)@@ -800,7 +1110,7 @@ def tokenizer(model_type="qwen35_2b"):return Qwen35ImageTokenizer_-def te(dtype_llama=None, llama_quantization_metadata=None, model_type="qwen35_2b"):+def te(dtype_llama=None, llama_quantization_metadata=None, model_type="qwen35_2b", mtp=False):class Qwen35TEModel_(Qwen35TEModel):def __init__(self, device="cpu", dtype=None, model_options={}):if dtype_llama is not None:@@ -808,5 +1118,5 @@ def te(dtype_llama=None, llama_quantization_metadata=None, model_type="qwen35_2bif llama_quantization_metadata is not None:model_options = model_options.copy()model_options["quantization_metadata"] = llama_quantization_metadata- super().__init__(device=device, dtype=dtype, model_options=model_options, model_type=model_type)+ super().__init__(device=device, dtype=dtype, model_options=model_options, model_type=model_type, mtp=mtp)return Qwen35TEModel_diff --git a/comfy/text_encoders/qwen3vl.py b/comfy/text_encoders/qwen3vl.pyindex fd7d0d267..f16b2a7e4 100644--- a/comfy/text_encoders/qwen3vl.py+++ b/comfy/text_encoders/qwen3vl.py@@ -127,7 +127,7 @@ class Qwen3VLClipModel(sd1_clip.SDClipModel):model_class=_make_qwen3vl_model(model_type), enable_attention_masks=attention_mask,return_attention_masks=attention_mask, model_options=model_options)- def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0):+ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0, mtp=True):if isinstance(tokens, dict):tokens = next(iter(tokens.values()))tokens_only = [[t[0] for t in b] for b in tokens]diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.pyindex ef5858a7d..243834cef 100644--- a/comfy_extras/nodes_textgen.py+++ b/comfy_extras/nodes_textgen.py@@ -40,6 +40,7 @@ class TextGenerate(io.ComfyNode):io.DynamicCombo.Input("sampling_mode", options=sampling_options, display_name="Sampling Mode"),io.Boolean.Input("thinking", optional=True, default=False, tooltip="Operate in thinking mode if the model supports it."),io.Boolean.Input("use_default_template", optional=True, default=True, tooltip="Use the built in system prompt/template if the model has one.", advanced=True),+ io.Combo.Input("mtp", options=["auto", "off", "2", "3", "4", "5"], default="auto", optional=True, tooltip="Speculative decoding with the checkpoint's multi-token-prediction head. No effect without MTP weights. auto adapts the draft depth; 2-5 pins it. Sampled output stays correctly distributed but differs from non-MTP output for the same seed."),],outputs=[io.String.Output(display_name="generated_text"),@@ -47,7 +48,9 @@ class TextGenerate(io.ComfyNode):)@classmethod- def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None) -> io.NodeOutput:+ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto") -> io.NodeOutput:++ mtp = False if mtp == "off" else (True if mtp == "auto" else int(mtp))tokens = clip.tokenize(prompt, image=image, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, audio=audio)@@ -71,7 +74,8 @@ class TextGenerate(io.ComfyNode):min_p=min_p,repetition_penalty=repetition_penalty,presence_penalty=presence_penalty,- seed=seed+ seed=seed,+ mtp=mtp)generated_text = clip.decode(generated_ids)@@ -225,7 +229,7 @@ class TextGenerateLTX2Prompt(TextGenerate):)@classmethod- def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None) -> io.NodeOutput:+ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto") -> io.NodeOutput:# Gemma 3 and Gemma 4 use different chat-turn markers and image tokens.# The Gemma 4 text encoder is the LTX 2.4 path; Gemma 3 is LTX 2.0.is_gemma4 = "gemma4" in getattr(clip.tokenizer, "clip_name", "")@@ -254,7 +258,7 @@ class TextGenerateLTX2Prompt(TextGenerate):f"<start_of_turn>model\n")- out = super().execute(clip, formatted_prompt, max_length, sampling_mode, image=image, thinking=thinking, use_default_template=use_default_template, video=video, audio=audio)+ out = super().execute(clip, formatted_prompt, max_length, sampling_mode, image=image, thinking=thinking, use_default_template=use_default_template, video=video, audio=audio, mtp=mtp)# Drop reasoning, including a block left unclosed by max_length. Both system prompts ask# for the original prompt back when there is nothing to give; empty conditions on nothing.