Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Qwen3/3.5/3.8 cudagraphs and w4a8 gemv support (CORE-390) (#15623)

Jukka Seppänen9a77c1db9eff
diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py
index 418faa51c..2b5cd042a 100644
--- a/comfy/model_prefetch.py
+++ b/comfy/model_prefetch.py
@@ -46,6 +46,22 @@ class _PauseMallocGraph:
def pause_malloc_graph(sync=False):
return _PauseMallocGraph(sync)
+class _MallocGraphScope:
+ def __init__(self, device):
+ self.device = device
+
+ def __enter__(self):
+ malloc_graph_begin(self.device)
+
+ def __exit__(self, exc_type, *args):
+ if exc_type is None:
+ malloc_graph_end()
+ else:
+ cleanup_malloc_graph()
+
+def malloc_graph_scope(device):
+ return _MallocGraphScope(device)
+
def malloc_graph_begin(device):
global MALLOC_GRAPH_USED
if not malloc_graph_enabled(device):
@@ -86,6 +102,24 @@ def cleanup_malloc_graph():
MALLOC_GRAPH_ROGUES += graph.rogue_count
del graph
+def pin_modules(comfy_modules, device, dtype=None):
+ registerable_size = 0
+ for s in comfy_modules:
+ registerable_size += comfy.memory_management.vram_aligned_size([s.weight, s.bias])
+ for param_key in ("weight", "bias"):
+ lowvram_fn = getattr(s, param_key + "_lowvram_function", None)
+ if lowvram_fn is not None:
+ registerable_size += lowvram_fn.memory_required()
+
+ offload_stream, fully_faulted = comfy.ops.cast_modules_with_vbar(comfy_modules, None, device, None, True, return_faulted=True)
+ if not (comfy_modules and comfy_modules[0]._pin_state["fast_disk"]):
+ comfy.model_management.ensure_pin_registerable(registerable_size)
+ comfy.model_management.sync_stream(device, offload_stream)
+ if fully_faulted and dtype is not None:
+ for comfy_module in comfy_modules:
+ comfy.ops.resolve_cast_module_with_vbar(comfy_module, dtype, device, dtype, None, False, return_weights=False)
+ return offload_stream, fully_faulted
+
def cleanup_prefetched_modules(module, comfy_modules):
for s in comfy_modules:
prefetch = getattr(s, "_prefetch", None)
@@ -201,21 +235,7 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap
if hasattr(s, "_v"):
comfy_modules.append(s)
- registerable_size = 0
- for s in comfy_modules:
- registerable_size += comfy.memory_management.vram_aligned_size([s.weight, s.bias])
- for param_key in ("weight", "bias"):
- lowvram_fn = getattr(s, param_key + "_lowvram_function", None)
- if lowvram_fn is not None:
- registerable_size += lowvram_fn.memory_required()
-
- offload_stream, fully_faulted = comfy.ops.cast_modules_with_vbar(comfy_modules, None, device, None, True, return_faulted=True)
- if not (comfy_modules and comfy_modules[0]._pin_state["fast_disk"]):
- comfy.model_management.ensure_pin_registerable(registerable_size)
- comfy.model_management.sync_stream(device, offload_stream)
- if fully_faulted and dtype is not None:
- for comfy_module in comfy_modules:
- comfy.ops.resolve_cast_module_with_vbar(comfy_module, dtype, device, dtype, None, False, return_weights=False)
+ offload_stream, fully_faulted = pin_modules(comfy_modules, device, dtype)
queue[0] = (offload_stream, (module, comfy_modules))
if core is not None:
diff --git a/comfy/sd.py b/comfy/sd.py
index 640d59352..230d4cf82 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -469,7 +469,7 @@ class CLIP:
def get_key_patches(self):
return self.patcher.get_key_patches()
- def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0):
+ def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0, mtp=True):
self.cond_stage_model.reset_clip_options()
self.load_model(tokens)
@@ -478,7 +478,7 @@ class CLIP:
self.cond_stage_model.set_clip_options({"execution_device": device})
with model_management.cuda_device_context(device), comfy.ops.use_quantized_matmul(self.cond_stage_model, device):
- return self.cond_stage_model.generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty)
+ return self.cond_stage_model.generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty, mtp=mtp)
def decode(self, token_ids, skip_special_tokens=True):
return self.tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)
@@ -1910,7 +1910,7 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip
elif te_model in (TEModel.QWEN35_08B, TEModel.QWEN35_2B, TEModel.QWEN35_4B, TEModel.QWEN35_9B, TEModel.QWEN35_27B):
clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
qwen35_type = {TEModel.QWEN35_08B: "qwen35_08b", TEModel.QWEN35_2B: "qwen35_2b", TEModel.QWEN35_4B: "qwen35_4b", TEModel.QWEN35_9B: "qwen35_9b", TEModel.QWEN35_27B: "qwen35_27b"}[te_model]
- clip_target.clip = comfy.text_encoders.qwen35.te(**llama_detect(clip_data), model_type=qwen35_type)
+ clip_target.clip = comfy.text_encoders.qwen35.te(**llama_detect(clip_data), model_type=qwen35_type, mtp="mtp.fc.weight" in clip_data[0])
clip_target.tokenizer = comfy.text_encoders.qwen35.tokenizer(model_type=qwen35_type)
elif te_model in (TEModel.QWEN3VL_4B, TEModel.QWEN3VL_8B):
if clip_type == CLIPType.IDEOGRAM4 and te_model == TEModel.QWEN3VL_8B: # Ideogram4 reuses the full Qwen3-VL-8B (13-layer tap for conditioning + multimodal generate).
diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py
index f0fdf1aa5..01651bd23 100644
--- a/comfy/sd1_clip.py
+++ b/comfy/sd1_clip.py
@@ -308,7 +308,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
def load_sd(self, sd):
return self.transformer.load_state_dict(sd, strict=False, assign=getattr(self, "can_assign_sd", False))
- def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0):
+ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0, mtp=True):
if isinstance(tokens, dict):
tokens_only = next(iter(tokens.values())) # todo: get this better?
else:
@@ -746,5 +746,11 @@ class SD1ClipModel(torch.nn.Module):
def load_sd(self, sd):
return getattr(self, self.clip).load_sd(sd)
- def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0):
- return getattr(self, self.clip).generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty)
+ def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0, mtp=True):
+ return getattr(self, self.clip).generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty, mtp=mtp)
+
+ def get_dynamic_vram__units(self):
+ # forward to the inner transformer so ModelPatcher can register vbar units (graph decode)
+ model = getattr(getattr(getattr(self, self.clip), "transformer", None), "model", None)
+ get_units = getattr(model, "get_dynamic_vram__units", None)
+ return get_units() if get_units is not None else ([], [])
diff --git a/comfy/text_encoders/gemma4.py b/comfy/text_encoders/gemma4.py
index ee035d522..0dcd226cb 100644
--- a/comfy/text_encoders/gemma4.py
+++ b/comfy/text_encoders/gemma4.py
@@ -1664,7 +1664,7 @@ class Gemma4Model(sd1_clip.SDClipModel):
self.dtypes.add(dtype)
super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={}, dtype=dtype, special_tokens={"start": 2, "pad": 0}, layer_norm_hidden_state=False, model_class=self.model_class, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, model_options=model_options)
- def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0):
+ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0, mtp=True):
if isinstance(tokens, dict):
tokens = next(iter(tokens.values()))
tokens_only = [[t[0] for t in b] for b in tokens]
@@ -1708,9 +1708,6 @@ def gemma4_te(dtype_llama=None, llama_quantization_metadata=None, model_class=No
if dtype_llama is not None:
dtype = dtype_llama
super().__init__(device=device, dtype=dtype, name="gemma4", clip_model=clip_model, model_options=model_options)
-
- def get_dynamic_vram__units(self):
- return getattr(self, self.clip).transformer.model.get_dynamic_vram__units()
return Gemma4TEModel_
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py
index d71daf3a3..6a778e4f0 100644
--- a/comfy/text_encoders/llama.py
+++ b/comfy/text_encoders/llama.py
@@ -32,6 +32,72 @@ class FixedKV:
def advance(self, num_tokens):
self.index += num_tokens
+ def rollback(self, discard=1):
+ # drop the rejected verify tail; stale slots and bias are overwritten by the next write/prepare
+ self.index -= discard
+
+@dataclass
+class FixedKVBias(FixedKV):
+ # full-capacity decode bias [1, 1, rows, capacity], last `seq` rows serve the queries; shared across layers
+ bias: torch.Tensor = None
+ tracker: dict = None
+
+ def prepare(self, num_tokens):
+ if self.tracker["step"] == (self.index, num_tokens):
+ return
+ self.tracker["step"] = (self.index, num_tokens)
+ i = self.index
+ rows = self.bias.shape[-2]
+ if num_tokens <= rows:
+ torch.arange(i, i + rows, out=self.position)
+ window = self.tracker[num_tokens]
+ start = i - rows + 1
+ skip = max(-start, 0)
+ end = min(i + rows, self.bias.shape[-1])
+ self.bias[..., :, start + skip:end] = window[:, skip:end - start]
+ else:
+ self.bias[..., :, i:i + num_tokens] = 0
+
+ @staticmethod
+ def shared(capacity, device, dtype):
+ # all layers advance in lockstep, so the bias caches share one position/bias/tracker
+ rows = 6
+ position = torch.empty((rows,), device=device, dtype=torch.int64)
+ bias = torch.full((1, 1, rows, capacity), torch.finfo(dtype).min, device=device, dtype=dtype)
+ tracker = {"step": -1}
+ # window templates per decode width: row r serves query j = r - (rows - n) and may see slots <= index + j
+ for n in range(1, rows + 1):
+ window = torch.full((rows, 2 * rows - 1), torch.finfo(dtype).min, device=device, dtype=dtype)
+ for r in range(rows):
+ window[r, :rows + max(r - (rows - n), 0)] = 0
+ tracker[n] = window
+ return position, bias, tracker
+
+ @classmethod
+ def zeros(cls, batch, kv_heads, capacity, head_dim, device, dtype, shared):
+ # zero-init: decode attends full capacity with masked tails, 0*0 stays finite
+ key = torch.zeros((batch, kv_heads, capacity, head_dim), device=device, dtype=dtype)
+ return cls(key, torch.zeros_like(key), 0, shared[0], None, shared[1], shared[2])
+
+ def append(self, xk, xv):
+ seq = xk.shape[2]
+ self.key[:, :, self.index:self.index + seq] = xk
+ self.value[:, :, self.index:self.index + seq] = xv
+ return self.key[:, :, :self.index + seq], self.value[:, :, :self.index + seq]
+
+ def decode(self, xq, xk, xv, num_kv_heads):
+ # CUDA-graphable: device-side write position, masked attention over the full capacity
+ batch_size, num_heads, seq, head_dim = xq.shape
+ self.key.index_copy_(2, self.position[:seq], xk)
+ self.value.index_copy_(2, self.position[:seq], xv)
+ groups = num_heads // num_kv_heads
+ q = xq.reshape(batch_size, num_kv_heads, groups, seq, head_dim) * head_dim ** -0.5
+ bias = self.bias[..., self.bias.shape[-2] - seq:, :].unsqueeze(1)
+ scores = (q @ self.key.transpose(-1, -2).unsqueeze(2)).add_(bias)
+ probs = torch.softmax(scores, dim=-1, dtype=torch.float32).to(xq.dtype)
+ out = probs @ self.value.unsqueeze(2)
+ return out.permute(0, 3, 1, 2, 4).reshape(batch_size, seq, num_heads * head_dim)
+
@dataclass
class Llama2Config:
vocab_size: int = 128320
@@ -278,6 +344,9 @@ class Qwen3VL_8BConfig(Qwen3_8BConfig):
rope_theta: float = 5000000.0
rope_dims = [24, 20, 20]
interleaved_mrope = True
+ fixed_kv: bool = True
+ graph_dynamic_vbar_blocks = True
+ prefetch_dynamic_vbars = True
@dataclass
class Qwen3VL_4BConfig(Qwen3VL_8BConfig):
@@ -776,7 +845,8 @@ class Llama2_(nn.Module):
super().__init__()
self.config = config
self.fixed_kv = getattr(config, "fixed_kv", False)
- self.graph_dynamic_vbar_blocks = False
+ self.graph_dynamic_vbar_blocks = getattr(config, "graph_dynamic_vbar_blocks", False)
+ self.prefetch_dynamic_vbars = getattr(config, "prefetch_dynamic_vbars", False)
self.vocab_size = config.vocab_size
if self.config.transformer_type == "gemma2" or self.config.transformer_type == "gemma3":
@@ -808,14 +878,15 @@ class Llama2_(nn.Module):
def init_kv_cache(self, batch, capacity, device, dtype):
caches = []
- fixed_kv = self.fixed_kv and comfy_kitchen.flash_attention_decode_is_available(device)
+ flash = getattr(comfy_kitchen, "flash_attention_decode_is_available", None)
+ flash_kv = self.fixed_kv and flash is not None and flash(device)
for _ in range(self.config.num_hidden_layers):
- if fixed_kv:
+ if flash_kv:
key = torch.empty((batch, capacity, self.config.num_key_value_heads, self.config.head_dim), device=device, dtype=dtype)
value = torch.empty_like(key)
- position = torch.empty((batch,), device=device, dtype=torch.int64)
+ pos = torch.empty((batch,), device=device, dtype=torch.int64)
seqlen = torch.zeros((batch,), device=device, dtype=torch.int32)
- caches.append(FixedKV(key, value, 0, position, seqlen))
+ caches.append(FixedKV(key, value, 0, pos, seqlen))
else:
key = torch.empty((batch, self.config.num_key_value_heads, capacity, self.config.head_dim), device=device, dtype=dtype)
caches.append((key, torch.empty_like(key), 0))
@@ -856,7 +927,8 @@ class Llama2_(nn.Module):
mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])).expand(attention_mask.shape[0], 1, seq_len, attention_mask.shape[-1])
mask = mask.masked_fill(mask.to(torch.bool), torch.finfo(x.dtype).min / 4)
- if seq_len > 1:
+ spec_decode = fixed_kv and any(isinstance(kv, FixedKVBias) for kv in past_key_values) and 2 <= seq_len <= 6 and past_len > 0 and attention_mask is None
+ if seq_len > 1 and not spec_decode: # spec verify: the staircase decode bias is causal
causal_mask = torch.empty(past_len + seq_len, past_len + seq_len, dtype=x.dtype, device=x.device).fill_(torch.finfo(x.dtype).min / 4).triu_(1)
if mask is not None:
mask += causal_mask
@@ -865,7 +937,7 @@ class Llama2_(nn.Module):
optimized_attention = optimized_attention_for_device(x.device, mask=mask is not None, small_input=True)
- enable_graph = self.graph_dynamic_vbar_blocks and fixed_kv_decode
+ enable_graph = self.graph_dynamic_vbar_blocks and (fixed_kv_decode or spec_decode)
if enable_graph:
if decode_buffers is None:
x = x.clone()
@@ -889,7 +961,7 @@ class Llama2_(nn.Module):
elif intermediate_output < 0:
intermediate_output = len(self.layers) + intermediate_output
- prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.layers), x.device, {"prefetch_dynamic_vbars": getattr(self, "prefetch_dynamic_vbars", False)})
+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.layers), x.device, {"prefetch_dynamic_vbars": self.prefetch_dynamic_vbars and past_key_values is not None})
next_key_values = list(past_key_values) if past_key_values is not None else []
for i, layer in enumerate(self.layers):
if all_intermediate is not None:
@@ -1001,6 +1073,18 @@ class BaseLlama:
def forward(self, input_ids, *args, **kwargs):
return self.model(input_ids, *args, **kwargs)
+def penalty_active(repetition_penalty, presence_penalty):
+ return repetition_penalty != 1.0 or (presence_penalty is not None and presence_penalty != 0.0)
+
+
+def apply_penalty(logits, repetition_penalty, presence_penalty):
+ if repetition_penalty != 1.0:
+ logits = torch.where(logits < 0, logits * repetition_penalty, logits / repetition_penalty)
+ if presence_penalty is not None and presence_penalty != 0.0:
+ logits = logits - presence_penalty
+ return logits
+
+
class BaseGenerate:
def logits(self, x):
input = x[:, -1:]
@@ -1010,7 +1094,7 @@ class BaseGenerate:
module = self.model.embed_tokens
if not module.comfy_cast_weights:
- return torch.nn.functional.linear(input, self.model.embed_tokens.weight.to(x), None)
+ return torch.nn.functional.linear(input, module.weight.to(x), None)
with comfy.ops.CastBiasWeightContext(module, input, offloadable=True) as (weight, _bias):
return torch.nn.functional.linear(input, weight, None)
@@ -1046,6 +1130,8 @@ class BaseGenerate:
compile_allocations = self.model.graph_dynamic_vbar_blocks and comfy.model_prefetch.malloc_graph_enabled(device)
decode_tokens = torch.empty((embeds.shape[0], 1), dtype=torch.long, device=device)
+ penalize = penalty_active(repetition_penalty, presence_penalty)
+ penalty_mask = None
# Generation loop
current_input_ids = initial_input_ids
@@ -1064,9 +1150,16 @@ class BaseGenerate:
extra["visual_pos_masks"] = visual_pos_masks
x, _, past_key_values = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=past_key_values, input_ids=current_input_ids, position_ids=position_ids, **extra, embeds_info=(embeds_info if step == 0 else None))
logits = self.logits(x)[:, -1]
- next_token = self.sample_token(logits, temperature, top_k, top_p, min_p, repetition_penalty, initial_tokens + generated_token_ids, generator, do_sample=do_sample, presence_penalty=presence_penalty)
+ if penalty_mask is None and do_sample and penalize:
+ # allocated on the (unbracketed) first step; later steps only index_fill_ it
+ penalty_mask = torch.zeros((logits.shape[-1],), dtype=torch.bool, device=device)
+ if len(initial_tokens) > 0:
+ penalty_mask.index_fill_(0, torch.tensor(initial_tokens, device=device), True)
+ next_token = self.sample_token(logits, temperature, top_k, top_p, min_p, repetition_penalty, [], generator, do_sample=do_sample, presence_penalty=presence_penalty, penalty_mask=penalty_mask)
decode_tokens.copy_(next_token)
+ if penalty_mask is not None:
+ penalty_mask.index_fill_(0, decode_tokens[0], True)
del next_token, logits, x, embeds, position_ids
if step > 0 and compile_allocations:
comfy.model_prefetch.malloc_graph_end()
@@ -1083,48 +1176,23 @@ class BaseGenerate:
return generated_token_ids
- def sample_token(self, logits, temperature, top_k, top_p, min_p, repetition_penalty, token_history, generator, do_sample=True, presence_penalty=0.0):
-
- if not do_sample or temperature == 0.0:
- return torch.argmax(logits, dim=-1, keepdim=True)
-
- # Sampling mode
- if len(token_history) > 0 and (repetition_penalty != 1.0 or (presence_penalty is not None and presence_penalty != 0.0)):
- token_ids = torch.tensor(list(set(token_history)), device=logits.device)
- token_logits = logits[:, token_ids]
- if repetition_penalty != 1.0:
- token_logits = torch.where(token_logits < 0, token_logits * repetition_penalty, token_logits / repetition_penalty)
- if presence_penalty is not None and presence_penalty != 0.0:
- token_logits = token_logits - presence_penalty
- logits[:, token_ids] = token_logits
+ def processed_probs(self, logits, temperature, top_k, top_p, min_p, repetition_penalty, token_history, presence_penalty=0.0, penalty_mask=None):
+ # returns (probs, indices); penalty_mask [vocab] bool stands in for token_history
+ if penalty_active(repetition_penalty, presence_penalty):
+ if penalty_mask is not None:
+ logits = torch.where(penalty_mask.unsqueeze(0), apply_penalty(logits, repetition_penalty, presence_penalty), logits)
+ elif len(token_history) > 0:
+ token_ids = torch.tensor(list(set(token_history)), device=logits.device)
+ logits[:, token_ids] = apply_penalty(logits[:, token_ids], repetition_penalty, presence_penalty)
if temperature != 1.0:
logits = logits / temperature
+ top_indices = None
if top_k > 0:
top_k = min(top_k, logits.shape[-1])
logits, top_indices = torch.topk(logits, top_k)
- if min_p > 0.0:
- probs_before_filter = torch.nn.functional.softmax(logits, dim=-1)
- top_probs, _ = probs_before_filter.max(dim=-1, keepdim=True)
- min_threshold = min_p * top_probs
- indices_to_remove = probs_before_filter < min_threshold
- logits[indices_to_remove] = torch.finfo(logits.dtype).min
-
- if top_p < 1.0:
- sorted_logits, sorted_indices = torch.sort(logits, descending=True)
- cumulative_probs = torch.cumsum(torch.nn.functional.softmax(sorted_logits, dim=-1), dim=-1)
- sorted_indices_to_remove = cumulative_probs > top_p
- sorted_indices_to_remove[..., 0] = False
- indices_to_remove = torch.zeros_like(logits, dtype=torch.bool)
- indices_to_remove.scatter_(1, sorted_indices, sorted_indices_to_remove)
- logits[indices_to_remove] = torch.finfo(logits.dtype).min
-
- probs = torch.nn.functional.softmax(logits, dim=-1)
- next_token = torch.multinomial(probs, num_samples=1, generator=generator)
- return top_indices.gather(1, next_token)
-
if min_p > 0.0:
probs_before_filter = torch.nn.functional.softmax(logits, dim=-1)
top_probs, _ = probs_before_filter.max(dim=-1, keepdim=True)
@@ -1141,9 +1209,18 @@ class BaseGenerate:
indices_to_remove.scatter_(1, sorted_indices, sorted_indices_to_remove)
logits[indices_to_remove] = torch.finfo(logits.dtype).min
- probs = torch.nn.functional.softmax(logits, dim=-1)
+ return torch.nn.functional.softmax(logits, dim=-1), top_indices
- return torch.multinomial(probs, num_samples=1, generator=generator)
+ def sample_token(self, logits, temperature, top_k, top_p, min_p, repetition_penalty, token_history, generator, do_sample=True, presence_penalty=0.0, penalty_mask=None):
+
+ if not do_sample or temperature == 0.0:
+ return torch.argmax(logits, dim=-1, keepdim=True)
+
+ probs, top_indices = self.processed_probs(logits, temperature, top_k, top_p, min_p, repetition_penalty, token_history, presence_penalty=presence_penalty, penalty_mask=penalty_mask)
+ next_token = torch.multinomial(probs, num_samples=1, generator=generator)
+ if top_indices is not None:
+ return top_indices.gather(1, next_token)
+ return next_token
class BaseQwen3:
def logits(self, x):
diff --git a/comfy/text_encoders/lt.py b/comfy/text_encoders/lt.py
index c512a7d48..2579e1b15 100644
--- a/comfy/text_encoders/lt.py
+++ b/comfy/text_encoders/lt.py
@@ -102,7 +102,7 @@ class Gemma3_12BModel(sd1_clip.SDClipModel):
self.dtypes.add(dtype)
super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={}, dtype=dtype, special_tokens={"start": 2, "pad": 0}, layer_norm_hidden_state=False, model_class=comfy.text_encoders.llama.Gemma3_12B, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, model_options=model_options)
- def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty):
+ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty, mtp=True):
tokens_only = [[t[0] for t in b] for b in tokens]
embeds, _, _, _ = self.process_tokens(tokens_only, self.execution_device)
return self.transformer.generate(embeds, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, stop_tokens=[106], presence_penalty=presence_penalty) # 106 is <end_of_turn>
@@ -205,7 +205,7 @@ class LTXAVTEModel(torch.nn.Module):
return out.to(device=out_device, dtype=torch.float), pooled, extra
- def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty):
+ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty, mtp=True):
return self.gemma3_12b.generate(tokens[self.text_encoder_key], do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty)
def load_sd(self, sd):
diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py
index 304a4357f..29b5d469d 100644
--- a/comfy/text_encoders/qwen35.py
+++ b/comfy/text_encoders/qwen35.py
@@ -2,14 +2,53 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from dataclasses import dataclass, field
+from tqdm import tqdm
+import contextlib
import os
+import warnings
import comfy.model_management
+import comfy.model_prefetch
+import comfy.ops
+import comfy_kitchen
+from comfy.quant_ops import QuantizedTensor
from comfy.ldm.modules.attention import optimized_attention_for_device
from comfy import sd1_clip
import comfy.text_encoders.qwen_vl
-from .llama import BaseLlama, BaseGenerate, Llama2_, MLP, RMSNorm, apply_rope
+from .llama import BaseLlama, BaseGenerate, FixedKV, FixedKVBias, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix
+
+
+@dataclass
+class LinearKV(FixedKV):
+ # DeltaNet state on the FixedKV interface: key=conv_state, value=recurrent_state (fp32)
+ g_decay: torch.Tensor = None
+ dt_bias: torch.Tensor = None
+ snapshots: list = None # [(recurrent, conv)] taken after step 1, 2, ... of the last verify
+ last_seq: int = 1
+ snap_backing: torch.Tensor = None
+ conv_snap_backing: torch.Tensor = None
+ norm_weight: torch.Tensor = None
+
+ def prepare(self, num_tokens):
+ pass
+
+ def rollback(self, discard=1):
+ # discard the rejected tail: restore the snapshot taken after the last kept token
+ rec, conv = self.snapshots[self.last_seq - discard - 1]
+ self.recurrent_state.copy_(rec)
+ self.conv_state.copy_(conv)
+ self.index -= discard
+
+ @property
+ def conv_state(self):
+ return self.key
+
+ @property
+ def recurrent_state(self):
+ return self.value
+
+
def _qwen35_layer_types(n):
@@ -48,6 +87,7 @@ class Qwen35Config:
transformer_type: str = "qwen35_2b"
rope_dims: list = None
rope_scale: float = None
+ mtp: bool = False
QWEN35_VISION_DEFAULTS = dict(hidden_size=1024, num_heads=16, intermediate_size=4096, depth=24, patch_size=16, temporal_patch_size=2, in_channels=3, spatial_merge_size=2, num_position_embeddings=2304)
@@ -135,18 +175,6 @@ def torch_chunk_gated_delta_rule(query, key, value, g, beta, chunk_size=64, init
return core_attn_out, last_recurrent_state
-def torch_causal_conv1d_update(x, conv_state, weight, bias=None):
- # conv_state: [B, channels, kernel_size-1], x: [B, channels, 1]
- # weight: [channels, kernel_size]
- state_len = conv_state.shape[-1]
- combined = torch.cat([conv_state, x], dim=-1).to(weight.dtype) # [B, channels, kernel_size]
- conv_state.copy_(combined[:, :, -state_len:])
- out = (combined * weight).sum(dim=-1, keepdim=True) # [B, channels, 1]
- if bias is not None:
- out = out + bias.unsqueeze(0).unsqueeze(-1)
- return F.silu(out).to(x.dtype)
-
-
# GatedDeltaNet - Linear Attention Layer
class GatedDeltaNet(nn.Module):
@@ -185,25 +213,55 @@ class GatedDeltaNet(nn.Module):
use_recurrent = (
past_key_value is not None
- and past_key_value[2] > 0
- and seq_len == 1
+ and past_key_value.index > 0
+ and seq_len <= 6
)
+ fused_available = getattr(comfy_kitchen, "gated_delta_decode_is_available", None)
+ use_fused = (use_recurrent and fused_available is not None and fused_available(x.device, self.key_head_dim, self.value_head_dim)
+ and (seq_len == 1 or past_key_value.snap_backing is not None))
+
# Projections (shared)
- mixed_qkv = self.in_proj_qkv(x).transpose(1, 2) # [B, conv_dim, seq_len]
+ proj = self.in_proj_qkv(x) # [B, seq_len, conv_dim]
z = self.in_proj_z(x)
+
+ if use_fused:
+ # decode: kitchen conv step, then gates + delta rule + gated norm in one kernel
+ if seq_len > 1:
+ past_key_value.last_seq = seq_len
+ with comfy.ops.CastBiasWeightContext(self.conv1d, proj, offloadable=True) as (conv_weight, conv_bias):
+ conv_out = comfy_kitchen.deltanet_conv_step(proj, past_key_value.conv_state, conv_weight, conv_bias,
+ past_key_value.conv_snap_backing[:seq_len - 1] if seq_len > 1 else None)
+ with comfy.ops.CastBiasWeightContext(self.in_proj_a, x, offloadable=True) as (w_a, _), \
+ comfy.ops.CastBiasWeightContext(self.in_proj_b, x, offloadable=True) as (w_b, _):
+ if isinstance(w_a, QuantizedTensor):
+ w_a = w_a.dequantize()
+ if isinstance(w_b, QuantizedTensor):
+ w_b = w_b.dequantize()
+ core_attn_out = comfy_kitchen.gated_delta_decode_fused(
+ conv_out, x, w_a, w_b, past_key_value.dt_bias, past_key_value.g_decay, past_key_value.recurrent_state,
+ self.key_dim, self.num_key_heads, self.key_head_dim ** -0.5, z, past_key_value.norm_weight, self.norm.eps,
+ past_key_value.snap_backing[:seq_len - 1] if seq_len > 1 else None)
+ return self.out_proj(core_attn_out.reshape(batch_size, seq_len, -1)), past_key_value
+
+ mixed_qkv = proj.transpose(1, 2) # [B, conv_dim, seq_len]
b = self.in_proj_b(x)
a = self.in_proj_a(x)
# Conv1d
if use_recurrent:
- recurrent_state, conv_state, step_index = past_key_value
- conv_weight = comfy.model_management.cast_to_device(self.conv1d.weight, mixed_qkv.device, mixed_qkv.dtype).squeeze(1)
- conv_bias = comfy.model_management.cast_to_device(self.conv1d.bias, mixed_qkv.device, mixed_qkv.dtype) if self.conv1d.bias is not None else None
- mixed_qkv = torch_causal_conv1d_update(mixed_qkv, conv_state, conv_weight, conv_bias)
+ # decode: exact-width causal window, weight resolved via the vbar-aware context
+ combined = torch.cat([past_key_value.conv_state, mixed_qkv], dim=-1)
+ if seq_len > 1:
+ past_key_value.last_seq = seq_len
+ for s in range(seq_len - 1):
+ past_key_value.snapshots[s][1].copy_(combined[:, :, 1 + s:1 + s + self.conv_kernel_size - 1])
+ past_key_value.conv_state.copy_(combined[:, :, seq_len:])
+ with comfy.ops.CastBiasWeightContext(self.conv1d, combined, offloadable=True) as (conv_weight, conv_bias):
+ mixed_qkv = F.silu(F.conv1d(combined, conv_weight, conv_bias, groups=self.conv1d.groups))
else:
if past_key_value is not None:
- recurrent_state, conv_state, step_index = past_key_value
+ conv_state = past_key_value.conv_state
conv_state_init = F.pad(mixed_qkv, (self.conv_kernel_size - mixed_qkv.shape[-1], 0))
conv_state.copy_(conv_state_init[:, :, -conv_state.shape[-1]:])
mixed_qkv = F.silu(self.conv1d(mixed_qkv)[:, :, :seq_len])
@@ -211,47 +269,51 @@ class GatedDeltaNet(nn.Module):
# Split QKV and compute beta/g
mixed_qkv = mixed_qkv.transpose(1, 2) # [B, seq_len, conv_dim]
query, key, value = mixed_qkv.split([self.key_dim, self.key_dim, self.value_dim], dim=-1)
+ if use_recurrent:
+ g_decay, dt_bias = past_key_value.g_decay, past_key_value.dt_bias
+ else:
+ g_decay = -comfy.model_management.cast_to_device(self.A_log, x.device, torch.float32).exp()
+ dt_bias = comfy.model_management.cast_to_device(self.dt_bias, x.device, torch.float32)
+ if past_key_value is not None:
+ past_key_value.g_decay = g_decay
+ past_key_value.dt_bias = dt_bias
+ past_key_value.norm_weight = comfy.model_management.cast_to(self.norm.weight, dtype=x.dtype, device=x.device)
+
beta = b.sigmoid()
- g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias.float())
+ g = g_decay * F.softplus(a.float() + dt_bias)
+ query = query.reshape(batch_size, seq_len, self.num_key_heads, self.key_head_dim)
+ key = key.reshape(batch_size, seq_len, self.num_key_heads, self.key_head_dim)
+ value = value.reshape(batch_size, seq_len, self.num_value_heads, self.value_head_dim)
+ if self.num_value_heads != self.num_key_heads:
+ rep = self.num_value_heads // self.num_key_heads
+ query = query.repeat_interleave(rep, dim=2)
+ key = key.repeat_interleave(rep, dim=2)
# Delta rule
if use_recurrent:
- # single-token path: work in [B, heads, dim] without seq dim
- query = query.reshape(batch_size, self.num_key_heads, self.key_head_dim)
- key = key.reshape(batch_size, self.num_key_heads, self.key_head_dim)
- value = value.reshape(batch_size, self.num_value_heads, self.value_head_dim)
-
- if self.num_value_heads != self.num_key_heads:
- rep = self.num_value_heads // self.num_key_heads
- query = query.repeat_interleave(rep, dim=1)
- key = key.repeat_interleave(rep, dim=1)
-
scale = self.key_head_dim ** -0.5
q = F.normalize(query.float(), dim=-1) * scale
k = F.normalize(key.float(), dim=-1)
v = value.float()
- beta_t = beta.reshape(batch_size, -1)
- g_t = g.reshape(batch_size, -1).exp()
+ beta_t = beta.reshape(batch_size, seq_len, -1)
+ g_t = g.reshape(batch_size, seq_len, -1).exp()
# In-place state update: [B, heads, k_dim, v_dim]
- recurrent_state.mul_(g_t[:, :, None, None])
- kv_mem = torch.einsum('bhk,bhkv->bhv', k, recurrent_state)
- delta = (v - kv_mem) * beta_t[:, :, None]
- recurrent_state.add_(k.unsqueeze(-1) * delta.unsqueeze(-2))
- core_attn_out = torch.einsum('bhk,bhkv->bhv', q, recurrent_state)
-
- core_attn_out = core_attn_out.to(x.dtype).unsqueeze(1)
- present_key_value = (recurrent_state, conv_state, step_index + 1)
+ recurrent_state = past_key_value.recurrent_state
+ outs = []
+ for s in range(seq_len):
+ recurrent_state.mul_(g_t[:, s, :, None, None])
+ kv_mem = torch.einsum('bhk,bhkv->bhv', k[:, s], recurrent_state)
+ delta = (v[:, s] - kv_mem) * beta_t[:, s, :, None]
+ # rank-1 update via baddbmm_: no materialized [B, H, D, D] outer-product temp
+ recurrent_state.view(-1, self.key_head_dim, self.value_head_dim).baddbmm_(
+ k[:, s].reshape(-1, self.key_head_dim, 1), delta.reshape(-1, 1, self.value_head_dim))
+ outs.append(torch.einsum('bhk,bhkv->bhv', q[:, s], recurrent_state))
+ if seq_len > 1 and s < seq_len - 1:
+ past_key_value.snapshots[s][0].copy_(recurrent_state)
+ core_attn_out = torch.stack(outs, dim=1).to(x.dtype)
+ present_key_value = past_key_value
else:
- query = query.reshape(batch_size, seq_len, -1, self.key_head_dim)
- key = key.reshape(batch_size, seq_len, -1, self.key_head_dim)
- value = value.reshape(batch_size, seq_len, -1, self.value_head_dim)
-
- if self.num_value_heads != self.num_key_heads:
- rep = self.num_value_heads // self.num_key_heads
- query = query.repeat_interleave(rep, dim=2)
- key = key.repeat_interleave(rep, dim=2)
-
core_attn_out, last_recurrent_state = torch_chunk_gated_delta_rule(
query, key, value, g=g, beta=beta,
initial_state=None,
@@ -261,8 +323,8 @@ class GatedDeltaNet(nn.Module):
present_key_value = None
if past_key_value is not None:
if last_recurrent_state is not None:
- recurrent_state.copy_(last_recurrent_state.to(recurrent_state.dtype))
- present_key_value = (recurrent_state, conv_state, step_index + seq_len)
+ past_key_value.recurrent_state.copy_(last_recurrent_state.to(past_key_value.recurrent_state.dtype))
+ present_key_value = past_key_value
# Gated norm + output projection (shared)
core_attn_out = self.norm(core_attn_out.reshape(-1, self.value_head_dim), z.reshape(-1, self.value_head_dim))
@@ -271,29 +333,6 @@ class GatedDeltaNet(nn.Module):
# GatedAttention - Full Attention with output gating
-def precompute_partial_rope(head_dim, rotary_dim, position_ids, theta, device=None, mrope_section=None):
- """Compute RoPE frequencies for partial rotary embeddings."""
- theta_numerator = torch.arange(0, rotary_dim, 2, device=device).float()
- inv_freq = 1.0 / (theta ** (theta_numerator / rotary_dim))
-
- inv_freq_expanded = inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
- position_ids_expanded = position_ids[:, None, :].float()
- freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
- emb = torch.cat((freqs, freqs), dim=-1)
- cos = emb.cos()
- sin = emb.sin()
-
- if mrope_section is not None and position_ids.shape[0] == 3:
- mrope_section_2 = [s * 2 for s in mrope_section]
- cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section_2, dim=-1))], dim=-1).unsqueeze(0)
- sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section_2, dim=-1))], dim=-1).unsqueeze(0)
-
- cos = cos.unsqueeze(1)
- sin = sin.unsqueeze(1)
- sin_split = sin.shape[-1] // 2
- return (cos, sin[..., :sin_split], -sin[..., sin_split:])
-
-
def apply_partial_rope(xq, xk, freqs_cis, rotary_dim):
"""Apply RoPE to only the first rotary_dim dimensions."""
xq_rot = xq[..., :rotary_dim]
@@ -350,26 +389,16 @@ class GatedAttention(nn.Module):
xq, xk = apply_partial_rope(xq, xk, freqs_cis, self.rotary_dim)
# KV cache
- present_key_value = None
- if past_key_value is not None:
- past_key, past_value, index = past_key_value
- num_tokens = xk.shape[2]
- if past_key.shape[2] >= (index + num_tokens):
- past_key[:, :, index:index + num_tokens] = xk
- past_value[:, :, index:index + num_tokens] = xv
- xk = past_key[:, :, :index + num_tokens]
- xv = past_value[:, :, :index + num_tokens]
- present_key_value = (past_key, past_value, index + num_tokens)
- else:
- if index > 0:
- xk = torch.cat((past_key[:, :, :index], xk), dim=2)
- xv = torch.cat((past_value[:, :, :index], xv), dim=2)
- present_key_value = (xk, xv, index + num_tokens)
+ present_key_value = past_key_value
+ if past_key_value is not None and seq_length <= 6 and attention_mask is None:
+ output = past_key_value.decode(xq, xk, xv, self.num_kv_heads)
+ else:
+ if past_key_value is not None:
+ xk, xv = past_key_value.append(xk, xv)
+ gqa_kwargs = {"enable_gqa": True} if self.num_heads != self.num_kv_heads else {}
+ output = optimized_attention(xq, xk, xv, self.num_heads, mask=attention_mask, skip_reshape=True, **gqa_kwargs)
- gqa_kwargs = {"enable_gqa": True} if self.num_heads != self.num_kv_heads else {}
- output = optimized_attention(xq, xk, xv, self.num_heads, mask=attention_mask, skip_reshape=True, **gqa_kwargs)
output = output * gate.sigmoid()
-
return self.o_proj(output), present_key_value
@@ -387,13 +416,15 @@ class Qwen35TransformerBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)
def forward(self, x, attention_mask=None, freqs_cis=None, optimized_attention=None, past_key_value=None):
+ output = x
if self.layer_type == "linear_attention":
h, present_key_value = self.linear_attn(self.input_layernorm(x), attention_mask=attention_mask, past_key_value=past_key_value)
else:
h, present_key_value = self.self_attn(self.input_layernorm(x), attention_mask=attention_mask, freqs_cis=freqs_cis, optimized_attention=optimized_attention, past_key_value=past_key_value)
- x = x + h
- x = x + self.mlp(self.post_attention_layernorm(x))
+ # in-place into the input buffer so CUDA-graph replays land in the static x
+ x = torch.add(x, h, out=output)
+ x = torch.add(x, self.mlp(self.post_attention_layernorm(x)), out=output)
return x, present_key_value
@@ -402,6 +433,8 @@ class Qwen35Transformer(Llama2_):
def __init__(self, config, device=None, dtype=None, ops=None):
nn.Module.__init__(self)
self.config = config
+ self.prefetch_dynamic_vbars = True
+ self.graph_dynamic_vbar_blocks = True
self.vocab_size = config.vocab_size
self.embed_tokens = ops.Embedding(config.vocab_size, config.hidden_size, device=device, dtype=dtype)
self.layers = nn.ModuleList([
@@ -417,21 +450,10 @@ class Qwen35Transformer(Llama2_):
if config.lm_head:
self.lm_head = ops.Linear(config.hidden_size, config.vocab_size, bias=False, device=device, dtype=dtype)
- def get_past_len(self, past_key_values):
- for i, layer in enumerate(self.layers):
- if layer.layer_type == "full_attention":
- if len(past_key_values) > i:
- return past_key_values[i][2]
- break
- return 0
-
def compute_freqs_cis(self, position_ids, device):
rotary_dim = int(self.config.head_dim * self.config.partial_rotary_factor)
- return precompute_partial_rope(
- self.config.head_dim, rotary_dim, position_ids,
- self.config.rope_theta, device=device,
- mrope_section=self.config.mrope_section,
- )
+ return precompute_freqs_cis(rotary_dim, position_ids, self.config.rope_theta,
+ rope_dims=self.config.mrope_section, interleaved_mrope=True, device=device)
# Vision Encoder
@@ -671,6 +693,23 @@ class Qwen35VisionModel(nn.Module):
return merged, deepstack_features
return merged
+class MTPHead(nn.Module):
+ # MTP draft head: fc(cat[norm(embed), norm(hidden)]) -> attention block (own KV) -> norm
+ def __init__(self, config, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.fc = ops.Linear(config.hidden_size * 2, config.hidden_size, bias=False, device=device, dtype=dtype)
+ self.pre_fc_norm_embedding = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)
+ self.pre_fc_norm_hidden = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)
+ self.layers = nn.ModuleList([Qwen35TransformerBlock(config, index=3, device=device, dtype=dtype, ops=ops)])
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)
+
+ def forward(self, embeds, hidden, freqs_cis, past_key_value):
+ x = self.fc(torch.cat([self.pre_fc_norm_embedding(embeds), self.pre_fc_norm_hidden(hidden)], dim=-1))
+ attention = optimized_attention_for_device(x.device, mask=False, small_input=True)
+ x, _ = self.layers[0](x, attention_mask=None, freqs_cis=freqs_cis, optimized_attention=attention, past_key_value=past_key_value)
+ return self.norm(x), x # (for logits, pre-norm hidden for recursive drafting)
+
+
# Model Wrapper
class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
model_type = "qwen35_2b"
@@ -680,6 +719,9 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
config = _make_config(self.model_type, config_dict)
self.num_layers = config.num_hidden_layers
self.model = Qwen35Transformer(config, device=device, dtype=dtype, ops=operations)
+ self.mtp = None
+ if config.mtp:
+ self.mtp = MTPHead(config, device=device, dtype=dtype, ops=operations)
vision_overrides = QWEN35_MODELS.get(self.model_type, {}).get("vision", {})
vision_config = {**QWEN35_VISION_DEFAULTS, **vision_overrides, "out_hidden_size": config.hidden_size}
self.visual = Qwen35VisionModel(vision_config, device=device, dtype=dtype, ops=operations)
@@ -687,7 +729,8 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
def preprocess_embed(self, embed, device):
if embed["type"] == "image":
- image, grid = comfy.text_encoders.qwen_vl.process_qwen2vl_images(embed["data"], patch_size=16)
+ # Qwen3.5 normalizes to [-1, 1] (mean/std 0.5), same as Qwen3-VL.
+ image, grid = comfy.text_encoders.qwen_vl.process_qwen2vl_images(embed["data"], patch_size=16, image_mean=[0.5, 0.5, 0.5], image_std=[0.5, 0.5, 0.5])
return self.visual(image.to(device, dtype=torch.float32), grid), grid
return None, None
@@ -695,9 +738,271 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, embeds.shape[1], embeds.device)
return super().forward(x, attention_mask=attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=final_layer_norm_intermediate, dtype=dtype, position_ids=position_ids, past_key_values=past_key_values)
+ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, **kwargs):
+ mtp = kwargs.pop("mtp", True)
+ if self.mtp is None or not mtp or kwargs.get("position_ids") is not None or kwargs.get("initial_input_ids") is not None:
+ return super().generate(embeds=embeds, do_sample=do_sample, max_length=max_length, temperature=temperature,
+ top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty,
+ seed=seed, stop_tokens=stop_tokens, **kwargs)
+ sampling = None
+ if do_sample and temperature != 0.0:
+ sampling = {"temperature": temperature, "top_k": top_k, "top_p": top_p, "min_p": min_p,
+ "repetition_penalty": repetition_penalty,
+ "presence_penalty": kwargs.get("presence_penalty", 0.0) or 0.0,
+ "seed": seed if seed is not None else 42}
+ fixed_depth = None if mtp is True else max(2, min(5, int(mtp)))
+ return self._generate_mtp(embeds, max_length, stop_tokens, sampling=sampling, fixed_depth=fixed_depth)
+
+ def _generate_mtp(self, embeds, max_length, stop_tokens, sampling=None, fixed_depth=None):
+ device = embeds.device
+ cfg = self.model.config
+ if stop_tokens is None:
+ stop_tokens = cfg.stop_tokens
+ dt = torch.bfloat16 if comfy.model_management.should_use_bf16(device) else torch.float32
+ embeds = embeds.to(dt)
+ if embeds.ndim == 2:
+ embeds = embeds.unsqueeze(0)
+ # greedy drafts 3 deep (5 after the probe); sampled stays at 2
+ depth = fixed_depth if fixed_depth is not None else (3 if sampling is None else 2)
+ cap = embeds.shape[1] + max_length + 7
+ pkv = self.init_kv_cache(embeds.shape[0], cap, device, dt)
+ # repair window: drafting ahead plus a near-full rollback
+ mtp_kv = FixedKVBias.zeros(embeds.shape[0], cfg.num_key_value_heads, cap, cfg.head_dim, device, dt,
+ FixedKVBias.shared(cap, device, dt))
+ head = self.model.lm_head if hasattr(self.model, "lm_head") else self.model.embed_tokens
+
+ def verify_logits(x):
+ if not head.comfy_cast_weights:
+ return F.linear(x, head.weight.to(x), None)
+ with comfy.ops.CastBiasWeightContext(head, x, offloadable=True) as (w, _bias):
+ return F.linear(x, w)
+
+ # the draft graph bakes these weights' addresses: keep them resident for the generate
+ hot = list({id(m): m for m in [head, self.model.embed_tokens, *self.mtp.modules()]}.values())
+ pinned = [m for m in hot if hasattr(m, "_v")]
+
+ generator = None
+ if sampling is not None:
+ generator = torch.Generator(device=device).manual_seed(sampling["seed"])
+ penalized = sampling is not None and penalty_active(sampling["repetition_penalty"], sampling["presence_penalty"])
+
+ # rope table once per generate, sliced per draft
+ ftab = rope_matrix(self.model.compute_freqs_cis(torch.arange(cap, device=device, dtype=torch.float).unsqueeze(0), device))
+
+ def freqs_at(p, n=1):
+ return ftab[:, :, p:p + n]
+
+ # cross-step state lives in static carriers: nothing allocated inside a step may outlive it
+ nt_buf = torch.empty((embeds.shape[0], 1), device=device, dtype=torch.long)
+ h_buf = torch.empty((embeds.shape[0], 1, cfg.hidden_size), device=device, dtype=dt)
+ x, _, _ = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=pkv)
+ lg0 = self.logits(x)[:, -1]
+ if sampling is None:
+ nt_buf.copy_(lg0.argmax(dim=-1, keepdim=True))
+ else:
+ nt_buf.copy_(self.sample_token(lg0, sampling["temperature"], sampling["top_k"], sampling["top_p"], sampling["min_p"],
+ sampling["repetition_penalty"], [], generator, presence_penalty=sampling["presence_penalty"]))
+ pen_mask = torch.zeros((lg0.shape[-1],), device=device, dtype=torch.bool) if penalized else None
+ h_buf.copy_(x[:, -1:, :])
+ del x, lg0
+ ids = [nt_buf[0].item()]
+ if penalized:
+ pen_mask.index_fill_(0, nt_buf.reshape(-1), True)
+ pos = embeds.shape[1]
+ progress = comfy.utils.ProgressBar(max_length)
+ console = tqdm(total=max_length, desc="Generating tokens", initial=1)
+
+ def update_progress(n):
+ progress.update(n)
+ console.update(n)
+
+ verify_buffers = None
+ snapshot_bytes = sum(kv.recurrent_state.numel() * 4 + kv.conv_state.numel() * kv.conv_state.element_size()
+ for kv in pkv if isinstance(kv, LinearKV))
+
+ def set_depth(d):
+ nonlocal depth, verify_buffers
+ depth = d
+ for kv in pkv:
+ if isinstance(kv, LinearKV):
+ # snapshot views share one backing slab so the fused kernel can write them
+ kv.snap_backing = torch.empty((d,) + tuple(kv.recurrent_state.shape), device=device, dtype=torch.float32)
+ kv.conv_snap_backing = torch.empty((d,) + tuple(kv.conv_state.shape), device=device, dtype=kv.conv_state.dtype)
+ kv.snapshots = [(kv.snap_backing[s], kv.conv_snap_backing[s]) for s in range(d)]
+ # static hidden/rope buffers: graphed layers bake their input addresses
+ verify_buffers = (torch.empty((embeds.shape[0], d + 1, cfg.hidden_size), device=device, dtype=dt), freqs_at(pos, d + 1).clone())
+
+ set_depth(depth)
+ use_graph = (device.type == "cuda"
+ and comfy.model_management.NUM_STREAMS > 0
+ and not comfy.model_management.args.disable_cuda_graphs)
+ compile_allocations = use_graph and self.model.graph_dynamic_vbar_blocks and comfy.model_prefetch.malloc_graph_enabled(device)
+ draft_state = {}
+
+ def drop_draft_graph():
+ # free inside torch API calls so the allocator's benign notices stay catchable
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ for t in (draft_state.get("d"), draft_state.get("r"), *draft_state.get("keep", ())):
+ if t is not None:
+ t.set_()
+ g = draft_state.pop("graph", None)
+ if g is not None:
+ g.reset()
+ draft_state.clear()
+
+ def draft_capture():
+ # captured outside the compiler bracket; its static buffers live for the generate
+ ds = draft_state
+ mtp_kv.prepare(1)
+ ds["tok"] = nt_buf.clone()
+ ds["hid"] = h_buf.clone()
+ ds["f"] = freqs_at(pos).clone()
+ side = torch.cuda.Stream()
+ side.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(side):
+ for _ in range(2):
+ n1, r1 = self.mtp(self.model.embed_tokens(ds["tok"]).to(dt), ds["hid"], ds["f"], mtp_kv)
+ self.logits(n1)[:, -1].argmax(dim=-1, keepdim=True)
+ torch.cuda.current_stream().wait_stream(side)
+ del n1, r1 # freed before the capture, not shadowed inside it
+ g = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(g, capture_error_mode="thread_local"):
+ n1, r1 = self.mtp(self.model.embed_tokens(ds["tok"]).to(dt), ds["hid"], ds["f"], mtp_kv)
+ lg1 = self.logits(n1)
+ ds["d"] = lg1[:, -1].argmax(dim=-1, keepdim=True)
+ ds["r"] = r1
+ ds["keep"] = (n1, lg1) # captured allocations must outlive the graph
+ ds["graph"] = g
+
+ def draft(token, hidden, p):
+ # one drafted token: mtp head + lm_head argmax, graph-replayed on cuda
+ mtp_kv.prepare(1)
+ f = freqs_at(p)
+ if not use_graph:
+ n1, r1 = self.mtp(self.model.embed_tokens(token).to(dt), hidden, f, mtp_kv)
+ mtp_kv.advance(1)
+ return self.logits(n1)[:, -1].argmax(dim=-1, keepdim=True), r1
+ ds = draft_state
+ ds["tok"].copy_(token)
+ ds["hid"].copy_(hidden)
+ ds["f"].copy_(f)
+ ds["graph"].replay()
+ mtp_kv.advance(1)
+ return ds["d"], ds["r"]
+
+ def verify_sample(lg, drafts):
+ # accept draft i w.p. p_i(draft), else sample the residual; all depth+1 columns as one batch
+ s = sampling
+ rows = lg[0].float()
+ if penalized:
+ # committed tokens penalize every column, each draft the columns after it
+ mask = pen_mask.unsqueeze(0).repeat(depth + 1, 1)
+ for c, d in enumerate(drafts):
+ mask[c + 1:].index_fill_(1, d.reshape(-1), True)
+ rows = torch.where(mask, apply_penalty(rows, s["repetition_penalty"], s["presence_penalty"]), rows)
+ probs, idx = self.processed_probs(rows, s["temperature"], s["top_k"], s["top_p"], s["min_p"], 1.0, [])
+ dr = torch.cat(drafts, dim=1).reshape(-1, 1)
+ if idx is None:
+ p_draft = probs[:depth].gather(1, dr).reshape(-1)
+ w = probs.clone()
+ w[:depth].scatter_(1, dr, 0.0)
+ else:
+ hit = idx[:depth] == dr
+ p_draft = (probs[:depth] * hit).sum(1)
+ w = probs.clone()
+ w[:depth].masked_fill_(hit, 0.0)
+ # residual: p with the draft zeroed (the rescue term only matters when discarded anyway)
+ w = w + (w.sum(1, keepdim=True) == 0) * probs
+ tok = torch.multinomial(w, num_samples=1, generator=generator)
+ corr = tok if idx is None else idx.gather(1, tok)
+ u = torch.rand(depth, device=device, generator=generator)
+ accepted = (u < p_draft).long().cumprod(0).sum()
+ return dr, corr, accepted
+
+ def step():
+ # scoped so every temporary dies before the compiler bracket closes
+ nonlocal pos
+ drafts = []
+ tok_in, hid_in = nt_buf, h_buf
+ for k in range(depth):
+ dk, rk = draft(tok_in, hid_in, pos + k)
+ if k < depth - 1:
+ dk = dk.clone() # later replays overwrite the static output
+ drafts.append(dk)
+ tok_in, hid_in = dk, rk
+ ev = self.model.embed_tokens(torch.cat([nt_buf] + drafts, dim=1)).to(dt)
+ x, _, _ = self.model.forward(None, embeds=ev, attention_mask=None, past_key_values=pkv, decode_buffers=verify_buffers)
+ # all verify positions in one lm_head GEMV, accept decided GPU-side, one sync
+ lg = verify_logits(x)
+ if sampling is None:
+ toks = lg.argmax(dim=-1)
+ vals = torch.cat([toks[0]] + [d[0] for d in drafts]).tolist()
+ t, dr = vals[:depth + 1], vals[depth + 1:]
+ accepts = 0
+ while accepts < depth and t[accepts] == dr[accepts]:
+ accepts += 1
+ next_toks = tuple(toks[:, i:i + 1] for i in range(depth + 1))
+ commit = tuple(dr[:accepts]) + (t[accepts],)
+ else:
+ dr, corr, accepted = verify_sample(lg, drafts)
+ vals = torch.cat([dr[:, 0], corr[:, 0], accepted.reshape(1)]).tolist()
+ dr, cv, accepts = vals[:depth], vals[depth:2 * depth + 1], vals[-1]
+ next_toks = tuple(corr[i:i + 1] for i in range(depth + 1))
+ commit = tuple(dr[:accepts]) + (cv[accepts],)
+ if accepts < depth:
+ for kv in pkv:
+ kv.rollback(depth - accepts)
+ if accepts < depth - 1:
+ mtp_kv.rollback(depth - 1 - accepts) # mtp entries fed by a rejected draft token
+ nt_buf.copy_(next_toks[accepts])
+ h_buf.copy_(x[:, accepts:accepts + 1, :])
+ if penalized:
+ for d in drafts[:accepts]:
+ pen_mask.index_fill_(0, d.reshape(-1), True)
+ pen_mask.index_fill_(0, nt_buf.reshape(-1), True)
+ pos += accepts + 1
+ return accepts, commit
+
+ probe = None if fixed_depth is not None else [0, 0] # steps, accepted drafts
+ try:
+ if pinned:
+ comfy.model_prefetch.pin_modules(pinned, device, dt)
+ if use_graph and len(ids) < max_length and ids[-1] not in stop_tokens:
+ draft_capture()
+ while len(ids) < max_length and ids[-1] not in stop_tokens:
+ with (comfy.model_prefetch.malloc_graph_scope(device) if compile_allocations else contextlib.nullcontext()):
+ accepts, commit = step()
+ commit = list(commit[:max_length - len(ids)])
+ stop = next((i for i, t in enumerate(commit) if t in stop_tokens), None)
+ if stop is not None:
+ del commit[stop + 1:]
+ ids.extend(commit)
+ update_progress(len(commit))
+ if probe is not None:
+ probe[0] += 1
+ probe[1] += accepts
+ if probe[0] == 32:
+ # deepen once acceptance sustains it and a recapture round can amortize
+ if (sampling is None and max_length - len(ids) > 512 and 1 + probe[1] / probe[0] >= 2.2
+ and 5 * snapshot_bytes < comfy.model_management.get_free_memory(device)):
+ comfy.model_prefetch.cleanup_prefetch_queues()
+ drop_draft_graph()
+ set_depth(5)
+ if use_graph:
+ draft_capture()
+ probe = None
+ finally:
+ console.close()
+ drop_draft_graph()
+ if pinned:
+ comfy.model_prefetch.cleanup_prefetched_modules(None, pinned)
+ return ids
+
def init_kv_cache(self, batch, max_cache_len, device, execution_dtype):
model_config = self.model.config
past_key_values = []
+ shared = FixedKVBias.shared(max_cache_len, device, execution_dtype)
for i in range(model_config.num_hidden_layers):
if model_config.layer_types[i] == "linear_attention":
recurrent_state = torch.zeros(
@@ -709,13 +1014,9 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
[batch, conv_dim, model_config.conv_kernel_size - 1],
device=device, dtype=execution_dtype
)
- past_key_values.append((recurrent_state, conv_state, 0))
+ past_key_values.append(LinearKV(conv_state, recurrent_state, 0, None, None))
else:
- past_key_values.append((
- torch.empty([batch, model_config.num_key_value_heads, max_cache_len, model_config.head_dim], device=device, dtype=execution_dtype),
- torch.empty([batch, model_config.num_key_value_heads, max_cache_len, model_config.head_dim], device=device, dtype=execution_dtype),
- 0
- ))
+ past_key_values.append(FixedKVBias.zeros(batch, model_config.num_key_value_heads, max_cache_len, model_config.head_dim, device, execution_dtype, shared))
return past_key_values
# Tokenizer and Text Encoder Wrappers
@@ -777,19 +1078,28 @@ class Qwen35ImageTokenizer(sd1_clip.SD1Tokenizer):
class Qwen35ClipModel(sd1_clip.SDClipModel):
- def __init__(self, device="cpu", layer="hidden", layer_idx=-2, dtype=None, attention_mask=True, model_options={}, model_type="qwen35_2b"):
+ def __init__(self, device="cpu", layer="hidden", layer_idx=-2, dtype=None, attention_mask=True, model_options={}, model_type="qwen35_2b", mtp=False):
class Qwen35_(Qwen35):
pass
Qwen35_.model_type = model_type
- super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={},
+ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={"mtp": True} if mtp else {},
dtype=dtype, special_tokens={"pad": 248044}, layer_norm_hidden_state=False,
model_class=Qwen35_, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, model_options=model_options)
+ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0, mtp=True):
+ if isinstance(tokens, dict):
+ tokens = next(iter(tokens.values()))
+ tokens_only = [[t[0] for t in b] for b in tokens]
+ embeds, _, _, embeds_info = self.process_tokens(tokens_only, self.execution_device)
+ position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, embeds.shape[1], embeds.device)
+ return self.transformer.generate(embeds, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed,
+ presence_penalty=presence_penalty, position_ids=position_ids, mtp=mtp)
+
class Qwen35TEModel(sd1_clip.SD1ClipModel):
- def __init__(self, device="cpu", dtype=None, model_options={}, model_type="qwen35_2b"):
- clip_model = lambda **kw: Qwen35ClipModel(**kw, model_type=model_type)
+ def __init__(self, device="cpu", dtype=None, model_options={}, model_type="qwen35_2b", mtp=False):
+ clip_model = lambda **kw: Qwen35ClipModel(**kw, model_type=model_type, mtp=mtp)
super().__init__(device=device, dtype=dtype, name=model_type, clip_model=clip_model, model_options=model_options)
@@ -800,7 +1110,7 @@ def tokenizer(model_type="qwen35_2b"):
return Qwen35ImageTokenizer_
-def te(dtype_llama=None, llama_quantization_metadata=None, model_type="qwen35_2b"):
+def te(dtype_llama=None, llama_quantization_metadata=None, model_type="qwen35_2b", mtp=False):
class Qwen35TEModel_(Qwen35TEModel):
def __init__(self, device="cpu", dtype=None, model_options={}):
if dtype_llama is not None:
@@ -808,5 +1118,5 @@ def te(dtype_llama=None, llama_quantization_metadata=None, model_type="qwen35_2b
if llama_quantization_metadata is not None:
model_options = model_options.copy()
model_options["quantization_metadata"] = llama_quantization_metadata
- super().__init__(device=device, dtype=dtype, model_options=model_options, model_type=model_type)
+ super().__init__(device=device, dtype=dtype, model_options=model_options, model_type=model_type, mtp=mtp)
return Qwen35TEModel_
diff --git a/comfy/text_encoders/qwen3vl.py b/comfy/text_encoders/qwen3vl.py
index fd7d0d267..f16b2a7e4 100644
--- a/comfy/text_encoders/qwen3vl.py
+++ b/comfy/text_encoders/qwen3vl.py
@@ -127,7 +127,7 @@ class Qwen3VLClipModel(sd1_clip.SDClipModel):
model_class=_make_qwen3vl_model(model_type), enable_attention_masks=attention_mask,
return_attention_masks=attention_mask, model_options=model_options)
- def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0):
+ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0, mtp=True):
if isinstance(tokens, dict):
tokens = next(iter(tokens.values()))
tokens_only = [[t[0] for t in b] for b in tokens]
diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.py
index ef5858a7d..243834cef 100644
--- a/comfy_extras/nodes_textgen.py
+++ b/comfy_extras/nodes_textgen.py
@@ -40,6 +40,7 @@ class TextGenerate(io.ComfyNode):
io.DynamicCombo.Input("sampling_mode", options=sampling_options, display_name="Sampling Mode"),
io.Boolean.Input("thinking", optional=True, default=False, tooltip="Operate in thinking mode if the model supports it."),
io.Boolean.Input("use_default_template", optional=True, default=True, tooltip="Use the built in system prompt/template if the model has one.", advanced=True),
+ io.Combo.Input("mtp", options=["auto", "off", "2", "3", "4", "5"], default="auto", optional=True, tooltip="Speculative decoding with the checkpoint's multi-token-prediction head. No effect without MTP weights. auto adapts the draft depth; 2-5 pins it. Sampled output stays correctly distributed but differs from non-MTP output for the same seed."),
],
outputs=[
io.String.Output(display_name="generated_text"),
@@ -47,7 +48,9 @@ class TextGenerate(io.ComfyNode):
)
@classmethod
- def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None) -> io.NodeOutput:
+ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto") -> io.NodeOutput:
+
+ mtp = False if mtp == "off" else (True if mtp == "auto" else int(mtp))
tokens = clip.tokenize(prompt, image=image, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, audio=audio)
@@ -71,7 +74,8 @@ class TextGenerate(io.ComfyNode):
min_p=min_p,
repetition_penalty=repetition_penalty,
presence_penalty=presence_penalty,
- seed=seed
+ seed=seed,
+ mtp=mtp
)
generated_text = clip.decode(generated_ids)
@@ -225,7 +229,7 @@ class TextGenerateLTX2Prompt(TextGenerate):
)
@classmethod
- def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None) -> io.NodeOutput:
+ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto") -> io.NodeOutput:
# Gemma 3 and Gemma 4 use different chat-turn markers and image tokens.
# The Gemma 4 text encoder is the LTX 2.4 path; Gemma 3 is LTX 2.0.
is_gemma4 = "gemma4" in getattr(clip.tokenizer, "clip_name", "")
@@ -254,7 +258,7 @@ class TextGenerateLTX2Prompt(TextGenerate):
f"<start_of_turn>model\n"
)
- out = super().execute(clip, formatted_prompt, max_length, sampling_mode, image=image, thinking=thinking, use_default_template=use_default_template, video=video, audio=audio)
+ out = super().execute(clip, formatted_prompt, max_length, sampling_mode, image=image, thinking=thinking, use_default_template=use_default_template, video=video, audio=audio, mtp=mtp)
# Drop reasoning, including a block left unclosed by max_length. Both system prompts ask
# for the original prompt back when there is nothing to give; empty conditions on nothing.