Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

qwen 3.x performance improvements part III - Adaptive tree MTP (#17273)

rattusf7c664378a9a
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py
index 4ff4158dd..d84d7b7cd 100644
--- a/comfy/text_encoders/llama.py
+++ b/comfy/text_encoders/llama.py
@@ -17,6 +17,9 @@ import comfy.clip_model
from . import qwen_vl
+# widest speculative verify (draft chain or tree rows) the decode caches serve in one step
+SPEC_ROWS = 8
+
@dataclass
class FixedKV:
@@ -42,18 +45,29 @@ class FixedKVState(FixedKV):
tracker: dict = None
def prepare(self, num_tokens):
- if self.tracker["step"] == (self.index, num_tokens):
+ tree = self.tracker["tree"]
+ if self.tracker["step"] == (self.index, num_tokens, tree):
return
- self.tracker["step"] = (self.index, num_tokens)
+ self.tracker["step"] = (self.index, num_tokens, tree)
torch.arange(self.index, self.index + self.position.shape[0], out=self.position)
- self.seqlen.fill_(self.index + num_tokens)
+ # A tree verify attends the committed prefix, then folds in its own rows.
+ self.seqlen.fill_(self.index if tree is not None else self.index + num_tokens)
+
+ def commit_path(self, base, path):
+ # the accepted tree rows path[1:] become tokens base + 1, ...; a row never sits above its
+ # depth, so ascending copies never clobber a later source. Modulo for a circular window.
+ for k, r in enumerate(path[1:], 1):
+ if r != k:
+ dst, src = (base + k) % self.key.shape[2], (base + r) % self.key.shape[2]
+ self.key[:, :, dst] = self.key[:, :, src]
+ self.value[:, :, dst] = self.value[:, :, src]
@staticmethod
def shared(batch, device):
# all layers advance in lockstep, so the caches share one position/seqlen/tracker
- position = torch.empty((6,), device=device, dtype=torch.int64)
+ position = torch.empty((SPEC_ROWS,), device=device, dtype=torch.int64)
seqlen = torch.empty((batch,), device=device, dtype=torch.int32)
- return position, seqlen, {"step": -1}
+ return position, seqlen, {"step": -1, "tree": None}
class FixedKVCache(FixedKVState):
@@ -77,13 +91,22 @@ class FixedKVCache(FixedKVState):
self.value.index_copy_(2, position, xv)
def attend(self, xq, xk, xv):
+ tree = self.tracker["tree"]
+ if tree is not None:
+ # every row attends the committed prefix, then its ancestors and itself through the log-sum-exp
+ out, lse = comfy_kitchen.flash_attention_decode_gqa(xq, self.key, self.value, self.seqlen, return_lse=True, causal=False)
+ return comfy_kitchen.flash_attention_decode_tree_merge(out, lse, xq, xk, xv, tree.mask, torch.empty_like(out))
return comfy_kitchen.flash_attention_decode_gqa(xq, self.key, self.value, self.seqlen)
def mask(self, seq):
# additive [B, 1, seq, capacity] mask with attend()'s visibility, for a generic attention over the
- # whole cache: a chain row sees slots up to its own
+ # whole cache: a chain row sees slots up to its own, a tree row the prefix plus its ancestors and itself
rel = torch.arange(self.key.shape[2], device=self.key.device) - self.seqlen.view(-1, 1, 1, 1)
- visible = rel < torch.arange(1 - seq, 1, device=self.key.device).view(seq, 1)
+ tree = self.tracker["tree"]
+ if tree is not None:
+ visible = (rel < 0) | ((rel < seq) & ((tree.mask.view(seq, 1) >> rel.clamp(0, seq - 1)) & 1).bool())
+ else:
+ visible = rel < torch.arange(1 - seq, 1, device=self.key.device).view(seq, 1)
return torch.zeros(visible.shape, device=self.key.device, dtype=self.key.dtype).masked_fill_(~visible, float("-inf"))
@@ -97,6 +120,7 @@ class Int8FixedKVCache(FixedKVState):
key = torch.zeros((batch, kv_heads, window, head_dim), device=device, dtype=dtype)
cache = cls(key, torch.zeros_like(key), 0, *shared)
cache.packed = packed
+ cache.chain_mask = torch.tensor([(1 << (r + 1)) - 1 for r in range(SPEC_ROWS)], dtype=torch.int32, device=device)
return cache
def append(self, xk, xv):
@@ -116,10 +140,13 @@ class Int8FixedKVCache(FixedKVState):
self.value.index_copy_(2, position, xv)
def attend(self, xq, xk, xv):
+ seq = xq.shape[2]
length = self.position[:1]
self.packed.update(self.key, self.value, length)
out, lse = self.packed.attend(xq, length)
- return comfy_kitchen.flash_attention_decode_step_merge(out, lse, xq, xk, xv, torch.empty_like(out))
+ tree = self.tracker["tree"]
+ mask = tree.mask if tree is not None else self.chain_mask[:seq]
+ return comfy_kitchen.flash_attention_decode_tree_merge(out, lse, xq, xk, xv, mask, torch.empty_like(out))
@dataclass
class Llama2Config:
@@ -987,7 +1014,7 @@ class Llama2_(nn.Module):
mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])).expand(attention_mask.shape[0], 1, seq_len, attention_mask.shape[-1])
mask = mask.masked_fill(mask.to(torch.bool), torch.finfo(x.dtype).min / 4)
- spec_decode = fixed_kv and any(isinstance(kv, FixedKVState) for kv in past_key_values) and 2 <= seq_len <= 6 and past_len > 0 and attention_mask is None
+ spec_decode = fixed_kv and any(isinstance(kv, FixedKVState) for kv in past_key_values) and 2 <= seq_len <= SPEC_ROWS and past_len > 0 and attention_mask is None
if seq_len > 1 and not spec_decode: # spec verify: the staircase decode bias is causal
causal_mask = torch.empty(past_len + seq_len, past_len + seq_len, dtype=x.dtype, device=x.device).fill_(torch.finfo(x.dtype).min / 4).triu_(1)
if mask is not None:
diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py
index 0a2368824..4719c52dd 100644
--- a/comfy/text_encoders/qwen35.py
+++ b/comfy/text_encoders/qwen35.py
@@ -5,6 +5,7 @@ from dataclasses import dataclass, field
from tqdm import tqdm
import contextlib
import os
+import time
import comfy.model_management
import comfy.model_prefetch
@@ -16,7 +17,7 @@ from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention
from comfy import sd1_clip
import comfy.text_encoders.qwen_vl
-from .llama import BaseLlama, BaseGenerate, FixedKV, FixedKVCache, Int8FixedKVCache, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix
+from .llama import SPEC_ROWS, BaseLlama, BaseGenerate, FixedKV, FixedKVCache, Int8FixedKVCache, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix
@dataclass
@@ -38,16 +39,17 @@ class LinearKV(FixedKV):
@staticmethod
def deferred_tracker(device):
- # ctl = {pending, parity}: replay the previous step's accepted prefix
- n = comfy_kitchen.gated_delta_ctl_ints
+ # ctl = {pending, parity, slot[sm], parent[sm], prog[sm]}; without MTP, only row 0 runs.
+ n, sm = comfy_kitchen.gated_delta_ctl_ints, comfy_kitchen.gated_delta_slot_max
+ single = torch.tensor(list(range(sm)) + [-1] * sm + list(range(sm)), dtype=torch.int32)
host = torch.zeros((2, n), dtype=torch.int32)
# the host rows are pinned and fenced by events only where the copy is asynchronous
cuda = torch.device(device).type == "cuda"
return {"ctl": torch.zeros((n,), dtype=torch.int32, device=device), "host": host.pin_memory() if cuda else host,
- "copied": [torch.cuda.Event(), torch.cuda.Event()] if cuda else None, "step": None, "parity": 0}
+ "copied": [torch.cuda.Event(), torch.cuda.Event()] if cuda else None, "step": None, "parity": 0, "single": single, "path": None, "tree": None}
def decode_step(self, num_tokens):
- return self.index > 0 and num_tokens <= 6
+ return self.index > 0 and num_tokens <= SPEC_ROWS
def prepare(self, num_tokens):
# runs eagerly before the layer (a replayed graph skips its Python): the first linear
@@ -62,11 +64,21 @@ class LinearKV(FixedKV):
host = t["host"][t["parity"]]
host[0] = self.uncommitted
host[1] = t["parity"]
+ host[2:] = t["single"]
+ if t["path"] is not None:
+ host[2:2 + len(t["path"])] = torch.tensor(t["path"], dtype=torch.int32)
+ t["path"] = None
+ if t["tree"] is not None:
+ host[-len(t["tree"].ctl):] = t["tree"].ctl # parent[], prog[]
self.ctl.copy_(host, non_blocking=True)
if t["copied"] is not None:
t["copied"][t["parity"]].record(torch.cuda.current_stream(self.ctl.device))
self.uncommitted = num_tokens if self.decode_step(num_tokens) else 0
+ def commit_path(self, base, path):
+ # the next step replays the accepted verify rows
+ self.ctl_tracker["path"] = path
+
def rollback(self, discard=1):
# nothing was written: the next step replays fewer tokens
self.index -= discard
@@ -81,6 +93,116 @@ class LinearKV(FixedKV):
return self.value
+class VerifyTree:
+ """Shape of one MTP verify step: draft tokens in a tree rooted at the committed next token.
+
+ Built from nested child lists. A node's first child is the MTP argmax, later children its
+ runner-ups, and only argmax-chain nodes have children. Rows are numbered depth-first, first
+ child first, so the chain is rows 0..chain and no row sits above its depth.
+ """
+
+ def __init__(self, spec, capacity, device):
+ self.device = device
+ self.rows = capacity
+ self.mask = self.positions = None
+ self.update(spec)
+
+ def update(self, spec):
+ # Equal-row topology changes retain the addresses captured by CUDA graphs.
+ self.parent, self.children, self.depth = [], [], []
+
+ def add(node, parent):
+ r = len(self.parent)
+ self.parent.append(parent)
+ self.children.append([])
+ self.depth.append(0 if parent < 0 else self.depth[parent] + 1)
+ if parent >= 0:
+ self.children[parent].append(r)
+ for child in node:
+ add(child, r)
+
+ add(spec, -1)
+ count = len(self.parent)
+ self.chain = 0 # drafts on the argmax chain
+ while self.children[self.chain]:
+ self.chain = self.children[self.chain][0]
+ self.fanout = max(len(c) for c in self.children)
+ self.parent.extend([-1] * (self.rows - count))
+ self.children.extend([] for _ in range(self.rows - count))
+ self.depth.extend([0] * (self.rows - count))
+ # kitchen flash_attention_decode_tree_merge: bit t of row r's mask lets it attend row t
+ mask = []
+ for r in range(self.rows):
+ bits, a = 0, r
+ while a >= 0:
+ bits |= 1 << a
+ a = self.parent[a]
+ mask.append(bits)
+ mask = torch.tensor(mask, dtype=torch.int32, device=self.device)
+ if self.mask is None:
+ self.mask = mask
+ else:
+ self.mask.copy_(mask)
+ # kitchen gated_delta_decode_deferred program: a chain row commits (bit 5) after its
+ # children's leaves ran against its state without committing
+ prog = []
+ for r in range(self.chain + 1):
+ prog.append(r | (32 if self.children[r] else 0))
+ prog.extend(self.children[r][1:])
+ sm = comfy_kitchen.gated_delta_slot_max
+ self.ctl = torch.tensor(self.parent + [-1] * (sm - self.rows) + prog + list(range(count, sm)), dtype=torch.int32)
+ positions = torch.tensor([self.depth], dtype=torch.long, device=self.device)
+ if self.positions is None:
+ self.positions = positions
+ else:
+ self.positions.copy_(positions)
+
+
+# Verify capacity and topology. A fixed depth is a tree with one child per node.
+MTP_MODES = {
+ "2": (4, [[[]]]),
+ "3": (4, [[[[]]]]),
+ "4": (8, [[[[[]]]]]),
+ "5": (8, [[[[[[]]]]]]),
+ "tree2s1": (8, [[[], []], []]),
+ "tree2s2": (8, [[[], [], []], [], []]),
+ "tree3s1": (8, [[[[], []], []], []]),
+ "tree4s1m8": (8, [[[[[]], []], []], []]),
+ "auto": (8, None),
+}
+
+# Every positive composition of seven: a primary chain with terminal siblings.
+_SHAPES = []
+for _cuts in range(64):
+ _ends = [0] + [i for i in range(1, 7) if _cuts & (1 << (i - 1))] + [7]
+ _SHAPES.append(tuple(b - a for a, b in zip(_ends, _ends[1:])))
+
+
+def tree_spec(widths):
+ node = []
+ for width in reversed(widths):
+ node = [node] + [[] for _ in range(width - 1)]
+ return node
+
+
+def best_widths(widths, hits, draft_ms, verify_ms):
+ """Reallocate seven draft rows using conditional rank hits and measured GPU cost."""
+ probabilities = []
+ prior = [0.125] * 8
+ for row in hits:
+ count = sum(row)
+ prior = [(hit + 4 * p) / (count + 4) for hit, p in zip(row, prior)]
+ probabilities.append(prior)
+
+ def score(shape):
+ reach, emitted = 1.0, 1.0
+ for width, probability in zip(shape, probabilities):
+ emitted += reach * sum(probability[:width])
+ reach *= probability[0]
+ return emitted / (verify_ms + len(shape) * draft_ms)
+
+ best = max(_SHAPES, key=score)
+ return best if score(best) > 1.03 * score(widths) else widths
def _qwen35_layer_types(n):
@@ -387,7 +509,7 @@ class GatedAttention(nn.Module):
# KV cache
present_key_value = past_key_value
- decode = past_key_value is not None and seq_length <= 6 and attention_mask is None
+ decode = past_key_value is not None and seq_length <= SPEC_ROWS and attention_mask is None
if decode:
past_key_value.write(xk, xv)
@@ -776,10 +898,9 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
"repetition_penalty": repetition_penalty,
"presence_penalty": kwargs.get("presence_penalty", 0.0) or 0.0,
"seed": seed if seed is not None else 42}
- fixed_depth = None if mtp is True else max(2, min(5, int(mtp)))
- return self._generate_mtp(embeds, max_length, stop_tokens, sampling=sampling, fixed_depth=fixed_depth, position_ids=kwargs.get("position_ids"))
+ return self._generate_mtp(embeds, max_length, stop_tokens, mtp, sampling=sampling, position_ids=kwargs.get("position_ids"))
- def _generate_mtp(self, embeds, max_length, stop_tokens, sampling=None, fixed_depth=None, position_ids=None):
+ def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, position_ids=None):
device = embeds.device
cfg = self.model.config
if stop_tokens is None:
@@ -788,8 +909,6 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
embeds = embeds.to(dt)
if embeds.ndim == 2:
embeds = embeds.unsqueeze(0)
- # greedy drafts 3 deep (5 after the probe); sampled stays at 2
- depth = fixed_depth if fixed_depth is not None else (3 if sampling is None else 2)
cap = embeds.shape[1] + max_length + 7
pkv = self.init_kv_cache(embeds.shape[0], cap, device, dt)
# repair window: drafting ahead plus a near-full rollback
@@ -844,18 +963,25 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
progress.update(n)
console.update(n)
- verify_buffers = None
- drafts = None
-
- def set_depth(d):
- nonlocal depth, verify_buffers, drafts
- depth = d
- # Saved draft IDs must survive capture of later draft units.
- drafts = [torch.empty_like(nt_buf) for _ in range(d)]
- # static hidden/rope buffers: graphed layers bake their input addresses
- verify_buffers = (torch.empty((embeds.shape[0], d + 1, cfg.hidden_size), device=device, dtype=dt), freqs_at(pos, d + 1).clone())
-
- set_depth(depth)
+ adaptive = mtp is True
+ # auto: seven draft rows as a chain of widths, reshaped from rank hits and measured step cost
+ widths = (4, 3)
+ hits = [[0.0] * 8 for _ in range(7)]
+ steps, draft_ms, verify_ms = 0, 0.0, 0.0
+ draft_ranks = torch.zeros((7, 7), device=device, dtype=torch.long) if adaptive else None
+ timers = [torch.cuda.Event(enable_timing=True) for _ in range(3)] if adaptive and device.type == "cuda" else None
+ clocks = [0.0] * 3
+ capacity, spec = MTP_MODES["auto" if adaptive else mtp]
+ tree = VerifyTree(tree_spec(widths) if adaptive else spec, capacity, device)
+ depth = tree.chain
+ for kv in pkv:
+ if isinstance(kv, LinearKV):
+ kv.ctl_tracker["tree"] = tree
+ else:
+ kv.tracker["tree"] = tree
+ verify_tokens = torch.zeros((embeds.shape[0], tree.rows), device=device, dtype=torch.long)
+ verify_buffers = (torch.empty((embeds.shape[0], tree.rows, cfg.hidden_size), device=device, dtype=dt), freqs_at(pos, tree.rows).clone())
+ draft_top = torch.empty((embeds.shape[0], 7 if adaptive else tree.fanout), device=device, dtype=torch.long)
use_graph = self.model.graph_dynamic_vbar_blocks
compile_allocations = use_graph and comfy.model_prefetch.malloc_graph_enabled(device)
embed = self.model.embed_tokens
@@ -865,7 +991,7 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
draft_freqs = freqs_at(pos).clone()
def draft(token, hidden, p):
- # one drafted token: mtp head + lm_head argmax, graph-replayed when resident
+ # one drafted token (and its runner-ups for a tree): mtp head + lm_head, graph-replayed when resident
mtp_kv.prepare(1)
draft_token.copy_(token)
draft_hidden.copy_(hidden)
@@ -881,7 +1007,14 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
def core():
n1, r1 = self.mtp(draft_emb, draft_hidden, draft_freqs, mtp_kv)
draft_hidden.copy_(r1)
- draft_token.copy_(self.logits(n1)[:, -1].argmax(dim=-1, keepdim=True))
+ logits = self.logits(n1)[:, -1]
+ if adaptive:
+ draft_top.copy_(logits.topk(7, dim=-1).indices)
+ draft_token.copy_(draft_top[:, :1])
+ else:
+ draft_token.copy_(logits.argmax(dim=-1, keepdim=True))
+ if tree.fanout > 1 and not adaptive:
+ draft_top.copy_(logits.topk(tree.fanout, dim=-1).indices)
# Host rows stay eager; a tied table uses its graph slot for MTP + head.
comfy.model_prefetch.prefetch_queue_pop(queue, device, embed, dt, core=embed_core,
@@ -890,104 +1023,134 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
enable_graph=use_graph, malloc_scope="mtp")
comfy.model_prefetch.prefetch_queue_pop(queue, device, None, malloc_scope="mtp")
mtp_kv.advance(1)
- return draft_token, draft_hidden
+ return draft_token, draft_top, draft_hidden
- def verify_sample(lg, drafts):
- # accept draft i w.p. p_i(draft), else sample the residual; all depth+1 columns as one batch
+ def verify_sample(lg):
s = sampling
rows = lg[0].float()
if penalized:
- # committed tokens penalize every column, each draft the columns after it
- mask = pen_mask.unsqueeze(0).repeat(depth + 1, 1)
- for c, d in enumerate(drafts):
- mask[c + 1:].index_fill_(1, d.reshape(-1), True)
+ # Each row sees committed history and its own path, never siblings.
+ mask = pen_mask.unsqueeze(0).repeat(tree.rows, 1)
+ for r in range(1, tree.rows):
+ mask[r].copy_(mask[tree.parent[r]])
+ mask[r].index_fill_(0, verify_tokens[:, r], True)
rows = torch.where(mask, apply_penalty(rows, s["repetition_penalty"], s["presence_penalty"]), rows)
- probs, idx = self.processed_probs(rows, s["temperature"], s["top_k"], s["top_p"], s["min_p"], 1.0, [])
- dr = torch.cat(drafts, dim=1).reshape(-1, 1)
- if idx is None:
- p_draft = probs[:depth].gather(1, dr).reshape(-1)
- w = probs.clone()
- w[:depth].scatter_(1, dr, 0.0)
- else:
- hit = idx[:depth] == dr
- p_draft = (probs[:depth] * hit).sum(1)
- w = probs.clone()
- w[:depth].masked_fill_(hit, 0.0)
- # residual: p with the draft zeroed (the rescue term only matters when discarded anyway)
- w = w + (w.sum(1, keepdim=True) == 0) * probs
- tok = torch.multinomial(w, num_samples=1, generator=generator)
- corr = tok if idx is None else idx.gather(1, tok)
- u = torch.rand(depth, device=device, generator=generator)
- accepted = (u < p_draft).long().cumprod(0).sum()
- return dr, corr, accepted
+ return self.sample_token(rows, s["temperature"], s["top_k"], s["top_p"], s["min_p"],
+ 1.0, [], generator).transpose(0, 1)
def step():
# scoped so every temporary dies before the compiler bracket closes
nonlocal pos
- tok_in, hid_in = nt_buf, h_buf
+ if adaptive:
+ if timers:
+ timers[0].record(torch.cuda.current_stream(device))
+ else:
+ clocks[0] = time.perf_counter()
+ # chain draft k fills row k + 1, its runner-ups that node's sibling rows
+ verify_tokens[:, :1].copy_(nt_buf)
+ hid_in = h_buf
for k in range(depth):
- dk, rk = draft(tok_in, hid_in, pos + k)
- drafts[k].copy_(dk)
- tok_in, hid_in = drafts[k], rk
- tokens = torch.cat([nt_buf] + drafts, dim=1)
+ dk, top, hid_in = draft(verify_tokens[:, k:k + 1], hid_in, pos + k)
+ if adaptive:
+ draft_ranks[k].copy_(top[0])
+ children = tree.children[k]
+ verify_tokens[:, children[0]:children[0] + 1].copy_(dk)
+ for j, c in enumerate(children[1:], 1):
+ verify_tokens[:, c:c + 1].copy_(top[:, j:j + 1])
+ if adaptive:
+ if timers:
+ timers[1].record(torch.cuda.current_stream(device))
+ else:
+ clocks[1] = time.perf_counter()
embed = self.model.embed_tokens
if hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed):
- ev = embed.host_rows(tokens, out_dtype=dt)
+ ev = embed.host_rows(verify_tokens, out_dtype=dt)
else:
- ev = embed(tokens).to(dt)
- verify_pos = torch.arange(pos + rope_offset, pos + rope_offset + depth + 1, device=device).unsqueeze(0)
+ ev = embed(verify_tokens).to(dt)
+ verify_pos = tree.positions + pos + rope_offset
x, _, _ = self.model.forward(None, embeds=ev, attention_mask=None, past_key_values=pkv, position_ids=verify_pos, decode_buffers=verify_buffers)
# all verify positions in one lm_head GEMV, accept decided GPU-side, one sync
lg = verify_logits(x)
- if sampling is None:
- toks = lg.argmax(dim=-1)
- vals = torch.cat([toks[0]] + [d[0] for d in drafts]).tolist()
- t, dr = vals[:depth + 1], vals[depth + 1:]
- accepts = 0
- while accepts < depth and t[accepts] == dr[accepts]:
- accepts += 1
- next_toks = tuple(toks[:, i:i + 1] for i in range(depth + 1))
- commit = tuple(dr[:accepts]) + (t[accepts],)
- else:
- dr, corr, accepted = verify_sample(lg, drafts)
- vals = torch.cat([dr[:, 0], corr[:, 0], accepted.reshape(1)]).tolist()
- dr, cv, accepts = vals[:depth], vals[depth:2 * depth + 1], vals[-1]
- next_toks = tuple(corr[i:i + 1] for i in range(depth + 1))
- commit = tuple(dr[:accepts]) + (cv[accepts],)
- if accepts < depth:
+ toks = lg.argmax(dim=-1) if sampling is None else verify_sample(lg)
+ if adaptive:
+ if timers:
+ timers[2].record(torch.cuda.current_stream(device))
+ else:
+ clocks[2] = time.perf_counter()
+ packet = [toks[0], verify_tokens[0, 1:]]
+ if adaptive:
+ packet.append(draft_ranks.reshape(-1))
+ vals = torch.cat(packet).tolist()
+ t, dr = vals[:tree.rows], [None] + vals[tree.rows:2 * tree.rows - 1]
+ # Continue only along a child containing the main model's chosen token.
+ path = [0]
+ while True:
+ node = next((c for c in tree.children[path[-1]] if dr[c] == t[path[-1]]), None)
+ if node is None:
+ break
+ path.append(node)
+ ranks = []
+ if adaptive:
+ for node in path:
+ if node >= depth:
+ break
+ candidates = vals[15 + 7 * node:22 + 7 * node]
+ ranks.append(next((r for r, token in enumerate(candidates) if token == t[node]), 7))
+ if timers:
+ clocks[0], clocks[1], clocks[2] = 0.0, timers[0].elapsed_time(timers[1]), timers[0].elapsed_time(timers[2])
+ else:
+ clocks[:] = [t * 1000 for t in clocks]
+ commit = tuple(dr[r] for r in path[1:]) + (t[path[-1]],)
+ row = path[-1]
+ next_tok = toks[:, row:row + 1]
+ if len(path) < tree.rows:
+ for kv in pkv:
+ kv.rollback(tree.rows - len(path))
+ on_chain = row == len(path) - 1
+ if not on_chain:
for kv in pkv:
- kv.rollback(depth - accepts)
- if accepts < depth - 1:
- mtp_kv.rollback(depth - 1 - accepts) # mtp entries fed by a rejected draft token
- nt_buf.copy_(next_toks[accepts])
- h_buf.copy_(x[:, accepts:accepts + 1, :])
+ kv.commit_path(pos, path)
+ # mtp entries fed by a rejected or off-chain draft token
+ kept = min(len(path) if on_chain else len(path) - 1, depth)
+ if kept < depth:
+ mtp_kv.rollback(depth - kept)
+ nt_buf.copy_(next_tok)
+ h_buf.copy_(x[:, row:row + 1, :])
if penalized:
- for d in drafts[:accepts]:
- pen_mask.index_fill_(0, d.reshape(-1), True)
+ for r in path[1:]:
+ pen_mask.index_fill_(0, verify_tokens[:, r], True)
pen_mask.index_fill_(0, nt_buf.reshape(-1), True)
- pos += accepts + 1
- return accepts, commit
+ pos += len(commit)
+ return commit, ranks
- probe = None if fixed_depth is not None else [0, 0] # steps, accepted drafts
try:
while len(ids) < max_length and ids[-1] not in stop_tokens:
with (comfy.model_prefetch.malloc_graph_scope(device) if compile_allocations else contextlib.nullcontext()):
- accepts, commit = step()
+ commit, ranks = step()
+ if adaptive:
+ steps += 1
+ for row in hits:
+ for r in range(8):
+ row[r] *= 0.984375
+ for level, rank in enumerate(ranks):
+ hits[level][rank] += 1
+ # Do not learn graph capture cost as steady-state execution cost.
+ if steps > 8:
+ mix = 1.0 if steps == 9 else 0.125
+ draft_ms += mix * ((clocks[1] - clocks[0]) / len(widths) - draft_ms)
+ verify_ms += mix * (clocks[2] - clocks[1] - verify_ms)
+ if steps % 32 == 0:
+ best = best_widths(widths, hits, draft_ms, verify_ms)
+ if best != widths:
+ widths = best
+ tree.update(tree_spec(widths))
+ depth = tree.chain
commit = list(commit[:max_length - len(ids)])
stop = next((i for i, t in enumerate(commit) if t in stop_tokens), None)
if stop is not None:
del commit[stop + 1:]
ids.extend(commit)
update_progress(len(commit))
- if probe is not None:
- probe[0] += 1
- probe[1] += accepts
- if probe[0] == 32:
- # deepen once acceptance sustains it and a recapture round can amortize
- if sampling is None and max_length - len(ids) > 512 and 1 + probe[1] / probe[0] >= 2.2:
- comfy.model_prefetch.cleanup_prefetch_queues()
- set_depth(5)
- probe = None
finally:
console.close()
comfy.model_prefetch.cleanup_prefetch_queues()
diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.py
index efd42b98b..0e7d3a9af 100644
--- a/comfy_extras/nodes_textgen.py
+++ b/comfy_extras/nodes_textgen.py
@@ -39,7 +39,7 @@ class TextGenerate(io.ComfyNode):
io.DynamicCombo.Input("sampling_mode", options=sampling_options, display_name="Sampling Mode"),
io.Boolean.Input("thinking", optional=True, default=False, tooltip="Operate in thinking mode if the model supports it."),
io.Boolean.Input("use_default_template", optional=True, default=True, tooltip="Use the built in system prompt/template if the model has one.", advanced=True),
- io.Combo.Input("mtp", options=["auto", "off", "2", "3", "4", "5"], default="auto", optional=True, tooltip="Speculative decoding with the checkpoint's multi-token-prediction head. No effect without MTP weights. auto adapts the draft depth; 2-5 pins it. Sampled output stays correctly distributed but differs from non-MTP output for the same seed."),
+ io.Combo.Input("mtp", options=["auto", "off", "2", "3", "4", "5", "tree2s1", "tree2s2", "tree3s1", "tree4s1m8"], default="auto", optional=True, tooltip="Speculative decoding with the checkpoint's multi-token-prediction head. No effect without MTP weights. auto adapts the draft depth and breadth to observed token matches and execution cost; 2-5 pins a constant draft depth; tree modes pin both breadth and depth. Sampled output stays correctly distributed but differs from non-MTP output for the same seed."),
io.String.Input("system_prompt", force_input=True, optional=True, tooltip="Replaces the system prompt in the model's chat template. Ignored when the default template is not used."),
],
outputs=[
@@ -51,7 +51,7 @@ class TextGenerate(io.ComfyNode):
@classmethod
def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="") -> io.NodeOutput:
- mtp = False if mtp == "off" else (True if mtp == "auto" else int(mtp))
+ mtp = {"off": False, "auto": True}.get(mtp, mtp)
tokens = clip.tokenize(prompt, image=image, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, audio=audio, system_prompt=system_prompt if use_default_template else "")
diff --git a/requirements.txt b/requirements.txt
index f46143b8b..0c12dd5ff 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -21,7 +21,7 @@ alembic
SQLAlchemy>=2.0.0
filelock
av>=17.0.0
-comfy-kitchen==0.2.39
+comfy-kitchen==0.2.40
comfy-aimdo==0.5.6
requests
simpleeval>=1.0.0
diff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.py
index e8d55ab9d..efd322865 100644
--- a/tests-unit/comfy_test/qwen35_decode_test.py
+++ b/tests-unit/comfy_test/qwen35_decode_test.py
@@ -176,15 +176,17 @@ def _masked_attention(q, k, v, mask):
def test_cache_mask_matches_attend(monkeypatch):
- # the additive mask reproduces attend()'s chain staircase over the whole cache
+ # the additive mask reproduces attend()'s chain staircase and tree ancestry over the whole cache
from comfy.text_encoders.llama import FixedKVCache
+ tree = _tree2s2()
torch.manual_seed(0)
shared = FixedKVCache.shared(2, "cpu")
kv = FixedKVCache.zeros(2, 1, 24, 64, "cpu", torch.float32, shared)
kv.key.normal_()
kv.value.normal_()
kv.index = 9
- for seq in (1, 3):
+ for seq, shaped in ((1, None), (3, None), (8, tree)):
+ kv.tracker["tree"] = shaped
kv.prepare(seq)
xq = torch.randn(2, 2, seq, 64)
xk = torch.randn(2, 1, seq, 64)
@@ -194,7 +196,11 @@ def test_cache_mask_matches_attend(monkeypatch):
mask = kv.mask(seq)
assert mask.shape == (2, 1, seq, 24) and mask.dtype == torch.float32
visible = (mask == 0)[0, 0]
- assert visible.sum(-1).tolist() == [10 + j for j in range(seq)]
+ if shaped is None:
+ assert visible.sum(-1).tolist() == [10 + j for j in range(seq)]
+ else:
+ assert visible[:, 9:17].int().tolist() == [[(m >> t) & 1 for t in range(8)] for m in tree.mask.tolist()]
+ assert visible[:, :9].all() and not visible[:, 17:].any()
torch.testing.assert_close(_masked_attention(xq, kv.key, kv.value, mask), expected, rtol=1e-4, atol=1e-5)
@@ -216,7 +222,7 @@ def test_deferred_ctl_rows_wait_for_their_previous_copy(monkeypatch):
tracker = LinearKV.deferred_tracker("cpu")
tracker["copied"] = [Event(0), Event(1)]
ctl = tracker["ctl"]
- assert ctl.shape == (2,) # {pending, parity}
+ assert ctl.shape == (26,) # {pending, parity, slots[8], parents[8], program[8]}
layers = [LinearKV(torch.zeros(1), torch.zeros(1), 0, None, None, ctl=ctl, ctl_tracker=tracker) for _ in range(2)]
def step(n, discard=0):
@@ -237,6 +243,85 @@ def test_deferred_ctl_rows_wait_for_their_previous_copy(monkeypatch):
assert step(1) == [3, 0]
+def _tree2s2():
+ from comfy.text_encoders.qwen35 import MTP_MODES, VerifyTree
+ capacity, spec = MTP_MODES["tree2s2"]
+ return VerifyTree(spec, capacity, "cpu")
+
+
+def test_verify_tree_shapes():
+ from comfy.text_encoders.qwen35 import MTP_MODES, VerifyTree
+ tree = _tree2s2()
+ assert (tree.rows, tree.chain, tree.fanout) == (8, 2, 3)
+ assert tree.parent == [-1, 0, 1, 1, 1, 0, 0, -1] and tree.depth == [0, 1, 2, 2, 2, 1, 1, 0]
+ assert tree.mask.tolist() == [1, 3, 7, 11, 19, 33, 65, 128]
+ assert tree.ctl.tolist() == [-1, 0, 1, 1, 1, 0, 0, -1, 32, 5, 6, 33, 3, 4, 2, 7]
+ tree = VerifyTree(MTP_MODES["tree4s1m8"][1], 8, "cpu")
+ assert (tree.rows, tree.chain, tree.fanout) == (8, 4, 2)
+ assert tree.parent == [-1, 0, 1, 2, 3, 2, 1, 0] and tree.depth == [0, 1, 2, 3, 4, 3, 2, 1]
+ assert tree.mask.tolist() == [1, 3, 7, 15, 31, 39, 67, 129]
+ assert tree.ctl.tolist() == [-1, 0, 1, 2, 3, 2, 1, 0, 32, 7, 33, 6, 34, 5, 35, 4]
+ mask_ptr, pos_ptr = tree.mask.data_ptr(), tree.positions.data_ptr()
+ tree.update([[[[[[[[]]]]]]]])
+ assert tree.mask.tolist() == [1, 3, 7, 15, 31, 63, 127, 255]
+ assert tree.positions.tolist() == [list(range(8))]
+ tree.update(MTP_MODES["tree4s1m8"][1])
+ assert tree.mask.data_ptr() == mask_ptr and tree.positions.data_ptr() == pos_ptr
+ assert tree.mask.tolist() == [1, 3, 7, 15, 31, 39, 67, 129]
+ assert tree.positions.tolist() == [[0, 1, 2, 3, 4, 3, 2, 1]]
+ for name, depth in (("tree2s1", 2), ("tree3s1", 3)):
+ tree = VerifyTree(MTP_MODES[name][1], MTP_MODES[name][0], "cpu")
+ assert (tree.rows, tree.chain, tree.fanout) == (8, depth, 2)
+ chain = VerifyTree(MTP_MODES["3"][1], 4, "cpu")
+ assert (chain.rows, chain.chain, chain.fanout, chain.children[:3]) == (4, 3, 1, [[1], [2], [3]])
+ assert chain.mask.tolist() == [1, 3, 7, 15]
+ for depth, capacity in ((2, 4), (4, 8), (5, 8)):
+ tree = VerifyTree(MTP_MODES[str(depth)][1], capacity, "cpu")
+ assert tree.chain == depth and tree.rows == capacity
+ assert tree.parent[depth + 1:] == [-1] * (capacity - depth - 1)
+ assert tree.ctl[8 + depth + 1:].tolist() == list(range(depth + 1, 8))
+ assert all(not children for children in tree.children[depth:])
+
+
+def test_tree_verify_cache_prefix_and_commit_path():
+ from comfy.text_encoders.llama import FixedKVCache
+ tree = _tree2s2()
+ shared = FixedKVCache.shared(1, "cpu")
+ kv = FixedKVCache.zeros(1, 1, 32, 256, "cpu", torch.bfloat16, shared)
+ kv.index = 10
+ kv.tracker["tree"] = tree
+ kv.prepare(8)
+ assert kv.seqlen.tolist() == [10]
+ kv.tracker["tree"] = None
+ kv.prepare(8)
+ assert kv.seqlen.tolist() == [18]
+ for r in range(7):
+ kv.key[:, :, 10 + r] = r
+ kv.value[:, :, 10 + r] = -r
+ kv.commit_path(10, [0, 1, 4])
+ assert kv.key[0, 0, 10:13, 0].tolist() == [0, 1, 4] and kv.value[0, 0, 12, 0].item() == -4
+
+
+def test_deferred_ctl_carries_accepted_path_and_tree_program():
+ from comfy.text_encoders.qwen35 import LinearKV
+ tree = _tree2s2()
+ tracker = LinearKV.deferred_tracker("cpu")
+ tracker["tree"] = tree
+ ctl = tracker["ctl"]
+ kv = LinearKV(torch.zeros(1), torch.zeros(1), 5, None, None, ctl=ctl, ctl_tracker=tracker)
+ kv.prepare(8)
+ assert ctl[2:10].tolist() == list(range(8)) and ctl[10:].tolist() == tree.ctl.tolist()
+ kv.advance(8)
+ kv.rollback(5)
+ kv.commit_path(5, [0, 1, 4])
+ kv.prepare(8)
+ assert ctl[0].item() == 3 and ctl[2:10].tolist() == [0, 1, 4, 3, 4, 5, 6, 7]
+ tracker["tree"] = None
+ kv.advance(8)
+ kv.prepare(1)
+ assert ctl[2:].tolist() == tracker["single"].tolist()
+
+
def _tiny_model(monkeypatch, layers=4, head_dim=64):
import comfy.text_encoders.qwen35 as qwen35
monkeypatch.setitem(qwen35.QWEN35_MODELS, "tiny", dict(vision=dict(hidden_size=32, num_heads=2, intermediate_size=64, depth=1)))
@@ -257,7 +342,7 @@ def _tiny_model(monkeypatch, layers=4, head_dim=64):
@pytest.mark.parametrize("image_prompt", [False, True])
def test_greedy_mtp_matches_plain_greedy(monkeypatch, image_prompt):
- # auto and fixed-depth chain verify through the kitchen torch paths: causal GQA, deferred DeltaNet
+ # chain and tree verify through the kitchen torch paths: GQA prefix + tree merge, deferred DeltaNet
from comfy.text_encoders.llama import FixedKVCache
from comfy.text_encoders.qwen_vl import qwen2vl_mrope_position_ids
model = _tiny_model(monkeypatch)
@@ -270,12 +355,61 @@ def test_greedy_mtp_matches_plain_greedy(monkeypatch, image_prompt):
if image_prompt:
assert positions.shape == (3, 24) and positions[:, -1].tolist() == [11, 11, 11]
plain = model.generate(embeds.clone(), do_sample=False, max_length=40, mtp=False, position_ids=positions)
- for mtp in (True, 3, 5):
+ for mtp in (True, "2", "3", "4", "5", "tree2s1", "tree2s2", "tree3s1", "tree4s1m8"):
discards.clear()
assert model.generate(embeds.clone(), do_sample=False, max_length=40, mtp=mtp, position_ids=positions) == plain
assert discards and min(discards) < max(discards) # partial acceptance happened
[email protected]("quantile", [0.42, 0.8, 0.92])
[email protected]("top_k", [0, 7])
+def test_sampled_tree_matches_sequential_with_same_draws(monkeypatch, quantile, top_k):
+ # Couple draws by quantile rather than RNG consumption: unused tree rows also sample.
+ def draw(weights, num_samples, generator=None):
+ assert num_samples == 1
+ cdf = weights.cumsum(-1) / weights.sum(-1, keepdim=True)
+ return (cdf < quantile).sum(-1, keepdim=True)
+
+ monkeypatch.setattr(torch, "multinomial", draw)
+ model = _tiny_model(monkeypatch)
+ embeds = torch.randn(1, 5, 64)
+ settings = dict(do_sample=True, max_length=65, temperature=0.7, top_k=top_k, top_p=0.95,
+ min_p=0.05, repetition_penalty=1.2, presence_penalty=0.8)
+ expected = model.generate(embeds.clone(), mtp=False, **settings)
+ for preset in (True, "2", "3", "4", "5", "tree2s1", "tree2s2", "tree3s1", "tree4s1m8"):
+ assert model.generate(embeds.clone(), mtp=preset, **settings) == expected, preset
+ stop = expected[4]
+ stopped = expected[:expected.index(stop) + 1]
+ assert model.generate(embeds.clone(), mtp=preset, stop_tokens=[stop], **settings) == stopped
+
+
[email protected]("sampled", [False, True])
+def test_adaptive_switch_preserves_sequential_output(monkeypatch, sampled):
+ import comfy.text_encoders.qwen35 as qwen35
+
+ def draw(weights, num_samples, generator=None):
+ cdf = weights.cumsum(-1) / weights.sum(-1, keepdim=True)
+ return (cdf < 0.8).sum(-1, keepdim=True)
+
+ switches = []
+
+ def best_widths(widths, hits, draft_ms, verify_ms):
+ # every 32nd step: cycle through shapes that differ in depth and fanout
+ assert len(hits) == 7 and all(h >= 0 for row in hits for h in row)
+ switches.append(((1,) * 7, (2, 1, 4), (7,), (4, 3))[len(switches) % 4])
+ return switches[-1]
+
+ monkeypatch.setattr(torch, "multinomial", draw)
+ monkeypatch.setattr(qwen35, "best_widths", best_widths)
+ model = _tiny_model(monkeypatch)
+ embeds = torch.randn(1, 5, 64)
+ settings = dict(do_sample=sampled, max_length=400, temperature=0.7, top_k=7,
+ top_p=0.95, min_p=0.05, repetition_penalty=1.2, presence_penalty=0.8)
+ expected = model.generate(embeds.clone(), mtp=False, **settings)
+ assert model.generate(embeds.clone(), mtp=True, **settings) == expected
+ assert len(set(switches)) == 4, switches
+
+
class _FakePacked:
# Int8DecodeCache boundary with 4-row pages: records the BF16 refreshes it is asked for
def __init__(self, batch, heads, capacity, device):
@@ -296,11 +430,11 @@ def _int8_cache(monkeypatch, capacity=32):
_fake_kitchen(monkeypatch)
merges = []
- def step_merge(out, lse, q, k, v, result):
- merges.append((k, v))
+ def tree_merge(out, lse, q, k, v, mask, result):
+ merges.append((k, v, mask.clone()))
return out
monkeypatch.setattr(llama, "Int8DecodeCache", _FakePacked, raising=False)
- monkeypatch.setattr(llama.comfy_kitchen, "flash_attention_decode_step_merge", step_merge)
+ monkeypatch.setattr(llama.comfy_kitchen, "flash_attention_decode_tree_merge", tree_merge)
shared = FixedKVCache.shared(1, "cpu")
return llama.Int8FixedKVCache.zeros(1, 1, capacity, 256, "cpu", torch.bfloat16, shared), merges
@@ -319,8 +453,8 @@ def test_int8_cache_merges_current_chain_rows(monkeypatch):
assert kv.attend(xq, xk, xv).shape == (1, 3, 512)
# the committed prefix is quantized once, then refreshed to the same length before attending
assert kv.packed.updates == [([10], True), ([10], False)]
- k, v = merges[-1]
- assert k is xk and v is xv
+ k, v, mask = merges[-1]
+ assert k is xk and v is xv and mask.tolist() == [1, 3, 7]
def test_int8_cache_keeps_bounded_circular_window(monkeypatch):
@@ -346,6 +480,27 @@ def test_int8_cache_keeps_bounded_circular_window(monkeypatch):
assert torch.equal(kv.key[:, :, 2:5], xk) and kv.packed.updates[-1] == ([14], False)
[email protected]("path", [[0, 1, 2, 3, 4], [0, 1, 2, 5], [0, 7]])
+def test_tree4s1m8_cache_commits_deep_and_sibling_paths(monkeypatch, path):
+ from comfy.text_encoders.qwen35 import MTP_MODES, VerifyTree
+ tree = VerifyTree(MTP_MODES["tree4s1m8"][1], 8, "cpu")
+ kv, merges = _int8_cache(monkeypatch)
+ kv.index = 10
+ kv.tracker["tree"] = tree
+ kv.prepare(8)
+ xk = torch.randn(1, 1, 8, 256, dtype=torch.bfloat16)
+ kv.write(xk, -xk)
+ kv.attend(torch.randn(1, 2, 8, 256, dtype=torch.bfloat16), xk, -xk)
+ assert merges[-1][2].tolist() == [1, 3, 7, 15, 31, 39, 67, 129]
+ kv.advance(8)
+ kv.rollback(8 - len(path))
+ kv.commit_path(10, path)
+ assert kv.index == 10 + len(path)
+ for token, row in enumerate(path, 10):
+ assert torch.equal(kv.key[:, :, token % 12], xk[:, :, row])
+ assert torch.equal(kv.value[:, :, token % 12], -xk[:, :, row])
+
+
def _attention_config(config):
return torch.tensor(list(json.dumps(config).encode()), dtype=torch.uint8)
@@ -405,8 +560,8 @@ def test_checkpoint_and_lora_attention_select_each_kv_layer(monkeypatch):
assert cache_types() == (FixedKVCache, FixedKVCache, FixedKVCache)
[email protected]("seq", [1, 3])
-def test_configured_decode_preserves_causal_visibility(monkeypatch, seq):
[email protected]("seq,tree_verify", [(1, False), (3, False), (8, True)])
+def test_configured_decode_preserves_causal_and_tree_visibility(monkeypatch, seq, tree_verify):
import comfy.ldm.modules.attention as attention
# Make the configured backend observably different from the default backend.
@attention.wrap_attn
@@ -423,13 +578,25 @@ def test_configured_decode_preserves_causal_visibility(monkeypatch, seq):
kv.prepare(9)
attn(prefix, attention_mask=torch.full((9, 9), -torch.inf).triu(1),
freqs_cis=torch.zeros(1, 1, 9, 32, 2, 2), optimized_attention=attention.attention_pytorch, past_key_value=kv)
- # Repeat after rollback; stale rows past the current chain must not become visible.
+ tree = _tree2s2() if tree_verify else None
+ kv.tracker["tree"] = tree
+ # Repeat after rollback; stale tree rows must not become visible.
for start in (9, 4):
kv.index = start
kv.prepare(seq)
out, _ = attn(current, freqs_cis=torch.zeros(1, 1, seq, 32, 2, 2), past_key_value=kv)
full = torch.cat((prefix[:, :start], current), dim=1)
mask = torch.full((start + seq, start + seq), -torch.inf).triu(1)
+ parents = [-1, 0, 1, 1, 1, 0, 0, -1]
+ for row in range(seq):
+ ancestors = []
+ r = row
+ while r >= 0:
+ ancestors.append(start + r)
+ r = parents[r] if tree_verify else r - 1
+ visible = list(range(start)) + sorted(ancestors)
+ mask[start + row] = -torch.inf
+ mask[start + row, visible] = 0
expected, _ = attn(full, attention_mask=mask, freqs_cis=torch.zeros(1, 1, start + seq, 32, 2, 2),
optimized_attention=attention.attention_pytorch)
torch.testing.assert_close(out, expected[:, start:])