The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
qwen 3.x performance improvements part III - Adaptive tree MTP (#17273)
rattusf7c664378a9a
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex 4ff4158dd..d84d7b7cd 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -17,6 +17,9 @@ import comfy.clip_modelfrom . import qwen_vl+# widest speculative verify (draft chain or tree rows) the decode caches serve in one step+SPEC_ROWS = 8+@dataclassclass FixedKV:@@ -42,18 +45,29 @@ class FixedKVState(FixedKV):tracker: dict = Nonedef prepare(self, num_tokens):- if self.tracker["step"] == (self.index, num_tokens):+ tree = self.tracker["tree"]+ if self.tracker["step"] == (self.index, num_tokens, tree):return- self.tracker["step"] = (self.index, num_tokens)+ self.tracker["step"] = (self.index, num_tokens, tree)torch.arange(self.index, self.index + self.position.shape[0], out=self.position)- self.seqlen.fill_(self.index + num_tokens)+ # A tree verify attends the committed prefix, then folds in its own rows.+ self.seqlen.fill_(self.index if tree is not None else self.index + num_tokens)++ def commit_path(self, base, path):+ # the accepted tree rows path[1:] become tokens base + 1, ...; a row never sits above its+ # depth, so ascending copies never clobber a later source. Modulo for a circular window.+ for k, r in enumerate(path[1:], 1):+ if r != k:+ dst, src = (base + k) % self.key.shape[2], (base + r) % self.key.shape[2]+ self.key[:, :, dst] = self.key[:, :, src]+ self.value[:, :, dst] = self.value[:, :, src]@staticmethoddef shared(batch, device):# all layers advance in lockstep, so the caches share one position/seqlen/tracker- position = torch.empty((6,), device=device, dtype=torch.int64)+ position = torch.empty((SPEC_ROWS,), device=device, dtype=torch.int64)seqlen = torch.empty((batch,), device=device, dtype=torch.int32)- return position, seqlen, {"step": -1}+ return position, seqlen, {"step": -1, "tree": None}class FixedKVCache(FixedKVState):@@ -77,13 +91,22 @@ class FixedKVCache(FixedKVState):self.value.index_copy_(2, position, xv)def attend(self, xq, xk, xv):+ tree = self.tracker["tree"]+ if tree is not None:+ # every row attends the committed prefix, then its ancestors and itself through the log-sum-exp+ out, lse = comfy_kitchen.flash_attention_decode_gqa(xq, self.key, self.value, self.seqlen, return_lse=True, causal=False)+ return comfy_kitchen.flash_attention_decode_tree_merge(out, lse, xq, xk, xv, tree.mask, torch.empty_like(out))return comfy_kitchen.flash_attention_decode_gqa(xq, self.key, self.value, self.seqlen)def mask(self, seq):# additive [B, 1, seq, capacity] mask with attend()'s visibility, for a generic attention over the- # whole cache: a chain row sees slots up to its own+ # whole cache: a chain row sees slots up to its own, a tree row the prefix plus its ancestors and itselfrel = torch.arange(self.key.shape[2], device=self.key.device) - self.seqlen.view(-1, 1, 1, 1)- visible = rel < torch.arange(1 - seq, 1, device=self.key.device).view(seq, 1)+ tree = self.tracker["tree"]+ if tree is not None:+ visible = (rel < 0) | ((rel < seq) & ((tree.mask.view(seq, 1) >> rel.clamp(0, seq - 1)) & 1).bool())+ else:+ visible = rel < torch.arange(1 - seq, 1, device=self.key.device).view(seq, 1)return torch.zeros(visible.shape, device=self.key.device, dtype=self.key.dtype).masked_fill_(~visible, float("-inf"))@@ -97,6 +120,7 @@ class Int8FixedKVCache(FixedKVState):key = torch.zeros((batch, kv_heads, window, head_dim), device=device, dtype=dtype)cache = cls(key, torch.zeros_like(key), 0, *shared)cache.packed = packed+ cache.chain_mask = torch.tensor([(1 << (r + 1)) - 1 for r in range(SPEC_ROWS)], dtype=torch.int32, device=device)return cachedef append(self, xk, xv):@@ -116,10 +140,13 @@ class Int8FixedKVCache(FixedKVState):self.value.index_copy_(2, position, xv)def attend(self, xq, xk, xv):+ seq = xq.shape[2]length = self.position[:1]self.packed.update(self.key, self.value, length)out, lse = self.packed.attend(xq, length)- return comfy_kitchen.flash_attention_decode_step_merge(out, lse, xq, xk, xv, torch.empty_like(out))+ tree = self.tracker["tree"]+ mask = tree.mask if tree is not None else self.chain_mask[:seq]+ return comfy_kitchen.flash_attention_decode_tree_merge(out, lse, xq, xk, xv, mask, torch.empty_like(out))@dataclassclass Llama2Config:@@ -987,7 +1014,7 @@ class Llama2_(nn.Module):mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])).expand(attention_mask.shape[0], 1, seq_len, attention_mask.shape[-1])mask = mask.masked_fill(mask.to(torch.bool), torch.finfo(x.dtype).min / 4)- spec_decode = fixed_kv and any(isinstance(kv, FixedKVState) for kv in past_key_values) and 2 <= seq_len <= 6 and past_len > 0 and attention_mask is None+ spec_decode = fixed_kv and any(isinstance(kv, FixedKVState) for kv in past_key_values) and 2 <= seq_len <= SPEC_ROWS and past_len > 0 and attention_mask is Noneif seq_len > 1 and not spec_decode: # spec verify: the staircase decode bias is causalcausal_mask = torch.empty(past_len + seq_len, past_len + seq_len, dtype=x.dtype, device=x.device).fill_(torch.finfo(x.dtype).min / 4).triu_(1)if mask is not None:diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.pyindex 0a2368824..4719c52dd 100644--- a/comfy/text_encoders/qwen35.py+++ b/comfy/text_encoders/qwen35.py@@ -5,6 +5,7 @@ from dataclasses import dataclass, fieldfrom tqdm import tqdmimport contextlibimport os+import timeimport comfy.model_managementimport comfy.model_prefetch@@ -16,7 +17,7 @@ from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttentionfrom comfy import sd1_clipimport comfy.text_encoders.qwen_vl-from .llama import BaseLlama, BaseGenerate, FixedKV, FixedKVCache, Int8FixedKVCache, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix+from .llama import SPEC_ROWS, BaseLlama, BaseGenerate, FixedKV, FixedKVCache, Int8FixedKVCache, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix@dataclass@@ -38,16 +39,17 @@ class LinearKV(FixedKV):@staticmethoddef deferred_tracker(device):- # ctl = {pending, parity}: replay the previous step's accepted prefix- n = comfy_kitchen.gated_delta_ctl_ints+ # ctl = {pending, parity, slot[sm], parent[sm], prog[sm]}; without MTP, only row 0 runs.+ n, sm = comfy_kitchen.gated_delta_ctl_ints, comfy_kitchen.gated_delta_slot_max+ single = torch.tensor(list(range(sm)) + [-1] * sm + list(range(sm)), dtype=torch.int32)host = torch.zeros((2, n), dtype=torch.int32)# the host rows are pinned and fenced by events only where the copy is asynchronouscuda = torch.device(device).type == "cuda"return {"ctl": torch.zeros((n,), dtype=torch.int32, device=device), "host": host.pin_memory() if cuda else host,- "copied": [torch.cuda.Event(), torch.cuda.Event()] if cuda else None, "step": None, "parity": 0}+ "copied": [torch.cuda.Event(), torch.cuda.Event()] if cuda else None, "step": None, "parity": 0, "single": single, "path": None, "tree": None}def decode_step(self, num_tokens):- return self.index > 0 and num_tokens <= 6+ return self.index > 0 and num_tokens <= SPEC_ROWSdef prepare(self, num_tokens):# runs eagerly before the layer (a replayed graph skips its Python): the first linear@@ -62,11 +64,21 @@ class LinearKV(FixedKV):host = t["host"][t["parity"]]host[0] = self.uncommittedhost[1] = t["parity"]+ host[2:] = t["single"]+ if t["path"] is not None:+ host[2:2 + len(t["path"])] = torch.tensor(t["path"], dtype=torch.int32)+ t["path"] = None+ if t["tree"] is not None:+ host[-len(t["tree"].ctl):] = t["tree"].ctl # parent[], prog[]self.ctl.copy_(host, non_blocking=True)if t["copied"] is not None:t["copied"][t["parity"]].record(torch.cuda.current_stream(self.ctl.device))self.uncommitted = num_tokens if self.decode_step(num_tokens) else 0+ def commit_path(self, base, path):+ # the next step replays the accepted verify rows+ self.ctl_tracker["path"] = path+def rollback(self, discard=1):# nothing was written: the next step replays fewer tokensself.index -= discard@@ -81,6 +93,116 @@ class LinearKV(FixedKV):return self.value+class VerifyTree:+ """Shape of one MTP verify step: draft tokens in a tree rooted at the committed next token.++ Built from nested child lists. A node's first child is the MTP argmax, later children its+ runner-ups, and only argmax-chain nodes have children. Rows are numbered depth-first, first+ child first, so the chain is rows 0..chain and no row sits above its depth.+ """++ def __init__(self, spec, capacity, device):+ self.device = device+ self.rows = capacity+ self.mask = self.positions = None+ self.update(spec)++ def update(self, spec):+ # Equal-row topology changes retain the addresses captured by CUDA graphs.+ self.parent, self.children, self.depth = [], [], []++ def add(node, parent):+ r = len(self.parent)+ self.parent.append(parent)+ self.children.append([])+ self.depth.append(0 if parent < 0 else self.depth[parent] + 1)+ if parent >= 0:+ self.children[parent].append(r)+ for child in node:+ add(child, r)++ add(spec, -1)+ count = len(self.parent)+ self.chain = 0 # drafts on the argmax chain+ while self.children[self.chain]:+ self.chain = self.children[self.chain][0]+ self.fanout = max(len(c) for c in self.children)+ self.parent.extend([-1] * (self.rows - count))+ self.children.extend([] for _ in range(self.rows - count))+ self.depth.extend([0] * (self.rows - count))+ # kitchen flash_attention_decode_tree_merge: bit t of row r's mask lets it attend row t+ mask = []+ for r in range(self.rows):+ bits, a = 0, r+ while a >= 0:+ bits |= 1 << a+ a = self.parent[a]+ mask.append(bits)+ mask = torch.tensor(mask, dtype=torch.int32, device=self.device)+ if self.mask is None:+ self.mask = mask+ else:+ self.mask.copy_(mask)+ # kitchen gated_delta_decode_deferred program: a chain row commits (bit 5) after its+ # children's leaves ran against its state without committing+ prog = []+ for r in range(self.chain + 1):+ prog.append(r | (32 if self.children[r] else 0))+ prog.extend(self.children[r][1:])+ sm = comfy_kitchen.gated_delta_slot_max+ self.ctl = torch.tensor(self.parent + [-1] * (sm - self.rows) + prog + list(range(count, sm)), dtype=torch.int32)+ positions = torch.tensor([self.depth], dtype=torch.long, device=self.device)+ if self.positions is None:+ self.positions = positions+ else:+ self.positions.copy_(positions)+++# Verify capacity and topology. A fixed depth is a tree with one child per node.+MTP_MODES = {+ "2": (4, [[[]]]),+ "3": (4, [[[[]]]]),+ "4": (8, [[[[[]]]]]),+ "5": (8, [[[[[[]]]]]]),+ "tree2s1": (8, [[[], []], []]),+ "tree2s2": (8, [[[], [], []], [], []]),+ "tree3s1": (8, [[[[], []], []], []]),+ "tree4s1m8": (8, [[[[[]], []], []], []]),+ "auto": (8, None),+}++# Every positive composition of seven: a primary chain with terminal siblings.+_SHAPES = []+for _cuts in range(64):+ _ends = [0] + [i for i in range(1, 7) if _cuts & (1 << (i - 1))] + [7]+ _SHAPES.append(tuple(b - a for a, b in zip(_ends, _ends[1:])))+++def tree_spec(widths):+ node = []+ for width in reversed(widths):+ node = [node] + [[] for _ in range(width - 1)]+ return node+++def best_widths(widths, hits, draft_ms, verify_ms):+ """Reallocate seven draft rows using conditional rank hits and measured GPU cost."""+ probabilities = []+ prior = [0.125] * 8+ for row in hits:+ count = sum(row)+ prior = [(hit + 4 * p) / (count + 4) for hit, p in zip(row, prior)]+ probabilities.append(prior)++ def score(shape):+ reach, emitted = 1.0, 1.0+ for width, probability in zip(shape, probabilities):+ emitted += reach * sum(probability[:width])+ reach *= probability[0]+ return emitted / (verify_ms + len(shape) * draft_ms)++ best = max(_SHAPES, key=score)+ return best if score(best) > 1.03 * score(widths) else widthsdef _qwen35_layer_types(n):@@ -387,7 +509,7 @@ class GatedAttention(nn.Module):# KV cachepresent_key_value = past_key_value- decode = past_key_value is not None and seq_length <= 6 and attention_mask is None+ decode = past_key_value is not None and seq_length <= SPEC_ROWS and attention_mask is Noneif decode:past_key_value.write(xk, xv)@@ -776,10 +898,9 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):"repetition_penalty": repetition_penalty,"presence_penalty": kwargs.get("presence_penalty", 0.0) or 0.0,"seed": seed if seed is not None else 42}- fixed_depth = None if mtp is True else max(2, min(5, int(mtp)))- return self._generate_mtp(embeds, max_length, stop_tokens, sampling=sampling, fixed_depth=fixed_depth, position_ids=kwargs.get("position_ids"))+ return self._generate_mtp(embeds, max_length, stop_tokens, mtp, sampling=sampling, position_ids=kwargs.get("position_ids"))- def _generate_mtp(self, embeds, max_length, stop_tokens, sampling=None, fixed_depth=None, position_ids=None):+ def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, position_ids=None):device = embeds.devicecfg = self.model.configif stop_tokens is None:@@ -788,8 +909,6 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):embeds = embeds.to(dt)if embeds.ndim == 2:embeds = embeds.unsqueeze(0)- # greedy drafts 3 deep (5 after the probe); sampled stays at 2- depth = fixed_depth if fixed_depth is not None else (3 if sampling is None else 2)cap = embeds.shape[1] + max_length + 7pkv = self.init_kv_cache(embeds.shape[0], cap, device, dt)# repair window: drafting ahead plus a near-full rollback@@ -844,18 +963,25 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):progress.update(n)console.update(n)- verify_buffers = None- drafts = None-- def set_depth(d):- nonlocal depth, verify_buffers, drafts- depth = d- # Saved draft IDs must survive capture of later draft units.- drafts = [torch.empty_like(nt_buf) for _ in range(d)]- # static hidden/rope buffers: graphed layers bake their input addresses- verify_buffers = (torch.empty((embeds.shape[0], d + 1, cfg.hidden_size), device=device, dtype=dt), freqs_at(pos, d + 1).clone())-- set_depth(depth)+ adaptive = mtp is True+ # auto: seven draft rows as a chain of widths, reshaped from rank hits and measured step cost+ widths = (4, 3)+ hits = [[0.0] * 8 for _ in range(7)]+ steps, draft_ms, verify_ms = 0, 0.0, 0.0+ draft_ranks = torch.zeros((7, 7), device=device, dtype=torch.long) if adaptive else None+ timers = [torch.cuda.Event(enable_timing=True) for _ in range(3)] if adaptive and device.type == "cuda" else None+ clocks = [0.0] * 3+ capacity, spec = MTP_MODES["auto" if adaptive else mtp]+ tree = VerifyTree(tree_spec(widths) if adaptive else spec, capacity, device)+ depth = tree.chain+ for kv in pkv:+ if isinstance(kv, LinearKV):+ kv.ctl_tracker["tree"] = tree+ else:+ kv.tracker["tree"] = tree+ verify_tokens = torch.zeros((embeds.shape[0], tree.rows), device=device, dtype=torch.long)+ verify_buffers = (torch.empty((embeds.shape[0], tree.rows, cfg.hidden_size), device=device, dtype=dt), freqs_at(pos, tree.rows).clone())+ draft_top = torch.empty((embeds.shape[0], 7 if adaptive else tree.fanout), device=device, dtype=torch.long)use_graph = self.model.graph_dynamic_vbar_blockscompile_allocations = use_graph and comfy.model_prefetch.malloc_graph_enabled(device)embed = self.model.embed_tokens@@ -865,7 +991,7 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):draft_freqs = freqs_at(pos).clone()def draft(token, hidden, p):- # one drafted token: mtp head + lm_head argmax, graph-replayed when resident+ # one drafted token (and its runner-ups for a tree): mtp head + lm_head, graph-replayed when residentmtp_kv.prepare(1)draft_token.copy_(token)draft_hidden.copy_(hidden)@@ -881,7 +1007,14 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):def core():n1, r1 = self.mtp(draft_emb, draft_hidden, draft_freqs, mtp_kv)draft_hidden.copy_(r1)- draft_token.copy_(self.logits(n1)[:, -1].argmax(dim=-1, keepdim=True))+ logits = self.logits(n1)[:, -1]+ if adaptive:+ draft_top.copy_(logits.topk(7, dim=-1).indices)+ draft_token.copy_(draft_top[:, :1])+ else:+ draft_token.copy_(logits.argmax(dim=-1, keepdim=True))+ if tree.fanout > 1 and not adaptive:+ draft_top.copy_(logits.topk(tree.fanout, dim=-1).indices)# Host rows stay eager; a tied table uses its graph slot for MTP + head.comfy.model_prefetch.prefetch_queue_pop(queue, device, embed, dt, core=embed_core,@@ -890,104 +1023,134 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):enable_graph=use_graph, malloc_scope="mtp")comfy.model_prefetch.prefetch_queue_pop(queue, device, None, malloc_scope="mtp")mtp_kv.advance(1)- return draft_token, draft_hidden+ return draft_token, draft_top, draft_hidden- def verify_sample(lg, drafts):- # accept draft i w.p. p_i(draft), else sample the residual; all depth+1 columns as one batch+ def verify_sample(lg):s = samplingrows = lg[0].float()if penalized:- # committed tokens penalize every column, each draft the columns after it- mask = pen_mask.unsqueeze(0).repeat(depth + 1, 1)- for c, d in enumerate(drafts):- mask[c + 1:].index_fill_(1, d.reshape(-1), True)+ # Each row sees committed history and its own path, never siblings.+ mask = pen_mask.unsqueeze(0).repeat(tree.rows, 1)+ for r in range(1, tree.rows):+ mask[r].copy_(mask[tree.parent[r]])+ mask[r].index_fill_(0, verify_tokens[:, r], True)rows = torch.where(mask, apply_penalty(rows, s["repetition_penalty"], s["presence_penalty"]), rows)- probs, idx = self.processed_probs(rows, s["temperature"], s["top_k"], s["top_p"], s["min_p"], 1.0, [])- dr = torch.cat(drafts, dim=1).reshape(-1, 1)- if idx is None:- p_draft = probs[:depth].gather(1, dr).reshape(-1)- w = probs.clone()- w[:depth].scatter_(1, dr, 0.0)- else:- hit = idx[:depth] == dr- p_draft = (probs[:depth] * hit).sum(1)- w = probs.clone()- w[:depth].masked_fill_(hit, 0.0)- # residual: p with the draft zeroed (the rescue term only matters when discarded anyway)- w = w + (w.sum(1, keepdim=True) == 0) * probs- tok = torch.multinomial(w, num_samples=1, generator=generator)- corr = tok if idx is None else idx.gather(1, tok)- u = torch.rand(depth, device=device, generator=generator)- accepted = (u < p_draft).long().cumprod(0).sum()- return dr, corr, accepted+ return self.sample_token(rows, s["temperature"], s["top_k"], s["top_p"], s["min_p"],+ 1.0, [], generator).transpose(0, 1)def step():# scoped so every temporary dies before the compiler bracket closesnonlocal pos- tok_in, hid_in = nt_buf, h_buf+ if adaptive:+ if timers:+ timers[0].record(torch.cuda.current_stream(device))+ else:+ clocks[0] = time.perf_counter()+ # chain draft k fills row k + 1, its runner-ups that node's sibling rows+ verify_tokens[:, :1].copy_(nt_buf)+ hid_in = h_buffor k in range(depth):- dk, rk = draft(tok_in, hid_in, pos + k)- drafts[k].copy_(dk)- tok_in, hid_in = drafts[k], rk- tokens = torch.cat([nt_buf] + drafts, dim=1)+ dk, top, hid_in = draft(verify_tokens[:, k:k + 1], hid_in, pos + k)+ if adaptive:+ draft_ranks[k].copy_(top[0])+ children = tree.children[k]+ verify_tokens[:, children[0]:children[0] + 1].copy_(dk)+ for j, c in enumerate(children[1:], 1):+ verify_tokens[:, c:c + 1].copy_(top[:, j:j + 1])+ if adaptive:+ if timers:+ timers[1].record(torch.cuda.current_stream(device))+ else:+ clocks[1] = time.perf_counter()embed = self.model.embed_tokensif hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed):- ev = embed.host_rows(tokens, out_dtype=dt)+ ev = embed.host_rows(verify_tokens, out_dtype=dt)else:- ev = embed(tokens).to(dt)- verify_pos = torch.arange(pos + rope_offset, pos + rope_offset + depth + 1, device=device).unsqueeze(0)+ ev = embed(verify_tokens).to(dt)+ verify_pos = tree.positions + pos + rope_offsetx, _, _ = self.model.forward(None, embeds=ev, attention_mask=None, past_key_values=pkv, position_ids=verify_pos, decode_buffers=verify_buffers)# all verify positions in one lm_head GEMV, accept decided GPU-side, one synclg = verify_logits(x)- if sampling is None:- toks = lg.argmax(dim=-1)- vals = torch.cat([toks[0]] + [d[0] for d in drafts]).tolist()- t, dr = vals[:depth + 1], vals[depth + 1:]- accepts = 0- while accepts < depth and t[accepts] == dr[accepts]:- accepts += 1- next_toks = tuple(toks[:, i:i + 1] for i in range(depth + 1))- commit = tuple(dr[:accepts]) + (t[accepts],)- else:- dr, corr, accepted = verify_sample(lg, drafts)- vals = torch.cat([dr[:, 0], corr[:, 0], accepted.reshape(1)]).tolist()- dr, cv, accepts = vals[:depth], vals[depth:2 * depth + 1], vals[-1]- next_toks = tuple(corr[i:i + 1] for i in range(depth + 1))- commit = tuple(dr[:accepts]) + (cv[accepts],)- if accepts < depth:+ toks = lg.argmax(dim=-1) if sampling is None else verify_sample(lg)+ if adaptive:+ if timers:+ timers[2].record(torch.cuda.current_stream(device))+ else:+ clocks[2] = time.perf_counter()+ packet = [toks[0], verify_tokens[0, 1:]]+ if adaptive:+ packet.append(draft_ranks.reshape(-1))+ vals = torch.cat(packet).tolist()+ t, dr = vals[:tree.rows], [None] + vals[tree.rows:2 * tree.rows - 1]+ # Continue only along a child containing the main model's chosen token.+ path = [0]+ while True:+ node = next((c for c in tree.children[path[-1]] if dr[c] == t[path[-1]]), None)+ if node is None:+ break+ path.append(node)+ ranks = []+ if adaptive:+ for node in path:+ if node >= depth:+ break+ candidates = vals[15 + 7 * node:22 + 7 * node]+ ranks.append(next((r for r, token in enumerate(candidates) if token == t[node]), 7))+ if timers:+ clocks[0], clocks[1], clocks[2] = 0.0, timers[0].elapsed_time(timers[1]), timers[0].elapsed_time(timers[2])+ else:+ clocks[:] = [t * 1000 for t in clocks]+ commit = tuple(dr[r] for r in path[1:]) + (t[path[-1]],)+ row = path[-1]+ next_tok = toks[:, row:row + 1]+ if len(path) < tree.rows:+ for kv in pkv:+ kv.rollback(tree.rows - len(path))+ on_chain = row == len(path) - 1+ if not on_chain:for kv in pkv:- kv.rollback(depth - accepts)- if accepts < depth - 1:- mtp_kv.rollback(depth - 1 - accepts) # mtp entries fed by a rejected draft token- nt_buf.copy_(next_toks[accepts])- h_buf.copy_(x[:, accepts:accepts + 1, :])+ kv.commit_path(pos, path)+ # mtp entries fed by a rejected or off-chain draft token+ kept = min(len(path) if on_chain else len(path) - 1, depth)+ if kept < depth:+ mtp_kv.rollback(depth - kept)+ nt_buf.copy_(next_tok)+ h_buf.copy_(x[:, row:row + 1, :])if penalized:- for d in drafts[:accepts]:- pen_mask.index_fill_(0, d.reshape(-1), True)+ for r in path[1:]:+ pen_mask.index_fill_(0, verify_tokens[:, r], True)pen_mask.index_fill_(0, nt_buf.reshape(-1), True)- pos += accepts + 1- return accepts, commit+ pos += len(commit)+ return commit, ranks- probe = None if fixed_depth is not None else [0, 0] # steps, accepted draftstry:while len(ids) < max_length and ids[-1] not in stop_tokens:with (comfy.model_prefetch.malloc_graph_scope(device) if compile_allocations else contextlib.nullcontext()):- accepts, commit = step()+ commit, ranks = step()+ if adaptive:+ steps += 1+ for row in hits:+ for r in range(8):+ row[r] *= 0.984375+ for level, rank in enumerate(ranks):+ hits[level][rank] += 1+ # Do not learn graph capture cost as steady-state execution cost.+ if steps > 8:+ mix = 1.0 if steps == 9 else 0.125+ draft_ms += mix * ((clocks[1] - clocks[0]) / len(widths) - draft_ms)+ verify_ms += mix * (clocks[2] - clocks[1] - verify_ms)+ if steps % 32 == 0:+ best = best_widths(widths, hits, draft_ms, verify_ms)+ if best != widths:+ widths = best+ tree.update(tree_spec(widths))+ depth = tree.chaincommit = list(commit[:max_length - len(ids)])stop = next((i for i, t in enumerate(commit) if t in stop_tokens), None)if stop is not None:del commit[stop + 1:]ids.extend(commit)update_progress(len(commit))- if probe is not None:- probe[0] += 1- probe[1] += accepts- if probe[0] == 32:- # deepen once acceptance sustains it and a recapture round can amortize- if sampling is None and max_length - len(ids) > 512 and 1 + probe[1] / probe[0] >= 2.2:- comfy.model_prefetch.cleanup_prefetch_queues()- set_depth(5)- probe = Nonefinally:console.close()comfy.model_prefetch.cleanup_prefetch_queues()diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.pyindex efd42b98b..0e7d3a9af 100644--- a/comfy_extras/nodes_textgen.py+++ b/comfy_extras/nodes_textgen.py@@ -39,7 +39,7 @@ class TextGenerate(io.ComfyNode):io.DynamicCombo.Input("sampling_mode", options=sampling_options, display_name="Sampling Mode"),io.Boolean.Input("thinking", optional=True, default=False, tooltip="Operate in thinking mode if the model supports it."),io.Boolean.Input("use_default_template", optional=True, default=True, tooltip="Use the built in system prompt/template if the model has one.", advanced=True),- io.Combo.Input("mtp", options=["auto", "off", "2", "3", "4", "5"], default="auto", optional=True, tooltip="Speculative decoding with the checkpoint's multi-token-prediction head. No effect without MTP weights. auto adapts the draft depth; 2-5 pins it. Sampled output stays correctly distributed but differs from non-MTP output for the same seed."),+ io.Combo.Input("mtp", options=["auto", "off", "2", "3", "4", "5", "tree2s1", "tree2s2", "tree3s1", "tree4s1m8"], default="auto", optional=True, tooltip="Speculative decoding with the checkpoint's multi-token-prediction head. No effect without MTP weights. auto adapts the draft depth and breadth to observed token matches and execution cost; 2-5 pins a constant draft depth; tree modes pin both breadth and depth. Sampled output stays correctly distributed but differs from non-MTP output for the same seed."),io.String.Input("system_prompt", force_input=True, optional=True, tooltip="Replaces the system prompt in the model's chat template. Ignored when the default template is not used."),],outputs=[@@ -51,7 +51,7 @@ class TextGenerate(io.ComfyNode):@classmethoddef execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="") -> io.NodeOutput:- mtp = False if mtp == "off" else (True if mtp == "auto" else int(mtp))+ mtp = {"off": False, "auto": True}.get(mtp, mtp)tokens = clip.tokenize(prompt, image=image, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, audio=audio, system_prompt=system_prompt if use_default_template else "")diff --git a/requirements.txt b/requirements.txtindex f46143b8b..0c12dd5ff 100644--- a/requirements.txt+++ b/requirements.txt@@ -21,7 +21,7 @@ alembicSQLAlchemy>=2.0.0filelockav>=17.0.0-comfy-kitchen==0.2.39+comfy-kitchen==0.2.40comfy-aimdo==0.5.6requestssimpleeval>=1.0.0diff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.pyindex e8d55ab9d..efd322865 100644--- a/tests-unit/comfy_test/qwen35_decode_test.py+++ b/tests-unit/comfy_test/qwen35_decode_test.py@@ -176,15 +176,17 @@ def _masked_attention(q, k, v, mask):def test_cache_mask_matches_attend(monkeypatch):- # the additive mask reproduces attend()'s chain staircase over the whole cache+ # the additive mask reproduces attend()'s chain staircase and tree ancestry over the whole cachefrom comfy.text_encoders.llama import FixedKVCache+ tree = _tree2s2()torch.manual_seed(0)shared = FixedKVCache.shared(2, "cpu")kv = FixedKVCache.zeros(2, 1, 24, 64, "cpu", torch.float32, shared)kv.key.normal_()kv.value.normal_()kv.index = 9- for seq in (1, 3):+ for seq, shaped in ((1, None), (3, None), (8, tree)):+ kv.tracker["tree"] = shapedkv.prepare(seq)xq = torch.randn(2, 2, seq, 64)xk = torch.randn(2, 1, seq, 64)@@ -194,7 +196,11 @@ def test_cache_mask_matches_attend(monkeypatch):mask = kv.mask(seq)assert mask.shape == (2, 1, seq, 24) and mask.dtype == torch.float32visible = (mask == 0)[0, 0]- assert visible.sum(-1).tolist() == [10 + j for j in range(seq)]+ if shaped is None:+ assert visible.sum(-1).tolist() == [10 + j for j in range(seq)]+ else:+ assert visible[:, 9:17].int().tolist() == [[(m >> t) & 1 for t in range(8)] for m in tree.mask.tolist()]+ assert visible[:, :9].all() and not visible[:, 17:].any()torch.testing.assert_close(_masked_attention(xq, kv.key, kv.value, mask), expected, rtol=1e-4, atol=1e-5)@@ -216,7 +222,7 @@ def test_deferred_ctl_rows_wait_for_their_previous_copy(monkeypatch):tracker = LinearKV.deferred_tracker("cpu")tracker["copied"] = [Event(0), Event(1)]ctl = tracker["ctl"]- assert ctl.shape == (2,) # {pending, parity}+ assert ctl.shape == (26,) # {pending, parity, slots[8], parents[8], program[8]}layers = [LinearKV(torch.zeros(1), torch.zeros(1), 0, None, None, ctl=ctl, ctl_tracker=tracker) for _ in range(2)]def step(n, discard=0):@@ -237,6 +243,85 @@ def test_deferred_ctl_rows_wait_for_their_previous_copy(monkeypatch):assert step(1) == [3, 0]+def _tree2s2():+ from comfy.text_encoders.qwen35 import MTP_MODES, VerifyTree+ capacity, spec = MTP_MODES["tree2s2"]+ return VerifyTree(spec, capacity, "cpu")+++def test_verify_tree_shapes():+ from comfy.text_encoders.qwen35 import MTP_MODES, VerifyTree+ tree = _tree2s2()+ assert (tree.rows, tree.chain, tree.fanout) == (8, 2, 3)+ assert tree.parent == [-1, 0, 1, 1, 1, 0, 0, -1] and tree.depth == [0, 1, 2, 2, 2, 1, 1, 0]+ assert tree.mask.tolist() == [1, 3, 7, 11, 19, 33, 65, 128]+ assert tree.ctl.tolist() == [-1, 0, 1, 1, 1, 0, 0, -1, 32, 5, 6, 33, 3, 4, 2, 7]+ tree = VerifyTree(MTP_MODES["tree4s1m8"][1], 8, "cpu")+ assert (tree.rows, tree.chain, tree.fanout) == (8, 4, 2)+ assert tree.parent == [-1, 0, 1, 2, 3, 2, 1, 0] and tree.depth == [0, 1, 2, 3, 4, 3, 2, 1]+ assert tree.mask.tolist() == [1, 3, 7, 15, 31, 39, 67, 129]+ assert tree.ctl.tolist() == [-1, 0, 1, 2, 3, 2, 1, 0, 32, 7, 33, 6, 34, 5, 35, 4]+ mask_ptr, pos_ptr = tree.mask.data_ptr(), tree.positions.data_ptr()+ tree.update([[[[[[[[]]]]]]]])+ assert tree.mask.tolist() == [1, 3, 7, 15, 31, 63, 127, 255]+ assert tree.positions.tolist() == [list(range(8))]+ tree.update(MTP_MODES["tree4s1m8"][1])+ assert tree.mask.data_ptr() == mask_ptr and tree.positions.data_ptr() == pos_ptr+ assert tree.mask.tolist() == [1, 3, 7, 15, 31, 39, 67, 129]+ assert tree.positions.tolist() == [[0, 1, 2, 3, 4, 3, 2, 1]]+ for name, depth in (("tree2s1", 2), ("tree3s1", 3)):+ tree = VerifyTree(MTP_MODES[name][1], MTP_MODES[name][0], "cpu")+ assert (tree.rows, tree.chain, tree.fanout) == (8, depth, 2)+ chain = VerifyTree(MTP_MODES["3"][1], 4, "cpu")+ assert (chain.rows, chain.chain, chain.fanout, chain.children[:3]) == (4, 3, 1, [[1], [2], [3]])+ assert chain.mask.tolist() == [1, 3, 7, 15]+ for depth, capacity in ((2, 4), (4, 8), (5, 8)):+ tree = VerifyTree(MTP_MODES[str(depth)][1], capacity, "cpu")+ assert tree.chain == depth and tree.rows == capacity+ assert tree.parent[depth + 1:] == [-1] * (capacity - depth - 1)+ assert tree.ctl[8 + depth + 1:].tolist() == list(range(depth + 1, 8))+ assert all(not children for children in tree.children[depth:])+++def test_tree_verify_cache_prefix_and_commit_path():+ from comfy.text_encoders.llama import FixedKVCache+ tree = _tree2s2()+ shared = FixedKVCache.shared(1, "cpu")+ kv = FixedKVCache.zeros(1, 1, 32, 256, "cpu", torch.bfloat16, shared)+ kv.index = 10+ kv.tracker["tree"] = tree+ kv.prepare(8)+ assert kv.seqlen.tolist() == [10]+ kv.tracker["tree"] = None+ kv.prepare(8)+ assert kv.seqlen.tolist() == [18]+ for r in range(7):+ kv.key[:, :, 10 + r] = r+ kv.value[:, :, 10 + r] = -r+ kv.commit_path(10, [0, 1, 4])+ assert kv.key[0, 0, 10:13, 0].tolist() == [0, 1, 4] and kv.value[0, 0, 12, 0].item() == -4+++def test_deferred_ctl_carries_accepted_path_and_tree_program():+ from comfy.text_encoders.qwen35 import LinearKV+ tree = _tree2s2()+ tracker = LinearKV.deferred_tracker("cpu")+ tracker["tree"] = tree+ ctl = tracker["ctl"]+ kv = LinearKV(torch.zeros(1), torch.zeros(1), 5, None, None, ctl=ctl, ctl_tracker=tracker)+ kv.prepare(8)+ assert ctl[2:10].tolist() == list(range(8)) and ctl[10:].tolist() == tree.ctl.tolist()+ kv.advance(8)+ kv.rollback(5)+ kv.commit_path(5, [0, 1, 4])+ kv.prepare(8)+ assert ctl[0].item() == 3 and ctl[2:10].tolist() == [0, 1, 4, 3, 4, 5, 6, 7]+ tracker["tree"] = None+ kv.advance(8)+ kv.prepare(1)+ assert ctl[2:].tolist() == tracker["single"].tolist()++def _tiny_model(monkeypatch, layers=4, head_dim=64):import comfy.text_encoders.qwen35 as qwen35monkeypatch.setitem(qwen35.QWEN35_MODELS, "tiny", dict(vision=dict(hidden_size=32, num_heads=2, intermediate_size=64, depth=1)))@@ -257,7 +342,7 @@ def _tiny_model(monkeypatch, layers=4, head_dim=64):@pytest.mark.parametrize("image_prompt", [False, True])def test_greedy_mtp_matches_plain_greedy(monkeypatch, image_prompt):- # auto and fixed-depth chain verify through the kitchen torch paths: causal GQA, deferred DeltaNet+ # chain and tree verify through the kitchen torch paths: GQA prefix + tree merge, deferred DeltaNetfrom comfy.text_encoders.llama import FixedKVCachefrom comfy.text_encoders.qwen_vl import qwen2vl_mrope_position_idsmodel = _tiny_model(monkeypatch)@@ -270,12 +355,61 @@ def test_greedy_mtp_matches_plain_greedy(monkeypatch, image_prompt):if image_prompt:assert positions.shape == (3, 24) and positions[:, -1].tolist() == [11, 11, 11]plain = model.generate(embeds.clone(), do_sample=False, max_length=40, mtp=False, position_ids=positions)- for mtp in (True, 3, 5):+ for mtp in (True, "2", "3", "4", "5", "tree2s1", "tree2s2", "tree3s1", "tree4s1m8"):discards.clear()assert model.generate(embeds.clone(), do_sample=False, max_length=40, mtp=mtp, position_ids=positions) == plainassert discards and min(discards) < max(discards) # partial acceptance happened[email protected]("quantile", [0.42, 0.8, 0.92])[email protected]("top_k", [0, 7])+def test_sampled_tree_matches_sequential_with_same_draws(monkeypatch, quantile, top_k):+ # Couple draws by quantile rather than RNG consumption: unused tree rows also sample.+ def draw(weights, num_samples, generator=None):+ assert num_samples == 1+ cdf = weights.cumsum(-1) / weights.sum(-1, keepdim=True)+ return (cdf < quantile).sum(-1, keepdim=True)++ monkeypatch.setattr(torch, "multinomial", draw)+ model = _tiny_model(monkeypatch)+ embeds = torch.randn(1, 5, 64)+ settings = dict(do_sample=True, max_length=65, temperature=0.7, top_k=top_k, top_p=0.95,+ min_p=0.05, repetition_penalty=1.2, presence_penalty=0.8)+ expected = model.generate(embeds.clone(), mtp=False, **settings)+ for preset in (True, "2", "3", "4", "5", "tree2s1", "tree2s2", "tree3s1", "tree4s1m8"):+ assert model.generate(embeds.clone(), mtp=preset, **settings) == expected, preset+ stop = expected[4]+ stopped = expected[:expected.index(stop) + 1]+ assert model.generate(embeds.clone(), mtp=preset, stop_tokens=[stop], **settings) == stopped++[email protected]("sampled", [False, True])+def test_adaptive_switch_preserves_sequential_output(monkeypatch, sampled):+ import comfy.text_encoders.qwen35 as qwen35++ def draw(weights, num_samples, generator=None):+ cdf = weights.cumsum(-1) / weights.sum(-1, keepdim=True)+ return (cdf < 0.8).sum(-1, keepdim=True)++ switches = []++ def best_widths(widths, hits, draft_ms, verify_ms):+ # every 32nd step: cycle through shapes that differ in depth and fanout+ assert len(hits) == 7 and all(h >= 0 for row in hits for h in row)+ switches.append(((1,) * 7, (2, 1, 4), (7,), (4, 3))[len(switches) % 4])+ return switches[-1]++ monkeypatch.setattr(torch, "multinomial", draw)+ monkeypatch.setattr(qwen35, "best_widths", best_widths)+ model = _tiny_model(monkeypatch)+ embeds = torch.randn(1, 5, 64)+ settings = dict(do_sample=sampled, max_length=400, temperature=0.7, top_k=7,+ top_p=0.95, min_p=0.05, repetition_penalty=1.2, presence_penalty=0.8)+ expected = model.generate(embeds.clone(), mtp=False, **settings)+ assert model.generate(embeds.clone(), mtp=True, **settings) == expected+ assert len(set(switches)) == 4, switches++class _FakePacked:# Int8DecodeCache boundary with 4-row pages: records the BF16 refreshes it is asked fordef __init__(self, batch, heads, capacity, device):@@ -296,11 +430,11 @@ def _int8_cache(monkeypatch, capacity=32):_fake_kitchen(monkeypatch)merges = []- def step_merge(out, lse, q, k, v, result):- merges.append((k, v))+ def tree_merge(out, lse, q, k, v, mask, result):+ merges.append((k, v, mask.clone()))return outmonkeypatch.setattr(llama, "Int8DecodeCache", _FakePacked, raising=False)- monkeypatch.setattr(llama.comfy_kitchen, "flash_attention_decode_step_merge", step_merge)+ monkeypatch.setattr(llama.comfy_kitchen, "flash_attention_decode_tree_merge", tree_merge)shared = FixedKVCache.shared(1, "cpu")return llama.Int8FixedKVCache.zeros(1, 1, capacity, 256, "cpu", torch.bfloat16, shared), merges@@ -319,8 +453,8 @@ def test_int8_cache_merges_current_chain_rows(monkeypatch):assert kv.attend(xq, xk, xv).shape == (1, 3, 512)# the committed prefix is quantized once, then refreshed to the same length before attendingassert kv.packed.updates == [([10], True), ([10], False)]- k, v = merges[-1]- assert k is xk and v is xv+ k, v, mask = merges[-1]+ assert k is xk and v is xv and mask.tolist() == [1, 3, 7]def test_int8_cache_keeps_bounded_circular_window(monkeypatch):@@ -346,6 +480,27 @@ def test_int8_cache_keeps_bounded_circular_window(monkeypatch):assert torch.equal(kv.key[:, :, 2:5], xk) and kv.packed.updates[-1] == ([14], False)[email protected]("path", [[0, 1, 2, 3, 4], [0, 1, 2, 5], [0, 7]])+def test_tree4s1m8_cache_commits_deep_and_sibling_paths(monkeypatch, path):+ from comfy.text_encoders.qwen35 import MTP_MODES, VerifyTree+ tree = VerifyTree(MTP_MODES["tree4s1m8"][1], 8, "cpu")+ kv, merges = _int8_cache(monkeypatch)+ kv.index = 10+ kv.tracker["tree"] = tree+ kv.prepare(8)+ xk = torch.randn(1, 1, 8, 256, dtype=torch.bfloat16)+ kv.write(xk, -xk)+ kv.attend(torch.randn(1, 2, 8, 256, dtype=torch.bfloat16), xk, -xk)+ assert merges[-1][2].tolist() == [1, 3, 7, 15, 31, 39, 67, 129]+ kv.advance(8)+ kv.rollback(8 - len(path))+ kv.commit_path(10, path)+ assert kv.index == 10 + len(path)+ for token, row in enumerate(path, 10):+ assert torch.equal(kv.key[:, :, token % 12], xk[:, :, row])+ assert torch.equal(kv.value[:, :, token % 12], -xk[:, :, row])++def _attention_config(config):return torch.tensor(list(json.dumps(config).encode()), dtype=torch.uint8)@@ -405,8 +560,8 @@ def test_checkpoint_and_lora_attention_select_each_kv_layer(monkeypatch):assert cache_types() == (FixedKVCache, FixedKVCache, FixedKVCache)[email protected]("seq", [1, 3])-def test_configured_decode_preserves_causal_visibility(monkeypatch, seq):[email protected]("seq,tree_verify", [(1, False), (3, False), (8, True)])+def test_configured_decode_preserves_causal_and_tree_visibility(monkeypatch, seq, tree_verify):import comfy.ldm.modules.attention as attention# Make the configured backend observably different from the default backend.@attention.wrap_attn@@ -423,13 +578,25 @@ def test_configured_decode_preserves_causal_visibility(monkeypatch, seq):kv.prepare(9)attn(prefix, attention_mask=torch.full((9, 9), -torch.inf).triu(1),freqs_cis=torch.zeros(1, 1, 9, 32, 2, 2), optimized_attention=attention.attention_pytorch, past_key_value=kv)- # Repeat after rollback; stale rows past the current chain must not become visible.+ tree = _tree2s2() if tree_verify else None+ kv.tracker["tree"] = tree+ # Repeat after rollback; stale tree rows must not become visible.for start in (9, 4):kv.index = startkv.prepare(seq)out, _ = attn(current, freqs_cis=torch.zeros(1, 1, seq, 32, 2, 2), past_key_value=kv)full = torch.cat((prefix[:, :start], current), dim=1)mask = torch.full((start + seq, start + seq), -torch.inf).triu(1)+ parents = [-1, 0, 1, 1, 1, 0, 0, -1]+ for row in range(seq):+ ancestors = []+ r = row+ while r >= 0:+ ancestors.append(start + r)+ r = parents[r] if tree_verify else r - 1+ visible = list(range(start)) + sorted(ancestors)+ mask[start + row] = -torch.inf+ mask[start + row, visible] = 0expected, _ = attn(full, attention_mask=mask, freqs_cis=torch.zeros(1, 1, start + seq, 32, 2, 2),optimized_attention=attention.attention_pytorch)torch.testing.assert_close(out, expected[:, start:])