The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Fix Qwen3.5 MTP embedding offload (#16877)
rattus46493d824fd7
diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.pyindex 292038a3d..f075d1298 100644--- a/comfy/text_encoders/qwen35.py+++ b/comfy/text_encoders/qwen35.py@@ -5,7 +5,6 @@ from dataclasses import dataclass, fieldfrom tqdm import tqdmimport contextlibimport os-import warningsimport comfy.model_managementimport comfy.model_prefetch@@ -732,10 +731,11 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):# head and MTP draft weights run every decode step: fault them resident ahead of the body.# a separate embedding goes last: evicted, its rows are gathered from the host copy instead.# a tied table is the head, so it stays first.- hot = [] if self.mtp is None else [self.mtp]+ head = self.model.lm_head if hasattr(self.model, "lm_head") else self.model.embed_tokens+ hot = head if self.mtp is None else (head, self.mtp)if hasattr(self.model, "lm_head"):- return [self.model.lm_head, *hot, *units], [*last_units, self.model.embed_tokens]- return [self.model.embed_tokens, *hot, *units], last_units+ return [hot, *units], [*last_units, self.model.embed_tokens]+ return [hot, *units], last_unitsdef preprocess_embed(self, embed, device):if embed["type"] == "image":@@ -787,10 +787,6 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):with comfy.ops.CastBiasWeightContext(head, x, offloadable=True) as (w, _bias):return F.linear(x, w)- # the draft graph bakes these weights' addresses: keep them resident for the generate- hot = list({id(m): m for m in [head, self.model.embed_tokens, *self.mtp.modules()]}.values())- pinned = [m for m in hot if hasattr(m, "_v")]-generator = Noneif sampling is not None:generator = torch.Generator(device=device).manual_seed(sampling["seed"])@@ -815,6 +811,10 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):pen_mask = torch.zeros((lg0.shape[-1],), device=device, dtype=torch.bool) if penalized else Noneh_buf.copy_(x[:, -1:, :])del x, lg0+ if hasattr(self.model.embed_tokens, "_v"):+ vbar = self.model.embed_tokens._v[0]+ comfy.model_management.reset_cast_buffers()+ vbar.set_watermark(vbar.max_size)ids = [nt_buf[0].item()]if penalized:pen_mask.index_fill_(0, nt_buf.reshape(-1), True)@@ -827,12 +827,15 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):console.update(n)verify_buffers = None+ drafts = Nonesnapshot_bytes = sum(kv.recurrent_state.numel() * 4 + kv.conv_state.numel() * kv.conv_state.element_size()for kv in pkv if isinstance(kv, LinearKV))def set_depth(d):- nonlocal depth, verify_buffers+ nonlocal depth, verify_buffers, draftsdepth = d+ # Saved draft IDs must survive capture of later draft units.+ drafts = [torch.empty_like(nt_buf) for _ in range(d)]for kv in pkv:if isinstance(kv, LinearKV):# snapshot views share one backing slab so the fused kernel can write them@@ -843,63 +846,41 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):verify_buffers = (torch.empty((embeds.shape[0], d + 1, cfg.hidden_size), device=device, dtype=dt), freqs_at(pos, d + 1).clone())set_depth(depth)- use_graph = (device.type == "cuda"- and comfy.model_management.NUM_STREAMS > 0- and not comfy.model_management.args.disable_cuda_graphs)- compile_allocations = use_graph and self.model.graph_dynamic_vbar_blocks and comfy.model_prefetch.malloc_graph_enabled(device)- draft_state = {}-- def drop_draft_graph():- # free inside torch API calls so the allocator's benign notices stay catchable- with warnings.catch_warnings():- warnings.simplefilter("ignore")- for t in (draft_state.get("d"), draft_state.get("r"), *draft_state.get("keep", ())):- if t is not None:- t.set_()- g = draft_state.pop("graph", None)- if g is not None:- g.reset()- draft_state.clear()-- def draft_capture():- # captured outside the compiler bracket; its static buffers live for the generate- ds = draft_state- mtp_kv.prepare(1)- ds["tok"] = nt_buf.clone()- ds["hid"] = h_buf.clone()- ds["f"] = freqs_at(pos).clone()- side = torch.cuda.Stream()- side.wait_stream(torch.cuda.current_stream())- with torch.cuda.stream(side):- for _ in range(2):- n1, r1 = self.mtp(self.model.embed_tokens(ds["tok"]).to(dt), ds["hid"], ds["f"], mtp_kv)- self.logits(n1)[:, -1].argmax(dim=-1, keepdim=True)- torch.cuda.current_stream().wait_stream(side)- del n1, r1 # freed before the capture, not shadowed inside it- g = torch.cuda.CUDAGraph()- with torch.cuda.graph(g, capture_error_mode="thread_local"):- n1, r1 = self.mtp(self.model.embed_tokens(ds["tok"]).to(dt), ds["hid"], ds["f"], mtp_kv)- lg1 = self.logits(n1)- ds["d"] = lg1[:, -1].argmax(dim=-1, keepdim=True)- ds["r"] = r1- ds["keep"] = (n1, lg1) # captured allocations must outlive the graph- ds["graph"] = g+ use_graph = self.model.graph_dynamic_vbar_blocks+ compile_allocations = use_graph and comfy.model_prefetch.malloc_graph_enabled(device)+ embed = self.model.embed_tokens+ draft_token = torch.empty_like(nt_buf)+ draft_emb = torch.empty_like(h_buf)+ draft_hidden = torch.empty_like(h_buf)+ draft_freqs = freqs_at(pos).clone()def draft(token, hidden, p):- # one drafted token: mtp head + lm_head argmax, graph-replayed on cuda+ # one drafted token: mtp head + lm_head argmax, graph-replayed when residentmtp_kv.prepare(1)- f = freqs_at(p)- if not use_graph:- n1, r1 = self.mtp(self.model.embed_tokens(token).to(dt), hidden, f, mtp_kv)- mtp_kv.advance(1)- return self.logits(n1)[:, -1].argmax(dim=-1, keepdim=True), r1- ds = draft_state- ds["tok"].copy_(token)- ds["hid"].copy_(hidden)- ds["f"].copy_(f)- ds["graph"].replay()+ draft_token.copy_(token)+ draft_hidden.copy_(hidden)+ draft_freqs.copy_(freqs_at(p))+ host_rows = hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed)+ queue = comfy.model_prefetch.make_prefetch_queue(+ [None if host_rows else embed, (head, self.mtp)], device,+ {"prefetch_dynamic_vbars": self.model.prefetch_dynamic_vbars})++ def embed_core():+ draft_emb.copy_(embed.host_rows(draft_token, out_dtype=dt) if host_rows else embed(draft_token).to(dt))++ def core():+ n1, r1 = self.mtp(draft_emb, draft_hidden, draft_freqs, mtp_kv)+ draft_hidden.copy_(r1)+ draft_token.copy_(self.logits(n1)[:, -1].argmax(dim=-1, keepdim=True))++ # Host rows stay eager; a tied table uses its graph slot for MTP + head.+ comfy.model_prefetch.prefetch_queue_pop(queue, device, embed, dt, core=embed_core,+ enable_graph=use_graph and not host_rows and embed is not head, malloc_scope="mtp")+ comfy.model_prefetch.prefetch_queue_pop(queue, device, head, dt, core=core,+ enable_graph=use_graph, malloc_scope="mtp")+ comfy.model_prefetch.prefetch_queue_pop(queue, device, None, malloc_scope="mtp")mtp_kv.advance(1)- return ds["d"], ds["r"]+ return draft_token, draft_hiddendef verify_sample(lg, drafts):# accept draft i w.p. p_i(draft), else sample the residual; all depth+1 columns as one batch@@ -933,15 +914,17 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):def step():# scoped so every temporary dies before the compiler bracket closesnonlocal pos- drafts = []tok_in, hid_in = nt_buf, h_buffor k in range(depth):dk, rk = draft(tok_in, hid_in, pos + k)- if k < depth - 1:- dk = dk.clone() # later replays overwrite the static output- drafts.append(dk)- tok_in, hid_in = dk, rk- ev = self.model.embed_tokens(torch.cat([nt_buf] + drafts, dim=1)).to(dt)+ drafts[k].copy_(dk)+ tok_in, hid_in = drafts[k], rk+ tokens = torch.cat([nt_buf] + drafts, dim=1)+ embed = self.model.embed_tokens+ if hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed):+ ev = embed.host_rows(tokens, out_dtype=dt)+ else:+ ev = embed(tokens).to(dt)x, _, _ = self.model.forward(None, embeds=ev, attention_mask=None, past_key_values=pkv, decode_buffers=verify_buffers)# all verify positions in one lm_head GEMV, accept decided GPU-side, one synclg = verify_logits(x)@@ -976,10 +959,6 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):probe = None if fixed_depth is not None else [0, 0] # steps, accepted draftstry:- if pinned:- comfy.model_prefetch.pin_modules(pinned, device, dt)- if use_graph and len(ids) < max_length and ids[-1] not in stop_tokens:- draft_capture()while len(ids) < max_length and ids[-1] not in stop_tokens:with (comfy.model_prefetch.malloc_graph_scope(device) if compile_allocations else contextlib.nullcontext()):accepts, commit = step()@@ -997,16 +976,11 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):if (sampling is None and max_length - len(ids) > 512 and 1 + probe[1] / probe[0] >= 2.2and 5 * snapshot_bytes < comfy.model_management.get_free_memory(device)):comfy.model_prefetch.cleanup_prefetch_queues()- drop_draft_graph()set_depth(5)- if use_graph:- draft_capture()probe = Nonefinally:console.close()- drop_draft_graph()- if pinned:- comfy.model_prefetch.cleanup_prefetched_modules(None, pinned)+ comfy.model_prefetch.cleanup_prefetch_queues()return idsdef init_kv_cache(self, batch, max_cache_len, device, execution_dtype):