The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Pause comfy compiler for long lived sparse allocations (#16148)
rattusa99d1f9c14f4
diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.pyindex 22a91cd6a..35be626ae 100644--- a/comfy/model_prefetch.py+++ b/comfy/model_prefetch.py@@ -1,3 +1,4 @@+import contextlibimport loggingimport threadingimport warnings@@ -28,6 +29,17 @@ def _malloc_graph_break():def malloc_graph_enabled(device):return not args.disable_comfy_compiler and comfy.memory_management.aimdo_enabled and comfy.model_management.is_device_cuda(device)+def pause_malloc_graph(sync=False):+ graph = ACTIVE_MALLOC_GRAPHS.get(threading.get_ident())+ if graph is not None:+ graph.pause(sync=sync)+ try:+ yield+ finally:+ if graph is not None:+ graph.resume(sync=sync)+def malloc_graph_begin(module, device):global MALLOC_GRAPH_USEDif not malloc_graph_enabled(device):diff --git a/comfy_extras/nodes_sparse_attention.py b/comfy_extras/nodes_sparse_attention.pyindex 38ab69703..a807a40f9 100644--- a/comfy_extras/nodes_sparse_attention.py+++ b/comfy_extras/nodes_sparse_attention.py@@ -10,6 +10,7 @@ import comfy_kitchen as ckimport torchimport comfy.model_management+import comfy.model_prefetchimport comfy.patcher_extensionfrom comfy.ldm.minimax.model import MiniMaxH3Modelfrom comfy_api.latest import ComfyExtension, io@@ -252,9 +253,22 @@ def h3_sparse_attention(attn, x, rope_freqs, transformer_options, patch: SparseAkw = comfy.model_management.cast_to(attn.k_norm.weight, device=x.device)extra, plan, gate = {}, None, Nonen, freqs = n_tokens, rope_freqs+ with comfy.model_prefetch.pause_malloc_graph():+ if patch.vsa:+ plan = patch.vsa_plan(transformer_options["minimax_h3_layout"], x.device)+ n = plan["n"]++ key = (block_index, n, tuple(transformer_options.get("uuids", ()))) # statistics per conditioning branch+ pooled = patch.pooled.get(key)+ first = pooled is None+ if first:+ pooled = (+ torch.empty((heads, head_dim), dtype=torch.float32, device=x.device),+ torch.empty((heads, head_dim), dtype=torch.float32, device=x.device),+ )+if patch.vsa:- plan = patch.vsa_plan(transformer_options["minimax_h3_layout"], x.device)- n, freqs = plan["n"], patch.vsa_rope_freqs(rope_freqs, plan)+ freqs = patch.vsa_rope_freqs(rope_freqs, plan)sink = sink_q = (0, plan["n_prefix"])extra = {"tail": False, "block_len": plan["block_len"]}gate = attn.to_gate_compress@@ -274,16 +288,16 @@ def h3_sparse_attention(attn, x, rope_freqs, transformer_options, patch: SparseAextra["coarse_gate"].view(n, heads * head_dim)[i:i + xc.shape[0]] = gate(xc)yield attn.qkv_proj(xc)- key = (block_index, n, tuple(transformer_options.get("uuids", ()))) # statistics per conditioning branch- pooled = patch.pooled.get(key)out, kmean, vscale = ck.sol_attn_chunked(chunks, n, heads, freqs, (qw, kw),- kmean=None if pooled is None else pooled[0],- vscale=None if pooled is None else pooled[1],+ kmean=None if first else pooled[0],+ vscale=None if first else pooled[1],tau=patch.tau, topk_ratio=patch.topk_ratio, token_aug=patch.extra_tokens,sink_blocks=list(sink), sink_q=list(sink_q),rope_eps=attn.q_norm.eps, **extra)- patch.pooled[key] = (kmean, vscale)+ pooled[0].copy_(kmean)+ pooled[1].copy_(vscale)+ patch.pooled[key] = pooledmode = f"VSA tiles ({n} padded rows, {sink[1]} prefix tiles)" if plan is not None else f"sinks {sink}/{sink_q}"patch.log_once(("producer", n), f"sparse producer path: {n_tokens} tokens, {mode}")out = out.view(n, heads * head_dim)