The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Speedup Gemma4 text generation (CORE-371) (#15054)
Jukka Seppänen7fe8a6138504
diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.pyindex 7aedab530..bdde5137a 100644--- a/comfy/model_prefetch.py+++ b/comfy/model_prefetch.py@@ -1,4 +1,5 @@import torch+import warningsimport weakrefimport comfy_aimdo.model_vbar@@ -28,6 +29,18 @@ def cleanup_prefetched_modules(module, comfy_modules):comfy_aimdo.model_vbar.vbar_unpin(module._v_block)del module._v_block_faulted+def _drop_graph(module):+ graph = getattr(module, "_comfy_graph", None)+ if graph is None:+ return+ # reset() through the bound method surfaces the allocator's benign+ # "uncaptured free of a captured allocation" as catchable Python warnings;+ # a plain del frees from the C++ dealloc path and spams stderr instead+ with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ graph["graph"].reset()+ del module._comfy_graph+def cleanup_prefetch_queues():global PREFETCH_QUEUES, GRAPH_CAPTURE_STREAMS@@ -41,7 +54,7 @@ def cleanup_prefetch_queues():cleanup_prefetched_modules(prefetched_module, comfy_modules)PREFETCH_QUEUES = []for module in GRAPH_MODULES:- del module._comfy_graph+ _drop_graph(module)GRAPH_MODULES.clear()GRAPH_WARMED_MODULES.clear()GRAPH_CAPTURE_STREAMS = {}@@ -117,6 +130,7 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grapif signature is not None:module._v_block_faulted = Trueif signature is not None:+ _drop_graph(module)graph = torch.cuda.CUDAGraph()if generator is not None:graph.register_generator_state(generator)diff --git a/comfy/ops.py b/comfy/ops.pyindex 73ae46674..ff64aad59 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -1639,7 +1639,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precself.norm_type, self.scale_grad_by_freq, self.sparse)target_dtype = out_dtype if out_dtype is not None else weight._params.orig_dtypex = x.to(dtype=target_dtype)- if scale is not None and scale != 1.0:+ if scale is not None:x = x * scale.to(dtype=target_dtype)return xdiff --git a/comfy/text_encoders/gemma4.py b/comfy/text_encoders/gemma4.pyindex 606f8993e..61bc3a3f0 100644--- a/comfy/text_encoders/gemma4.py+++ b/comfy/text_encoders/gemma4.py@@ -9,10 +9,12 @@ import mathfrom comfy import sd1_clipimport comfy.model_management+import comfy.model_prefetchimport comfy.ops+import comfy.quant_opsfrom comfy.ldm.modules.attention import optimized_attention_for_devicefrom comfy.rmsnorm import rms_norm-from comfy.text_encoders.llama import RMSNorm, MLP, BaseLlama, BaseGenerate, _make_scaled_embedding+from comfy.text_encoders.llama import RMSNorm, MLP, BaseLlama, BaseGenerate, FixedKV, _make_scaled_embedding# Intentional minor divergences from transformers -reference implementation:@@ -109,7 +111,28 @@ class Gemma4_12B_Config(Gemma4Config):suppress_tokens = [258883, 258882]-# unfused RoPE as addcmul_ RoPE diverges from reference code+class RingKV(FixedKV):+ # sliding-window ring: writes wrap at capacity, validity saturates+ def prepare(self, num_tokens):+ capacity = self.key.shape[2]+ self.position.fill_(self.index % capacity)+ self.seqlen.fill_(min(self.index + num_tokens, capacity))+++def _fixed_kv_decode_mask(mask, cache, min_val):+ capacity = cache.key.shape[2]+ valid = min(cache.index + 1, capacity)+ output = mask.new_full((*mask.shape[:-1], capacity), min_val)+ if isinstance(cache, RingKV):+ positions = torch.arange(cache.index + 1 - valid, cache.index + 1, device=mask.device) % capacity+ output.index_copy_(-1, positions, mask[..., -valid:])+ else:+ output[..., :valid] = mask[..., :valid]+ return output+++# unfused RoPE as addcmul_ RoPE diverges from reference code (vision only; text+# layers use the kitchen split-half kernel, bitwise-equal to this with bf16 freqs)def _apply_rotary_pos_emb(x, freqs_cis):cos, sin = freqs_cis[0], freqs_cis[1]half = x.shape[-1] // 2@@ -140,6 +163,23 @@ class Gemma4Attention(nn.Module):if config.k_norm == "gemma3":self.k_norm = RMSNorm(head_dim, eps=config.rms_norm_eps, device=device, dtype=dtype)+ def _decode_attention(self, xq, cache, bias):+ if bias is None:+ # eager decode: slice the cache to the valid length (python-side index,+ # no mask needed; a full ring is order-invariant under softmax)+ n = min(cache.index + 1, cache.key.shape[2])+ gqa_kwargs = {"enable_gqa": True} if self.num_heads != self.num_kv_heads else {}+ attention = optimized_attention_for_device(xq.device, mask=False, small_input=True)+ return attention(xq, cache.key[:, :, :n], cache.value[:, :, :n], self.num_heads, skip_reshape=True, scale=1.0, **gqa_kwargs)+ # graph capture: fixed-length masked attention over the full capacity, explicit+ # math (SDPA leaves its fast path on broadcast-bias + GQA and costs ~0.5ms/layer)+ batch_size = xq.shape[0]+ groups = self.num_heads // self.num_kv_heads+ q = xq.reshape(batch_size, self.num_kv_heads, groups, self.head_dim)+ scores = q @ cache.key.transpose(-1, -2) + bias+ probs = torch.softmax(scores.float(), dim=-1).to(xq.dtype)+ return (probs @ cache.value).reshape(batch_size, 1, self.inner_size)+def forward(self,hidden_states: torch.Tensor,@@ -156,10 +196,16 @@ class Gemma4Attention(nn.Module):if self.q_norm is not None:xq = self.q_norm(xq)+ if isinstance(shared_kv, FixedKV):+ # decode on a KV-shared layer: attend the source layer's fixed cache+ xq = comfy.quant_ops.ck.apply_rope_split_half1(xq, freqs_cis)+ output = self._decode_attention(xq, shared_kv, attention_mask)+ return self.o_proj(output), None, None+if shared_kv is not None:xk, xv = shared_kv# Apply RoPE to Q only (K already has RoPE from source layer)- xq = _apply_rotary_pos_emb(xq, freqs_cis)+ xq = comfy.quant_ops.ck.apply_rope_split_half1(xq, freqs_cis)present_key_value = Noneshareable_kv = Noneelse:@@ -173,11 +219,39 @@ class Gemma4Attention(nn.Module):xv = rms_norm(xv)xk = xk.transpose(1, 2)xv = xv.transpose(1, 2)- xq = _apply_rotary_pos_emb(xq, freqs_cis)- xk = _apply_rotary_pos_emb(xk, freqs_cis)+ xq = comfy.quant_ops.ck.apply_rope_split_half1(xq, freqs_cis)+ xk = comfy.quant_ops.ck.apply_rope_split_half1(xk, freqs_cis)present_key_value = None- if past_key_value is not None:+ fixed_cache = past_key_value if isinstance(past_key_value, FixedKV) else None+ if fixed_cache is not None:+ if seq_length == 1:+ # CUDA-graphable decode: write at the device-side ring/linear position+ fixed_cache.key.index_copy_(2, fixed_cache.position, xk)+ fixed_cache.value.index_copy_(2, fixed_cache.position, xv)+ output = self._decode_attention(xq, fixed_cache, attention_mask)+ return self.o_proj(output), fixed_cache, None++ # prefill: attend the local sequence, persist the tail into the cache+ capacity = fixed_cache.key.shape[2]+ index = fixed_cache.index+ if index + seq_length <= capacity:+ fixed_cache.key[:, :, index:index + seq_length] = xk+ fixed_cache.value[:, :, index:index + seq_length] = xv+ if index > 0:+ xk = fixed_cache.key[:, :, :index + seq_length]+ xv = fixed_cache.value[:, :, :index + seq_length]+ elif index == 0:+ # prefill longer than the sliding ring: attend the full local K/V+ # (per-query windows come from the prefill sliding mask), cache only+ # the last `capacity` keys at their wrapped slots (position % capacity)+ slots = torch.arange(seq_length - capacity, seq_length, device=xk.device) % capacity+ fixed_cache.key.index_copy_(2, slots, xk[:, :, -capacity:])+ fixed_cache.value.index_copy_(2, slots, xv[:, :, -capacity:])+ else:+ raise RuntimeError("gemma4: chunked prefill past the sliding window is not supported")+ present_key_value = fixed_cache+ elif past_key_value is not None:cumulative_len = 0if len(past_key_value) > 0:past_key, past_value, cumulative_len = past_key_value@@ -245,6 +319,7 @@ class TransformerBlockGemma4(nn.Module):self.register_buffer("layer_scalar", torch.empty(1, device=device, dtype=dtype))def forward(self, x, attention_mask=None, freqs_cis=None, past_key_value=None, per_layer_input=None, shared_kv=None):+ output = xsliding_window = Noneif self.sliding_attention:sliding_window = self.sliding_attention@@ -281,7 +356,8 @@ class TransformerBlockGemma4(nn.Module):x = self.post_per_layer_input_norm(x)x = residual + x- x = x * comfy.ops.cast_to_input(self.layer_scalar, x)+ # in-place into the input buffer so CUDA-graph replays land in the static x+ x = torch.mul(x, comfy.ops.cast_to_input(self.layer_scalar, x), out=output)return x, present_key_value, shareable_kv@@ -290,6 +366,9 @@ class Gemma4Transformer(nn.Module):def __init__(self, config, device=None, dtype=None, ops=None):super().__init__()self.config = config+ self.fixed_kv = True+ self.prefetch_dynamic_vbars = True+ self.graph_dynamic_vbar_blocks = Trueself.embed_tokens = _make_scaled_embedding(ops, config.vocab_size, config.hidden_size, config.hidden_size ** 0.5, device, dtype)@@ -298,6 +377,19 @@ class Gemma4Transformer(nn.Module):for i in range(config.num_hidden_layers)])+ # KV-shared layers never run k_proj/v_proj/k_norm: their never-resolved vbar+ # signatures would block layer graph capture, so prefetch only what executes+ first_kv_shared = config.num_hidden_layers - config.num_kv_shared_layers if config.num_kv_shared_layers > 0 else config.num_hidden_layers+ self._prefetch_units = []+ for i, layer in enumerate(self.layers):+ if i >= first_kv_shared:+ dead = {layer.self_attn.k_proj, layer.self_attn.v_proj, layer.self_attn.k_norm}+ self._prefetch_units.append([+ m for m in layer.modules() if next(m.children(), None) is None and m not in dead+ ])+ else:+ self._prefetch_units.append(layer)+self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, device=device, dtype=dtype) if config.final_norm else None# Precompute RoPE inv_freq on CPU to match reference code's exact value@@ -311,6 +403,9 @@ class Gemma4Transformer(nn.Module):sliding_inv = 1.0 / (config.rope_theta[1] ** (torch.arange(0, config.head_dim, 2).float() / config.head_dim))self.register_buffer("_sliding_inv_freq", sliding_inv, persistent=False)+ if config.suppress_tokens:+ self.register_buffer("_suppress_tokens", torch.tensor(config.suppress_tokens, dtype=torch.long), persistent=False)+# Per-layer input mechanismself.hidden_size_per_layer_input = config.hidden_size_per_layer_inputif self.hidden_size_per_layer_input:@@ -322,19 +417,26 @@ class Gemma4Transformer(nn.Module):self.hidden_size_per_layer_input, eps=config.rms_norm_eps,device=device, dtype=dtype)+ def get_dynamic_vram__units(self):+ return (list(self.layers), []) if self.graph_dynamic_vbar_blocks else ([], [])+def get_past_len(self, past_key_values):for kv in past_key_values:+ if isinstance(kv, FixedKV):+ return kv.indexif len(kv) >= 3:return kv[2]return 0def _freqs_from_inv(self, inv_freq, position_ids, device, dtype):- """Compute cos/sin from stored inv_freq"""+ """Compute per-pair 2x2 rotation matrices [B, 1, S, d/2, 2, 2] from stored inv_freq"""inv_exp = inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(device)pos_exp = position_ids[:, None, :].float()freqs = (inv_exp @ pos_exp).transpose(1, 2)- emb = torch.cat((freqs, freqs), dim=-1)- return emb.cos().unsqueeze(1).to(dtype), emb.sin().unsqueeze(1).to(dtype)+ cos, sin = freqs.cos(), freqs.sin()+ mat = torch.stack((torch.stack((cos, -sin), dim=-1),+ torch.stack((sin, cos), dim=-1)), dim=-2)+ return mat.unsqueeze(1).to(dtype)def compute_freqs_cis(self, position_ids, device, dtype=None):global_freqs = self._freqs_from_inv(self._global_inv_freq, position_ids, device, dtype)@@ -401,6 +503,71 @@ class Gemma4Transformer(nn.Module):first_kv_shared = self.config.num_hidden_layers - num_kv_shared if num_kv_shared > 0 else self.config.num_hidden_layersshared_sliding_kv = None # KV from last non-shared sliding layershared_global_kv = None # KV from last non-shared global layer+ share_source = {}+ if num_kv_shared > 0:+ for i in range(first_kv_shared):+ share_source[bool(self.layers[i].sliding_attention)] = i++ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(+ list(self._prefetch_units), x.device,+ {"prefetch_dynamic_vbars": self.prefetch_dynamic_vbars and past_key_values is not None})++ fixed_kv = (past_key_values is not None and len(past_key_values) > 0+ and isinstance(past_key_values[0], FixedKV))+ decode = fixed_kv and seq_len == 1+ # mirror the conditions under which prefetch_queue_pop can actually capture, so+ # eager fallbacks keep the sliced decode path instead of the full-capacity one+ enable_graph = (decode and mask is None and self.graph_dynamic_vbar_blocks+ and prefetch_queue is not None+ and hasattr(self.layers[0], "_v_block")+ and not comfy.model_management.args.disable_cuda_graphs+ and comfy.model_management.is_device_cuda(x.device))+ decode_bias = None+ decode_masks = None+ if decode:+ prepared = set()+ for kv in past_key_values:+ if isinstance(kv, FixedKV) and id(kv.position) not in prepared:+ kv.prepare(seq_len)+ prepared.add(id(kv.position))+ if mask is not None:+ decode_masks = {}+ for kv in past_key_values:+ if isinstance(kv, FixedKV) and id(kv.position) not in decode_masks:+ decode_masks[id(kv.position)] = _fixed_kv_decode_mask(mask, kv, min_val)+ if enable_graph:+ # static buffers + per-capacity attention biases: layer graphs replay against+ # stable storage, refreshed eagerly each step+ capacities = tuple(sorted({kv.key.shape[2] for kv in past_key_values if isinstance(kv, FixedKV)}))+ state_key = (x.shape, x.dtype, x.device, tuple(t.shape for t in freqs_cis), capacities,+ None if per_layer_inputs is None else per_layer_inputs.shape)+ state = getattr(self, "_comfy_cross_step_state", None)+ if state is None or state["key"] != state_key:+ state = {"key": state_key,+ "x": torch.empty_like(x),+ "freqs_cis": [torch.empty_like(t) for t in freqs_cis],+ "bias": {c: torch.empty((1, 1, 1, c), dtype=x.dtype, device=x.device) for c in capacities},+ "per_layer": None if per_layer_inputs is None else torch.empty_like(per_layer_inputs),+ "bias_valid": -1}+ self._comfy_cross_step_state = state+ comfy.model_management._register_cross_step(self)+ state["x"].copy_(x)+ for source, target in zip(freqs_cis, state["freqs_cis"]):+ target.copy_(source)+ x = state["x"]+ freqs_cis = state["freqs_cis"]+ if per_layer_inputs is not None:+ state["per_layer"].copy_(per_layer_inputs)+ per_layer_inputs = state["per_layer"]+ valid = past_len + 1+ for capacity, bias in state["bias"].items():+ if state["bias_valid"] != past_len:+ bias.fill_(min_val)+ bias[..., :min(valid, capacity)] = 0+ elif past_len < capacity:+ bias[..., past_len:valid] = 0+ state["bias_valid"] = valid+ decode_bias = state["bias"]intermediate = Noneall_intermediate = None@@ -429,12 +596,36 @@ class Gemma4Transformer(nn.Module):is_sliding = hasattr(layer, 'sliding_attention') and layer.sliding_attentionif i >= first_kv_shared and num_kv_shared > 0:- shared = shared_sliding_kv if is_sliding else shared_global_kv- if shared is not None:- layer_kwargs['shared_kv'] = shared+ if decode:+ layer_kwargs['shared_kv'] = past_key_values[share_source[bool(is_sliding)]]+ else:+ shared = shared_sliding_kv if is_sliding else shared_global_kv+ if shared is not None:+ layer_kwargs['shared_kv'] = shared++ if enable_graph:+ bias_cache = layer_kwargs.get('shared_kv', past_kv)+ layer_mask = decode_bias[bias_cache.key.shape[2]]+ elif decode:+ bias_cache = layer_kwargs.get('shared_kv', past_kv)+ layer_mask = None if decode_masks is None else decode_masks[id(bias_cache.position)]+ else:+ layer_mask = mask++ result = []- x, current_kv, shareable_kv = layer(x=x, attention_mask=mask, freqs_cis=freqs_cis, past_key_value=past_kv, **layer_kwargs)+ def core():+ nonlocal x+ x, current_kv, shareable_kv = layer(x=x, attention_mask=layer_mask, freqs_cis=freqs_cis, past_key_value=past_kv, **layer_kwargs)+ result.append((current_kv, shareable_kv))+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, x.device, layer, x.dtype, core=core, enable_graph=enable_graph)++ if result:+ current_kv, shareable_kv = result[0]+ else:+ # graph replay: the cache already holds this step's write+ current_kv, shareable_kv = past_kv, Nonenext_key_values.append(current_kv if current_kv is not None else ())# Only track the last sliding/global before the sharing boundary@@ -447,6 +638,14 @@ class Gemma4Transformer(nn.Module):if i == intermediate_output:intermediate = x.clone()+ if prefetch_queue is not None:+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, x.device, None)++ if fixed_kv:+ for kv in past_key_values:+ if isinstance(kv, FixedKV):+ kv.advance(seq_len)+if self.norm is not None:x = self.norm(x)@@ -481,14 +680,37 @@ class Gemma4Base(BaseLlama, BaseGenerate, torch.nn.Module):if cap:logits = cap * torch.tanh(logits / cap)if self.model.config.suppress_tokens:- logits[..., self.model.config.suppress_tokens] = torch.finfo(logits.dtype).min+ logits.index_fill_(-1, self.model._suppress_tokens, torch.finfo(logits.dtype).min)return logitsdef init_kv_cache(self, batch, max_cache_len, device, execution_dtype):- past_key_values = []- for _ in range(self.model.config.num_hidden_layers):- past_key_values.append(())- return past_key_values+ cfg = self.model.config+ num_layers = cfg.num_hidden_layers+ if not self.model.fixed_kv:+ return [() for _ in range(num_layers)]+ first_shared = num_layers - cfg.num_kv_shared_layers if cfg.num_kv_shared_layers > 0 else num_layers+ # position/seqlen device tensors are shared per cache geometry and filled once per step+ trackers = {}+ caches = []+ for i in range(num_layers):+ if i >= first_shared:+ caches.append(())+ continue+ sliding = cfg.sliding_attention[i % len(cfg.sliding_attention)] if cfg.sliding_attention else False+ head_dim = cfg.head_dim if sliding else cfg.global_head_dim+ k_eq_v = cfg.attention_k_eq_v and not sliding+ kv_heads = cfg.num_global_key_value_heads if k_eq_v else cfg.num_key_value_heads+ length = min(sliding, max_cache_len) if sliding else max_cache_len+ cache_cls = RingKV if sliding else FixedKV+ tracker = trackers.get((cache_cls, length))+ if tracker is None:+ tracker = (torch.empty((1,), device=device, dtype=torch.int64),+ torch.empty((batch,), device=device, dtype=torch.int32))+ trackers[(cache_cls, length)] = tracker+ # zero-init: decode attends full capacity with masked tails, 0*0 stays finite+ key = torch.zeros((batch, kv_heads, length, head_dim), device=device, dtype=execution_dtype)+ caches.append(cache_cls(key, torch.zeros_like(key), 0, tracker[0], tracker[1]))+ return cachesdef preprocess_embed(self, embed, device):if embed["type"] == "image":@@ -1497,6 +1719,9 @@ def gemma4_te(dtype_llama=None, llama_quantization_metadata=None, model_class=Noif dtype_llama is not None:dtype = dtype_llamasuper().__init__(device=device, dtype=dtype, name="gemma4", clip_model=clip_model, model_options=model_options)++ def get_dynamic_vram__units(self):+ return getattr(self, self.clip).transformer.model.get_dynamic_vram__units()return Gemma4TEModel_