The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Speedup generation on qwen3.5/3.8 with long contexts. (#16638)
comfyanonymous67f11ccbaaca
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex ba55fbf14..b5c2b7c12 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -43,6 +43,8 @@ class FixedKVBias(FixedKV):tracker: dict = Nonedef prepare(self, num_tokens):+ if self.seqlen is not None:+ self.seqlen.fill_(self.index + num_tokens)if self.tracker["step"] == (self.index, num_tokens):returnself.tracker["step"] = (self.index, num_tokens)@@ -77,7 +79,10 @@ class FixedKVBias(FixedKV):def zeros(cls, batch, kv_heads, capacity, head_dim, device, dtype, shared):# zero-init: decode attends full capacity with masked tails, 0*0 stays finitekey = torch.zeros((batch, kv_heads, capacity, head_dim), device=device, dtype=dtype)- return cls(key, torch.zeros_like(key), 0, shared[0], None, shared[1], shared[2])+ seqlen = None+ if dtype == torch.bfloat16 and head_dim in (128, 256) and comfy_kitchen.flash_attention_decode_is_available(key.device):+ seqlen = torch.empty((batch,), device=device, dtype=torch.int32)+ return cls(key, torch.zeros_like(key), 0, shared[0], seqlen, shared[1], shared[2])def append(self, xk, xv):seq = xk.shape[2]@@ -86,10 +91,13 @@ class FixedKVBias(FixedKV):return self.key[:, :, :self.index + seq], self.value[:, :, :self.index + seq]def decode(self, xq, xk, xv, num_kv_heads):- # CUDA-graphable: device-side write position, masked attention over the full capacity+ # CUDA-graphable: device-side write position and cache lengthbatch_size, num_heads, seq, head_dim = xq.shapeself.key.index_copy_(2, self.position[:seq], xk)self.value.index_copy_(2, self.position[:seq], xv)+ if seq == 1 and self.seqlen is not None:+ out = comfy_kitchen.flash_attention_decode(xq.transpose(1, 2), self.key.transpose(1, 2), self.value.transpose(1, 2), self.seqlen)+ return out.reshape(batch_size, seq, num_heads * head_dim)groups = num_heads // num_kv_headsq = xq.reshape(batch_size, num_kv_heads, groups, seq, head_dim) * head_dim ** -0.5bias = self.bias[..., self.bias.shape[-2] - seq:, :].unsqueeze(1)