Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Performance improvements for qwen 3.x (Part 2) (#17007)

rattus4d7b594592c0
diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py
index 77ffd15d9..abed292bf 100644
--- a/comfy/ldm/modules/attention.py
+++ b/comfy/ldm/modules/attention.py
@@ -79,6 +79,7 @@ class ComfyAttention(ConfigurableModule):
super().__init__()
self.config = None
self.function = None
+ self.method = None
def with_config(self, encoded_config):
attention = ComfyAttention()
@@ -88,6 +89,7 @@ class ComfyAttention(ConfigurableModule):
def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs):
self.config = None
self.function = None
+ self.method = None
metadata = state_dict.pop(prefix + "config", None)
if metadata is not None:
self.config = json.loads(metadata.numpy().tobytes())
@@ -103,6 +105,7 @@ class ComfyAttention(ConfigurableModule):
else:
logging.warning(f"Ignoring unknown attention method {method!r} for {prefix.rstrip('.')}")
if self.function is not None:
+ self.method = method
break
super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs)
diff --git a/comfy/ops.py b/comfy/ops.py
index 07dd351d0..c1c029108 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -304,6 +304,10 @@ def resolve_cast_module_with_vbar(s, dtype, device, bias_dtype, compute_dtype, w
weight = params[0]
bias = params[1]
if prefetch["signature"] is not None:
+ # FIXME: Make this non_gemv_specific.
+ if (isinstance(s, MixedPrecisionOp) and s.comfy_gemv_optimized and isinstance(weight, QuantizedTensor) and not s._full_precision_mm
+ and s.weight_lowvram_function is None and len(s.weight_function) == 0):
+ weight = weight.optimize_for_gemv(xfer_dest)
s._v_weight = weight
s._v_bias = bias
s._v_signature = prefetch["signature"]
@@ -968,7 +972,6 @@ from .quant_ops import (
QuantizedTensor,
QUANT_ALGOS,
TensorCoreFP8Layout,
- TensorWiseINT8Layout,
get_layout_class,
)
@@ -1022,39 +1025,46 @@ def _fp16_linear_wanted(x):
and x.dtype == torch.float16 and x.is_cuda)
+def _try_quantized_linear_input_act(x, weight, bias, input_act, act_weight, act_eps,
+ residual, residual_scale):
+ if comfy.model_management.in_training or not isinstance(weight, QuantizedTensor):
+ return NotImplemented
+ with _input_act_weight(x, input_act, act_weight, act_eps) as (act_weight, act_eps):
+ return weight.linear_input_act(
+ x, bias, input_act=input_act, input_act_weight=act_weight, input_act_eps=act_eps,
+ residual=residual, residual_scale=residual_scale,
+ )
+
+
def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0,
residual=None, residual_scale=None):
run_every_op()
weight = linear.weight
quantized = isinstance(weight, QuantizedTensor)
if (comfy.model_management.in_training or getattr(linear, "comfy_force_forward", False) or getattr(linear, "_full_precision_mm", False)
- or not ((quantized and weight._layout_cls == "TensorWiseINT8Layout"
- and not getattr(weight._params, "transposed", False))
- or (not quantized and _fp16_linear_wanted(x)))):
+ or not (quantized or _fp16_linear_wanted(x))):
out = linear(_eager_input_act(x, input_act, act_weight, act_eps))
return _linear_residual(out, residual, residual_scale)
with CastBiasWeightContext(linear, x, offloadable=True,
compute_dtype=x.dtype if quantized else None,
want_requant=quantized) as (weight, bias):
- return linear_input_act_(x, weight, bias, input_act, act_weight, act_eps,
- residual, residual_scale, fp16_accumulation=not quantized)
+ if not quantized:
+ return linear_input_act_(x, weight, bias, input_act, act_weight, act_eps,
+ residual, residual_scale)
+ out = _try_quantized_linear_input_act(x, weight, bias, input_act, act_weight, act_eps,
+ residual, residual_scale)
+ if out is not NotImplemented:
+ return out
+
+ # A tensor-level fallback cannot replace module-owned input processing.
+ out = linear(_eager_input_act(x, input_act, act_weight, act_eps))
+ return _linear_residual(out, residual, residual_scale)
def linear_input_act_(x, weight, bias, input_act=None, act_weight=None, act_eps=0.0,
residual=None, residual_scale=None, *, fp16_accumulation=True):
- """``linear(act(x))``, with ``act`` folded into an INT8 activation quantizer.
-
- An INT8 linear quantizes its input anyway, so an elementwise activation can
- ride along inside that kernel instead of writing a full-size intermediate to
- HBM and reading it straight back. Worth it for an MLP's down-projection,
- where the intermediate is several times the hidden size, and for a pre-norm
- block's ``linear(rms_norm(x))`` ("rms_norm", which reads the norm's weight
- and eps from act_weight/act_eps).
-
- With ``residual``/``residual_scale`` the result is the pre-norm block's
- addcmul, ``residual + residual_scale * linear(act(x))``, fused into the
- INT8 GEMM epilogue where supported.
+ """``linear(act(x))`` with optional residual addcmul, fused where the weight supports it.
Linear weights are already prepared. An RMSNorm module in act_weight owns
its norm-weight cast context; a tensor uses the supplied act_eps directly.
@@ -1062,29 +1072,18 @@ def linear_input_act_(x, weight, bias, input_act=None, act_weight=None, act_eps=
if input_act is None and residual is None:
return torch.nn.functional.linear(x, weight, bias)
- if (comfy.model_management.in_training
- or not isinstance(weight, QuantizedTensor)
- or weight._layout_cls != "TensorWiseINT8Layout"
- or getattr(weight._params, "transposed", False)):
- x = _eager_input_act(x, input_act, act_weight, act_eps)
- if (not comfy.model_management.in_training
- and not isinstance(weight, QuantizedTensor)
- and fp16_accumulation and _fp16_linear_wanted(x)):
- return quant_ops.ck.fp16_linear(x, weight, bias, residual=residual, residual_scale=residual_scale)
- return _linear_residual(torch.nn.functional.linear(x, weight, bias), residual, residual_scale)
-
- qdata, scale = TensorWiseINT8Layout.get_plain_tensors(weight)
- with _input_act_weight(x, input_act, act_weight, act_eps) as (act_weight, act_eps):
- return quant_ops.ck.int8_linear(
- x, qdata, scale, bias, x.dtype,
- convrot=getattr(weight._params, "convrot", False),
- convrot_groupsize=getattr(weight._params, "convrot_groupsize", 256),
- input_act=input_act,
- input_act_weight=act_weight,
- input_act_eps=act_eps,
- residual=residual,
- residual_scale=residual_scale,
- )
+ out = _try_quantized_linear_input_act(x, weight, bias, input_act, act_weight, act_eps,
+ residual, residual_scale)
+ if out is not NotImplemented:
+ return out
+ x = _eager_input_act(x, input_act, act_weight, act_eps)
+
+ if (not comfy.model_management.in_training
+ and not isinstance(weight, QuantizedTensor)
+ and fp16_accumulation and _fp16_linear_wanted(x)):
+ return quant_ops.ck.fp16_linear(x, weight, bias, residual=residual, residual_scale=residual_scale)
+
+ return _linear_residual(torch.nn.functional.linear(x, weight, bias), residual, residual_scale)
# Grouped-scale integer weights that run through the shared INT8 GEMM (comfy_kitchen
@@ -1397,6 +1396,10 @@ def _quantized_weight_state_dict(module, sd, prefix, extra_quant_conf=None, extr
class MixedPrecisionOp(CastWeightBiasOp):
quant_format = None
+ # Prefer small-M execution; Kitchen owns the layout and preserves GEMM support.
+ # Keep opt-in until extensive testing allows unconditional relayout and a single
+ # resident layout/kernel path instead of parallel canonical and packed paths.
+ comfy_gemv_optimized = False
def can_use_quantized_matmul(self, disabled_formats):
return (self.quant_format in QUANT_ALGOS
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py
index 161e65411..542c4590e 100644
--- a/comfy/text_encoders/llama.py
+++ b/comfy/text_encoders/llama.py
@@ -6,6 +6,7 @@ import math
from tqdm import tqdm
import comfy.utils
import comfy_kitchen
+from comfy_kitchen.int8_decode import Int8DecodeCache
from comfy.ldm.modules.attention import optimized_attention_for_device
import comfy.model_management
@@ -37,52 +38,31 @@ class FixedKV:
self.index -= discard
@dataclass
-class FixedKVBias(FixedKV):
- # full-capacity decode bias [1, 1, rows, capacity], last `seq` rows serve the queries; shared across layers
- bias: torch.Tensor = None
+class FixedKVState(FixedKV):
tracker: dict = None
def prepare(self, num_tokens):
- if num_tokens == 1 and self.seqlen is not None:
- self.seqlen.fill_(self.index + num_tokens)
if self.tracker["step"] == (self.index, num_tokens):
return
self.tracker["step"] = (self.index, num_tokens)
- i = self.index
- rows = self.bias.shape[-2]
- if num_tokens <= rows:
- torch.arange(i, i + rows, out=self.position)
- window = self.tracker[num_tokens]
- start = i - rows + 1
- skip = max(-start, 0)
- end = min(i + rows, self.bias.shape[-1])
- self.bias[..., :, start + skip:end] = window[:, skip:end - start]
- else:
- self.bias[..., :, i:i + num_tokens] = 0
+ torch.arange(self.index, self.index + self.position.shape[0], out=self.position)
+ self.seqlen.fill_(self.index + num_tokens)
@staticmethod
- def shared(capacity, device, dtype):
- # all layers advance in lockstep, so the bias caches share one position/bias/tracker
- rows = 6
- position = torch.empty((rows,), device=device, dtype=torch.int64)
- bias = torch.full((1, 1, rows, capacity), torch.finfo(dtype).min, device=device, dtype=dtype)
- tracker = {"step": -1}
- # window templates per decode width: row r serves query j = r - (rows - n) and may see slots <= index + j
- for n in range(1, rows + 1):
- window = torch.full((rows, 2 * rows - 1), torch.finfo(dtype).min, device=device, dtype=dtype)
- for r in range(rows):
- window[r, :rows + max(r - (rows - n), 0)] = 0
- tracker[n] = window
- return position, bias, tracker
+ def shared(batch, device):
+ # all layers advance in lockstep, so the caches share one position/seqlen/tracker
+ position = torch.empty((6,), device=device, dtype=torch.int64)
+ seqlen = torch.empty((batch,), device=device, dtype=torch.int32)
+ return position, seqlen, {"step": -1}
+
+class FixedKVCache(FixedKVState):
+ # [B, kv heads, capacity, D] storage with shared decode position state.
@classmethod
def zeros(cls, batch, kv_heads, capacity, head_dim, device, dtype, shared):
# zero-init: decode attends full capacity with masked tails, 0*0 stays finite
key = torch.zeros((batch, kv_heads, capacity, head_dim), device=device, dtype=dtype)
- seqlen = None
- if dtype == torch.bfloat16 and head_dim in (128, 256) and comfy_kitchen.flash_attention_decode_is_available(key.device):
- seqlen = torch.empty((batch,), device=device, dtype=torch.int32)
- return cls(key, torch.zeros_like(key), 0, shared[0], seqlen, shared[1], shared[2])
+ return cls(key, torch.zeros_like(key), 0, *shared)
def append(self, xk, xv):
seq = xk.shape[2]
@@ -90,21 +70,56 @@ class FixedKVBias(FixedKV):
self.value[:, :, self.index:self.index + seq] = xv
return self.key[:, :, :self.index + seq], self.value[:, :, :self.index + seq]
- def decode(self, xq, xk, xv, num_kv_heads):
- # CUDA-graphable: device-side write position and cache length
- batch_size, num_heads, seq, head_dim = xq.shape
- self.key.index_copy_(2, self.position[:seq], xk)
- self.value.index_copy_(2, self.position[:seq], xv)
- if seq == 1 and self.seqlen is not None:
- out = comfy_kitchen.flash_attention_decode(xq.transpose(1, 2), self.key.transpose(1, 2), self.value.transpose(1, 2), self.seqlen)
- return out.reshape(batch_size, seq, num_heads * head_dim)
- groups = num_heads // num_kv_heads
- q = xq.reshape(batch_size, num_kv_heads, groups, seq, head_dim) * head_dim ** -0.5
- bias = self.bias[..., self.bias.shape[-2] - seq:, :].unsqueeze(1)
- scores = (q @ self.key.transpose(-1, -2).unsqueeze(2)).add_(bias)
- probs = torch.softmax(scores, dim=-1, dtype=torch.float32).to(xq.dtype)
- out = probs @ self.value.unsqueeze(2)
- return out.permute(0, 3, 1, 2, 4).reshape(batch_size, seq, num_heads * head_dim)
+ def write(self, xk, xv):
+ # CUDA-graphable: device-side write position
+ position = self.position[:xk.shape[2]]
+ self.key.index_copy_(2, position, xk)
+ self.value.index_copy_(2, position, xv)
+
+ def attend(self, xq, xk, xv):
+ return comfy_kitchen.flash_attention_decode_gqa(xq, self.key, self.value, self.seqlen)
+
+ def mask(self, seq):
+ # additive [B, 1, seq, capacity] mask with attend()'s visibility, for a generic attention over the
+ # whole cache: a chain row sees slots up to its own
+ rel = torch.arange(self.key.shape[2], device=self.key.device) - self.seqlen.view(-1, 1, 1, 1)
+ visible = rel < torch.arange(1 - seq, 1, device=self.key.device).view(seq, 1)
+ return torch.zeros(visible.shape, device=self.key.device, dtype=self.key.dtype).masked_fill_(~visible, float("-inf"))
+
+
+class Int8FixedKVCache(FixedKVState):
+ # INT8 prefix with a bounded BF16 window for page updates and speculative commits.
+ @classmethod
+ def zeros(cls, batch, kv_heads, capacity, head_dim, device, dtype, shared):
+ packed = Int8DecodeCache(batch, kv_heads, capacity, device)
+ # Two pages may be refreshed after a rollback while new rows enter the next page.
+ window = min(capacity, 3 * packed.key.shape[3])
+ key = torch.zeros((batch, kv_heads, window, head_dim), device=device, dtype=dtype)
+ cache = cls(key, torch.zeros_like(key), 0, *shared)
+ cache.packed = packed
+ return cache
+
+ def append(self, xk, xv):
+ # Prefill attends its original tensors; only the quantized prefix survives it.
+ seq = xk.shape[2]
+ length = torch.full_like(self.position[:1], seq)
+ self.packed.update(xk.contiguous(), xv.contiguous(), length, initialize=True)
+ start = max(0, seq - self.key.shape[2])
+ slots = torch.arange(start, seq, device=xk.device).remainder(self.key.shape[2])
+ self.key.index_copy_(2, slots, xk[:, :, start:])
+ self.value.index_copy_(2, slots, xv[:, :, start:])
+ return xk, xv
+
+ def write(self, xk, xv):
+ position = self.position[:xk.shape[2]].remainder(self.key.shape[2])
+ self.key.index_copy_(2, position, xk)
+ self.value.index_copy_(2, position, xv)
+
+ def attend(self, xq, xk, xv):
+ length = self.position[:1]
+ self.packed.update(self.key, self.value, length)
+ out, lse = self.packed.attend(xq, length)
+ return comfy_kitchen.flash_attention_decode_step_merge(out, lse, xq, xk, xv, torch.empty_like(out))
@dataclass
class Llama2Config:
@@ -510,12 +525,11 @@ class RMSNorm(nn.Module):
self.weight = nn.Parameter(torch.empty(dim, device=device, dtype=dtype))
self.add = add
- def forward(self, x: torch.Tensor):
- w = self.weight
- if self.add:
- w = w + 1.0
+ def scale(self):
+ return self.weight + 1.0 if self.add else self.weight
- return comfy.ldm.common_dit.rms_norm(x, w, self.eps)
+ def forward(self, x: torch.Tensor):
+ return comfy.ldm.common_dit.rms_norm(x, self.scale(), self.eps)
@@ -750,13 +764,19 @@ class MLP(nn.Module):
self.activation = lambda a: torch.nn.functional.gelu(a, approximate="tanh")
self.merged_input_act = None
- def forward(self, x):
+ def forward(self, x, norm=None):
+ # norm: the block's pre-norm, folded into the merged projection's input quantizer where supported
if self.merged_mlp:
- x = self.gate_up_proj(x)
+ if norm is not None:
+ x = comfy.ops.linear_input_act(self.gate_up_proj, x, "rms_norm", norm.scale(), norm.eps)
+ else:
+ x = self.gate_up_proj(x)
if self.merged_input_act is not None:
return comfy.ops.linear_input_act(self.down_proj, x, self.merged_input_act)
gate, up = x.chunk(2, dim=-1)
return self.down_proj(self.activation(gate) * up)
+ if norm is not None:
+ x = norm(x)
return self.down_proj(self.activation(self.gate_proj(x)) * self.up_proj(x))
class TransformerBlock(nn.Module):
@@ -914,12 +934,13 @@ class Llama2_(nn.Module):
caches = []
flash = getattr(comfy_kitchen, "flash_attention_decode_is_available", None)
flash_kv = self.fixed_kv and flash is not None and flash(device)
+ if flash_kv:
+ pos = torch.empty((batch,), device=device, dtype=torch.int64)
+ seqlen = torch.zeros((batch,), device=device, dtype=torch.int32)
for _ in range(self.config.num_hidden_layers):
if flash_kv:
key = torch.empty((batch, capacity, self.config.num_key_value_heads, self.config.head_dim), device=device, dtype=dtype)
value = torch.empty_like(key)
- pos = torch.empty((batch,), device=device, dtype=torch.int64)
- seqlen = torch.zeros((batch,), device=device, dtype=torch.int32)
caches.append(FixedKV(key, value, 0, pos, seqlen))
else:
key = torch.empty((batch, self.config.num_key_value_heads, capacity, self.config.head_dim), device=device, dtype=dtype)
@@ -941,7 +962,7 @@ class Llama2_(nn.Module):
device=device)
def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True,
- dtype=None, position_ids=None, embeds_info=[], past_key_values=None, input_ids=None,deepstack_embeds=None, visual_pos_masks=None, decode_buffers=None):
+ dtype=None, position_ids=None, embeds_info=[], past_key_values=None, input_ids=None,deepstack_embeds=None, visual_pos_masks=None, decode_buffers=None, freqs_cis=None):
if embeds is not None:
x = embeds
else:
@@ -956,17 +977,17 @@ class Llama2_(nn.Module):
if fixed_kv_decode:
attention_mask = None
- if position_ids is None:
- position_ids = torch.arange(past_len, past_len + seq_len, device=x.device).unsqueeze(0)
-
- freqs_cis = self.compute_freqs_cis(position_ids, x.device)
+ if freqs_cis is None:
+ if position_ids is None:
+ position_ids = torch.arange(past_len, past_len + seq_len, device=x.device).unsqueeze(0)
+ freqs_cis = self.compute_freqs_cis(position_ids, x.device)
mask = None
if attention_mask is not None:
mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])).expand(attention_mask.shape[0], 1, seq_len, attention_mask.shape[-1])
mask = mask.masked_fill(mask.to(torch.bool), torch.finfo(x.dtype).min / 4)
- spec_decode = fixed_kv and any(isinstance(kv, FixedKVBias) for kv in past_key_values) and 2 <= seq_len <= 6 and past_len > 0 and attention_mask is None
+ spec_decode = fixed_kv and any(isinstance(kv, FixedKVState) for kv in past_key_values) and 2 <= seq_len <= 6 and past_len > 0 and attention_mask is None
if seq_len > 1 and not spec_decode: # spec verify: the staircase decode bias is causal
causal_mask = torch.empty(past_len + seq_len, past_len + seq_len, dtype=x.dtype, device=x.device).fill_(torch.finfo(x.dtype).min / 4).triu_(1)
if mask is not None:
@@ -1011,11 +1032,14 @@ class Llama2_(nn.Module):
if past_key_values is not None:
past_kv = past_key_values[i] if len(past_key_values) > 0 else []
- if fixed_kv:
+ if fixed_kv and type(past_kv) is not FixedKV:
past_kv.prepare(seq_len)
def core():
nonlocal x
+ # Plain FixedKV shares GPU counters; prepare once, eager or graph-replayed.
+ if i == 0 and type(past_kv) is FixedKV:
+ past_kv.prepare(seq_len)
output, current_kv = layer(
x=x,
attention_mask=mask,
@@ -1174,6 +1198,10 @@ class BaseGenerate:
if init_decode_buffers is not None:
decode_buffers = init_decode_buffers(embeds.shape[0], device, execution_dtype)
decode_tokens = torch.empty((embeds.shape[0], 1), dtype=torch.long, device=device)
+ # Precompute decode RoPE once; explicit MRoPE positions keep their existing path.
+ rope = None
+ if decode_buffers is not None and next_pos is None:
+ rope = rope_matrix(self.model.compute_freqs_cis(torch.arange(max_cache_len, device=device, dtype=torch.float).unsqueeze(0), device))
penalize = penalty_active(repetition_penalty, presence_penalty)
penalty_mask = None
@@ -1195,6 +1223,9 @@ class BaseGenerate:
extra = {}
if decode_buffers is not None:
extra["decode_buffers"] = decode_buffers
+ if rope is not None and step > 0:
+ past_len = self.model.get_past_len(past_key_values)
+ extra["freqs_cis"] = rope[:, :, past_len:past_len + 1]
if step == 0 and deepstack_embeds is not None:
extra["deepstack_embeds"] = deepstack_embeds
extra["visual_pos_masks"] = visual_pos_masks
diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py
index 4f0f291e2..c114869bb 100644
--- a/comfy/text_encoders/qwen35.py
+++ b/comfy/text_encoders/qwen35.py
@@ -10,12 +10,13 @@ import comfy.model_management
import comfy.model_prefetch
import comfy.ops
import comfy_kitchen
+from comfy_kitchen.int8_decode import is_available as int8_decode_is_available
from comfy.quant_ops import QuantizedTensor
-from comfy.ldm.modules.attention import optimized_attention_for_device
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_for_device
from comfy import sd1_clip
import comfy.text_encoders.qwen_vl
-from .llama import BaseLlama, BaseGenerate, FixedKV, FixedKVBias, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix
+from .llama import BaseLlama, BaseGenerate, FixedKV, FixedKVCache, Int8FixedKVCache, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix
@dataclass
@@ -23,21 +24,53 @@ class LinearKV(FixedKV):
# DeltaNet state on the FixedKV interface: key=conv_state, value=recurrent_state (fp32)
g_decay: torch.Tensor = None
dt_bias: torch.Tensor = None
- snapshots: list = None # [(recurrent, conv)] taken after step 1, 2, ... of the last verify
- last_seq: int = 1
- snap_backing: torch.Tensor = None
- conv_snap_backing: torch.Tensor = None
norm_weight: torch.Tensor = None
+ # deferred commit (kitchen gated_delta_decode_deferred): a decode step leaves the state untouched
+ # and the next step first replays its ctl[0] still-accepted tokens from the [1 - ctl[1]] side
+ # buffers. ctl and its tracker are shared by every layer.
+ qkv_buf: torch.Tensor = None
+ proj_buf: torch.Tensor = None
+ gates_buf: torch.Tensor = None
+ sumsq_buf: torch.Tensor = None
+ ctl: torch.Tensor = None
+ ctl_tracker: dict = None
+ uncommitted: int = 0
+
+ @staticmethod
+ def deferred_tracker(device):
+ # ctl = {pending, parity}: replay the previous step's accepted prefix
+ n = comfy_kitchen.gated_delta_ctl_ints
+ host = torch.zeros((2, n), dtype=torch.int32)
+ # the host rows are pinned and fenced by events only where the copy is asynchronous
+ cuda = torch.device(device).type == "cuda"
+ return {"ctl": torch.zeros((n,), dtype=torch.int32, device=device), "host": host.pin_memory() if cuda else host,
+ "copied": [torch.cuda.Event(), torch.cuda.Event()] if cuda else None, "step": None, "parity": 0}
+
+ def decode_step(self, num_tokens):
+ return self.index > 0 and num_tokens <= 6
def prepare(self, num_tokens):
- pass
+ # runs eagerly before the layer (a replayed graph skips its Python): the first linear
+ # layer of a step hands the kernels what to commit, every layer records what it leaves
+ t = self.ctl_tracker
+ if t["step"] != (self.index, num_tokens):
+ t["step"] = (self.index, num_tokens)
+ t["parity"] ^= 1
+ # the pinned row is reused every other step: its previous copy must have landed
+ if t["copied"] is not None:
+ t["copied"][t["parity"]].synchronize()
+ host = t["host"][t["parity"]]
+ host[0] = self.uncommitted
+ host[1] = t["parity"]
+ self.ctl.copy_(host, non_blocking=True)
+ if t["copied"] is not None:
+ t["copied"][t["parity"]].record(torch.cuda.current_stream(self.ctl.device))
+ self.uncommitted = num_tokens if self.decode_step(num_tokens) else 0
def rollback(self, discard=1):
- # discard the rejected tail: restore the snapshot taken after the last kept token
- rec, conv = self.snapshots[self.last_seq - discard - 1]
- self.recurrent_state.copy_(rec)
- self.conv_state.copy_(conv)
+ # nothing was written: the next step replays fewer tokens
self.index -= discard
+ self.uncommitted -= discard
@property
def conv_state(self):
@@ -210,37 +243,26 @@ class GatedDeltaNet(nn.Module):
def forward(self, x, past_key_value=None, **kwargs):
batch_size, seq_len, _ = x.shape
- use_recurrent = (
- past_key_value is not None
- and past_key_value.index > 0
- and seq_len <= 6
- )
-
- fused_available = getattr(comfy_kitchen, "gated_delta_decode_is_available", None)
- use_fused = (use_recurrent and fused_available is not None and fused_available(x.device, self.key_head_dim, self.value_head_dim)
- and (seq_len == 1 or past_key_value.snap_backing is not None))
-
# Projections (shared)
proj = self.in_proj_qkv(x) # [B, seq_len, conv_dim]
z = self.in_proj_z(x)
- if use_fused:
- # decode: kitchen conv step, then gates + delta rule + gated norm in one kernel
- if seq_len > 1:
- past_key_value.last_seq = seq_len
+ if past_key_value is not None and past_key_value.decode_step(seq_len):
+ # decode: conv step into the side buffer, then gates + delta rule + gated norm in one
+ # kernel; both first commit the previous step's accepted tokens
+ kv = past_key_value
with comfy.ops.CastBiasWeightContext(self.conv1d, proj, offloadable=True) as (conv_weight, conv_bias):
- conv_out = comfy_kitchen.deltanet_conv_step(proj, past_key_value.conv_state, conv_weight, conv_bias,
- past_key_value.conv_snap_backing[:seq_len - 1] if seq_len > 1 else None)
+ comfy_kitchen.deltanet_conv_step_deferred(proj, kv.conv_state, conv_weight, conv_bias, kv.proj_buf, kv.qkv_buf, kv.ctl)
with comfy.ops.CastBiasWeightContext(self.in_proj_a, x, offloadable=True) as (w_a, _), \
comfy.ops.CastBiasWeightContext(self.in_proj_b, x, offloadable=True) as (w_b, _):
if isinstance(w_a, QuantizedTensor):
w_a = w_a.dequantize()
if isinstance(w_b, QuantizedTensor):
w_b = w_b.dequantize()
- core_attn_out = comfy_kitchen.gated_delta_decode_fused(
- conv_out, x, w_a, w_b, past_key_value.dt_bias, past_key_value.g_decay, past_key_value.recurrent_state,
- self.key_dim, self.num_key_heads, self.key_head_dim ** -0.5, z, past_key_value.norm_weight, self.norm.eps,
- past_key_value.snap_backing[:seq_len - 1] if seq_len > 1 else None)
+ core_attn_out = comfy_kitchen.gated_delta_decode_deferred(
+ x, w_a, w_b, kv.dt_bias, kv.g_decay, kv.recurrent_state,
+ self.key_dim, self.num_key_heads, self.key_head_dim ** -0.5, z, kv.norm_weight, self.norm.eps,
+ kv.qkv_buf, kv.gates_buf, kv.sumsq_buf, kv.ctl)
return self.out_proj(core_attn_out.reshape(batch_size, seq_len, -1)), past_key_value
mixed_qkv = proj.transpose(1, 2) # [B, conv_dim, seq_len]
@@ -248,35 +270,21 @@ class GatedDeltaNet(nn.Module):
a = self.in_proj_a(x)
# Conv1d
- if use_recurrent:
- # decode: exact-width causal window, weight resolved via the vbar-aware context
- combined = torch.cat([past_key_value.conv_state, mixed_qkv], dim=-1)
- if seq_len > 1:
- past_key_value.last_seq = seq_len
- for s in range(seq_len - 1):
- past_key_value.snapshots[s][1].copy_(combined[:, :, 1 + s:1 + s + self.conv_kernel_size - 1])
- past_key_value.conv_state.copy_(combined[:, :, seq_len:])
- with comfy.ops.CastBiasWeightContext(self.conv1d, combined, offloadable=True) as (conv_weight, conv_bias):
- mixed_qkv = F.silu(F.conv1d(combined, conv_weight, conv_bias, groups=self.conv1d.groups))
- else:
- if past_key_value is not None:
- conv_state = past_key_value.conv_state
- conv_state_init = F.pad(mixed_qkv, (self.conv_kernel_size - mixed_qkv.shape[-1], 0))
- conv_state.copy_(conv_state_init[:, :, -conv_state.shape[-1]:])
- mixed_qkv = F.silu(self.conv1d(mixed_qkv)[:, :, :seq_len])
+ if past_key_value is not None:
+ conv_state = past_key_value.conv_state
+ conv_state_init = F.pad(mixed_qkv, (self.conv_kernel_size - mixed_qkv.shape[-1], 0))
+ conv_state.copy_(conv_state_init[:, :, -conv_state.shape[-1]:])
+ mixed_qkv = F.silu(self.conv1d(mixed_qkv)[:, :, :seq_len])
# Split QKV and compute beta/g
mixed_qkv = mixed_qkv.transpose(1, 2) # [B, seq_len, conv_dim]
query, key, value = mixed_qkv.split([self.key_dim, self.key_dim, self.value_dim], dim=-1)
- if use_recurrent:
- g_decay, dt_bias = past_key_value.g_decay, past_key_value.dt_bias
- else:
- g_decay = -comfy.model_management.cast_to_device(self.A_log, x.device, torch.float32).exp()
- dt_bias = comfy.model_management.cast_to_device(self.dt_bias, x.device, torch.float32)
- if past_key_value is not None:
- past_key_value.g_decay = g_decay
- past_key_value.dt_bias = dt_bias
- past_key_value.norm_weight = comfy.model_management.cast_to(self.norm.weight, dtype=x.dtype, device=x.device)
+ g_decay = -comfy.model_management.cast_to_device(self.A_log, x.device, torch.float32).exp()
+ dt_bias = comfy.model_management.cast_to_device(self.dt_bias, x.device, torch.float32)
+ if past_key_value is not None:
+ past_key_value.g_decay = g_decay
+ past_key_value.dt_bias = dt_bias
+ past_key_value.norm_weight = comfy.model_management.cast_to(self.norm.weight, dtype=x.dtype, device=x.device)
beta = b.sigmoid()
g = g_decay * F.softplus(a.float() + dt_bias)
@@ -289,41 +297,17 @@ class GatedDeltaNet(nn.Module):
key = key.repeat_interleave(rep, dim=2)
# Delta rule
- if use_recurrent:
- scale = self.key_head_dim ** -0.5
- q = F.normalize(query.float(), dim=-1) * scale
- k = F.normalize(key.float(), dim=-1)
- v = value.float()
- beta_t = beta.reshape(batch_size, seq_len, -1)
- g_t = g.reshape(batch_size, seq_len, -1).exp()
-
- # In-place state update: [B, heads, k_dim, v_dim]
- recurrent_state = past_key_value.recurrent_state
- outs = []
- for s in range(seq_len):
- recurrent_state.mul_(g_t[:, s, :, None, None])
- kv_mem = torch.einsum('bhk,bhkv->bhv', k[:, s], recurrent_state)
- delta = (v[:, s] - kv_mem) * beta_t[:, s, :, None]
- # rank-1 update via baddbmm_: no materialized [B, H, D, D] outer-product temp
- recurrent_state.view(-1, self.key_head_dim, self.value_head_dim).baddbmm_(
- k[:, s].reshape(-1, self.key_head_dim, 1), delta.reshape(-1, 1, self.value_head_dim))
- outs.append(torch.einsum('bhk,bhkv->bhv', q[:, s], recurrent_state))
- if seq_len > 1 and s < seq_len - 1:
- past_key_value.snapshots[s][0].copy_(recurrent_state)
- core_attn_out = torch.stack(outs, dim=1).to(x.dtype)
- present_key_value = past_key_value
- else:
- core_attn_out, last_recurrent_state = torch_chunk_gated_delta_rule(
- query, key, value, g=g, beta=beta,
- initial_state=None,
- output_final_state=past_key_value is not None,
- )
+ core_attn_out, last_recurrent_state = torch_chunk_gated_delta_rule(
+ query, key, value, g=g, beta=beta,
+ initial_state=None,
+ output_final_state=past_key_value is not None,
+ )
- present_key_value = None
- if past_key_value is not None:
- if last_recurrent_state is not None:
- past_key_value.recurrent_state.copy_(last_recurrent_state.to(past_key_value.recurrent_state.dtype))
- present_key_value = past_key_value
+ present_key_value = None
+ if past_key_value is not None:
+ if last_recurrent_state is not None:
+ past_key_value.recurrent_state.copy_(last_recurrent_state.to(past_key_value.recurrent_state.dtype))
+ present_key_value = past_key_value
# Gated norm + output projection (shared)
core_attn_out = self.norm(core_attn_out.reshape(-1, self.value_head_dim), z.reshape(-1, self.value_head_dim))
@@ -366,6 +350,20 @@ class GatedAttention(nn.Module):
# QK norms with (1+weight) scaling
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)
+ self.comfy_attention = ComfyAttention()
+
+ def kv_cache_type(self, device, dtype):
+ method = self.comfy_attention.method
+ if method == "comfy_kitchen_int8" or method is None and comfy.model_management.comfy_kitchen_attention_enabled():
+ if dtype == torch.bfloat16 and self.head_dim == 256 and int8_decode_is_available(device):
+ return Int8FixedKVCache
+ return FixedKVCache
+
+ def init_kv_cache(self, batch, capacity, device, dtype, shared):
+ cache_type = self.kv_cache_type(device, dtype)
+ if cache_type not in shared:
+ shared[cache_type] = cache_type.shared(batch, device)
+ return cache_type.zeros(batch, self.num_kv_heads, capacity, self.head_dim, device, dtype, shared[cache_type])
def forward(self, x, attention_mask=None, freqs_cis=None, optimized_attention=None, past_key_value=None):
batch_size, seq_length, _ = x.shape
@@ -389,13 +387,25 @@ class GatedAttention(nn.Module):
# KV cache
present_key_value = past_key_value
- if past_key_value is not None and seq_length <= 6 and attention_mask is None:
- output = past_key_value.decode(xq, xk, xv, self.num_kv_heads)
+ decode = past_key_value is not None and seq_length <= 6 and attention_mask is None
+ if decode:
+ past_key_value.write(xk, xv)
+
+ if decode and (isinstance(past_key_value, Int8FixedKVCache) or
+ self.comfy_attention.method is None and not comfy.model_management.comfy_kitchen_attention_enabled()):
+ output = past_key_value.attend(xq, xk, xv)
else:
- if past_key_value is not None:
+ if decode:
+ xk, xv = past_key_value.key, past_key_value.value
+ attention_mask = past_key_value.mask(seq_length)
+ optimized_attention = optimized_attention_for_device(x.device)
+ elif past_key_value is not None:
xk, xv = past_key_value.append(xk, xv)
gqa_kwargs = {"enable_gqa": True} if self.num_heads != self.num_kv_heads else {}
- output = optimized_attention(xq, xk, xv, self.num_heads, mask=attention_mask, skip_reshape=True, **gqa_kwargs)
+ q, k, v = map(AttentionTensorContainer, (xq, xk, xv))
+ del xq, xk, xv
+ output = optimized_attention(q, k, v, self.num_heads, mask=attention_mask, skip_reshape=True,
+ preferred_attention=self.comfy_attention, **gqa_kwargs)
output = output * gate.sigmoid()
return self.o_proj(output), present_key_value
@@ -423,7 +433,7 @@ class Qwen35TransformerBlock(nn.Module):
# in-place into the input buffer so CUDA-graph replays land in the static x
x = torch.add(x, h, out=output)
- x = torch.add(x, self.mlp(self.post_attention_layernorm(x)), out=output)
+ x = torch.add(x, self.mlp(x, norm=self.post_attention_layernorm), out=output)
return x, present_key_value
@@ -721,6 +731,12 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
self.mtp = None
if config.mtp:
self.mtp = MTPHead(config, device=device, dtype=dtype, ops=operations)
+ # Prefer small-M execution for autoregressive linears, including the draft/head.
+ decode_units = [self.model.layers] + ([self.mtp] if config.mtp else []) + ([self.model.lm_head] if config.lm_head else [])
+ for unit in decode_units:
+ for m in unit.modules():
+ if isinstance(m, comfy.ops.MixedPrecisionOp):
+ m.comfy_gemv_optimized = True
vision_overrides = QWEN35_MODELS.get(self.model_type, {}).get("vision", {})
vision_config = {**QWEN35_VISION_DEFAULTS, **vision_overrides, "out_hidden_size": config.hidden_size}
self.visual = Qwen35VisionModel(vision_config, device=device, dtype=dtype, ops=operations)
@@ -777,8 +793,7 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
cap = embeds.shape[1] + max_length + 7
pkv = self.init_kv_cache(embeds.shape[0], cap, device, dt)
# repair window: drafting ahead plus a near-full rollback
- mtp_kv = FixedKVBias.zeros(embeds.shape[0], cfg.num_key_value_heads, cap, cfg.head_dim, device, dt,
- FixedKVBias.shared(cap, device, dt))
+ mtp_kv = self.mtp.layers[0].self_attn.init_kv_cache(embeds.shape[0], cap, device, dt, {})
head = self.model.lm_head if hasattr(self.model, "lm_head") else self.model.embed_tokens
def verify_logits(x):
@@ -831,20 +846,12 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
verify_buffers = None
drafts = None
- snapshot_bytes = sum(kv.recurrent_state.numel() * 4 + kv.conv_state.numel() * kv.conv_state.element_size()
- for kv in pkv if isinstance(kv, LinearKV))
def set_depth(d):
nonlocal depth, verify_buffers, drafts
depth = d
# Saved draft IDs must survive capture of later draft units.
drafts = [torch.empty_like(nt_buf) for _ in range(d)]
- for kv in pkv:
- if isinstance(kv, LinearKV):
- # snapshot views share one backing slab so the fused kernel can write them
- kv.snap_backing = torch.empty((d,) + tuple(kv.recurrent_state.shape), device=device, dtype=torch.float32)
- kv.conv_snap_backing = torch.empty((d,) + tuple(kv.conv_state.shape), device=device, dtype=kv.conv_state.dtype)
- kv.snapshots = [(kv.snap_backing[s], kv.conv_snap_backing[s]) for s in range(d)]
# static hidden/rope buffers: graphed layers bake their input addresses
verify_buffers = (torch.empty((embeds.shape[0], d + 1, cfg.hidden_size), device=device, dtype=dt), freqs_at(pos, d + 1).clone())
@@ -977,8 +984,7 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
probe[1] += accepts
if probe[0] == 32:
# deepen once acceptance sustains it and a recapture round can amortize
- if (sampling is None and max_length - len(ids) > 512 and 1 + probe[1] / probe[0] >= 2.2
- and 5 * snapshot_bytes < comfy.model_management.get_free_memory(device)):
+ if sampling is None and max_length - len(ids) > 512 and 1 + probe[1] / probe[0] >= 2.2:
comfy.model_prefetch.cleanup_prefetch_queues()
set_depth(5)
probe = None
@@ -990,7 +996,8 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
def init_kv_cache(self, batch, max_cache_len, device, execution_dtype):
model_config = self.model.config
past_key_values = []
- shared = FixedKVBias.shared(max_cache_len, device, execution_dtype)
+ shared = {}
+ tracker = LinearKV.deferred_tracker(device)
for i in range(model_config.num_hidden_layers):
if model_config.layer_types[i] == "linear_attention":
recurrent_state = torch.zeros(
@@ -1002,9 +1009,13 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
[batch, conv_dim, model_config.conv_kernel_size - 1],
device=device, dtype=execution_dtype
)
- past_key_values.append(LinearKV(conv_state, recurrent_state, 0, None, None))
+ kv = LinearKV(conv_state, recurrent_state, 0, None, None)
+ kv.qkv_buf, kv.proj_buf, kv.gates_buf, kv.sumsq_buf = comfy_kitchen.gated_delta_deferred_buffers(
+ batch, conv_dim, model_config.linear_num_value_heads, model_config.linear_num_key_heads, execution_dtype, device)
+ kv.ctl, kv.ctl_tracker = tracker["ctl"], tracker
+ past_key_values.append(kv)
else:
- past_key_values.append(FixedKVBias.zeros(batch, model_config.num_key_value_heads, max_cache_len, model_config.head_dim, device, execution_dtype, shared))
+ past_key_values.append(self.model.layers[i].self_attn.init_kv_cache(batch, max_cache_len, device, execution_dtype, shared))
return past_key_values
# Tokenizer and Text Encoder Wrappers
diff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.py
new file mode 100644
index 000000000..e8d55ab9d
--- /dev/null
+++ b/tests-unit/comfy_test/qwen35_decode_test.py
@@ -0,0 +1,445 @@
+import json
+import types
+
+import pytest
+import torch
+
+from comfy.cli_args import args as cli_args
+
+if not torch.cuda.is_available():
+ cli_args.cpu = True
+
+import comfy.ops # noqa: E402
+from comfy.text_encoders.llama import MLP, RMSNorm # noqa: E402
+
+
+def _mlp(merged):
+ config = types.SimpleNamespace(hidden_size=16, intermediate_size=24, mlp_activation="silu", merged_mlp=merged)
+ mlp = MLP(config, dtype=torch.float32, ops=comfy.ops.manual_cast)
+ for p in mlp.parameters():
+ p.data = torch.randn_like(p)
+ return mlp
+
+
+def _norm(add):
+ norm = RMSNorm(16, eps=1e-6, add=add, dtype=torch.float32)
+ norm.weight.data = torch.randn(16) * 0.1
+ return norm
+
+
+def test_mlp_folded_norm_matches_eager_norm():
+ torch.manual_seed(0)
+ x = torch.randn(1, 3, 16)
+ for merged in (False, True):
+ mlp = _mlp(merged)
+ for add in (False, True):
+ norm = _norm(add)
+ assert torch.equal(mlp(x, norm=norm), mlp(norm(x)))
+
+
+def test_rms_norm_scale_adds_one_without_caching():
+ norm = _norm(True)
+ assert torch.equal(norm.scale(), norm.weight + 1.0)
+ norm.weight.data.add_(1.0)
+ assert torch.equal(norm.scale(), norm.weight + 1.0)
+ plain = _norm(False)
+ assert plain.scale() is plain.weight
+
+
[email protected]("bits", [4, 6, 8, "fp8", "transposed"])
[email protected]("activation", ["rms_norm", "swiglu"])
+def test_input_activation_preserves_linear_and_patched_forward(bits, activation):
+ from comfy.quant_ops import QuantizedTensor
+
+ torch.manual_seed(7)
+ w = torch.randn(32, 256, dtype=torch.bfloat16)
+ if bits in (4, 6):
+ weight = QuantizedTensor.from_float(w, "AsymW4A8Int8Layout", bits=bits)
+ elif bits == "fp8":
+ weight = QuantizedTensor.from_float(w, "TensorCoreFP8Layout")
+ elif bits == "transposed":
+ weight = QuantizedTensor.from_float(w.t().contiguous(), "TensorWiseINT8Layout").t()
+ else:
+ weight = QuantizedTensor.from_float(w, "TensorWiseINT8Layout")
+ x = torch.randn(3, 512 if activation == "swiglu" else 256, dtype=torch.bfloat16)
+ norm_weight = torch.randn(256, dtype=torch.bfloat16)
+ bias = torch.randn(32, dtype=torch.bfloat16)
+ residual = torch.randn(3, 32, dtype=torch.bfloat16)
+ scale = torch.linspace(-1, 2, 32, dtype=torch.bfloat16)
+ if activation == "rms_norm":
+ activated = torch.nn.functional.rms_norm(x, (256,), norm_weight, 1e-6)
+ else:
+ gate, up = x.chunk(2, dim=-1)
+ activated = torch.nn.functional.silu(gate) * up
+ linear = torch.nn.functional.linear(activated, weight, bias)
+ expected = torch.addcmul(residual, linear, scale)
+ got = comfy.ops.linear_input_act_(x, weight, bias, activation, norm_weight, 1e-6, residual, scale)
+ assert torch.equal(got, expected)
+
+ layer = comfy.ops.manual_cast.Linear(256, 32, dtype=torch.bfloat16)
+ layer.weight = torch.nn.Parameter(weight, requires_grad=False)
+ layer.bias = torch.nn.Parameter(bias, requires_grad=False)
+ got = comfy.ops.linear_input_act(layer, x, activation, norm_weight, 1e-6, residual, scale)
+ assert torch.equal(got, expected)
+
+ if bits in ("fp8", "transposed"):
+ layer = comfy.ops.mixed_precision_ops(compute_dtype=torch.bfloat16).Linear(256, 32)
+ layer.weight = torch.nn.Parameter(weight, requires_grad=False)
+ layer.bias = torch.nn.Parameter(bias, requires_grad=False)
+ layer.layout_type = weight._layout_cls
+ layer.quant_format = "float8_e4m3fn" if bits == "fp8" else "int8_tensorwise"
+ layer.pre_quant_scale = torch.linspace(0.25, 1.75, 256, dtype=torch.bfloat16)
+ layer.input_scale = torch.tensor(0.01)
+ expected = torch.addcmul(residual, layer(activated), scale)
+ got = comfy.ops.linear_input_act(layer, x, activation, norm_weight, 1e-6, residual, scale)
+ assert torch.equal(got, expected)
+
+ class Patched(torch.nn.Module):
+ comfy_force_forward = True
+
+ def __init__(self):
+ super().__init__()
+ self.weight = weight
+
+ def forward(self, value):
+ return torch.nn.functional.linear(value, self.weight, bias) + 7
+
+ got = comfy.ops.linear_input_act(Patched(), x, activation, norm_weight, 1e-6, residual, scale)
+ assert torch.equal(got, torch.addcmul(residual, linear + 7, scale))
+
+
+def _fake_kitchen(monkeypatch):
+ import comfy.text_encoders.llama as llama
+ calls = []
+
+ def flash_gqa(q, k, v, kv_lengths, **kwargs):
+ calls.append((q.shape, kv_lengths.clone(), kwargs))
+ return torch.zeros(q.shape[0], q.shape[2], q.shape[1] * q.shape[3], dtype=q.dtype)
+ monkeypatch.setattr(llama.comfy_kitchen, "flash_attention_decode_gqa", flash_gqa)
+ return calls
+
+
+def _gated_attention():
+ from comfy.text_encoders.qwen35 import GatedAttention, Qwen35Config
+ config = Qwen35Config(hidden_size=32, num_attention_heads=2, num_key_value_heads=1, head_dim=256)
+ attn = GatedAttention(config, dtype=torch.bfloat16, ops=comfy.ops.manual_cast)
+ for p in attn.parameters():
+ p.data = torch.randn_like(p) * 0.1
+ return attn
+
+
+def test_int8_cache_requires_cli_flag_and_native_support(monkeypatch):
+ import comfy.text_encoders.qwen35 as qwen35
+ from comfy.text_encoders.llama import FixedKVCache
+ attn = _gated_attention()
+ device = torch.device("cpu")
+ monkeypatch.setattr(qwen35, "int8_decode_is_available", lambda device: True)
+ monkeypatch.setattr(cli_args, "use_ck_attention", True)
+ monkeypatch.setenv("COMFY_INT8_KV", "0")
+ assert attn.kv_cache_type(device, torch.bfloat16) is qwen35.Int8FixedKVCache
+ for dtype, head_dim in ((torch.float16, 256), (torch.float32, 256), (torch.bfloat16, 128)):
+ attn.head_dim = head_dim
+ assert attn.kv_cache_type(device, dtype) is FixedKVCache
+ attn.head_dim = 256
+ monkeypatch.setattr(qwen35, "int8_decode_is_available", lambda device: False)
+ assert attn.kv_cache_type(device, torch.bfloat16) is FixedKVCache
+ monkeypatch.setattr(qwen35, "int8_decode_is_available", lambda device: True)
+ monkeypatch.setattr(cli_args, "use_ck_attention", False)
+ monkeypatch.setenv("COMFY_INT8_KV", "1")
+ assert attn.kv_cache_type(device, torch.bfloat16) is FixedKVCache
+ for dtype, head_dim in ((torch.float16, 256), (torch.float32, 256), (torch.bfloat16, 128)):
+ attn.head_dim = head_dim
+ assert attn.kv_cache_type(device, dtype) is FixedKVCache
+
+
+def test_cache_masks_with_device_length(monkeypatch):
+ from comfy.text_encoders.llama import FixedKVCache
+ calls = _fake_kitchen(monkeypatch)
+ shared = FixedKVCache.shared(1, "cpu")
+ kv = FixedKVCache.zeros(1, 2, 32, 256, "cpu", torch.bfloat16, shared)
+ kv.index = 10
+ kv.prepare(3)
+ assert kv.position.tolist()[:3] == [10, 11, 12] and kv.seqlen.tolist() == [13]
+ xq = torch.randn(1, 4, 3, 256, dtype=torch.bfloat16)
+ xk = torch.randn(1, 2, 3, 256, dtype=torch.bfloat16)
+ kv.write(xk, xk)
+ out = kv.attend(xq, xk, xk)
+ assert out.shape == (1, 3, 1024) and calls[-1][1].tolist() == [13]
+ assert torch.equal(kv.key[:, :, 10:13], xk)
+
+
+def _masked_attention(q, k, v, mask):
+ groups = q.shape[1] // k.shape[1]
+ scores = q.float() @ k.float().repeat_interleave(groups, 1).transpose(-1, -2) * q.shape[-1] ** -0.5 + mask.float()
+ out = scores.softmax(-1) @ v.float().repeat_interleave(groups, 1)
+ return out.transpose(1, 2).reshape(q.shape[0], q.shape[2], -1)
+
+
+def test_cache_mask_matches_attend(monkeypatch):
+ # the additive mask reproduces attend()'s chain staircase over the whole cache
+ from comfy.text_encoders.llama import FixedKVCache
+ torch.manual_seed(0)
+ shared = FixedKVCache.shared(2, "cpu")
+ kv = FixedKVCache.zeros(2, 1, 24, 64, "cpu", torch.float32, shared)
+ kv.key.normal_()
+ kv.value.normal_()
+ kv.index = 9
+ for seq in (1, 3):
+ kv.prepare(seq)
+ xq = torch.randn(2, 2, seq, 64)
+ xk = torch.randn(2, 1, seq, 64)
+ xv = torch.randn(2, 1, seq, 64)
+ kv.write(xk, xv)
+ expected = kv.attend(xq, xk, xv)
+ mask = kv.mask(seq)
+ assert mask.shape == (2, 1, seq, 24) and mask.dtype == torch.float32
+ visible = (mask == 0)[0, 0]
+ assert visible.sum(-1).tolist() == [10 + j for j in range(seq)]
+ torch.testing.assert_close(_masked_attention(xq, kv.key, kv.value, mask), expected, rtol=1e-4, atol=1e-5)
+
+
+def test_deferred_ctl_rows_wait_for_their_previous_copy(monkeypatch):
+ from comfy.text_encoders.qwen35 import LinearKV
+ log = []
+
+ class Event:
+ def __init__(self, i):
+ self.i = i
+
+ def synchronize(self):
+ log.append(("sync", self.i))
+
+ def record(self, stream=None):
+ log.append(("record", self.i))
+
+ monkeypatch.setattr(torch.cuda, "current_stream", lambda device=None: None)
+ tracker = LinearKV.deferred_tracker("cpu")
+ tracker["copied"] = [Event(0), Event(1)]
+ ctl = tracker["ctl"]
+ assert ctl.shape == (2,) # {pending, parity}
+ layers = [LinearKV(torch.zeros(1), torch.zeros(1), 0, None, None, ctl=ctl, ctl_tracker=tracker) for _ in range(2)]
+
+ def step(n, discard=0):
+ for kv in layers:
+ kv.prepare(n)
+ for kv in layers:
+ kv.advance(n)
+ if discard:
+ kv.rollback(discard)
+ return ctl[:2].tolist()
+
+ assert step(10) == [0, 1] # prefill commits directly
+ assert step(3, discard=2) == [0, 0]
+ assert [kv.uncommitted for kv in layers] == [1, 1] and layers[0].index == 11
+ log.clear()
+ assert step(3) == [1, 1]
+ assert log == [("sync", 1), ("record", 1)] # one copy per step, after its row's previous copy
+ assert step(1) == [3, 0]
+
+
+def _tiny_model(monkeypatch, layers=4, head_dim=64):
+ import comfy.text_encoders.qwen35 as qwen35
+ monkeypatch.setitem(qwen35.QWEN35_MODELS, "tiny", dict(vision=dict(hidden_size=32, num_heads=2, intermediate_size=64, depth=1)))
+
+ class Tiny(qwen35.Qwen35):
+ model_type = "tiny"
+ config = dict(vocab_size=97, hidden_size=64, intermediate_size=128, num_hidden_layers=layers, layer_types=qwen35._qwen35_layer_types(layers),
+ num_attention_heads=2, num_key_value_heads=1, head_dim=head_dim, linear_num_key_heads=2, linear_num_value_heads=2,
+ linear_key_head_dim=16, linear_value_head_dim=16, mtp=True, lm_head=True, stop_tokens=[])
+ torch.manual_seed(0)
+ model = Tiny(config, torch.float32, "cpu", comfy.ops.manual_cast)
+ model.requires_grad_(False)
+ for name, p in model.named_parameters():
+ p.data = torch.randn_like(p) * (0.05 if "norm" in name else 0.3)
+ model.model.lm_head.weight.data[:8] *= 4 # a few likely tokens, so some drafts survive
+ return model
+
+
[email protected]("image_prompt", [False, True])
+def test_greedy_mtp_matches_plain_greedy(monkeypatch, image_prompt):
+ # auto and fixed-depth chain verify through the kitchen torch paths: causal GQA, deferred DeltaNet
+ from comfy.text_encoders.llama import FixedKVCache
+ from comfy.text_encoders.qwen_vl import qwen2vl_mrope_position_ids
+ model = _tiny_model(monkeypatch)
+ discards = []
+ rollback = FixedKVCache.rollback
+ monkeypatch.setattr(FixedKVCache, "rollback", lambda self, discard=1: discards.append(discard) or rollback(self, discard))
+ embeds = torch.randn(1, 24 if image_prompt else 5, 64)
+ image_info = [{"type": "image", "index": 3, "size": 16, "extra": torch.tensor([[1, 8, 8]])}] if image_prompt else []
+ positions = qwen2vl_mrope_position_ids(image_info, embeds.shape[1], "cpu")
+ if image_prompt:
+ assert positions.shape == (3, 24) and positions[:, -1].tolist() == [11, 11, 11]
+ plain = model.generate(embeds.clone(), do_sample=False, max_length=40, mtp=False, position_ids=positions)
+ for mtp in (True, 3, 5):
+ discards.clear()
+ assert model.generate(embeds.clone(), do_sample=False, max_length=40, mtp=mtp, position_ids=positions) == plain
+ assert discards and min(discards) < max(discards) # partial acceptance happened
+
+
+class _FakePacked:
+ # Int8DecodeCache boundary with 4-row pages: records the BF16 refreshes it is asked for
+ def __init__(self, batch, heads, capacity, device):
+ self.key = torch.zeros(batch, (capacity + 3) // 4, heads, 4, 256, dtype=torch.int8)
+ self.updates = []
+
+ def update(self, key, value, length, initialize=False):
+ self.updates.append((length.tolist(), initialize))
+
+ def attend(self, query, length):
+ batch, heads, seq, dim = query.shape
+ return torch.zeros(batch, seq, heads * dim, dtype=query.dtype), torch.zeros(batch, heads, seq)
+
+
+def _int8_cache(monkeypatch, capacity=32):
+ import comfy.text_encoders.llama as llama
+ from comfy.text_encoders.llama import FixedKVCache
+ _fake_kitchen(monkeypatch)
+ merges = []
+
+ def step_merge(out, lse, q, k, v, result):
+ merges.append((k, v))
+ return out
+ monkeypatch.setattr(llama, "Int8DecodeCache", _FakePacked, raising=False)
+ monkeypatch.setattr(llama.comfy_kitchen, "flash_attention_decode_step_merge", step_merge)
+ shared = FixedKVCache.shared(1, "cpu")
+ return llama.Int8FixedKVCache.zeros(1, 1, capacity, 256, "cpu", torch.bfloat16, shared), merges
+
+
+def test_int8_cache_merges_current_chain_rows(monkeypatch):
+ kv, merges = _int8_cache(monkeypatch)
+ prefill = torch.randn(1, 1, 10, 256, dtype=torch.bfloat16)
+ kv.prepare(10)
+ kv.append(prefill, prefill)
+ kv.advance(10)
+ kv.prepare(3)
+ xq = torch.randn(1, 2, 3, 256, dtype=torch.bfloat16)
+ xk = torch.randn(1, 1, 3, 256, dtype=torch.bfloat16)
+ xv = torch.randn(1, 1, 3, 256, dtype=torch.bfloat16)
+ kv.write(xk, xv)
+ assert kv.attend(xq, xk, xv).shape == (1, 3, 512)
+ # the committed prefix is quantized once, then refreshed to the same length before attending
+ assert kv.packed.updates == [([10], True), ([10], False)]
+ k, v = merges[-1]
+ assert k is xk and v is xv
+
+
+def test_int8_cache_keeps_bounded_circular_window(monkeypatch):
+ for capacity in (3, 5, 11, 12, 13):
+ kv, _ = _int8_cache(monkeypatch, capacity)
+ assert kv.key.shape == (1, 1, min(capacity, 12), 256)
+ kv, _ = _int8_cache(monkeypatch)
+ # three 4-row pages of BF16 window for a 32-row capacity
+ assert kv.key.shape == (1, 1, 12, 256)
+ prefill = torch.randn(1, 1, 14, 256, dtype=torch.bfloat16)
+ kv.prepare(14)
+ out_k, _ = kv.append(prefill, prefill)
+ assert out_k is prefill and kv.packed.updates == [([14], True)]
+ # only the newest rows survive, absolute row i at slot i % 12
+ assert torch.equal(kv.key[:, :, 2:12], prefill[:, :, 2:12])
+ assert torch.equal(kv.key[:, :, :2], prefill[:, :, 12:14])
+ kv.advance(14)
+ kv.prepare(3)
+ assert kv.position[:3].tolist() == [14, 15, 16]
+ xk = torch.randn(1, 1, 3, 256, dtype=torch.bfloat16)
+ kv.write(xk, xk)
+ kv.attend(torch.randn(1, 2, 3, 256, dtype=torch.bfloat16), xk, xk)
+ assert torch.equal(kv.key[:, :, 2:5], xk) and kv.packed.updates[-1] == ([14], False)
+
+
+def _attention_config(config):
+ return torch.tensor(list(json.dumps(config).encode()), dtype=torch.uint8)
+
+
+def test_checkpoint_and_lora_attention_select_each_kv_layer(monkeypatch):
+ import comfy.ldm.modules.attention as attention
+ import comfy.lora
+ from comfy.model_patcher import ModelPatcher
+ import comfy.text_encoders.llama as llama
+ import comfy.text_encoders.qwen35 as qwen35
+ from comfy.text_encoders.llama import FixedKVCache
+
+ monkeypatch.setattr(attention, "COMFY_KITCHEN_INT8_ATTENTION_IS_AVAILABLE", True)
+ monkeypatch.setattr(qwen35.comfy_kitchen, "int8_attention_is_available", lambda device: True)
+ monkeypatch.setattr(qwen35.comfy_kitchen, "sol_attn_is_available", lambda device: False)
+ monkeypatch.setattr(qwen35, "int8_decode_is_available", lambda device: True)
+ monkeypatch.setattr(llama, "Int8DecodeCache", _FakePacked)
+ monkeypatch.setattr(cli_args, "use_ck_attention", False)
+ model = _tiny_model(monkeypatch, layers=8, head_dim=256)
+ key = "model.layers.3.self_attn.comfy_attention.config"
+ mtp_key = "mtp.layers.0.self_attn.comfy_attention.config"
+ encoded = _attention_config([{"attention": "comfy_kitchen_sol"}, {"attention": "comfy_kitchen_int8"}])
+ sd = model.state_dict()
+ sd[key] = sd[mtp_key] = encoded
+ model.load_state_dict(sd, strict=True)
+ assert torch.equal(model.state_dict()[key], encoded)
+ device = torch.device("cpu")
+
+ def cache_types():
+ kv = model.init_kv_cache(1, 32, device, torch.bfloat16)
+ draft = model.mtp.layers[0].self_attn.init_kv_cache(1, 32, device, torch.bfloat16, {})
+ return type(kv[3]), type(kv[7]), type(draft)
+
+ assert cache_types() == (qwen35.Int8FixedKVCache, FixedKVCache, qwen35.Int8FixedKVCache)
+ monkeypatch.setattr(qwen35.comfy_kitchen, "sol_attn_is_available", lambda device: True)
+ sol = _attention_config({"attention": "comfy_kitchen_sol", "tau": 0.75})
+ patcher = ModelPatcher(model, device, device)
+ patches = comfy.lora.load_lora({key: sol, mtp_key: sol}, {})
+ assert set(patcher.add_patches(patches, strength_patch=0)) == {key, mtp_key}
+ patcher.patch_model(load_weights=False)
+ assert cache_types() == (qwen35.Int8FixedKVCache, FixedKVCache, qwen35.Int8FixedKVCache)
+ patcher.unpatch_model(unpatch_weights=False)
+ patcher.add_patches(patches)
+ patcher.patch_model(load_weights=False)
+ try:
+ # A LoRA's supported setting beats both the checkpoint and the CLI default.
+ monkeypatch.setattr(cli_args, "use_ck_attention", True)
+ assert cache_types() == (FixedKVCache, qwen35.Int8FixedKVCache, FixedKVCache)
+ finally:
+ patcher.unpatch_model(unpatch_weights=False)
+ monkeypatch.setattr(cli_args, "use_ck_attention", False)
+ assert cache_types() == (qwen35.Int8FixedKVCache, FixedKVCache, qwen35.Int8FixedKVCache)
+ # Reloading an unavailable setting clears the previously resolved INT8 selection.
+ monkeypatch.setattr(qwen35.comfy_kitchen, "int8_attention_is_available", lambda device: False)
+ sd[key] = sd[mtp_key] = _attention_config({"attention": "comfy_kitchen_int8"})
+ model.load_state_dict(sd, strict=True)
+ assert cache_types() == (FixedKVCache, FixedKVCache, FixedKVCache)
+
+
[email protected]("seq", [1, 3])
+def test_configured_decode_preserves_causal_visibility(monkeypatch, seq):
+ import comfy.ldm.modules.attention as attention
+ # Make the configured backend observably different from the default backend.
+ @attention.wrap_attn
+ def scaled_attention(q, k, v, heads, *, tau, **kwargs):
+ return tau * attention.attention_pytorch(q, k, v, heads, **kwargs)
+
+ monkeypatch.setattr(attention.comfy_kitchen, "sol_attn_is_available", lambda device: True)
+ monkeypatch.setattr(attention, "attention_comfy_kitchen_sol", scaled_attention)
+ attn = _gated_attention().float()
+ attn.load_state_dict({"comfy_attention.config": _attention_config({"attention": "comfy_kitchen_sol", "tau": 0.25})}, strict=False)
+ kv = attn.init_kv_cache(2, 24, torch.device("cpu"), torch.float32, {})
+ torch.manual_seed(35)
+ prefix, current = torch.randn(2, 9, 32), torch.randn(2, seq, 32)
+ kv.prepare(9)
+ attn(prefix, attention_mask=torch.full((9, 9), -torch.inf).triu(1),
+ freqs_cis=torch.zeros(1, 1, 9, 32, 2, 2), optimized_attention=attention.attention_pytorch, past_key_value=kv)
+ # Repeat after rollback; stale rows past the current chain must not become visible.
+ for start in (9, 4):
+ kv.index = start
+ kv.prepare(seq)
+ out, _ = attn(current, freqs_cis=torch.zeros(1, 1, seq, 32, 2, 2), past_key_value=kv)
+ full = torch.cat((prefix[:, :start], current), dim=1)
+ mask = torch.full((start + seq, start + seq), -torch.inf).triu(1)
+ expected, _ = attn(full, attention_mask=mask, freqs_cis=torch.zeros(1, 1, start + seq, 32, 2, 2),
+ optimized_attention=attention.attention_pytorch)
+ torch.testing.assert_close(out, expected[:, start:])
+
+ # The same checkpoint setting must also affect ordinary prefill/encode.
+ x = torch.randn(1, 3, 32)
+ freqs = torch.zeros(1, 1, 3, 32, 2, 2)
+ mask = torch.full((3, 3), -torch.inf).triu(1)
+ configured, _ = attn(x, attention_mask=mask, freqs_cis=freqs, optimized_attention=attention.attention_pytorch)
+ attn.comfy_attention.load_state_dict({})
+ default, _ = attn(x, attention_mask=mask, freqs_cis=freqs, optimized_attention=attention.attention_pytorch)
+ assert default.abs().max() > 0
+ torch.testing.assert_close(configured, default * 0.25)