The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Performance improvements for qwen 3.x (Part 2) (#17007)
rattus4d7b594592c0
diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.pyindex 77ffd15d9..abed292bf 100644--- a/comfy/ldm/modules/attention.py+++ b/comfy/ldm/modules/attention.py@@ -79,6 +79,7 @@ class ComfyAttention(ConfigurableModule):super().__init__()self.config = Noneself.function = None+ self.method = Nonedef with_config(self, encoded_config):attention = ComfyAttention()@@ -88,6 +89,7 @@ class ComfyAttention(ConfigurableModule):def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs):self.config = Noneself.function = None+ self.method = Nonemetadata = state_dict.pop(prefix + "config", None)if metadata is not None:self.config = json.loads(metadata.numpy().tobytes())@@ -103,6 +105,7 @@ class ComfyAttention(ConfigurableModule):else:logging.warning(f"Ignoring unknown attention method {method!r} for {prefix.rstrip('.')}")if self.function is not None:+ self.method = methodbreaksuper()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs)diff --git a/comfy/ops.py b/comfy/ops.pyindex 07dd351d0..c1c029108 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -304,6 +304,10 @@ def resolve_cast_module_with_vbar(s, dtype, device, bias_dtype, compute_dtype, wweight = params[0]bias = params[1]if prefetch["signature"] is not None:+ # FIXME: Make this non_gemv_specific.+ if (isinstance(s, MixedPrecisionOp) and s.comfy_gemv_optimized and isinstance(weight, QuantizedTensor) and not s._full_precision_mm+ and s.weight_lowvram_function is None and len(s.weight_function) == 0):+ weight = weight.optimize_for_gemv(xfer_dest)s._v_weight = weights._v_bias = biass._v_signature = prefetch["signature"]@@ -968,7 +972,6 @@ from .quant_ops import (QuantizedTensor,QUANT_ALGOS,TensorCoreFP8Layout,- TensorWiseINT8Layout,get_layout_class,)@@ -1022,39 +1025,46 @@ def _fp16_linear_wanted(x):and x.dtype == torch.float16 and x.is_cuda)+def _try_quantized_linear_input_act(x, weight, bias, input_act, act_weight, act_eps,+ residual, residual_scale):+ if comfy.model_management.in_training or not isinstance(weight, QuantizedTensor):+ return NotImplemented+ with _input_act_weight(x, input_act, act_weight, act_eps) as (act_weight, act_eps):+ return weight.linear_input_act(+ x, bias, input_act=input_act, input_act_weight=act_weight, input_act_eps=act_eps,+ residual=residual, residual_scale=residual_scale,+ )++def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0,residual=None, residual_scale=None):run_every_op()weight = linear.weightquantized = isinstance(weight, QuantizedTensor)if (comfy.model_management.in_training or getattr(linear, "comfy_force_forward", False) or getattr(linear, "_full_precision_mm", False)- or not ((quantized and weight._layout_cls == "TensorWiseINT8Layout"- and not getattr(weight._params, "transposed", False))- or (not quantized and _fp16_linear_wanted(x)))):+ or not (quantized or _fp16_linear_wanted(x))):out = linear(_eager_input_act(x, input_act, act_weight, act_eps))return _linear_residual(out, residual, residual_scale)with CastBiasWeightContext(linear, x, offloadable=True,compute_dtype=x.dtype if quantized else None,want_requant=quantized) as (weight, bias):- return linear_input_act_(x, weight, bias, input_act, act_weight, act_eps,- residual, residual_scale, fp16_accumulation=not quantized)+ if not quantized:+ return linear_input_act_(x, weight, bias, input_act, act_weight, act_eps,+ residual, residual_scale)+ out = _try_quantized_linear_input_act(x, weight, bias, input_act, act_weight, act_eps,+ residual, residual_scale)+ if out is not NotImplemented:+ return out++ # A tensor-level fallback cannot replace module-owned input processing.+ out = linear(_eager_input_act(x, input_act, act_weight, act_eps))+ return _linear_residual(out, residual, residual_scale)def linear_input_act_(x, weight, bias, input_act=None, act_weight=None, act_eps=0.0,residual=None, residual_scale=None, *, fp16_accumulation=True):- """``linear(act(x))``, with ``act`` folded into an INT8 activation quantizer.-- An INT8 linear quantizes its input anyway, so an elementwise activation can- ride along inside that kernel instead of writing a full-size intermediate to- HBM and reading it straight back. Worth it for an MLP's down-projection,- where the intermediate is several times the hidden size, and for a pre-norm- block's ``linear(rms_norm(x))`` ("rms_norm", which reads the norm's weight- and eps from act_weight/act_eps).-- With ``residual``/``residual_scale`` the result is the pre-norm block's- addcmul, ``residual + residual_scale * linear(act(x))``, fused into the- INT8 GEMM epilogue where supported.+ """``linear(act(x))`` with optional residual addcmul, fused where the weight supports it.Linear weights are already prepared. An RMSNorm module in act_weight ownsits norm-weight cast context; a tensor uses the supplied act_eps directly.@@ -1062,29 +1072,18 @@ def linear_input_act_(x, weight, bias, input_act=None, act_weight=None, act_eps=if input_act is None and residual is None:return torch.nn.functional.linear(x, weight, bias)- if (comfy.model_management.in_training- or not isinstance(weight, QuantizedTensor)- or weight._layout_cls != "TensorWiseINT8Layout"- or getattr(weight._params, "transposed", False)):- x = _eager_input_act(x, input_act, act_weight, act_eps)- if (not comfy.model_management.in_training- and not isinstance(weight, QuantizedTensor)- and fp16_accumulation and _fp16_linear_wanted(x)):- return quant_ops.ck.fp16_linear(x, weight, bias, residual=residual, residual_scale=residual_scale)- return _linear_residual(torch.nn.functional.linear(x, weight, bias), residual, residual_scale)-- qdata, scale = TensorWiseINT8Layout.get_plain_tensors(weight)- with _input_act_weight(x, input_act, act_weight, act_eps) as (act_weight, act_eps):- return quant_ops.ck.int8_linear(- x, qdata, scale, bias, x.dtype,- convrot=getattr(weight._params, "convrot", False),- convrot_groupsize=getattr(weight._params, "convrot_groupsize", 256),- input_act=input_act,- input_act_weight=act_weight,- input_act_eps=act_eps,- residual=residual,- residual_scale=residual_scale,- )+ out = _try_quantized_linear_input_act(x, weight, bias, input_act, act_weight, act_eps,+ residual, residual_scale)+ if out is not NotImplemented:+ return out+ x = _eager_input_act(x, input_act, act_weight, act_eps)++ if (not comfy.model_management.in_training+ and not isinstance(weight, QuantizedTensor)+ and fp16_accumulation and _fp16_linear_wanted(x)):+ return quant_ops.ck.fp16_linear(x, weight, bias, residual=residual, residual_scale=residual_scale)++ return _linear_residual(torch.nn.functional.linear(x, weight, bias), residual, residual_scale)# Grouped-scale integer weights that run through the shared INT8 GEMM (comfy_kitchen@@ -1397,6 +1396,10 @@ def _quantized_weight_state_dict(module, sd, prefix, extra_quant_conf=None, extrclass MixedPrecisionOp(CastWeightBiasOp):quant_format = None+ # Prefer small-M execution; Kitchen owns the layout and preserves GEMM support.+ # Keep opt-in until extensive testing allows unconditional relayout and a single+ # resident layout/kernel path instead of parallel canonical and packed paths.+ comfy_gemv_optimized = Falsedef can_use_quantized_matmul(self, disabled_formats):return (self.quant_format in QUANT_ALGOSdiff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex 161e65411..542c4590e 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -6,6 +6,7 @@ import mathfrom tqdm import tqdmimport comfy.utilsimport comfy_kitchen+from comfy_kitchen.int8_decode import Int8DecodeCachefrom comfy.ldm.modules.attention import optimized_attention_for_deviceimport comfy.model_management@@ -37,52 +38,31 @@ class FixedKV:self.index -= discard@dataclass-class FixedKVBias(FixedKV):- # full-capacity decode bias [1, 1, rows, capacity], last `seq` rows serve the queries; shared across layers- bias: torch.Tensor = None+class FixedKVState(FixedKV):tracker: dict = Nonedef prepare(self, num_tokens):- if num_tokens == 1 and self.seqlen is not None:- self.seqlen.fill_(self.index + num_tokens)if self.tracker["step"] == (self.index, num_tokens):returnself.tracker["step"] = (self.index, num_tokens)- i = self.index- rows = self.bias.shape[-2]- if num_tokens <= rows:- torch.arange(i, i + rows, out=self.position)- window = self.tracker[num_tokens]- start = i - rows + 1- skip = max(-start, 0)- end = min(i + rows, self.bias.shape[-1])- self.bias[..., :, start + skip:end] = window[:, skip:end - start]- else:- self.bias[..., :, i:i + num_tokens] = 0+ torch.arange(self.index, self.index + self.position.shape[0], out=self.position)+ self.seqlen.fill_(self.index + num_tokens)@staticmethod- def shared(capacity, device, dtype):- # all layers advance in lockstep, so the bias caches share one position/bias/tracker- rows = 6- position = torch.empty((rows,), device=device, dtype=torch.int64)- bias = torch.full((1, 1, rows, capacity), torch.finfo(dtype).min, device=device, dtype=dtype)- tracker = {"step": -1}- # window templates per decode width: row r serves query j = r - (rows - n) and may see slots <= index + j- for n in range(1, rows + 1):- window = torch.full((rows, 2 * rows - 1), torch.finfo(dtype).min, device=device, dtype=dtype)- for r in range(rows):- window[r, :rows + max(r - (rows - n), 0)] = 0- tracker[n] = window- return position, bias, tracker+ def shared(batch, device):+ # all layers advance in lockstep, so the caches share one position/seqlen/tracker+ position = torch.empty((6,), device=device, dtype=torch.int64)+ seqlen = torch.empty((batch,), device=device, dtype=torch.int32)+ return position, seqlen, {"step": -1}++class FixedKVCache(FixedKVState):+ # [B, kv heads, capacity, D] storage with shared decode position state.@classmethoddef zeros(cls, batch, kv_heads, capacity, head_dim, device, dtype, shared):# zero-init: decode attends full capacity with masked tails, 0*0 stays finitekey = torch.zeros((batch, kv_heads, capacity, head_dim), device=device, dtype=dtype)- seqlen = None- if dtype == torch.bfloat16 and head_dim in (128, 256) and comfy_kitchen.flash_attention_decode_is_available(key.device):- seqlen = torch.empty((batch,), device=device, dtype=torch.int32)- return cls(key, torch.zeros_like(key), 0, shared[0], seqlen, shared[1], shared[2])+ return cls(key, torch.zeros_like(key), 0, *shared)def append(self, xk, xv):seq = xk.shape[2]@@ -90,21 +70,56 @@ class FixedKVBias(FixedKV):self.value[:, :, self.index:self.index + seq] = xvreturn self.key[:, :, :self.index + seq], self.value[:, :, :self.index + seq]- def decode(self, xq, xk, xv, num_kv_heads):- # CUDA-graphable: device-side write position and cache length- batch_size, num_heads, seq, head_dim = xq.shape- self.key.index_copy_(2, self.position[:seq], xk)- self.value.index_copy_(2, self.position[:seq], xv)- if seq == 1 and self.seqlen is not None:- out = comfy_kitchen.flash_attention_decode(xq.transpose(1, 2), self.key.transpose(1, 2), self.value.transpose(1, 2), self.seqlen)- return out.reshape(batch_size, seq, num_heads * head_dim)- groups = num_heads // num_kv_heads- q = xq.reshape(batch_size, num_kv_heads, groups, seq, head_dim) * head_dim ** -0.5- bias = self.bias[..., self.bias.shape[-2] - seq:, :].unsqueeze(1)- scores = (q @ self.key.transpose(-1, -2).unsqueeze(2)).add_(bias)- probs = torch.softmax(scores, dim=-1, dtype=torch.float32).to(xq.dtype)- out = probs @ self.value.unsqueeze(2)- return out.permute(0, 3, 1, 2, 4).reshape(batch_size, seq, num_heads * head_dim)+ def write(self, xk, xv):+ # CUDA-graphable: device-side write position+ position = self.position[:xk.shape[2]]+ self.key.index_copy_(2, position, xk)+ self.value.index_copy_(2, position, xv)++ def attend(self, xq, xk, xv):+ return comfy_kitchen.flash_attention_decode_gqa(xq, self.key, self.value, self.seqlen)++ def mask(self, seq):+ # additive [B, 1, seq, capacity] mask with attend()'s visibility, for a generic attention over the+ # whole cache: a chain row sees slots up to its own+ rel = torch.arange(self.key.shape[2], device=self.key.device) - self.seqlen.view(-1, 1, 1, 1)+ visible = rel < torch.arange(1 - seq, 1, device=self.key.device).view(seq, 1)+ return torch.zeros(visible.shape, device=self.key.device, dtype=self.key.dtype).masked_fill_(~visible, float("-inf"))+++class Int8FixedKVCache(FixedKVState):+ # INT8 prefix with a bounded BF16 window for page updates and speculative commits.+ @classmethod+ def zeros(cls, batch, kv_heads, capacity, head_dim, device, dtype, shared):+ packed = Int8DecodeCache(batch, kv_heads, capacity, device)+ # Two pages may be refreshed after a rollback while new rows enter the next page.+ window = min(capacity, 3 * packed.key.shape[3])+ key = torch.zeros((batch, kv_heads, window, head_dim), device=device, dtype=dtype)+ cache = cls(key, torch.zeros_like(key), 0, *shared)+ cache.packed = packed+ return cache++ def append(self, xk, xv):+ # Prefill attends its original tensors; only the quantized prefix survives it.+ seq = xk.shape[2]+ length = torch.full_like(self.position[:1], seq)+ self.packed.update(xk.contiguous(), xv.contiguous(), length, initialize=True)+ start = max(0, seq - self.key.shape[2])+ slots = torch.arange(start, seq, device=xk.device).remainder(self.key.shape[2])+ self.key.index_copy_(2, slots, xk[:, :, start:])+ self.value.index_copy_(2, slots, xv[:, :, start:])+ return xk, xv++ def write(self, xk, xv):+ position = self.position[:xk.shape[2]].remainder(self.key.shape[2])+ self.key.index_copy_(2, position, xk)+ self.value.index_copy_(2, position, xv)++ def attend(self, xq, xk, xv):+ length = self.position[:1]+ self.packed.update(self.key, self.value, length)+ out, lse = self.packed.attend(xq, length)+ return comfy_kitchen.flash_attention_decode_step_merge(out, lse, xq, xk, xv, torch.empty_like(out))@dataclassclass Llama2Config:@@ -510,12 +525,11 @@ class RMSNorm(nn.Module):self.weight = nn.Parameter(torch.empty(dim, device=device, dtype=dtype))self.add = add- def forward(self, x: torch.Tensor):- w = self.weight- if self.add:- w = w + 1.0+ def scale(self):+ return self.weight + 1.0 if self.add else self.weight- return comfy.ldm.common_dit.rms_norm(x, w, self.eps)+ def forward(self, x: torch.Tensor):+ return comfy.ldm.common_dit.rms_norm(x, self.scale(), self.eps)@@ -750,13 +764,19 @@ class MLP(nn.Module):self.activation = lambda a: torch.nn.functional.gelu(a, approximate="tanh")self.merged_input_act = None- def forward(self, x):+ def forward(self, x, norm=None):+ # norm: the block's pre-norm, folded into the merged projection's input quantizer where supportedif self.merged_mlp:- x = self.gate_up_proj(x)+ if norm is not None:+ x = comfy.ops.linear_input_act(self.gate_up_proj, x, "rms_norm", norm.scale(), norm.eps)+ else:+ x = self.gate_up_proj(x)if self.merged_input_act is not None:return comfy.ops.linear_input_act(self.down_proj, x, self.merged_input_act)gate, up = x.chunk(2, dim=-1)return self.down_proj(self.activation(gate) * up)+ if norm is not None:+ x = norm(x)return self.down_proj(self.activation(self.gate_proj(x)) * self.up_proj(x))class TransformerBlock(nn.Module):@@ -914,12 +934,13 @@ class Llama2_(nn.Module):caches = []flash = getattr(comfy_kitchen, "flash_attention_decode_is_available", None)flash_kv = self.fixed_kv and flash is not None and flash(device)+ if flash_kv:+ pos = torch.empty((batch,), device=device, dtype=torch.int64)+ seqlen = torch.zeros((batch,), device=device, dtype=torch.int32)for _ in range(self.config.num_hidden_layers):if flash_kv:key = torch.empty((batch, capacity, self.config.num_key_value_heads, self.config.head_dim), device=device, dtype=dtype)value = torch.empty_like(key)- pos = torch.empty((batch,), device=device, dtype=torch.int64)- seqlen = torch.zeros((batch,), device=device, dtype=torch.int32)caches.append(FixedKV(key, value, 0, pos, seqlen))else:key = torch.empty((batch, self.config.num_key_value_heads, capacity, self.config.head_dim), device=device, dtype=dtype)@@ -941,7 +962,7 @@ class Llama2_(nn.Module):device=device)def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True,- dtype=None, position_ids=None, embeds_info=[], past_key_values=None, input_ids=None,deepstack_embeds=None, visual_pos_masks=None, decode_buffers=None):+ dtype=None, position_ids=None, embeds_info=[], past_key_values=None, input_ids=None,deepstack_embeds=None, visual_pos_masks=None, decode_buffers=None, freqs_cis=None):if embeds is not None:x = embedselse:@@ -956,17 +977,17 @@ class Llama2_(nn.Module):if fixed_kv_decode:attention_mask = None- if position_ids is None:- position_ids = torch.arange(past_len, past_len + seq_len, device=x.device).unsqueeze(0)-- freqs_cis = self.compute_freqs_cis(position_ids, x.device)+ if freqs_cis is None:+ if position_ids is None:+ position_ids = torch.arange(past_len, past_len + seq_len, device=x.device).unsqueeze(0)+ freqs_cis = self.compute_freqs_cis(position_ids, x.device)mask = Noneif attention_mask is not None:mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])).expand(attention_mask.shape[0], 1, seq_len, attention_mask.shape[-1])mask = mask.masked_fill(mask.to(torch.bool), torch.finfo(x.dtype).min / 4)- spec_decode = fixed_kv and any(isinstance(kv, FixedKVBias) for kv in past_key_values) and 2 <= seq_len <= 6 and past_len > 0 and attention_mask is None+ spec_decode = fixed_kv and any(isinstance(kv, FixedKVState) for kv in past_key_values) and 2 <= seq_len <= 6 and past_len > 0 and attention_mask is Noneif seq_len > 1 and not spec_decode: # spec verify: the staircase decode bias is causalcausal_mask = torch.empty(past_len + seq_len, past_len + seq_len, dtype=x.dtype, device=x.device).fill_(torch.finfo(x.dtype).min / 4).triu_(1)if mask is not None:@@ -1011,11 +1032,14 @@ class Llama2_(nn.Module):if past_key_values is not None:past_kv = past_key_values[i] if len(past_key_values) > 0 else []- if fixed_kv:+ if fixed_kv and type(past_kv) is not FixedKV:past_kv.prepare(seq_len)def core():nonlocal x+ # Plain FixedKV shares GPU counters; prepare once, eager or graph-replayed.+ if i == 0 and type(past_kv) is FixedKV:+ past_kv.prepare(seq_len)output, current_kv = layer(x=x,attention_mask=mask,@@ -1174,6 +1198,10 @@ class BaseGenerate:if init_decode_buffers is not None:decode_buffers = init_decode_buffers(embeds.shape[0], device, execution_dtype)decode_tokens = torch.empty((embeds.shape[0], 1), dtype=torch.long, device=device)+ # Precompute decode RoPE once; explicit MRoPE positions keep their existing path.+ rope = None+ if decode_buffers is not None and next_pos is None:+ rope = rope_matrix(self.model.compute_freqs_cis(torch.arange(max_cache_len, device=device, dtype=torch.float).unsqueeze(0), device))penalize = penalty_active(repetition_penalty, presence_penalty)penalty_mask = None@@ -1195,6 +1223,9 @@ class BaseGenerate:extra = {}if decode_buffers is not None:extra["decode_buffers"] = decode_buffers+ if rope is not None and step > 0:+ past_len = self.model.get_past_len(past_key_values)+ extra["freqs_cis"] = rope[:, :, past_len:past_len + 1]if step == 0 and deepstack_embeds is not None:extra["deepstack_embeds"] = deepstack_embedsextra["visual_pos_masks"] = visual_pos_masksdiff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.pyindex 4f0f291e2..c114869bb 100644--- a/comfy/text_encoders/qwen35.py+++ b/comfy/text_encoders/qwen35.py@@ -10,12 +10,13 @@ import comfy.model_managementimport comfy.model_prefetchimport comfy.opsimport comfy_kitchen+from comfy_kitchen.int8_decode import is_available as int8_decode_is_availablefrom comfy.quant_ops import QuantizedTensor-from comfy.ldm.modules.attention import optimized_attention_for_device+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_for_devicefrom comfy import sd1_clipimport comfy.text_encoders.qwen_vl-from .llama import BaseLlama, BaseGenerate, FixedKV, FixedKVBias, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix+from .llama import BaseLlama, BaseGenerate, FixedKV, FixedKVCache, Int8FixedKVCache, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix@dataclass@@ -23,21 +24,53 @@ class LinearKV(FixedKV):# DeltaNet state on the FixedKV interface: key=conv_state, value=recurrent_state (fp32)g_decay: torch.Tensor = Nonedt_bias: torch.Tensor = None- snapshots: list = None # [(recurrent, conv)] taken after step 1, 2, ... of the last verify- last_seq: int = 1- snap_backing: torch.Tensor = None- conv_snap_backing: torch.Tensor = Nonenorm_weight: torch.Tensor = None+ # deferred commit (kitchen gated_delta_decode_deferred): a decode step leaves the state untouched+ # and the next step first replays its ctl[0] still-accepted tokens from the [1 - ctl[1]] side+ # buffers. ctl and its tracker are shared by every layer.+ qkv_buf: torch.Tensor = None+ proj_buf: torch.Tensor = None+ gates_buf: torch.Tensor = None+ sumsq_buf: torch.Tensor = None+ ctl: torch.Tensor = None+ ctl_tracker: dict = None+ uncommitted: int = 0++ @staticmethod+ def deferred_tracker(device):+ # ctl = {pending, parity}: replay the previous step's accepted prefix+ n = comfy_kitchen.gated_delta_ctl_ints+ host = torch.zeros((2, n), dtype=torch.int32)+ # the host rows are pinned and fenced by events only where the copy is asynchronous+ cuda = torch.device(device).type == "cuda"+ return {"ctl": torch.zeros((n,), dtype=torch.int32, device=device), "host": host.pin_memory() if cuda else host,+ "copied": [torch.cuda.Event(), torch.cuda.Event()] if cuda else None, "step": None, "parity": 0}++ def decode_step(self, num_tokens):+ return self.index > 0 and num_tokens <= 6def prepare(self, num_tokens):- pass+ # runs eagerly before the layer (a replayed graph skips its Python): the first linear+ # layer of a step hands the kernels what to commit, every layer records what it leaves+ t = self.ctl_tracker+ if t["step"] != (self.index, num_tokens):+ t["step"] = (self.index, num_tokens)+ t["parity"] ^= 1+ # the pinned row is reused every other step: its previous copy must have landed+ if t["copied"] is not None:+ t["copied"][t["parity"]].synchronize()+ host = t["host"][t["parity"]]+ host[0] = self.uncommitted+ host[1] = t["parity"]+ self.ctl.copy_(host, non_blocking=True)+ if t["copied"] is not None:+ t["copied"][t["parity"]].record(torch.cuda.current_stream(self.ctl.device))+ self.uncommitted = num_tokens if self.decode_step(num_tokens) else 0def rollback(self, discard=1):- # discard the rejected tail: restore the snapshot taken after the last kept token- rec, conv = self.snapshots[self.last_seq - discard - 1]- self.recurrent_state.copy_(rec)- self.conv_state.copy_(conv)+ # nothing was written: the next step replays fewer tokensself.index -= discard+ self.uncommitted -= discard@propertydef conv_state(self):@@ -210,37 +243,26 @@ class GatedDeltaNet(nn.Module):def forward(self, x, past_key_value=None, **kwargs):batch_size, seq_len, _ = x.shape- use_recurrent = (- past_key_value is not None- and past_key_value.index > 0- and seq_len <= 6- )-- fused_available = getattr(comfy_kitchen, "gated_delta_decode_is_available", None)- use_fused = (use_recurrent and fused_available is not None and fused_available(x.device, self.key_head_dim, self.value_head_dim)- and (seq_len == 1 or past_key_value.snap_backing is not None))-# Projections (shared)proj = self.in_proj_qkv(x) # [B, seq_len, conv_dim]z = self.in_proj_z(x)- if use_fused:- # decode: kitchen conv step, then gates + delta rule + gated norm in one kernel- if seq_len > 1:- past_key_value.last_seq = seq_len+ if past_key_value is not None and past_key_value.decode_step(seq_len):+ # decode: conv step into the side buffer, then gates + delta rule + gated norm in one+ # kernel; both first commit the previous step's accepted tokens+ kv = past_key_valuewith comfy.ops.CastBiasWeightContext(self.conv1d, proj, offloadable=True) as (conv_weight, conv_bias):- conv_out = comfy_kitchen.deltanet_conv_step(proj, past_key_value.conv_state, conv_weight, conv_bias,- past_key_value.conv_snap_backing[:seq_len - 1] if seq_len > 1 else None)+ comfy_kitchen.deltanet_conv_step_deferred(proj, kv.conv_state, conv_weight, conv_bias, kv.proj_buf, kv.qkv_buf, kv.ctl)with comfy.ops.CastBiasWeightContext(self.in_proj_a, x, offloadable=True) as (w_a, _), \comfy.ops.CastBiasWeightContext(self.in_proj_b, x, offloadable=True) as (w_b, _):if isinstance(w_a, QuantizedTensor):w_a = w_a.dequantize()if isinstance(w_b, QuantizedTensor):w_b = w_b.dequantize()- core_attn_out = comfy_kitchen.gated_delta_decode_fused(- conv_out, x, w_a, w_b, past_key_value.dt_bias, past_key_value.g_decay, past_key_value.recurrent_state,- self.key_dim, self.num_key_heads, self.key_head_dim ** -0.5, z, past_key_value.norm_weight, self.norm.eps,- past_key_value.snap_backing[:seq_len - 1] if seq_len > 1 else None)+ core_attn_out = comfy_kitchen.gated_delta_decode_deferred(+ x, w_a, w_b, kv.dt_bias, kv.g_decay, kv.recurrent_state,+ self.key_dim, self.num_key_heads, self.key_head_dim ** -0.5, z, kv.norm_weight, self.norm.eps,+ kv.qkv_buf, kv.gates_buf, kv.sumsq_buf, kv.ctl)return self.out_proj(core_attn_out.reshape(batch_size, seq_len, -1)), past_key_valuemixed_qkv = proj.transpose(1, 2) # [B, conv_dim, seq_len]@@ -248,35 +270,21 @@ class GatedDeltaNet(nn.Module):a = self.in_proj_a(x)# Conv1d- if use_recurrent:- # decode: exact-width causal window, weight resolved via the vbar-aware context- combined = torch.cat([past_key_value.conv_state, mixed_qkv], dim=-1)- if seq_len > 1:- past_key_value.last_seq = seq_len- for s in range(seq_len - 1):- past_key_value.snapshots[s][1].copy_(combined[:, :, 1 + s:1 + s + self.conv_kernel_size - 1])- past_key_value.conv_state.copy_(combined[:, :, seq_len:])- with comfy.ops.CastBiasWeightContext(self.conv1d, combined, offloadable=True) as (conv_weight, conv_bias):- mixed_qkv = F.silu(F.conv1d(combined, conv_weight, conv_bias, groups=self.conv1d.groups))- else:- if past_key_value is not None:- conv_state = past_key_value.conv_state- conv_state_init = F.pad(mixed_qkv, (self.conv_kernel_size - mixed_qkv.shape[-1], 0))- conv_state.copy_(conv_state_init[:, :, -conv_state.shape[-1]:])- mixed_qkv = F.silu(self.conv1d(mixed_qkv)[:, :, :seq_len])+ if past_key_value is not None:+ conv_state = past_key_value.conv_state+ conv_state_init = F.pad(mixed_qkv, (self.conv_kernel_size - mixed_qkv.shape[-1], 0))+ conv_state.copy_(conv_state_init[:, :, -conv_state.shape[-1]:])+ mixed_qkv = F.silu(self.conv1d(mixed_qkv)[:, :, :seq_len])# Split QKV and compute beta/gmixed_qkv = mixed_qkv.transpose(1, 2) # [B, seq_len, conv_dim]query, key, value = mixed_qkv.split([self.key_dim, self.key_dim, self.value_dim], dim=-1)- if use_recurrent:- g_decay, dt_bias = past_key_value.g_decay, past_key_value.dt_bias- else:- g_decay = -comfy.model_management.cast_to_device(self.A_log, x.device, torch.float32).exp()- dt_bias = comfy.model_management.cast_to_device(self.dt_bias, x.device, torch.float32)- if past_key_value is not None:- past_key_value.g_decay = g_decay- past_key_value.dt_bias = dt_bias- past_key_value.norm_weight = comfy.model_management.cast_to(self.norm.weight, dtype=x.dtype, device=x.device)+ g_decay = -comfy.model_management.cast_to_device(self.A_log, x.device, torch.float32).exp()+ dt_bias = comfy.model_management.cast_to_device(self.dt_bias, x.device, torch.float32)+ if past_key_value is not None:+ past_key_value.g_decay = g_decay+ past_key_value.dt_bias = dt_bias+ past_key_value.norm_weight = comfy.model_management.cast_to(self.norm.weight, dtype=x.dtype, device=x.device)beta = b.sigmoid()g = g_decay * F.softplus(a.float() + dt_bias)@@ -289,41 +297,17 @@ class GatedDeltaNet(nn.Module):key = key.repeat_interleave(rep, dim=2)# Delta rule- if use_recurrent:- scale = self.key_head_dim ** -0.5- q = F.normalize(query.float(), dim=-1) * scale- k = F.normalize(key.float(), dim=-1)- v = value.float()- beta_t = beta.reshape(batch_size, seq_len, -1)- g_t = g.reshape(batch_size, seq_len, -1).exp()-- # In-place state update: [B, heads, k_dim, v_dim]- recurrent_state = past_key_value.recurrent_state- outs = []- for s in range(seq_len):- recurrent_state.mul_(g_t[:, s, :, None, None])- kv_mem = torch.einsum('bhk,bhkv->bhv', k[:, s], recurrent_state)- delta = (v[:, s] - kv_mem) * beta_t[:, s, :, None]- # rank-1 update via baddbmm_: no materialized [B, H, D, D] outer-product temp- recurrent_state.view(-1, self.key_head_dim, self.value_head_dim).baddbmm_(- k[:, s].reshape(-1, self.key_head_dim, 1), delta.reshape(-1, 1, self.value_head_dim))- outs.append(torch.einsum('bhk,bhkv->bhv', q[:, s], recurrent_state))- if seq_len > 1 and s < seq_len - 1:- past_key_value.snapshots[s][0].copy_(recurrent_state)- core_attn_out = torch.stack(outs, dim=1).to(x.dtype)- present_key_value = past_key_value- else:- core_attn_out, last_recurrent_state = torch_chunk_gated_delta_rule(- query, key, value, g=g, beta=beta,- initial_state=None,- output_final_state=past_key_value is not None,- )+ core_attn_out, last_recurrent_state = torch_chunk_gated_delta_rule(+ query, key, value, g=g, beta=beta,+ initial_state=None,+ output_final_state=past_key_value is not None,+ )- present_key_value = None- if past_key_value is not None:- if last_recurrent_state is not None:- past_key_value.recurrent_state.copy_(last_recurrent_state.to(past_key_value.recurrent_state.dtype))- present_key_value = past_key_value+ present_key_value = None+ if past_key_value is not None:+ if last_recurrent_state is not None:+ past_key_value.recurrent_state.copy_(last_recurrent_state.to(past_key_value.recurrent_state.dtype))+ present_key_value = past_key_value# Gated norm + output projection (shared)core_attn_out = self.norm(core_attn_out.reshape(-1, self.value_head_dim), z.reshape(-1, self.value_head_dim))@@ -366,6 +350,20 @@ class GatedAttention(nn.Module):# QK norms with (1+weight) scalingself.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)+ self.comfy_attention = ComfyAttention()++ def kv_cache_type(self, device, dtype):+ method = self.comfy_attention.method+ if method == "comfy_kitchen_int8" or method is None and comfy.model_management.comfy_kitchen_attention_enabled():+ if dtype == torch.bfloat16 and self.head_dim == 256 and int8_decode_is_available(device):+ return Int8FixedKVCache+ return FixedKVCache++ def init_kv_cache(self, batch, capacity, device, dtype, shared):+ cache_type = self.kv_cache_type(device, dtype)+ if cache_type not in shared:+ shared[cache_type] = cache_type.shared(batch, device)+ return cache_type.zeros(batch, self.num_kv_heads, capacity, self.head_dim, device, dtype, shared[cache_type])def forward(self, x, attention_mask=None, freqs_cis=None, optimized_attention=None, past_key_value=None):batch_size, seq_length, _ = x.shape@@ -389,13 +387,25 @@ class GatedAttention(nn.Module):# KV cachepresent_key_value = past_key_value- if past_key_value is not None and seq_length <= 6 and attention_mask is None:- output = past_key_value.decode(xq, xk, xv, self.num_kv_heads)+ decode = past_key_value is not None and seq_length <= 6 and attention_mask is None+ if decode:+ past_key_value.write(xk, xv)++ if decode and (isinstance(past_key_value, Int8FixedKVCache) or+ self.comfy_attention.method is None and not comfy.model_management.comfy_kitchen_attention_enabled()):+ output = past_key_value.attend(xq, xk, xv)else:- if past_key_value is not None:+ if decode:+ xk, xv = past_key_value.key, past_key_value.value+ attention_mask = past_key_value.mask(seq_length)+ optimized_attention = optimized_attention_for_device(x.device)+ elif past_key_value is not None:xk, xv = past_key_value.append(xk, xv)gqa_kwargs = {"enable_gqa": True} if self.num_heads != self.num_kv_heads else {}- output = optimized_attention(xq, xk, xv, self.num_heads, mask=attention_mask, skip_reshape=True, **gqa_kwargs)+ q, k, v = map(AttentionTensorContainer, (xq, xk, xv))+ del xq, xk, xv+ output = optimized_attention(q, k, v, self.num_heads, mask=attention_mask, skip_reshape=True,+ preferred_attention=self.comfy_attention, **gqa_kwargs)output = output * gate.sigmoid()return self.o_proj(output), present_key_value@@ -423,7 +433,7 @@ class Qwen35TransformerBlock(nn.Module):# in-place into the input buffer so CUDA-graph replays land in the static xx = torch.add(x, h, out=output)- x = torch.add(x, self.mlp(self.post_attention_layernorm(x)), out=output)+ x = torch.add(x, self.mlp(x, norm=self.post_attention_layernorm), out=output)return x, present_key_value@@ -721,6 +731,12 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):self.mtp = Noneif config.mtp:self.mtp = MTPHead(config, device=device, dtype=dtype, ops=operations)+ # Prefer small-M execution for autoregressive linears, including the draft/head.+ decode_units = [self.model.layers] + ([self.mtp] if config.mtp else []) + ([self.model.lm_head] if config.lm_head else [])+ for unit in decode_units:+ for m in unit.modules():+ if isinstance(m, comfy.ops.MixedPrecisionOp):+ m.comfy_gemv_optimized = Truevision_overrides = QWEN35_MODELS.get(self.model_type, {}).get("vision", {})vision_config = {**QWEN35_VISION_DEFAULTS, **vision_overrides, "out_hidden_size": config.hidden_size}self.visual = Qwen35VisionModel(vision_config, device=device, dtype=dtype, ops=operations)@@ -777,8 +793,7 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):cap = embeds.shape[1] + max_length + 7pkv = self.init_kv_cache(embeds.shape[0], cap, device, dt)# repair window: drafting ahead plus a near-full rollback- mtp_kv = FixedKVBias.zeros(embeds.shape[0], cfg.num_key_value_heads, cap, cfg.head_dim, device, dt,- FixedKVBias.shared(cap, device, dt))+ mtp_kv = self.mtp.layers[0].self_attn.init_kv_cache(embeds.shape[0], cap, device, dt, {})head = self.model.lm_head if hasattr(self.model, "lm_head") else self.model.embed_tokensdef verify_logits(x):@@ -831,20 +846,12 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):verify_buffers = Nonedrafts = None- snapshot_bytes = sum(kv.recurrent_state.numel() * 4 + kv.conv_state.numel() * kv.conv_state.element_size()- for kv in pkv if isinstance(kv, LinearKV))def set_depth(d):nonlocal depth, verify_buffers, draftsdepth = d# Saved draft IDs must survive capture of later draft units.drafts = [torch.empty_like(nt_buf) for _ in range(d)]- for kv in pkv:- if isinstance(kv, LinearKV):- # snapshot views share one backing slab so the fused kernel can write them- kv.snap_backing = torch.empty((d,) + tuple(kv.recurrent_state.shape), device=device, dtype=torch.float32)- kv.conv_snap_backing = torch.empty((d,) + tuple(kv.conv_state.shape), device=device, dtype=kv.conv_state.dtype)- kv.snapshots = [(kv.snap_backing[s], kv.conv_snap_backing[s]) for s in range(d)]# static hidden/rope buffers: graphed layers bake their input addressesverify_buffers = (torch.empty((embeds.shape[0], d + 1, cfg.hidden_size), device=device, dtype=dt), freqs_at(pos, d + 1).clone())@@ -977,8 +984,7 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):probe[1] += acceptsif probe[0] == 32:# deepen once acceptance sustains it and a recapture round can amortize- if (sampling is None and max_length - len(ids) > 512 and 1 + probe[1] / probe[0] >= 2.2- and 5 * snapshot_bytes < comfy.model_management.get_free_memory(device)):+ if sampling is None and max_length - len(ids) > 512 and 1 + probe[1] / probe[0] >= 2.2:comfy.model_prefetch.cleanup_prefetch_queues()set_depth(5)probe = None@@ -990,7 +996,8 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):def init_kv_cache(self, batch, max_cache_len, device, execution_dtype):model_config = self.model.configpast_key_values = []- shared = FixedKVBias.shared(max_cache_len, device, execution_dtype)+ shared = {}+ tracker = LinearKV.deferred_tracker(device)for i in range(model_config.num_hidden_layers):if model_config.layer_types[i] == "linear_attention":recurrent_state = torch.zeros(@@ -1002,9 +1009,13 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):[batch, conv_dim, model_config.conv_kernel_size - 1],device=device, dtype=execution_dtype)- past_key_values.append(LinearKV(conv_state, recurrent_state, 0, None, None))+ kv = LinearKV(conv_state, recurrent_state, 0, None, None)+ kv.qkv_buf, kv.proj_buf, kv.gates_buf, kv.sumsq_buf = comfy_kitchen.gated_delta_deferred_buffers(+ batch, conv_dim, model_config.linear_num_value_heads, model_config.linear_num_key_heads, execution_dtype, device)+ kv.ctl, kv.ctl_tracker = tracker["ctl"], tracker+ past_key_values.append(kv)else:- past_key_values.append(FixedKVBias.zeros(batch, model_config.num_key_value_heads, max_cache_len, model_config.head_dim, device, execution_dtype, shared))+ past_key_values.append(self.model.layers[i].self_attn.init_kv_cache(batch, max_cache_len, device, execution_dtype, shared))return past_key_values# Tokenizer and Text Encoder Wrappersdiff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.pynew file mode 100644index 000000000..e8d55ab9d--- /dev/null+++ b/tests-unit/comfy_test/qwen35_decode_test.py@@ -0,0 +1,445 @@+import json+import types++import pytest+import torch++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++import comfy.ops # noqa: E402+from comfy.text_encoders.llama import MLP, RMSNorm # noqa: E402+++def _mlp(merged):+ config = types.SimpleNamespace(hidden_size=16, intermediate_size=24, mlp_activation="silu", merged_mlp=merged)+ mlp = MLP(config, dtype=torch.float32, ops=comfy.ops.manual_cast)+ for p in mlp.parameters():+ p.data = torch.randn_like(p)+ return mlp+++def _norm(add):+ norm = RMSNorm(16, eps=1e-6, add=add, dtype=torch.float32)+ norm.weight.data = torch.randn(16) * 0.1+ return norm+++def test_mlp_folded_norm_matches_eager_norm():+ torch.manual_seed(0)+ x = torch.randn(1, 3, 16)+ for merged in (False, True):+ mlp = _mlp(merged)+ for add in (False, True):+ norm = _norm(add)+ assert torch.equal(mlp(x, norm=norm), mlp(norm(x)))+++def test_rms_norm_scale_adds_one_without_caching():+ norm = _norm(True)+ assert torch.equal(norm.scale(), norm.weight + 1.0)+ norm.weight.data.add_(1.0)+ assert torch.equal(norm.scale(), norm.weight + 1.0)+ plain = _norm(False)+ assert plain.scale() is plain.weight++[email protected]("bits", [4, 6, 8, "fp8", "transposed"])[email protected]("activation", ["rms_norm", "swiglu"])+def test_input_activation_preserves_linear_and_patched_forward(bits, activation):+ from comfy.quant_ops import QuantizedTensor++ torch.manual_seed(7)+ w = torch.randn(32, 256, dtype=torch.bfloat16)+ if bits in (4, 6):+ weight = QuantizedTensor.from_float(w, "AsymW4A8Int8Layout", bits=bits)+ elif bits == "fp8":+ weight = QuantizedTensor.from_float(w, "TensorCoreFP8Layout")+ elif bits == "transposed":+ weight = QuantizedTensor.from_float(w.t().contiguous(), "TensorWiseINT8Layout").t()+ else:+ weight = QuantizedTensor.from_float(w, "TensorWiseINT8Layout")+ x = torch.randn(3, 512 if activation == "swiglu" else 256, dtype=torch.bfloat16)+ norm_weight = torch.randn(256, dtype=torch.bfloat16)+ bias = torch.randn(32, dtype=torch.bfloat16)+ residual = torch.randn(3, 32, dtype=torch.bfloat16)+ scale = torch.linspace(-1, 2, 32, dtype=torch.bfloat16)+ if activation == "rms_norm":+ activated = torch.nn.functional.rms_norm(x, (256,), norm_weight, 1e-6)+ else:+ gate, up = x.chunk(2, dim=-1)+ activated = torch.nn.functional.silu(gate) * up+ linear = torch.nn.functional.linear(activated, weight, bias)+ expected = torch.addcmul(residual, linear, scale)+ got = comfy.ops.linear_input_act_(x, weight, bias, activation, norm_weight, 1e-6, residual, scale)+ assert torch.equal(got, expected)++ layer = comfy.ops.manual_cast.Linear(256, 32, dtype=torch.bfloat16)+ layer.weight = torch.nn.Parameter(weight, requires_grad=False)+ layer.bias = torch.nn.Parameter(bias, requires_grad=False)+ got = comfy.ops.linear_input_act(layer, x, activation, norm_weight, 1e-6, residual, scale)+ assert torch.equal(got, expected)++ if bits in ("fp8", "transposed"):+ layer = comfy.ops.mixed_precision_ops(compute_dtype=torch.bfloat16).Linear(256, 32)+ layer.weight = torch.nn.Parameter(weight, requires_grad=False)+ layer.bias = torch.nn.Parameter(bias, requires_grad=False)+ layer.layout_type = weight._layout_cls+ layer.quant_format = "float8_e4m3fn" if bits == "fp8" else "int8_tensorwise"+ layer.pre_quant_scale = torch.linspace(0.25, 1.75, 256, dtype=torch.bfloat16)+ layer.input_scale = torch.tensor(0.01)+ expected = torch.addcmul(residual, layer(activated), scale)+ got = comfy.ops.linear_input_act(layer, x, activation, norm_weight, 1e-6, residual, scale)+ assert torch.equal(got, expected)++ class Patched(torch.nn.Module):+ comfy_force_forward = True++ def __init__(self):+ super().__init__()+ self.weight = weight++ def forward(self, value):+ return torch.nn.functional.linear(value, self.weight, bias) + 7++ got = comfy.ops.linear_input_act(Patched(), x, activation, norm_weight, 1e-6, residual, scale)+ assert torch.equal(got, torch.addcmul(residual, linear + 7, scale))+++def _fake_kitchen(monkeypatch):+ import comfy.text_encoders.llama as llama+ calls = []++ def flash_gqa(q, k, v, kv_lengths, **kwargs):+ calls.append((q.shape, kv_lengths.clone(), kwargs))+ return torch.zeros(q.shape[0], q.shape[2], q.shape[1] * q.shape[3], dtype=q.dtype)+ monkeypatch.setattr(llama.comfy_kitchen, "flash_attention_decode_gqa", flash_gqa)+ return calls+++def _gated_attention():+ from comfy.text_encoders.qwen35 import GatedAttention, Qwen35Config+ config = Qwen35Config(hidden_size=32, num_attention_heads=2, num_key_value_heads=1, head_dim=256)+ attn = GatedAttention(config, dtype=torch.bfloat16, ops=comfy.ops.manual_cast)+ for p in attn.parameters():+ p.data = torch.randn_like(p) * 0.1+ return attn+++def test_int8_cache_requires_cli_flag_and_native_support(monkeypatch):+ import comfy.text_encoders.qwen35 as qwen35+ from comfy.text_encoders.llama import FixedKVCache+ attn = _gated_attention()+ device = torch.device("cpu")+ monkeypatch.setattr(qwen35, "int8_decode_is_available", lambda device: True)+ monkeypatch.setattr(cli_args, "use_ck_attention", True)+ monkeypatch.setenv("COMFY_INT8_KV", "0")+ assert attn.kv_cache_type(device, torch.bfloat16) is qwen35.Int8FixedKVCache+ for dtype, head_dim in ((torch.float16, 256), (torch.float32, 256), (torch.bfloat16, 128)):+ attn.head_dim = head_dim+ assert attn.kv_cache_type(device, dtype) is FixedKVCache+ attn.head_dim = 256+ monkeypatch.setattr(qwen35, "int8_decode_is_available", lambda device: False)+ assert attn.kv_cache_type(device, torch.bfloat16) is FixedKVCache+ monkeypatch.setattr(qwen35, "int8_decode_is_available", lambda device: True)+ monkeypatch.setattr(cli_args, "use_ck_attention", False)+ monkeypatch.setenv("COMFY_INT8_KV", "1")+ assert attn.kv_cache_type(device, torch.bfloat16) is FixedKVCache+ for dtype, head_dim in ((torch.float16, 256), (torch.float32, 256), (torch.bfloat16, 128)):+ attn.head_dim = head_dim+ assert attn.kv_cache_type(device, dtype) is FixedKVCache+++def test_cache_masks_with_device_length(monkeypatch):+ from comfy.text_encoders.llama import FixedKVCache+ calls = _fake_kitchen(monkeypatch)+ shared = FixedKVCache.shared(1, "cpu")+ kv = FixedKVCache.zeros(1, 2, 32, 256, "cpu", torch.bfloat16, shared)+ kv.index = 10+ kv.prepare(3)+ assert kv.position.tolist()[:3] == [10, 11, 12] and kv.seqlen.tolist() == [13]+ xq = torch.randn(1, 4, 3, 256, dtype=torch.bfloat16)+ xk = torch.randn(1, 2, 3, 256, dtype=torch.bfloat16)+ kv.write(xk, xk)+ out = kv.attend(xq, xk, xk)+ assert out.shape == (1, 3, 1024) and calls[-1][1].tolist() == [13]+ assert torch.equal(kv.key[:, :, 10:13], xk)+++def _masked_attention(q, k, v, mask):+ groups = q.shape[1] // k.shape[1]+ scores = q.float() @ k.float().repeat_interleave(groups, 1).transpose(-1, -2) * q.shape[-1] ** -0.5 + mask.float()+ out = scores.softmax(-1) @ v.float().repeat_interleave(groups, 1)+ return out.transpose(1, 2).reshape(q.shape[0], q.shape[2], -1)+++def test_cache_mask_matches_attend(monkeypatch):+ # the additive mask reproduces attend()'s chain staircase over the whole cache+ from comfy.text_encoders.llama import FixedKVCache+ torch.manual_seed(0)+ shared = FixedKVCache.shared(2, "cpu")+ kv = FixedKVCache.zeros(2, 1, 24, 64, "cpu", torch.float32, shared)+ kv.key.normal_()+ kv.value.normal_()+ kv.index = 9+ for seq in (1, 3):+ kv.prepare(seq)+ xq = torch.randn(2, 2, seq, 64)+ xk = torch.randn(2, 1, seq, 64)+ xv = torch.randn(2, 1, seq, 64)+ kv.write(xk, xv)+ expected = kv.attend(xq, xk, xv)+ mask = kv.mask(seq)+ assert mask.shape == (2, 1, seq, 24) and mask.dtype == torch.float32+ visible = (mask == 0)[0, 0]+ assert visible.sum(-1).tolist() == [10 + j for j in range(seq)]+ torch.testing.assert_close(_masked_attention(xq, kv.key, kv.value, mask), expected, rtol=1e-4, atol=1e-5)+++def test_deferred_ctl_rows_wait_for_their_previous_copy(monkeypatch):+ from comfy.text_encoders.qwen35 import LinearKV+ log = []++ class Event:+ def __init__(self, i):+ self.i = i++ def synchronize(self):+ log.append(("sync", self.i))++ def record(self, stream=None):+ log.append(("record", self.i))++ monkeypatch.setattr(torch.cuda, "current_stream", lambda device=None: None)+ tracker = LinearKV.deferred_tracker("cpu")+ tracker["copied"] = [Event(0), Event(1)]+ ctl = tracker["ctl"]+ assert ctl.shape == (2,) # {pending, parity}+ layers = [LinearKV(torch.zeros(1), torch.zeros(1), 0, None, None, ctl=ctl, ctl_tracker=tracker) for _ in range(2)]++ def step(n, discard=0):+ for kv in layers:+ kv.prepare(n)+ for kv in layers:+ kv.advance(n)+ if discard:+ kv.rollback(discard)+ return ctl[:2].tolist()++ assert step(10) == [0, 1] # prefill commits directly+ assert step(3, discard=2) == [0, 0]+ assert [kv.uncommitted for kv in layers] == [1, 1] and layers[0].index == 11+ log.clear()+ assert step(3) == [1, 1]+ assert log == [("sync", 1), ("record", 1)] # one copy per step, after its row's previous copy+ assert step(1) == [3, 0]+++def _tiny_model(monkeypatch, layers=4, head_dim=64):+ import comfy.text_encoders.qwen35 as qwen35+ monkeypatch.setitem(qwen35.QWEN35_MODELS, "tiny", dict(vision=dict(hidden_size=32, num_heads=2, intermediate_size=64, depth=1)))++ class Tiny(qwen35.Qwen35):+ model_type = "tiny"+ config = dict(vocab_size=97, hidden_size=64, intermediate_size=128, num_hidden_layers=layers, layer_types=qwen35._qwen35_layer_types(layers),+ num_attention_heads=2, num_key_value_heads=1, head_dim=head_dim, linear_num_key_heads=2, linear_num_value_heads=2,+ linear_key_head_dim=16, linear_value_head_dim=16, mtp=True, lm_head=True, stop_tokens=[])+ torch.manual_seed(0)+ model = Tiny(config, torch.float32, "cpu", comfy.ops.manual_cast)+ model.requires_grad_(False)+ for name, p in model.named_parameters():+ p.data = torch.randn_like(p) * (0.05 if "norm" in name else 0.3)+ model.model.lm_head.weight.data[:8] *= 4 # a few likely tokens, so some drafts survive+ return model++[email protected]("image_prompt", [False, True])+def test_greedy_mtp_matches_plain_greedy(monkeypatch, image_prompt):+ # auto and fixed-depth chain verify through the kitchen torch paths: causal GQA, deferred DeltaNet+ from comfy.text_encoders.llama import FixedKVCache+ from comfy.text_encoders.qwen_vl import qwen2vl_mrope_position_ids+ model = _tiny_model(monkeypatch)+ discards = []+ rollback = FixedKVCache.rollback+ monkeypatch.setattr(FixedKVCache, "rollback", lambda self, discard=1: discards.append(discard) or rollback(self, discard))+ embeds = torch.randn(1, 24 if image_prompt else 5, 64)+ image_info = [{"type": "image", "index": 3, "size": 16, "extra": torch.tensor([[1, 8, 8]])}] if image_prompt else []+ positions = qwen2vl_mrope_position_ids(image_info, embeds.shape[1], "cpu")+ if image_prompt:+ assert positions.shape == (3, 24) and positions[:, -1].tolist() == [11, 11, 11]+ plain = model.generate(embeds.clone(), do_sample=False, max_length=40, mtp=False, position_ids=positions)+ for mtp in (True, 3, 5):+ discards.clear()+ assert model.generate(embeds.clone(), do_sample=False, max_length=40, mtp=mtp, position_ids=positions) == plain+ assert discards and min(discards) < max(discards) # partial acceptance happened+++class _FakePacked:+ # Int8DecodeCache boundary with 4-row pages: records the BF16 refreshes it is asked for+ def __init__(self, batch, heads, capacity, device):+ self.key = torch.zeros(batch, (capacity + 3) // 4, heads, 4, 256, dtype=torch.int8)+ self.updates = []++ def update(self, key, value, length, initialize=False):+ self.updates.append((length.tolist(), initialize))++ def attend(self, query, length):+ batch, heads, seq, dim = query.shape+ return torch.zeros(batch, seq, heads * dim, dtype=query.dtype), torch.zeros(batch, heads, seq)+++def _int8_cache(monkeypatch, capacity=32):+ import comfy.text_encoders.llama as llama+ from comfy.text_encoders.llama import FixedKVCache+ _fake_kitchen(monkeypatch)+ merges = []++ def step_merge(out, lse, q, k, v, result):+ merges.append((k, v))+ return out+ monkeypatch.setattr(llama, "Int8DecodeCache", _FakePacked, raising=False)+ monkeypatch.setattr(llama.comfy_kitchen, "flash_attention_decode_step_merge", step_merge)+ shared = FixedKVCache.shared(1, "cpu")+ return llama.Int8FixedKVCache.zeros(1, 1, capacity, 256, "cpu", torch.bfloat16, shared), merges+++def test_int8_cache_merges_current_chain_rows(monkeypatch):+ kv, merges = _int8_cache(monkeypatch)+ prefill = torch.randn(1, 1, 10, 256, dtype=torch.bfloat16)+ kv.prepare(10)+ kv.append(prefill, prefill)+ kv.advance(10)+ kv.prepare(3)+ xq = torch.randn(1, 2, 3, 256, dtype=torch.bfloat16)+ xk = torch.randn(1, 1, 3, 256, dtype=torch.bfloat16)+ xv = torch.randn(1, 1, 3, 256, dtype=torch.bfloat16)+ kv.write(xk, xv)+ assert kv.attend(xq, xk, xv).shape == (1, 3, 512)+ # the committed prefix is quantized once, then refreshed to the same length before attending+ assert kv.packed.updates == [([10], True), ([10], False)]+ k, v = merges[-1]+ assert k is xk and v is xv+++def test_int8_cache_keeps_bounded_circular_window(monkeypatch):+ for capacity in (3, 5, 11, 12, 13):+ kv, _ = _int8_cache(monkeypatch, capacity)+ assert kv.key.shape == (1, 1, min(capacity, 12), 256)+ kv, _ = _int8_cache(monkeypatch)+ # three 4-row pages of BF16 window for a 32-row capacity+ assert kv.key.shape == (1, 1, 12, 256)+ prefill = torch.randn(1, 1, 14, 256, dtype=torch.bfloat16)+ kv.prepare(14)+ out_k, _ = kv.append(prefill, prefill)+ assert out_k is prefill and kv.packed.updates == [([14], True)]+ # only the newest rows survive, absolute row i at slot i % 12+ assert torch.equal(kv.key[:, :, 2:12], prefill[:, :, 2:12])+ assert torch.equal(kv.key[:, :, :2], prefill[:, :, 12:14])+ kv.advance(14)+ kv.prepare(3)+ assert kv.position[:3].tolist() == [14, 15, 16]+ xk = torch.randn(1, 1, 3, 256, dtype=torch.bfloat16)+ kv.write(xk, xk)+ kv.attend(torch.randn(1, 2, 3, 256, dtype=torch.bfloat16), xk, xk)+ assert torch.equal(kv.key[:, :, 2:5], xk) and kv.packed.updates[-1] == ([14], False)+++def _attention_config(config):+ return torch.tensor(list(json.dumps(config).encode()), dtype=torch.uint8)+++def test_checkpoint_and_lora_attention_select_each_kv_layer(monkeypatch):+ import comfy.ldm.modules.attention as attention+ import comfy.lora+ from comfy.model_patcher import ModelPatcher+ import comfy.text_encoders.llama as llama+ import comfy.text_encoders.qwen35 as qwen35+ from comfy.text_encoders.llama import FixedKVCache++ monkeypatch.setattr(attention, "COMFY_KITCHEN_INT8_ATTENTION_IS_AVAILABLE", True)+ monkeypatch.setattr(qwen35.comfy_kitchen, "int8_attention_is_available", lambda device: True)+ monkeypatch.setattr(qwen35.comfy_kitchen, "sol_attn_is_available", lambda device: False)+ monkeypatch.setattr(qwen35, "int8_decode_is_available", lambda device: True)+ monkeypatch.setattr(llama, "Int8DecodeCache", _FakePacked)+ monkeypatch.setattr(cli_args, "use_ck_attention", False)+ model = _tiny_model(monkeypatch, layers=8, head_dim=256)+ key = "model.layers.3.self_attn.comfy_attention.config"+ mtp_key = "mtp.layers.0.self_attn.comfy_attention.config"+ encoded = _attention_config([{"attention": "comfy_kitchen_sol"}, {"attention": "comfy_kitchen_int8"}])+ sd = model.state_dict()+ sd[key] = sd[mtp_key] = encoded+ model.load_state_dict(sd, strict=True)+ assert torch.equal(model.state_dict()[key], encoded)+ device = torch.device("cpu")++ def cache_types():+ kv = model.init_kv_cache(1, 32, device, torch.bfloat16)+ draft = model.mtp.layers[0].self_attn.init_kv_cache(1, 32, device, torch.bfloat16, {})+ return type(kv[3]), type(kv[7]), type(draft)++ assert cache_types() == (qwen35.Int8FixedKVCache, FixedKVCache, qwen35.Int8FixedKVCache)+ monkeypatch.setattr(qwen35.comfy_kitchen, "sol_attn_is_available", lambda device: True)+ sol = _attention_config({"attention": "comfy_kitchen_sol", "tau": 0.75})+ patcher = ModelPatcher(model, device, device)+ patches = comfy.lora.load_lora({key: sol, mtp_key: sol}, {})+ assert set(patcher.add_patches(patches, strength_patch=0)) == {key, mtp_key}+ patcher.patch_model(load_weights=False)+ assert cache_types() == (qwen35.Int8FixedKVCache, FixedKVCache, qwen35.Int8FixedKVCache)+ patcher.unpatch_model(unpatch_weights=False)+ patcher.add_patches(patches)+ patcher.patch_model(load_weights=False)+ try:+ # A LoRA's supported setting beats both the checkpoint and the CLI default.+ monkeypatch.setattr(cli_args, "use_ck_attention", True)+ assert cache_types() == (FixedKVCache, qwen35.Int8FixedKVCache, FixedKVCache)+ finally:+ patcher.unpatch_model(unpatch_weights=False)+ monkeypatch.setattr(cli_args, "use_ck_attention", False)+ assert cache_types() == (qwen35.Int8FixedKVCache, FixedKVCache, qwen35.Int8FixedKVCache)+ # Reloading an unavailable setting clears the previously resolved INT8 selection.+ monkeypatch.setattr(qwen35.comfy_kitchen, "int8_attention_is_available", lambda device: False)+ sd[key] = sd[mtp_key] = _attention_config({"attention": "comfy_kitchen_int8"})+ model.load_state_dict(sd, strict=True)+ assert cache_types() == (FixedKVCache, FixedKVCache, FixedKVCache)++[email protected]("seq", [1, 3])+def test_configured_decode_preserves_causal_visibility(monkeypatch, seq):+ import comfy.ldm.modules.attention as attention+ # Make the configured backend observably different from the default backend.+ @attention.wrap_attn+ def scaled_attention(q, k, v, heads, *, tau, **kwargs):+ return tau * attention.attention_pytorch(q, k, v, heads, **kwargs)++ monkeypatch.setattr(attention.comfy_kitchen, "sol_attn_is_available", lambda device: True)+ monkeypatch.setattr(attention, "attention_comfy_kitchen_sol", scaled_attention)+ attn = _gated_attention().float()+ attn.load_state_dict({"comfy_attention.config": _attention_config({"attention": "comfy_kitchen_sol", "tau": 0.25})}, strict=False)+ kv = attn.init_kv_cache(2, 24, torch.device("cpu"), torch.float32, {})+ torch.manual_seed(35)+ prefix, current = torch.randn(2, 9, 32), torch.randn(2, seq, 32)+ kv.prepare(9)+ attn(prefix, attention_mask=torch.full((9, 9), -torch.inf).triu(1),+ freqs_cis=torch.zeros(1, 1, 9, 32, 2, 2), optimized_attention=attention.attention_pytorch, past_key_value=kv)+ # Repeat after rollback; stale rows past the current chain must not become visible.+ for start in (9, 4):+ kv.index = start+ kv.prepare(seq)+ out, _ = attn(current, freqs_cis=torch.zeros(1, 1, seq, 32, 2, 2), past_key_value=kv)+ full = torch.cat((prefix[:, :start], current), dim=1)+ mask = torch.full((start + seq, start + seq), -torch.inf).triu(1)+ expected, _ = attn(full, attention_mask=mask, freqs_cis=torch.zeros(1, 1, start + seq, 32, 2, 2),+ optimized_attention=attention.attention_pytorch)+ torch.testing.assert_close(out, expected[:, start:])++ # The same checkpoint setting must also affect ordinary prefill/encode.+ x = torch.randn(1, 3, 32)+ freqs = torch.zeros(1, 1, 3, 32, 2, 2)+ mask = torch.full((3, 3), -torch.inf).triu(1)+ configured, _ = attn(x, attention_mask=mask, freqs_cis=freqs, optimized_attention=attention.attention_pytorch)+ attn.comfy_attention.load_state_dict({})+ default, _ = attn(x, attention_mask=mask, freqs_cis=freqs, optimized_attention=attention.attention_pytorch)+ assert default.abs().max() > 0+ torch.testing.assert_close(configured, default * 0.25)