The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Fix quantized text encoder matmul gating and Gemma4 prefill cache positions (#16185)
Jukka Seppänenb7ebfd73c5a5
diff --git a/comfy/ops.py b/comfy/ops.pyindex d9df909bb..d53886de0 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -1342,11 +1342,8 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_preccompute_dtype=compute_dtype,want_requant=want_requant,) as (weight, bias):- if self._full_precision_mm and isinstance(weight, QuantizedTensor):- # cast_bias_weight only dequantizes on a dtype change, which is a- # no-op here when the quantized weight's orig_dtype already equals- # the compute dtype. Force it so the disabled/unsupported-format- # fallback doesn't hand a QuantizedTensor to a plain linear() call.+ if isinstance(weight, QuantizedTensor) and (self._full_precision_mm_config or getattr(self, "quant_format", None) in self._disabled_formats or not weight.layout_cls.supports_fast_matmul()):+ # explicit per-layer full precision, or a format this device can't run: don't reach the fast quantized matmulweight = weight.dequantize()return self._forward(input, weight, bias)diff --git a/comfy/text_encoders/gemma4.py b/comfy/text_encoders/gemma4.pyindex 96a547b60..ee035d522 100644--- a/comfy/text_encoders/gemma4.py+++ b/comfy/text_encoders/gemma4.py@@ -528,17 +528,18 @@ class Gemma4Transformer(nn.Module):and comfy.model_management.is_device_cuda(x.device))decode_bias = Nonedecode_masks = None- if decode:+ if fixed_kv:+ # prefill must advance the device-side write position of the global caches tooprepared = set()for kv in past_key_values:if isinstance(kv, FixedKV) and id(kv.position) not in prepared:kv.prepare(seq_len)prepared.add(id(kv.position))- if mask is not None:- decode_masks = {}- for kv in past_key_values:- if isinstance(kv, FixedKV) and id(kv.position) not in decode_masks:- decode_masks[id(kv.position)] = _fixed_kv_decode_mask(mask, kv, min_val)+ if decode and mask is not None:+ decode_masks = {}+ for kv in past_key_values:+ if isinstance(kv, FixedKV) and id(kv.position) not in decode_masks:+ decode_masks[id(kv.position)] = _fixed_kv_decode_mask(mask, kv, min_val)if compiled_decode:capacities = tuple(sorted({kv.key.shape[2] for kv in past_key_values if isinstance(kv, FixedKV)}))valid = past_len + 1