Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Performance improvements for qwen 3.x (Part 4) - Combine Parallel Linear Weights (#17361)

rattus48b090cb6429
diff --git a/comfy/memory_management.py b/comfy/memory_management.py
index abb387239..0a40b58e3 100644
--- a/comfy/memory_management.py
+++ b/comfy/memory_management.py
@@ -5,7 +5,7 @@ import torch
from typing import NamedTuple
import comfy_aimdo.host_buffer
-from comfy.quant_ops import QuantizedTensor
+from comfy.quant_ops import QuantizedTensor, ck
class TensorFileSlice(NamedTuple):
@@ -190,6 +190,50 @@ def interpret_gathered_like(tensors, gathered):
return dest_views
+def cat_gathered(tensors, gathered):
+ """Concatenate quantized tensors laid out in order in gathered into one, in place.
+
+ Per-row fields are concatenated; every other field and parameter must match and is
+ shared. The codes are moved down over the earlier tensors' metadata, which is saved
+ first and rewritten after the last codes, so the result is laid out exactly like
+ interpret_gathered_like would lay out one tensor of the summed rows. Returns None
+ when the tensors cannot be concatenated."""
+ inner_tensors, ctx = tensors[0].__tensor_flatten__()
+ rows = [t._params.orig_shape[0] for t in tensors]
+ orig_shape = (sum(rows), *tensors[0]._params.orig_shape[1:])
+ for t in tensors[1:]:
+ other_inner, other_ctx = t.__tensor_flatten__()
+ other_ctx["non_tensor_fields"]["orig_shape"] = (rows[0], *t._params.orig_shape[1:])
+ if (other_inner != inner_tensors or other_ctx["layout_cls"] != ctx["layout_cls"]
+ or other_ctx["non_tensor_fields"] != ctx["non_tensor_fields"]):
+ return None
+ fields = {}
+ for attr in inner_tensors[1:]:
+ values = [getattr(t, attr) for t in tensors]
+ if all(v.ndim > 0 and v.shape[0] == n and v.shape[1:] == values[0].shape[1:] and v.dtype == values[0].dtype for v, n in zip(values, rows)):
+ fields[attr] = torch.cat(values)
+ elif all(torch.equal(v, values[0]) for v in values[1:]):
+ fields[attr] = values[0].clone()
+ else:
+ return None
+
+ offset = 0
+ for t in tensors:
+ qdata = t._qdata
+ if qdata.data_ptr() < gathered.data_ptr() + offset:
+ raise RuntimeError("Tensors must sit in order within gathered")
+ ck.memmove(gathered[offset:offset + qdata.nbytes], qdata.view(torch.uint8).view(-1))
+ offset += qdata.nbytes
+ qdata = tensors[0]._qdata
+ actuals = {"_qdata": gathered[:offset].view(qdata.dtype).view(orig_shape[0], *qdata.shape[1:])}
+ offset = vram_aligned_size(actuals["_qdata"])
+ for attr, value in fields.items():
+ actuals[attr] = gathered[offset:offset + value.nbytes].view(value.dtype).view(value.shape)
+ actuals[attr].copy_(value)
+ offset += vram_aligned_size(value)
+ ctx["non_tensor_fields"]["orig_shape"] = orig_shape
+ return QuantizedTensor.__tensor_unflatten__(actuals, ctx, 0, 0)
+
aimdo_enabled = False
extra_ram_release_callback = None
diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py
index 83b92edb5..9105b5a4b 100644
--- a/comfy/model_patcher.py
+++ b/comfy/model_patcher.py
@@ -1986,18 +1986,26 @@ class ModelPatcherDynamic(ModelPatcher):
dynamic_units = list(dynamic_units)
last_dynamic_units = list(last_dynamic_units)
loading_by_module = {entry[-2]: entry for entry in loading}
+
+ def unit_loading(units, loading):
+ for unit in units:
+ unit_modules = unit if isinstance(unit, (list, tuple)) else (unit,)
+ # VBAR is a bump allocator, so loading a CombinedLinear's members back to
+ # back in member order is what makes their weights consecutive.
+ modules = []
+ for root in unit_modules:
+ for module in root.modules():
+ if module not in loading_by_module or module in modules:
+ continue
+ group = getattr(module, "_combined_linear", None)
+ modules.extend(group.linears if group is not None else (module,))
+ for index, module in enumerate(modules):
+ loading.append((*loading_by_module.pop(module), unit if index == len(modules) - 1 else None))
+
loading = []
- for unit in dynamic_units:
- unit_modules = unit if isinstance(unit, (list, tuple)) else (unit,)
- modules = [module for root in unit_modules for module in root.modules() if module in loading_by_module]
- for index, module in enumerate(modules):
- loading.append((*loading_by_module.pop(module), unit if index == len(modules) - 1 else None))
+ unit_loading(dynamic_units, loading)
last_loading = []
- for unit in last_dynamic_units:
- unit_modules = unit if isinstance(unit, (list, tuple)) else (unit,)
- modules = [module for root in unit_modules for module in root.modules() if module in loading_by_module]
- for index, module in enumerate(modules):
- last_loading.append((*loading_by_module.pop(module), unit if index == len(modules) - 1 else None))
+ unit_loading(last_dynamic_units, last_loading)
loading.extend((*entry, None) for entry in loading_by_module.values())
loading.extend(last_loading)
v_block = None
diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py
index 679789f81..c8a9bbcdf 100644
--- a/comfy/model_prefetch.py
+++ b/comfy/model_prefetch.py
@@ -8,10 +8,12 @@ import torch
import comfy_aimdo.malloc_graph
import comfy_aimdo.model_vbar
+import comfy_aimdo.torch
from comfy.cli_args import args
import comfy.memory_management
import comfy.model_management
import comfy.ops
+from comfy.quant_ops import QuantizedTensor
PREFETCH_QUEUES = []
GRAPH_WARMED_MODULES = weakref.WeakSet()
@@ -123,6 +125,28 @@ def pin_modules(comfy_modules, device, dtype=None):
if fully_faulted and dtype is not None:
for comfy_module in comfy_modules:
comfy.ops.resolve_cast_module_with_vbar(comfy_module, dtype, device, dtype, None, False, return_weights=False)
+ # A miss whose modules all have VBAR slots and no runtime weight functions is resolved
+ # here rather than in the forward, so the block is combined and relaid out on its first
+ # residency and the eager warm pass runs the kernels the graph captures on the next hit.
+ elif dtype is not None and all(s._prefetch["signature"] is not None and not getattr(s, param_key + "_function", []) for s in comfy_modules for param_key in ("weight", "bias")):
+ for comfy_module in comfy_modules:
+ comfy.ops.resolve_cast_module_with_vbar(comfy_module, dtype, device, dtype, None, False, return_weights=False)
+ else:
+ return offload_stream, fully_faulted
+ relayout = []
+ for s in comfy_modules:
+ if s._v_relayout:
+ continue
+ s._v_relayout = True
+ group = s._combined_linear
+ if group is not None and s is group.linears[0]:
+ group.prepare(device)
+ relayout.append(group)
+ relayout.append(s)
+ for s in relayout:
+ # FIXME: Make this non_gemv_specific.
+ if isinstance(s, (comfy.ops.MixedPrecisionOp, comfy.ops.CombinedLinear)) and s.comfy_gemv_optimized and isinstance(s._v_weight, QuantizedTensor) and not s._full_precision_mm:
+ s._v_weight = s._v_weight.optimize_for_gemv(comfy_aimdo.torch.aimdo_to_tensor(s._v, device))
return offload_stream, fully_faulted
def cleanup_prefetched_modules(module, comfy_modules):
diff --git a/comfy/ops.py b/comfy/ops.py
index d450860fe..5eecb865c 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -180,9 +180,30 @@ def cast_modules_with_vbar(comfy_modules, dtype, device, bias_dtype, non_blockin
cast_buffer_offset += buffer_size
return buffer
+ group_faults = {}
+ dropped_groups = []
for s in comfy_modules:
- signature = comfy_aimdo.model_vbar.vbar_fault(s._v)
- resident = comfy_aimdo.model_vbar.vbar_signature_compare(signature, s._v_signature)
+ group = s._combined_linear
+ if group is None or group._v_weight is None:
+ signature = comfy_aimdo.model_vbar.vbar_fault(s._v)
+ resident = comfy_aimdo.model_vbar.vbar_signature_compare(signature, s._v_signature)
+ else:
+ # A combined group's members hold each other's bytes, so one member's miss
+ # is a miss for all of them. Fault the peers now; each keeps that fault's
+ # pin as its own, so every member must be in comfy_modules. An uncombined
+ # group's members fault individually above, which also keeps the
+ # single-module cast path from pinning peers it never releases.
+ if s not in group_faults:
+ for m in group.linears:
+ signature = comfy_aimdo.model_vbar.vbar_fault(m._v)
+ group_faults[m] = (signature, comfy_aimdo.model_vbar.vbar_signature_compare(signature, m._v_signature))
+ if not all(group_faults[m][1] for m in group.linears):
+ # Dropped after the loop: a member that already holds its group fault
+ # must not fall into the uncombined branch and take a second pin.
+ dropped_groups.append(group)
+ for m in group.linears:
+ group_faults[m] = (group_faults[m][0], False)
+ signature, resident = group_faults.pop(s)
if return_faulted and (signature is None or not resident):
fully_faulted = False
prefetch = {
@@ -279,6 +300,8 @@ def cast_modules_with_vbar(comfy_modules, dtype, device, bias_dtype, non_blockin
prefetch["needs_cast"] = needs_cast
s._prefetch = prefetch
+ for group in dropped_groups:
+ group._v_weight = None
if return_faulted:
return offload_stream, fully_faulted
return offload_stream
@@ -305,12 +328,9 @@ def resolve_cast_module_with_vbar(s, dtype, device, bias_dtype, compute_dtype, w
weight = params[0]
bias = params[1]
if prefetch["signature"] is not None:
- # FIXME: Make this non_gemv_specific.
- if (isinstance(s, MixedPrecisionOp) and s.comfy_gemv_optimized and isinstance(weight, QuantizedTensor) and not s._full_precision_mm
- and s.weight_lowvram_function is None and len(s.weight_function) == 0):
- weight = weight.optimize_for_gemv(xfer_dest)
s._v_weight = weight
s._v_bias = bias
+ s._v_relayout = False
s._v_signature = prefetch["signature"]
def post_cast(s, param_key, x, dtype, resident, update_weight):
@@ -508,6 +528,7 @@ class CastWeightBiasOp:
comfy_force_forward = False
weight_function = []
bias_function = []
+ _combined_linear = None
class WritableWeight(torch.nn.Module, CastWeightBiasOp):
"""One runtime tensor, either realized or cast from writable file backing."""
@@ -1079,8 +1100,68 @@ def _try_quantized_linear_input_act(x, weight, bias, input_act, act_weight, act_
)
+class CombinedLinear:
+ """A residency-scoped combination of consecutive VBAR linears, not a parameter-owning module.
+
+ All members must belong to one prefetch unit, so they fault, resolve and miss together.
+ Once combined, the members' weights are gone and the model is committed to the
+ combined weight until a miss reloads them. The model owns the split path.
+ """
+ # The model opts the combined weight into small-M layout, as it does per MixedPrecisionOp.
+ comfy_gemv_optimized = False
+
+ @property
+ def _full_precision_mm(self):
+ # The members' flag is live: a text encoder lifts it for generate (use_quantized_matmul)
+ # and restores it for encode, and the combined weight must follow the same contract.
+ return self.linears[0]._full_precision_mm
+
+ def __init__(self, linears):
+ self.linears = linears
+ self._v = None
+ self._v_weight = None
+ for linear in self.linears:
+ linear._combined_linear = self
+
+ def prepare(self, device):
+ """Concatenate the members' resident weights in place; they are gone afterwards."""
+ weights = []
+ for s in self.linears:
+ # FIXME: add bias support if ever needed
+ if (not isinstance(s, MixedPrecisionOp) or not isinstance(s._v_weight, QuantizedTensor)
+ or s._full_precision_mm or s.comfy_force_forward or s.bias is not None):
+ return
+ weights.append(s._v_weight)
+ # Members are loaded back to back, so their VBAR allocations are consecutive in member order.
+ v = (self.linears[0]._v[0], min(s._v[1] for s in self.linears), sum(s._v[2] for s in self.linears))
+ weight = comfy.memory_management.cat_gathered(weights, comfy_aimdo.torch.aimdo_to_tensor(v, device))
+ if weight is None:
+ return
+ self._v = v
+ self._v_weight = weight
+ for s in self.linears:
+ s._v_weight = None
+
+ def forward(self, x):
+ return self.linear_input_act(x, None)
+
+ def linear_input_act(self, x, input_act, act_weight=None, act_eps=0.0,
+ residual=None, residual_scale=None):
+ run_every_op()
+ if self._v_weight is None:
+ return NotImplemented
+ if self._full_precision_mm:
+ x = _eager_input_act(x, input_act, act_weight, act_eps)
+ out = torch.nn.functional.linear(x, self._v_weight.to(dtype=x.dtype).dequantize())
+ return _linear_residual(out, residual, residual_scale)
+ return _try_quantized_linear_input_act(x, self._v_weight, None, input_act, act_weight, act_eps,
+ residual, residual_scale)
+
+
def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0,
residual=None, residual_scale=None):
+ if isinstance(linear, CombinedLinear):
+ return linear.linear_input_act(x, input_act, act_weight, act_eps, residual, residual_scale)
run_every_op()
weight = linear.weight
quantized = isinstance(weight, QuantizedTensor)
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py
index d84d7b7cd..367701904 100644
--- a/comfy/text_encoders/llama.py
+++ b/comfy/text_encoders/llama.py
@@ -677,6 +677,7 @@ class Attention(nn.Module):
self.q_proj = ops.Linear(config.hidden_size, self.inner_size, bias=config.qkv_bias, device=device, dtype=dtype)
self.k_proj = ops.Linear(config.hidden_size, self.kv_size, bias=config.qkv_bias, device=device, dtype=dtype)
self.v_proj = ops.Linear(config.hidden_size, self.kv_size, bias=config.qkv_bias, device=device, dtype=dtype)
+ self.combined_qkv = comfy.ops.CombinedLinear((self.q_proj, self.k_proj, self.v_proj))
self.o_proj = ops.Linear(self.inner_size, config.hidden_size, bias=False, device=device, dtype=dtype)
self.q_norm = None
@@ -701,9 +702,13 @@ class Attention(nn.Module):
if self.merged_qkv:
xq, xk, xv = self.qkv_proj(hidden_states).split((self.inner_size, self.kv_size, self.kv_size), dim=-1)
else:
- xq = self.q_proj(hidden_states)
- xk = self.k_proj(hidden_states)
- xv = self.v_proj(hidden_states)
+ qkv = self.combined_qkv.forward(hidden_states)
+ if qkv is NotImplemented:
+ xq = self.q_proj(hidden_states)
+ xk = self.k_proj(hidden_states)
+ xv = self.v_proj(hidden_states)
+ else:
+ xq, xk, xv = qkv.split((self.inner_size, self.kv_size, self.kv_size), dim=-1)
xq = xq.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2)
xk = xk.view(batch_size, seq_length, self.num_kv_heads, self.head_dim).transpose(1, 2)
@@ -783,6 +788,7 @@ class MLP(nn.Module):
else:
self.gate_proj = ops.Linear(config.hidden_size, intermediate_size, bias=False, device=device, dtype=dtype)
self.up_proj = ops.Linear(config.hidden_size, intermediate_size, bias=False, device=device, dtype=dtype)
+ self.combined_gate_up = comfy.ops.CombinedLinear((self.gate_proj, self.up_proj))
self.down_proj = ops.Linear(intermediate_size, config.hidden_size, bias=False, device=device, dtype=dtype)
if config.mlp_activation == "silu":
self.activation = torch.nn.functional.silu
@@ -793,14 +799,15 @@ class MLP(nn.Module):
def forward(self, x, norm=None):
# norm: the block's pre-norm, folded into the merged projection's input quantizer where supported
- if self.merged_mlp:
- if norm is not None:
- x = comfy.ops.linear_input_act(self.gate_up_proj, x, "rms_norm", norm.scale(), norm.eps)
- else:
- x = self.gate_up_proj(x)
+ gate_up = self.gate_up_proj if self.merged_mlp else self.combined_gate_up
+ if norm is not None:
+ combined = comfy.ops.linear_input_act(gate_up, x, "rms_norm", norm.scale(), norm.eps)
+ else:
+ combined = comfy.ops.linear_input_act(gate_up, x, None)
+ if combined is not NotImplemented:
if self.merged_input_act is not None:
- return comfy.ops.linear_input_act(self.down_proj, x, self.merged_input_act)
- gate, up = x.chunk(2, dim=-1)
+ return comfy.ops.linear_input_act(self.down_proj, combined, self.merged_input_act)
+ gate, up = combined.chunk(2, dim=-1)
return self.down_proj(self.activation(gate) * up)
if norm is not None:
x = norm(x)
diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py
index d4f241c5f..196f5327b 100644
--- a/comfy/text_encoders/qwen35.py
+++ b/comfy/text_encoders/qwen35.py
@@ -350,6 +350,7 @@ class GatedDeltaNet(nn.Module):
self.in_proj_qkv = ops.Linear(hidden, conv_dim, bias=False, device=device, dtype=dtype)
self.in_proj_z = ops.Linear(hidden, value_dim, bias=False, device=device, dtype=dtype)
+ self.combined_qkvz = comfy.ops.CombinedLinear((self.in_proj_qkv, self.in_proj_z))
self.in_proj_b = ops.Linear(hidden, self.num_value_heads, bias=False, device=device, dtype=dtype)
self.in_proj_a = ops.Linear(hidden, self.num_value_heads, bias=False, device=device, dtype=dtype)
self.out_proj = ops.Linear(value_dim, hidden, bias=False, device=device, dtype=dtype)
@@ -366,8 +367,12 @@ class GatedDeltaNet(nn.Module):
batch_size, seq_len, _ = x.shape
# Projections (shared)
- proj = self.in_proj_qkv(x) # [B, seq_len, conv_dim]
- z = self.in_proj_z(x)
+ qkvz = self.combined_qkvz.forward(x)
+ if qkvz is NotImplemented:
+ proj = self.in_proj_qkv(x) # [B, seq_len, conv_dim]
+ z = self.in_proj_z(x)
+ else:
+ proj, z = qkvz.split((self.key_dim * 2 + self.value_dim, self.value_dim), dim=-1)
if past_key_value is not None and past_key_value.decode_step(seq_len):
# decode: conv step into the side buffer, then gates + delta rule + gated norm in one
@@ -467,6 +472,7 @@ class GatedAttention(nn.Module):
self.q_proj = ops.Linear(config.hidden_size, self.inner_size * 2, bias=config.qkv_bias, device=device, dtype=dtype)
self.k_proj = ops.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=config.qkv_bias, device=device, dtype=dtype)
self.v_proj = ops.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=config.qkv_bias, device=device, dtype=dtype)
+ self.combined_qkv = comfy.ops.CombinedLinear((self.q_proj, self.k_proj, self.v_proj))
self.o_proj = ops.Linear(self.inner_size, config.hidden_size, bias=False, device=device, dtype=dtype)
# QK norms with (1+weight) scaling
@@ -491,15 +497,19 @@ class GatedAttention(nn.Module):
batch_size, seq_length, _ = x.shape
# Project Q (with gate), K, V
- qg = self.q_proj(x)
+ qkv = self.combined_qkv.forward(x)
+ if qkv is NotImplemented:
+ qg = self.q_proj(x)
+ xk = self.k_proj(x)
+ xv = self.v_proj(x)
+ else:
+ kv_size = self.num_kv_heads * self.head_dim
+ qg, xk, xv = qkv.split((self.inner_size * 2, kv_size, kv_size), dim=-1)
# Split into query and gate: each is [B, seq, inner_size]
qg = qg.view(batch_size, seq_length, self.num_heads, self.head_dim * 2)
xq, gate = qg[..., :self.head_dim], qg[..., self.head_dim:]
gate = gate.reshape(batch_size, seq_length, -1) # [B, seq, inner_size]
- xk = self.k_proj(x)
- xv = self.v_proj(x)
-
xq = self.q_norm(xq).transpose(1, 2) # [B, heads, seq, head_dim]
xk = self.k_norm(xk.view(batch_size, seq_length, self.num_kv_heads, self.head_dim)).transpose(1, 2)
xv = xv.view(batch_size, seq_length, self.num_kv_heads, self.head_dim).transpose(1, 2)
@@ -859,6 +869,8 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
for m in unit.modules():
if isinstance(m, comfy.ops.MixedPrecisionOp):
m.comfy_gemv_optimized = True
+ if m._combined_linear is not None:
+ m._combined_linear.comfy_gemv_optimized = True
vision_overrides = QWEN35_MODELS.get(self.model_type, {}).get("vision", {})
vision_config = {**QWEN35_VISION_DEFAULTS, **vision_overrides, "out_hidden_size": config.hidden_size}
self.visual = Qwen35VisionModel(vision_config, device=device, dtype=dtype, ops=operations)
diff --git a/tests-unit/comfy_test/combined_linear_test.py b/tests-unit/comfy_test/combined_linear_test.py
new file mode 100644
index 000000000..9b419c162
--- /dev/null
+++ b/tests-unit/comfy_test/combined_linear_test.py
@@ -0,0 +1,184 @@
+import pytest
+import torch
+
+from comfy.cli_args import args
+
+if not torch.cuda.is_available():
+ args.cpu = True
+
+import comfy.lora # noqa: E402
+import comfy.model_patcher # noqa: E402
+import comfy.model_prefetch # noqa: E402
+import comfy.ops # noqa: E402
+from comfy.quant_ops import QuantizedTensor # noqa: E402
+
+
+def test_combined_linear_declines_without_running_split_or_activation():
+ linears = (comfy.ops.manual_cast.Linear(16, 24, bias=False), comfy.ops.manual_cast.Linear(16, 40, bias=False))
+ combined = comfy.ops.CombinedLinear(linears)
+ x = torch.randn(2, 16)
+ original = x.clone()
+ assert combined.forward(x) is NotImplemented
+ assert comfy.ops.linear_input_act(combined, x, "model_owned_activation") is NotImplemented
+ assert torch.equal(x, original)
+
+
+def _gathered(layout, rows, k, **kwargs):
+ weights = [QuantizedTensor.from_float(torch.randn(n, k), layout, **kwargs) for n in rows]
+ gathered = torch.empty(comfy.memory_management.vram_aligned_size(weights), dtype=torch.uint8)
+ views = comfy.memory_management.interpret_gathered_like(weights, gathered)
+ for weight, view in zip(weights, views):
+ inner_tensors, _ = weight.__tensor_flatten__()
+ view.__tensor_flatten__()
+ for attr in inner_tensors:
+ getattr(view, attr).copy_(getattr(weight, attr))
+ return weights, views, gathered
+
+
[email protected]("layout, kwargs", [
+ ("TensorWiseINT8Layout", {"per_channel": True, "convrot": True}),
+ ("AsymW4A8Int8Layout", {}),
+ ("AsymW4A8Int8Layout", {"bits": 6}),
+])
+def test_cat_gathered_concatenates_rows_in_place(layout, kwargs):
+ torch.manual_seed(7)
+ rows = (256, 128, 384)
+ weights, views, gathered = _gathered(layout, rows, 512, **kwargs)
+ combined = comfy.memory_management.cat_gathered(views, gathered)
+ assert tuple(combined.shape) == (sum(rows), 512)
+ assert combined._qdata.data_ptr() == gathered.data_ptr()
+ assert torch.equal(combined.dequantize(), torch.cat([w.dequantize() for w in weights]))
+ # Laid out like one gathered weight of the summed rows, so optimize_for_gemv's storage contract holds.
+ inner_tensors, _ = combined.__tensor_flatten__()
+ like = comfy.memory_management.interpret_gathered_like([combined], gathered)[0]
+ like.__tensor_flatten__()
+ for attr in inner_tensors:
+ assert getattr(combined, attr).data_ptr() == getattr(like, attr).data_ptr()
+ assert comfy.memory_management.vram_aligned_size(combined) <= gathered.numel()
+
+
+def test_combined_linear_follows_members_full_precision():
+ # A text encoder combines during generate (use_quantized_matmul) and then encodes with
+ # full precision restored on the members; the combined weight must dequantize like they do.
+ torch.manual_seed(7)
+ ops = comfy.ops.mixed_precision_ops(full_precision_mm=True)
+ linears = (ops.Linear(512, 128, bias=False), ops.Linear(512, 256, bias=False))
+ combined = comfy.ops.CombinedLinear(linears)
+ _, views, gathered = _gathered("AsymW4A8Int8Layout", (128, 256), 512)
+ combined._v_weight = comfy.memory_management.cat_gathered(views, gathered)
+ for s in linears:
+ s._full_precision_mm = False
+ assert not combined._full_precision_mm
+ for s in linears:
+ s._full_precision_mm = True
+ assert combined._full_precision_mm
+ x = torch.randn(3, 512, dtype=torch.float32)
+ out = combined.forward(x)
+ assert out.dtype == torch.float32
+ assert torch.equal(out, torch.nn.functional.linear(x, combined._v_weight.to(dtype=torch.float32).dequantize()))
+
+
+def test_cat_gathered_declines_incompatible_weights():
+ torch.manual_seed(7)
+ # A tensor-wise scale cannot be shared across rows.
+ weights, views, gathered = _gathered("TensorWiseINT8Layout", (128, 128), 512)
+ assert comfy.memory_management.cat_gathered(views, gathered) is None
+ assert all(torch.equal(v.dequantize(), w.dequantize()) for v, w in zip(views, weights))
+ # Different K.
+ a = _gathered("TensorWiseINT8Layout", (128,), 512, per_channel=True)[0][0]
+ b = _gathered("TensorWiseINT8Layout", (128,), 256, per_channel=True)[0][0]
+ gathered = torch.empty(comfy.memory_management.vram_aligned_size([a, b]), dtype=torch.uint8)
+ views = comfy.memory_management.interpret_gathered_like([a, b], gathered)
+ assert comfy.memory_management.cat_gathered(views, gathered) is None
+ # Out of order.
+ _, views, gathered = _gathered("TensorWiseINT8Layout", (128, 128), 512, per_channel=True)
+ with pytest.raises(RuntimeError):
+ comfy.memory_management.cat_gathered(views[::-1], gathered)
+
+
+def test_int8_combined_on_hit_reloaded_on_partial_miss_including_lora():
+ if not comfy.memory_management.aimdo_enabled or not torch.cuda.is_available():
+ pytest.skip("requires initialized DynamicVRAM")
+
+ torch.manual_seed(53)
+ device = torch.device("cuda:0")
+ ops = comfy.ops.mixed_precision_ops(compute_dtype=torch.bfloat16)
+
+ class Projections(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.a = ops.Linear(512, 96, bias=False)
+ self.b = ops.Linear(512, 160, bias=False)
+ for layer in (self.a, self.b):
+ weight = QuantizedTensor.from_float(
+ torch.randn(layer.out_features, 512, dtype=torch.bfloat16) * 0.03,
+ "TensorWiseINT8Layout", per_channel=True, convrot=True, convrot_groupsize=256,
+ )
+ layer.weight = torch.nn.Parameter(weight, requires_grad=False)
+ layer.quant_format = "int8_tensorwise"
+ layer.layout_type = weight.layout_cls
+ layer.comfy_gemv_optimized = True
+ self.combined = comfy.ops.CombinedLinear((self.a, self.b))
+
+ def get_dynamic_vram__units(self):
+ return [self], []
+
+ def forward(self, x):
+ ab = self.combined.forward(x)
+ return torch.cat((self.a(x), self.b(x)), dim=-1) if ab is NotImplemented else ab
+
+ model = Projections()
+ members = (model.a, model.b)
+ canonical = [(s.weight._qdata.to(device), s.weight.params.scale.flatten().to(device)) for s in members]
+ base = comfy.model_patcher.ModelPatcherDynamic(model, device, torch.device("cpu"))
+ patches = comfy.lora.load_lora({
+ "a.lora_up.weight": torch.randn(96, 4) * 0.1,
+ "a.lora_down.weight": torch.randn(4, 512) * 0.1,
+ }, {"a": "a.weight"})
+ lora = base.clone()
+ lora.add_patches(patches, strength_patch=0.75)
+ x = torch.randn(3, 512, device=device, dtype=torch.bfloat16)
+
+ def assert_unpinned():
+ assert not any(flags & 2 for flags in model.a._v[0].get_residency())
+
+ def step(combined, expected=None):
+ comfy.model_prefetch.pin_modules(members, device, x.dtype)
+ try:
+ assert (model.combined._v_weight is not None) == combined
+ out = model(x)
+ if combined:
+ assert all(s._v_weight is None for s in members)
+ if expected:
+ # The combined weight is the members' canonical bytes, back to back.
+ weight = model.combined._v_weight
+ assert torch.equal(weight._qdata, torch.cat([q for q, _ in expected]))
+ assert torch.equal(weight.params.scale.flatten(), torch.cat([s for _, s in expected]))
+ return out.clone()
+ finally:
+ comfy.model_prefetch.cleanup_prefetched_modules(model, members)
+ assert_unpinned()
+
+ try:
+ base.partially_load(device)
+ # An unprefetched member forward casts only itself and must not leave its peers pinned.
+ split = torch.cat((model.a(x), model.b(x)), dim=-1)
+ assert_unpinned()
+ # The first pin after a load misses, resolves and combines in the same pin.
+ results = [split, step(True, canonical), step(True, canonical)]
+ # A member missing from a combined group reloads the group's canonical bytes and recombines.
+ model.b._v_signature = None
+ results += [step(True, canonical)]
+ assert all(torch.equal(results[0], r) for r in results[1:])
+
+ # The baked patch is what gets combined.
+ lora.partially_load(device)
+ patched = [step(True), step(True)]
+ assert torch.equal(patched[0], patched[1])
+ assert not torch.equal(results[0], patched[0])
+
+ base.partially_load(device)
+ assert torch.equal(results[0], step(True, canonical))
+ finally:
+ torch.cuda.synchronize()
+ base.unpatch_model(unpatch_weights=True)
diff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.py
index efd322865..e7cf9ca81 100644
--- a/tests-unit/comfy_test/qwen35_decode_test.py
+++ b/tests-unit/comfy_test/qwen35_decode_test.py
@@ -9,7 +9,11 @@ from comfy.cli_args import args as cli_args
if not torch.cuda.is_available():
cli_args.cpu = True
+import comfy.lora # noqa: E402
+import comfy.model_patcher # noqa: E402
+import comfy.model_prefetch # noqa: E402
import comfy.ops # noqa: E402
+from comfy.quant_ops import QuantizedTensor # noqa: E402
from comfy.text_encoders.llama import MLP, RMSNorm # noqa: E402
@@ -32,11 +36,124 @@ def test_mlp_folded_norm_matches_eager_norm():
x = torch.randn(1, 3, 16)
for merged in (False, True):
mlp = _mlp(merged)
+ assert set(mlp.state_dict()) == (
+ {"gate_up_proj.weight", "down_proj.weight"} if merged else
+ {"gate_proj.weight", "up_proj.weight", "down_proj.weight"}
+ )
for add in (False, True):
norm = _norm(add)
assert torch.equal(mlp(x, norm=norm), mlp(norm(x)))
+def test_combined_gate_up_declines_without_running_split_or_activation():
+ mlp = _mlp(False)
+ combined = mlp.combined_gate_up
+ x = torch.randn(2, 16)
+ original = x.clone()
+ assert combined.forward(x) is NotImplemented
+ # An unsupported request must escape to the model, not enter the eager
+ # activation fallback (which would reject this activation name).
+ assert comfy.ops.linear_input_act(combined, x, "model_owned_activation") is NotImplemented
+ assert torch.equal(x, original)
+ assert [name for name, _ in mlp.named_modules()] == ["", "gate_proj", "up_proj", "down_proj"]
+
+
+def test_w6a8_packed_and_combined_on_hit_including_lora():
+ if not comfy.memory_management.aimdo_enabled or not torch.cuda.is_available():
+ pytest.skip("requires initialized CUDA DynamicVRAM")
+
+ torch.manual_seed(53)
+ device = torch.device("cuda:0")
+ ops = comfy.ops.mixed_precision_ops(compute_dtype=torch.bfloat16)
+
+ class Projections(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.a = ops.Linear(512, 96, bias=False)
+ self.b = ops.Linear(512, 160, bias=False)
+ self.c = ops.Linear(512, 128, bias=False)
+ for layer in (self.a, self.b, self.c):
+ weight = QuantizedTensor.from_float(
+ torch.randn(layer.out_features, 512, dtype=torch.bfloat16) * 0.03,
+ "AsymW4A8Int8Layout", bits=6, convrot_groupsize=256,
+ )
+ layer.weight = torch.nn.Parameter(weight, requires_grad=False)
+ layer.quant_format = "w6a8_int8"
+ layer.layout_type = weight.layout_cls
+ layer.comfy_gemv_optimized = True
+ self.combined = comfy.ops.CombinedLinear((self.a, self.b))
+ self.combined.comfy_gemv_optimized = True
+
+ def get_dynamic_vram__units(self):
+ return [self], []
+
+ def forward(self, x):
+ ab = self.combined.forward(x)
+ if ab is NotImplemented:
+ ab = torch.cat((self.a(x), self.b(x)), dim=-1)
+ return torch.cat((ab, self.c(x)), dim=-1)
+
+ model = Projections()
+ members = (model.a, model.b, model.c)
+ canonical = [s.weight.dequantize().to(device) for s in members]
+ base = comfy.model_patcher.ModelPatcherDynamic(model, device, torch.device("cpu"))
+ tensors = {}
+ for name in ("a", "c"):
+ tensors[f"{name}.lora_up.weight"] = torch.randn(getattr(model, name).out_features, 4) * 0.1
+ tensors[f"{name}.lora_down.weight"] = torch.randn(4, 512) * 0.1
+ patches = comfy.lora.load_lora(tensors, {"a": "a.weight", "c": "c.weight"})
+ lora = base.clone()
+ assert set(lora.add_patches(patches, strength_patch=0.75)) == {"a.weight", "c.weight"}
+ x = torch.randn(3, 512, device=device, dtype=torch.bfloat16)
+
+ def step(packed, expected=None):
+ comfy.model_prefetch.pin_modules(members, device, x.dtype)
+ try:
+ assert (model.combined._v_weight is not None) == packed
+ out = model(x)
+ if packed:
+ # Combined members give up their own weights; the combined weight and c pack.
+ assert model.a._v_weight is None and model.b._v_weight is None
+ assert model.combined._v_weight.params.mma_packed and model.c._v_weight.params.mma_packed
+ if expected:
+ # Un-rotation on dequantize is a batched matmul whose rounding depends on the row count.
+ torch.testing.assert_close(model.combined._v_weight.dequantize(), torch.cat(expected[:2]))
+ torch.testing.assert_close(model.c._v_weight.dequantize(), expected[2])
+ else:
+ assert not any(s._v_weight.params.mma_packed for s in members)
+ return out.clone()
+ finally:
+ comfy.model_prefetch.cleanup_prefetched_modules(model, members)
+
+ def check(split, packed):
+ # The packed GEMM shares the int8 grid and int32 accumulation; only the fp32 epilogue order differs.
+ torch.testing.assert_close(packed.float(), split.float(), atol=2e-2, rtol=1e-2)
+
+ try:
+ # An unprefetched forward runs the canonical layout; the first pin after a
+ # load misses, resolves, packs and combines in the same pin.
+ base.partially_load(device)
+ split = model(x).clone()
+ packed = [step(True, canonical), step(True, canonical)]
+ assert torch.equal(packed[0], packed[1])
+ check(split, packed[0])
+
+ # The baked patch is what gets packed and combined. A reload dirties the members but
+ # not the group; the pin that precedes every block forward drops the stale group on
+ # the members' miss, so the unprefetched canonical reference drops it here.
+ lora.partially_load(device)
+ model.combined._v_weight = None
+ patched = [model(x).clone(), step(True)]
+ assert not torch.equal(split, patched[0])
+ check(*patched)
+
+ base.partially_load(device)
+ assert torch.equal(packed[0], step(True, canonical))
+ finally:
+ torch.cuda.synchronize()
+ base.unpatch_model(unpatch_weights=True)
+
+
def test_rms_norm_scale_adds_one_without_caching():
norm = _norm(True)
assert torch.equal(norm.scale(), norm.weight + 1.0)