The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Performance improvements for qwen 3.x (Part 4) - Combine Parallel Linear Weights (#17361)
rattus48b090cb6429
diff --git a/comfy/memory_management.py b/comfy/memory_management.pyindex abb387239..0a40b58e3 100644--- a/comfy/memory_management.py+++ b/comfy/memory_management.py@@ -5,7 +5,7 @@ import torchfrom typing import NamedTupleimport comfy_aimdo.host_buffer-from comfy.quant_ops import QuantizedTensor+from comfy.quant_ops import QuantizedTensor, ckclass TensorFileSlice(NamedTuple):@@ -190,6 +190,50 @@ def interpret_gathered_like(tensors, gathered):return dest_views+def cat_gathered(tensors, gathered):+ """Concatenate quantized tensors laid out in order in gathered into one, in place.++ Per-row fields are concatenated; every other field and parameter must match and is+ shared. The codes are moved down over the earlier tensors' metadata, which is saved+ first and rewritten after the last codes, so the result is laid out exactly like+ interpret_gathered_like would lay out one tensor of the summed rows. Returns None+ when the tensors cannot be concatenated."""+ inner_tensors, ctx = tensors[0].__tensor_flatten__()+ rows = [t._params.orig_shape[0] for t in tensors]+ orig_shape = (sum(rows), *tensors[0]._params.orig_shape[1:])+ for t in tensors[1:]:+ other_inner, other_ctx = t.__tensor_flatten__()+ other_ctx["non_tensor_fields"]["orig_shape"] = (rows[0], *t._params.orig_shape[1:])+ if (other_inner != inner_tensors or other_ctx["layout_cls"] != ctx["layout_cls"]+ or other_ctx["non_tensor_fields"] != ctx["non_tensor_fields"]):+ return None+ fields = {}+ for attr in inner_tensors[1:]:+ values = [getattr(t, attr) for t in tensors]+ if all(v.ndim > 0 and v.shape[0] == n and v.shape[1:] == values[0].shape[1:] and v.dtype == values[0].dtype for v, n in zip(values, rows)):+ fields[attr] = torch.cat(values)+ elif all(torch.equal(v, values[0]) for v in values[1:]):+ fields[attr] = values[0].clone()+ else:+ return None++ offset = 0+ for t in tensors:+ qdata = t._qdata+ if qdata.data_ptr() < gathered.data_ptr() + offset:+ raise RuntimeError("Tensors must sit in order within gathered")+ ck.memmove(gathered[offset:offset + qdata.nbytes], qdata.view(torch.uint8).view(-1))+ offset += qdata.nbytes+ qdata = tensors[0]._qdata+ actuals = {"_qdata": gathered[:offset].view(qdata.dtype).view(orig_shape[0], *qdata.shape[1:])}+ offset = vram_aligned_size(actuals["_qdata"])+ for attr, value in fields.items():+ actuals[attr] = gathered[offset:offset + value.nbytes].view(value.dtype).view(value.shape)+ actuals[attr].copy_(value)+ offset += vram_aligned_size(value)+ ctx["non_tensor_fields"]["orig_shape"] = orig_shape+ return QuantizedTensor.__tensor_unflatten__(actuals, ctx, 0, 0)+aimdo_enabled = Falseextra_ram_release_callback = Nonediff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex 83b92edb5..9105b5a4b 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -1986,18 +1986,26 @@ class ModelPatcherDynamic(ModelPatcher):dynamic_units = list(dynamic_units)last_dynamic_units = list(last_dynamic_units)loading_by_module = {entry[-2]: entry for entry in loading}++ def unit_loading(units, loading):+ for unit in units:+ unit_modules = unit if isinstance(unit, (list, tuple)) else (unit,)+ # VBAR is a bump allocator, so loading a CombinedLinear's members back to+ # back in member order is what makes their weights consecutive.+ modules = []+ for root in unit_modules:+ for module in root.modules():+ if module not in loading_by_module or module in modules:+ continue+ group = getattr(module, "_combined_linear", None)+ modules.extend(group.linears if group is not None else (module,))+ for index, module in enumerate(modules):+ loading.append((*loading_by_module.pop(module), unit if index == len(modules) - 1 else None))+loading = []- for unit in dynamic_units:- unit_modules = unit if isinstance(unit, (list, tuple)) else (unit,)- modules = [module for root in unit_modules for module in root.modules() if module in loading_by_module]- for index, module in enumerate(modules):- loading.append((*loading_by_module.pop(module), unit if index == len(modules) - 1 else None))+ unit_loading(dynamic_units, loading)last_loading = []- for unit in last_dynamic_units:- unit_modules = unit if isinstance(unit, (list, tuple)) else (unit,)- modules = [module for root in unit_modules for module in root.modules() if module in loading_by_module]- for index, module in enumerate(modules):- last_loading.append((*loading_by_module.pop(module), unit if index == len(modules) - 1 else None))+ unit_loading(last_dynamic_units, last_loading)loading.extend((*entry, None) for entry in loading_by_module.values())loading.extend(last_loading)v_block = Nonediff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.pyindex 679789f81..c8a9bbcdf 100644--- a/comfy/model_prefetch.py+++ b/comfy/model_prefetch.py@@ -8,10 +8,12 @@ import torchimport comfy_aimdo.malloc_graphimport comfy_aimdo.model_vbar+import comfy_aimdo.torchfrom comfy.cli_args import argsimport comfy.memory_managementimport comfy.model_managementimport comfy.ops+from comfy.quant_ops import QuantizedTensorPREFETCH_QUEUES = []GRAPH_WARMED_MODULES = weakref.WeakSet()@@ -123,6 +125,28 @@ def pin_modules(comfy_modules, device, dtype=None):if fully_faulted and dtype is not None:for comfy_module in comfy_modules:comfy.ops.resolve_cast_module_with_vbar(comfy_module, dtype, device, dtype, None, False, return_weights=False)+ # A miss whose modules all have VBAR slots and no runtime weight functions is resolved+ # here rather than in the forward, so the block is combined and relaid out on its first+ # residency and the eager warm pass runs the kernels the graph captures on the next hit.+ elif dtype is not None and all(s._prefetch["signature"] is not None and not getattr(s, param_key + "_function", []) for s in comfy_modules for param_key in ("weight", "bias")):+ for comfy_module in comfy_modules:+ comfy.ops.resolve_cast_module_with_vbar(comfy_module, dtype, device, dtype, None, False, return_weights=False)+ else:+ return offload_stream, fully_faulted+ relayout = []+ for s in comfy_modules:+ if s._v_relayout:+ continue+ s._v_relayout = True+ group = s._combined_linear+ if group is not None and s is group.linears[0]:+ group.prepare(device)+ relayout.append(group)+ relayout.append(s)+ for s in relayout:+ # FIXME: Make this non_gemv_specific.+ if isinstance(s, (comfy.ops.MixedPrecisionOp, comfy.ops.CombinedLinear)) and s.comfy_gemv_optimized and isinstance(s._v_weight, QuantizedTensor) and not s._full_precision_mm:+ s._v_weight = s._v_weight.optimize_for_gemv(comfy_aimdo.torch.aimdo_to_tensor(s._v, device))return offload_stream, fully_faulteddef cleanup_prefetched_modules(module, comfy_modules):diff --git a/comfy/ops.py b/comfy/ops.pyindex d450860fe..5eecb865c 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -180,9 +180,30 @@ def cast_modules_with_vbar(comfy_modules, dtype, device, bias_dtype, non_blockincast_buffer_offset += buffer_sizereturn buffer+ group_faults = {}+ dropped_groups = []for s in comfy_modules:- signature = comfy_aimdo.model_vbar.vbar_fault(s._v)- resident = comfy_aimdo.model_vbar.vbar_signature_compare(signature, s._v_signature)+ group = s._combined_linear+ if group is None or group._v_weight is None:+ signature = comfy_aimdo.model_vbar.vbar_fault(s._v)+ resident = comfy_aimdo.model_vbar.vbar_signature_compare(signature, s._v_signature)+ else:+ # A combined group's members hold each other's bytes, so one member's miss+ # is a miss for all of them. Fault the peers now; each keeps that fault's+ # pin as its own, so every member must be in comfy_modules. An uncombined+ # group's members fault individually above, which also keeps the+ # single-module cast path from pinning peers it never releases.+ if s not in group_faults:+ for m in group.linears:+ signature = comfy_aimdo.model_vbar.vbar_fault(m._v)+ group_faults[m] = (signature, comfy_aimdo.model_vbar.vbar_signature_compare(signature, m._v_signature))+ if not all(group_faults[m][1] for m in group.linears):+ # Dropped after the loop: a member that already holds its group fault+ # must not fall into the uncombined branch and take a second pin.+ dropped_groups.append(group)+ for m in group.linears:+ group_faults[m] = (group_faults[m][0], False)+ signature, resident = group_faults.pop(s)if return_faulted and (signature is None or not resident):fully_faulted = Falseprefetch = {@@ -279,6 +300,8 @@ def cast_modules_with_vbar(comfy_modules, dtype, device, bias_dtype, non_blockinprefetch["needs_cast"] = needs_casts._prefetch = prefetch+ for group in dropped_groups:+ group._v_weight = Noneif return_faulted:return offload_stream, fully_faultedreturn offload_stream@@ -305,12 +328,9 @@ def resolve_cast_module_with_vbar(s, dtype, device, bias_dtype, compute_dtype, wweight = params[0]bias = params[1]if prefetch["signature"] is not None:- # FIXME: Make this non_gemv_specific.- if (isinstance(s, MixedPrecisionOp) and s.comfy_gemv_optimized and isinstance(weight, QuantizedTensor) and not s._full_precision_mm- and s.weight_lowvram_function is None and len(s.weight_function) == 0):- weight = weight.optimize_for_gemv(xfer_dest)s._v_weight = weights._v_bias = bias+ s._v_relayout = Falses._v_signature = prefetch["signature"]def post_cast(s, param_key, x, dtype, resident, update_weight):@@ -508,6 +528,7 @@ class CastWeightBiasOp:comfy_force_forward = Falseweight_function = []bias_function = []+ _combined_linear = Noneclass WritableWeight(torch.nn.Module, CastWeightBiasOp):"""One runtime tensor, either realized or cast from writable file backing."""@@ -1079,8 +1100,68 @@ def _try_quantized_linear_input_act(x, weight, bias, input_act, act_weight, act_)+class CombinedLinear:+ """A residency-scoped combination of consecutive VBAR linears, not a parameter-owning module.++ All members must belong to one prefetch unit, so they fault, resolve and miss together.+ Once combined, the members' weights are gone and the model is committed to the+ combined weight until a miss reloads them. The model owns the split path.+ """+ # The model opts the combined weight into small-M layout, as it does per MixedPrecisionOp.+ comfy_gemv_optimized = False++ @property+ def _full_precision_mm(self):+ # The members' flag is live: a text encoder lifts it for generate (use_quantized_matmul)+ # and restores it for encode, and the combined weight must follow the same contract.+ return self.linears[0]._full_precision_mm++ def __init__(self, linears):+ self.linears = linears+ self._v = None+ self._v_weight = None+ for linear in self.linears:+ linear._combined_linear = self++ def prepare(self, device):+ """Concatenate the members' resident weights in place; they are gone afterwards."""+ weights = []+ for s in self.linears:+ # FIXME: add bias support if ever needed+ if (not isinstance(s, MixedPrecisionOp) or not isinstance(s._v_weight, QuantizedTensor)+ or s._full_precision_mm or s.comfy_force_forward or s.bias is not None):+ return+ weights.append(s._v_weight)+ # Members are loaded back to back, so their VBAR allocations are consecutive in member order.+ v = (self.linears[0]._v[0], min(s._v[1] for s in self.linears), sum(s._v[2] for s in self.linears))+ weight = comfy.memory_management.cat_gathered(weights, comfy_aimdo.torch.aimdo_to_tensor(v, device))+ if weight is None:+ return+ self._v = v+ self._v_weight = weight+ for s in self.linears:+ s._v_weight = None++ def forward(self, x):+ return self.linear_input_act(x, None)++ def linear_input_act(self, x, input_act, act_weight=None, act_eps=0.0,+ residual=None, residual_scale=None):+ run_every_op()+ if self._v_weight is None:+ return NotImplemented+ if self._full_precision_mm:+ x = _eager_input_act(x, input_act, act_weight, act_eps)+ out = torch.nn.functional.linear(x, self._v_weight.to(dtype=x.dtype).dequantize())+ return _linear_residual(out, residual, residual_scale)+ return _try_quantized_linear_input_act(x, self._v_weight, None, input_act, act_weight, act_eps,+ residual, residual_scale)++def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0,residual=None, residual_scale=None):+ if isinstance(linear, CombinedLinear):+ return linear.linear_input_act(x, input_act, act_weight, act_eps, residual, residual_scale)run_every_op()weight = linear.weightquantized = isinstance(weight, QuantizedTensor)diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex d84d7b7cd..367701904 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -677,6 +677,7 @@ class Attention(nn.Module):self.q_proj = ops.Linear(config.hidden_size, self.inner_size, bias=config.qkv_bias, device=device, dtype=dtype)self.k_proj = ops.Linear(config.hidden_size, self.kv_size, bias=config.qkv_bias, device=device, dtype=dtype)self.v_proj = ops.Linear(config.hidden_size, self.kv_size, bias=config.qkv_bias, device=device, dtype=dtype)+ self.combined_qkv = comfy.ops.CombinedLinear((self.q_proj, self.k_proj, self.v_proj))self.o_proj = ops.Linear(self.inner_size, config.hidden_size, bias=False, device=device, dtype=dtype)self.q_norm = None@@ -701,9 +702,13 @@ class Attention(nn.Module):if self.merged_qkv:xq, xk, xv = self.qkv_proj(hidden_states).split((self.inner_size, self.kv_size, self.kv_size), dim=-1)else:- xq = self.q_proj(hidden_states)- xk = self.k_proj(hidden_states)- xv = self.v_proj(hidden_states)+ qkv = self.combined_qkv.forward(hidden_states)+ if qkv is NotImplemented:+ xq = self.q_proj(hidden_states)+ xk = self.k_proj(hidden_states)+ xv = self.v_proj(hidden_states)+ else:+ xq, xk, xv = qkv.split((self.inner_size, self.kv_size, self.kv_size), dim=-1)xq = xq.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2)xk = xk.view(batch_size, seq_length, self.num_kv_heads, self.head_dim).transpose(1, 2)@@ -783,6 +788,7 @@ class MLP(nn.Module):else:self.gate_proj = ops.Linear(config.hidden_size, intermediate_size, bias=False, device=device, dtype=dtype)self.up_proj = ops.Linear(config.hidden_size, intermediate_size, bias=False, device=device, dtype=dtype)+ self.combined_gate_up = comfy.ops.CombinedLinear((self.gate_proj, self.up_proj))self.down_proj = ops.Linear(intermediate_size, config.hidden_size, bias=False, device=device, dtype=dtype)if config.mlp_activation == "silu":self.activation = torch.nn.functional.silu@@ -793,14 +799,15 @@ class MLP(nn.Module):def forward(self, x, norm=None):# norm: the block's pre-norm, folded into the merged projection's input quantizer where supported- if self.merged_mlp:- if norm is not None:- x = comfy.ops.linear_input_act(self.gate_up_proj, x, "rms_norm", norm.scale(), norm.eps)- else:- x = self.gate_up_proj(x)+ gate_up = self.gate_up_proj if self.merged_mlp else self.combined_gate_up+ if norm is not None:+ combined = comfy.ops.linear_input_act(gate_up, x, "rms_norm", norm.scale(), norm.eps)+ else:+ combined = comfy.ops.linear_input_act(gate_up, x, None)+ if combined is not NotImplemented:if self.merged_input_act is not None:- return comfy.ops.linear_input_act(self.down_proj, x, self.merged_input_act)- gate, up = x.chunk(2, dim=-1)+ return comfy.ops.linear_input_act(self.down_proj, combined, self.merged_input_act)+ gate, up = combined.chunk(2, dim=-1)return self.down_proj(self.activation(gate) * up)if norm is not None:x = norm(x)diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.pyindex d4f241c5f..196f5327b 100644--- a/comfy/text_encoders/qwen35.py+++ b/comfy/text_encoders/qwen35.py@@ -350,6 +350,7 @@ class GatedDeltaNet(nn.Module):self.in_proj_qkv = ops.Linear(hidden, conv_dim, bias=False, device=device, dtype=dtype)self.in_proj_z = ops.Linear(hidden, value_dim, bias=False, device=device, dtype=dtype)+ self.combined_qkvz = comfy.ops.CombinedLinear((self.in_proj_qkv, self.in_proj_z))self.in_proj_b = ops.Linear(hidden, self.num_value_heads, bias=False, device=device, dtype=dtype)self.in_proj_a = ops.Linear(hidden, self.num_value_heads, bias=False, device=device, dtype=dtype)self.out_proj = ops.Linear(value_dim, hidden, bias=False, device=device, dtype=dtype)@@ -366,8 +367,12 @@ class GatedDeltaNet(nn.Module):batch_size, seq_len, _ = x.shape# Projections (shared)- proj = self.in_proj_qkv(x) # [B, seq_len, conv_dim]- z = self.in_proj_z(x)+ qkvz = self.combined_qkvz.forward(x)+ if qkvz is NotImplemented:+ proj = self.in_proj_qkv(x) # [B, seq_len, conv_dim]+ z = self.in_proj_z(x)+ else:+ proj, z = qkvz.split((self.key_dim * 2 + self.value_dim, self.value_dim), dim=-1)if past_key_value is not None and past_key_value.decode_step(seq_len):# decode: conv step into the side buffer, then gates + delta rule + gated norm in one@@ -467,6 +472,7 @@ class GatedAttention(nn.Module):self.q_proj = ops.Linear(config.hidden_size, self.inner_size * 2, bias=config.qkv_bias, device=device, dtype=dtype)self.k_proj = ops.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=config.qkv_bias, device=device, dtype=dtype)self.v_proj = ops.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=config.qkv_bias, device=device, dtype=dtype)+ self.combined_qkv = comfy.ops.CombinedLinear((self.q_proj, self.k_proj, self.v_proj))self.o_proj = ops.Linear(self.inner_size, config.hidden_size, bias=False, device=device, dtype=dtype)# QK norms with (1+weight) scaling@@ -491,15 +497,19 @@ class GatedAttention(nn.Module):batch_size, seq_length, _ = x.shape# Project Q (with gate), K, V- qg = self.q_proj(x)+ qkv = self.combined_qkv.forward(x)+ if qkv is NotImplemented:+ qg = self.q_proj(x)+ xk = self.k_proj(x)+ xv = self.v_proj(x)+ else:+ kv_size = self.num_kv_heads * self.head_dim+ qg, xk, xv = qkv.split((self.inner_size * 2, kv_size, kv_size), dim=-1)# Split into query and gate: each is [B, seq, inner_size]qg = qg.view(batch_size, seq_length, self.num_heads, self.head_dim * 2)xq, gate = qg[..., :self.head_dim], qg[..., self.head_dim:]gate = gate.reshape(batch_size, seq_length, -1) # [B, seq, inner_size]- xk = self.k_proj(x)- xv = self.v_proj(x)-xq = self.q_norm(xq).transpose(1, 2) # [B, heads, seq, head_dim]xk = self.k_norm(xk.view(batch_size, seq_length, self.num_kv_heads, self.head_dim)).transpose(1, 2)xv = xv.view(batch_size, seq_length, self.num_kv_heads, self.head_dim).transpose(1, 2)@@ -859,6 +869,8 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):for m in unit.modules():if isinstance(m, comfy.ops.MixedPrecisionOp):m.comfy_gemv_optimized = True+ if m._combined_linear is not None:+ m._combined_linear.comfy_gemv_optimized = Truevision_overrides = QWEN35_MODELS.get(self.model_type, {}).get("vision", {})vision_config = {**QWEN35_VISION_DEFAULTS, **vision_overrides, "out_hidden_size": config.hidden_size}self.visual = Qwen35VisionModel(vision_config, device=device, dtype=dtype, ops=operations)diff --git a/tests-unit/comfy_test/combined_linear_test.py b/tests-unit/comfy_test/combined_linear_test.pynew file mode 100644index 000000000..9b419c162--- /dev/null+++ b/tests-unit/comfy_test/combined_linear_test.py@@ -0,0 +1,184 @@+import pytest+import torch++from comfy.cli_args import args++if not torch.cuda.is_available():+ args.cpu = True++import comfy.lora # noqa: E402+import comfy.model_patcher # noqa: E402+import comfy.model_prefetch # noqa: E402+import comfy.ops # noqa: E402+from comfy.quant_ops import QuantizedTensor # noqa: E402+++def test_combined_linear_declines_without_running_split_or_activation():+ linears = (comfy.ops.manual_cast.Linear(16, 24, bias=False), comfy.ops.manual_cast.Linear(16, 40, bias=False))+ combined = comfy.ops.CombinedLinear(linears)+ x = torch.randn(2, 16)+ original = x.clone()+ assert combined.forward(x) is NotImplemented+ assert comfy.ops.linear_input_act(combined, x, "model_owned_activation") is NotImplemented+ assert torch.equal(x, original)+++def _gathered(layout, rows, k, **kwargs):+ weights = [QuantizedTensor.from_float(torch.randn(n, k), layout, **kwargs) for n in rows]+ gathered = torch.empty(comfy.memory_management.vram_aligned_size(weights), dtype=torch.uint8)+ views = comfy.memory_management.interpret_gathered_like(weights, gathered)+ for weight, view in zip(weights, views):+ inner_tensors, _ = weight.__tensor_flatten__()+ view.__tensor_flatten__()+ for attr in inner_tensors:+ getattr(view, attr).copy_(getattr(weight, attr))+ return weights, views, gathered++[email protected]("layout, kwargs", [+ ("TensorWiseINT8Layout", {"per_channel": True, "convrot": True}),+ ("AsymW4A8Int8Layout", {}),+ ("AsymW4A8Int8Layout", {"bits": 6}),+])+def test_cat_gathered_concatenates_rows_in_place(layout, kwargs):+ torch.manual_seed(7)+ rows = (256, 128, 384)+ weights, views, gathered = _gathered(layout, rows, 512, **kwargs)+ combined = comfy.memory_management.cat_gathered(views, gathered)+ assert tuple(combined.shape) == (sum(rows), 512)+ assert combined._qdata.data_ptr() == gathered.data_ptr()+ assert torch.equal(combined.dequantize(), torch.cat([w.dequantize() for w in weights]))+ # Laid out like one gathered weight of the summed rows, so optimize_for_gemv's storage contract holds.+ inner_tensors, _ = combined.__tensor_flatten__()+ like = comfy.memory_management.interpret_gathered_like([combined], gathered)[0]+ like.__tensor_flatten__()+ for attr in inner_tensors:+ assert getattr(combined, attr).data_ptr() == getattr(like, attr).data_ptr()+ assert comfy.memory_management.vram_aligned_size(combined) <= gathered.numel()+++def test_combined_linear_follows_members_full_precision():+ # A text encoder combines during generate (use_quantized_matmul) and then encodes with+ # full precision restored on the members; the combined weight must dequantize like they do.+ torch.manual_seed(7)+ ops = comfy.ops.mixed_precision_ops(full_precision_mm=True)+ linears = (ops.Linear(512, 128, bias=False), ops.Linear(512, 256, bias=False))+ combined = comfy.ops.CombinedLinear(linears)+ _, views, gathered = _gathered("AsymW4A8Int8Layout", (128, 256), 512)+ combined._v_weight = comfy.memory_management.cat_gathered(views, gathered)+ for s in linears:+ s._full_precision_mm = False+ assert not combined._full_precision_mm+ for s in linears:+ s._full_precision_mm = True+ assert combined._full_precision_mm+ x = torch.randn(3, 512, dtype=torch.float32)+ out = combined.forward(x)+ assert out.dtype == torch.float32+ assert torch.equal(out, torch.nn.functional.linear(x, combined._v_weight.to(dtype=torch.float32).dequantize()))+++def test_cat_gathered_declines_incompatible_weights():+ torch.manual_seed(7)+ # A tensor-wise scale cannot be shared across rows.+ weights, views, gathered = _gathered("TensorWiseINT8Layout", (128, 128), 512)+ assert comfy.memory_management.cat_gathered(views, gathered) is None+ assert all(torch.equal(v.dequantize(), w.dequantize()) for v, w in zip(views, weights))+ # Different K.+ a = _gathered("TensorWiseINT8Layout", (128,), 512, per_channel=True)[0][0]+ b = _gathered("TensorWiseINT8Layout", (128,), 256, per_channel=True)[0][0]+ gathered = torch.empty(comfy.memory_management.vram_aligned_size([a, b]), dtype=torch.uint8)+ views = comfy.memory_management.interpret_gathered_like([a, b], gathered)+ assert comfy.memory_management.cat_gathered(views, gathered) is None+ # Out of order.+ _, views, gathered = _gathered("TensorWiseINT8Layout", (128, 128), 512, per_channel=True)+ with pytest.raises(RuntimeError):+ comfy.memory_management.cat_gathered(views[::-1], gathered)+++def test_int8_combined_on_hit_reloaded_on_partial_miss_including_lora():+ if not comfy.memory_management.aimdo_enabled or not torch.cuda.is_available():+ pytest.skip("requires initialized DynamicVRAM")++ torch.manual_seed(53)+ device = torch.device("cuda:0")+ ops = comfy.ops.mixed_precision_ops(compute_dtype=torch.bfloat16)++ class Projections(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.a = ops.Linear(512, 96, bias=False)+ self.b = ops.Linear(512, 160, bias=False)+ for layer in (self.a, self.b):+ weight = QuantizedTensor.from_float(+ torch.randn(layer.out_features, 512, dtype=torch.bfloat16) * 0.03,+ "TensorWiseINT8Layout", per_channel=True, convrot=True, convrot_groupsize=256,+ )+ layer.weight = torch.nn.Parameter(weight, requires_grad=False)+ layer.quant_format = "int8_tensorwise"+ layer.layout_type = weight.layout_cls+ layer.comfy_gemv_optimized = True+ self.combined = comfy.ops.CombinedLinear((self.a, self.b))++ def get_dynamic_vram__units(self):+ return [self], []++ def forward(self, x):+ ab = self.combined.forward(x)+ return torch.cat((self.a(x), self.b(x)), dim=-1) if ab is NotImplemented else ab++ model = Projections()+ members = (model.a, model.b)+ canonical = [(s.weight._qdata.to(device), s.weight.params.scale.flatten().to(device)) for s in members]+ base = comfy.model_patcher.ModelPatcherDynamic(model, device, torch.device("cpu"))+ patches = comfy.lora.load_lora({+ "a.lora_up.weight": torch.randn(96, 4) * 0.1,+ "a.lora_down.weight": torch.randn(4, 512) * 0.1,+ }, {"a": "a.weight"})+ lora = base.clone()+ lora.add_patches(patches, strength_patch=0.75)+ x = torch.randn(3, 512, device=device, dtype=torch.bfloat16)++ def assert_unpinned():+ assert not any(flags & 2 for flags in model.a._v[0].get_residency())++ def step(combined, expected=None):+ comfy.model_prefetch.pin_modules(members, device, x.dtype)+ try:+ assert (model.combined._v_weight is not None) == combined+ out = model(x)+ if combined:+ assert all(s._v_weight is None for s in members)+ if expected:+ # The combined weight is the members' canonical bytes, back to back.+ weight = model.combined._v_weight+ assert torch.equal(weight._qdata, torch.cat([q for q, _ in expected]))+ assert torch.equal(weight.params.scale.flatten(), torch.cat([s for _, s in expected]))+ return out.clone()+ finally:+ comfy.model_prefetch.cleanup_prefetched_modules(model, members)+ assert_unpinned()++ try:+ base.partially_load(device)+ # An unprefetched member forward casts only itself and must not leave its peers pinned.+ split = torch.cat((model.a(x), model.b(x)), dim=-1)+ assert_unpinned()+ # The first pin after a load misses, resolves and combines in the same pin.+ results = [split, step(True, canonical), step(True, canonical)]+ # A member missing from a combined group reloads the group's canonical bytes and recombines.+ model.b._v_signature = None+ results += [step(True, canonical)]+ assert all(torch.equal(results[0], r) for r in results[1:])++ # The baked patch is what gets combined.+ lora.partially_load(device)+ patched = [step(True), step(True)]+ assert torch.equal(patched[0], patched[1])+ assert not torch.equal(results[0], patched[0])++ base.partially_load(device)+ assert torch.equal(results[0], step(True, canonical))+ finally:+ torch.cuda.synchronize()+ base.unpatch_model(unpatch_weights=True)diff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.pyindex efd322865..e7cf9ca81 100644--- a/tests-unit/comfy_test/qwen35_decode_test.py+++ b/tests-unit/comfy_test/qwen35_decode_test.py@@ -9,7 +9,11 @@ from comfy.cli_args import args as cli_argsif not torch.cuda.is_available():cli_args.cpu = True+import comfy.lora # noqa: E402+import comfy.model_patcher # noqa: E402+import comfy.model_prefetch # noqa: E402import comfy.ops # noqa: E402+from comfy.quant_ops import QuantizedTensor # noqa: E402from comfy.text_encoders.llama import MLP, RMSNorm # noqa: E402@@ -32,11 +36,124 @@ def test_mlp_folded_norm_matches_eager_norm():x = torch.randn(1, 3, 16)for merged in (False, True):mlp = _mlp(merged)+ assert set(mlp.state_dict()) == (+ {"gate_up_proj.weight", "down_proj.weight"} if merged else+ {"gate_proj.weight", "up_proj.weight", "down_proj.weight"}+ )for add in (False, True):norm = _norm(add)assert torch.equal(mlp(x, norm=norm), mlp(norm(x)))+def test_combined_gate_up_declines_without_running_split_or_activation():+ mlp = _mlp(False)+ combined = mlp.combined_gate_up+ x = torch.randn(2, 16)+ original = x.clone()+ assert combined.forward(x) is NotImplemented+ # An unsupported request must escape to the model, not enter the eager+ # activation fallback (which would reject this activation name).+ assert comfy.ops.linear_input_act(combined, x, "model_owned_activation") is NotImplemented+ assert torch.equal(x, original)+ assert [name for name, _ in mlp.named_modules()] == ["", "gate_proj", "up_proj", "down_proj"]+++def test_w6a8_packed_and_combined_on_hit_including_lora():+ if not comfy.memory_management.aimdo_enabled or not torch.cuda.is_available():+ pytest.skip("requires initialized CUDA DynamicVRAM")++ torch.manual_seed(53)+ device = torch.device("cuda:0")+ ops = comfy.ops.mixed_precision_ops(compute_dtype=torch.bfloat16)++ class Projections(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.a = ops.Linear(512, 96, bias=False)+ self.b = ops.Linear(512, 160, bias=False)+ self.c = ops.Linear(512, 128, bias=False)+ for layer in (self.a, self.b, self.c):+ weight = QuantizedTensor.from_float(+ torch.randn(layer.out_features, 512, dtype=torch.bfloat16) * 0.03,+ "AsymW4A8Int8Layout", bits=6, convrot_groupsize=256,+ )+ layer.weight = torch.nn.Parameter(weight, requires_grad=False)+ layer.quant_format = "w6a8_int8"+ layer.layout_type = weight.layout_cls+ layer.comfy_gemv_optimized = True+ self.combined = comfy.ops.CombinedLinear((self.a, self.b))+ self.combined.comfy_gemv_optimized = True++ def get_dynamic_vram__units(self):+ return [self], []++ def forward(self, x):+ ab = self.combined.forward(x)+ if ab is NotImplemented:+ ab = torch.cat((self.a(x), self.b(x)), dim=-1)+ return torch.cat((ab, self.c(x)), dim=-1)++ model = Projections()+ members = (model.a, model.b, model.c)+ canonical = [s.weight.dequantize().to(device) for s in members]+ base = comfy.model_patcher.ModelPatcherDynamic(model, device, torch.device("cpu"))+ tensors = {}+ for name in ("a", "c"):+ tensors[f"{name}.lora_up.weight"] = torch.randn(getattr(model, name).out_features, 4) * 0.1+ tensors[f"{name}.lora_down.weight"] = torch.randn(4, 512) * 0.1+ patches = comfy.lora.load_lora(tensors, {"a": "a.weight", "c": "c.weight"})+ lora = base.clone()+ assert set(lora.add_patches(patches, strength_patch=0.75)) == {"a.weight", "c.weight"}+ x = torch.randn(3, 512, device=device, dtype=torch.bfloat16)++ def step(packed, expected=None):+ comfy.model_prefetch.pin_modules(members, device, x.dtype)+ try:+ assert (model.combined._v_weight is not None) == packed+ out = model(x)+ if packed:+ # Combined members give up their own weights; the combined weight and c pack.+ assert model.a._v_weight is None and model.b._v_weight is None+ assert model.combined._v_weight.params.mma_packed and model.c._v_weight.params.mma_packed+ if expected:+ # Un-rotation on dequantize is a batched matmul whose rounding depends on the row count.+ torch.testing.assert_close(model.combined._v_weight.dequantize(), torch.cat(expected[:2]))+ torch.testing.assert_close(model.c._v_weight.dequantize(), expected[2])+ else:+ assert not any(s._v_weight.params.mma_packed for s in members)+ return out.clone()+ finally:+ comfy.model_prefetch.cleanup_prefetched_modules(model, members)++ def check(split, packed):+ # The packed GEMM shares the int8 grid and int32 accumulation; only the fp32 epilogue order differs.+ torch.testing.assert_close(packed.float(), split.float(), atol=2e-2, rtol=1e-2)++ try:+ # An unprefetched forward runs the canonical layout; the first pin after a+ # load misses, resolves, packs and combines in the same pin.+ base.partially_load(device)+ split = model(x).clone()+ packed = [step(True, canonical), step(True, canonical)]+ assert torch.equal(packed[0], packed[1])+ check(split, packed[0])++ # The baked patch is what gets packed and combined. A reload dirties the members but+ # not the group; the pin that precedes every block forward drops the stale group on+ # the members' miss, so the unprefetched canonical reference drops it here.+ lora.partially_load(device)+ model.combined._v_weight = None+ patched = [model(x).clone(), step(True)]+ assert not torch.equal(split, patched[0])+ check(*patched)++ base.partially_load(device)+ assert torch.equal(packed[0], step(True, canonical))+ finally:+ torch.cuda.synchronize()+ base.unpatch_model(unpatch_weights=True)++def test_rms_norm_scale_adds_one_without_caching():norm = _norm(True)assert torch.equal(norm.scale(), norm.weight + 1.0)