The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Improve low VRAM performance of qwen 3.5 LLM (#16820)
rattus6a8dcf514bc0
diff --git a/comfy/ops.py b/comfy/ops.pyindex 4ca876fe7..bb671dc9e 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -124,6 +124,14 @@ def materialize_meta_param(s, param_keys):setattr(s, param_key, torch.nn.Parameter(torch.zeros(param.shape, dtype=param.dtype), requires_grad=param.requires_grad))+def vbar_above_watermark(s):+ # prefetch ring manages residency itself, hands off+ if getattr(s, "_prefetch", None) is not None:+ return False+ vbar, alloc, size = s._v+ return alloc - vbar.base_addr + size > vbar.get_watermark() * (32 << 20) # watermark is in VBAR pages++# FIXME: add n=1 cache hit fast pathdef cast_modules_with_vbar(comfy_modules, dtype, device, bias_dtype, non_blocking, return_faulted=False):offload_stream = None@@ -797,6 +805,11 @@ class disable_weight_init:with CastBiasWeightContext(self, device=input.device, dtype=out_dtype, offloadable=True) as (weight, bias):return torch.nn.functional.embedding(input, weight, self.padding_idx, self.max_norm, self.norm_type, self.scale_grad_by_freq, self.sparse).to(dtype=output_dtype)+ def host_rows(self, input, out_dtype=None):+ # table not resident: gather the rows from the host copy instead of streaming the table+ x = self.weight[input.reshape(-1).cpu()].to(input.device)+ x = x.view(*input.shape, x.shape[-1])+ return x if out_dtype is None else x.to(dtype=out_dtype)def forward(self, *args, **kwargs):run_every_op()@@ -1726,6 +1739,22 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precsd = destination if destination is not None else {}return _quantized_weight_state_dict(self, sd, prefix)+ def host_rows(self, input, out_dtype=None):+ weight = self.weight+ if not isinstance(weight, QuantizedTensor):+ return super().host_rows(input, out_dtype=out_dtype)+ if self.quant_format != "int8_tensorwise":+ # TODO: fp8 tables still stream through the cast buffer+ return self.forward_comfy_cast_weights(input, out_dtype=out_dtype)+ idx = input.reshape(-1).cpu()+ params = weight._params+ scale = params.scale[idx] if params.scale.dim() >= 2 else params.scale # per-row scale is [vocab, 1], as dequantize_embedding expects+ params = dataclasses.replace(params, scale=scale.to(input.device))+ rows = weight._qdata[idx].to(input.device)+ x = get_layout_class(self.layout_type).dequantize_embedding(rows, params, torch.arange(idx.numel(), device=input.device))+ x = x.view(*input.shape, x.shape[-1])+ return x if out_dtype is None else x.to(dtype=out_dtype)+def forward_comfy_cast_weights(self, input, out_dtype=None):weight = self.weightdiff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.pyindex 01651bd23..39280bb5e 100644--- a/comfy/sd1_clip.py+++ b/comfy/sd1_clip.py@@ -751,6 +751,6 @@ class SD1ClipModel(torch.nn.Module):def get_dynamic_vram__units(self):# forward to the inner transformer so ModelPatcher can register vbar units (graph decode)- model = getattr(getattr(getattr(self, self.clip), "transformer", None), "model", None)- get_units = getattr(model, "get_dynamic_vram__units", None)+ transformer = getattr(getattr(self, self.clip), "transformer", None)+ get_units = getattr(transformer, "get_dynamic_vram__units", None) or getattr(getattr(transformer, "model", None), "get_dynamic_vram__units", None)return get_units() if get_units is not None else ([], [])diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex db36fc37e..161e65411 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -865,6 +865,9 @@ def _make_scaled_embedding(ops, vocab_size, hidden_size, scale, device, dtype):class ScaledEmbedding(ops.Embedding):def forward(self, input_ids, out_dtype=None):return super().forward(input_ids, out_dtype=out_dtype) * scale++ def host_rows(self, input_ids, out_dtype=None):+ return super().host_rows(input_ids, out_dtype=out_dtype) * scalereturn ScaledEmbedding(vocab_size, hidden_size, device=device, dtype=dtype)@@ -1180,7 +1183,11 @@ class BaseGenerate:if step > 0:if compile_allocations:comfy.model_prefetch.malloc_graph_begin(device)- embeds = self.model.embed_tokens(decode_tokens).to(execution_dtype)+ embed = self.model.embed_tokens+ if hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed):+ embeds = embed.host_rows(decode_tokens, out_dtype=execution_dtype)+ else:+ embeds = embed(decode_tokens).to(execution_dtype)current_input_ids = decode_tokens if initial_input_ids is not None else Noneposition_ids = torch.tensor([[next_pos]], device=device) if next_pos is not None else None@@ -1210,6 +1217,12 @@ class BaseGenerate:token_id = decode_tokens[0].item()generated_token_ids.append(token_id)+ if step == 0 and hasattr(self.model.embed_tokens, "_v"):+ # prefill's transients can evict body pages and lower the VBAR watermark: let the decode fault them back+ vbar = self.model.embed_tokens._v[0]+ comfy.model_management.reset_cast_buffers()+ vbar.set_watermark(vbar.max_size)+if step > 0 and next_pos is not None:next_pos += 1pbar.update(1)diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.pyindex 29ce136ef..292038a3d 100644--- a/comfy/text_encoders/qwen35.py+++ b/comfy/text_encoders/qwen35.py@@ -727,6 +727,16 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):self.visual = Qwen35VisionModel(vision_config, device=device, dtype=dtype, ops=operations)self.dtype = dtype+ def get_dynamic_vram__units(self):+ units, last_units = self.model.get_dynamic_vram__units()+ # head and MTP draft weights run every decode step: fault them resident ahead of the body.+ # a separate embedding goes last: evicted, its rows are gathered from the host copy instead.+ # a tied table is the head, so it stays first.+ hot = [] if self.mtp is None else [self.mtp]+ if hasattr(self.model, "lm_head"):+ return [self.model.lm_head, *hot, *units], [*last_units, self.model.embed_tokens]+ return [self.model.embed_tokens, *hot, *units], last_units+def preprocess_embed(self, embed, device):if embed["type"] == "image":# Qwen3.5 normalizes to [-1, 1] (mean/std 0.5), same as Qwen3-VL.