Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

dynamic_vram: Fix windows Aimdo crash + Fix LLM performance (#12408)

rattusd297a749a2fa
diff --git a/comfy/model_management.py b/comfy/model_management.py
index 304931eb0..38c3e482b 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -1213,8 +1213,12 @@ def cast_to(weight, dtype=None, device=None, non_blocking=False, copy=False, str
signature = comfy_aimdo.model_vbar.vbar_fault(weight._v)
if signature is not None:
- v_tensor = comfy.memory_management.interpret_gathered_like(cast_geometry, weight._v_tensor)[0]
- if not comfy_aimdo.model_vbar.vbar_signature_compare(signature, weight._v_signature):
+ if comfy_aimdo.model_vbar.vbar_signature_compare(signature, weight._v_signature):
+ v_tensor = weight._v_tensor
+ else:
+ raw_tensor = comfy_aimdo.torch.aimdo_to_tensor(weight._v, device)
+ v_tensor = comfy.memory_management.interpret_gathered_like(cast_geometry, raw_tensor)[0]
+ weight._v_tensor = v_tensor
weight._v_signature = signature
#Send it over
v_tensor.copy_(weight, non_blocking=non_blocking)
diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py
index 19c9031ea..224e218e3 100644
--- a/comfy/model_patcher.py
+++ b/comfy/model_patcher.py
@@ -1542,7 +1542,6 @@ class ModelPatcherDynamic(ModelPatcher):
if vbar is not None and not hasattr(m, "_v"):
m._v = vbar.alloc(v_weight_size)
- m._v_tensor = comfy_aimdo.torch.aimdo_to_tensor(m._v, device_to)
allocated_size += v_weight_size
else:
@@ -1557,7 +1556,6 @@ class ModelPatcherDynamic(ModelPatcher):
weight_size = geometry.numel() * geometry.element_size()
if vbar is not None and not hasattr(weight, "_v"):
weight._v = vbar.alloc(weight_size)
- weight._v_tensor = comfy_aimdo.torch.aimdo_to_tensor(weight._v, device_to)
weight._model_dtype = model_dtype
allocated_size += weight_size
vbar.set_watermark_limit(allocated_size)
diff --git a/comfy/ops.py b/comfy/ops.py
index 33803b223..688937e43 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -83,14 +83,18 @@ def cast_to_input(weight, input, non_blocking=False, copy=True):
def cast_bias_weight_with_vbar(s, dtype, device, bias_dtype, non_blocking, compute_dtype):
offload_stream = None
xfer_dest = None
- cast_geometry = comfy.memory_management.tensors_to_geometries([ s.weight, s.bias ])
signature = comfy_aimdo.model_vbar.vbar_fault(s._v)
- if signature is not None:
- xfer_dest = s._v_tensor
resident = comfy_aimdo.model_vbar.vbar_signature_compare(signature, s._v_signature)
+ if signature is not None:
+ if resident:
+ weight = s._v_weight
+ bias = s._v_bias
+ else:
+ xfer_dest = comfy_aimdo.torch.aimdo_to_tensor(s._v, device)
if not resident:
+ cast_geometry = comfy.memory_management.tensors_to_geometries([ s.weight, s.bias ])
cast_dest = None
xfer_source = [ s.weight, s.bias ]
@@ -140,9 +144,13 @@ def cast_bias_weight_with_vbar(s, dtype, device, bias_dtype, non_blocking, compu
post_cast.copy_(pre_cast)
xfer_dest = cast_dest
- params = comfy.memory_management.interpret_gathered_like(cast_geometry, xfer_dest)
- weight = params[0]
- bias = params[1]
+ params = comfy.memory_management.interpret_gathered_like(cast_geometry, xfer_dest)
+ weight = params[0]
+ bias = params[1]
+ if signature is not None:
+ s._v_weight = weight
+ s._v_bias = bias
+ s._v_signature=signature
def post_cast(s, param_key, x, dtype, resident, update_weight):
lowvram_fn = getattr(s, param_key + "_lowvram_function", None)
@@ -182,7 +190,6 @@ def cast_bias_weight_with_vbar(s, dtype, device, bias_dtype, non_blocking, compu
weight = post_cast(s, "weight", weight, dtype, resident, update_weight)
if s.bias is not None:
bias = post_cast(s, "bias", bias, bias_dtype, resident, update_weight)
- s._v_signature=signature
#FIXME: weird offload return protocol
return weight, bias, (offload_stream, device if signature is not None else None, None)