The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
dynamic_vram: Fix windows Aimdo crash + Fix LLM performance (#12408)
rattusd297a749a2fa
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex 304931eb0..38c3e482b 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -1213,8 +1213,12 @@ def cast_to(weight, dtype=None, device=None, non_blocking=False, copy=False, strsignature = comfy_aimdo.model_vbar.vbar_fault(weight._v)if signature is not None:- v_tensor = comfy.memory_management.interpret_gathered_like(cast_geometry, weight._v_tensor)[0]- if not comfy_aimdo.model_vbar.vbar_signature_compare(signature, weight._v_signature):+ if comfy_aimdo.model_vbar.vbar_signature_compare(signature, weight._v_signature):+ v_tensor = weight._v_tensor+ else:+ raw_tensor = comfy_aimdo.torch.aimdo_to_tensor(weight._v, device)+ v_tensor = comfy.memory_management.interpret_gathered_like(cast_geometry, raw_tensor)[0]+ weight._v_tensor = v_tensorweight._v_signature = signature#Send it overv_tensor.copy_(weight, non_blocking=non_blocking)diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex 19c9031ea..224e218e3 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -1542,7 +1542,6 @@ class ModelPatcherDynamic(ModelPatcher):if vbar is not None and not hasattr(m, "_v"):m._v = vbar.alloc(v_weight_size)- m._v_tensor = comfy_aimdo.torch.aimdo_to_tensor(m._v, device_to)allocated_size += v_weight_sizeelse:@@ -1557,7 +1556,6 @@ class ModelPatcherDynamic(ModelPatcher):weight_size = geometry.numel() * geometry.element_size()if vbar is not None and not hasattr(weight, "_v"):weight._v = vbar.alloc(weight_size)- weight._v_tensor = comfy_aimdo.torch.aimdo_to_tensor(weight._v, device_to)weight._model_dtype = model_dtypeallocated_size += weight_sizevbar.set_watermark_limit(allocated_size)diff --git a/comfy/ops.py b/comfy/ops.pyindex 33803b223..688937e43 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -83,14 +83,18 @@ def cast_to_input(weight, input, non_blocking=False, copy=True):def cast_bias_weight_with_vbar(s, dtype, device, bias_dtype, non_blocking, compute_dtype):offload_stream = Nonexfer_dest = None- cast_geometry = comfy.memory_management.tensors_to_geometries([ s.weight, s.bias ])signature = comfy_aimdo.model_vbar.vbar_fault(s._v)- if signature is not None:- xfer_dest = s._v_tensorresident = comfy_aimdo.model_vbar.vbar_signature_compare(signature, s._v_signature)+ if signature is not None:+ if resident:+ weight = s._v_weight+ bias = s._v_bias+ else:+ xfer_dest = comfy_aimdo.torch.aimdo_to_tensor(s._v, device)if not resident:+ cast_geometry = comfy.memory_management.tensors_to_geometries([ s.weight, s.bias ])cast_dest = Nonexfer_source = [ s.weight, s.bias ]@@ -140,9 +144,13 @@ def cast_bias_weight_with_vbar(s, dtype, device, bias_dtype, non_blocking, compupost_cast.copy_(pre_cast)xfer_dest = cast_dest- params = comfy.memory_management.interpret_gathered_like(cast_geometry, xfer_dest)- weight = params[0]- bias = params[1]+ params = comfy.memory_management.interpret_gathered_like(cast_geometry, xfer_dest)+ weight = params[0]+ bias = params[1]+ if signature is not None:+ s._v_weight = weight+ s._v_bias = bias+ s._v_signature=signaturedef post_cast(s, param_key, x, dtype, resident, update_weight):lowvram_fn = getattr(s, param_key + "_lowvram_function", None)@@ -182,7 +190,6 @@ def cast_bias_weight_with_vbar(s, dtype, device, bias_dtype, non_blocking, compuweight = post_cast(s, "weight", weight, dtype, resident, update_weight)if s.bias is not None:bias = post_cast(s, "bias", bias, bias_dtype, resident, update_weight)- s._v_signature=signature#FIXME: weird offload return protocolreturn weight, bias, (offload_stream, device if signature is not None else None, None)