The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
dynamic_vram: respect argument cast dtypes in non-comfy weights (#12209)
rattus794d05bdb12b
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex 758e718e8..6b1166b94 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -1202,27 +1202,36 @@ def cast_to(weight, dtype=None, device=None, non_blocking=False, copy=False, strassert r is Noneassert stream is None- r = torch.empty_like(weight, dtype=weight._model_dtype, device=device)+ cast_geometry = comfy.memory_management.tensors_to_geometries([ weight ])++ if dtype is None:+ dtype = weight._model_dtype++ r = torch.empty_like(weight, dtype=dtype, device=device)signature = comfy_aimdo.model_vbar.vbar_fault(weight._v)if signature is not None:raw_tensor = comfy_aimdo.torch.aimdo_to_tensor(weight._v, device)- v_tensor = comfy.memory_management.interpret_gathered_like([r], raw_tensor)[0]-- if comfy_aimdo.model_vbar.vbar_signature_compare(signature, weight._v_signature):+ v_tensor = comfy.memory_management.interpret_gathered_like(cast_geometry, raw_tensor)[0]+ if not comfy_aimdo.model_vbar.vbar_signature_compare(signature, weight._v_signature):+ weight._v_signature = signature+ #Send it over+ v_tensor.copy_(weight, non_blocking=non_blocking)#always take a deep copy even if _v is good, as we have no reasonable point to unpin#a non comfy weightr.copy_(v_tensor)comfy_aimdo.model_vbar.vbar_unpin(weight._v)return r+ if weight.dtype != r.dtype and weight.dtype != weight._model_dtype:+ #Offloaded casting could skip this, however it would make the quantizations+ #inconsistent between loaded and offloaded weights. So force the double casting+ #that would happen in regular flow to make offload deterministic.+ cast_buffer = torch.empty_like(weight, dtype=weight._model_dtype, device=device)+ cast_buffer.copy_(weight, non_blocking=non_blocking)+ weight = cast_bufferr.copy_(weight, non_blocking=non_blocking)- if signature is not None:- weight._v_signature = signature- v_tensor.copy_(r)- comfy_aimdo.model_vbar.vbar_unpin(weight._v)-return rif device is None or weight.device == device: