The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
comfy aimdo 0.2.11 + Improved RAM Pressure release strategies - Windows speedups (#12925)
rattus7810f49702ea
diff --git a/comfy/memory_management.py b/comfy/memory_management.pyindex 0b7da2852..563224098 100644--- a/comfy/memory_management.py+++ b/comfy/memory_management.py@@ -1,9 +1,68 @@import math+import ctypes+import threading+import dataclassesimport torchfrom typing import NamedTuplefrom comfy.quant_ops import QuantizedTensor++class TensorFileSlice(NamedTuple):+ file_ref: object+ thread_id: int+ offset: int+ size: int+++def read_tensor_file_slice_into(tensor, destination):++ if isinstance(tensor, QuantizedTensor):+ if not isinstance(destination, QuantizedTensor):+ return False+ if tensor._layout_cls != destination._layout_cls:+ return False++ if not read_tensor_file_slice_into(tensor._qdata, destination._qdata):+ return False++ dst_orig_dtype = destination._params.orig_dtype+ destination._params.copy_from(tensor._params, non_blocking=False)+ destination._params = dataclasses.replace(destination._params, orig_dtype=dst_orig_dtype)+ return True++ info = getattr(tensor.untyped_storage(), "_comfy_tensor_file_slice", None)+ if info is None:+ return False++ file_obj = info.file_ref+ if (destination.device.type != "cpu"+ or file_obj is None+ or threading.get_ident() != info.thread_id+ or destination.numel() * destination.element_size() < info.size):+ return False++ if info.size == 0:+ return True++ buf_type = ctypes.c_ubyte * info.size+ view = memoryview(buf_type.from_address(destination.data_ptr()))++ try:+ file_obj.seek(info.offset)+ done = 0+ while done < info.size:+ try:+ n = file_obj.readinto(view[done:])+ except OSError:+ return False+ if n <= 0:+ return False+ done += n+ return True+ finally:+ view.release()+class TensorGeometry(NamedTuple):shape: anydtype: torch.dtypediff --git a/comfy/model_management.py b/comfy/model_management.pyindex 81c89b180..4d5851bc0 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -505,6 +505,28 @@ def module_size(module):module_mem += t.nbytesreturn module_mem+def module_mmap_residency(module, free=False):+ mmap_touched_mem = 0+ module_mem = 0+ bounced_mmaps = set()+ sd = module.state_dict()+ for k in sd:+ t = sd[k]+ module_mem += t.nbytes+ storage = t._qdata.untyped_storage() if isinstance(t, comfy.quant_ops.QuantizedTensor) else t.untyped_storage()+ if not getattr(storage, "_comfy_tensor_mmap_touched", False):+ continue+ mmap_touched_mem += t.nbytes+ if not free:+ continue+ storage._comfy_tensor_mmap_touched = False+ mmap_obj = storage._comfy_tensor_mmap_refs[0]+ if mmap_obj in bounced_mmaps:+ continue+ mmap_obj.bounce()+ bounced_mmaps.add(mmap_obj)+ return mmap_touched_mem, module_mem+class LoadedModel:def __init__(self, model):self._set_model(model)@@ -532,6 +554,9 @@ class LoadedModel:def model_memory(self):return self.model.model_size()+ def model_mmap_residency(self, free=False):+ return self.model.model_mmap_residency(free=free)+def model_loaded_memory(self):return self.model.loaded_size()@@ -633,7 +658,7 @@ def extra_reserved_memory():def minimum_inference_memory():return (1024 * 1024 * 1024) * 0.8 + extra_reserved_memory()-def free_memory(memory_required, device, keep_loaded=[], for_dynamic=False, ram_required=0):+def free_memory(memory_required, device, keep_loaded=[], for_dynamic=False, pins_required=0, ram_required=0):cleanup_models_gc()unloaded_model = []can_unload = []@@ -646,13 +671,14 @@ def free_memory(memory_required, device, keep_loaded=[], for_dynamic=False, ram_can_unload.append((-shift_model.model_offloaded_memory(), sys.getrefcount(shift_model.model), shift_model.model_memory(), i))shift_model.currently_used = False- for x in sorted(can_unload):+ can_unload_sorted = sorted(can_unload)+ for x in can_unload_sorted:i = x[-1]memory_to_free = 1e32- ram_to_free = 1e32+ pins_to_free = 1e32if not DISABLE_SMART_MEMORY:memory_to_free = memory_required - get_free_memory(device)- ram_to_free = ram_required - get_free_ram()+ pins_to_free = pins_required - get_free_ram()if current_loaded_models[i].model.is_dynamic() and for_dynamic:#don't actually unload dynamic models for the sake of other dynamic models#as that works on-demand.@@ -661,9 +687,18 @@ def free_memory(memory_required, device, keep_loaded=[], for_dynamic=False, ram_if memory_to_free > 0 and current_loaded_models[i].model_unload(memory_to_free):logging.debug(f"Unloading {current_loaded_models[i].model.model.__class__.__name__}")unloaded_model.append(i)- if ram_to_free > 0:+ if pins_to_free > 0:+ logging.debug(f"PIN Unloading {current_loaded_models[i].model.model.__class__.__name__}")+ current_loaded_models[i].model.partially_unload_ram(pins_to_free)++ for x in can_unload_sorted:+ i = x[-1]+ ram_to_free = ram_required - psutil.virtual_memory().available+ if ram_to_free <= 0 and i not in unloaded_model:+ continue+ resident_memory, _ = current_loaded_models[i].model_mmap_residency(free=True)+ if resident_memory > 0:logging.debug(f"RAM Unloading {current_loaded_models[i].model.model.__class__.__name__}")- current_loaded_models[i].model.partially_unload_ram(ram_to_free)for i in sorted(unloaded_model, reverse=True):unloaded_models.append(current_loaded_models.pop(i))@@ -729,17 +764,27 @@ def load_models_gpu(models, memory_required=0, force_patch_weights=False, minimutotal_memory_required = {}+ total_pins_required = {}total_ram_required = {}for loaded_model in models_to_load:- total_memory_required[loaded_model.device] = total_memory_required.get(loaded_model.device, 0) + loaded_model.model_memory_required(loaded_model.device)- #x2, one to make sure the OS can fit the model for loading in disk cache, and for us to do any pinning we- #want to do.- #FIXME: This should subtract off the to_load current pin consumption.- total_ram_required[loaded_model.device] = total_ram_required.get(loaded_model.device, 0) + loaded_model.model_memory() * 2+ device = loaded_model.device+ total_memory_required[device] = total_memory_required.get(device, 0) + loaded_model.model_memory_required(device)+ resident_memory, model_memory = loaded_model.model_mmap_residency()+ pinned_memory = loaded_model.model.pinned_memory_size()+ #FIXME: This can over-free the pins as it budgets to pin the entire model. We should+ #make this JIT to keep as much pinned as possible.+ pins_required = model_memory - pinned_memory+ ram_required = model_memory - resident_memory+ total_pins_required[device] = total_pins_required.get(device, 0) + pins_required+ total_ram_required[device] = total_ram_required.get(device, 0) + ram_requiredfor device in total_memory_required:if device != torch.device("cpu"):- free_memory(total_memory_required[device] * 1.1 + extra_mem, device, for_dynamic=free_for_dynamic, ram_required=total_ram_required[device])+ free_memory(total_memory_required[device] * 1.1 + extra_mem,+ device,+ for_dynamic=free_for_dynamic,+ pins_required=total_pins_required[device],+ ram_required=total_ram_required[device])for device in total_memory_required:if device != torch.device("cpu"):@@ -1225,6 +1270,11 @@ def cast_to_gathered(tensors, r, non_blocking=False, stream=None):dest_view = dest_views.pop(0)if tensor is None:continue+ if comfy.memory_management.read_tensor_file_slice_into(tensor, dest_view):+ continue+ storage = tensor._qdata.untyped_storage() if isinstance(tensor, comfy.quant_ops.QuantizedTensor) else tensor.untyped_storage()+ if hasattr(storage, "_comfy_tensor_mmap_touched"):+ storage._comfy_tensor_mmap_touched = Truedest_view.copy_(tensor, non_blocking=non_blocking)diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex bc3a8f446..c26d37db2 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -297,6 +297,9 @@ class ModelPatcher:self.size = comfy.model_management.module_size(self.model)return self.size+ def model_mmap_residency(self, free=False):+ return comfy.model_management.module_mmap_residency(self.model, free=free)+def get_ram_usage(self):return self.model_size()@@ -1063,6 +1066,10 @@ class ModelPatcher:return self.model.model_loaded_weight_memory - current_used+ def pinned_memory_size(self):+ # Pinned memory pressure tracking is only implemented for DynamicVram loading+ return 0+def partially_unload_ram(self, ram_to_unload):pass@@ -1653,6 +1660,16 @@ class ModelPatcherDynamic(ModelPatcher):return freed+ def pinned_memory_size(self):+ total = 0+ loading = self._load_list(for_dynamic=True)+ for x in loading:+ _, _, _, _, m, _ = x+ pin = comfy.pinned_memory.get_pin(m)+ if pin is not None:+ total += pin.numel() * pin.element_size()+ return total+def partially_unload_ram(self, ram_to_unload):loading = self._load_list(for_dynamic=True, default_device=self.offload_device)for x in loading:diff --git a/comfy/ops.py b/comfy/ops.pyindex 87b36b5c5..3f2da4e63 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -306,6 +306,33 @@ class CastWeightBiasOp:bias_function = []class disable_weight_init:+ @staticmethod+ def _lazy_load_from_state_dict(module, state_dict, prefix, local_metadata,+ missing_keys, unexpected_keys, weight_shape,+ bias_shape=None):+ assign_to_params_buffers = local_metadata.get("assign_to_params_buffers", False)+ prefix_len = len(prefix)+ for k, v in state_dict.items():+ key = k[prefix_len:]+ if key == "weight":+ if not assign_to_params_buffers:+ v = v.clone()+ module.weight = torch.nn.Parameter(v, requires_grad=False)+ elif bias_shape is not None and key == "bias" and v is not None:+ if not assign_to_params_buffers:+ v = v.clone()+ module.bias = torch.nn.Parameter(v, requires_grad=False)+ else:+ unexpected_keys.append(k)++ if module.weight is None:+ module.weight = torch.nn.Parameter(torch.zeros(weight_shape), requires_grad=False)+ missing_keys.append(prefix + "weight")++ if bias_shape is not None and module.bias is None and getattr(module, "comfy_need_lazy_init_bias", False):+ module.bias = torch.nn.Parameter(torch.zeros(bias_shape), requires_grad=False)+ missing_keys.append(prefix + "bias")+class Linear(torch.nn.Linear, CastWeightBiasOp):def __init__(self, in_features, out_features, bias=True, device=None, dtype=None):@@ -333,29 +360,16 @@ class disable_weight_init:if not comfy.model_management.WINDOWS or not comfy.memory_management.aimdo_enabled:return super()._load_from_state_dict(state_dict, prefix, local_metadata, strict,missing_keys, unexpected_keys, error_msgs)- assign_to_params_buffers = local_metadata.get("assign_to_params_buffers", False)- prefix_len = len(prefix)- for k,v in state_dict.items():- if k[prefix_len:] == "weight":- if not assign_to_params_buffers:- v = v.clone()- self.weight = torch.nn.Parameter(v, requires_grad=False)- elif k[prefix_len:] == "bias" and v is not None:- if not assign_to_params_buffers:- v = v.clone()- self.bias = torch.nn.Parameter(v, requires_grad=False)- else:- unexpected_keys.append(k)-- #Reconcile default construction of the weight if its missing.- if self.weight is None:- v = torch.zeros(self.in_features, self.out_features)- self.weight = torch.nn.Parameter(v, requires_grad=False)- missing_keys.append(prefix+"weight")- if self.bias is None and self.comfy_need_lazy_init_bias:- v = torch.zeros(self.out_features,)- self.bias = torch.nn.Parameter(v, requires_grad=False)- missing_keys.append(prefix+"bias")+ disable_weight_init._lazy_load_from_state_dict(+ self,+ state_dict,+ prefix,+ local_metadata,+ missing_keys,+ unexpected_keys,+ weight_shape=(self.in_features, self.out_features),+ bias_shape=(self.out_features,),+ )def reset_parameters(self):@@ -547,6 +561,48 @@ class disable_weight_init:return super().forward(*args, **kwargs)class Embedding(torch.nn.Embedding, CastWeightBiasOp):+ def __init__(self, num_embeddings, embedding_dim, padding_idx=None, max_norm=None,+ norm_type=2.0, scale_grad_by_freq=False, sparse=False, _weight=None,+ _freeze=False, device=None, dtype=None):+ if not comfy.model_management.WINDOWS or not comfy.memory_management.aimdo_enabled:+ super().__init__(num_embeddings, embedding_dim, padding_idx, max_norm,+ norm_type, scale_grad_by_freq, sparse, _weight,+ _freeze, device, dtype)+ return++ torch.nn.Module.__init__(self)+ self.num_embeddings = num_embeddings+ self.embedding_dim = embedding_dim+ self.padding_idx = padding_idx+ self.max_norm = max_norm+ self.norm_type = norm_type+ self.scale_grad_by_freq = scale_grad_by_freq+ self.sparse = sparse+ # Keep shape/dtype visible for module introspection without reserving storage.+ embedding_dtype = dtype if dtype is not None else torch.get_default_dtype()+ self.weight = torch.nn.Parameter(+ torch.empty((num_embeddings, embedding_dim), device="meta", dtype=embedding_dtype),+ requires_grad=False,+ )+ self.bias = None+ self.weight_comfy_model_dtype = dtype++ def _load_from_state_dict(self, state_dict, prefix, local_metadata,+ strict, missing_keys, unexpected_keys, error_msgs):++ if not comfy.model_management.WINDOWS or not comfy.memory_management.aimdo_enabled:+ return super()._load_from_state_dict(state_dict, prefix, local_metadata, strict,+ missing_keys, unexpected_keys, error_msgs)+ disable_weight_init._lazy_load_from_state_dict(+ self,+ state_dict,+ prefix,+ local_metadata,+ missing_keys,+ unexpected_keys,+ weight_shape=(self.num_embeddings, self.embedding_dim),+ )+def reset_parameters(self):self.bias = Nonereturn Nonediff --git a/comfy/pinned_memory.py b/comfy/pinned_memory.pyindex 8acc327a7..f6fb806c4 100644--- a/comfy/pinned_memory.py+++ b/comfy/pinned_memory.py@@ -1,6 +1,7 @@-import torchimport comfy.model_managementimport comfy.memory_management+import comfy_aimdo.host_buffer+import comfy_aimdo.torchfrom comfy.cli_args import args@@ -12,18 +13,31 @@ def pin_memory(module):return#FIXME: This is a RAM cache trigger eventsize = comfy.memory_management.vram_aligned_size([ module.weight, module.bias ])- pin = torch.empty((size,), dtype=torch.uint8)- if comfy.model_management.pin_memory(pin):- module._pin = pin- else:++ if comfy.model_management.MAX_PINNED_MEMORY <= 0 or (comfy.model_management.TOTAL_PINNED_MEMORY + size) > comfy.model_management.MAX_PINNED_MEMORY:+ module.pin_failed = True+ return False++ try:+ hostbuf = comfy_aimdo.host_buffer.HostBuffer(size)+ except RuntimeError:module.pin_failed = Truereturn False++ module._pin = comfy_aimdo.torch.hostbuf_to_tensor(hostbuf)+ module._pin_hostbuf = hostbuf+ comfy.model_management.TOTAL_PINNED_MEMORY += sizereturn Truedef unpin_memory(module):if get_pin(module) is None:return 0size = module._pin.numel() * module._pin.element_size()- comfy.model_management.unpin_memory(module._pin)++ comfy.model_management.TOTAL_PINNED_MEMORY -= size+ if comfy.model_management.TOTAL_PINNED_MEMORY < 0:+ comfy.model_management.TOTAL_PINNED_MEMORY = 0+del module._pin+ del module._pin_hostbufreturn sizediff --git a/comfy/utils.py b/comfy/utils.pyindex 6e1d14419..9931fe3b4 100644--- a/comfy/utils.py+++ b/comfy/utils.py@@ -20,6 +20,8 @@import torchimport mathimport struct+import ctypes+import osimport comfy.memory_managementimport safetensors.torchimport numpy as np@@ -32,7 +34,7 @@ from einops import rearrangefrom comfy.cli_args import argsimport jsonimport time-import mmap+import threadingimport warningsMMAP_TORCH_FILES = args.mmap_torch_files@@ -81,14 +83,17 @@ _TYPES = {}def load_safetensors(ckpt):- f = open(ckpt, "rb")- mapping = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)- mv = memoryview(mapping)+ import comfy_aimdo.model_mmap- header_size = struct.unpack("<Q", mapping[:8])[0]- header = json.loads(mapping[8:8+header_size].decode("utf-8"))+ f = open(ckpt, "rb", buffering=0)+ model_mmap = comfy_aimdo.model_mmap.ModelMMAP(ckpt)+ file_size = os.path.getsize(ckpt)+ mv = memoryview((ctypes.c_uint8 * file_size).from_address(model_mmap.get()))- mv = mv[8 + header_size:]+ header_size = struct.unpack("<Q", mv[:8])[0]+ header = json.loads(mv[8:8 + header_size].tobytes().decode("utf-8"))++ mv = mv[(data_base_offset := 8 + header_size):]sd = {}for name, info in header.items():@@ -102,7 +107,14 @@ def load_safetensors(ckpt):with warnings.catch_warnings():#We are working with read-only RAM by designwarnings.filterwarnings("ignore", message="The given buffer is not writable")- sd[name] = torch.frombuffer(mv[start:end], dtype=_TYPES[info["dtype"]]).view(info["shape"])+ tensor = torch.frombuffer(mv[start:end], dtype=_TYPES[info["dtype"]]).view(info["shape"])+ storage = tensor.untyped_storage()+ setattr(storage,+ "_comfy_tensor_file_slice",+ comfy.memory_management.TensorFileSlice(f, threading.get_ident(), data_base_offset + start, end - start))+ setattr(storage, "_comfy_tensor_mmap_refs", (model_mmap, mv))+ setattr(storage, "_comfy_tensor_mmap_touched", False)+ sd[name] = tensorreturn sd, header.get("__metadata__", {}),diff --git a/requirements.txt b/requirements.txtindex 6efb77f29..52bc0fd12 100644--- a/requirements.txt+++ b/requirements.txt@@ -23,7 +23,7 @@ SQLAlchemyfilelockav>=14.2.0comfy-kitchen>=0.2.8-comfy-aimdo>=0.2.10+comfy-aimdo>=0.2.11requestssimpleeval>=1.0.0blake3