Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Comfy Aimdo 0.2.5 + Fix offload performance in DynamicVram (#12754)

rattus9b85cf955858
diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py
index 168ce8430..7e5ad7aa4 100644
--- a/comfy/model_patcher.py
+++ b/comfy/model_patcher.py
@@ -307,7 +307,13 @@ class ModelPatcher:
return self.model.lowvram_patch_counter
def get_free_memory(self, device):
- return comfy.model_management.get_free_memory(device)
+ #Prioritize batching (incl. CFG/conds etc) over keeping the model resident. In
+ #the vast majority of setups a little bit of offloading on the giant model more
+ #than pays for CFG. So return everything both torch and Aimdo could give us
+ aimdo_mem = 0
+ if comfy.memory_management.aimdo_enabled:
+ aimdo_mem = comfy_aimdo.model_vbar.vbars_analyze()
+ return comfy.model_management.get_free_memory(device) + aimdo_mem
def get_clone_model_override(self):
return self.model, (self.backup, self.backup_buffers, self.object_patches_backup, self.pinned)
@@ -1465,12 +1471,6 @@ class ModelPatcherDynamic(ModelPatcher):
vbar = self._vbar_get()
return (vbar.loaded_size() if vbar is not None else 0) + self.model.model_loaded_weight_memory
- def get_free_memory(self, device):
- #NOTE: on high condition / batch counts, estimate should have already vacated
- #all non-dynamic models so this is safe even if its not 100% true that this
- #would all be avaiable for inference use.
- return comfy.model_management.get_total_memory(device) - self.model_size()
-
#Pinning is deferred to ops time. Assert against this API to avoid pin leaks.
def pin_weight_to_device(self, key):
diff --git a/comfy/ops.py b/comfy/ops.py
index 6ee6075fb..8275dd0a5 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -269,8 +269,8 @@ def uncast_bias_weight(s, weight, bias, offload_stream):
return
os, weight_a, bias_a = offload_stream
device=None
- #FIXME: This is not good RTTI
- if not isinstance(weight_a, torch.Tensor):
+ #FIXME: This is really bad RTTI
+ if weight_a is not None and not isinstance(weight_a, torch.Tensor):
comfy_aimdo.model_vbar.vbar_unpin(s._v)
device = weight_a
if os is None:
diff --git a/requirements.txt b/requirements.txt
index 608b0cfa6..110568cd3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -22,7 +22,7 @@ alembic
SQLAlchemy
av>=14.2.0
comfy-kitchen>=0.2.7
-comfy-aimdo>=0.2.4
+comfy-aimdo>=0.2.5
requests
#non essential dependencies: