Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Loras can now replace the per layer attention config on a model. (#16880)

comfyanonymousf49c531e5494
diff --git a/comfy/configurable.py b/comfy/configurable.py
new file mode 100644
index 000000000..799ea5d90
--- /dev/null
+++ b/comfy/configurable.py
@@ -0,0 +1,7 @@
+from torch import nn
+
+
+class ConfigurableModule(nn.Module):
+ def with_config(self, encoded_config):
+ """Return a new module configured from a uint8 JSON tensor, without modifying this module."""
+ raise NotImplementedError
diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py
index d3ea61fa0..77ffd15d9 100644
--- a/comfy/ldm/modules/attention.py
+++ b/comfy/ldm/modules/attention.py
@@ -17,6 +17,7 @@ from .diffusionmodules.util import AlphaBlender, timestep_embedding
from .sub_quadratic_attention import efficient_dot_product_attention
from comfy import model_management
+from comfy.configurable import ConfigurableModule
if model_management.xformers_enabled():
import xformers
@@ -73,12 +74,17 @@ def get_attention_function(name: str, default: Any=...) -> Union[Callable, None]
return REGISTERED_ATTENTION_FUNCTIONS[name]
-class ComfyAttention(nn.Module):
+class ComfyAttention(ConfigurableModule):
def __init__(self):
super().__init__()
self.config = None
self.function = None
+ def with_config(self, encoded_config):
+ attention = ComfyAttention()
+ attention.load_state_dict({"config": encoded_config})
+ return attention
+
def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs):
self.config = None
self.function = None
diff --git a/comfy/lora.py b/comfy/lora.py
index 117f50ce8..36f286cc3 100644
--- a/comfy/lora.py
+++ b/comfy/lora.py
@@ -87,6 +87,11 @@ def load_lora(lora, to_load, log_missing=True):
patch_dict[to_load[x]] = ("set", (set_weight,))
loaded_keys.add(set_weight_name)
+ for key in lora:
+ if key.endswith(".config"):
+ patch_dict[key] = ("config", (lora[key],))
+ loaded_keys.add(key)
+
if log_missing:
for x in lora.keys():
if x not in loaded_keys:
diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py
index ca4c54e24..8472b4a45 100644
--- a/comfy/model_patcher.py
+++ b/comfy/model_patcher.py
@@ -38,6 +38,7 @@ import comfy.patcher_extension
import comfy.utils
import comfy_aimdo.host_buffer
from comfy.comfy_types import UnetWrapperFunction
+from comfy.configurable import ConfigurableModule
from comfy.internal_logging import detail
from comfy.quant_ops import QuantizedTensor
from comfy.patcher_extension import CallbacksMP, PatcherInjection, WrappersMP
@@ -843,6 +844,7 @@ class ModelPatcher:
def add_patches(self, patches, strength_patch=1.0, strength_model=1.0):
with self.use_ejected():
p = set()
+ weights_changed = False
model_sd = self.model.state_dict()
for k in patches:
offset = None
@@ -855,13 +857,28 @@ class ModelPatcher:
if len(k) > 2:
function = k[2]
+ patch = patches[k]
+ if isinstance(patch, tuple) and len(patch) == 2 and patch[0] == "config":
+ module_name, _, attribute = key.rpartition(".")
+ try:
+ module = self.get_model_object(module_name)
+ except AttributeError:
+ continue
+ if attribute == "config" and isinstance(module, ConfigurableModule):
+ p.add(k)
+ if strength_patch != 0:
+ self.add_object_patch(module_name, module.with_config(patch[1][0]))
+ continue
+
if key in model_sd:
p.add(k)
current_patches = self.patches.get(key, [])
current_patches.append((strength_patch, patches[k], strength_model, offset, function))
self.patches[key] = current_patches
+ weights_changed = True
- self.patches_uuid = uuid.uuid4()
+ if weights_changed:
+ self.patches_uuid = uuid.uuid4()
return list(p)
def get_key_patches(self, filter_prefix=None):
@@ -1544,6 +1561,8 @@ class ModelPatcher:
p = set()
model_sd = self.model.state_dict()
for k in patches:
+ if isinstance(patches[k], tuple) and len(patches[k]) == 2 and patches[k][0] == "config":
+ continue
offset = None
function = None
if isinstance(k, str):