The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Loras can now replace the per layer attention config on a model. (#16880)
comfyanonymousf49c531e5494
diff --git a/comfy/configurable.py b/comfy/configurable.pynew file mode 100644index 000000000..799ea5d90--- /dev/null+++ b/comfy/configurable.py@@ -0,0 +1,7 @@+from torch import nn+++class ConfigurableModule(nn.Module):+ def with_config(self, encoded_config):+ """Return a new module configured from a uint8 JSON tensor, without modifying this module."""+ raise NotImplementedErrordiff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.pyindex d3ea61fa0..77ffd15d9 100644--- a/comfy/ldm/modules/attention.py+++ b/comfy/ldm/modules/attention.py@@ -17,6 +17,7 @@ from .diffusionmodules.util import AlphaBlender, timestep_embeddingfrom .sub_quadratic_attention import efficient_dot_product_attentionfrom comfy import model_management+from comfy.configurable import ConfigurableModuleif model_management.xformers_enabled():import xformers@@ -73,12 +74,17 @@ def get_attention_function(name: str, default: Any=...) -> Union[Callable, None]return REGISTERED_ATTENTION_FUNCTIONS[name]-class ComfyAttention(nn.Module):+class ComfyAttention(ConfigurableModule):def __init__(self):super().__init__()self.config = Noneself.function = None+ def with_config(self, encoded_config):+ attention = ComfyAttention()+ attention.load_state_dict({"config": encoded_config})+ return attention+def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs):self.config = Noneself.function = Nonediff --git a/comfy/lora.py b/comfy/lora.pyindex 117f50ce8..36f286cc3 100644--- a/comfy/lora.py+++ b/comfy/lora.py@@ -87,6 +87,11 @@ def load_lora(lora, to_load, log_missing=True):patch_dict[to_load[x]] = ("set", (set_weight,))loaded_keys.add(set_weight_name)+ for key in lora:+ if key.endswith(".config"):+ patch_dict[key] = ("config", (lora[key],))+ loaded_keys.add(key)+if log_missing:for x in lora.keys():if x not in loaded_keys:diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex ca4c54e24..8472b4a45 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -38,6 +38,7 @@ import comfy.patcher_extensionimport comfy.utilsimport comfy_aimdo.host_bufferfrom comfy.comfy_types import UnetWrapperFunction+from comfy.configurable import ConfigurableModulefrom comfy.internal_logging import detailfrom comfy.quant_ops import QuantizedTensorfrom comfy.patcher_extension import CallbacksMP, PatcherInjection, WrappersMP@@ -843,6 +844,7 @@ class ModelPatcher:def add_patches(self, patches, strength_patch=1.0, strength_model=1.0):with self.use_ejected():p = set()+ weights_changed = Falsemodel_sd = self.model.state_dict()for k in patches:offset = None@@ -855,13 +857,28 @@ class ModelPatcher:if len(k) > 2:function = k[2]+ patch = patches[k]+ if isinstance(patch, tuple) and len(patch) == 2 and patch[0] == "config":+ module_name, _, attribute = key.rpartition(".")+ try:+ module = self.get_model_object(module_name)+ except AttributeError:+ continue+ if attribute == "config" and isinstance(module, ConfigurableModule):+ p.add(k)+ if strength_patch != 0:+ self.add_object_patch(module_name, module.with_config(patch[1][0]))+ continue+if key in model_sd:p.add(k)current_patches = self.patches.get(key, [])current_patches.append((strength_patch, patches[k], strength_model, offset, function))self.patches[key] = current_patches+ weights_changed = True- self.patches_uuid = uuid.uuid4()+ if weights_changed:+ self.patches_uuid = uuid.uuid4()return list(p)def get_key_patches(self, filter_prefix=None):@@ -1544,6 +1561,8 @@ class ModelPatcher:p = set()model_sd = self.model.state_dict()for k in patches:+ if isinstance(patches[k], tuple) and len(patches[k]) == 2 and patches[k][0] == "config":+ continueoffset = Nonefunction = Noneif isinstance(k, str):