The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add OFT/BOFT algorithm in weight adapter (#7725)
Kohaku-Blueleaf966c43ce2683
diff --git a/comfy/weight_adapter/__init__.py b/comfy/weight_adapter/__init__.pyindex e6cd805b6..d2a1d0151 100644--- a/comfy/weight_adapter/__init__.py+++ b/comfy/weight_adapter/__init__.py@@ -3,6 +3,8 @@ from .lora import LoRAAdapterfrom .loha import LoHaAdapterfrom .lokr import LoKrAdapterfrom .glora import GLoRAAdapter+from .oft import OFTAdapter+from .boft import BOFTAdapteradapters: list[type[WeightAdapterBase]] = [@@ -10,4 +12,6 @@ adapters: list[type[WeightAdapterBase]] = [LoHaAdapter,LoKrAdapter,GLoRAAdapter,+ OFTAdapter,+ BOFTAdapter,]diff --git a/comfy/weight_adapter/boft.py b/comfy/weight_adapter/boft.pynew file mode 100644index 000000000..c85adc7ab--- /dev/null+++ b/comfy/weight_adapter/boft.py@@ -0,0 +1,115 @@+import logging+from typing import Optional++import torch+import comfy.model_management+from .base import WeightAdapterBase, weight_decompose+++class BOFTAdapter(WeightAdapterBase):+ name = "boft"++ def __init__(self, loaded_keys, weights):+ self.loaded_keys = loaded_keys+ self.weights = weights++ @classmethod+ def load(+ cls,+ x: str,+ lora: dict[str, torch.Tensor],+ alpha: float,+ dora_scale: torch.Tensor,+ loaded_keys: set[str] = None,+ ) -> Optional["BOFTAdapter"]:+ if loaded_keys is None:+ loaded_keys = set()+ blocks_name = "{}.boft_blocks".format(x)+ rescale_name = "{}.rescale".format(x)++ blocks = None+ if blocks_name in lora.keys():+ blocks = lora[blocks_name]+ if blocks.ndim == 4:+ loaded_keys.add(blocks_name)++ rescale = None+ if rescale_name in lora.keys():+ rescale = lora[rescale_name]+ loaded_keys.add(rescale_name)++ if blocks is not None:+ weights = (blocks, rescale, alpha, dora_scale)+ return cls(loaded_keys, weights)+ else:+ return None++ def calculate_weight(+ self,+ weight,+ key,+ strength,+ strength_model,+ offset,+ function,+ intermediate_dtype=torch.float32,+ original_weight=None,+ ):+ v = self.weights+ blocks = v[0]+ rescale = v[1]+ alpha = v[2]+ dora_scale = v[3]++ blocks = comfy.model_management.cast_to_device(blocks, weight.device, intermediate_dtype)+ if rescale is not None:+ rescale = comfy.model_management.cast_to_device(rescale, weight.device, intermediate_dtype)++ boft_m, block_num, boft_b, *_ = blocks.shape++ try:+ # Get r+ I = torch.eye(boft_b, device=blocks.device, dtype=blocks.dtype)+ # for Q = -Q^T+ q = blocks - blocks.transpose(1, 2)+ normed_q = q+ if alpha > 0: # alpha in boft/bboft is for constraint+ q_norm = torch.norm(q) + 1e-8+ if q_norm > alpha:+ normed_q = q * alpha / q_norm+ # use float() to prevent unsupported type in .inverse()+ r = (I + normed_q) @ (I - normed_q).float().inverse()+ r = r.to(original_weight)++ inp = org = original_weight++ r_b = boft_b//2+ for i in range(boft_m):+ bi = r[i]+ g = 2+ k = 2**i * r_b+ if strength != 1:+ bi = bi * strength + (1-strength) * I+ inp = (+ inp.unflatten(-1, (-1, g, k))+ .transpose(-2, -1)+ .flatten(-3)+ .unflatten(-1, (-1, boft_b))+ )+ inp = torch.einsum("b n m, b n ... -> b m ...", inp, bi)+ inp = (+ inp.flatten(-2).unflatten(-1, (-1, k, g)).transpose(-2, -1).flatten(-3)+ )++ if rescale is not None:+ inp = inp * rescale++ lora_diff = inp - org+ lora_diff = comfy.model_management.cast_to_device(lora_diff, weight.device, intermediate_dtype)+ if dora_scale is not None:+ weight = weight_decompose(dora_scale, weight, lora_diff, alpha, strength, intermediate_dtype, function)+ else:+ weight += function(((strength * alpha) * lora_diff).type(weight.dtype))+ except Exception as e:+ logging.error("ERROR {} {} {}".format(self.name, key, e))+ return weightdiff --git a/comfy/weight_adapter/oft.py b/comfy/weight_adapter/oft.pynew file mode 100644index 000000000..0ea229b79--- /dev/null+++ b/comfy/weight_adapter/oft.py@@ -0,0 +1,94 @@+import logging+from typing import Optional++import torch+import comfy.model_management+from .base import WeightAdapterBase, weight_decompose+++class OFTAdapter(WeightAdapterBase):+ name = "oft"++ def __init__(self, loaded_keys, weights):+ self.loaded_keys = loaded_keys+ self.weights = weights++ @classmethod+ def load(+ cls,+ x: str,+ lora: dict[str, torch.Tensor],+ alpha: float,+ dora_scale: torch.Tensor,+ loaded_keys: set[str] = None,+ ) -> Optional["OFTAdapter"]:+ if loaded_keys is None:+ loaded_keys = set()+ blocks_name = "{}.oft_blocks".format(x)+ rescale_name = "{}.rescale".format(x)++ blocks = None+ if blocks_name in lora.keys():+ blocks = lora[blocks_name]+ if blocks.ndim == 3:+ loaded_keys.add(blocks_name)++ rescale = None+ if rescale_name in lora.keys():+ rescale = lora[rescale_name]+ loaded_keys.add(rescale_name)++ if blocks is not None:+ weights = (blocks, rescale, alpha, dora_scale)+ return cls(loaded_keys, weights)+ else:+ return None++ def calculate_weight(+ self,+ weight,+ key,+ strength,+ strength_model,+ offset,+ function,+ intermediate_dtype=torch.float32,+ original_weight=None,+ ):+ v = self.weights+ blocks = v[0]+ rescale = v[1]+ alpha = v[2]+ dora_scale = v[3]++ blocks = comfy.model_management.cast_to_device(blocks, weight.device, intermediate_dtype)+ if rescale is not None:+ rescale = comfy.model_management.cast_to_device(rescale, weight.device, intermediate_dtype)++ block_num, block_size, *_ = blocks.shape++ try:+ # Get r+ I = torch.eye(block_size, device=blocks.device, dtype=blocks.dtype)+ # for Q = -Q^T+ q = blocks - blocks.transpose(1, 2)+ normed_q = q+ if alpha > 0: # alpha in oft/boft is for constraint+ q_norm = torch.norm(q) + 1e-8+ if q_norm > alpha:+ normed_q = q * alpha / q_norm+ # use float() to prevent unsupported type in .inverse()+ r = (I + normed_q) @ (I - normed_q).float().inverse()+ r = r.to(original_weight)+ lora_diff = torch.einsum(+ "k n m, k n ... -> k m ...",+ (r * strength) - strength * I,+ original_weight,+ )+ if dora_scale is not None:+ weight = weight_decompose(dora_scale, weight, lora_diff, alpha, strength, intermediate_dtype, function)+ else:+ weight += function(((strength * alpha) * lora_diff).type(weight.dtype))+ except Exception as e:+ logging.error("ERROR {} {} {}".format(self.name, key, e))+ return weight