The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add support for BiRefNet background remove model (CORE-46) (#12747)
Yousef R. Gamaleldind3c18c163665
diff --git a/comfy/background_removal/birefnet.json b/comfy/background_removal/birefnet.jsonnew file mode 100644index 000000000..f0960af39--- /dev/null+++ b/comfy/background_removal/birefnet.json@@ -0,0 +1,7 @@+{+ "model_type": "birefnet",+ "image_std": [1.0, 1.0, 1.0],+ "image_mean": [0.0, 0.0, 0.0],+ "image_size": 1024,+ "resize_to_original": true+}diff --git a/comfy/background_removal/birefnet.py b/comfy/background_removal/birefnet.pynew file mode 100644index 000000000..df54b2b90--- /dev/null+++ b/comfy/background_removal/birefnet.py@@ -0,0 +1,689 @@+import torch+import comfy.ops+import numpy as np+import torch.nn as nn+from functools import partial+import torch.nn.functional as F+from torchvision.ops import deform_conv2d+from comfy.ldm.modules.attention import optimized_attention_for_device++CXT = [3072, 1536, 768, 384][1:][::-1][-3:]++class Attention(nn.Module):+ def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, device=None, dtype=None, operations=None):+ super().__init__()++ self.dim = dim+ self.num_heads = num_heads+ head_dim = dim // num_heads+ self.scale = qk_scale or head_dim ** -0.5++ self.q = operations.Linear(dim, dim, bias=qkv_bias, device=device, dtype=dtype)+ self.kv = operations.Linear(dim, dim * 2, bias=qkv_bias, device=device, dtype=dtype)+ self.proj = operations.Linear(dim, dim, device=device, dtype=dtype)++ def forward(self, x):+ B, N, C = x.shape+ optimized_attention = optimized_attention_for_device(x.device, mask=False, small_input=True)+ q = self.q(x).reshape(B, N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3)+ kv = self.kv(x).reshape(B, -1, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)+ k, v = kv[0], kv[1]++ x = optimized_attention(+ q, k, v, heads=self.num_heads, skip_output_reshape=True, skip_reshape=True+ ).transpose(1, 2).reshape(B, N, C)+ x = self.proj(x)++ return x++class Mlp(nn.Module):+ def __init__(self, in_features, hidden_features=None, out_features=None, device=None, dtype=None, operations=None):+ super().__init__()+ out_features = out_features or in_features+ hidden_features = hidden_features or in_features+ self.fc1 = operations.Linear(in_features, hidden_features, device=device, dtype=dtype)+ self.act = nn.GELU()+ self.fc2 = operations.Linear(hidden_features, out_features, device=device, dtype=dtype)++ def forward(self, x):+ x = self.fc1(x)+ x = self.act(x)+ x = self.fc2(x)+ return x+++def window_partition(x, window_size):+ B, H, W, C = x.shape+ x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)+ windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)+ return windows+++def window_reverse(windows, window_size, H, W):+ B = int(windows.shape[0] / (H * W / window_size / window_size))+ x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)+ x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)+ return x+++class WindowAttention(nn.Module):+ def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, device=None, dtype=None, operations=None):++ super().__init__()+ self.dim = dim+ self.window_size = window_size # Wh, Ww+ self.num_heads = num_heads+ head_dim = dim // num_heads+ self.scale = qk_scale or head_dim ** -0.5++ self.relative_position_bias_table = nn.Parameter(+ torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads, device=device, dtype=dtype))++ coords_h = torch.arange(self.window_size[0])+ coords_w = torch.arange(self.window_size[1])+ coords = torch.stack(torch.meshgrid([coords_h, coords_w], indexing='ij')) # 2, Wh, Ww+ coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww+ relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww+ relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2+ relative_coords[:, :, 0] += self.window_size[0] - 1+ relative_coords[:, :, 1] += self.window_size[1] - 1+ relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1+ relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww+ self.register_buffer("relative_position_index", relative_position_index)++ self.qkv = operations.Linear(dim, dim * 3, bias=qkv_bias, device=device, dtype=dtype)+ self.proj = operations.Linear(dim, dim, device=device, dtype=dtype)+ self.softmax = nn.Softmax(dim=-1)++ def forward(self, x, mask=None):+ B_, N, C = x.shape+ qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)+ q, k, v = qkv[0], qkv[1], qkv[2]++ q = q * self.scale+ attn = (q @ k.transpose(-2, -1))++ relative_position_bias = self.relative_position_bias_table[self.relative_position_index.long().view(-1)].view(+ self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1) # Wh*Ww,Wh*Ww,nH+ relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww+ attn = attn + relative_position_bias.unsqueeze(0)++ if mask is not None:+ nW = mask.shape[0]+ attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)+ attn = attn.view(-1, self.num_heads, N, N)+ attn = self.softmax(attn)+ else:+ attn = self.softmax(attn)++ x = (attn @ v).transpose(1, 2).reshape(B_, N, C)+ x = self.proj(x)+ return x+++class SwinTransformerBlock(nn.Module):+ def __init__(self, dim, num_heads, window_size=7, shift_size=0,+ mlp_ratio=4., qkv_bias=True, qk_scale=None,+ norm_layer=nn.LayerNorm, device=None, dtype=None, operations=None):+ super().__init__()+ self.dim = dim+ self.num_heads = num_heads+ self.window_size = window_size+ self.shift_size = shift_size+ self.mlp_ratio = mlp_ratio++ self.norm1 = norm_layer(dim, device=device, dtype=dtype)+ self.attn = WindowAttention(+ dim, window_size=(self.window_size, self.window_size), num_heads=num_heads,+ qkv_bias=qkv_bias, qk_scale=qk_scale, device=device, dtype=dtype, operations=operations)++ self.norm2 = norm_layer(dim, device=device, dtype=dtype)+ mlp_hidden_dim = int(dim * mlp_ratio)+ self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, device=device, dtype=dtype, operations=operations)++ self.H = None+ self.W = None++ def forward(self, x, mask_matrix):+ B, L, C = x.shape+ H, W = self.H, self.W++ shortcut = x+ x = self.norm1(x)+ x = x.view(B, H, W, C)++ pad_l = pad_t = 0+ pad_r = (self.window_size - W % self.window_size) % self.window_size+ pad_b = (self.window_size - H % self.window_size) % self.window_size+ x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b))+ _, Hp, Wp, _ = x.shape++ if self.shift_size > 0:+ shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))+ attn_mask = mask_matrix+ else:+ shifted_x = x+ attn_mask = None++ x_windows = window_partition(shifted_x, self.window_size)+ x_windows = x_windows.view(-1, self.window_size * self.window_size, C)++ attn_windows = self.attn(x_windows, mask=attn_mask)++ attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)+ shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp) # B H' W' C++ if self.shift_size > 0:+ x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))+ else:+ x = shifted_x++ if pad_r > 0 or pad_b > 0:+ x = x[:, :H, :W, :].contiguous()++ x = x.view(B, H * W, C)++ x = shortcut + x+ x = x + self.mlp(self.norm2(x))++ return x+++class PatchMerging(nn.Module):+ def __init__(self, dim, device=None, dtype=None, operations=None):+ super().__init__()+ self.dim = dim+ self.reduction = operations.Linear(4 * dim, 2 * dim, bias=False, device=device, dtype=dtype)+ self.norm = operations.LayerNorm(4 * dim, device=device, dtype=dtype)++ def forward(self, x, H, W):+ B, L, C = x.shape+ x = x.view(B, H, W, C)++ # padding+ pad_input = (H % 2 == 1) or (W % 2 == 1)+ if pad_input:+ x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2))++ x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C+ x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C+ x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C+ x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C+ x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C+ x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C++ x = self.norm(x)+ x = self.reduction(x)++ return x+++class BasicLayer(nn.Module):+ def __init__(self,+ dim,+ depth,+ num_heads,+ window_size=7,+ mlp_ratio=4.,+ qkv_bias=True,+ qk_scale=None,+ norm_layer=nn.LayerNorm,+ downsample=None,+ device=None, dtype=None, operations=None):+ super().__init__()+ self.window_size = window_size+ self.shift_size = window_size // 2+ self.depth = depth++ # build blocks+ self.blocks = nn.ModuleList([+ SwinTransformerBlock(+ dim=dim,+ num_heads=num_heads,+ window_size=window_size,+ shift_size=0 if (i % 2 == 0) else window_size // 2,+ mlp_ratio=mlp_ratio,+ qkv_bias=qkv_bias,+ qk_scale=qk_scale,+ norm_layer=norm_layer,+ device=device, dtype=dtype, operations=operations)+ for i in range(depth)])++ # patch merging layer+ if downsample is not None:+ self.downsample = downsample(dim=dim, device=device, dtype=dtype, operations=operations)+ else:+ self.downsample = None++ def forward(self, x, H, W):+ Hp = int(np.ceil(H / self.window_size)) * self.window_size+ Wp = int(np.ceil(W / self.window_size)) * self.window_size+ img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1+ h_slices = (slice(0, -self.window_size),+ slice(-self.window_size, -self.shift_size),+ slice(-self.shift_size, None))+ w_slices = (slice(0, -self.window_size),+ slice(-self.window_size, -self.shift_size),+ slice(-self.shift_size, None))+ cnt = 0+ for h in h_slices:+ for w in w_slices:+ img_mask[:, h, w, :] = cnt+ cnt += 1++ mask_windows = window_partition(img_mask, self.window_size)+ mask_windows = mask_windows.view(-1, self.window_size * self.window_size)+ attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)+ attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))++ for blk in self.blocks:+ blk.H, blk.W = H, W+ x = blk(x, attn_mask)+ if self.downsample is not None:+ x_down = self.downsample(x, H, W)+ Wh, Ww = (H + 1) // 2, (W + 1) // 2+ return x, H, W, x_down, Wh, Ww+ else:+ return x, H, W, x, H, W+++class PatchEmbed(nn.Module):+ def __init__(self, patch_size=4, in_channels=3, embed_dim=96, norm_layer=None, device=None, dtype=None, operations=None):+ super().__init__()+ patch_size = (patch_size, patch_size)+ self.patch_size = patch_size++ self.in_channels = in_channels+ self.embed_dim = embed_dim++ self.proj = operations.Conv2d(in_channels, embed_dim, kernel_size=patch_size, stride=patch_size, device=device, dtype=dtype)+ if norm_layer is not None:+ self.norm = norm_layer(embed_dim, device=device, dtype=dtype)+ else:+ self.norm = None++ def forward(self, x):+ _, _, H, W = x.size()+ if W % self.patch_size[1] != 0:+ x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1]))+ if H % self.patch_size[0] != 0:+ x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0]))++ x = self.proj(x) # B C Wh Ww+ if self.norm is not None:+ Wh, Ww = x.size(2), x.size(3)+ x = x.flatten(2).transpose(1, 2)+ x = self.norm(x)+ x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww)++ return x+++class SwinTransformer(nn.Module):+ def __init__(self,+ pretrain_img_size=224,+ patch_size=4,+ in_channels=3,+ embed_dim=96,+ depths=[2, 2, 6, 2],+ num_heads=[3, 6, 12, 24],+ window_size=7,+ mlp_ratio=4.,+ qkv_bias=True,+ qk_scale=None,+ patch_norm=True,+ out_indices=(0, 1, 2, 3),+ frozen_stages=-1,+ device=None, dtype=None, operations=None):+ super().__init__()++ norm_layer = partial(operations.LayerNorm, device=device, dtype=dtype)+ self.pretrain_img_size = pretrain_img_size+ self.num_layers = len(depths)+ self.embed_dim = embed_dim+ self.patch_norm = patch_norm+ self.out_indices = out_indices+ self.frozen_stages = frozen_stages++ self.patch_embed = PatchEmbed(+ patch_size=patch_size, in_channels=in_channels, embed_dim=embed_dim,+ device=device, dtype=dtype, operations=operations,+ norm_layer=norm_layer if self.patch_norm else None)++ self.layers = nn.ModuleList()+ for i_layer in range(self.num_layers):+ layer = BasicLayer(+ dim=int(embed_dim * 2 ** i_layer),+ depth=depths[i_layer],+ num_heads=num_heads[i_layer],+ window_size=window_size,+ mlp_ratio=mlp_ratio,+ qkv_bias=qkv_bias,+ qk_scale=qk_scale,+ norm_layer=norm_layer,+ downsample=PatchMerging if (i_layer < self.num_layers - 1) else None,+ device=device, dtype=dtype, operations=operations)+ self.layers.append(layer)++ num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)]+ self.num_features = num_features++ for i_layer in out_indices:+ layer = norm_layer(num_features[i_layer])+ layer_name = f'norm{i_layer}'+ self.add_module(layer_name, layer)+++ def forward(self, x):+ x = self.patch_embed(x)++ Wh, Ww = x.size(2), x.size(3)++ outs = []+ x = x.flatten(2).transpose(1, 2)+ for i in range(self.num_layers):+ layer = self.layers[i]+ x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww)++ if i in self.out_indices:+ norm_layer = getattr(self, f'norm{i}')+ x_out = norm_layer(x_out)++ out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous()+ outs.append(out)++ return tuple(outs)++class DeformableConv2d(nn.Module):+ def __init__(self,+ in_channels,+ out_channels,+ kernel_size=3,+ stride=1,+ padding=1,+ bias=False, device=None, dtype=None, operations=None):++ super(DeformableConv2d, self).__init__()++ kernel_size = kernel_size if type(kernel_size) is tuple else (kernel_size, kernel_size)+ self.stride = stride if type(stride) is tuple else (stride, stride)+ self.padding = padding++ self.offset_conv = operations.Conv2d(in_channels,+ 2 * kernel_size[0] * kernel_size[1],+ kernel_size=kernel_size,+ stride=stride,+ padding=self.padding,+ bias=True, device=device, dtype=dtype)++ self.modulator_conv = operations.Conv2d(in_channels,+ 1 * kernel_size[0] * kernel_size[1],+ kernel_size=kernel_size,+ stride=stride,+ padding=self.padding,+ bias=True, device=device, dtype=dtype)++ self.regular_conv = operations.Conv2d(in_channels,+ out_channels=out_channels,+ kernel_size=kernel_size,+ stride=stride,+ padding=self.padding,+ bias=bias, device=device, dtype=dtype)++ def forward(self, x):+ offset = self.offset_conv(x)+ modulator = 2. * torch.sigmoid(self.modulator_conv(x))+ weight, bias, offload_info = comfy.ops.cast_bias_weight(self.regular_conv, x, offloadable=True)++ x = deform_conv2d(+ input=x,+ offset=offset,+ weight=weight,+ bias=None,+ padding=self.padding,+ mask=modulator,+ stride=self.stride,+ )+ comfy.ops.uncast_bias_weight(self.regular_conv, weight, bias, offload_info)+ return x++class BasicDecBlk(nn.Module):+ def __init__(self, in_channels=64, out_channels=64, inter_channels=64, device=None, dtype=None, operations=None):+ super(BasicDecBlk, self).__init__()+ inter_channels = 64+ self.conv_in = operations.Conv2d(in_channels, inter_channels, 3, 1, padding=1, device=device, dtype=dtype)+ self.relu_in = nn.ReLU(inplace=True)+ self.dec_att = ASPPDeformable(in_channels=inter_channels, device=device, dtype=dtype, operations=operations)+ self.conv_out = operations.Conv2d(inter_channels, out_channels, 3, 1, padding=1, device=device, dtype=dtype)+ self.bn_in = operations.BatchNorm2d(inter_channels, device=device, dtype=dtype)+ self.bn_out = operations.BatchNorm2d(out_channels, device=device, dtype=dtype)++ def forward(self, x):+ x = self.conv_in(x)+ x = self.bn_in(x)+ x = self.relu_in(x)+ x = self.dec_att(x)+ x = self.conv_out(x)+ x = self.bn_out(x)+ return x+++class BasicLatBlk(nn.Module):+ def __init__(self, in_channels=64, out_channels=64, device=None, dtype=None, operations=None):+ super(BasicLatBlk, self).__init__()+ self.conv = operations.Conv2d(in_channels, out_channels, 1, 1, 0, device=device, dtype=dtype)++ def forward(self, x):+ x = self.conv(x)+ return x+++class _ASPPModuleDeformable(nn.Module):+ def __init__(self, in_channels, planes, kernel_size, padding, device, dtype, operations):+ super(_ASPPModuleDeformable, self).__init__()+ self.atrous_conv = DeformableConv2d(in_channels, planes, kernel_size=kernel_size,+ stride=1, padding=padding, bias=False, device=device, dtype=dtype, operations=operations)+ self.bn = operations.BatchNorm2d(planes, device=device, dtype=dtype)+ self.relu = nn.ReLU(inplace=True)++ def forward(self, x):+ x = self.atrous_conv(x)+ x = self.bn(x)++ return self.relu(x)+++class ASPPDeformable(nn.Module):+ def __init__(self, in_channels, out_channels=None, parallel_block_sizes=[1, 3, 7], device=None, dtype=None, operations=None):+ super(ASPPDeformable, self).__init__()+ self.down_scale = 1+ if out_channels is None:+ out_channels = in_channels+ self.in_channelster = 256 // self.down_scale++ self.aspp1 = _ASPPModuleDeformable(in_channels, self.in_channelster, 1, padding=0, device=device, dtype=dtype, operations=operations)+ self.aspp_deforms = nn.ModuleList([+ _ASPPModuleDeformable(in_channels, self.in_channelster, conv_size, padding=int(conv_size//2), device=device, dtype=dtype, operations=operations)+ for conv_size in parallel_block_sizes+ ])++ self.global_avg_pool = nn.Sequential(nn.AdaptiveAvgPool2d((1, 1)),+ operations.Conv2d(in_channels, self.in_channelster, 1, stride=1, bias=False, device=device, dtype=dtype),+ operations.BatchNorm2d(self.in_channelster, device=device, dtype=dtype),+ nn.ReLU(inplace=True))+ self.conv1 = operations.Conv2d(self.in_channelster * (2 + len(self.aspp_deforms)), out_channels, 1, bias=False, device=device, dtype=dtype)+ self.bn1 = operations.BatchNorm2d(out_channels, device=device, dtype=dtype)+ self.relu = nn.ReLU(inplace=True)++ def forward(self, x):+ x1 = self.aspp1(x)+ x_aspp_deforms = [aspp_deform(x) for aspp_deform in self.aspp_deforms]+ x5 = self.global_avg_pool(x)+ x5 = F.interpolate(x5, size=x1.size()[2:], mode='bilinear', align_corners=True)+ x = torch.cat((x1, *x_aspp_deforms, x5), dim=1)++ x = self.conv1(x)+ x = self.bn1(x)+ x = self.relu(x)++ return x++class BiRefNet(nn.Module):+ def __init__(self, config=None, dtype=None, device=None, operations=None):+ super(BiRefNet, self).__init__()+ self.bb = SwinTransformer(embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48], window_size=12, device=device, dtype=dtype, operations=operations)++ channels = [1536, 768, 384, 192]+ channels = [c * 2 for c in channels]+ self.cxt = channels[1:][::-1][-3:]+ self.squeeze_module = nn.Sequential(*[+ BasicDecBlk(channels[0]+sum(self.cxt), channels[0], device=device, dtype=dtype, operations=operations)+ for _ in range(1)+ ])++ self.decoder = Decoder(channels, device=device, dtype=dtype, operations=operations)++ def forward_enc(self, x):+ x1, x2, x3, x4 = self.bb(x)+ B, C, H, W = x.shape+ x1_, x2_, x3_, x4_ = self.bb(F.interpolate(x, size=(H//2, W//2), mode='bilinear', align_corners=True))+ x1 = torch.cat([x1, F.interpolate(x1_, size=x1.shape[2:], mode='bilinear', align_corners=True)], dim=1)+ x2 = torch.cat([x2, F.interpolate(x2_, size=x2.shape[2:], mode='bilinear', align_corners=True)], dim=1)+ x3 = torch.cat([x3, F.interpolate(x3_, size=x3.shape[2:], mode='bilinear', align_corners=True)], dim=1)+ x4 = torch.cat([x4, F.interpolate(x4_, size=x4.shape[2:], mode='bilinear', align_corners=True)], dim=1)+ x4 = torch.cat(+ (+ *[+ F.interpolate(x1, size=x4.shape[2:], mode='bilinear', align_corners=True),+ F.interpolate(x2, size=x4.shape[2:], mode='bilinear', align_corners=True),+ F.interpolate(x3, size=x4.shape[2:], mode='bilinear', align_corners=True),+ ][-len(CXT):],+ x4+ ),+ dim=1+ )+ return (x1, x2, x3, x4)++ def forward_ori(self, x):+ (x1, x2, x3, x4) = self.forward_enc(x)+ x4 = self.squeeze_module(x4)+ features = [x, x1, x2, x3, x4]+ scaled_preds = self.decoder(features)+ return scaled_preds++ def forward(self, pixel_values, intermediate_output=None):+ scaled_preds = self.forward_ori(pixel_values)+ return scaled_preds+++class Decoder(nn.Module):+ def __init__(self, channels, device, dtype, operations):+ super(Decoder, self).__init__()+ # factory kwargs+ fk = {"device":device, "dtype":dtype, "operations":operations}+ DecoderBlock = partial(BasicDecBlk, **fk)+ LateralBlock = partial(BasicLatBlk, **fk)+ DBlock = partial(SimpleConvs, **fk)++ self.split = True+ N_dec_ipt = 64+ ic = 64+ ipt_cha_opt = 1+ self.ipt_blk5 = DBlock(2**10*3 if self.split else 3, [N_dec_ipt, channels[0]//8][ipt_cha_opt], inter_channels=ic)+ self.ipt_blk4 = DBlock(2**8*3 if self.split else 3, [N_dec_ipt, channels[0]//8][ipt_cha_opt], inter_channels=ic)+ self.ipt_blk3 = DBlock(2**6*3 if self.split else 3, [N_dec_ipt, channels[1]//8][ipt_cha_opt], inter_channels=ic)+ self.ipt_blk2 = DBlock(2**4*3 if self.split else 3, [N_dec_ipt, channels[2]//8][ipt_cha_opt], inter_channels=ic)+ self.ipt_blk1 = DBlock(2**0*3 if self.split else 3, [N_dec_ipt, channels[3]//8][ipt_cha_opt], inter_channels=ic)++ self.decoder_block4 = DecoderBlock(channels[0]+([N_dec_ipt, channels[0]//8][ipt_cha_opt]), channels[1])+ self.decoder_block3 = DecoderBlock(channels[1]+([N_dec_ipt, channels[0]//8][ipt_cha_opt]), channels[2])+ self.decoder_block2 = DecoderBlock(channels[2]+([N_dec_ipt, channels[1]//8][ipt_cha_opt]), channels[3])+ self.decoder_block1 = DecoderBlock(channels[3]+([N_dec_ipt, channels[2]//8][ipt_cha_opt]), channels[3]//2)++ fk = {"device":device, "dtype":dtype}++ self.conv_out1 = nn.Sequential(operations.Conv2d(channels[3]//2+([N_dec_ipt, channels[3]//8][ipt_cha_opt]), 1, 1, 1, 0, **fk))++ self.lateral_block4 = LateralBlock(channels[1], channels[1])+ self.lateral_block3 = LateralBlock(channels[2], channels[2])+ self.lateral_block2 = LateralBlock(channels[3], channels[3])++ self.conv_ms_spvn_4 = operations.Conv2d(channels[1], 1, 1, 1, 0, **fk)+ self.conv_ms_spvn_3 = operations.Conv2d(channels[2], 1, 1, 1, 0, **fk)+ self.conv_ms_spvn_2 = operations.Conv2d(channels[3], 1, 1, 1, 0, **fk)++ _N = 16++ self.gdt_convs_4 = nn.Sequential(operations.Conv2d(channels[0] // 2, _N, 3, 1, 1, **fk), operations.BatchNorm2d(_N, **fk), nn.ReLU(inplace=True))+ self.gdt_convs_3 = nn.Sequential(operations.Conv2d(channels[1] // 2, _N, 3, 1, 1, **fk), operations.BatchNorm2d(_N, **fk), nn.ReLU(inplace=True))+ self.gdt_convs_2 = nn.Sequential(operations.Conv2d(channels[2] // 2, _N, 3, 1, 1, **fk), operations.BatchNorm2d(_N, **fk), nn.ReLU(inplace=True))++ [setattr(self, f"gdt_convs_pred_{i}", nn.Sequential(operations.Conv2d(_N, 1, 1, 1, 0, **fk))) for i in range(2, 5)]+ [setattr(self, f"gdt_convs_attn_{i}", nn.Sequential(operations.Conv2d(_N, 1, 1, 1, 0, **fk))) for i in range(2, 5)]++ def get_patches_batch(self, x, p):+ _size_h, _size_w = p.shape[2:]+ patches_batch = []+ for idx in range(x.shape[0]):+ columns_x = torch.split(x[idx], split_size_or_sections=_size_w, dim=-1)+ patches_x = []+ for column_x in columns_x:+ patches_x += [p.unsqueeze(0) for p in torch.split(column_x, split_size_or_sections=_size_h, dim=-2)]+ patch_sample = torch.cat(patches_x, dim=1)+ patches_batch.append(patch_sample)+ return torch.cat(patches_batch, dim=0)++ def forward(self, features):+ x, x1, x2, x3, x4 = features++ patches_batch = self.get_patches_batch(x, x4) if self.split else x+ x4 = torch.cat((x4, self.ipt_blk5(F.interpolate(patches_batch, size=x4.shape[2:], mode='bilinear', align_corners=True))), 1)+ p4 = self.decoder_block4(x4)+ p4_gdt = self.gdt_convs_4(p4)+ gdt_attn_4 = self.gdt_convs_attn_4(p4_gdt).sigmoid()+ p4 = p4 * gdt_attn_4+ _p4 = F.interpolate(p4, size=x3.shape[2:], mode='bilinear', align_corners=True)+ _p3 = _p4 + self.lateral_block4(x3)++ patches_batch = self.get_patches_batch(x, _p3) if self.split else x+ _p3 = torch.cat((_p3, self.ipt_blk4(F.interpolate(patches_batch, size=x3.shape[2:], mode='bilinear', align_corners=True))), 1)+ p3 = self.decoder_block3(_p3)++ p3_gdt = self.gdt_convs_3(p3)+ gdt_attn_3 = self.gdt_convs_attn_3(p3_gdt).sigmoid()+ p3 = p3 * gdt_attn_3+ _p3 = F.interpolate(p3, size=x2.shape[2:], mode='bilinear', align_corners=True)+ _p2 = _p3 + self.lateral_block3(x2)++ patches_batch = self.get_patches_batch(x, _p2) if self.split else x+ _p2 = torch.cat((_p2, self.ipt_blk3(F.interpolate(patches_batch, size=x2.shape[2:], mode='bilinear', align_corners=True))), 1)+ p2 = self.decoder_block2(_p2)++ p2_gdt = self.gdt_convs_2(p2)+ gdt_attn_2 = self.gdt_convs_attn_2(p2_gdt).sigmoid()+ p2 = p2 * gdt_attn_2++ _p2 = F.interpolate(p2, size=x1.shape[2:], mode='bilinear', align_corners=True)+ _p1 = _p2 + self.lateral_block2(x1)++ patches_batch = self.get_patches_batch(x, _p1) if self.split else x+ _p1 = torch.cat((_p1, self.ipt_blk2(F.interpolate(patches_batch, size=x1.shape[2:], mode='bilinear', align_corners=True))), 1)+ _p1 = self.decoder_block1(_p1)+ _p1 = F.interpolate(_p1, size=x.shape[2:], mode='bilinear', align_corners=True)++ patches_batch = self.get_patches_batch(x, _p1) if self.split else x+ _p1 = torch.cat((_p1, self.ipt_blk1(F.interpolate(patches_batch, size=x.shape[2:], mode='bilinear', align_corners=True))), 1)+ p1_out = self.conv_out1(_p1)+ return p1_out+++class SimpleConvs(nn.Module):+ def __init__(+ self, in_channels: int, out_channels: int, inter_channels=64, device=None, dtype=None, operations=None+ ) -> None:+ super().__init__()+ self.conv1 = operations.Conv2d(in_channels, inter_channels, 3, 1, 1, device=device, dtype=dtype)+ self.conv_out = operations.Conv2d(inter_channels, out_channels, 3, 1, 1, device=device, dtype=dtype)++ def forward(self, x):+ return self.conv_out(self.conv1(x))diff --git a/comfy/bg_removal_model.py b/comfy/bg_removal_model.pynew file mode 100644index 000000000..cb7c2ee53--- /dev/null+++ b/comfy/bg_removal_model.py@@ -0,0 +1,78 @@+from .utils import load_torch_file+import os+import json+import torch+import logging++import comfy.ops+import comfy.model_patcher+import comfy.model_management+import comfy.clip_model+import comfy.background_removal.birefnet++BG_REMOVAL_MODELS = {+ "birefnet": comfy.background_removal.birefnet.BiRefNet+}++class BackgroundRemovalModel():+ def __init__(self, json_config):+ with open(json_config) as f:+ config = json.load(f)++ self.image_size = config.get("image_size", 1024)+ self.image_mean = config.get("image_mean", [0.0, 0.0, 0.0])+ self.image_std = config.get("image_std", [1.0, 1.0, 1.0])+ self.model_type = config.get("model_type", "birefnet")+ self.config = config.copy()+ model_class = BG_REMOVAL_MODELS.get(self.model_type)++ self.load_device = comfy.model_management.text_encoder_device()+ offload_device = comfy.model_management.text_encoder_offload_device()+ self.dtype = comfy.model_management.text_encoder_dtype(self.load_device)+ self.model = model_class(config, self.dtype, offload_device, comfy.ops.manual_cast)+ self.model.eval()++ self.patcher = comfy.model_patcher.CoreModelPatcher(self.model, load_device=self.load_device, offload_device=offload_device)++ def load_sd(self, sd):+ return self.model.load_state_dict(sd, strict=False, assign=self.patcher.is_dynamic())++ def get_sd(self):+ return self.model.state_dict()++ def encode_image(self, image):+ comfy.model_management.load_model_gpu(self.patcher)+ H, W = image.shape[1], image.shape[2]+ pixel_values = comfy.clip_model.clip_preprocess(image.to(self.load_device), size=self.image_size, mean=self.image_mean, std=self.image_std, crop=False)+ out = self.model(pixel_values=pixel_values)+ out = torch.nn.functional.interpolate(out, size=(H, W), mode="bicubic", antialias=False)++ mask = out.sigmoid()+ if mask.ndim == 3:+ mask = mask.unsqueeze(0)+ if mask.shape[1] != 1:+ mask = mask.movedim(-1, 1)++ return mask+++def load_background_removal_model(sd):+ if "bb.layers.1.blocks.0.attn.relative_position_index" in sd:+ json_config = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "background_removal"), "birefnet.json")+ else:+ return None++ bg_model = BackgroundRemovalModel(json_config)+ m, u = bg_model.load_sd(sd)+ if len(m) > 0:+ logging.warning("missing background removal: {}".format(m))+ u = set(u)+ keys = list(sd.keys())+ for k in keys:+ if k not in u:+ sd.pop(k)+ return bg_model++def load(ckpt_path):+ sd = load_torch_file(ckpt_path)+ return load_background_removal_model(sd)diff --git a/comfy/ops.py b/comfy/ops.pyindex 585c185a3..77ad1d527 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -562,6 +562,25 @@ class disable_weight_init:else:return super().forward(*args, **kwargs)+ class BatchNorm2d(torch.nn.BatchNorm2d, CastWeightBiasOp):+ def reset_parameters(self):+ return None++ def forward_comfy_cast_weights(self, input):+ weight, bias, offload_stream = cast_bias_weight(self, input, offloadable=True)+ running_mean = self.running_mean.to(device=input.device, dtype=weight.dtype) if self.running_mean is not None else None+ running_var = self.running_var.to(device=input.device, dtype=weight.dtype) if self.running_var is not None else None+ x = torch.nn.functional.batch_norm(input, running_mean, running_var, weight, bias, self.training, self.momentum, self.eps)+ uncast_bias_weight(self, weight, bias, offload_stream)+ return x++ def forward(self, *args, **kwargs):+ run_every_op()+ if self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:+ return self.forward_comfy_cast_weights(*args, **kwargs)+ else:+ return super().forward(*args, **kwargs)+class LayerNorm(torch.nn.LayerNorm, CastWeightBiasOp):def reset_parameters(self):return None@@ -749,6 +768,9 @@ class manual_cast(disable_weight_init):class Conv3d(disable_weight_init.Conv3d):comfy_cast_weights = True+ class BatchNorm2d(disable_weight_init.BatchNorm2d):+ comfy_cast_weights = True+class GroupNorm(disable_weight_init.GroupNorm):comfy_cast_weights = Truediff --git a/comfy_api/latest/_io.py b/comfy_api/latest/_io.pyindex e50266bc5..5ed968960 100644--- a/comfy_api/latest/_io.py+++ b/comfy_api/latest/_io.py@@ -17,6 +17,7 @@ if TYPE_CHECKING:from spandrel import ImageModelDescriptorfrom comfy.clip_vision import ClipVisionModelfrom comfy.clip_vision import Output as ClipVisionOutput_+ from comfy.bg_removal_model import BackgroundRemovalModelfrom comfy.controlnet import ControlNetfrom comfy.hooks import HookGroup, HookKeyframeGroupfrom comfy.model_patcher import ModelPatcher@@ -614,6 +615,11 @@ class Model(ComfyTypeIO):if TYPE_CHECKING:Type = ModelPatcher+@comfytype(io_type="BACKGROUND_REMOVAL")+class BackgroundRemoval(ComfyTypeIO):+ if TYPE_CHECKING:+ Type = BackgroundRemovalModel+@comfytype(io_type="CLIP_VISION")class ClipVision(ComfyTypeIO):if TYPE_CHECKING:@@ -2257,6 +2263,7 @@ __all__ = ["ModelPatch","ClipVision","ClipVisionOutput",+ "BackgroundRemoval","AudioEncoder","AudioEncoderOutput","StyleModel",diff --git a/comfy_extras/nodes_bg_removal.py b/comfy_extras/nodes_bg_removal.pynew file mode 100644index 000000000..8d046b8d4--- /dev/null+++ b/comfy_extras/nodes_bg_removal.py@@ -0,0 +1,60 @@+import folder_paths+from typing_extensions import override+from comfy_api.latest import ComfyExtension, IO+from comfy.bg_removal_model import load+++class LoadBackgroundRemovalModel(IO.ComfyNode):+ @classmethod+ def define_schema(cls):+ files = folder_paths.get_filename_list("background_removal")+ return IO.Schema(+ node_id="LoadBackgroundRemovalModel",+ display_name="Load Background Removal Model",+ category="loaders",+ inputs=[+ IO.Combo.Input("bg_removal_name", options=sorted(files), tooltip="The model used to remove backgrounds from images"),+ ],+ outputs=[+ IO.BackgroundRemoval.Output("bg_model")+ ]+ )+ @classmethod+ def execute(cls, bg_removal_name):+ path = folder_paths.get_full_path_or_raise("background_removal", bg_removal_name)+ bg = load(path)+ if bg is None:+ raise RuntimeError("ERROR: background model file is invalid and does not contain a valid background removal model.")+ return IO.NodeOutput(bg)++class RemoveBackground(IO.ComfyNode):+ @classmethod+ def define_schema(cls):+ return IO.Schema(+ node_id="RemoveBackground",+ display_name="Remove Background",+ category="image/background removal",+ inputs=[+ IO.Image.Input("image", tooltip="Input image to remove the background from"),+ IO.BackgroundRemoval.Input("bg_removal_model", tooltip="Background removal model used to generate the mask")+ ],+ outputs=[+ IO.Mask.Output("mask", tooltip="Generated foreground mask")+ ]+ )+ @classmethod+ def execute(cls, image, bg_removal_model):+ mask = bg_removal_model.encode_image(image)+ return IO.NodeOutput(mask)++class BackgroundRemovalExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[IO.ComfyNode]]:+ return [+ LoadBackgroundRemovalModel,+ RemoveBackground+ ]+++async def comfy_entrypoint() -> BackgroundRemovalExtension:+ return BackgroundRemovalExtension()diff --git a/comfy_extras/nodes_mask.py b/comfy_extras/nodes_mask.pyindex 43a933dac..c9b2a84d9 100644--- a/comfy_extras/nodes_mask.py+++ b/comfy_extras/nodes_mask.py@@ -40,10 +40,21 @@ def composite(destination, source, x, y, mask = None, multiplier = 8, resize_souinverse_mask = torch.ones_like(mask) - mask- source_portion = mask * source[..., :visible_height, :visible_width]- destination_portion = inverse_mask * destination[..., top:bottom, left:right]+ source_rgb = source[:, :3, :visible_height, :visible_width]+ dest_slice = destination[..., top:bottom, left:right]++ if destination.shape[1] == 4:+ if torch.max(dest_slice) == 0:+ destination[:, :3, top:bottom, left:right] = source_rgb+ destination[:, 3:4, top:bottom, left:right] = mask+ else:+ destination[:, :3, top:bottom, left:right] = (mask * source_rgb) + (inverse_mask * dest_slice[:, :3])+ destination[:, 3:4, top:bottom, left:right] = torch.max(mask, dest_slice[:, 3:4])+ else:+ source_portion = mask * source_rgb+ destination_portion = inverse_mask * dest_slice+ destination[..., top:bottom, left:right] = source_portion + destination_portion- destination[..., top:bottom, left:right] = source_portion + destination_portionreturn destinationclass LatentCompositeMasked(IO.ComfyNode):@@ -84,18 +95,23 @@ class ImageCompositeMasked(IO.ComfyNode):display_name="Image Composite Masked",category="image",inputs=[- IO.Image.Input("destination"),IO.Image.Input("source"),IO.Int.Input("x", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1),IO.Int.Input("y", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1),IO.Boolean.Input("resize_source", default=False),+ IO.Image.Input("destination", optional=True),IO.Mask.Input("mask", optional=True),],outputs=[IO.Image.Output()],)@classmethod- def execute(cls, destination, source, x, y, resize_source, mask = None) -> IO.NodeOutput:+ def execute(cls, source, x, y, resize_source, destination = None, mask = None) -> IO.NodeOutput:+ if destination is None: # transparent rgba+ B, H, W, C = source.shape+ destination = torch.zeros((B, H, W, 4), dtype=source.dtype, device=source.device)+ if C == 3:+ source = torch.nn.functional.pad(source, (0, 1), value=1.0)destination, source = node_helpers.image_alpha_fix(destination, source)destination = destination.clone().movedim(-1, 1)output = composite(destination, source.movedim(-1, 1), x, y, mask, 1, resize_source).movedim(1, -1)@@ -381,7 +397,6 @@ class GrowMask(IO.ComfyNode):expand_mask = execute # TODO: remove-class ThresholdMask(IO.ComfyNode):@classmethoddef define_schema(cls):diff --git a/folder_paths.py b/folder_paths.pyindex 98d3b1880..92e8df3cf 100644--- a/folder_paths.py+++ b/folder_paths.py@@ -52,6 +52,8 @@ folder_names_and_paths["model_patches"] = ([os.path.join(models_dir, "model_patcfolder_names_and_paths["audio_encoders"] = ([os.path.join(models_dir, "audio_encoders")], supported_pt_extensions)+folder_names_and_paths["background_removal"] = ([os.path.join(models_dir, "background_removal")], supported_pt_extensions)+folder_names_and_paths["frame_interpolation"] = ([os.path.join(models_dir, "frame_interpolation")], supported_pt_extensions)folder_names_and_paths["optical_flow"] = ([os.path.join(models_dir, "optical_flow")], supported_pt_extensions)diff --git a/models/background_removal/put_background_removal_models_here b/models/background_removal/put_background_removal_models_herenew file mode 100644index 000000000..e69de29bbdiff --git a/nodes.py b/nodes.pyindex ae9e70cb9..5755f0bb8 100644--- a/nodes.py+++ b/nodes.py@@ -2429,6 +2429,7 @@ async def init_builtin_extra_nodes():"nodes_number_convert.py","nodes_painter.py","nodes_curve.py",+ "nodes_bg_removal.py","nodes_rtdetr.py","nodes_frame_interpolation.py","nodes_sam3.py",