Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

feat: RIFE and FILM frame interpolation model support (CORE-29) (#13258)

Jukka Seppänendb85cf03ff33
diff --git a/comfy_extras/frame_interpolation_models/film_net.py b/comfy_extras/frame_interpolation_models/film_net.py
new file mode 100644
index 000000000..cf4f6e1e1
--- /dev/null
+++ b/comfy_extras/frame_interpolation_models/film_net.py
@@ -0,0 +1,258 @@
+"""FILM: Frame Interpolation for Large Motion (ECCV 2022)."""
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+import comfy.ops
+
+ops = comfy.ops.disable_weight_init
+
+
+class FilmConv2d(nn.Module):
+ """Conv2d with optional LeakyReLU and FILM-style padding."""
+
+ def __init__(self, in_channels, out_channels, size, activation=True, device=None, dtype=None, operations=ops):
+ super().__init__()
+ self.even_pad = not size % 2
+ self.conv = operations.Conv2d(in_channels, out_channels, kernel_size=size, padding=size // 2 if size % 2 else 0, device=device, dtype=dtype)
+ self.activation = nn.LeakyReLU(0.2) if activation else None
+
+ def forward(self, x):
+ if self.even_pad:
+ x = F.pad(x, (0, 1, 0, 1))
+ x = self.conv(x)
+ if self.activation is not None:
+ x = self.activation(x)
+ return x
+
+
+def _warp_core(image, flow, grid_x, grid_y):
+ dtype = image.dtype
+ H, W = flow.shape[2], flow.shape[3]
+ dx = flow[:, 0].float() / (W * 0.5)
+ dy = flow[:, 1].float() / (H * 0.5)
+ grid = torch.stack([grid_x[None, None, :] + dx, grid_y[None, :, None] + dy], dim=3)
+ return F.grid_sample(image.float(), grid, mode="bilinear", padding_mode="border", align_corners=False).to(dtype)
+
+
+def build_image_pyramid(image, pyramid_levels):
+ pyramid = [image]
+ for _ in range(1, pyramid_levels):
+ image = F.avg_pool2d(image, 2, 2)
+ pyramid.append(image)
+ return pyramid
+
+
+def flow_pyramid_synthesis(residual_pyramid):
+ flow = residual_pyramid[-1]
+ flow_pyramid = [flow]
+ for residual_flow in residual_pyramid[:-1][::-1]:
+ flow = F.interpolate(flow, size=residual_flow.shape[2:4], mode="bilinear", scale_factor=None).mul_(2).add_(residual_flow)
+ flow_pyramid.append(flow)
+ flow_pyramid.reverse()
+ return flow_pyramid
+
+
+def multiply_pyramid(pyramid, scalar):
+ return [image * scalar[:, None, None, None] for image in pyramid]
+
+
+def pyramid_warp(feature_pyramid, flow_pyramid, warp_fn):
+ return [warp_fn(features, flow) for features, flow in zip(feature_pyramid, flow_pyramid)]
+
+
+def concatenate_pyramids(pyramid1, pyramid2):
+ return [torch.cat([f1, f2], dim=1) for f1, f2 in zip(pyramid1, pyramid2)]
+
+
+class SubTreeExtractor(nn.Module):
+ def __init__(self, in_channels=3, channels=64, n_layers=4, device=None, dtype=None, operations=ops):
+ super().__init__()
+ convs = []
+ for i in range(n_layers):
+ out_ch = channels << i
+ convs.append(nn.Sequential(
+ FilmConv2d(in_channels, out_ch, 3, device=device, dtype=dtype, operations=operations),
+ FilmConv2d(out_ch, out_ch, 3, device=device, dtype=dtype, operations=operations)))
+ in_channels = out_ch
+ self.convs = nn.ModuleList(convs)
+
+ def forward(self, image, n):
+ head = image
+ pyramid = []
+ for i, layer in enumerate(self.convs):
+ head = layer(head)
+ pyramid.append(head)
+ if i < n - 1:
+ head = F.avg_pool2d(head, 2, 2)
+ return pyramid
+
+
+class FeatureExtractor(nn.Module):
+ def __init__(self, in_channels=3, channels=64, sub_levels=4, device=None, dtype=None, operations=ops):
+ super().__init__()
+ self.extract_sublevels = SubTreeExtractor(in_channels, channels, sub_levels, device=device, dtype=dtype, operations=operations)
+ self.sub_levels = sub_levels
+
+ def forward(self, image_pyramid):
+ sub_pyramids = [self.extract_sublevels(image_pyramid[i], min(len(image_pyramid) - i, self.sub_levels))
+ for i in range(len(image_pyramid))]
+ feature_pyramid = []
+ for i in range(len(image_pyramid)):
+ features = sub_pyramids[i][0]
+ for j in range(1, self.sub_levels):
+ if j <= i:
+ features = torch.cat([features, sub_pyramids[i - j][j]], dim=1)
+ feature_pyramid.append(features)
+ # Free sub-pyramids no longer needed by future levels
+ if i >= self.sub_levels - 1:
+ sub_pyramids[i - self.sub_levels + 1] = None
+ return feature_pyramid
+
+
+class FlowEstimator(nn.Module):
+ def __init__(self, in_channels, num_convs, num_filters, device=None, dtype=None, operations=ops):
+ super().__init__()
+ self._convs = nn.ModuleList()
+ for _ in range(num_convs):
+ self._convs.append(FilmConv2d(in_channels, num_filters, 3, device=device, dtype=dtype, operations=operations))
+ in_channels = num_filters
+ self._convs.append(FilmConv2d(in_channels, num_filters // 2, 1, device=device, dtype=dtype, operations=operations))
+ self._convs.append(FilmConv2d(num_filters // 2, 2, 1, activation=False, device=device, dtype=dtype, operations=operations))
+
+ def forward(self, features_a, features_b):
+ net = torch.cat([features_a, features_b], dim=1)
+ for conv in self._convs:
+ net = conv(net)
+ return net
+
+
+class PyramidFlowEstimator(nn.Module):
+ def __init__(self, filters=64, flow_convs=(3, 3, 3, 3), flow_filters=(32, 64, 128, 256), device=None, dtype=None, operations=ops):
+ super().__init__()
+ in_channels = filters << 1
+ predictors = []
+ for i in range(len(flow_convs)):
+ predictors.append(FlowEstimator(in_channels, flow_convs[i], flow_filters[i], device=device, dtype=dtype, operations=operations))
+ in_channels += filters << (i + 2)
+ self._predictor = predictors[-1]
+ self._predictors = nn.ModuleList(predictors[:-1][::-1])
+
+ def forward(self, feature_pyramid_a, feature_pyramid_b, warp_fn):
+ levels = len(feature_pyramid_a)
+ v = self._predictor(feature_pyramid_a[-1], feature_pyramid_b[-1])
+ residuals = [v]
+ # Coarse-to-fine: shared predictor for deep levels, then specialized predictors for fine levels
+ steps = [(i, self._predictor) for i in range(levels - 2, len(self._predictors) - 1, -1)]
+ steps += [(len(self._predictors) - 1 - k, p) for k, p in enumerate(self._predictors)]
+ for i, predictor in steps:
+ v = F.interpolate(v, size=feature_pyramid_a[i].shape[2:4], mode="bilinear").mul_(2)
+ v_residual = predictor(feature_pyramid_a[i], warp_fn(feature_pyramid_b[i], v))
+ residuals.append(v_residual)
+ v = v.add_(v_residual)
+ residuals.reverse()
+ return residuals
+
+
+def _get_fusion_channels(level, filters):
+ # Per direction: multi-scale features + RGB image (3ch) + flow (2ch), doubled for both directions
+ return (sum(filters << i for i in range(level)) + 3 + 2) * 2
+
+
+class Fusion(nn.Module):
+ def __init__(self, n_layers=4, specialized_layers=3, filters=64, device=None, dtype=None, operations=ops):
+ super().__init__()
+ self.output_conv = operations.Conv2d(filters, 3, kernel_size=1, device=device, dtype=dtype)
+ self.convs = nn.ModuleList()
+ in_channels = _get_fusion_channels(n_layers, filters)
+ increase = 0
+ for i in range(n_layers)[::-1]:
+ num_filters = (filters << i) if i < specialized_layers else (filters << specialized_layers)
+ self.convs.append(nn.ModuleList([
+ FilmConv2d(in_channels, num_filters, 2, activation=False, device=device, dtype=dtype, operations=operations),
+ FilmConv2d(in_channels + (increase or num_filters), num_filters, 3, device=device, dtype=dtype, operations=operations),
+ FilmConv2d(num_filters, num_filters, 3, device=device, dtype=dtype, operations=operations)]))
+ in_channels = num_filters
+ increase = _get_fusion_channels(i, filters) - num_filters // 2
+
+ def forward(self, pyramid):
+ net = pyramid[-1]
+ for k, layers in enumerate(self.convs):
+ i = len(self.convs) - 1 - k
+ net = layers[0](F.interpolate(net, size=pyramid[i].shape[2:4], mode="nearest"))
+ net = layers[2](layers[1](torch.cat([pyramid[i], net], dim=1)))
+ return self.output_conv(net)
+
+
+class FILMNet(nn.Module):
+ def __init__(self, pyramid_levels=7, fusion_pyramid_levels=5, specialized_levels=3, sub_levels=4,
+ filters=64, flow_convs=(3, 3, 3, 3), flow_filters=(32, 64, 128, 256), device=None, dtype=None, operations=ops):
+ super().__init__()
+ self.pyramid_levels = pyramid_levels
+ self.fusion_pyramid_levels = fusion_pyramid_levels
+ self.extract = FeatureExtractor(3, filters, sub_levels, device=device, dtype=dtype, operations=operations)
+ self.predict_flow = PyramidFlowEstimator(filters, flow_convs, flow_filters, device=device, dtype=dtype, operations=operations)
+ self.fuse = Fusion(sub_levels, specialized_levels, filters, device=device, dtype=dtype, operations=operations)
+ self._warp_grids = {}
+
+ def get_dtype(self):
+ return self.extract.extract_sublevels.convs[0][0].conv.weight.dtype
+
+ def _build_warp_grids(self, H, W, device):
+ """Pre-compute warp grids for all pyramid levels."""
+ if (H, W) in self._warp_grids:
+ return
+ self._warp_grids = {} # clear old resolution grids to prevent memory leaks
+ for _ in range(self.pyramid_levels):
+ self._warp_grids[(H, W)] = (
+ torch.linspace(-(1 - 1 / W), 1 - 1 / W, W, dtype=torch.float32, device=device),
+ torch.linspace(-(1 - 1 / H), 1 - 1 / H, H, dtype=torch.float32, device=device),
+ )
+ H, W = H // 2, W // 2
+
+ def warp(self, image, flow):
+ grid_x, grid_y = self._warp_grids[(flow.shape[2], flow.shape[3])]
+ return _warp_core(image, flow, grid_x, grid_y)
+
+ def extract_features(self, img):
+ """Extract image and feature pyramids for a single frame. Can be cached across pairs."""
+ image_pyramid = build_image_pyramid(img, self.pyramid_levels)
+ feature_pyramid = self.extract(image_pyramid)
+ return image_pyramid, feature_pyramid
+
+ def forward(self, img0, img1, timestep=0.5, cache=None):
+ # FILM uses a scalar timestep per batch element (spatially-varying timesteps not supported)
+ t = timestep.mean(dim=(1, 2, 3)).item() if isinstance(timestep, torch.Tensor) else timestep
+ return self.forward_multi_timestep(img0, img1, [t], cache=cache)
+
+ def forward_multi_timestep(self, img0, img1, timesteps, cache=None):
+ """Compute flow once, synthesize at multiple timesteps. Expects batch=1 inputs."""
+ self._build_warp_grids(img0.shape[2], img0.shape[3], img0.device)
+
+ image_pyr0, feat_pyr0 = cache["img0"] if cache and "img0" in cache else self.extract_features(img0)
+ image_pyr1, feat_pyr1 = cache["img1"] if cache and "img1" in cache else self.extract_features(img1)
+
+ fwd_flow = flow_pyramid_synthesis(self.predict_flow(feat_pyr0, feat_pyr1, self.warp))[:self.fusion_pyramid_levels]
+ bwd_flow = flow_pyramid_synthesis(self.predict_flow(feat_pyr1, feat_pyr0, self.warp))[:self.fusion_pyramid_levels]
+
+ # Build warp targets and free full pyramids (only first fpl levels needed from here)
+ fpl = self.fusion_pyramid_levels
+ p2w = [concatenate_pyramids(image_pyr0[:fpl], feat_pyr0[:fpl]),
+ concatenate_pyramids(image_pyr1[:fpl], feat_pyr1[:fpl])]
+ del image_pyr0, image_pyr1, feat_pyr0, feat_pyr1
+
+ results = []
+ dt_tensors = torch.tensor(timesteps, device=img0.device, dtype=img0.dtype)
+ for idx in range(len(timesteps)):
+ batch_dt = dt_tensors[idx:idx + 1]
+ bwd_scaled = multiply_pyramid(bwd_flow, batch_dt)
+ fwd_scaled = multiply_pyramid(fwd_flow, 1 - batch_dt)
+ fwd_warped = pyramid_warp(p2w[0], bwd_scaled, self.warp)
+ bwd_warped = pyramid_warp(p2w[1], fwd_scaled, self.warp)
+ aligned = [torch.cat([fw, bw, bf, ff], dim=1)
+ for fw, bw, bf, ff in zip(fwd_warped, bwd_warped, bwd_scaled, fwd_scaled)]
+ del fwd_warped, bwd_warped, bwd_scaled, fwd_scaled
+ results.append(self.fuse(aligned))
+ del aligned
+ return torch.cat(results, dim=0)
diff --git a/comfy_extras/frame_interpolation_models/ifnet.py b/comfy_extras/frame_interpolation_models/ifnet.py
new file mode 100644
index 000000000..03cb34c50
--- /dev/null
+++ b/comfy_extras/frame_interpolation_models/ifnet.py
@@ -0,0 +1,128 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+import comfy.ops
+
+ops = comfy.ops.disable_weight_init
+
+
+def _warp(img, flow, warp_grids):
+ B, _, H, W = img.shape
+ base_grid, flow_div = warp_grids[(H, W)]
+ flow_norm = torch.cat([flow[:, 0:1] / flow_div[0], flow[:, 1:2] / flow_div[1]], 1).float()
+ grid = (base_grid.expand(B, -1, -1, -1) + flow_norm).permute(0, 2, 3, 1)
+ return F.grid_sample(img.float(), grid, mode="bilinear", padding_mode="border", align_corners=True).to(img.dtype)
+
+
+class Head(nn.Module):
+ def __init__(self, out_ch=4, device=None, dtype=None, operations=ops):
+ super().__init__()
+ self.cnn0 = operations.Conv2d(3, 16, 3, 2, 1, device=device, dtype=dtype)
+ self.cnn1 = operations.Conv2d(16, 16, 3, 1, 1, device=device, dtype=dtype)
+ self.cnn2 = operations.Conv2d(16, 16, 3, 1, 1, device=device, dtype=dtype)
+ self.cnn3 = operations.ConvTranspose2d(16, out_ch, 4, 2, 1, device=device, dtype=dtype)
+ self.relu = nn.LeakyReLU(0.2, True)
+
+ def forward(self, x):
+ x = self.relu(self.cnn0(x))
+ x = self.relu(self.cnn1(x))
+ x = self.relu(self.cnn2(x))
+ return self.cnn3(x)
+
+
+class ResConv(nn.Module):
+ def __init__(self, c, device=None, dtype=None, operations=ops):
+ super().__init__()
+ self.conv = operations.Conv2d(c, c, 3, 1, 1, device=device, dtype=dtype)
+ self.beta = nn.Parameter(torch.ones((1, c, 1, 1), device=device, dtype=dtype))
+ self.relu = nn.LeakyReLU(0.2, True)
+
+ def forward(self, x):
+ return self.relu(torch.addcmul(x, self.conv(x), self.beta))
+
+
+class IFBlock(nn.Module):
+ def __init__(self, in_planes, c=64, device=None, dtype=None, operations=ops):
+ super().__init__()
+ self.conv0 = nn.Sequential(
+ nn.Sequential(operations.Conv2d(in_planes, c // 2, 3, 2, 1, device=device, dtype=dtype), nn.LeakyReLU(0.2, True)),
+ nn.Sequential(operations.Conv2d(c // 2, c, 3, 2, 1, device=device, dtype=dtype), nn.LeakyReLU(0.2, True)))
+ self.convblock = nn.Sequential(*(ResConv(c, device=device, dtype=dtype, operations=operations) for _ in range(8)))
+ self.lastconv = nn.Sequential(operations.ConvTranspose2d(c, 4 * 13, 4, 2, 1, device=device, dtype=dtype), nn.PixelShuffle(2))
+
+ def forward(self, x, flow=None, scale=1):
+ x = F.interpolate(x, scale_factor=1.0 / scale, mode="bilinear")
+ if flow is not None:
+ flow = F.interpolate(flow, scale_factor=1.0 / scale, mode="bilinear").div_(scale)
+ x = torch.cat((x, flow), 1)
+ feat = self.convblock(self.conv0(x))
+ tmp = F.interpolate(self.lastconv(feat), scale_factor=scale, mode="bilinear")
+ return tmp[:, :4] * scale, tmp[:, 4:5], tmp[:, 5:]
+
+
+class IFNet(nn.Module):
+ def __init__(self, head_ch=4, channels=(192, 128, 96, 64, 32), device=None, dtype=None, operations=ops):
+ super().__init__()
+ self.encode = Head(out_ch=head_ch, device=device, dtype=dtype, operations=operations)
+ block_in = [7 + 2 * head_ch] + [8 + 4 + 8 + 2 * head_ch] * 4
+ self.blocks = nn.ModuleList([IFBlock(block_in[i], channels[i], device=device, dtype=dtype, operations=operations) for i in range(5)])
+ self.scale_list = [16, 8, 4, 2, 1]
+ self.pad_align = 64
+ self._warp_grids = {}
+
+ def get_dtype(self):
+ return self.encode.cnn0.weight.dtype
+
+ def _build_warp_grids(self, H, W, device):
+ if (H, W) in self._warp_grids:
+ return
+ self._warp_grids = {} # clear old resolution grids to prevent memory leaks
+ grid_y, grid_x = torch.meshgrid(
+ torch.linspace(-1.0, 1.0, H, device=device, dtype=torch.float32),
+ torch.linspace(-1.0, 1.0, W, device=device, dtype=torch.float32), indexing="ij")
+ self._warp_grids[(H, W)] = (
+ torch.stack((grid_x, grid_y), dim=0).unsqueeze(0),
+ torch.tensor([(W - 1.0) / 2.0, (H - 1.0) / 2.0], dtype=torch.float32, device=device))
+
+ def warp(self, img, flow):
+ return _warp(img, flow, self._warp_grids)
+
+ def extract_features(self, img):
+ """Extract head features for a single frame. Can be cached across pairs."""
+ return self.encode(img)
+
+ def forward(self, img0, img1, timestep=0.5, cache=None):
+ if not isinstance(timestep, torch.Tensor):
+ timestep = torch.full((img0.shape[0], 1, img0.shape[2], img0.shape[3]), timestep, device=img0.device, dtype=img0.dtype)
+
+ self._build_warp_grids(img0.shape[2], img0.shape[3], img0.device)
+
+ B = img0.shape[0]
+ f0 = cache["img0"].expand(B, -1, -1, -1) if cache and "img0" in cache else self.encode(img0)
+ f1 = cache["img1"].expand(B, -1, -1, -1) if cache and "img1" in cache else self.encode(img1)
+ flow = mask = feat = None
+ warped_img0, warped_img1 = img0, img1
+ for i, block in enumerate(self.blocks):
+ if flow is None:
+ flow, mask, feat = block(torch.cat((img0, img1, f0, f1, timestep), 1), None, scale=self.scale_list[i])
+ else:
+ fd, mask, feat = block(
+ torch.cat((warped_img0, warped_img1, self.warp(f0, flow[:, :2]), self.warp(f1, flow[:, 2:4]), timestep, mask, feat), 1),
+ flow, scale=self.scale_list[i])
+ flow = flow.add_(fd)
+ warped_img0 = self.warp(img0, flow[:, :2])
+ warped_img1 = self.warp(img1, flow[:, 2:4])
+ return torch.lerp(warped_img1, warped_img0, torch.sigmoid(mask))
+
+
+def detect_rife_config(state_dict):
+ head_ch = state_dict["encode.cnn3.weight"].shape[1] # ConvTranspose2d: (in_ch, out_ch, kH, kW)
+ channels = []
+ for i in range(5):
+ key = f"blocks.{i}.conv0.1.0.weight"
+ if key in state_dict:
+ channels.append(state_dict[key].shape[0])
+ if len(channels) != 5:
+ raise ValueError(f"Unsupported RIFE model: expected 5 blocks, found {len(channels)}")
+ return head_ch, channels
diff --git a/comfy_extras/nodes_frame_interpolation.py b/comfy_extras/nodes_frame_interpolation.py
new file mode 100644
index 000000000..a3b00d36e
--- /dev/null
+++ b/comfy_extras/nodes_frame_interpolation.py
@@ -0,0 +1,211 @@
+import torch
+from tqdm import tqdm
+from typing_extensions import override
+
+import comfy.model_patcher
+import comfy.utils
+import folder_paths
+from comfy import model_management
+from comfy_extras.frame_interpolation_models.ifnet import IFNet, detect_rife_config
+from comfy_extras.frame_interpolation_models.film_net import FILMNet
+from comfy_api.latest import ComfyExtension, io
+
+FrameInterpolationModel = io.Custom("INTERP_MODEL")
+
+
+class FrameInterpolationModelLoader(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="FrameInterpolationModelLoader",
+ display_name="Load Frame Interpolation Model",
+ category="loaders",
+ inputs=[
+ io.Combo.Input("model_name", options=folder_paths.get_filename_list("frame_interpolation"),
+ tooltip="Select a frame interpolation model to load. Models must be placed in the 'frame_interpolation' folder."),
+ ],
+ outputs=[
+ FrameInterpolationModel.Output(),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, model_name) -> io.NodeOutput:
+ model_path = folder_paths.get_full_path_or_raise("frame_interpolation", model_name)
+ sd = comfy.utils.load_torch_file(model_path, safe_load=True)
+
+ model = cls._detect_and_load(sd)
+ dtype = torch.float16 if model_management.should_use_fp16(model_management.get_torch_device()) else torch.float32
+ model.eval().to(dtype)
+ patcher = comfy.model_patcher.ModelPatcher(
+ model,
+ load_device=model_management.get_torch_device(),
+ offload_device=model_management.unet_offload_device(),
+ )
+ return io.NodeOutput(patcher)
+
+ @classmethod
+ def _detect_and_load(cls, sd):
+ # Try FILM
+ if "extract.extract_sublevels.convs.0.0.conv.weight" in sd:
+ model = FILMNet()
+ model.load_state_dict(sd)
+ return model
+
+ # Try RIFE (needs key remapping for raw checkpoints)
+ sd = comfy.utils.state_dict_prefix_replace(sd, {"module.": "", "flownet.": ""})
+ key_map = {}
+ for k in sd:
+ for i in range(5):
+ if k.startswith(f"block{i}."):
+ key_map[k] = f"blocks.{i}.{k[len(f'block{i}.'):]}"
+ if key_map:
+ sd = {key_map.get(k, k): v for k, v in sd.items()}
+ sd = {k: v for k, v in sd.items() if not k.startswith(("teacher.", "caltime."))}
+
+ try:
+ head_ch, channels = detect_rife_config(sd)
+ except (KeyError, ValueError):
+ raise ValueError("Unrecognized frame interpolation model format")
+ model = IFNet(head_ch=head_ch, channels=channels)
+ model.load_state_dict(sd)
+ return model
+
+
+class FrameInterpolate(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="FrameInterpolate",
+ display_name="Frame Interpolate",
+ category="image/video",
+ search_aliases=["rife", "film", "frame interpolation", "slow motion", "interpolate frames", "vfi"],
+ inputs=[
+ FrameInterpolationModel.Input("interp_model"),
+ io.Image.Input("images"),
+ io.Int.Input("multiplier", default=2, min=2, max=16),
+ ],
+ outputs=[
+ io.Image.Output(),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, interp_model, images, multiplier) -> io.NodeOutput:
+ offload_device = model_management.intermediate_device()
+
+ num_frames = images.shape[0]
+ if num_frames < 2 or multiplier < 2:
+ return io.NodeOutput(images)
+
+ model_management.load_model_gpu(interp_model)
+ device = interp_model.load_device
+ dtype = interp_model.model_dtype()
+ inference_model = interp_model.model
+
+ # Free VRAM for inference activations (model weights + ~20x a single frame's worth)
+ H, W = images.shape[1], images.shape[2]
+ activation_mem = H * W * 3 * images.element_size() * 20
+ model_management.free_memory(activation_mem, device)
+ align = getattr(inference_model, "pad_align", 1)
+
+ # Prepare a single padded frame on device for determining output dimensions
+ def prepare_frame(idx):
+ frame = images[idx:idx + 1].movedim(-1, 1).to(dtype=dtype, device=device)
+ if align > 1:
+ from comfy.ldm.common_dit import pad_to_patch_size
+ frame = pad_to_patch_size(frame, (align, align), padding_mode="reflect")
+ return frame
+
+ # Count total interpolation passes for progress bar
+ total_pairs = num_frames - 1
+ num_interp = multiplier - 1
+ total_steps = total_pairs * num_interp
+ pbar = comfy.utils.ProgressBar(total_steps)
+ tqdm_bar = tqdm(total=total_steps, desc="Frame interpolation")
+
+ batch = num_interp # reduced on OOM and persists across pairs (same resolution = same limit)
+ t_values = [t / multiplier for t in range(1, multiplier)]
+
+ out_dtype = model_management.intermediate_dtype()
+ total_out_frames = total_pairs * multiplier + 1
+ result = torch.empty((total_out_frames, 3, H, W), dtype=out_dtype, device=offload_device)
+ result[0] = images[0].movedim(-1, 0).to(out_dtype)
+ out_idx = 1
+
+ # Pre-compute timestep tensor on device (padded dimensions needed)
+ sample = prepare_frame(0)
+ pH, pW = sample.shape[2], sample.shape[3]
+ ts_full = torch.tensor(t_values, device=device, dtype=dtype).reshape(num_interp, 1, 1, 1)
+ ts_full = ts_full.expand(-1, 1, pH, pW)
+ del sample
+
+ multi_fn = getattr(inference_model, "forward_multi_timestep", None)
+ feat_cache = {}
+ prev_frame = None
+
+ try:
+ for i in range(total_pairs):
+ img0_single = prev_frame if prev_frame is not None else prepare_frame(i)
+ img1_single = prepare_frame(i + 1)
+ prev_frame = img1_single
+
+ # Cache features: img1 of pair N becomes img0 of pair N+1
+ feat_cache["img0"] = feat_cache.pop("next") if "next" in feat_cache else inference_model.extract_features(img0_single)
+ feat_cache["img1"] = inference_model.extract_features(img1_single)
+ feat_cache["next"] = feat_cache["img1"]
+
+ used_multi = False
+ if multi_fn is not None:
+ # Models with timestep-independent flow can compute it once for all timesteps
+ try:
+ mids = multi_fn(img0_single, img1_single, t_values, cache=feat_cache)
+ result[out_idx:out_idx + num_interp] = mids[:, :, :H, :W].to(out_dtype)
+ out_idx += num_interp
+ pbar.update(num_interp)
+ tqdm_bar.update(num_interp)
+ used_multi = True
+ except model_management.OOM_EXCEPTION:
+ model_management.soft_empty_cache()
+ multi_fn = None # fall through to single-timestep path
+
+ if not used_multi:
+ j = 0
+ while j < num_interp:
+ b = min(batch, num_interp - j)
+ try:
+ img0 = img0_single.expand(b, -1, -1, -1)
+ img1 = img1_single.expand(b, -1, -1, -1)
+ mids = inference_model(img0, img1, timestep=ts_full[j:j + b], cache=feat_cache)
+ result[out_idx:out_idx + b] = mids[:, :, :H, :W].to(out_dtype)
+ out_idx += b
+ pbar.update(b)
+ tqdm_bar.update(b)
+ j += b
+ except model_management.OOM_EXCEPTION:
+ if batch <= 1:
+ raise
+ batch = max(1, batch // 2)
+ model_management.soft_empty_cache()
+
+ result[out_idx] = images[i + 1].movedim(-1, 0).to(out_dtype)
+ out_idx += 1
+ finally:
+ tqdm_bar.close()
+
+ # BCHW -> BHWC
+ result = result.movedim(1, -1).clamp_(0.0, 1.0)
+ return io.NodeOutput(result)
+
+
+class FrameInterpolationExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ FrameInterpolationModelLoader,
+ FrameInterpolate,
+ ]
+
+
+async def comfy_entrypoint() -> FrameInterpolationExtension:
+ return FrameInterpolationExtension()
diff --git a/folder_paths.py b/folder_paths.py
index 9c96540e3..80f4b291a 100644
--- a/folder_paths.py
+++ b/folder_paths.py
@@ -52,6 +52,8 @@ folder_names_and_paths["model_patches"] = ([os.path.join(models_dir, "model_patc
folder_names_and_paths["audio_encoders"] = ([os.path.join(models_dir, "audio_encoders")], supported_pt_extensions)
+folder_names_and_paths["frame_interpolation"] = ([os.path.join(models_dir, "frame_interpolation")], supported_pt_extensions)
+
output_directory = os.path.join(base_path, "output")
temp_directory = os.path.join(base_path, "temp")
input_directory = os.path.join(base_path, "input")
diff --git a/models/frame_interpolation/put_frame_interpolation_models_here b/models/frame_interpolation/put_frame_interpolation_models_here
new file mode 100644
index 000000000..e69de29bb
diff --git a/nodes.py b/nodes.py
index 299b3d758..bb38e07b8 100644
--- a/nodes.py
+++ b/nodes.py
@@ -2457,7 +2457,8 @@ async def init_builtin_extra_nodes():
"nodes_number_convert.py",
"nodes_painter.py",
"nodes_curve.py",
- "nodes_rtdetr.py"
+ "nodes_rtdetr.py",
+ "nodes_frame_interpolation.py",
]
import_failed = []