The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Nodes to properly use the SDV img2vid checkpoint.
comfyanonymous42dfae63312f
diff --git a/comfy_extras/nodes_video_model.py b/comfy_extras/nodes_video_model.pynew file mode 100644index 000000000..92bd883ae--- /dev/null+++ b/comfy_extras/nodes_video_model.py@@ -0,0 +1,89 @@+import nodes+import torch+import comfy.utils+import comfy.sd+import folder_paths+++class ImageOnlyCheckpointLoader:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": { "ckpt_name": (folder_paths.get_filename_list("checkpoints"), ),+ }}+ RETURN_TYPES = ("MODEL", "CLIP_VISION", "VAE")+ FUNCTION = "load_checkpoint"++ CATEGORY = "loaders/video_models"++ def load_checkpoint(self, ckpt_name, output_vae=True, output_clip=True):+ ckpt_path = folder_paths.get_full_path("checkpoints", ckpt_name)+ out = comfy.sd.load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=False, output_clipvision=True, embedding_directory=folder_paths.get_folder_paths("embeddings"))+ return (out[0], out[3], out[2])+++class SDV_img2vid_Conditioning:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": { "clip_vision": ("CLIP_VISION",),+ "init_image": ("IMAGE",),+ "vae": ("VAE",),+ "width": ("INT", {"default": 1024, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),+ "height": ("INT", {"default": 576, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),+ "video_frames": ("INT", {"default": 14, "min": 1, "max": 4096}),+ "motion_bucket_id": ("INT", {"default": 127, "min": 1, "max": 1023}),+ "fps": ("INT", {"default": 6, "min": 1, "max": 1024}),+ "augmentation_level": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 10.0, "step": 0.01})+ }}+ RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")+ RETURN_NAMES = ("positive", "negative", "latent")++ FUNCTION = "encode"++ CATEGORY = "conditioning/video_models"++ def encode(self, clip_vision, init_image, vae, width, height, video_frames, motion_bucket_id, fps, augmentation_level):+ output = clip_vision.encode_image(init_image)+ pooled = output.image_embeds.unsqueeze(0)+ pixels = comfy.utils.common_upscale(init_image.movedim(-1,1), width, height, "bilinear", "center").movedim(1,-1)+ encode_pixels = pixels[:,:,:,:3]+ if augmentation_level > 0:+ encode_pixels += torch.randn_like(pixels) * augmentation_level+ t = vae.encode(encode_pixels)+ positive = [[pooled, {"motion_bucket_id": motion_bucket_id, "fps": fps, "augmentation_level": augmentation_level, "concat_latent_image": t}]]+ negative = [[torch.zeros_like(pooled), {"motion_bucket_id": motion_bucket_id, "fps": fps, "augmentation_level": augmentation_level, "concat_latent_image": torch.zeros_like(t)}]]+ latent = torch.zeros([video_frames, 4, height // 8, width // 8])+ return (positive, negative, {"samples":latent})++class VideoLinearCFGGuidance:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": { "model": ("MODEL",),+ "min_cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.5, "round": 0.01}),+ }}+ RETURN_TYPES = ("MODEL",)+ FUNCTION = "patch"++ CATEGORY = "sampling/video_models"++ def patch(self, model, min_cfg):+ def linear_cfg(args):+ cond = args["cond"]+ uncond = args["uncond"]+ cond_scale = args["cond_scale"]++ scale = torch.linspace(min_cfg, cond_scale, cond.shape[0], device=cond.device).reshape((cond.shape[0], 1, 1, 1))+ return uncond + scale * (cond - uncond)++ m = model.clone()+ m.set_model_sampler_cfg_function(linear_cfg)+ return (m, )++NODE_CLASS_MAPPINGS = {+ "ImageOnlyCheckpointLoader": ImageOnlyCheckpointLoader,+ "SDV_img2vid_Conditioning": SDV_img2vid_Conditioning,+ "VideoLinearCFGGuidance": VideoLinearCFGGuidance,+}++NODE_DISPLAY_NAME_MAPPINGS = {+ "ImageOnlyCheckpointLoader": "Image Only Checkpoint Loader (img2vid model)",+}diff --git a/nodes.py b/nodes.pyindex 2de468da7..bb24bc6e8 100644--- a/nodes.py+++ b/nodes.py@@ -1850,6 +1850,7 @@ def init_custom_nodes():"nodes_model_advanced.py","nodes_model_downscale.py","nodes_images.py",+ "nodes_video_model.py",]for node_file in extras_files: