Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Dataset Processing Nodes and Improved LoRA Trainer Nodes with multi resolution supports. (#10708)

Kohaku-Blueleafcc6a8dcd1ad9
diff --git a/comfy_extras/nodes_dataset.py b/comfy_extras/nodes_dataset.py
new file mode 100644
index 000000000..b23867505
--- /dev/null
+++ b/comfy_extras/nodes_dataset.py
@@ -0,0 +1,1532 @@
+import logging
+import os
+import math
+import json
+
+import numpy as np
+import torch
+from PIL import Image
+from typing_extensions import override
+
+import folder_paths
+import node_helpers
+from comfy_api.latest import ComfyExtension, io
+
+
+def load_and_process_images(image_files, input_dir):
+ """Utility function to load and process a list of images.
+
+ Args:
+ image_files: List of image filenames
+ input_dir: Base directory containing the images
+ resize_method: How to handle images of different sizes ("None", "Stretch", "Crop", "Pad")
+
+ Returns:
+ torch.Tensor: Batch of processed images
+ """
+ if not image_files:
+ raise ValueError("No valid images found in input")
+
+ output_images = []
+
+ for file in image_files:
+ image_path = os.path.join(input_dir, file)
+ img = node_helpers.pillow(Image.open, image_path)
+
+ if img.mode == "I":
+ img = img.point(lambda i: i * (1 / 255))
+ img = img.convert("RGB")
+ img_array = np.array(img).astype(np.float32) / 255.0
+ img_tensor = torch.from_numpy(img_array)[None,]
+ output_images.append(img_tensor)
+
+ return output_images
+
+
+class LoadImageDataSetFromFolderNode(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="LoadImageDataSetFromFolder",
+ display_name="Load Image Dataset from Folder",
+ category="dataset",
+ is_experimental=True,
+ inputs=[
+ io.Combo.Input(
+ "folder",
+ options=folder_paths.get_input_subfolders(),
+ tooltip="The folder to load images from.",
+ )
+ ],
+ outputs=[
+ io.Image.Output(
+ display_name="images",
+ is_output_list=True,
+ tooltip="List of loaded images",
+ )
+ ],
+ )
+
+ @classmethod
+ def execute(cls, folder):
+ sub_input_dir = os.path.join(folder_paths.get_input_directory(), folder)
+ valid_extensions = [".png", ".jpg", ".jpeg", ".webp"]
+ image_files = [
+ f
+ for f in os.listdir(sub_input_dir)
+ if any(f.lower().endswith(ext) for ext in valid_extensions)
+ ]
+ output_tensor = load_and_process_images(image_files, sub_input_dir)
+ return io.NodeOutput(output_tensor)
+
+
+class LoadImageTextDataSetFromFolderNode(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="LoadImageTextDataSetFromFolder",
+ display_name="Load Image and Text Dataset from Folder",
+ category="dataset",
+ is_experimental=True,
+ inputs=[
+ io.Combo.Input(
+ "folder",
+ options=folder_paths.get_input_subfolders(),
+ tooltip="The folder to load images from.",
+ )
+ ],
+ outputs=[
+ io.Image.Output(
+ display_name="images",
+ is_output_list=True,
+ tooltip="List of loaded images",
+ ),
+ io.String.Output(
+ display_name="texts",
+ is_output_list=True,
+ tooltip="List of text captions",
+ ),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, folder):
+ logging.info(f"Loading images from folder: {folder}")
+
+ sub_input_dir = os.path.join(folder_paths.get_input_directory(), folder)
+ valid_extensions = [".png", ".jpg", ".jpeg", ".webp"]
+
+ image_files = []
+ for item in os.listdir(sub_input_dir):
+ path = os.path.join(sub_input_dir, item)
+ if any(item.lower().endswith(ext) for ext in valid_extensions):
+ image_files.append(path)
+ elif os.path.isdir(path):
+ # Support kohya-ss/sd-scripts folder structure
+ repeat = 1
+ if item.split("_")[0].isdigit():
+ repeat = int(item.split("_")[0])
+ image_files.extend(
+ [
+ os.path.join(path, f)
+ for f in os.listdir(path)
+ if any(f.lower().endswith(ext) for ext in valid_extensions)
+ ]
+ * repeat
+ )
+
+ caption_file_path = [
+ f.replace(os.path.splitext(f)[1], ".txt") for f in image_files
+ ]
+ captions = []
+ for caption_file in caption_file_path:
+ caption_path = os.path.join(sub_input_dir, caption_file)
+ if os.path.exists(caption_path):
+ with open(caption_path, "r", encoding="utf-8") as f:
+ caption = f.read().strip()
+ captions.append(caption)
+ else:
+ captions.append("")
+
+ output_tensor = load_and_process_images(image_files, sub_input_dir)
+
+ logging.info(f"Loaded {len(output_tensor)} images from {sub_input_dir}.")
+ return io.NodeOutput(output_tensor, captions)
+
+
+def save_images_to_folder(image_list, output_dir, prefix="image"):
+ """Utility function to save a list of image tensors to disk.
+
+ Args:
+ image_list: List of image tensors (each [1, H, W, C] or [H, W, C] or [C, H, W])
+ output_dir: Directory to save images to
+ prefix: Filename prefix
+
+ Returns:
+ List of saved filenames
+ """
+ os.makedirs(output_dir, exist_ok=True)
+ saved_files = []
+
+ for idx, img_tensor in enumerate(image_list):
+ # Handle different tensor shapes
+ if isinstance(img_tensor, torch.Tensor):
+ # Remove batch dimension if present [1, H, W, C] -> [H, W, C]
+ if img_tensor.dim() == 4 and img_tensor.shape[0] == 1:
+ img_tensor = img_tensor.squeeze(0)
+
+ # If tensor is [C, H, W], permute to [H, W, C]
+ if img_tensor.dim() == 3 and img_tensor.shape[0] in [1, 3, 4]:
+ if (
+ img_tensor.shape[0] <= 4
+ and img_tensor.shape[1] > 4
+ and img_tensor.shape[2] > 4
+ ):
+ img_tensor = img_tensor.permute(1, 2, 0)
+
+ # Convert to numpy and scale to 0-255
+ img_array = img_tensor.cpu().numpy()
+ img_array = np.clip(img_array * 255.0, 0, 255).astype(np.uint8)
+
+ # Convert to PIL Image
+ img = Image.fromarray(img_array)
+ else:
+ raise ValueError(f"Expected torch.Tensor, got {type(img_tensor)}")
+
+ # Save image
+ filename = f"{prefix}_{idx:05d}.png"
+ filepath = os.path.join(output_dir, filename)
+ img.save(filepath)
+ saved_files.append(filename)
+
+ return saved_files
+
+
+class SaveImageDataSetToFolderNode(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="SaveImageDataSetToFolder",
+ display_name="Save Image Dataset to Folder",
+ category="dataset",
+ is_experimental=True,
+ is_output_node=True,
+ is_input_list=True, # Receive images as list
+ inputs=[
+ io.Image.Input("images", tooltip="List of images to save."),
+ io.String.Input(
+ "folder_name",
+ default="dataset",
+ tooltip="Name of the folder to save images to (inside output directory).",
+ ),
+ io.String.Input(
+ "filename_prefix",
+ default="image",
+ tooltip="Prefix for saved image filenames.",
+ ),
+ ],
+ outputs=[],
+ )
+
+ @classmethod
+ def execute(cls, images, folder_name, filename_prefix):
+ # Extract scalar values
+ folder_name = folder_name[0]
+ filename_prefix = filename_prefix[0]
+
+ output_dir = os.path.join(folder_paths.get_output_directory(), folder_name)
+ saved_files = save_images_to_folder(images, output_dir, filename_prefix)
+
+ logging.info(f"Saved {len(saved_files)} images to {output_dir}.")
+ return io.NodeOutput()
+
+
+class SaveImageTextDataSetToFolderNode(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="SaveImageTextDataSetToFolder",
+ display_name="Save Image and Text Dataset to Folder",
+ category="dataset",
+ is_experimental=True,
+ is_output_node=True,
+ is_input_list=True, # Receive both images and texts as lists
+ inputs=[
+ io.Image.Input("images", tooltip="List of images to save."),
+ io.String.Input("texts", tooltip="List of text captions to save."),
+ io.String.Input(
+ "folder_name",
+ default="dataset",
+ tooltip="Name of the folder to save images to (inside output directory).",
+ ),
+ io.String.Input(
+ "filename_prefix",
+ default="image",
+ tooltip="Prefix for saved image filenames.",
+ ),
+ ],
+ outputs=[],
+ )
+
+ @classmethod
+ def execute(cls, images, texts, folder_name, filename_prefix):
+ # Extract scalar values
+ folder_name = folder_name[0]
+ filename_prefix = filename_prefix[0]
+
+ output_dir = os.path.join(folder_paths.get_output_directory(), folder_name)
+ saved_files = save_images_to_folder(images, output_dir, filename_prefix)
+
+ # Save captions
+ for idx, (filename, caption) in enumerate(zip(saved_files, texts)):
+ caption_filename = filename.replace(".png", ".txt")
+ caption_path = os.path.join(output_dir, caption_filename)
+ with open(caption_path, "w", encoding="utf-8") as f:
+ f.write(caption)
+
+ logging.info(f"Saved {len(saved_files)} images and captions to {output_dir}.")
+ return io.NodeOutput()
+
+
+# ========== Helper Functions for Transform Nodes ==========
+
+
+def tensor_to_pil(img_tensor):
+ """Convert tensor to PIL Image."""
+ if img_tensor.dim() == 4 and img_tensor.shape[0] == 1:
+ img_tensor = img_tensor.squeeze(0)
+ img_array = (img_tensor.cpu().numpy() * 255).clip(0, 255).astype(np.uint8)
+ return Image.fromarray(img_array)
+
+
+def pil_to_tensor(img):
+ """Convert PIL Image to tensor."""
+ img_array = np.array(img).astype(np.float32) / 255.0
+ return torch.from_numpy(img_array)[None,]
+
+
+# ========== Base Classes for Transform Nodes ==========
+
+
+class ImageProcessingNode(io.ComfyNode):
+ """Base class for image processing nodes that operate on images.
+
+ Child classes should set:
+ node_id: Unique node identifier (required)
+ display_name: Display name (optional, defaults to node_id)
+ description: Node description (optional)
+ extra_inputs: List of additional io.Input objects beyond "images" (optional)
+ is_group_process: None (auto-detect), True (group), or False (individual) (optional)
+ is_output_list: True (list output) or False (single output) (optional, default True)
+
+ Child classes must implement ONE of:
+ _process(cls, image, **kwargs) -> tensor (for single-item processing)
+ _group_process(cls, images, **kwargs) -> list[tensor] (for group processing)
+ """
+
+ node_id = None
+ display_name = None
+ description = None
+ extra_inputs = []
+ is_group_process = None # None = auto-detect, True/False = explicit
+ is_output_list = None # None = auto-detect based on processing mode
+
+ @classmethod
+ def _detect_processing_mode(cls):
+ """Detect whether this node uses group or individual processing.
+
+ Returns:
+ bool: True if group processing, False if individual processing
+ """
+ # Explicit setting takes precedence
+ if cls.is_group_process is not None:
+ return cls.is_group_process
+
+ # Check which method is overridden by looking at the defining class in MRO
+ base_class = ImageProcessingNode
+
+ # Find which class in MRO defines _process
+ process_definer = None
+ for klass in cls.__mro__:
+ if "_process" in klass.__dict__:
+ process_definer = klass
+ break
+
+ # Find which class in MRO defines _group_process
+ group_definer = None
+ for klass in cls.__mro__:
+ if "_group_process" in klass.__dict__:
+ group_definer = klass
+ break
+
+ # Check what was overridden (not defined in base class)
+ has_process = process_definer is not None and process_definer is not base_class
+ has_group = group_definer is not None and group_definer is not base_class
+
+ if has_process and has_group:
+ raise ValueError(
+ f"{cls.__name__}: Cannot override both _process and _group_process. "
+ "Override only one, or set is_group_process explicitly."
+ )
+ if not has_process and not has_group:
+ raise ValueError(
+ f"{cls.__name__}: Must override either _process or _group_process"
+ )
+
+ return has_group
+
+ @classmethod
+ def define_schema(cls):
+ if cls.node_id is None:
+ raise NotImplementedError(f"{cls.__name__} must set node_id class variable")
+
+ is_group = cls._detect_processing_mode()
+
+ # Auto-detect is_output_list if not explicitly set
+ # Single processing: False (backend collects results into list)
+ # Group processing: True by default (can be False for single-output nodes)
+ output_is_list = (
+ cls.is_output_list if cls.is_output_list is not None else is_group
+ )
+
+ inputs = [
+ io.Image.Input(
+ "images",
+ tooltip=(
+ "List of images to process." if is_group else "Image to process."
+ ),
+ )
+ ]
+ inputs.extend(cls.extra_inputs)
+
+ return io.Schema(
+ node_id=cls.node_id,
+ display_name=cls.display_name or cls.node_id,
+ category="dataset/image",
+ is_experimental=True,
+ is_input_list=is_group, # True for group, False for individual
+ inputs=inputs,
+ outputs=[
+ io.Image.Output(
+ display_name="images",
+ is_output_list=output_is_list,
+ tooltip="Processed images",
+ )
+ ],
+ )
+
+ @classmethod
+ def execute(cls, images, **kwargs):
+ """Execute the node. Routes to _process or _group_process based on mode."""
+ is_group = cls._detect_processing_mode()
+
+ # Extract scalar values from lists for parameters
+ params = {}
+ for k, v in kwargs.items():
+ if isinstance(v, list) and len(v) == 1:
+ params[k] = v[0]
+ else:
+ params[k] = v
+
+ if is_group:
+ # Group processing: images is list, call _group_process
+ result = cls._group_process(images, **params)
+ else:
+ # Individual processing: images is single item, call _process
+ result = cls._process(images, **params)
+
+ return io.NodeOutput(result)
+
+ @classmethod
+ def _process(cls, image, **kwargs):
+ """Override this method for single-item processing.
+
+ Args:
+ image: tensor - Single image tensor
+ **kwargs: Additional parameters (already extracted from lists)
+
+ Returns:
+ tensor - Processed image
+ """
+ raise NotImplementedError(f"{cls.__name__} must implement _process method")
+
+ @classmethod
+ def _group_process(cls, images, **kwargs):
+ """Override this method for group processing.
+
+ Args:
+ images: list[tensor] - List of image tensors
+ **kwargs: Additional parameters (already extracted from lists)
+
+ Returns:
+ list[tensor] - Processed images
+ """
+ raise NotImplementedError(
+ f"{cls.__name__} must implement _group_process method"
+ )
+
+
+class TextProcessingNode(io.ComfyNode):
+ """Base class for text processing nodes that operate on texts.
+
+ Child classes should set:
+ node_id: Unique node identifier (required)
+ display_name: Display name (optional, defaults to node_id)
+ description: Node description (optional)
+ extra_inputs: List of additional io.Input objects beyond "texts" (optional)
+ is_group_process: None (auto-detect), True (group), or False (individual) (optional)
+ is_output_list: True (list output) or False (single output) (optional, default True)
+
+ Child classes must implement ONE of:
+ _process(cls, text, **kwargs) -> str (for single-item processing)
+ _group_process(cls, texts, **kwargs) -> list[str] (for group processing)
+ """
+
+ node_id = None
+ display_name = None
+ description = None
+ extra_inputs = []
+ is_group_process = None # None = auto-detect, True/False = explicit
+ is_output_list = None # None = auto-detect based on processing mode
+
+ @classmethod
+ def _detect_processing_mode(cls):
+ """Detect whether this node uses group or individual processing.
+
+ Returns:
+ bool: True if group processing, False if individual processing
+ """
+ # Explicit setting takes precedence
+ if cls.is_group_process is not None:
+ return cls.is_group_process
+
+ # Check which method is overridden by looking at the defining class in MRO
+ base_class = TextProcessingNode
+
+ # Find which class in MRO defines _process
+ process_definer = None
+ for klass in cls.__mro__:
+ if "_process" in klass.__dict__:
+ process_definer = klass
+ break
+
+ # Find which class in MRO defines _group_process
+ group_definer = None
+ for klass in cls.__mro__:
+ if "_group_process" in klass.__dict__:
+ group_definer = klass
+ break
+
+ # Check what was overridden (not defined in base class)
+ has_process = process_definer is not None and process_definer is not base_class
+ has_group = group_definer is not None and group_definer is not base_class
+
+ if has_process and has_group:
+ raise ValueError(
+ f"{cls.__name__}: Cannot override both _process and _group_process. "
+ "Override only one, or set is_group_process explicitly."
+ )
+ if not has_process and not has_group:
+ raise ValueError(
+ f"{cls.__name__}: Must override either _process or _group_process"
+ )
+
+ return has_group
+
+ @classmethod
+ def define_schema(cls):
+ if cls.node_id is None:
+ raise NotImplementedError(f"{cls.__name__} must set node_id class variable")
+
+ is_group = cls._detect_processing_mode()
+
+ inputs = [
+ io.String.Input(
+ "texts",
+ tooltip="List of texts to process." if is_group else "Text to process.",
+ )
+ ]
+ inputs.extend(cls.extra_inputs)
+
+ return io.Schema(
+ node_id=cls.node_id,
+ display_name=cls.display_name or cls.node_id,
+ category="dataset/text",
+ is_experimental=True,
+ is_input_list=is_group, # True for group, False for individual
+ inputs=inputs,
+ outputs=[
+ io.String.Output(
+ display_name="texts",
+ is_output_list=cls.is_output_list,
+ tooltip="Processed texts",
+ )
+ ],
+ )
+
+ @classmethod
+ def execute(cls, texts, **kwargs):
+ """Execute the node. Routes to _process or _group_process based on mode."""
+ is_group = cls._detect_processing_mode()
+
+ # Extract scalar values from lists for parameters
+ params = {}
+ for k, v in kwargs.items():
+ if isinstance(v, list) and len(v) == 1:
+ params[k] = v[0]
+ else:
+ params[k] = v
+
+ if is_group:
+ # Group processing: texts is list, call _group_process
+ result = cls._group_process(texts, **params)
+ else:
+ # Individual processing: texts is single item, call _process
+ result = cls._process(texts, **params)
+
+ # Wrap result based on is_output_list
+ if cls.is_output_list:
+ # Result should already be a list (or will be for individual)
+ return io.NodeOutput(result if is_group else [result])
+ else:
+ # Single output - wrap in list for NodeOutput
+ return io.NodeOutput([result])
+
+ @classmethod
+ def _process(cls, text, **kwargs):
+ """Override this method for single-item processing.
+
+ Args:
+ text: str - Single text string
+ **kwargs: Additional parameters (already extracted from lists)
+
+ Returns:
+ str - Processed text
+ """
+ raise NotImplementedError(f"{cls.__name__} must implement _process method")
+
+ @classmethod
+ def _group_process(cls, texts, **kwargs):
+ """Override this method for group processing.
+
+ Args:
+ texts: list[str] - List of text strings
+ **kwargs: Additional parameters (already extracted from lists)
+
+ Returns:
+ list[str] - Processed texts
+ """
+ raise NotImplementedError(
+ f"{cls.__name__} must implement _group_process method"
+ )
+
+
+# ========== Image Transform Nodes ==========
+
+
+class ResizeImagesToSameSizeNode(ImageProcessingNode):
+ node_id = "ResizeImagesToSameSize"
+ display_name = "Resize Images to Same Size"
+ description = "Resize all images to the same width and height."
+ extra_inputs = [
+ io.Int.Input("width", default=512, min=1, max=8192, tooltip="Target width."),
+ io.Int.Input("height", default=512, min=1, max=8192, tooltip="Target height."),
+ io.Combo.Input(
+ "mode",
+ options=["stretch", "crop_center", "pad"],
+ default="stretch",
+ tooltip="Resize mode.",
+ ),
+ ]
+
+ @classmethod
+ def _process(cls, image, width, height, mode):
+ img = tensor_to_pil(image)
+
+ if mode == "stretch":
+ img = img.resize((width, height), Image.Resampling.LANCZOS)
+ elif mode == "crop_center":
+ left = max(0, (img.width - width) // 2)
+ top = max(0, (img.height - height) // 2)
+ right = min(img.width, left + width)
+ bottom = min(img.height, top + height)
+ img = img.crop((left, top, right, bottom))
+ if img.width != width or img.height != height:
+ img = img.resize((width, height), Image.Resampling.LANCZOS)
+ elif mode == "pad":
+ img.thumbnail((width, height), Image.Resampling.LANCZOS)
+ new_img = Image.new("RGB", (width, height), (0, 0, 0))
+ paste_x = (width - img.width) // 2
+ paste_y = (height - img.height) // 2
+ new_img.paste(img, (paste_x, paste_y))
+ img = new_img
+
+ return pil_to_tensor(img)
+
+
+class ResizeImagesToPixelCountNode(ImageProcessingNode):
+ node_id = "ResizeImagesToPixelCount"
+ display_name = "Resize Images to Pixel Count"
+ description = "Resize images so that the total pixel count matches the specified number while preserving aspect ratio."
+ extra_inputs = [
+ io.Int.Input(
+ "pixel_count",
+ default=512 * 512,
+ min=1,
+ max=8192 * 8192,
+ tooltip="Target pixel count.",
+ ),
+ io.Int.Input(
+ "steps",
+ default=64,
+ min=1,
+ max=128,
+ tooltip="The stepping for resize width/height.",
+ ),
+ ]
+
+ @classmethod
+ def _process(cls, image, pixel_count, steps):
+ img = tensor_to_pil(image)
+ w, h = img.size
+ pixel_count_ratio = math.sqrt(pixel_count / (w * h))
+ new_w = int(w * pixel_count_ratio / steps) * steps
+ new_h = int(h * pixel_count_ratio / steps) * steps
+ logging.info(f"Resizing from {w}x{h} to {new_w}x{new_h}")
+ img = img.resize((new_w, new_h), Image.Resampling.LANCZOS)
+ return pil_to_tensor(img)
+
+
+class ResizeImagesByShorterEdgeNode(ImageProcessingNode):
+ node_id = "ResizeImagesByShorterEdge"
+ display_name = "Resize Images by Shorter Edge"
+ description = "Resize images so that the shorter edge matches the specified length while preserving aspect ratio."
+ extra_inputs = [
+ io.Int.Input(
+ "shorter_edge",
+ default=512,
+ min=1,
+ max=8192,
+ tooltip="Target length for the shorter edge.",
+ ),
+ ]
+
+ @classmethod
+ def _process(cls, image, shorter_edge):
+ img = tensor_to_pil(image)
+ w, h = img.size
+ if w < h:
+ new_w = shorter_edge
+ new_h = int(h * (shorter_edge / w))
+ else:
+ new_h = shorter_edge
+ new_w = int(w * (shorter_edge / h))
+ img = img.resize((new_w, new_h), Image.Resampling.LANCZOS)
+ return pil_to_tensor(img)
+
+
+class ResizeImagesByLongerEdgeNode(ImageProcessingNode):
+ node_id = "ResizeImagesByLongerEdge"
+ display_name = "Resize Images by Longer Edge"
+ description = "Resize images so that the longer edge matches the specified length while preserving aspect ratio."
+ extra_inputs = [
+ io.Int.Input(
+ "longer_edge",
+ default=1024,
+ min=1,
+ max=8192,
+ tooltip="Target length for the longer edge.",
+ ),
+ ]
+
+ @classmethod
+ def _process(cls, image, longer_edge):
+ img = tensor_to_pil(image)
+ w, h = img.size
+ if w > h:
+ new_w = longer_edge
+ new_h = int(h * (longer_edge / w))
+ else:
+ new_h = longer_edge
+ new_w = int(w * (longer_edge / h))
+ img = img.resize((new_w, new_h), Image.Resampling.LANCZOS)
+ return pil_to_tensor(img)
+
+
+class CenterCropImagesNode(ImageProcessingNode):
+ node_id = "CenterCropImages"
+ display_name = "Center Crop Images"
+ description = "Center crop all images to the specified dimensions."
+ extra_inputs = [
+ io.Int.Input("width", default=512, min=1, max=8192, tooltip="Crop width."),
+ io.Int.Input("height", default=512, min=1, max=8192, tooltip="Crop height."),
+ ]
+
+ @classmethod
+ def _process(cls, image, width, height):
+ img = tensor_to_pil(image)
+ left = max(0, (img.width - width) // 2)
+ top = max(0, (img.height - height) // 2)
+ right = min(img.width, left + width)
+ bottom = min(img.height, top + height)
+ img = img.crop((left, top, right, bottom))
+ return pil_to_tensor(img)
+
+
+class RandomCropImagesNode(ImageProcessingNode):
+ node_id = "RandomCropImages"
+ display_name = "Random Crop Images"
+ description = (
+ "Randomly crop all images to the specified dimensions (for data augmentation)."
+ )
+ extra_inputs = [
+ io.Int.Input("width", default=512, min=1, max=8192, tooltip="Crop width."),
+ io.Int.Input("height", default=512, min=1, max=8192, tooltip="Crop height."),
+ io.Int.Input(
+ "seed", default=0, min=0, max=0xFFFFFFFFFFFFFFFF, tooltip="Random seed."
+ ),
+ ]
+
+ @classmethod
+ def _process(cls, image, width, height, seed):
+ np.random.seed(seed % (2**32 - 1))
+ img = tensor_to_pil(image)
+ max_left = max(0, img.width - width)
+ max_top = max(0, img.height - height)
+ left = np.random.randint(0, max_left + 1) if max_left > 0 else 0
+ top = np.random.randint(0, max_top + 1) if max_top > 0 else 0
+ right = min(img.width, left + width)
+ bottom = min(img.height, top + height)
+ img = img.crop((left, top, right, bottom))
+ return pil_to_tensor(img)
+
+
+class FlipImagesNode(ImageProcessingNode):
+ node_id = "FlipImages"
+ display_name = "Flip Images"
+ description = "Flip all images horizontally or vertically."
+ extra_inputs = [
+ io.Combo.Input(
+ "direction",
+ options=["horizontal", "vertical"],
+ default="horizontal",
+ tooltip="Flip direction.",
+ ),
+ ]
+
+ @classmethod
+ def _process(cls, image, direction):
+ img = tensor_to_pil(image)
+ if direction == "horizontal":
+ img = img.transpose(Image.FLIP_LEFT_RIGHT)
+ else:
+ img = img.transpose(Image.FLIP_TOP_BOTTOM)
+ return pil_to_tensor(img)
+
+
+class NormalizeImagesNode(ImageProcessingNode):
+ node_id = "NormalizeImages"
+ display_name = "Normalize Images"
+ description = "Normalize images using mean and standard deviation."
+ extra_inputs = [
+ io.Float.Input(
+ "mean",
+ default=0.5,
+ min=0.0,
+ max=1.0,
+ tooltip="Mean value for normalization.",
+ ),
+ io.Float.Input(
+ "std",
+ default=0.5,
+ min=0.001,
+ max=1.0,
+ tooltip="Standard deviation for normalization.",
+ ),
+ ]
+
+ @classmethod
+ def _process(cls, image, mean, std):
+ return (image - mean) / std
+
+
+class AdjustBrightnessNode(ImageProcessingNode):
+ node_id = "AdjustBrightness"
+ display_name = "Adjust Brightness"
+ description = "Adjust brightness of all images."
+ extra_inputs = [
+ io.Float.Input(
+ "factor",
+ default=1.0,
+ min=0.0,
+ max=2.0,
+ tooltip="Brightness factor. 1.0 = no change, <1.0 = darker, >1.0 = brighter.",
+ ),
+ ]
+
+ @classmethod
+ def _process(cls, image, factor):
+ return (image * factor).clamp(0.0, 1.0)
+
+
+class AdjustContrastNode(ImageProcessingNode):
+ node_id = "AdjustContrast"
+ display_name = "Adjust Contrast"
+ description = "Adjust contrast of all images."
+ extra_inputs = [
+ io.Float.Input(
+ "factor",
+ default=1.0,
+ min=0.0,
+ max=2.0,
+ tooltip="Contrast factor. 1.0 = no change, <1.0 = less contrast, >1.0 = more contrast.",
+ ),
+ ]
+
+ @classmethod
+ def _process(cls, image, factor):
+ return ((image - 0.5) * factor + 0.5).clamp(0.0, 1.0)
+
+
+class ShuffleDatasetNode(ImageProcessingNode):
+ node_id = "ShuffleDataset"
+ display_name = "Shuffle Image Dataset"
+ description = "Randomly shuffle the order of images in the dataset."
+ is_group_process = True # Requires full list to shuffle
+ extra_inputs = [
+ io.Int.Input(
+ "seed", default=0, min=0, max=0xFFFFFFFFFFFFFFFF, tooltip="Random seed."
+ ),
+ ]
+
+ @classmethod
+ def _group_process(cls, images, seed):
+ np.random.seed(seed % (2**32 - 1))
+ indices = np.random.permutation(len(images))
+ return [images[i] for i in indices]
+
+
+class ShuffleImageTextDatasetNode(io.ComfyNode):
+ """Special node that shuffles both images and texts together."""
+
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="ShuffleImageTextDataset",
+ display_name="Shuffle Image-Text Dataset",
+ category="dataset/image",
+ is_experimental=True,
+ is_input_list=True,
+ inputs=[
+ io.Image.Input("images", tooltip="List of images to shuffle."),
+ io.String.Input("texts", tooltip="List of texts to shuffle."),
+ io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=0xFFFFFFFFFFFFFFFF,
+ tooltip="Random seed.",
+ ),
+ ],
+ outputs=[
+ io.Image.Output(
+ display_name="images",
+ is_output_list=True,
+ tooltip="Shuffled images",
+ ),
+ io.String.Output(
+ display_name="texts", is_output_list=True, tooltip="Shuffled texts"
+ ),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, images, texts, seed):
+ seed = seed[0] # Extract scalar
+ np.random.seed(seed % (2**32 - 1))
+ indices = np.random.permutation(len(images))
+ shuffled_images = [images[i] for i in indices]
+ shuffled_texts = [texts[i] for i in indices]
+ return io.NodeOutput(shuffled_images, shuffled_texts)
+
+
+# ========== Text Transform Nodes ==========
+
+
+class TextToLowercaseNode(TextProcessingNode):
+ node_id = "TextToLowercase"
+ display_name = "Text to Lowercase"
+ description = "Convert all texts to lowercase."
+
+ @classmethod
+ def _process(cls, text):
+ return text.lower()
+
+
+class TextToUppercaseNode(TextProcessingNode):
+ node_id = "TextToUppercase"
+ display_name = "Text to Uppercase"
+ description = "Convert all texts to uppercase."
+
+ @classmethod
+ def _process(cls, text):
+ return text.upper()
+
+
+class TruncateTextNode(TextProcessingNode):
+ node_id = "TruncateText"
+ display_name = "Truncate Text"
+ description = "Truncate all texts to a maximum length."
+ extra_inputs = [
+ io.Int.Input(
+ "max_length", default=77, min=1, max=10000, tooltip="Maximum text length."
+ ),
+ ]
+
+ @classmethod
+ def _process(cls, text, max_length):
+ return text[:max_length]
+
+
+class AddTextPrefixNode(TextProcessingNode):
+ node_id = "AddTextPrefix"
+ display_name = "Add Text Prefix"
+ description = "Add a prefix to all texts."
+ extra_inputs = [
+ io.String.Input("prefix", default="", tooltip="Prefix to add."),
+ ]
+
+ @classmethod
+ def _process(cls, text, prefix):
+ return prefix + text
+
+
+class AddTextSuffixNode(TextProcessingNode):
+ node_id = "AddTextSuffix"
+ display_name = "Add Text Suffix"
+ description = "Add a suffix to all texts."
+ extra_inputs = [
+ io.String.Input("suffix", default="", tooltip="Suffix to add."),
+ ]
+
+ @classmethod
+ def _process(cls, text, suffix):
+ return text + suffix
+
+
+class ReplaceTextNode(TextProcessingNode):
+ node_id = "ReplaceText"
+ display_name = "Replace Text"
+ description = "Replace text in all texts."
+ extra_inputs = [
+ io.String.Input("find", default="", tooltip="Text to find."),
+ io.String.Input("replace", default="", tooltip="Text to replace with."),
+ ]
+
+ @classmethod
+ def _process(cls, text, find, replace):
+ return text.replace(find, replace)
+
+
+class StripWhitespaceNode(TextProcessingNode):
+ node_id = "StripWhitespace"
+ display_name = "Strip Whitespace"
+ description = "Strip leading and trailing whitespace from all texts."
+
+ @classmethod
+ def _process(cls, text):
+ return text.strip()
+
+
+# ========== Group Processing Example Nodes ==========
+
+
+class ImageDeduplicationNode(ImageProcessingNode):
+ """Remove duplicate or very similar images from the dataset using perceptual hashing."""
+
+ node_id = "ImageDeduplication"
+ display_name = "Image Deduplication"
+ description = "Remove duplicate or very similar images from the dataset."
+ is_group_process = True # Requires full list to compare images
+ extra_inputs = [
+ io.Float.Input(
+ "similarity_threshold",
+ default=0.95,
+ min=0.0,
+ max=1.0,
+ tooltip="Similarity threshold (0-1). Higher means more similar. Images above this threshold are considered duplicates.",
+ ),
+ ]
+
+ @classmethod
+ def _group_process(cls, images, similarity_threshold):
+ """Remove duplicate images using perceptual hashing."""
+ if len(images) == 0:
+ return []
+
+ # Compute simple perceptual hash for each image
+ def compute_hash(img_tensor):
+ """Compute a simple perceptual hash by resizing to 8x8 and comparing to average."""
+ img = tensor_to_pil(img_tensor)
+ # Resize to 8x8
+ img_small = img.resize((8, 8), Image.Resampling.LANCZOS).convert("L")
+ # Get pixels
+ pixels = list(img_small.getdata())
+ # Compute average
+ avg = sum(pixels) / len(pixels)
+ # Create hash (1 if above average, 0 otherwise)
+ hash_bits = "".join("1" if p > avg else "0" for p in pixels)
+ return hash_bits
+
+ def hamming_distance(hash1, hash2):
+ """Compute Hamming distance between two hash strings."""
+ return sum(c1 != c2 for c1, c2 in zip(hash1, hash2))
+
+ # Compute hashes for all images
+ hashes = [compute_hash(img) for img in images]
+
+ # Find duplicates
+ keep_indices = []
+ for i in range(len(images)):
+ is_duplicate = False
+ for j in keep_indices:
+ # Compare hashes
+ distance = hamming_distance(hashes[i], hashes[j])
+ similarity = 1.0 - (distance / 64.0) # 64 bits total
+ if similarity >= similarity_threshold:
+ is_duplicate = True
+ logging.info(
+ f"Image {i} is similar to image {j} (similarity: {similarity:.3f}), skipping"
+ )
+ break
+
+ if not is_duplicate:
+ keep_indices.append(i)
+
+ # Return only unique images
+ unique_images = [images[i] for i in keep_indices]
+ logging.info(
+ f"Deduplication: kept {len(unique_images)} out of {len(images)} images"
+ )
+ return unique_images
+
+
+class ImageGridNode(ImageProcessingNode):
+ """Combine multiple images into a single grid/collage."""
+
+ node_id = "ImageGrid"
+ display_name = "Image Grid"
+ description = "Arrange multiple images into a grid layout."
+ is_group_process = True # Requires full list to create grid
+ is_output_list = False # Outputs single grid image
+ extra_inputs = [
+ io.Int.Input(
+ "columns",
+ default=4,
+ min=1,
+ max=20,
+ tooltip="Number of columns in the grid.",
+ ),
+ io.Int.Input(
+ "cell_width",
+ default=256,
+ min=32,
+ max=2048,
+ tooltip="Width of each cell in the grid.",
+ ),
+ io.Int.Input(
+ "cell_height",
+ default=256,
+ min=32,
+ max=2048,
+ tooltip="Height of each cell in the grid.",
+ ),
+ io.Int.Input(
+ "padding", default=4, min=0, max=50, tooltip="Padding between images."
+ ),
+ ]
+
+ @classmethod
+ def _group_process(cls, images, columns, cell_width, cell_height, padding):
+ """Arrange images into a grid."""
+ if len(images) == 0:
+ raise ValueError("Cannot create grid from empty image list")
+
+ # Calculate grid dimensions
+ num_images = len(images)
+ rows = (num_images + columns - 1) // columns # Ceiling division
+
+ # Calculate total grid size
+ grid_width = columns * cell_width + (columns - 1) * padding
+ grid_height = rows * cell_height + (rows - 1) * padding
+
+ # Create blank grid
+ grid = Image.new("RGB", (grid_width, grid_height), (0, 0, 0))
+
+ # Place images
+ for idx, img_tensor in enumerate(images):
+ row = idx // columns
+ col = idx % columns
+
+ # Convert to PIL and resize to cell size
+ img = tensor_to_pil(img_tensor)
+ img = img.resize((cell_width, cell_height), Image.Resampling.LANCZOS)
+
+ # Calculate position
+ x = col * (cell_width + padding)
+ y = row * (cell_height + padding)
+
+ # Paste into grid
+ grid.paste(img, (x, y))
+
+ logging.info(
+ f"Created {columns}x{rows} grid with {num_images} images ({grid_width}x{grid_height})"
+ )
+ return pil_to_tensor(grid)
+
+
+class MergeImageListsNode(ImageProcessingNode):
+ """Merge multiple image lists into a single list."""
+
+ node_id = "MergeImageLists"
+ display_name = "Merge Image Lists"
+ description = "Concatenate multiple image lists into one."
+ is_group_process = True # Receives images as list
+
+ @classmethod
+ def _group_process(cls, images):
+ """Simply return the images list (already merged by input handling)."""
+ # When multiple list inputs are connected, they're concatenated
+ # For now, this is a simple pass-through
+ logging.info(f"Merged image list contains {len(images)} images")
+ return images
+
+
+class MergeTextListsNode(TextProcessingNode):
+ """Merge multiple text lists into a single list."""
+
+ node_id = "MergeTextLists"
+ display_name = "Merge Text Lists"
+ description = "Concatenate multiple text lists into one."
+ is_group_process = True # Receives texts as list
+
+ @classmethod
+ def _group_process(cls, texts):
+ """Simply return the texts list (already merged by input handling)."""
+ # When multiple list inputs are connected, they're concatenated
+ # For now, this is a simple pass-through
+ logging.info(f"Merged text list contains {len(texts)} texts")
+ return texts
+
+
+# ========== Training Dataset Nodes ==========
+
+
+class MakeTrainingDataset(io.ComfyNode):
+ """Encode images with VAE and texts with CLIP to create a training dataset."""
+
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="MakeTrainingDataset",
+ display_name="Make Training Dataset",
+ category="dataset",
+ is_experimental=True,
+ is_input_list=True, # images and texts as lists
+ inputs=[
+ io.Image.Input("images", tooltip="List of images to encode."),
+ io.Vae.Input(
+ "vae", tooltip="VAE model for encoding images to latents."
+ ),
+ io.Clip.Input(
+ "clip", tooltip="CLIP model for encoding text to conditioning."
+ ),
+ io.String.Input(
+ "texts",
+ optional=True,
+ tooltip="List of text captions. Can be length n (matching images), 1 (repeated for all), or omitted (uses empty string).",
+ ),
+ ],
+ outputs=[
+ io.Latent.Output(
+ display_name="latents",
+ is_output_list=True,
+ tooltip="List of latent dicts",
+ ),
+ io.Conditioning.Output(
+ display_name="conditioning",
+ is_output_list=True,
+ tooltip="List of conditioning lists",
+ ),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, images, vae, clip, texts=None):
+ # Extract scalars (vae and clip are single values wrapped in lists)
+ vae = vae[0]
+ clip = clip[0]
+
+ # Handle text list
+ num_images = len(images)
+
+ if texts is None or len(texts) == 0:
+ # Treat as [""] for unconditional training
+ texts = [""]
+
+ if len(texts) == 1 and num_images > 1:
+ # Repeat single text for all images
+ texts = texts * num_images
+ elif len(texts) != num_images:
+ raise ValueError(
+ f"Number of texts ({len(texts)}) does not match number of images ({num_images}). "
+ f"Text list should have length {num_images}, 1, or 0."
+ )
+
+ # Encode images with VAE
+ logging.info(f"Encoding {num_images} images with VAE...")
+ latents_list = [] # list[{"samples": tensor}]
+ for img_tensor in images:
+ # img_tensor is [1, H, W, 3]
+ latent_tensor = vae.encode(img_tensor[:, :, :, :3])
+ latents_list.append({"samples": latent_tensor})
+
+ # Encode texts with CLIP
+ logging.info(f"Encoding {len(texts)} texts with CLIP...")
+ conditioning_list = [] # list[list[cond]]
+ for text in texts:
+ if text == "":
+ cond = clip.encode_from_tokens_scheduled(clip.tokenize(""))
+ else:
+ tokens = clip.tokenize(text)
+ cond = clip.encode_from_tokens_scheduled(tokens)
+ conditioning_list.append(cond)
+
+ logging.info(
+ f"Created dataset with {len(latents_list)} latents and {len(conditioning_list)} conditioning."
+ )
+ return io.NodeOutput(latents_list, conditioning_list)
+
+
+class SaveTrainingDataset(io.ComfyNode):
+ """Save encoded training dataset (latents + conditioning) to disk."""
+
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="SaveTrainingDataset",
+ display_name="Save Training Dataset",
+ category="dataset",
+ is_experimental=True,
+ is_output_node=True,
+ is_input_list=True, # Receive lists
+ inputs=[
+ io.Latent.Input(
+ "latents",
+ tooltip="List of latent dicts from MakeTrainingDataset.",
+ ),
+ io.Conditioning.Input(
+ "conditioning",
+ tooltip="List of conditioning lists from MakeTrainingDataset.",
+ ),
+ io.String.Input(
+ "folder_name",
+ default="training_dataset",
+ tooltip="Name of folder to save dataset (inside output directory).",
+ ),
+ io.Int.Input(
+ "shard_size",
+ default=1000,
+ min=1,
+ max=100000,
+ tooltip="Number of samples per shard file.",
+ ),
+ ],
+ outputs=[],
+ )
+
+ @classmethod
+ def execute(cls, latents, conditioning, folder_name, shard_size):
+ # Extract scalars
+ folder_name = folder_name[0]
+ shard_size = shard_size[0]
+
+ # latents: list[{"samples": tensor}]
+ # conditioning: list[list[cond]]
+
+ # Validate lengths match
+ if len(latents) != len(conditioning):
+ raise ValueError(
+ f"Number of latents ({len(latents)}) does not match number of conditions ({len(conditioning)}). "
+ f"Something went wrong in dataset preparation."
+ )
+
+ # Create output directory
+ output_dir = os.path.join(folder_paths.get_output_directory(), folder_name)
+ os.makedirs(output_dir, exist_ok=True)
+
+ # Prepare data pairs
+ num_samples = len(latents)
+ num_shards = (num_samples + shard_size - 1) // shard_size # Ceiling division
+
+ logging.info(
+ f"Saving {num_samples} samples to {num_shards} shards in {output_dir}..."
+ )
+
+ # Save data in shards
+ for shard_idx in range(num_shards):
+ start_idx = shard_idx * shard_size
+ end_idx = min(start_idx + shard_size, num_samples)
+
+ # Get shard data (list of latent dicts and conditioning lists)
+ shard_data = {
+ "latents": latents[start_idx:end_idx],
+ "conditioning": conditioning[start_idx:end_idx],
+ }
+
+ # Save shard
+ shard_filename = f"shard_{shard_idx:04d}.pkl"
+ shard_path = os.path.join(output_dir, shard_filename)
+
+ with open(shard_path, "wb") as f:
+ torch.save(shard_data, f)
+
+ logging.info(
+ f"Saved shard {shard_idx + 1}/{num_shards}: {shard_filename} ({end_idx - start_idx} samples)"
+ )
+
+ # Save metadata
+ metadata = {
+ "num_samples": num_samples,
+ "num_shards": num_shards,
+ "shard_size": shard_size,
+ }
+ metadata_path = os.path.join(output_dir, "metadata.json")
+ with open(metadata_path, "w") as f:
+ json.dump(metadata, f, indent=2)
+
+ logging.info(f"Successfully saved {num_samples} samples to {output_dir}.")
+ return io.NodeOutput()
+
+
+class LoadTrainingDataset(io.ComfyNode):
+ """Load encoded training dataset from disk."""
+
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="LoadTrainingDataset",
+ display_name="Load Training Dataset",
+ category="dataset",
+ is_experimental=True,
+ inputs=[
+ io.String.Input(
+ "folder_name",
+ default="training_dataset",
+ tooltip="Name of folder containing the saved dataset (inside output directory).",
+ ),
+ ],
+ outputs=[
+ io.Latent.Output(
+ display_name="latents",
+ is_output_list=True,
+ tooltip="List of latent dicts",
+ ),
+ io.Conditioning.Output(
+ display_name="conditioning",
+ is_output_list=True,
+ tooltip="List of conditioning lists",
+ ),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, folder_name):
+ # Get dataset directory
+ dataset_dir = os.path.join(folder_paths.get_output_directory(), folder_name)
+
+ if not os.path.exists(dataset_dir):
+ raise ValueError(f"Dataset directory not found: {dataset_dir}")
+
+ # Find all shard files
+ shard_files = sorted(
+ [
+ f
+ for f in os.listdir(dataset_dir)
+ if f.startswith("shard_") and f.endswith(".pkl")
+ ]
+ )
+
+ if not shard_files:
+ raise ValueError(f"No shard files found in {dataset_dir}")
+
+ logging.info(f"Loading {len(shard_files)} shards from {dataset_dir}...")
+
+ # Load all shards
+ all_latents = [] # list[{"samples": tensor}]
+ all_conditioning = [] # list[list[cond]]
+
+ for shard_file in shard_files:
+ shard_path = os.path.join(dataset_dir, shard_file)
+
+ with open(shard_path, "rb") as f:
+ shard_data = torch.load(f)
+
+ all_latents.extend(shard_data["latents"])
+ all_conditioning.extend(shard_data["conditioning"])
+
+ logging.info(f"Loaded {shard_file}: {len(shard_data['latents'])} samples")
+
+ logging.info(
+ f"Successfully loaded {len(all_latents)} samples from {dataset_dir}."
+ )
+ return io.NodeOutput(all_latents, all_conditioning)
+
+
+# ========== Extension Setup ==========
+
+
+class DatasetExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ # Data loading/saving nodes
+ LoadImageDataSetFromFolderNode,
+ LoadImageTextDataSetFromFolderNode,
+ SaveImageDataSetToFolderNode,
+ SaveImageTextDataSetToFolderNode,
+ # Image transform nodes
+ ResizeImagesToSameSizeNode,
+ ResizeImagesToPixelCountNode,
+ ResizeImagesByShorterEdgeNode,
+ ResizeImagesByLongerEdgeNode,
+ CenterCropImagesNode,
+ RandomCropImagesNode,
+ FlipImagesNode,
+ NormalizeImagesNode,
+ AdjustBrightnessNode,
+ AdjustContrastNode,
+ ShuffleDatasetNode,
+ ShuffleImageTextDatasetNode,
+ # Text transform nodes
+ TextToLowercaseNode,
+ TextToUppercaseNode,
+ TruncateTextNode,
+ AddTextPrefixNode,
+ AddTextSuffixNode,
+ ReplaceTextNode,
+ StripWhitespaceNode,
+ # Group processing examples
+ ImageDeduplicationNode,
+ ImageGridNode,
+ MergeImageListsNode,
+ MergeTextListsNode,
+ # Training dataset nodes
+ MakeTrainingDataset,
+ SaveTrainingDataset,
+ LoadTrainingDataset,
+ ]
+
+
+async def comfy_entrypoint() -> DatasetExtension:
+ return DatasetExtension()
diff --git a/comfy_extras/nodes_train.py b/comfy_extras/nodes_train.py
index 9e6ec6780..cb24ab709 100644
--- a/comfy_extras/nodes_train.py
+++ b/comfy_extras/nodes_train.py
@@ -1,15 +1,13 @@
-import datetime
-import json
import logging
import os
import numpy as np
import safetensors
import torch
-from PIL import Image, ImageDraw, ImageFont
-from PIL.PngImagePlugin import PngInfo
import torch.utils.checkpoint
-import tqdm
+from tqdm.auto import trange
+from PIL import Image, ImageDraw, ImageFont
+from typing_extensions import override
import comfy.samplers
import comfy.sd
@@ -18,9 +16,9 @@ import comfy.model_management
import comfy_extras.nodes_custom_sampler
import folder_paths
import node_helpers
-from comfy.cli_args import args
-from comfy.comfy_types.node_typing import IO
from comfy.weight_adapter import adapters, adapter_maps
+from comfy_api.latest import ComfyExtension, io, ui
+from comfy.utils import ProgressBar
def make_batch_extra_option_dict(d, indicies, full_size=None):
@@ -56,7 +54,18 @@ def process_cond_list(d, prefix=""):
class TrainSampler(comfy.samplers.Sampler):
- def __init__(self, loss_fn, optimizer, loss_callback=None, batch_size=1, grad_acc=1, total_steps=1, seed=0, training_dtype=torch.bfloat16):
+ def __init__(
+ self,
+ loss_fn,
+ optimizer,
+ loss_callback=None,
+ batch_size=1,
+ grad_acc=1,
+ total_steps=1,
+ seed=0,
+ training_dtype=torch.bfloat16,
+ real_dataset=None,
+ ):
self.loss_fn = loss_fn
self.optimizer = optimizer
self.loss_callback = loss_callback
@@ -65,54 +74,138 @@ class TrainSampler(comfy.samplers.Sampler):
self.grad_acc = grad_acc
self.seed = seed
self.training_dtype = training_dtype
+ self.real_dataset: list[torch.Tensor] | None = real_dataset
+
+ def fwd_bwd(
+ self,
+ model_wrap,
+ batch_sigmas,
+ batch_noise,
+ batch_latent,
+ cond,
+ indicies,
+ extra_args,
+ dataset_size,
+ bwd=True,
+ ):
+ xt = model_wrap.inner_model.model_sampling.noise_scaling(
+ batch_sigmas, batch_noise, batch_latent, False
+ )
+ x0 = model_wrap.inner_model.model_sampling.noise_scaling(
+ torch.zeros_like(batch_sigmas),
+ torch.zeros_like(batch_noise),
+ batch_latent,
+ False,
+ )
+
+ model_wrap.conds["positive"] = [cond[i] for i in indicies]
+ batch_extra_args = make_batch_extra_option_dict(
+ extra_args, indicies, full_size=dataset_size
+ )
+
+ with torch.autocast(xt.device.type, dtype=self.training_dtype):
+ x0_pred = model_wrap(
+ xt.requires_grad_(True),
+ batch_sigmas.requires_grad_(True),
+ **batch_extra_args,
+ )
+ loss = self.loss_fn(x0_pred, x0)
+ if bwd:
+ bwd_loss = loss / self.grad_acc
+ bwd_loss.backward()
+ return loss
- def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
+ def sample(
+ self,
+ model_wrap,
+ sigmas,
+ extra_args,
+ callback,
+ noise,
+ latent_image=None,
+ denoise_mask=None,
+ disable_pbar=False,
+ ):
model_wrap.conds = process_cond_list(model_wrap.conds)
cond = model_wrap.conds["positive"]
dataset_size = sigmas.size(0)
torch.cuda.empty_cache()
- for i in (pbar:=tqdm.trange(self.total_steps, desc="Training LoRA", smoothing=0.01, disable=not comfy.utils.PROGRESS_BAR_ENABLED)):
- noisegen = comfy_extras.nodes_custom_sampler.Noise_RandomNoise(self.seed + i * 1000)
- indicies = torch.randperm(dataset_size)[:self.batch_size].tolist()
-
- batch_latent = torch.stack([latent_image[i] for i in indicies])
- batch_noise = noisegen.generate_noise({"samples": batch_latent}).to(batch_latent.device)
- batch_sigmas = [
- model_wrap.inner_model.model_sampling.percent_to_sigma(
- torch.rand((1,)).item()
- ) for _ in range(min(self.batch_size, dataset_size))
- ]
- batch_sigmas = torch.tensor(batch_sigmas).to(batch_latent.device)
-
- xt = model_wrap.inner_model.model_sampling.noise_scaling(
- batch_sigmas,
- batch_noise,
- batch_latent,
- False
+ ui_pbar = ProgressBar(self.total_steps)
+ for i in (
+ pbar := trange(
+ self.total_steps,
+ desc="Training LoRA",
+ smoothing=0.01,
+ disable=not comfy.utils.PROGRESS_BAR_ENABLED,
)
- x0 = model_wrap.inner_model.model_sampling.noise_scaling(
- torch.zeros_like(batch_sigmas),
- torch.zeros_like(batch_noise),
- batch_latent,
- False
+ ):
+ noisegen = comfy_extras.nodes_custom_sampler.Noise_RandomNoise(
+ self.seed + i * 1000
)
+ indicies = torch.randperm(dataset_size)[: self.batch_size].tolist()
- model_wrap.conds["positive"] = [
- cond[i] for i in indicies
- ]
- batch_extra_args = make_batch_extra_option_dict(extra_args, indicies, full_size=dataset_size)
-
- with torch.autocast(xt.device.type, dtype=self.training_dtype):
- x0_pred = model_wrap(xt, batch_sigmas, **batch_extra_args)
- loss = self.loss_fn(x0_pred, x0)
- loss.backward()
- if self.loss_callback:
- self.loss_callback(loss.item())
- pbar.set_postfix({"loss": f"{loss.item():.4f}"})
-
- if (i+1) % self.grad_acc == 0:
+ if self.real_dataset is None:
+ batch_latent = torch.stack([latent_image[i] for i in indicies])
+ batch_noise = noisegen.generate_noise({"samples": batch_latent}).to(
+ batch_latent.device
+ )
+ batch_sigmas = [
+ model_wrap.inner_model.model_sampling.percent_to_sigma(
+ torch.rand((1,)).item()
+ )
+ for _ in range(min(self.batch_size, dataset_size))
+ ]
+ batch_sigmas = torch.tensor(batch_sigmas).to(batch_latent.device)
+
+ loss = self.fwd_bwd(
+ model_wrap,
+ batch_sigmas,
+ batch_noise,
+ batch_latent,
+ cond,
+ indicies,
+ extra_args,
+ dataset_size,
+ bwd=True,
+ )
+ if self.loss_callback:
+ self.loss_callback(loss.item())
+ pbar.set_postfix({"loss": f"{loss.item():.4f}"})
+ else:
+ total_loss = 0
+ for index in indicies:
+ single_latent = self.real_dataset[index].to(latent_image)
+ batch_noise = noisegen.generate_noise(
+ {"samples": single_latent}
+ ).to(single_latent.device)
+ batch_sigmas = (
+ model_wrap.inner_model.model_sampling.percent_to_sigma(
+ torch.rand((1,)).item()
+ )
+ )
+ batch_sigmas = torch.tensor([batch_sigmas]).to(single_latent.device)
+ loss = self.fwd_bwd(
+ model_wrap,
+ batch_sigmas,
+ batch_noise,
+ single_latent,
+ cond,
+ [index],
+ extra_args,
+ dataset_size,
+ bwd=False,
+ )
+ total_loss += loss
+ total_loss = total_loss / self.grad_acc / len(indicies)
+ total_loss.backward()
+ if self.loss_callback:
+ self.loss_callback(total_loss.item())
+ pbar.set_postfix({"loss": f"{total_loss.item():.4f}"})
+
+ if (i + 1) % self.grad_acc == 0:
self.optimizer.step()
self.optimizer.zero_grad()
+ ui_pbar.update(1)
torch.cuda.empty_cache()
return torch.zeros_like(latent_image)
@@ -134,233 +227,6 @@ class BiasDiff(torch.nn.Module):
return self.passive_memory_usage()
-def load_and_process_images(image_files, input_dir, resize_method="None", w=None, h=None):
- """Utility function to load and process a list of images.
-
- Args:
- image_files: List of image filenames
- input_dir: Base directory containing the images
- resize_method: How to handle images of different sizes ("None", "Stretch", "Crop", "Pad")
-
- Returns:
- torch.Tensor: Batch of processed images
- """
- if not image_files:
- raise ValueError("No valid images found in input")
-
- output_images = []
-
- for file in image_files:
- image_path = os.path.join(input_dir, file)
- img = node_helpers.pillow(Image.open, image_path)
-
- if img.mode == "I":
- img = img.point(lambda i: i * (1 / 255))
- img = img.convert("RGB")
-
- if w is None and h is None:
- w, h = img.size[0], img.size[1]
-
- # Resize image to first image
- if img.size[0] != w or img.size[1] != h:
- if resize_method == "Stretch":
- img = img.resize((w, h), Image.Resampling.LANCZOS)
- elif resize_method == "Crop":
- img = img.crop((0, 0, w, h))
- elif resize_method == "Pad":
- img = img.resize((w, h), Image.Resampling.LANCZOS)
- elif resize_method == "None":
- raise ValueError(
- "Your input image size does not match the first image in the dataset. Either select a valid resize method or use the same size for all images."
- )
-
- img_array = np.array(img).astype(np.float32) / 255.0
- img_tensor = torch.from_numpy(img_array)[None,]
- output_images.append(img_tensor)
-
- return torch.cat(output_images, dim=0)
-
-
-class LoadImageSetNode:
- @classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "images": (
- [
- f
- for f in os.listdir(folder_paths.get_input_directory())
- if f.endswith((".png", ".jpg", ".jpeg", ".webp", ".bmp", ".gif", ".jpe", ".apng", ".tif", ".tiff"))
- ],
- {"image_upload": True, "allow_batch": True},
- )
- },
- "optional": {
- "resize_method": (
- ["None", "Stretch", "Crop", "Pad"],
- {"default": "None"},
- ),
- },
- }
-
- INPUT_IS_LIST = True
- RETURN_TYPES = ("IMAGE",)
- FUNCTION = "load_images"
- CATEGORY = "loaders"
- EXPERIMENTAL = True
- DESCRIPTION = "Loads a batch of images from a directory for training."
-
- @classmethod
- def VALIDATE_INPUTS(s, images, resize_method):
- filenames = images[0] if isinstance(images[0], list) else images
-
- for image in filenames:
- if not folder_paths.exists_annotated_filepath(image):
- return "Invalid image file: {}".format(image)
- return True
-
- def load_images(self, input_files, resize_method):
- input_dir = folder_paths.get_input_directory()
- valid_extensions = [".png", ".jpg", ".jpeg", ".webp", ".bmp", ".gif", ".jpe", ".apng", ".tif", ".tiff"]
- image_files = [
- f
- for f in input_files
- if any(f.lower().endswith(ext) for ext in valid_extensions)
- ]
- output_tensor = load_and_process_images(image_files, input_dir, resize_method)
- return (output_tensor,)
-
-
-class LoadImageSetFromFolderNode:
- @classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "folder": (folder_paths.get_input_subfolders(), {"tooltip": "The folder to load images from."})
- },
- "optional": {
- "resize_method": (
- ["None", "Stretch", "Crop", "Pad"],
- {"default": "None"},
- ),
- },
- }
-
- RETURN_TYPES = ("IMAGE",)
- FUNCTION = "load_images"
- CATEGORY = "loaders"
- EXPERIMENTAL = True
- DESCRIPTION = "Loads a batch of images from a directory for training."
-
- def load_images(self, folder, resize_method):
- sub_input_dir = os.path.join(folder_paths.get_input_directory(), folder)
- valid_extensions = [".png", ".jpg", ".jpeg", ".webp"]
- image_files = [
- f
- for f in os.listdir(sub_input_dir)
- if any(f.lower().endswith(ext) for ext in valid_extensions)
- ]
- output_tensor = load_and_process_images(image_files, sub_input_dir, resize_method)
- return (output_tensor,)
-
-
-class LoadImageTextSetFromFolderNode:
- @classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "folder": (folder_paths.get_input_subfolders(), {"tooltip": "The folder to load images from."}),
- "clip": (IO.CLIP, {"tooltip": "The CLIP model used for encoding the text."}),
- },
- "optional": {
- "resize_method": (
- ["None", "Stretch", "Crop", "Pad"],
- {"default": "None"},
- ),
- "width": (
- IO.INT,
- {
- "default": -1,
- "min": -1,
- "max": 10000,
- "step": 1,
- "tooltip": "The width to resize the images to. -1 means use the original width.",
- },
- ),
- "height": (
- IO.INT,
- {
- "default": -1,
- "min": -1,
- "max": 10000,
- "step": 1,
- "tooltip": "The height to resize the images to. -1 means use the original height.",
- },
- )
- },
- }
-
- RETURN_TYPES = ("IMAGE", IO.CONDITIONING,)
- FUNCTION = "load_images"
- CATEGORY = "loaders"
- EXPERIMENTAL = True
- DESCRIPTION = "Loads a batch of images and caption from a directory for training."
-
- def load_images(self, folder, clip, resize_method, width=None, height=None):
- if clip is None:
- raise RuntimeError("ERROR: clip input is invalid: None\n\nIf the clip is from a checkpoint loader node your checkpoint does not contain a valid clip or text encoder model.")
-
- logging.info(f"Loading images from folder: {folder}")
-
- sub_input_dir = os.path.join(folder_paths.get_input_directory(), folder)
- valid_extensions = [".png", ".jpg", ".jpeg", ".webp"]
-
- image_files = []
- for item in os.listdir(sub_input_dir):
- path = os.path.join(sub_input_dir, item)
- if any(item.lower().endswith(ext) for ext in valid_extensions):
- image_files.append(path)
- elif os.path.isdir(path):
- # Support kohya-ss/sd-scripts folder structure
- repeat = 1
- if item.split("_")[0].isdigit():
- repeat = int(item.split("_")[0])
- image_files.extend([
- os.path.join(path, f) for f in os.listdir(path) if any(f.lower().endswith(ext) for ext in valid_extensions)
- ] * repeat)
-
- caption_file_path = [
- f.replace(os.path.splitext(f)[1], ".txt")
- for f in image_files
- ]
- captions = []
- for caption_file in caption_file_path:
- caption_path = os.path.join(sub_input_dir, caption_file)
- if os.path.exists(caption_path):
- with open(caption_path, "r", encoding="utf-8") as f:
- caption = f.read().strip()
- captions.append(caption)
- else:
- captions.append("")
-
- width = width if width != -1 else None
- height = height if height != -1 else None
- output_tensor = load_and_process_images(image_files, sub_input_dir, resize_method, width, height)
-
- logging.info(f"Loaded {len(output_tensor)} images from {sub_input_dir}.")
-
- logging.info(f"Encoding captions from {sub_input_dir}.")
- conditions = []
- empty_cond = clip.encode_from_tokens_scheduled(clip.tokenize(""))
- for text in captions:
- if text == "":
- conditions.append(empty_cond)
- tokens = clip.tokenize(text)
- conditions.extend(clip.encode_from_tokens_scheduled(tokens))
- logging.info(f"Encoded {len(conditions)} captions from {sub_input_dir}.")
- return (output_tensor, conditions)
-
-
def draw_loss_graph(loss_map, steps):
width, height = 500, 300
img = Image.new("RGB", (width, height), "white")
@@ -379,10 +245,14 @@ def draw_loss_graph(loss_map, steps):
return img
-def find_all_highest_child_module_with_forward(model: torch.nn.Module, result = None, name = None):
+def find_all_highest_child_module_with_forward(
+ model: torch.nn.Module, result=None, name=None
+):
if result is None:
result = []
- elif hasattr(model, "forward") and not isinstance(model, (torch.nn.ModuleList, torch.nn.Sequential, torch.nn.ModuleDict)):
+ elif hasattr(model, "forward") and not isinstance(
+ model, (torch.nn.ModuleList, torch.nn.Sequential, torch.nn.ModuleDict)
+ ):
result.append(model)
logging.debug(f"Found module with forward: {name} ({model.__class__.__name__})")
return result
@@ -396,12 +266,13 @@ def patch(m):
if not hasattr(m, "forward"):
return
org_forward = m.forward
+
def fwd(args, kwargs):
return org_forward(*args, **kwargs)
+
def checkpointing_fwd(*args, **kwargs):
- return torch.utils.checkpoint.checkpoint(
- fwd, args, kwargs, use_reentrant=False
- )
+ return torch.utils.checkpoint.checkpoint(fwd, args, kwargs, use_reentrant=False)
+
m.org_forward = org_forward
m.forward = checkpointing_fwd
@@ -412,130 +283,126 @@ def unpatch(m):
del m.org_forward
-class TrainLoraNode:
+class TrainLoraNode(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "model": (IO.MODEL, {"tooltip": "The model to train the LoRA on."}),
- "latents": (
- "LATENT",
- {
- "tooltip": "The Latents to use for training, serve as dataset/input of the model."
- },
+ def define_schema(cls):
+ return io.Schema(
+ node_id="TrainLoraNode",
+ display_name="Train LoRA",
+ category="training",
+ is_experimental=True,
+ is_input_list=True, # All inputs become lists
+ inputs=[
+ io.Model.Input("model", tooltip="The model to train the LoRA on."),
+ io.Latent.Input(
+ "latents",
+ tooltip="The Latents to use for training, serve as dataset/input of the model.",
),
- "positive": (
- IO.CONDITIONING,
- {"tooltip": "The positive conditioning to use for training."},
+ io.Conditioning.Input(
+ "positive", tooltip="The positive conditioning to use for training."
),
- "batch_size": (
- IO.INT,
- {
- "default": 1,
- "min": 1,
- "max": 10000,
- "step": 1,
- "tooltip": "The batch size to use for training.",
- },
+ io.Int.Input(
+ "batch_size",
+ default=1,
+ min=1,
+ max=10000,
+ tooltip="The batch size to use for training.",
),
- "grad_accumulation_steps": (
- IO.INT,
- {
- "default": 1,
- "min": 1,
- "max": 1024,
- "step": 1,
- "tooltip": "The number of gradient accumulation steps to use for training.",
- }
+ io.Int.Input(
+ "grad_accumulation_steps",
+ default=1,
+ min=1,
+ max=1024,
+ tooltip="The number of gradient accumulation steps to use for training.",
),
- "steps": (
- IO.INT,
- {
- "default": 16,
- "min": 1,
- "max": 100000,
- "tooltip": "The number of steps to train the LoRA for.",
- },
+ io.Int.Input(
+ "steps",
+ default=16,
+ min=1,
+ max=100000,
+ tooltip="The number of steps to train the LoRA for.",
),
- "learning_rate": (
- IO.FLOAT,
- {
- "default": 0.0005,
- "min": 0.0000001,
- "max": 1.0,
- "step": 0.000001,
- "tooltip": "The learning rate to use for training.",
- },
+ io.Float.Input(
+ "learning_rate",
+ default=0.0005,
+ min=0.0000001,
+ max=1.0,
+ step=0.0000001,
+ tooltip="The learning rate to use for training.",
),
- "rank": (
- IO.INT,
- {
- "default": 8,
- "min": 1,
- "max": 128,
- "tooltip": "The rank of the LoRA layers.",
- },
+ io.Int.Input(
+ "rank",
+ default=8,
+ min=1,
+ max=128,
+ tooltip="The rank of the LoRA layers.",
),
- "optimizer": (
- ["AdamW", "Adam", "SGD", "RMSprop"],
- {
- "default": "AdamW",
- "tooltip": "The optimizer to use for training.",
- },
+ io.Combo.Input(
+ "optimizer",
+ options=["AdamW", "Adam", "SGD", "RMSprop"],
+ default="AdamW",
+ tooltip="The optimizer to use for training.",
),
- "loss_function": (
- ["MSE", "L1", "Huber", "SmoothL1"],
- {
- "default": "MSE",
- "tooltip": "The loss function to use for training.",
- },
+ io.Combo.Input(
+ "loss_function",
+ options=["MSE", "L1", "Huber", "SmoothL1"],
+ default="MSE",
+ tooltip="The loss function to use for training.",
),
- "seed": (
- IO.INT,
- {
- "default": 0,
- "min": 0,
- "max": 0xFFFFFFFFFFFFFFFF,
- "tooltip": "The seed to use for training (used in generator for LoRA weight initialization and noise sampling)",
- },
+ io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=0xFFFFFFFFFFFFFFFF,
+ tooltip="The seed to use for training (used in generator for LoRA weight initialization and noise sampling)",
),
- "training_dtype": (
- ["bf16", "fp32"],
- {"default": "bf16", "tooltip": "The dtype to use for training."},
+ io.Combo.Input(
+ "training_dtype",
+ options=["bf16", "fp32"],
+ default="bf16",
+ tooltip="The dtype to use for training.",
),
- "lora_dtype": (
- ["bf16", "fp32"],
- {"default": "bf16", "tooltip": "The dtype to use for lora."},
+ io.Combo.Input(
+ "lora_dtype",
+ options=["bf16", "fp32"],
+ default="bf16",
+ tooltip="The dtype to use for lora.",
),
- "algorithm": (
- list(adapter_maps.keys()),
- {"default": list(adapter_maps.keys())[0], "tooltip": "The algorithm to use for training."},
+ io.Combo.Input(
+ "algorithm",
+ options=list(adapter_maps.keys()),
+ default=list(adapter_maps.keys())[0],
+ tooltip="The algorithm to use for training.",
),
- "gradient_checkpointing": (
- IO.BOOLEAN,
- {
- "default": True,
- "tooltip": "Use gradient checkpointing for training.",
- }
+ io.Boolean.Input(
+ "gradient_checkpointing",
+ default=True,
+ tooltip="Use gradient checkpointing for training.",
),
- "existing_lora": (
- folder_paths.get_filename_list("loras") + ["[None]"],
- {
- "default": "[None]",
- "tooltip": "The existing LoRA to append to. Set to None for new LoRA.",
- },
+ io.Combo.Input(
+ "existing_lora",
+ options=folder_paths.get_filename_list("loras") + ["[None]"],
+ default="[None]",
+ tooltip="The existing LoRA to append to. Set to None for new LoRA.",
),
- },
- }
-
- RETURN_TYPES = (IO.MODEL, IO.LORA_MODEL, IO.LOSS_MAP, IO.INT)
- RETURN_NAMES = ("model_with_lora", "lora", "loss", "steps")
- FUNCTION = "train"
- CATEGORY = "training"
- EXPERIMENTAL = True
+ ],
+ outputs=[
+ io.Model.Output(
+ display_name="model", tooltip="Model with LoRA applied"
+ ),
+ io.Custom("LORA_MODEL").Output(
+ display_name="lora", tooltip="LoRA weights"
+ ),
+ io.Custom("LOSS_MAP").Output(
+ display_name="loss_map", tooltip="Loss history"
+ ),
+ io.Int.Output(display_name="steps", tooltip="Total training steps"),
+ ],
+ )
- def train(
- self,
+ @classmethod
+ def execute(
+ cls,
model,
latents,
positive,
@@ -553,13 +420,74 @@ class TrainLoraNode:
gradient_checkpointing,
existing_lora,
):
+ # Extract scalars from lists (due to is_input_list=True)
+ model = model[0]
+ batch_size = batch_size[0]
+ steps = steps[0]
+ grad_accumulation_steps = grad_accumulation_steps[0]
+ learning_rate = learning_rate[0]
+ rank = rank[0]
+ optimizer = optimizer[0]
+ loss_function = loss_function[0]
+ seed = seed[0]
+ training_dtype = training_dtype[0]
+ lora_dtype = lora_dtype[0]
+ algorithm = algorithm[0]
+ gradient_checkpointing = gradient_checkpointing[0]
+ existing_lora = existing_lora[0]
+
+ # Handle latents - either single dict or list of dicts
+ if len(latents) == 1:
+ latents = latents[0]["samples"] # Single latent dict
+ else:
+ latent_list = []
+ for latent in latents:
+ latent = latent["samples"]
+ bs = latent.shape[0]
+ if bs != 1:
+ for sub_latent in latent:
+ latent_list.append(sub_latent[None])
+ else:
+ latent_list.append(latent)
+ latents = latent_list
+
+ # Handle conditioning - either single list or list of lists
+ if len(positive) == 1:
+ positive = positive[0] # Single conditioning list
+ else:
+ # Multiple conditioning lists - flatten
+ flat_positive = []
+ for cond in positive:
+ if isinstance(cond, list):
+ flat_positive.extend(cond)
+ else:
+ flat_positive.append(cond)
+ positive = flat_positive
+
mp = model.clone()
dtype = node_helpers.string_to_torch_dtype(training_dtype)
lora_dtype = node_helpers.string_to_torch_dtype(lora_dtype)
mp.set_model_compute_dtype(dtype)
- latents = latents["samples"].to(dtype)
- num_images = latents.shape[0]
+ # latents here can be list of different size latent or one large batch
+ if isinstance(latents, list):
+ all_shapes = set()
+ latents = [t.to(dtype) for t in latents]
+ for latent in latents:
+ all_shapes.add(latent.shape)
+ logging.info(f"Latent shapes: {all_shapes}")
+ if len(all_shapes) > 1:
+ multi_res = True
+ else:
+ multi_res = False
+ latents = torch.cat(latents, dim=0)
+ num_images = len(latents)
+ elif isinstance(latents, torch.Tensor):
+ latents = latents.to(dtype)
+ num_images = latents.shape[0]
+ else:
+ logging.error(f"Invalid latents type: {type(latents)}")
+
logging.info(f"Total Images: {num_images}, Total Captions: {len(positive)}")
if len(positive) == 1 and num_images > 1:
positive = positive * num_images
@@ -591,9 +519,7 @@ class TrainLoraNode:
shape = m.weight.shape
if len(shape) >= 2:
alpha = float(existing_weights.get(f"{key}.alpha", 1.0))
- dora_scale = existing_weights.get(
- f"{key}.dora_scale", None
- )
+ dora_scale = existing_weights.get(f"{key}.dora_scale", None)
for adapter_cls in adapters:
existing_adapter = adapter_cls.load(
n, existing_weights, alpha, dora_scale
@@ -605,7 +531,9 @@ class TrainLoraNode:
adapter_cls = adapter_maps[algorithm]
if existing_adapter is not None:
- train_adapter = existing_adapter.to_train().to(lora_dtype)
+ train_adapter = existing_adapter.to_train().to(
+ lora_dtype
+ )
else:
# Use LoRA with alpha=1.0 by default
train_adapter = adapter_cls.create_train(
@@ -629,7 +557,9 @@ class TrainLoraNode:
if hasattr(m, "bias") and m.bias is not None:
key = "{}.bias".format(n)
bias = torch.nn.Parameter(
- torch.zeros(m.bias.shape, dtype=lora_dtype, requires_grad=True)
+ torch.zeros(
+ m.bias.shape, dtype=lora_dtype, requires_grad=True
+ )
)
bias_module = BiasDiff(bias)
lora_sd["{}.diff_b".format(n)] = bias
@@ -657,24 +587,31 @@ class TrainLoraNode:
# setup models
if gradient_checkpointing:
- for m in find_all_highest_child_module_with_forward(mp.model.diffusion_model):
+ for m in find_all_highest_child_module_with_forward(
+ mp.model.diffusion_model
+ ):
patch(m)
mp.model.requires_grad_(False)
- comfy.model_management.load_models_gpu([mp], memory_required=1e20, force_full_load=True)
+ comfy.model_management.load_models_gpu(
+ [mp], memory_required=1e20, force_full_load=True
+ )
# Setup sampler and guider like in test script
loss_map = {"loss": []}
+
def loss_callback(loss):
loss_map["loss"].append(loss)
+
train_sampler = TrainSampler(
criterion,
optimizer,
loss_callback=loss_callback,
batch_size=batch_size,
grad_acc=grad_accumulation_steps,
- total_steps=steps*grad_accumulation_steps,
+ total_steps=steps * grad_accumulation_steps,
seed=seed,
- training_dtype=dtype
+ training_dtype=dtype,
+ real_dataset=latents if multi_res else None,
)
guider = comfy_extras.nodes_custom_sampler.Guider_Basic(mp)
guider.set_conds(positive) # Set conditioning from input
@@ -684,12 +621,15 @@ class TrainLoraNode:
# Generate dummy sigmas and noise
sigmas = torch.tensor(range(num_images))
noise = comfy_extras.nodes_custom_sampler.Noise_RandomNoise(seed)
+ if multi_res:
+ # use first latent as dummy latent if multi_res
+ latents = latents[0].repeat(num_images, 1, 1, 1)
guider.sample(
noise.generate_noise({"samples": latents}),
latents,
train_sampler,
sigmas,
- seed=noise.seed
+ seed=noise.seed,
)
finally:
for m in mp.model.modules():
@@ -702,111 +642,118 @@ class TrainLoraNode:
for param in lora_sd:
lora_sd[param] = lora_sd[param].to(lora_dtype)
- return (mp, lora_sd, loss_map, steps + existing_steps)
+ return io.NodeOutput(mp, lora_sd, loss_map, steps + existing_steps)
-class LoraModelLoader:
- def __init__(self):
- self.loaded_lora = None
+class LoraModelLoader(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="LoraModelLoader",
+ display_name="Load LoRA Model",
+ category="loaders",
+ is_experimental=True,
+ inputs=[
+ io.Model.Input(
+ "model", tooltip="The diffusion model the LoRA will be applied to."
+ ),
+ io.Custom("LORA_MODEL").Input(
+ "lora", tooltip="The LoRA model to apply to the diffusion model."
+ ),
+ io.Float.Input(
+ "strength_model",
+ default=1.0,
+ min=-100.0,
+ max=100.0,
+ tooltip="How strongly to modify the diffusion model. This value can be negative.",
+ ),
+ ],
+ outputs=[
+ io.Model.Output(
+ display_name="model", tooltip="The modified diffusion model."
+ ),
+ ],
+ )
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "model": ("MODEL", {"tooltip": "The diffusion model the LoRA will be applied to."}),
- "lora": (IO.LORA_MODEL, {"tooltip": "The LoRA model to apply to the diffusion model."}),
- "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01, "tooltip": "How strongly to modify the diffusion model. This value can be negative."}),
- }
- }
-
- RETURN_TYPES = ("MODEL",)
- OUTPUT_TOOLTIPS = ("The modified diffusion model.",)
- FUNCTION = "load_lora_model"
-
- CATEGORY = "loaders"
- DESCRIPTION = "Load Trained LoRA weights from Train LoRA node."
- EXPERIMENTAL = True
-
- def load_lora_model(self, model, lora, strength_model):
+ def execute(cls, model, lora, strength_model):
if strength_model == 0:
- return (model, )
-
- model_lora, _ = comfy.sd.load_lora_for_models(model, None, lora, strength_model, 0)
- return (model_lora, )
+ return io.NodeOutput(model)
+ model_lora, _ = comfy.sd.load_lora_for_models(
+ model, None, lora, strength_model, 0
+ )
+ return io.NodeOutput(model_lora)
-class SaveLoRA:
- def __init__(self):
- self.output_dir = folder_paths.get_output_directory()
+class SaveLoRA(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "lora": (
- IO.LORA_MODEL,
- {
- "tooltip": "The LoRA model to save. Do not use the model with LoRA layers."
- },
+ def define_schema(cls):
+ return io.Schema(
+ node_id="SaveLoRA",
+ display_name="Save LoRA Weights",
+ category="loaders",
+ is_experimental=True,
+ is_output_node=True,
+ inputs=[
+ io.Custom("LORA_MODEL").Input(
+ "lora",
+ tooltip="The LoRA model to save. Do not use the model with LoRA layers.",
),
- "prefix": (
- "STRING",
- {
- "default": "loras/ComfyUI_trained_lora",
- "tooltip": "The prefix to use for the saved LoRA file.",
- },
+ io.String.Input(
+ "prefix",
+ default="loras/ComfyUI_trained_lora",
+ tooltip="The prefix to use for the saved LoRA file.",
),
- },
- "optional": {
- "steps": (
- IO.INT,
- {
- "forceInput": True,
- "tooltip": "Optional: The number of steps to LoRA has been trained for, used to name the saved file.",
- },
+ io.Int.Input(
+ "steps",
+ optional=True,
+ tooltip="Optional: The number of steps to LoRA has been trained for, used to name the saved file.",
),
- },
- }
-
- RETURN_TYPES = ()
- FUNCTION = "save"
- CATEGORY = "loaders"
- EXPERIMENTAL = True
- OUTPUT_NODE = True
+ ],
+ outputs=[],
+ )
- def save(self, lora, prefix, steps=None):
- full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(prefix, self.output_dir)
+ @classmethod
+ def execute(cls, lora, prefix, steps=None):
+ output_dir = folder_paths.get_output_directory()
+ full_output_folder, filename, counter, subfolder, filename_prefix = (
+ folder_paths.get_save_image_path(prefix, output_dir)
+ )
if steps is None:
output_checkpoint = f"{filename}_{counter:05}_.safetensors"
else:
output_checkpoint = f"{filename}_{steps}_steps_{counter:05}_.safetensors"
output_checkpoint = os.path.join(full_output_folder, output_checkpoint)
safetensors.torch.save_file(lora, output_checkpoint)
- return {}
+ return io.NodeOutput()
-class LossGraphNode:
- def __init__(self):
- self.output_dir = folder_paths.get_temp_directory()
+class LossGraphNode(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="LossGraphNode",
+ display_name="Plot Loss Graph",
+ category="training",
+ is_experimental=True,
+ is_output_node=True,
+ inputs=[
+ io.Custom("LOSS_MAP").Input(
+ "loss", tooltip="Loss map from training node."
+ ),
+ io.String.Input(
+ "filename_prefix",
+ default="loss_graph",
+ tooltip="Prefix for the saved loss graph image.",
+ ),
+ ],
+ outputs=[],
+ hidden=[io.Hidden.prompt, io.Hidden.extra_pnginfo],
+ )
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "loss": (IO.LOSS_MAP, {"default": {}}),
- "filename_prefix": (IO.STRING, {"default": "loss_graph"}),
- },
- "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
- }
-
- RETURN_TYPES = ()
- FUNCTION = "plot_loss"
- OUTPUT_NODE = True
- CATEGORY = "training"
- EXPERIMENTAL = True
- DESCRIPTION = "Plots the loss graph and saves it to the output directory."
-
- def plot_loss(self, loss, filename_prefix, prompt=None, extra_pnginfo=None):
+ def execute(cls, loss, filename_prefix, prompt=None, extra_pnginfo=None):
loss_values = loss["loss"]
width, height = 800, 480
margin = 40
@@ -849,47 +796,27 @@ class LossGraphNode:
(margin - 30, height - 10), f"{min_loss:.2f}", font=font, fill="black"
)
- metadata = None
- if not args.disable_metadata:
- metadata = PngInfo()
- if prompt is not None:
- metadata.add_text("prompt", json.dumps(prompt))
- if extra_pnginfo is not None:
- for x in extra_pnginfo:
- metadata.add_text(x, json.dumps(extra_pnginfo[x]))
-
- date = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
- img.save(
- os.path.join(self.output_dir, f"{filename_prefix}_{date}.png"),
- pnginfo=metadata,
- )
- return {
- "ui": {
- "images": [
- {
- "filename": f"{filename_prefix}_{date}.png",
- "subfolder": "",
- "type": "temp",
- }
- ]
- }
- }
-
-
-NODE_CLASS_MAPPINGS = {
- "TrainLoraNode": TrainLoraNode,
- "SaveLoRANode": SaveLoRA,
- "LoraModelLoader": LoraModelLoader,
- "LoadImageSetFromFolderNode": LoadImageSetFromFolderNode,
- "LoadImageTextSetFromFolderNode": LoadImageTextSetFromFolderNode,
- "LossGraphNode": LossGraphNode,
-}
-
-NODE_DISPLAY_NAME_MAPPINGS = {
- "TrainLoraNode": "Train LoRA",
- "SaveLoRANode": "Save LoRA Weights",
- "LoraModelLoader": "Load LoRA Model",
- "LoadImageSetFromFolderNode": "Load Image Dataset from Folder",
- "LoadImageTextSetFromFolderNode": "Load Image and Text Dataset from Folder",
- "LossGraphNode": "Plot Loss Graph",
-}
+ # Convert PIL image to tensor for PreviewImage
+ img_array = np.array(img).astype(np.float32) / 255.0
+ img_tensor = torch.from_numpy(img_array)[None,] # [1, H, W, 3]
+
+ # Return preview UI
+ return io.NodeOutput(ui=ui.PreviewImage(img_tensor, cls=cls))
+
+
+# ========== Extension Setup ==========
+
+
+class TrainingExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ TrainLoraNode,
+ LoraModelLoader,
+ SaveLoRA,
+ LossGraphNode,
+ ]
+
+
+async def comfy_entrypoint() -> TrainingExtension:
+ return TrainingExtension()
diff --git a/nodes.py b/nodes.py
index f4835c02e..bf73eb90e 100644
--- a/nodes.py
+++ b/nodes.py
@@ -2278,6 +2278,7 @@ async def init_builtin_extra_nodes():
"nodes_images.py",
"nodes_video_model.py",
"nodes_train.py",
+ "nodes_dataset.py",
"nodes_sag.py",
"nodes_perpneg.py",
"nodes_stable3d.py",