The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat: Bounding boxes canvas and Ideogram JSON prompt (#14537)
Terry Jiadac4ea3a80ff
diff --git a/comfy_api/latest/_io.py b/comfy_api/latest/_io.pyindex 012fae3ac..58e49d8e2 100644--- a/comfy_api/latest/_io.py+++ b/comfy_api/latest/_io.py@@ -891,6 +891,14 @@ class Tracks(ComfyTypeIO):track_visibility: torch.TensorType = TrackDict+@comfytype(io_type="DICT")+class Dict(ComfyTypeIO):+ Type = dict++@comfytype(io_type="ARRAY")+class Array(ComfyTypeIO):+ Type = list+@comfytype(io_type="COMFY_MULTITYPED_V3")class MultiType:Type = Any@@ -1279,6 +1287,19 @@ class Color(ComfyTypeIO):def as_dict(self):return super().as_dict()++@comfytype(io_type="COLORS")+class Colors(ComfyTypeIO):+ Type = list[Color.Type]++ class Input(WidgetInput):+ def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None,+ socketless: bool=True, default: list[str]=None, advanced: bool=None):+ super().__init__(id, display_name, optional, tooltip, None, default, socketless, None, None, None, None, advanced)+ if default is None:+ self.default = []++@comfytype(io_type="BOUNDING_BOX")class BoundingBox(ComfyTypeIO):class BoundingBoxDict(TypedDict):@@ -1326,6 +1347,20 @@ class Curve(ComfyTypeIO):return d+@comfytype(io_type="BOUNDING_BOXES")+class BoundingBoxes(ComfyTypeIO):+ class BoundingBoxWithMetadata(BoundingBox.BoundingBoxDict):+ metadata: dict+ Type = list[BoundingBoxWithMetadata]++ class Input(WidgetInput):+ def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None,+ socketless: bool=True, default: list[dict]=None, advanced: bool=None):+ super().__init__(id, display_name, optional, tooltip, None, default, socketless, None, None, None, None, advanced)+ if default is None:+ self.default = []++@comfytype(io_type="HISTOGRAM")class Histogram(ComfyTypeIO):"""A histogram represented as a list of bin counts."""@@ -2376,6 +2411,8 @@ __all__ = ["AnyType","MultiType","Tracks",+ "Dict",+ "Array","Color",# Dynamic Types"MatchType",@@ -2394,6 +2431,8 @@ __all__ = ["PriceBadgeDepends","PriceBadge","BoundingBox",+ "BoundingBoxes",+ "Colors","Curve","Histogram","Range",diff --git a/comfy_extras/color_util.py b/comfy_extras/color_util.pynew file mode 100644index 000000000..d50795ae3--- /dev/null+++ b/comfy_extras/color_util.py@@ -0,0 +1,23 @@+def hex_to_rgb(value: str) -> tuple[int, int, int]:+ h = value.lstrip("#")+ if len(h) != 6:+ return (255, 255, 255)+ try:+ return (int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16))+ except ValueError:+ return (255, 255, 255)+++def readable_color(rgb: tuple[int, int, int]) -> tuple[int, int, int]:+ r, g, b = rgb+ lum = 0.299 * r + 0.587 * g + 0.114 * b+ if lum >= 130:+ return (r, g, b)+ t = (130 - lum) / (255 - lum)+ return (round(r + (255 - r) * t), round(g + (255 - g) * t), round(b + (255 - b) * t))+++def normalize_palette(colors) -> list[str]:+ if isinstance(colors, dict):+ colors = colors.values()+ return [c.upper() for c in colors if isinstance(c, str) and c]diff --git a/comfy_extras/nodes_bounding_boxes.py b/comfy_extras/nodes_bounding_boxes.pynew file mode 100644index 000000000..77cbf8649--- /dev/null+++ b/comfy_extras/nodes_bounding_boxes.py@@ -0,0 +1,253 @@+import numpy as np+import torch+from PIL import Image, ImageDraw, ImageEnhance, ImageFont+from typing_extensions import override++from comfy_api.latest import ComfyExtension, io+from comfy_extras.color_util import hex_to_rgb, normalize_palette, readable_color++_PREVIEW_LONG_EDGE = 1024+_PREVIEW_DIM = 0.25+++def pixels_to_fractions(box: dict, width: int, height: int) -> dict:+ w = width or 1+ h = height or 1+ return {+ "x": box.get("x", 0) / w,+ "y": box.get("y", 0) / h,+ "w": box.get("width", 0) / w,+ "h": box.get("height", 0) / h,+ }+++def fractions_to_pixels(box: dict, width: int, height: int) -> dict:+ x, y = box.get("x", 0.0), box.get("y", 0.0)+ w, h = box.get("w", 0.0), box.get("h", 0.0)+ if w < 0:+ x, w = x + w, -w+ if h < 0:+ y, h = y + h, -h+ return {+ "x": round(x * width),+ "y": round(y * height),+ "width": round(w * width),+ "height": round(h * height),+ }+++def fractions_to_bbox_frame(boxes: list, width: int, height: int) -> list:+ pixels = [+ fractions_to_pixels(box, width, height)+ for box in boxes+ if isinstance(box, dict)+ ]+ return [pixels] if pixels else []+++def _font(size: int):+ try:+ return ImageFont.load_default(size)+ except Exception:+ return ImageFont.load_default()+++def _wrap(draw, text: str, font, max_w: float) -> list[str]:+ lines = []+ for para in text.split("\n"):+ line = ""+ for word in para.split():+ test = word if not line else line + " " + word+ if line and draw.textlength(test, font=font) > max_w:+ lines.append(line)+ line = word+ else:+ line = test+ lines.append(line)+ return lines+++def _bg_from_image(image) -> Image.Image | None:+ if image is None:+ return None+ try:+ arr = (image[0].detach().cpu().numpy() * 255).clip(0, 255).astype(np.uint8)+ return Image.fromarray(arr)+ except Exception:+ return None+++def render_preview(regions, width, height, bg=None):+ if bg is not None:+ iw, ih = bg.size+ long_edge = max(iw, ih) or 1+ scale = min(1.0, _PREVIEW_LONG_EDGE / long_edge)+ rw, rh = max(1, round(iw * scale)), max(1, round(ih * scale))+ base = bg.convert("RGB").resize((rw, rh), Image.LANCZOS)+ base = ImageEnhance.Brightness(base).enhance(_PREVIEW_DIM)+ img = base.convert("RGBA")+ else:+ long_edge = max(width, height) or 1+ scale = min(1.0, _PREVIEW_LONG_EDGE / long_edge)+ rw, rh = max(1, round(width * scale)), max(1, round(height * scale))+ grey = round(_PREVIEW_DIM * 128)+ img = Image.new("RGBA", (rw, rh), (grey, grey, grey, 255))++ overlay = Image.new("RGBA", (rw, rh), (0, 0, 0, 0))+ draw = ImageDraw.Draw(overlay)+ fs = max(10, round(rh / 64))+ font = _font(fs)+ tag_font = _font(max(9, fs - 2))+ line_h = fs + 2++ for i, region in enumerate(regions):+ if not isinstance(region, dict):+ continue+ palette = [c for c in (region.get("palette") or []) if c]+ r, g, b = hex_to_rgb(palette[0]) if palette else (140, 140, 140)+ x1 = max(0, min(rw, round(region.get("x", 0) * rw)))+ y1 = max(0, min(rh, round(region.get("y", 0) * rh)))+ x2 = max(0, min(rw, round((region.get("x", 0) + region.get("w", 0)) * rw)))+ y2 = max(0, min(rh, round((region.get("y", 0) + region.get("h", 0)) * rh)))+ if x2 < x1:+ x1, x2 = x2, x1+ if y2 < y1:+ y1, y2 = y2, y1++ draw.rectangle([x1, y1, x2, y2], outline=(r, g, b, 255), width=2)++ swatches = palette[:5]+ if swatches and (x2 - x1) > 2:+ sh = max(5, fs // 2)+ seg = (x2 - x1) / len(swatches)+ for p, hexc in enumerate(swatches):+ sx = x1 + round(p * seg)+ draw.rectangle([sx, y1, x1 + round((p + 1) * seg), y1 + sh], fill=hex_to_rgb(hexc))++ etype = "text" if region.get("type") == "text" else "obj"+ tag = str(i + 1).zfill(2)+ tw = draw.textlength(tag, font=tag_font)+ draw.rectangle([x1, y1, x1 + tw + 6, y1 + fs + 2], fill=(r, g, b, 255))+ tag_fill = (0, 0, 0, 255) if (0.299 * r + 0.587 * g + 0.114 * b) > 140 else (255, 255, 255, 255)+ draw.text((x1 + 3, y1 + 1), tag, fill=tag_fill, font=tag_font)++ body = region.get("desc", "") or ""+ if etype == "text" and region.get("text"):+ body = '"%s"%s' % (region["text"], " — " + body if body else "")+ if body and (x2 - x1) > 8:+ ty = y1 + fs + 5+ for line in _wrap(draw, body, font, x2 - x1 - 8):+ if ty > y2:+ break+ draw.text((x1 + 4, ty), line, fill=readable_color((r, g, b)) + (255,), font=font)+ ty += line_h++ composed = Image.alpha_composite(img, overlay).convert("RGB")+ arr = np.asarray(composed, dtype=np.float32) / 255.0+ return torch.from_numpy(arr).unsqueeze(0)+++def boxes_to_regions(boxes, width: int, height: int) -> list:+ regions: list = []+ if not isinstance(boxes, list):+ return regions+ for box in boxes:+ if not isinstance(box, dict):+ continue+ meta = box.get("metadata")+ meta = meta if isinstance(meta, dict) else {}+ regions.append({+ **pixels_to_fractions(box, width, height),+ "type": meta.get("type", "obj"),+ "text": meta.get("text", ""),+ "desc": meta.get("desc", ""),+ "palette": meta.get("palette", []),+ })+ return regions+++def _norm_bbox(region: dict) -> list[int]:+ def grid(value: float) -> int:+ return max(0, min(1000, round(value * 1000)))++ x, y = region.get("x", 0.0), region.get("y", 0.0)+ w, h = region.get("w", 0.0), region.get("h", 0.0)+ ymin, xmin, ymax, xmax = grid(y), grid(x), grid(y + h), grid(x + w)+ if ymin > ymax:+ ymin, ymax = ymax, ymin+ if xmin > xmax:+ xmin, xmax = xmax, xmin+ return [ymin, xmin, ymax, xmax]+++def build_elements(regions: list) -> list:+ elements = []+ for region in regions:+ if not isinstance(region, dict):+ continue+ etype = "text" if region.get("type") == "text" else "obj"+ element = {"type": etype}+ element["bbox"] = _norm_bbox(region)+ if etype == "text":+ element["text"] = region.get("text", "")+ element["desc"] = region.get("desc", "")+ palette = normalize_palette(region.get("palette", []))+ if palette:+ element["color_palette"] = palette[:5]+ elements.append(element)+ return elements+++class CreateBoundingBoxes(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ editor_state = io.BoundingBoxes.Input(+ "editor_state",+ socketless=False,+ tooltip="Draw bounding boxes and set each box type, text, description, color palette. Start with background element first and foreground last.",+ )+ return io.Schema(+ node_id="CreateBoundingBoxes",+ display_name="Create Bounding Boxes",+ category="utilities",+ description="Draw bounding boxes in a canvas. Outputs Ideogram prompt elements, pixel-space bounding boxes, and a preview image.",+ inputs=[+ io.Image.Input(+ "background",+ optional=True,+ tooltip="Optional image used as background in the canvas and preview.",+ ),+ io.Int.Input("width", default=1024, min=64, max=16384, step=16,+ tooltip="Width of the canvas and the pixel grid for the bounding boxes."),+ io.Int.Input("height", default=1024, min=64, max=16384, step=16,+ tooltip="Height of the canvas and the pixel grid for the bounding boxes."),+ editor_state,+ ],+ outputs=[+ io.Image.Output(display_name="preview"),+ io.BoundingBox.Output(display_name="bboxes"),+ io.Array.Output(display_name="elements"),+ ],+ is_experimental=True,+ )++ @classmethod+ def execute(cls, width, height, editor_state=None, background=None) -> io.NodeOutput:+ regions = boxes_to_regions(editor_state, width, height)+ preview = render_preview(regions, width, height, _bg_from_image(background))+ return io.NodeOutput(+ preview,+ fractions_to_bbox_frame(regions, width, height),+ build_elements(regions),+ ui={"dims": [width, height]},+ )+++class BoundingBoxesExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[io.ComfyNode]]:+ return [CreateBoundingBoxes]+++async def comfy_entrypoint() -> BoundingBoxesExtension:+ return BoundingBoxesExtension()diff --git a/comfy_extras/nodes_color.py b/comfy_extras/nodes_color.pyindex 688254e4e..f58e51bff 100644--- a/comfy_extras/nodes_color.py+++ b/comfy_extras/nodes_color.py@@ -1,5 +1,6 @@from typing_extensions import overridefrom comfy_api.latest import ComfyExtension, io+from comfy_extras.color_util import hex_to_rgbclass ColorToRGBInt(io.ComfyNode):@@ -24,9 +25,11 @@ class ColorToRGBInt(io.ComfyNode):# expect format #RRGGBBif len(color) != 7 or color[0] != "#":raise ValueError("Color must be in format #RRGGBB")- r = int(color[1:3], 16)- g = int(color[3:5], 16)- b = int(color[5:7], 16)+ try:+ int(color[1:], 16)+ except ValueError:+ raise ValueError("Color must be in format #RRGGBB") from None+ r, g, b = hex_to_rgb(color)rgb_int = r * 256 * 256 + g * 256 + breturn io.NodeOutput(rgb_int, color)diff --git a/comfy_extras/nodes_json_prompt.py b/comfy_extras/nodes_json_prompt.pynew file mode 100644index 000000000..206f5aa71--- /dev/null+++ b/comfy_extras/nodes_json_prompt.py@@ -0,0 +1,77 @@+from typing_extensions import override++from comfy_api.latest import ComfyExtension, io+from comfy_extras.color_util import normalize_palette+++class BuildJsonPromptIdeogram(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ color_palette = io.Colors.Input(+ "color_palette",+ socketless=False,+ tooltip="Hex color codes that steer the image's dominant colors. Up to 16 entries.",+ )+ return io.Schema(+ node_id="BuildJsonPromptIdeogram",+ display_name="Build JSON Prompt (Ideogram)",+ category="text",+ description="Build a JSON prompt for the Ideogram 4 model.",+ inputs=[+ io.Array.Input("element", tooltip="Prompt elements from the node Create Bounding Boxes."),+ io.String.Input("high_level_description", multiline=True, default="",+ tooltip="Optional description of the image in one or two sentences. Strongly recommended."),+ io.String.Input("background", multiline=True, default="",+ tooltip="Mandatory description of the image background or environment."),+ io.DynamicCombo.Input("style", options=[+ io.DynamicCombo.Option("none", []),+ io.DynamicCombo.Option("photo", [io.String.Input("photo", default="", tooltip="Camera or lens details for photographic outputs (e.g. 35mm, f/1.4, bokeh).")]),+ io.DynamicCombo.Option("art_style", [io.String.Input("art_style", default="", tooltip="Art style description (e.g. flat vector illustration, bold outlines).")]),+ ]),+ io.String.Input("aesthetics", default="", tooltip="Mandatory aesthetic keywords (e.g. moody, cinematic, desaturated)."),+ io.String.Input("lighting", default="", tooltip="Mandatory lighting description (e.g. golden hour, rim light, dramatic shadows)."),+ io.String.Input("medium", default="", tooltip="Mandatory medium type (e.g. photograph, illustration, 3d_render, painting, graphic_design). When style = photo, set to photograph."),+ color_palette,+ ],+ outputs=[io.Dict.Output(display_name="prompt")],+ is_experimental=True,+ )++ @classmethod+ def execute(cls, element, style, high_level_description="", background="",+ aesthetics="", lighting="", medium="", color_palette=None) -> io.NodeOutput:+ elements = element if isinstance(element, list) else []+ kind = style.get("style", "none") if isinstance(style, dict) else "none"+ photo = style.get("photo", "") if isinstance(style, dict) else ""+ art_style = style.get("art_style", "") if isinstance(style, dict) else ""+ palette = normalize_palette(color_palette or [])++ caption: dict = {}+ if high_level_description.strip():+ caption["high_level_description"] = high_level_description+ if kind != "none":+ style_desc: dict = {"aesthetics": aesthetics, "lighting": lighting}+ if kind == "photo":+ style_desc["photo"] = photo+ style_desc["medium"] = medium+ else:+ style_desc["medium"] = medium+ style_desc["art_style"] = art_style+ if palette:+ style_desc["color_palette"] = palette+ caption["style_description"] = style_desc+ caption["compositional_deconstruction"] = {+ "background": background,+ "elements": elements,+ }+ return io.NodeOutput(caption)+++class JsonPromptExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[io.ComfyNode]]:+ return [BuildJsonPromptIdeogram]+++async def comfy_entrypoint() -> JsonPromptExtension:+ return JsonPromptExtension()diff --git a/comfy_extras/nodes_string.py b/comfy_extras/nodes_string.pyindex 97485c8c5..21929ae63 100644--- a/comfy_extras/nodes_string.py+++ b/comfy_extras/nodes_string.py@@ -440,6 +440,57 @@ class JsonExtractString(io.ComfyNode):except (json.JSONDecodeError, TypeError):return io.NodeOutput("")++def _dump_json(value, indent):+ return json.dumps(value, ensure_ascii=False, indent=indent or None)+++class ConvertDictionaryToString(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="ConvertDictionaryToString",+ display_name="Convert Dictionary to String",+ category="text",+ search_aliases=["json", "dict to json", "stringify", "serialize", "dict to string"],+ inputs=[+ io.Dict.Input("dictionary"),+ io.Int.Input("indent", default=2, min=0, max=8,+ tooltip="Spaces per indent level. 0 produces compact single-line string."),+ ],+ outputs=[+ io.String.Output(),+ ],+ )++ @classmethod+ def execute(cls, dictionary, indent=2):+ return io.NodeOutput(_dump_json(dictionary, indent))+++class ConvertArrayToString(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="ConvertArrayToString",+ display_name="Convert Array to String",+ category="text",+ search_aliases=["json", "list to json", "stringify", "serialize", "list to string", "array to json"],+ inputs=[+ io.Array.Input("array"),+ io.Int.Input("indent", default=2, min=0, max=8,+ tooltip="Spaces per indent level. 0 produces compact single-line string."),+ ],+ outputs=[+ io.String.Output(),+ ],+ )++ @classmethod+ def execute(cls, array, indent=2):+ return io.NodeOutput(_dump_json(array, indent))++class StringExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[io.ComfyNode]]:@@ -457,6 +508,8 @@ class StringExtension(ComfyExtension):RegexExtract,RegexReplace,JsonExtractString,+ ConvertDictionaryToString,+ ConvertArrayToString,]async def comfy_entrypoint() -> StringExtension:diff --git a/nodes.py b/nodes.pyindex ad172890d..028e58c77 100644--- a/nodes.py+++ b/nodes.py@@ -2374,6 +2374,8 @@ async def init_builtin_extra_nodes():"nodes_images.py","nodes_video_model.py","nodes_ideogram4.py",+ "nodes_bounding_boxes.py",+ "nodes_json_prompt.py","nodes_train.py","nodes_dataset.py","nodes_sag.py",