Skip to content
Python
9.9 KB
Raw Blame
#!/usr/bin/env python3
"""Check API workflows against a running ComfyUI without a GPU or real model files.

ComfyUI validates every prompt when it is queued: node types, required inputs, value
ranges, links, and that each model file name is in its folder list. With --placeholders,
this script creates empty files with the expected names (models and example inputs) so
that check can run on any machine, even a CPU-only container. A prompt that passes is
cancelled right away so nothing heavy runs.

    python tools/comfy_validate.py --url http://127.0.0.1:8188 --comfy-dir /path/to/ComfyUI --placeholders
    python tools/comfy_validate.py --url ... apps/z-image/workflow.api.json other.api.json

With no workflow paths, it checks every apps/*/app.json that has a workflow, applying the
app's test values, with no preset and with each preset. With --lab, it checks the prompts a lab run would submit
(values, overrides and inserted LoRAs included), with tiny placeholders for chained inputs:

    python tools/comfy_validate.py --url ... --comfy-dir ... --placeholders --lab lab/cases.json
"""
from __future__ import annotations

import argparse
import json
import struct
import sys
import time
import urllib.error
import urllib.request
import zlib
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
MODEL_EXT = (".safetensors", ".gguf", ".pt", ".pth", ".bin", ".ckpt", ".sft")
MODEL_DIRS = ["diffusion_models", "text_encoders", "vae", "loras", "checkpoints", "clip_vision",
              "latent_upscale_models", "upscale_models", "model_patches", "geometry_estimation",
              "background_removal", "frame_interpolation", "audio_encoders", "embeddings", "detection", "unet", "clip"]
IMAGE_EXT = (".png", ".jpg", ".jpeg", ".webp")
VIDEO_EXT = (".mp4", ".webm", ".mov", ".mkv")
AUDIO_EXT = (".wav", ".mp3", ".flac", ".ogg", ".m4a")


def _post(url: str, payload: dict) -> tuple[int, dict]:
    req = urllib.request.Request(url, data=json.dumps(payload).encode(), headers={"content-type": "application/json"})
    try:
        with urllib.request.urlopen(req, timeout=60) as r:
            body = r.read()
            return r.status, (json.loads(body) if body else {})
    except urllib.error.HTTPError as e:
        body = e.read()
        try:
            return e.code, json.loads(body)
        except Exception:
            return e.code, {"raw": body.decode(errors="replace")[:500]}


def _tiny_png(path: Path) -> None:
    def chunk(t: bytes, d: bytes) -> bytes:
        return struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF)
    w = h = 64
    raw = b"".join(b"\x00" + b"\x80\x80\x80" * w for _ in range(h))
    data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) \
        + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
    path.write_bytes(data)


def _tiny_wav(path: Path) -> None:
    import wave
    with wave.open(str(path), "wb") as w:
        w.setnchannels(1)
        w.setsampwidth(2)
        w.setframerate(16000)
        w.writeframes(b"\x00\x00" * 16000)


def _tiny_mp4(path: Path) -> bool:
    try:
        import av  # ships with ComfyUI's requirements
        import numpy as np
    except ImportError:
        return False
    with av.open(str(path), "w") as c:
        s = c.add_stream("mpeg4", rate=24)
        s.width, s.height, s.pix_fmt = 64, 64, "yuv420p"
        for i in range(24):
            frame = av.VideoFrame.from_ndarray(np.full((64, 64, 3), i * 8, dtype=np.uint8), format="rgb24")
            for p in s.encode(frame):
                c.mux(p)
        for p in s.encode():
            c.mux(p)
    return True


def strings_in(api: dict) -> set[str]:
    out = set()
    for node in api.values():
        for v in node.get("inputs", {}).values():
            if isinstance(v, str):
                out.add(v)
    return out


def make_placeholders(api: dict, comfy_dir: Path) -> None:
    for s in strings_in(api):
        name = Path(s).name
        low = name.lower()
        if low.endswith(MODEL_EXT):
            for d in MODEL_DIRS:
                p = comfy_dir / "models" / d / s
                p.parent.mkdir(parents=True, exist_ok=True)
                if not p.exists():
                    p.write_bytes(b"")
        elif low.endswith(IMAGE_EXT + VIDEO_EXT + AUDIO_EXT) and "/" not in s:
            p = comfy_dir / "input" / s
            p.parent.mkdir(parents=True, exist_ok=True)
            if p.exists():
                continue
            if low.endswith(IMAGE_EXT):
                _tiny_png(p)
            elif low.endswith(AUDIO_EXT):
                _tiny_wav(p)
            elif not _tiny_mp4(p):
                p.write_bytes(b"")


def validate(url: str, api: dict, label: str) -> bool:
    status, body = _post(f"{url}/prompt", {"prompt": api, "client_id": "freefield-validate"})
    if status == 200 and "prompt_id" in body:
        _post(f"{url}/queue", {"delete": [body["prompt_id"]]})
        _post(f"{url}/interrupt", {})
        print(f"✓ {label}")
        return True
    print(f"✗ {label}")
    err = body.get("error") or {}
    if err:
        print(f"    {err.get('type', '')}: {err.get('message', '')} {err.get('details', '')}".rstrip())
    for nid, ne in (body.get("node_errors") or {}).items():
        for e in ne.get("errors", []):
            print(f"    node {nid} ({ne.get('class_type')}): {e.get('message')} — {e.get('details', '')}")
    if not err and not body.get("node_errors"):
        print(f"    HTTP {status}: {json.dumps(body)[:300]}")
    return False


def app_variants(app_dir: Path) -> list[tuple[str, dict]]:
    """The app's workflow with its test values applied, once per preset (or once with none)."""
    manifest = json.loads((app_dir / "app.json").read_text(encoding="utf-8"))
    if not manifest.get("workflow") or not (app_dir / manifest["workflow"]).exists():
        return []
    base = json.loads((app_dir / manifest["workflow"]).read_text(encoding="utf-8"))
    test = manifest.get("test") or {}
    values = {i["id"]: test.get(i["id"], i.get("default")) for i in manifest.get("inputs", [])}
    for i in manifest.get("inputs", []):   # the runner turns -1 into a random seed; any fixed one will do here
        if i["type"] == "seed" and values.get(i["id"]) in (None, -1, "-1"):
            values[i["id"]] = 42
    for i in manifest.get("inputs", []):   # a multiple select reaches the workflow joined, as server/runner.ts writes it
        if i.get("multiple") and isinstance(values.get(i["id"]), list):
            m, items = i["multiple"], values[i["id"]]
            values[i["id"]] = f'{m.get("prefix", "")}{m.get("join", ", ").join(map(str, items))}{m.get("suffix", "")}' if items else ""

    def render(t: str) -> str:
        import re
        return re.sub(r"\{([a-z][a-z0-9_]*)\}", lambda m: "" if values.get(m.group(1)) is None else str(values[m.group(1)]), t)

    def apply(api: dict, preset: dict | None) -> dict:
        api = json.loads(json.dumps(api))
        for inp in manifest.get("inputs", []):
            v = values.get(inp["id"])
            if v is None:
                continue
            for b in inp.get("bind", []):
                if inp["type"] in ("image", "video", "audio"):
                    api[b["node"]]["inputs"][b["field"]] = Path(str(v)).name
                else:
                    api[b["node"]]["inputs"][b["field"]] = render(b["template"]) if b.get("template") else v
            for opt in inp.get("options") or []:
                if opt["value"] == v:
                    for s in opt.get("sets", []):
                        api[s["node"]]["inputs"][s["field"]] = render(s["value"]) if isinstance(s["value"], str) else s["value"]
        for s in (preset or {}).get("sets", []):
            api[s["node"]]["inputs"][s["field"]] = render(s["value"]) if isinstance(s["value"], str) else s["value"]
        return api

    # no preset is a valid choice in the UI too, so check it alongside every preset
    return [(manifest["id"], apply(base, None))] + \
        [(f"{manifest['id']} [{p['id']}]", apply(base, p)) for p in manifest.get("presets") or []]


def lab_variants(cases_path: Path) -> list[tuple[str, dict]]:
    """Every lab case as the prompt the lab would queue; file inputs become placeholders."""
    sys.path.insert(0, str(ROOT / "lab"))
    import core  # lab/core.py
    ext = {"image": "png", "video": "mp4", "audio": "wav"}
    cases = json.loads(cases_path.read_text(encoding="utf-8"))["cases"]
    out = []
    for c in cases:
        _, api, _ = core.build_case(ROOT / "apps", c, lambda _cid, _raw, kind: f"freefield_check_{kind}.{ext[kind]}")
        out.append((f"lab:{c['id']} ({c['app']})", api))
    return out


def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("workflows", nargs="*", type=Path)
    ap.add_argument("--url", default="http://127.0.0.1:8188")
    ap.add_argument("--comfy-dir", type=Path, help="ComfyUI folder, needed for --placeholders")
    ap.add_argument("--placeholders", action="store_true", help="create empty model files and tiny example inputs")
    ap.add_argument("--lab", type=Path, help="check the prompts of a lab cases file instead of the apps")
    a = ap.parse_args(argv)

    jobs: list[tuple[str, dict]] = []
    if a.lab:
        jobs = lab_variants(a.lab)
    elif a.workflows:
        for p in a.workflows:
            jobs.append((str(p), json.loads(p.read_text(encoding="utf-8"))))
    else:
        for app_dir in sorted((ROOT / "apps").iterdir()):
            if (app_dir / "app.json").exists():
                jobs += app_variants(app_dir)
    ok = 0
    for label, api in jobs:
        if a.placeholders:
            if not a.comfy_dir:
                print("--placeholders needs --comfy-dir", file=sys.stderr)
                return 2
            make_placeholders(api, a.comfy_dir)
        ok += validate(a.url.rstrip("/"), api, label)
        time.sleep(0.2)
    print(f"{ok}/{len(jobs)} workflows valid")
    return 0 if ok == len(jobs) else 1


if __name__ == "__main__":
    sys.exit(main())