One-click character swap for videos: an open-source, local Higgsfield Genjutsu alternative built on ComfyUI and MiniMax H3, with FLUX.3 Klein relighting.
Make Montage mask with saved tracks in one continuous chain
D3voz985222191418
diff --git a/README.md b/README.mdindex d96aaef..8f9703a 100644--- a/README.md+++ b/README.md@@ -77,7 +77,7 @@ On Windows, double-click `start_openjutsu.bat` (ComfyUI must be running, or starA 6 s clip at 480p takes about 6–13 minutes on a 4060 Ti, most of it in H3 sampling. 736p is the default; 1080p is about 2.5× slower than 736p and needs much more memory. Custom sets an exact width × height from 256 to 2048 px a side (rounded to 32); the video is scaled and centre-cropped to that shape first. It needs the `short_edge` input of the H3 Swap Plan nodes in comfyui-h3-oneclick-swap.-Advanced motion transfer has a **Montage** tab for edited clips where different people take turns within one shot without a hard cut (whip pans, selfie handovers, travel montages). The shot list and H3 are told when each person is on screen, so nobody turns into someone else. It needs the `timing` input of the pack's Clip Shots and Fill Shots nodes. Write roles as "who - looks" ("the dancer - white polo, jeans"): only the part before " - " reaches H3, and clothes named there are drawn on the new person.+Advanced motion transfer has a **Montage** tab for edited clips where different people take turns within one shot without a hard cut (whip pans, selfie handovers, travel montages). Write roles as "who - looks" ("the dancer - white polo, jeans"): the looks help match the people, and only the part before " - " reaches H3. **Find people** (about 3 minutes for 18 s of video) tracks everyone with SAM over the whole video and matches each appearance to the roles with Qwen (a person SAM loses and finds again is matched again, so they keep their photo), then lists the appearances with a photo picker each. Find people saves the tracks, and the render masks with exactly those; nothing is tracked again. The video is made as one chain like Standard, with overlapping clips: each clip has one H3 pass per photo on screen in it (the two with the most screen time) that masks only that photo's appearances, with a prompt about that person only. It needs the pack's `H3MontagePeople`, `H3MontageTracks`, `H3MontagePlan` and `H3MontagePass` nodes. The saved tracks live in ComfyUI's temp folder, so after a ComfyUI restart, Find people again.## Tipsdiff --git a/app/api/jobs/[id]/events/route.ts b/app/api/jobs/[id]/events/route.tsindex deafe2e..063cace 100644--- a/app/api/jobs/[id]/events/route.ts+++ b/app/api/jobs/[id]/events/route.ts@@ -35,7 +35,7 @@ export async function GET(request: Request, { params }: { params: Promise<{ id:}} catch (error) {// The proxy forgets jobs when it restarts; the page then shows the job as no longer available.- if (error instanceof NotFound) send({ type: "job", job: { id, status: "expired", queuePosition: null, error: null, video: null } });+ if (error instanceof NotFound) send({ type: "job", job: { id, status: "expired", queuePosition: null, error: null, video: null, data: null } });else if (!request.signal.aborted) send({ type: "error", message: error instanceof Error ? error.message : "Lost the job stream." });}if (!request.signal.aborted) controller.close();diff --git a/app/api/jobs/route.ts b/app/api/jobs/route.tsindex f429224..071813f 100644--- a/app/api/jobs/route.ts+++ b/app/api/jobs/route.ts@@ -1,5 +1,6 @@import { NextResponse } from "next/server";-import { listModels, submitAdvanced, submitColor, submitFace, submitMask, submitObject, submitSimple } from "../../../lib/comfy.ts";+import { listModels, submitAdvanced, submitColor, submitFace, submitMask, submitMontage, submitObject, submitPeople, submitSimple } from "../../../lib/comfy.ts";+import { TRACKS_FILE, type Label } from "../../../lib/montage.ts";import { DEFAULT_TONE, DETAIL, TONE_MODES, TONE_RANGES, type ColorSettings, type ToneMode, type ToneSettings, type ToneValue } from "../../../lib/color.ts";import { CHUNK_SECONDS, FACE_STYLES, type FaceSettings, type FaceStyle } from "../../../lib/face.ts";import { MAX_CHARACTERS } from "../../../lib/advanced.ts";@@ -38,14 +39,18 @@ function parsePicture(picture: FormDataEntryValue | null) {return picture;}-function parseRun(form: FormData) {+function parseResolution(form: FormData): Resolution {const entry = String(form.get("resolution"));const custom = entry.match(/^(\d+)x(\d+)$/);const [width, height] = custom ? [custom[1], custom[2]].map((side) => Math.round(Number(side) / CUSTOM_SIZE.step) * CUSTOM_SIZE.step) : [0, 0];if (custom ? !(Math.min(width, height) >= CUSTOM_SIZE.min && Math.max(width, height) <= CUSTOM_SIZE.max) : !RESOLUTIONS.includes(entry as Preset)) {throw new Error(`Choose a resolution, or a custom size of ${CUSTOM_SIZE.min}-${CUSTOM_SIZE.max} px a side.`);}- const resolution: Resolution = custom ? { width, height } : entry as Preset;+ return custom ? { width, height } : entry as Preset;+}++function parseRun(form: FormData) {+ const resolution = parseResolution(form);const motion = Number(form.get("motion"));if (!MOTION_SIZES.includes(motion as MotionSize)) throw new Error("Choose a supported motion reference size.");const seed = Number(form.get("seed"));@@ -137,7 +142,36 @@ function parseAdvanced(form: FormData) {const maxQuality = form.get("max_quality") === "true";if (maxQuality && pictures.length !== 2) throw new Error("One pass per person needs exactly two characters.");const roles = form.getAll("role").map((role) => String(role).slice(0, 200));- return { pictures, maxQuality, settings: { roles, noLipSync: form.get("no_lip_sync") === "true", montage: form.get("montage") === "true", ...parseRun(form), ...parseMask(form) } };+ return { pictures, maxQuality, settings: { roles, noLipSync: form.get("no_lip_sync") === "true", ...parseRun(form), ...parseMask(form) } };+}++// Find people matches the tracks to the roles' looks, so every photo needs one.+function parsePeople(form: FormData) {+ const roles = form.getAll("role").map((role) => String(role).replace(/\s+/g, " ").trim().slice(0, 200));+ if (roles.length < 1 || roles.length > MAX_CHARACTERS) throw new Error(`Add 1-${MAX_CHARACTERS} character photos.`);+ if (roles.some((role) => !role)) throw new Error("Say who each photo replaces and how they look, e.g. “the dancer - white polo, jeans”.");+ return { roles, resolution: parseResolution(form) };+}++// The appearances picked in the person list: a SAM track within its frames, and the photo it becomes.+function parseLabels(entry: FormDataEntryValue | null, photos: number): Label[] {+ let labels: Label[] = [];+ try {+ labels = JSON.parse(String(entry));+ } catch {}+ const whole = (value: unknown): value is number => Number.isInteger(value) && (value as number) >= 0;+ const valid = Array.isArray(labels) && labels.length > 0+ && labels.every((l) => whole(l?.track) && whole(l.start) && whole(l.end) && l.end > l.start && whole(l.role) && l.role >= 1 && l.role <= photos);+ if (!valid) throw new Error("Pick a photo for at least one person.");+ return labels.map(({ track, start, end, role }) => ({ track, start, end, role }));+}++function parseMontage(form: FormData) {+ const { pictures, settings } = parseAdvanced(form);+ if (pictures.length !== parsePeople(form).roles.length) throw new Error("Give each photo a role.");+ const tracks = String(form.get("tracks") ?? "");+ if (!TRACKS_FILE.test(tracks)) throw new Error("Find people again.");+ return { pictures, settings, tracks, labels: parseLabels(form.get("labels"), pictures.length) };}export async function POST(request: Request) {@@ -150,6 +184,13 @@ export async function POST(request: Request) {if (mode === "mask") {const { hatFix, margin, target } = parseMask(form);submit = () => submitMask(video, section, hatFix, margin, target);+ } else if (mode === "people") {+ const { roles, resolution } = parsePeople(form);+ submit = () => submitPeople(video, section, roles, resolution);+ } else if (mode === "montage") {+ const { pictures, settings, tracks, labels } = parseMontage(form);+ const model = parseModel(form, await listModels());+ submit = () => submitMontage(video, section, pictures, settings, tracks, labels, model);} else if (mode === "simple") {const { picture, settings } = parseSimple(form);const model = parseModel(form, await listModels());diff --git a/app/globals.css b/app/globals.cssindex c43ae7c..af595ed 100644--- a/app/globals.css+++ b/app/globals.css@@ -159,3 +159,10 @@ input[type="range"] { accent-color: var(--accent); width: 220px; max-width: 100%.lora-list { display: flex; flex-direction: column; align-items: flex-start; gap: 8px; flex: 1 1 100%; }.lora { display: grid; grid-template-columns: minmax(0, 1fr) 4.5rem auto; gap: 8px; width: 100%; }.lora .file-name { width: 100%; }++.people-list { display: flex; gap: 12px; flex-wrap: wrap; }+.person { display: grid; gap: 6px; background: var(--panel-2); border-radius: 12px; padding: 8px; max-width: 100%; }+.person > img { height: 150px; max-width: 100%; object-fit: contain; border-radius: 8px; }+.photo-pick { background: var(--bg); align-items: center; }+.photo-pick button { padding: 3px; }+.photo-pick img { width: 34px; height: 44px; object-fit: cover; border-radius: 5px; display: block; }diff --git a/components/openjutsu.tsx b/components/openjutsu.tsxindex c88533a..988dcad 100644--- a/components/openjutsu.tsx+++ b/components/openjutsu.tsx@@ -10,13 +10,15 @@ import { DEFAULT_MODEL, MAX_LORAS, STEPS, STRENGTH, type Lora } from "../lib/modimport type { EditMode } from "../lib/object.ts";import { CHUNK_SECONDS, faceChunkCount, type FaceStyle } from "../lib/face.ts";import { DEFAULT_PRESET, DEFAULT_TONE, DETAIL, TONE_MODES, TONE_RANGES, toneFromPreset, type ToneSettings, type ToneValue } from "../lib/color.ts";+import { appearances, renderLabels, type Appearance, type Label, type People } from "../lib/montage.ts";type Character = { file: File; url: string; role: string };type Mode = "simple" | "advanced" | "object" | "post";type Post = "face" | "color";-type Kind = "simple" | "mask" | "advanced" | "advanced_maxq" | "object" | "face" | "color";+type Kind = "simple" | "mask" | "advanced" | "advanced_maxq" | "object" | "face" | "color" | "people" | "montage";type TonePreset = { name: string; settings: Record<string, unknown> };-type Run = { id: string; kind: Kind; margin: Margin; clips: number; clipSeconds: number; startedAt: number };+// frames: the video, part and crop SAM saw+type Run = { id: string; kind: Kind; margin: Margin; clips: number; clipSeconds: number; startedAt: number; frames: string };// pass counts the sampler runs started so far: Fix faces samples its chunks one after another in the same node.type Progress = NodeInfo & { step: number | null; steps: number | null; pass: number };@@ -292,6 +294,8 @@ export function Openjutsu() {const canvas = typeof resolution === "object" ? [resolution.width, resolution.height] : video?.width ? canvasSize(video.width, video.height, resolution) : null;const resolutionField = typeof resolution === "object" ? `${resolution.width}x${resolution.height}` : resolution;+ // A montage render masks with the tracks Find people saved, so it must use the frames Find people saw.+ const frames = video ? `${video.file.name}:${video.file.size} ${whole ? "whole" : `${part.start.toFixed(2)}+${part.length.toFixed(2)}`} ${typeof resolution === "object" ? resolutionField : ""}` : "";function chooseSize(side: keyof Size, value: number) {const snapped = Math.min(CUSTOM_SIZE.max, Math.max(CUSTOM_SIZE.min, Math.round(value / CUSTOM_SIZE.step) * CUSTOM_SIZE.step || customSize[side]));@@ -314,8 +318,8 @@ export function Openjutsu() {setLook({ file: files[0], url: URL.createObjectURL(files[0]) });}- async function submit(kind: Kind) {- if (!video || (kind !== "mask" && !ready)) return;+ async function submit(kind: Kind, montagePicks?: { tracks: string; labels: Label[] }) {+ if (!video || (kind !== "mask" && kind !== "people" && !ready)) return;setSubmitting(true);setFormError("");const form = new FormData();@@ -348,7 +352,21 @@ export function Openjutsu() {form.append("role", c.role);}form.set("max_quality", String(kind === "advanced_maxq"));- form.set("montage", String(montage));+ form.set("resolution", resolutionField);+ form.set("seed", String(seed));+ form.set("clip_seconds", String(clipSeconds));+ }+ if (kind === "people") {+ for (const c of shown) form.append("role", c.role);+ form.set("resolution", resolutionField);+ }+ if (kind === "montage") {+ for (const c of shown) {+ form.append("picture", c.file);+ form.append("role", c.role);+ }+ form.set("tracks", montagePicks?.tracks ?? "");+ form.set("labels", JSON.stringify(montagePicks?.labels ?? []));form.set("resolution", resolutionField);form.set("seed", String(seed));form.set("clip_seconds", String(clipSeconds));@@ -377,7 +395,7 @@ export function Openjutsu() {form.set("seed", String(seed));form.set("clip_seconds", String(clipSeconds));}- if (kind !== "mask" && kind !== "color") {+ if (kind !== "mask" && kind !== "color" && kind !== "people") {form.set("motion", String(motion));form.set("steps", String(steps));form.set("model", model);@@ -390,7 +408,7 @@ export function Openjutsu() {const response = await fetch("/api/jobs", { method: "POST", body: form });const body = await response.json();if (!response.ok) throw new Error(body.error ?? "Could not start the job.");- const next: Run = { id: body.id, kind, margin, clips: Math.max(clips, 1), clipSeconds: kind === "face" ? chunkSeconds : clipSeconds, startedAt: Date.now() };+ const next: Run = { id: body.id, kind, margin, clips: Math.max(clips, 1), clipSeconds: kind === "face" ? chunkSeconds : clipSeconds, startedAt: Date.now(), frames };writeRun(next);setJob(body);setProgress(null);@@ -411,6 +429,7 @@ export function Openjutsu() {}function reset() {+ setFormError("");writeRun(null);setRun(null);setJob(null);@@ -444,11 +463,14 @@ export function Openjutsu() {))}</nav>+ {formError && <p className="error" role="alert">{formError}</p>}{run ? (<RunViewrun={run} job={job} progress={progress} preview={preview} streamError={streamError} now={now}onCancel={cancel} onReset={reset} onGenerate={video && character && !submitting && mode === "simple" ? () => submit("simple") : null}margin={margin} onMargin={setMargin} onRecheck={video && !submitting ? () => submit("mask") : null}+ photos={shown.map((c) => c.url)}+ onMontage={video && character && !submitting ? (tracks, labels) => submit("montage", { tracks, labels }) : null} frames={frames}/>) : (<>@@ -511,8 +533,9 @@ export function Openjutsu() {{montage && (<p className="hint">For edited clips where different people take turns within one shot, with no hard cut between them: whip pans,- selfie handovers, travel and “vibe” montages. H3 is told when each person is on screen, so nobody turns- into someone else. Use Standard for everything else.+ selfie handovers, travel and “vibe” montages. Find people tracks everyone over the whole video and matches+ them to the photos; you check each appearance and pick its photo, then the video is made in one go. Give every photo a role with a look after+ “ - ” (“the dancer - white polo, jeans”) so the people can be told apart. Use Standard for everything else.</p>)}</>)}@@ -707,7 +730,7 @@ export function Openjutsu() {<input type="checkbox" checked={noLipSync} onChange={(event) => setNoLipSync(event.target.checked)} /><i aria-hidden /></label>}- {mode === "advanced" && (+ {mode === "advanced" && !montage && (<label className="setting toggle"><span>One pass per person <small>Each character gets their own H3 pass with only their photo, so about twice the generation time.{shown.length === 2 ? "" : " Needs exactly two characters."}</small></span><input type="checkbox" checked={onePassEach} disabled={shown.length !== 2} onChange={(event) => setMaxQuality(event.target.checked)} />@@ -782,17 +805,21 @@ export function Openjutsu() {)}<div className="go">- {formError && <p className="error">{formError}</p>}<div className="row">{mode === "simple" && <button className="secondary" disabled={!video || submitting} onClick={() => submit("mask")}>Check mask</button>}- <button className="primary" disabled={!video || !ready || submitting} onClick={() => submit(mode === "object" ? fromPicture ? "simple" : "object" : mode === "post" ? post : mode === "simple" ? "simple" : onePassEach ? "advanced_maxq" : "advanced")}>- {submitting ? "Uploading…" : "Generate"}- </button>+ {mode === "advanced" && montage ? (+ <button className="primary" disabled={!video || !character || submitting} onClick={() => submit("people")}>{submitting ? "Uploading…" : "Find people"}</button>+ ) : (+ <button className="primary" disabled={!video || !ready || submitting} onClick={() => submit(mode === "object" ? fromPicture ? "simple" : "object" : mode === "post" ? post : mode === "simple" ? "simple" : onePassEach ? "advanced_maxq" : "advanced")}>+ {submitting ? "Uploading…" : "Generate"}+ </button>+ )}</div><small>{colorFix? !video ? "Add a video." : ready ? `${(part.length || video.duration).toFixed(1)} s of video` : "Turn on detail contrast, the tone mapper or both.": clips && ready- ? faceFix ? `Up to ${clips} chunk${clips > 1 ? "s" : ""} of face` : `${clips} clip${clips > 1 ? "s" : ""} at ${canvas ? `${canvas[0]}×${canvas[1]}` : resolution}`+ ? mode === "advanced" && montage ? "About 3 minutes per 18 s of video; then you check who becomes which photo and generate."+ : faceFix ? `Up to ${clips} chunk${clips > 1 ? "s" : ""} of face` : `${clips} clip${clips > 1 ? "s" : ""} at ${canvas ? `${canvas[0]}×${canvas[1]}` : resolution}`: `Add a video and ${mode === "object" ? "say what it becomes" : faceFix ? "the photo of the person" : mode === "simple" ? "a character photo" : "character photos"}.`}</small></div></>@@ -801,16 +828,27 @@ export function Openjutsu() {);}-function RunView({ run, job, progress, preview, streamError, now, onCancel, onReset, onGenerate, margin, onMargin, onRecheck }: {+function RunView({ run, job, progress, preview, streamError, now, onCancel, onReset, onGenerate, margin, onMargin, onRecheck, photos, onMontage, frames }: {run: Run; job: PublicJob | null; progress: Progress | null; preview: string; streamError: string; now: number;onCancel: () => void; onReset: () => void; onGenerate: (() => void) | null;margin: Margin; onMargin: (margin: Margin) => void; onRecheck: (() => void) | null;+ photos: string[]; onMontage: ((tracks: string, labels: Label[]) => void) | null; frames: string;}) {const status = job?.status ?? "queued";const clips = Math.max(run.clips, progress?.clip ?? 1);- const mask = run.kind === "mask";+ const people = run.kind === "people";+ const mask = run.kind === "mask" || people;++ if (people && status === "succeeded" && job?.data) {+ return (+ <PersonList+ url={job.data} photos={photos} onBack={onReset}+ onGenerate={run.frames === frames ? onMontage : null}+ />+ );+ }- if (mask && status === "succeeded" && job?.video) {+ if (run.kind === "mask" && status === "succeeded" && job?.video) {return (<MaskCheckurl={job.video.url} clipSeconds={run.clipSeconds} checked={run.margin ?? "normal"} margin={margin}@@ -856,7 +894,7 @@ function RunView({ run, job, progress, preview, streamError, now, onCancel, onRereturn (<section className="panel progress"><div className="row">- <h2>{status === "queued" ? `Queued${job?.queuePosition ? ` (#${job.queuePosition})` : ""}` : mask ? "Checking the mask" : color ? "Fixing the colours"+ <h2>{status === "queued" ? `Queued${job?.queuePosition ? ` (#${job.queuePosition})` : ""}` : people ? "Finding people" : mask ? "Checking the mask" : color ? "Fixing the colours": face ? (sampling ? `Chunk ${chunk} of up to ${run.clips}` : chunk ? "Pasting the face back" : "Tracking the face"): progress?.clip ? `Clip ${progress.clip} of ${clips}` : "Preparing"}</h2><span className="time">{elapsed(now - run.startedAt)}</span>@@ -868,7 +906,7 @@ function RunView({ run, job, progress, preview, streamError, now, onCancel, onRe</p>{preview && <img className="preview" src={preview} alt="Mask preview: white areas are regenerated" />}{streamError && <p className="error">{streamError}</p>}- <p className="hint">{mask ? "SAM tracks the whole video once, about a minute per 3 s of video." : color ? "Every frame goes through the filters, then the video is saved with its soundtrack." : face ? "The face is tracked over the whole video, then each chunk is regenerated; nothing is saved until the end." : "Each clip is saved as it finishes, so a rerun with the same inputs resumes where it stopped."} You can close this page and come back.</p>+ <p className="hint">{people ? "SAM tracks everyone over the whole video, then Qwen matches each person to a role." : mask ? "SAM tracks the whole video once, about a minute per 3 s of video." : color ? "Every frame goes through the filters, then the video is saved with its soundtrack." : face ? "The face is tracked over the whole video, then each chunk is regenerated; nothing is saved until the end." : "Each clip is saved as it finishes, so a rerun with the same inputs resumes where it stopped."} You can close this page and come back.</p><button className="secondary" onClick={onCancel}>Cancel</button></section>);@@ -940,3 +978,60 @@ function MaskCheck({ url, clipSeconds, checked, margin, onMargin, onRecheck, onG</section>);}++function PersonList({ url, photos, onBack, onGenerate }: {+ url: string; photos: string[]; onBack: () => void; onGenerate: ((tracks: string, labels: Label[]) => void) | null;+}) {+ const [found, setFound] = useState<{ tracks: string | null; list: Appearance[] } | null>(null);+ const [error, setError] = useState("");++ useEffect(() => {+ fetch(url).then((response) => response.json()).then((people: People) => setFound({ tracks: people.tracks, list: appearances(people) }),+ () => setError("Could not read what Find people found."));+ }, [url]);++ const labels = found ? renderLabels(found.list) : [];++ function pick(index: number, photo: number) {+ setFound((current) => current && { ...current, list: current.list.map((a, i) => (i === index ? { ...a, pick: photo } : a)) });+ }++ return (+ <section className="panel">+ <h2>People {found && <small>{found.list.length} appearance{found.list.length === 1 ? "" : "s"}</small>}</h2>+ <p className="hint">+ Everyone SAM tracked, in order of appearance, matched to the photos. Check each one and pick the photo they become,+ or Keep to leave them as they are. The whole video is then made in one go, with one H3 pass per photo on screen.+ </p>+ {error && <p className="error">{error}</p>}+ {!found && !error && <p className="hint">Loading…</p>}+ <div className="people-list">+ {found?.list.map((a, i) => (+ <div className="person" key={`${a.track}:${a.start}`}>+ <img src={a.image} alt={`Appearance ${i + 1}`} />+ <small>+ {timecode(a.on[0][0] / 24)}–{timecode(a.on.at(-1)![1] / 24)} · {a.role ? `looks like photo ${a.role}` : "none of the photos"}+ {a.small ? " · small in the frame" : ""}+ </small>+ <div className="segmented photo-pick">+ {photos.map((photo, k) => (+ <button key={k} className={a.pick === k + 1 ? "on" : ""} onClick={() => pick(i, k + 1)} aria-label={`Photo ${k + 1}`}>+ <img src={photo} alt="" />+ </button>+ ))}+ <button className={a.pick ? "" : "on"} onClick={() => pick(i, 0)}>Keep</button>+ </div>+ </div>+ ))}+ </div>+ {found && !found.list.length && <p className="hint">Nobody found.</p>}+ {found && !onGenerate && <p className="hint">The video, the part to use or the custom size changed since Find people, or a photo is missing. Go back and find the people again.</p>}+ <div className="row">+ <button className="primary" disabled={!onGenerate || !found?.tracks || !labels.length} onClick={() => onGenerate?.(found!.tracks!, labels)}>+ {labels.length ? "Generate" : "Pick a photo to generate"}+ </button>+ <button className="secondary" onClick={onBack}>Back to settings</button>+ </div>+ </section>+ );+}diff --git a/lib/advanced.ts b/lib/advanced.tsindex 0821c7b..03917e4 100644--- a/lib/advanced.ts+++ b/lib/advanced.ts@@ -2,7 +2,7 @@// workflows/advanced.api.json (one pass) and advanced_maxq.api.json (one pass per person, 2 characters) are exported// with every picture, the hat fix and No lip-sync enabled; each request prunes what it does not use. Nodes are found by title, as// the builder (ComfyUI user/default/oneclick_advanced_build.js) names them.-import { MARGINS, dropNoLipSync, isLink, setMotionSize, setResolution, suffixRunName, type ApiGraph, type Margin, type MotionSize, type Resolution } from "./simple.ts";+import { MARGINS, dropNoLipSync, isLink, setMotionSize, setResolution, type ApiGraph, type Margin, type MotionSize, type Resolution } from "./simple.ts";export const MAX_CHARACTERS = 4;@@ -12,14 +12,13 @@ export type AdvancedSettings = {motion: MotionSize;hatFix: boolean;noLipSync: boolean;- montage: boolean; // different people take turns within one shot (whip pans, selfie handovers)margin: Margin;seed: number;clipSeconds: number;};const HAT = /\bhats?\b/i;-const PLAN = "Swap Plan (roles, seed, size)";+export const PLAN = "Swap Plan (roles, seed, size)";export const VIDEO = "Source video (people get replaced)";export function nodeByTitle(graph: ApiGraph, title: string) {@@ -64,17 +63,12 @@ export function pruneAdvanced(source: ApiGraph, pictures: number, settings: Advafor (const [name, value] of Object.entries(node.inputs)) {if (!isLink(value) || !removed.has(value[0])) continue;if (bypass.has(value[0])) node.inputs[name] = bypass.get(value[0]);- else if (node._meta?.title === PLAN || name.startsWith("ref_images.")) delete node.inputs[name];+ else if (/^picture_\d/.test(name) || name.startsWith("ref_images.")) delete node.inputs[name];else throw new Error(`Node ${id}.${name} still links to removed node ${value[0]}.`);}}if (!settings.noLipSync) dropNoLipSync(graph);setMotionSize(graph, settings.motion);- // Montage: the shot list and H3 are told when each person is on screen, so nobody turns into another subject within a shot.- if (settings.montage) {- for (const node of Object.values(graph)) if (node.class_type === "H3ClipShots" || node.class_type === "H3FillShots") node.inputs.timing = true;- suffixRunName(graph, "montage_run_name", "_montage", "Run name for Montage");- }for (const node of Object.values(graph)) {if (node.class_type === "GrowMaskWithBlur" && / grow mask /.test(node._meta?.title ?? "")) node.inputs.expand = MARGINS[settings.margin];diff --git a/lib/comfy.ts b/lib/comfy.tsindex 5c4e309..877e97e 100644--- a/lib/comfy.ts+++ b/lib/comfy.ts@@ -2,12 +2,13 @@ import { readFile } from "node:fs/promises";import { join } from "node:path";import { Comfy, COMFY_CLOUD_BASE_URL, resolveCredentials, type Job } from "@comfyorg/sdk";import { ComfyLow } from "@comfyorg/sdk/low";-import { MASK_NODES, NODES, pruneMask, pruneSimple, setSection, type ApiGraph, type Margin, type Section, type SimpleSettings } from "./simple.ts";+import { MASK_NODES, NODES, cropSource, pruneMask, pruneSimple, setSection, type ApiGraph, type Margin, type Resolution, type Section, type SimpleSettings } from "./simple.ts";import { VIDEO, nodeByTitle, pruneAdvanced, type AdvancedSettings } from "./advanced.ts";import { applyModel, type ModelSettings } from "./model.ts";import { OBJECT_PICTURE, OBJECT_VIDEO, pruneObject, type ObjectSettings } from "./object.ts";import { FACE_PICTURE, FACE_VIDEO, pruneFace, type FaceSettings } from "./face.ts";import { COLOR_VIDEO, pruneColor, type ColorSettings } from "./color.ts";+import { PEOPLE_NODES, peopleRoles, pruneMontage, type Label } from "./montage.ts";export type PublicJob = {id: string;@@ -15,6 +16,7 @@ export type PublicJob = {queuePosition: number | null;error: string | null;video: { id: string; name: string; url: string } | null;+ data: string | null; // what Find people found, as JSON};export const TERMINAL = new Set(["succeeded", "failed", "canceled", "expired"]);@@ -36,7 +38,7 @@ export function rawJobEvents(job: Job, signal: AbortSignal) {return low().getJobEvents(job.urls.events || job.id, { signal });}-export const JOB_KINDS = ["simple", "mask", "advanced", "advanced_maxq", "object", "face", "color"] as const;+export const JOB_KINDS = ["simple", "mask", "advanced", "advanced_maxq", "object", "face", "color", "people", "montage"] as const;export type JobKind = (typeof JOB_KINDS)[number];export async function loadGraph(kind: JobKind): Promise<ApiGraph> {@@ -54,12 +56,14 @@ function jobErrorMessage(error: unknown) {export function publicJob(job: Job): PublicJob {const video = job.outputs.find((output) => output.contentType.startsWith("video/"));+ const data = job.outputs.find((output) => output.contentType === "application/json");return {id: job.id,status: job.status,queuePosition: job.queuePosition,error: job.error ? jobErrorMessage(job.error) : null,video: video ? { id: video.id, name: video.name, url: `/api/jobs/${job.id}/outputs/${video.id}` } : null,+ data: data ? `/api/jobs/${job.id}/outputs/${data.id}` : null,};}@@ -134,6 +138,28 @@ export async function submitMask(video: File, section: Section, hatFix: boolean,return publicJob(await comfy.submit(workflow));}+export async function submitMontage(video: File, section: Section, pictures: File[], settings: AdvancedSettings, tracks: string, labels: Label[], model: ModelSettings) {+ const comfy = client();+ const graph = pruneMontage(await loadGraph("montage"), pictures.length, settings, tracks, labels);+ applyModel(graph, model);+ setSection(graph, nodeByTitle(graph, VIDEO), section);+ const workflow = comfy.workflows.fromJson(graph);+ workflow.setInput(nodeByTitle(graph, VIDEO), "video", await upload(comfy, video));+ for (const [i, picture] of pictures.entries()) workflow.setInput(nodeByTitle(graph, `Picture ${i + 1}`), "image", await upload(comfy, picture));+ return publicJob(await comfy.submit(workflow));+}++export async function submitPeople(video: File, section: Section, roles: string[], resolution: Resolution) {+ const comfy = client();+ const graph = await loadGraph("people");+ graph[PEOPLE_NODES.crops].inputs.people = peopleRoles(roles);+ if (typeof resolution === "object") cropSource(graph, PEOPLE_NODES.video, resolution);+ setSection(graph, PEOPLE_NODES.video, section);+ const workflow = comfy.workflows.fromJson(graph);+ workflow.setInput(PEOPLE_NODES.video, "video", await upload(comfy, video));+ return publicJob(await comfy.submit(workflow));+}+const COMFYUI_URL = process.env.COMFYUI_URL || "http://127.0.0.1:8188";// The proxy has no model listing, so the pickers read ComfyUI's own node info. COMFYUI_URL points at ComfyUI itselfdiff --git a/lib/model.ts b/lib/model.tsindex c5ba0d4..1ad35c8 100644--- a/lib/model.ts+++ b/lib/model.ts@@ -18,7 +18,7 @@ export const STRENGTH = { min: -4, max: 4 };// Mutates graph: the steps (on the Swap Plan, or the Schedule in Fix faces, which has no plan), the chosen diffusion model,// then the LoRAs in order into whatever read the old LoRA node.export function applyModel(graph: ApiGraph, settings: ModelSettings) {- const plans = Object.values(graph).filter((node) => ["H3OneClickSwapPlan", "H3AdvancedSwapPlan", "H3ObjectSwapPlan"].includes(node.class_type));+ const plans = Object.values(graph).filter((node) => ["H3OneClickSwapPlan", "H3AdvancedSwapPlan", "H3MontagePlan", "H3ObjectSwapPlan"].includes(node.class_type));if (plans.length > 1) throw new Error(`Expected one Swap Plan node in the workflow, found ${plans.length}.`);(plans[0] ?? graph[nodeByTitle(graph, "Schedule")]).inputs.steps = settings.steps;const unet = nodeByTitle(graph, "H3 Diffusion Model");diff --git a/lib/montage.ts b/lib/montage.tsnew file mode 100644index 0000000..72c29ea--- /dev/null+++ b/lib/montage.ts@@ -0,0 +1,53 @@+// Montage: for edited clips where people take turns within one shot. Find people (workflows/people.api.json) tracks every+// person over the whole video, labels each appearance with a photo and saves the tracks; H3MontagePeople in+// comfyui-h3-oneclick-swap writes the result. The render (workflows/montage.api.json) loads those same tracks and makes the+// video as one chain, with one H3 pass per photo on screen in each clip.+import { PLAN, nodeByTitle, pruneAdvanced, type AdvancedSettings } from "./advanced.ts";+import type { ApiGraph } from "./simple.ts";++// One SAM track's appearance within one shot. It spans start..end (to the middle of the gaps around it), is on screen during+// `on`, takes at most `area` of the frame and looks like photo `role` (0 = none). Frames count from the start of the part used, at 24 fps.+export type Track = { track: number; start: number; end: number; role: number; on: [number, number][]; area: number; image: string };+// tracks: the file Find people saved them to, in ComfyUI's temp folder+export type People = { frames: number; tracks: string | null; people: Track[] };++// pick: the photo this appearance becomes, 0 to keep it. small: never big enough for H3 to draw them.+export type Appearance = Track & { frames: number; small: boolean; pick: number };+export type Label = { track: number; start: number; end: number; role: number };++// Node ids in workflows/people.api.json.+export const PEOPLE_NODES = { video: "1", crops: "6" };+export const TRACKS = "Tracks from Find people";+export const TRACKS_FILE = /^h3_montage_people_\d+_\.npz$/;++// Appearances start as kept unless they are on screen this long and this big: H3 draws a brief or small,+// far-off person as the grey figure of the motion reference instead of the photo.+const BRIEF = 12;+const MIN_SHARE = 0.02;++export function peopleRoles(roles: string[]) {+ return roles.map((role, i) => `<Subject ${i + 1}>: ${role.replace(/\s+/g, " ").trim()}`).join("\n");+}++export function appearances({ people }: People): Appearance[] {+ return people+ .map((t) => {+ const frames = t.on.reduce((sum, [a, b]) => sum + b - a, 0);+ const small = t.area < MIN_SHARE;+ return { ...t, frames, small, pick: t.role && !small && frames >= BRIEF ? t.role : 0 };+ })+ // unlabelled small flickers stay as they are either way; a big one may be part of someone (an arm, a knee)+ .filter((a) => a.role || a.frames >= BRIEF || !a.small)+ .sort((a, b) => a.on[0][0] - b.on[0][0] || a.track - b.track);+}++export function renderLabels(list: Appearance[]): Label[] {+ return list.filter((a) => a.pick).map(({ track, start, end, pick }) => ({ track, start, end, role: pick }));+}++export function pruneMontage(source: ApiGraph, pictures: number, settings: AdvancedSettings, tracks: string, labels: Label[]): ApiGraph {+ const graph = pruneAdvanced(source, pictures, settings);+ graph[nodeByTitle(graph, PLAN)].inputs.labels = JSON.stringify(labels);+ graph[nodeByTitle(graph, TRACKS)].inputs.file = tracks;+ return graph;+}diff --git a/lib/simple.ts b/lib/simple.tsindex 8e355f7..f006d50 100644--- a/lib/simple.ts+++ b/lib/simple.ts@@ -25,11 +25,16 @@ export function setResolution(graph: ApiGraph, plan: string, video: string, resograph[plan].inputs.resolution = resolution;return;}+ cropSource(graph, video, resolution);+ graph[plan].inputs.short_edge = Math.min(resolution.width, resolution.height);+}++// Also for Find people, whose SAM must see the same frames as the montage render.+export function cropSource(graph: ApiGraph, video: string, size: Size) {for (const node of Object.values(graph)) {for (const [name, value] of Object.entries(node.inputs)) if (isLink(value) && value[0] === video && value[1] === 0) node.inputs[name] = ["custom_size", 0];}- graph.custom_size = { class_type: "ImageScale", inputs: { image: [video, 0], upscale_method: "bilinear", width: resolution.width, height: resolution.height, crop: "center" }, _meta: { title: "Custom size (centre crop)" } };- graph[plan].inputs.short_edge = Math.min(resolution.width, resolution.height);+ graph.custom_size = { class_type: "ImageScale", inputs: { image: [video, 0], upscale_method: "bilinear", width: size.width, height: size.height, crop: "center" }, _meta: { title: "Custom size (centre crop)" } };}export const CLIP_SECONDS = { min: 2.5, max: 15, default: 6 };diff --git a/test/advanced.test.ts b/test/advanced.test.tsindex d08daa2..9e7566c 100644--- a/test/advanced.test.ts+++ b/test/advanced.test.ts@@ -8,7 +8,7 @@ import { isLink, type ApiGraph } from "../lib/simple.ts";const load = (name: string): ApiGraph => JSON.parse(readFileSync(new URL(`../workflows/${name}.api.json`, import.meta.url), "utf8"));const graph = load("advanced");const maxq = load("advanced_maxq");-const settings = (hatFix: boolean, roles: string[] = [], noLipSync = true): AdvancedSettings => ({ roles, resolution: "480p", motion: 512, hatFix, noLipSync, montage: false, margin: "tight", seed: 3, clipSeconds: 6 });+const settings = (hatFix: boolean, roles: string[] = [], noLipSync = true): AdvancedSettings => ({ roles, resolution: "480p", motion: 512, hatFix, noLipSync, margin: "tight", seed: 3, clipSeconds: 6 });const titles = (g: ApiGraph) => Object.values(g).map((node) => node._meta?.title ?? "");const dangling = (g: ApiGraph) => Object.entries(g).flatMap(([id, node]) =>Object.entries(node.inputs).filter(([, v]) => isLink(v) && !g[v[0]]).map(([name]) => `${id}.${name}`));@@ -94,19 +94,6 @@ test("the motion reference size reaches both passes", () => {assert.deepEqual(sizes(pruneAdvanced(maxq, 2, { ...settings(false), motion: 1376 })), [1376, 1376]);});-test("Montage turns on the on-screen timing in both passes and gets its own run", () => {- const timing = (g: typeof graph) => Object.values(g).filter((node) => node.class_type === "H3ClipShots" || node.class_type === "H3FillShots").map((node) => node.inputs.timing);- for (const source of [graph, maxq]) {- const off = pruneAdvanced(source, 2, { ...settings(false), motion: 768 });- const on = pruneAdvanced(source, 2, { ...settings(false), motion: 768, montage: true });- assert.ok(timing(off).every((v) => v === undefined));- assert.ok(timing(on).length >= 2 && timing(on).every((v) => v === true));- assert.equal(off.montage_run_name, undefined);- assert.deepEqual(on.montage_run_name.inputs, { string_a: ["motion_run_name", 0], string_b: "_montage", delimiter: "" });- assert.ok(Object.values(on).some((node) => node.class_type === "MiniMaxH3ChainPlanModern" && isLink(node.inputs.run_name) && node.inputs.run_name[0] === "montage_run_name"));- }-});-test("picture counts outside the export are refused", () => {assert.throws(() => pruneAdvanced(graph, 0, settings(true)));assert.throws(() => pruneAdvanced(maxq, 3, settings(true)));diff --git a/test/montage.test.ts b/test/montage.test.tsnew file mode 100644index 0000000..e5dc79b--- /dev/null+++ b/test/montage.test.ts@@ -0,0 +1,78 @@+import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import test from "node:test";+import { PEOPLE_NODES, TRACKS, TRACKS_FILE, appearances, peopleRoles, pruneMontage, renderLabels, type Track } from "../lib/montage.ts";+import { PLAN, nodeByTitle, type AdvancedSettings } from "../lib/advanced.ts";+import { isLink, type ApiGraph } from "../lib/simple.ts";+import { DEFAULT_MODEL, applyModel } from "../lib/model.ts";++const track = (id: number, role: number, start: number, end: number, on: [number, number][], area = 0.1): Track => ({ track: id, start, end, role, on, area, image: `img${id}` });++test("appearances are listed in order and start as their photo unless brief or small", () => {+ const list = appearances({+ frames: 300, tracks: "h3_montage_people_00001_.npz", people: [+ track(1, 2, 0, 160, [[104, 160]]),+ track(0, 1, 0, 160, [[0, 110]]),+ track(3, 1, 0, 160, [[138, 158]], 0.006), // the dancer far off behind the filmer+ track(4, 2, 160, 300, [[160, 166]]), // a flicker+ track(5, 0, 160, 300, [[170, 175]], 0.005), // a small unlabelled flicker: not listed+ track(7, 0, 160, 300, [[176, 180]], 0.06), // a big one may be part of someone: listed, kept+ track(6, 0, 160, 300, [[170, 300]]), // nobody Qwen knew+ ],+ });+ assert.deepEqual(list.map((a) => [a.track, a.role, a.pick, a.small]), [[0, 1, 1, false], [1, 2, 2, false], [3, 1, 0, true], [4, 2, 0, false], [6, 0, 0, false], [7, 0, 0, false]]);+});++test("a person whose track broke is the same photo in every appearance", () => {+ // SAM lost the woman at 174 and found her again as a new track+ const list = appearances({ frames: 216, tracks: null, people: [track(6, 3, 131, 174, [[131, 174]], 0.8), track(9, 3, 174, 216, [[174, 216]], 0.9)] });+ assert.deepEqual(renderLabels(list), [{ track: 6, start: 131, end: 174, role: 3 }, { track: 9, start: 174, end: 216, role: 3 }]);+});++test("the render gets the picks, keeping each appearance's frames, and leaves out kept people", () => {+ const list = appearances({ frames: 200, tracks: null, people: [track(0, 1, 0, 100, [[0, 90]]), track(1, 2, 0, 100, [[60, 100]])] });+ list[1].pick = 1;+ list[0].pick = 0;+ assert.deepEqual(renderLabels(list), [{ track: 1, start: 0, end: 100, role: 1 }]);+});++test("the montage export loads the tracks Find people saved and takes the picks", () => {+ const source: ApiGraph = JSON.parse(readFileSync(new URL("../workflows/montage.api.json", import.meta.url), "utf8"));+ const settings: AdvancedSettings = { roles: ["the dancer - white polo", "the filmer - bucket hat"], resolution: "480p", motion: 512, hatFix: false, noLipSync: false, margin: "normal", seed: 3, clipSeconds: 6 };+ const labels = [{ track: 0, start: 0, end: 107, role: 2 }];+ const graph = pruneMontage(source, 2, settings, "h3_montage_people_00003_.npz", labels);+ const dangling = Object.entries(graph).flatMap(([id, node]) => Object.entries(node.inputs).filter(([, v]) => isLink(v) && !graph[v[0]]).map(([name]) => `${id}.${name}`));+ assert.deepEqual(dangling, []);+ const plan = graph[nodeByTitle(graph, PLAN)];+ assert.equal(plan.class_type, "H3MontagePlan");+ assert.deepEqual(JSON.parse(String(plan.inputs.labels)), labels);+ assert.equal(plan.inputs.roles, "the dancer - white polo\nthe filmer - bucket hat");+ assert.ok(!("picture_3" in plan.inputs) && isLink(plan.inputs.picture_2));+ const tracks = graph[nodeByTitle(graph, TRACKS)];+ assert.equal(tracks.inputs.file, "h3_montage_people_00003_.npz");+ assert.deepEqual(plan.inputs.track_data, [nodeByTitle(graph, TRACKS), 0]);+ applyModel(graph, { ...DEFAULT_MODEL, steps: 4 });+ assert.equal(plan.inputs.steps, 4);+ const passes = Object.values(graph).filter((node) => node.class_type === "H3MontagePass");+ assert.deepEqual(passes.map((node) => node.inputs.pass_number), [1, 2]);+ assert.ok(passes.every((node) => isLink(node.inputs.picture_2) && !("picture_3" in node.inputs)));+ // no person tracking at render time: the masks are the saved ones+ assert.ok(!Object.values(graph).some((node) => node.class_type === "SAM3_VideoTrack" && /every person/.test(node._meta?.title ?? "")));+ // the hats and faces SAM finds per clip line up with the saved tracks: both at the same 1008 px frames+ const people: ApiGraph = JSON.parse(readFileSync(new URL("../workflows/people.api.json", import.meta.url), "utf8"));+ const find = Object.values(people).find((node) => node.class_type === "SAM3_VideoTrack")!;+ const size = (g: ApiGraph, title: RegExp) => Object.values(g).find((node) => node.class_type === "ImageScaleToMaxDimension" && title.test(node._meta?.title ?? ""))!.inputs.largest_size;+ assert.equal(size(graph, /^SAM video/), people[(find.inputs.images as [string, number])[0]].inputs.largest_size);+});++test("only files Find people saves pass as tracks", () => {+ assert.ok(TRACKS_FILE.test("h3_montage_people_00012_.npz"));+ for (const bad of ["../h3_montage_people_00012_.npz", "h3_montage_people_00012_.json", "x.npz", "h3_montage_people_1_.npz/.."]) assert.ok(!TRACKS_FILE.test(bad), bad);+});++test("Find people gets the roles as the subjects to match", () => {+ const graph: ApiGraph = JSON.parse(readFileSync(new URL("../workflows/people.api.json", import.meta.url), "utf8"));+ assert.equal(graph[PEOPLE_NODES.video].class_type, "VHS_LoadVideo");+ assert.equal(graph[PEOPLE_NODES.crops].class_type, "H3TrackCrops");+ assert.equal(peopleRoles(["the dancer - white polo", "the filmer - bucket hat"]), "<Subject 1>: the dancer - white polo\n<Subject 2>: the filmer - bucket hat");+});diff --git a/workflows/montage.api.json b/workflows/montage.api.jsonnew file mode 100644index 0000000..477e66e--- /dev/null+++ b/workflows/montage.api.json@@ -0,0 +1,93 @@+{+ "2": {"inputs": {"image": "apt_vest_reference.png"}, "class_type": "LoadImage", "_meta": {"title": "Picture 1"}},+ "3": {"inputs": {"upscale_method": "area", "largest_size": 768, "image": ["2", 0]}, "class_type": "ImageScaleToMaxDimension", "_meta": {"title": "Shrink for the describer"}},+ "4": {"inputs": {"prompt": "Describe the person in this picture for a casting sheet, in one sentence of about 40 words: apparent age and gender, hair, facial hair, skin tone, build, and their complete outfit including footwear. No preamble, no background.", "max_length": 120, "sampling_mode": "off", "thinking": false, "use_default_template": true, "mtp": "auto", "clip": ["16", 0], "image": ["3", 0]}, "class_type": "TextGenerate", "_meta": {"title": "Describe picture 1"}},+ "5": {"inputs": {"image": "apt_kurta_reference.png"}, "class_type": "LoadImage", "_meta": {"title": "Picture 2"}},+ "6": {"inputs": {"upscale_method": "area", "largest_size": 768, "image": ["5", 0]}, "class_type": "ImageScaleToMaxDimension", "_meta": {"title": "Shrink for the describer"}},+ "7": {"inputs": {"prompt": "Describe the person in this picture for a casting sheet, in one sentence of about 40 words: apparent age and gender, hair, facial hair, skin tone, build, and their complete outfit including footwear. No preamble, no background.", "max_length": 120, "sampling_mode": "off", "thinking": false, "use_default_template": true, "mtp": "auto", "clip": ["16", 0], "image": ["6", 0]}, "class_type": "TextGenerate", "_meta": {"title": "Describe picture 2"}},+ "8": {"inputs": {"video": "apt_rose_16s.mkv", "force_rate": 24, "custom_width": 0, "custom_height": 0, "frame_load_cap": 0, "skip_first_frames": 0, "select_every_nth": 1, "format": "None"}, "class_type": "VHS_LoadVideo", "_meta": {"title": "Source video (people get replaced)"}},+ "9": {"inputs": {"start_index": 0, "duration": 2048, "audio": ["8", 2]}, "class_type": "TrimAudioDuration", "_meta": {"title": "Soundtrack (whole video)"}},+ "14": {"class_type": "H3MontagePlan", "_meta": {"title": "Swap Plan (roles, seed, size)"}, "inputs": {"clip_seconds": 6, "overlap_frames": 39, "resolution": "736p", "steps": 6, "seed": 0, "roles": "", "no_lip_sync": ["97", 0], "source_frames": ["8", 0], "picture_1": ["2", 0], "picture_2": ["5", 0], "picture_3": ["103", 0], "picture_4": ["106", 0], "picture_1_text": ["4", 0], "picture_2_text": ["7", 0], "picture_3_text": ["105", 0], "picture_4_text": ["108", 0], "track_data": ["109", 0], "labels": "[]"}},+ "16": {"inputs": {"clip_name": "qwen3vl_8b_int8_convrot.safetensors", "type": "stable_diffusion", "device": "default"}, "class_type": "CLIPLoader", "_meta": {"title": "Describer: Qwen3-VL 8B"}},+ "18": {"inputs": {"unet_name": "minimax_h3_ref2va_pruned_int8_convrot.safetensors", "weight_dtype": "default"}, "class_type": "UNETLoader", "_meta": {"title": "H3 Diffusion Model"}},+ "19": {"inputs": {"lora_name": "minimax_h3_DMAD_4step_full_lora_avg_rank_39_bf16.safetensors", "strength_model": 1, "model": ["18", 0]}, "class_type": "LoraLoaderModelOnly", "_meta": {"title": "DMAD LoRA"}},+ "20": {"inputs": {"attention": "comfy kitchen attention", "model": ["19", 0]}, "class_type": "ModelAttentionBackend", "_meta": {"title": "Comfy Kitchen Attention"}},+ "21": {"inputs": {"shift_video": 12, "shift_audio": 2, "model": ["20", 0]}, "class_type": "MiniMaxH3SigmaShift", "_meta": {"title": "Sigma Shift"}},+ "22": {"inputs": {"clip_name": "qwen3vl_32b_minimax_h3_int4_convrot.safetensors", "type": "minimax", "device": "default"}, "class_type": "CLIPLoader", "_meta": {"title": "H3 Text Encoder"}},+ "23": {"inputs": {"vae_name": "minimax_h3_video_vae_int8_convrot.safetensors"}, "class_type": "VAELoader", "_meta": {"title": "Video VAE"}},+ "24": {"inputs": {"vae_name": "minimax_h3_audio_vae_fp32.safetensors"}, "class_type": "VAELoader", "_meta": {"title": "Audio VAE"}},+ "25": {"inputs": {"sampler_name": "euler"}, "class_type": "KSamplerSelect", "_meta": {"title": "Sampler"}},+ "26": {"inputs": {"ckpt_name": "sam3.1_multiplex_fp16.safetensors"}, "class_type": "CheckpointLoaderSimple", "_meta": {"title": "SAM 3.1"}},+ "27": {"inputs": {"text": "person", "clip": ["26", 1]}, "class_type": "CLIPTextEncode", "_meta": {"title": "SAM: what to replace"}},+ "28": {"inputs": {"scene_continuity": "Hard picture + protected audio", "audio_profile": "Generate audio"}, "class_type": "MiniMaxH3GenerationProfile", "_meta": {"title": "Generation Profile"}},+ "29": {"inputs": {"plan_json": "{\n \"shots\": [\n {\n \"id\": \"intro\",\n \"prompt\": \"Describe the opening shot.\"\n },\n {\n \"id\": \"continuation\",\n \"prompt\": \"Continue the same take.\"\n }\n ]\n}", "run_name": ["14", 3], "generation_fingerprint": "", "width": ["14", 1], "height": ["14", 2], "encode_mode": "video", "crop": "center", "default_duration_seconds": 15, "default_steps": 6, "base_seed": 0, "segment_crf": 16, "video_blend_frames": 0, "chain_policy": ["28", 0], "plan_json_input": ["14", 0]}, "class_type": "MiniMaxH3ChainPlanModern", "_meta": {"title": "Production Plan (filled by Swap Plan)"}},+ "30": {"inputs": {"video_path": "", "audio_path": "", "embedded_audio": "auto", "skip_first_frames": 0, "source_audio": ["9", 0]}, "class_type": "MiniMaxH3SourceTimeline", "_meta": {"title": "Source soundtrack"}},+ "31": {"inputs": {"start_clip": 1, "scene_range": "", "verify_resume_history": true, "plan": ["29", 0], "source_timeline": ["30", 0]}, "class_type": "MiniMaxH3ChainLoopStart", "_meta": {"title": "Loop Start"}},+ "32": {"inputs": {"align_audio_reference": false, "state": ["31", 1]}, "class_type": "MiniMaxH3ChainCurrent", "_meta": {"title": "Current Clip"}},+ "33": {"inputs": {"expression": "round(a * 24)", "values.a": ["32", 10]}, "class_type": "ComfyMathExpression", "_meta": {"title": "Clip start frame"}},+ "34": {"inputs": {"upscale_method": "area", "largest_size": 512, "image": ["8", 0]}, "class_type": "ImageScaleToMaxDimension", "_meta": {"title": "Motion reference video (~512 px)"}},+ "35": {"inputs": {"batch_index": ["33", 1], "length": ["32", 6], "image": ["34", 0]}, "class_type": "ImageFromBatch", "_meta": {"title": "Motion reference for this clip"}},+ "36": {"inputs": {"upscale_method": "area", "largest_size": 1008, "image": ["8", 0]}, "class_type": "ImageScaleToMaxDimension", "_meta": {"title": "SAM video (1008 px, SAM's own size)"}},+ "37": {"inputs": {"batch_index": ["33", 1], "length": ["32", 6], "image": ["36", 0]}, "class_type": "ImageFromBatch", "_meta": {"title": "SAM frames for this clip"}},+ "47": {"inputs": {"text": "hat", "clip": ["26", 1]}, "class_type": "CLIPTextEncode", "_meta": {"title": "SAM: hats"}},+ "48": {"inputs": {"detection_threshold": 0.35, "max_objects": 0, "detect_interval": 1, "images": ["37", 0], "model": ["26", 0], "conditioning": ["47", 0]}, "class_type": "SAM3_VideoTrack", "_meta": {"title": "SAM 3.1 track hats"}},+ "49": {"inputs": {"object_indices": "", "track_data": ["48", 0]}, "class_type": "SAM3_TrackToMask", "_meta": {"title": "All hats"}},+ "50": {"inputs": {"source_fps": 24, "crop": "center", "state": ["32", 0], "latent": ["59", 1], "audio_vae": ["24", 0], "source_frames": ["8", 0], "source_audio": ["101", 0]}, "class_type": "MiniMaxH3ContexLoopSourceSlice", "_meta": {"title": "Source clip (frames + audio)"}},+ "51": {"inputs": {"noise_seed": ["32", 5]}, "class_type": "RandomNoise", "_meta": {"title": "Clip Seed"}},+ "52": {"inputs": {"headroom": 0.1, "hold_frames": 2, "subject": 0, "track_data": ["111", 0], "labels": ["111", 1]}, "class_type": "H3RoleMask", "_meta": {"title": "Pass 1: labelled people (+ headroom)"}},+ "53": {"inputs": {"headroom": 0, "hold_frames": 1, "subject": 0, "track_data": ["111", 0], "labels": ["111", 1]}, "class_type": "H3RoleMask", "_meta": {"title": "Pass 1: labelled people, no headroom (mannequin)"}},+ "54": {"inputs": {"expand": 40, "incremental_expandrate": 0, "tapered_corners": true, "flip_input": false, "blur_radius": 0, "lerp_alpha": 1, "decay_factor": 1, "fill_holes": true, "mask": ["58", 0]}, "class_type": "GrowMaskWithBlur", "_meta": {"title": "Pass 1: grow mask (~4% of the frame)"}},+ "55": {"inputs": {"blur": 0.04, "face_blur": 0.05, "images": ["35", 0], "mask": ["53", 0], "size": ["53", 1], "faces": ["100", 0], "face_size": ["100", 1]}, "class_type": "H3GreyMannequins", "_meta": {"title": "Pass 1: hide them (grey mannequins)"}},+ "56": {"inputs": {"expand": 60, "incremental_expandrate": 0, "tapered_corners": true, "flip_input": false, "blur_radius": 0, "lerp_alpha": 1, "decay_factor": 1, "fill_holes": true, "mask": ["52", 0]}, "class_type": "GrowMaskWithBlur", "_meta": {"title": "Pass 1: hat search area (person + 60 px)"}},+ "57": {"inputs": {"x": 0, "y": 0, "operation": "multiply", "destination": ["49", 0], "source": ["56", 0]}, "class_type": "MaskComposite", "_meta": {"title": "Pass 1: only their hats"}},+ "58": {"inputs": {"x": 0, "y": 0, "operation": "add", "destination": ["52", 0], "source": ["57", 0]}, "class_type": "MaskComposite", "_meta": {"title": "Pass 1: people + hats"}},+ "59": {"inputs": {"prompt": ["111", 2], "width": ["32", 8], "height": ["32", 9], "length": ["32", 6], "ref_image_size": "match", "clip": ["22", 0], "vae": ["23", 0], "audio_vae": ["24", 0], "ref_images.ref_image_0": ["111", 3], "ref_videos.ref_video_0": ["55", 0], "ref_video_audios.ref_video_audio_0": ["102", 0]}, "class_type": "MiniMaxH3ReferenceToVideo", "_meta": {"title": "Pass 1: reference conditioning"}},+ "61": {"inputs": {"source": ["111", 2]}, "class_type": "PreviewAny", "_meta": {"title": "Pass 1: clip prompt (what H3 gets)"}},+ "63": {"inputs": {"grow": 0, "keep_audio": true, "megapixels": 0, "vae": ["23", 0], "images": ["50", 0], "mask": ["54", 0], "audio_vae": ["24", 0], "audio": ["50", 1]}, "class_type": "MiniMaxH3FantasticVideoEditLatent", "_meta": {"title": "Pass 1: edit latent (source + mask + audio)"}},+ "64": {"inputs": {"mask_meaning": "white = generate", "audio_mode": "preserve source audio", "mask_conversion": "H3 exact (causal/token max)", "target_latent": ["63", 0], "mask": ["54", 0]}, "class_type": "MiniMaxH3ContexMaskedTarget", "_meta": {"title": "Pass 1: masked target"}},+ "65": {"inputs": {"state": ["32", 0], "conditioning": ["59", 0], "vae": ["23", 0], "latent": ["64", 0], "audio_vae": ["24", 0]}, "class_type": "MiniMaxH3ChainContext", "_meta": {"title": "Pass 1: apply clip context"}},+ "66": {"inputs": {"model": ["21", 0], "conditioning": ["65", 0]}, "class_type": "BasicGuider", "_meta": {"title": "Pass 1: guider"}},+ "67": {"inputs": {"scheduler": "simple", "steps": ["32", 7], "denoise": 1, "model": ["21", 0]}, "class_type": "BasicScheduler", "_meta": {"title": "Pass 1: schedule"}},+ "68": {"inputs": {"noise": ["51", 0], "guider": ["66", 0], "sampler": ["25", 0], "sigmas": ["67", 0], "latent_image": ["65", 3]}, "class_type": "SamplerCustomAdvanced", "_meta": {"title": "Pass 1: sample video + audio"}},+ "69": {"inputs": {"samples": ["68", 0], "vae": ["23", 0]}, "class_type": "VAEDecode", "_meta": {"title": "Pass 1: decode video"}},+ "70": {"inputs": {"upscale_method": "area", "largest_size": 512, "image": ["69", 0]}, "class_type": "ImageScaleToMaxDimension", "_meta": {"title": "Pass 2: motion reference from pass 1 (~512 px)"}},+ "71": {"inputs": {"headroom": 0.1, "hold_frames": 2, "subject": 0, "track_data": ["112", 0], "labels": ["112", 1]}, "class_type": "H3RoleMask", "_meta": {"title": "Pass 2: labelled people (+ headroom)"}},+ "72": {"inputs": {"headroom": 0, "hold_frames": 1, "subject": 0, "track_data": ["112", 0], "labels": ["112", 1]}, "class_type": "H3RoleMask", "_meta": {"title": "Pass 2: labelled people, no headroom (mannequin)"}},+ "73": {"inputs": {"expand": 40, "incremental_expandrate": 0, "tapered_corners": true, "flip_input": false, "blur_radius": 0, "lerp_alpha": 1, "decay_factor": 1, "fill_holes": true, "mask": ["77", 0]}, "class_type": "GrowMaskWithBlur", "_meta": {"title": "Pass 2: grow mask (~4% of the frame)"}},+ "74": {"inputs": {"blur": 0.04, "face_blur": 0.05, "images": ["70", 0], "mask": ["72", 0], "size": ["72", 1], "faces": ["100", 0], "face_size": ["100", 1]}, "class_type": "H3GreyMannequins", "_meta": {"title": "Pass 2: hide them (grey mannequins)"}},+ "75": {"inputs": {"expand": 60, "incremental_expandrate": 0, "tapered_corners": true, "flip_input": false, "blur_radius": 0, "lerp_alpha": 1, "decay_factor": 1, "fill_holes": true, "mask": ["71", 0]}, "class_type": "GrowMaskWithBlur", "_meta": {"title": "Pass 2: hat search area (person + 60 px)"}},+ "76": {"inputs": {"x": 0, "y": 0, "operation": "multiply", "destination": ["49", 0], "source": ["75", 0]}, "class_type": "MaskComposite", "_meta": {"title": "Pass 2: only their hats"}},+ "77": {"inputs": {"x": 0, "y": 0, "operation": "add", "destination": ["71", 0], "source": ["76", 0]}, "class_type": "MaskComposite", "_meta": {"title": "Pass 2: people + hats"}},+ "78": {"inputs": {"prompt": ["112", 2], "width": ["32", 8], "height": ["32", 9], "length": ["32", 6], "ref_image_size": "match", "clip": ["22", 0], "vae": ["23", 0], "audio_vae": ["24", 0], "ref_images.ref_image_0": ["112", 3], "ref_videos.ref_video_0": ["74", 0], "ref_video_audios.ref_video_audio_0": ["102", 0]}, "class_type": "MiniMaxH3ReferenceToVideo", "_meta": {"title": "Pass 2: reference conditioning"}},+ "82": {"inputs": {"grow": 0, "keep_audio": true, "megapixels": 0, "vae": ["23", 0], "images": ["69", 0], "mask": ["73", 0], "audio_vae": ["24", 0], "audio": ["50", 1]}, "class_type": "MiniMaxH3FantasticVideoEditLatent", "_meta": {"title": "Pass 2: edit latent (source + mask + audio)"}},+ "83": {"inputs": {"mask_meaning": "white = generate", "audio_mode": "preserve source audio", "mask_conversion": "H3 exact (causal/token max)", "target_latent": ["82", 0], "mask": ["73", 0]}, "class_type": "MiniMaxH3ContexMaskedTarget", "_meta": {"title": "Pass 2: masked target"}},+ "84": {"inputs": {"state": ["32", 0], "conditioning": ["78", 0], "vae": ["23", 0], "latent": ["83", 0], "audio_vae": ["24", 0]}, "class_type": "MiniMaxH3ChainContext", "_meta": {"title": "Pass 2: apply clip context"}},+ "85": {"inputs": {"model": ["21", 0], "conditioning": ["84", 0]}, "class_type": "BasicGuider", "_meta": {"title": "Pass 2: guider"}},+ "86": {"inputs": {"scheduler": "simple", "steps": ["32", 7], "denoise": 1, "model": ["21", 0]}, "class_type": "BasicScheduler", "_meta": {"title": "Pass 2: schedule"}},+ "87": {"inputs": {"noise": ["51", 0], "guider": ["85", 0], "sampler": ["25", 0], "sigmas": ["86", 0], "latent_image": ["84", 3]}, "class_type": "SamplerCustomAdvanced", "_meta": {"title": "Pass 2: sample video + audio"}},+ "88": {"inputs": {"samples": ["113", 0], "vae": ["23", 0]}, "class_type": "VAEDecode", "_meta": {"title": "Decode video"}},+ "89": {"inputs": {"cell_selection": "runtime exact (latent max)", "cell_adjust": 0, "preview_frame": 0, "overlay_opacity": 0.38, "show_grid": true, "show_source_outline": true, "mask_meaning": "white = generate", "image": ["50", 0], "mask": ["54", 0]}, "class_type": "MiniMaxH3ContexMaskGridPreview", "_meta": {"title": "Mask preview"}},+ "90": {"inputs": {"images": ["89", 1]}, "class_type": "PreviewImage", "_meta": {"title": "Mask preview (white = regenerated)"}},+ "91": {"inputs": {"samples": ["113", 0], "vae": ["24", 0]}, "class_type": "VAEDecodeAudio", "_meta": {"title": "Decode Audio"}},+ "92": {"inputs": {"trim_frames": ["65", 1], "fps": 24, "match_tail": true, "audio_trim_mode": "sync_with_video", "images": ["88", 0], "audio": ["91", 0], "state": ["32", 0]}, "class_type": "MiniMaxH3LoopTrim", "_meta": {"title": "Trim Overlap"}},+ "93": {"inputs": {"state": ["32", 0], "images": ["92", 0], "sampled_latent": ["113", 0], "audio": ["92", 1], "images_with_overlap": ["92", 2]}, "class_type": "MiniMaxH3ChainSegmentSave", "_meta": {"title": "Save Clip + Checkpoint"}},+ "94": {"inputs": {"enabled": false, "play_notification_sound": false, "auto_continue_timeout_minutes": 0, "unload_models_while_waiting": false, "assemble_partial_on_stop": true, "partial_audio_source": "checkpointed", "candidate_count": 1, "review_each_candidate": false, "state": ["32", 0], "segment": ["93", 0], "audio": ["92", 1]}, "class_type": "MiniMaxH3ChainReview", "_meta": {"title": "Review (off = no pause)"}},+ "95": {"inputs": {"between_scene_cleanup": "off", "execution_mode": "recursive", "flow": ["31", 0], "state": ["32", 0], "images": ["92", 0], "sampled_latent": ["113", 0], "segment": ["94", 0]}, "class_type": "MiniMaxH3ChainLoopEnd", "_meta": {"title": "Loop End"}},+ "96": {"inputs": {"audio_source": "source", "filename": "montage_swap_%date:yyyy-MM-dd%", "audio_bitrate": 256, "copy_to_output": true, "output_subfolder": "h3_oneclick_swap", "blend_schedule": "plan", "boundary_tone_match": "off", "color_stabilization": "off", "delete_checkpoints_after_assembly": false, "generated_audio_join": "av_overlap", "manifest": ["95", 0]}, "class_type": "MiniMaxH3ChainAssemble", "_meta": {"title": "Final Video"}},+ "97": {"inputs": {"value": true}, "class_type": "PrimitiveBoolean", "_meta": {"title": "No lip-sync in the prompt"}},+ "98": {"inputs": {"text": "face", "clip": ["26", 1]}, "class_type": "CLIPTextEncode", "_meta": {"title": "SAM: faces"}},+ "99": {"inputs": {"detection_threshold": 0.2, "max_objects": 0, "detect_interval": 1, "images": ["37", 0], "model": ["26", 0], "conditioning": ["98", 0]}, "class_type": "SAM3_VideoTrack", "_meta": {"title": "SAM 3.1 track faces"}},+ "100": {"inputs": {"headroom": 0, "hold_frames": 0, "max_people": 0, "track_data": ["99", 0]}, "class_type": "H3PeopleMask", "_meta": {"title": "Faces (shown on the mannequins)"}},+ "101": {"inputs": {"volume": -100, "audio": ["9", 0]}, "class_type": "AudioAdjustVolume", "_meta": {"title": "Mute the clip audio H3 hears"}},+ "102": {"inputs": {"volume": -100, "audio": ["32", 12]}, "class_type": "AudioAdjustVolume", "_meta": {"title": "Mute the reference audio H3 hears"}},+ "103": {"inputs": {"image": "example.png"}, "class_type": "LoadImage", "_meta": {"title": "Picture 3"}},+ "104": {"inputs": {"upscale_method": "area", "largest_size": 768, "image": ["103", 0]}, "class_type": "ImageScaleToMaxDimension", "_meta": {"title": "Shrink for the describer"}},+ "105": {"inputs": {"prompt": "Describe the person in this picture for a casting sheet, in one sentence of about 40 words: apparent age and gender, hair, facial hair, skin tone, build, and their complete outfit including footwear. No preamble, no background.", "max_length": 120, "sampling_mode": "off", "thinking": false, "use_default_template": true, "mtp": "auto", "clip": ["16", 0], "image": ["104", 0]}, "class_type": "TextGenerate", "_meta": {"title": "Describe picture 3"}},+ "106": {"inputs": {"image": "example.png"}, "class_type": "LoadImage", "_meta": {"title": "Picture 4"}},+ "107": {"inputs": {"upscale_method": "area", "largest_size": 768, "image": ["106", 0]}, "class_type": "ImageScaleToMaxDimension", "_meta": {"title": "Shrink for the describer"}},+ "108": {"inputs": {"prompt": "Describe the person in this picture for a casting sheet, in one sentence of about 40 words: apparent age and gender, hair, facial hair, skin tone, build, and their complete outfit including footwear. No preamble, no background.", "max_length": 120, "sampling_mode": "off", "thinking": false, "use_default_template": true, "mtp": "auto", "clip": ["16", 0], "image": ["107", 0]}, "class_type": "TextGenerate", "_meta": {"title": "Describe picture 4"}},+ "109": {"inputs": {"file": ""}, "class_type": "H3MontageTracks", "_meta": {"title": "Tracks from Find people"}},+ "111": {"class_type": "H3MontagePass", "_meta": {"title": "Pass 1: this clip's person"}, "inputs": {"pass_number": 1, "montage": ["14", 4], "track_data": ["109", 0], "clip_index": ["32", 1], "picture_1": ["2", 0], "picture_2": ["5", 0], "picture_3": ["103", 0], "picture_4": ["106", 0]}},+ "112": {"class_type": "H3MontagePass", "_meta": {"title": "Pass 2: this clip's second person"}, "inputs": {"pass_number": 2, "montage": ["14", 4], "track_data": ["109", 0], "clip_index": ["32", 1], "picture_1": ["2", 0], "picture_2": ["5", 0], "picture_3": ["103", 0], "picture_4": ["106", 0]}},+ "113": {"class_type": "ComfySwitchNode", "_meta": {"title": "Pass 2 only where the piece has a second person"}, "inputs": {"switch": ["111", 4], "on_false": ["68", 0], "on_true": ["87", 0]}}+}diff --git a/workflows/people.api.json b/workflows/people.api.jsonnew file mode 100644index 0000000..273aedc--- /dev/null+++ b/workflows/people.api.json@@ -0,0 +1,12 @@+{+ "1": {"class_type": "VHS_LoadVideo", "_meta": {"title": "Source video"}, "inputs": {"video": "", "force_rate": 24, "custom_width": 0, "custom_height": 0, "frame_load_cap": 0, "skip_first_frames": 0, "select_every_nth": 1, "format": "None"}},+ "2": {"class_type": "CheckpointLoaderSimple", "_meta": {"title": "SAM 3.1"}, "inputs": {"ckpt_name": "sam3.1_multiplex_fp16.safetensors"}},+ "3": {"class_type": "CLIPTextEncode", "_meta": {"title": "SAM: people"}, "inputs": {"text": "person", "clip": ["2", 1]}},+ "4": {"class_type": "ImageScaleToMaxDimension", "_meta": {"title": "SAM video (1008 px, as the render)"}, "inputs": {"upscale_method": "area", "largest_size": 1008, "image": ["1", 0]}},+ "5": {"class_type": "SAM3_VideoTrack", "_meta": {"title": "SAM 3.1 track every person"}, "inputs": {"detection_threshold": 0.35, "max_objects": 0, "detect_interval": 1, "images": ["4", 0], "model": ["2", 0], "conditioning": ["3", 0]}},+ "6": {"class_type": "H3TrackCrops", "_meta": {"title": "Show each track per shot"}, "inputs": {"cut_threshold": 40, "max_crops": 32, "people": "<Subject 1>: the main person", "track_data": ["5", 0], "images": ["4", 0]}},+ "7": {"class_type": "CLIPLoader", "_meta": {"title": "Describer: Qwen3-VL 8B"}, "inputs": {"clip_name": "qwen3vl_8b_int8_convrot.safetensors", "type": "stable_diffusion", "device": "default"}},+ "8": {"class_type": "TextGenerate", "_meta": {"title": "Who is each track (Qwen3-VL)"}, "inputs": {"prompt": ["6", 1], "max_length": 1200, "sampling_mode": "off", "thinking": false, "use_default_template": true, "mtp": "auto", "clip": ["7", 0], "image": ["6", 0]}},+ "9": {"class_type": "H3TrackRoles", "_meta": {"title": "Read the track labels"}, "inputs": {"segments": ["6", 2], "answer": ["8", 0]}},+ "10": {"class_type": "H3MontagePeople", "_meta": {"title": "People per shot (for the piece list)"}, "inputs": {"track_data": ["5", 0], "crops": ["6", 0], "labels": ["9", 0]}}+}