Skip to content

One-click character swap for videos: an open-source, local Higgsfield Genjutsu alternative built on ComfyUI and MiniMax H3, with FLUX.3 Klein relighting.

Add moving-area previews for Add mode (#21)

mixbag0c7fa5faebcc
diff --git a/app/api/jobs/route.ts b/app/api/jobs/route.ts
index c66f7ff..8fc0d8b 100644
--- a/app/api/jobs/route.ts
+++ b/app/api/jobs/route.ts
@@ -1,6 +1,6 @@
import { apiAccess, HttpError, IMAGE_SIGNATURE, readFormData, StoredFile, VIDEO_SIGNATURE } from "../../../lib/http.ts";
import { NextResponse } from "next/server";
-import { listModels, submitAdvanced, submitColor, submitControl, submitFace, submitMask, submitMontage, submitObject, submitPeople, submitSimple } from "../../../lib/comfy.ts";
+import { listModels, submitAdvanced, submitColor, submitControl, submitFace, submitFit, submitMask, submitMontage, submitObject, submitPeople, submitSimple } from "../../../lib/comfy.ts";
import type { ControlSettings } from "../../../lib/control.ts";
import { TRACKS_FILE, type Label } from "../../../lib/montage.ts";
import { DEFAULT_TONE, DETAIL, TONE_MODES, TONE_RANGES, type ColorSettings, type ToneMode, type ToneSettings, type ToneValue } from "../../../lib/color.ts";
@@ -235,6 +235,11 @@ async function submitForm(form: FormData) {
if (mode === "mask") {
const { hatFix, margin, target } = parseMask(form);
submit = () => submitMask(video, section, hatFix, margin, target);
+ } else if (mode === "fit") {
+ const placement = parsePlacement(form.get("placement"));
+ const resolution = parseResolution(form);
+ const action = String(form.get("action") ?? "").replace(/\s+/g, " ").trim().slice(0, 300);
+ submit = () => submitFit(video, section, placement, resolution, action);
} else if (mode === "people") {
const { roles, resolution } = parsePeople(form);
submit = () => submitPeople(video, section, roles, resolution);
diff --git a/app/globals.css b/app/globals.css
index 7a2f651..1d864ca 100644
--- a/app/globals.css
+++ b/app/globals.css
@@ -162,6 +162,8 @@ input[type="range"] { accent-color: var(--accent); width: 220px; max-width: 100%
.swatch { display: inline-block; width: 12px; height: 12px; border-radius: 3px; }
.swatch.cyan { background: rgb(0, 200, 255); }
.swatch.orange { background: rgb(255, 140, 0); }
+.swatch.green { background: rgb(26, 204, 51); }
+.swatch.red { background: rgb(255, 26, 26); }
.file-name { min-width: 0; width: min(100%, 420px); }
.steps-input { width: 6rem; }
diff --git a/components/openjutsu.tsx b/components/openjutsu.tsx
index c5f3782..46753ba 100644
--- a/components/openjutsu.tsx
+++ b/components/openjutsu.tsx
@@ -8,7 +8,7 @@ import { clipCount, clipStarts } from "../lib/clips.ts";
import { CLIP_SECONDS, DEFAULT_TARGET, MARGINS, MAX_SEED, MIN_SECTION_SECONDS, MOTION_SIZES, RESOLUTIONS, CUSTOM_SIZE, canvasSize, type Margin, type MotionSize, type Resolution, type Section, type Size } from "../lib/simple.ts";
import { MAX_CHARACTERS } from "../lib/advanced.ts";
import { DEFAULT_MODEL, MAX_LORAS, STEPS, STRENGTH, type Lora } from "../lib/model.ts";
-import type { EditMode, PaintKeyframe, PlacementKeyframe } from "../lib/object.ts";
+import type { EditMode, FitSummary, PaintKeyframe, PlacementKeyframe } from "../lib/object.ts";
import { CONTROL_MAX_SECONDS } from "../lib/control.ts";
import { ObjectPlacement } from "./object-placement.tsx";
import { CHUNK_SECONDS, FACE_STYLES, faceChunkCount, type FaceStyle } from "../lib/face.ts";
@@ -18,7 +18,7 @@ import { appearances, renderLabels, type Appearance, type Kept, type Label, type
type Character = { file: File; url: string; role: string };
type Mode = "simple" | "advanced" | "object" | "post";
type Post = "face" | "color";
-type Kind = "simple" | "mask" | "advanced" | "advanced_maxq" | "object" | "control" | "face" | "color" | "people" | "montage";
+type Kind = "simple" | "mask" | "advanced" | "advanced_maxq" | "object" | "fit" | "control" | "face" | "color" | "people" | "montage";
type TonePreset = { name: string; settings: Record<string, unknown> };
// frames: the video, part and crop SAM saw
type Run = { id: string; kind: Kind; margin: Margin; clips: number; clipSeconds: number; startedAt: number; frames: string };
@@ -33,7 +33,7 @@ const HISTORY_KEY = "openjutsu.history";
const HISTORY_LENGTH = 20;
const KIND_LABELS: Record<Kind, string> = {
- simple: "Person swap", mask: "Mask check", advanced: "Advanced swap", advanced_maxq: "Advanced swap", object: "Object swap",
+ simple: "Person swap", mask: "Mask check", advanced: "Advanced swap", advanced_maxq: "Advanced swap", object: "Object swap", fit: "Fit check",
control: "ControlNet", face: "Fix faces", color: "Fix colours", people: "Find people", montage: "Montage",
};
const STATUS_LABELS: Record<string, string> = { succeeded: "Done", failed: "Failed", canceled: "Canceled", expired: "Expired" };
@@ -481,7 +481,7 @@ export function Openjutsu() {
}
async function submit(kind: Kind, review?: Review) {
- if (!video || (kind !== "mask" && kind !== "people" && !ready)) return;
+ if (!video || (kind === "fit" ? !placement.length : kind !== "mask" && kind !== "people" && !ready)) return;
setSubmitting(true);
setCheckingSetup(true);
setSetup(null);
@@ -521,6 +521,11 @@ export function Openjutsu() {
form.set("seed", String(seed));
form.set("clip_seconds", String(clipSeconds));
}
+ if (kind === "fit") {
+ form.set("action", action);
+ form.set("placement", JSON.stringify(placement.map((p) => ({ ...p, frame: p.frame - Math.round((whole ? 0 : part.start) * 24) }))));
+ form.set("resolution", resolutionField);
+ }
if (kind === "people") {
for (const c of shown) form.append("role", c.role);
form.set("resolution", resolutionField);
@@ -567,7 +572,7 @@ export function Openjutsu() {
form.set("seed", String(seed));
form.set("clip_seconds", String(clipSeconds));
}
- if (kind !== "mask" && kind !== "color" && kind !== "people") {
+ if (kind !== "mask" && kind !== "color" && kind !== "people" && kind !== "fit") {
form.set("motion", String(motion));
form.set("steps", String(steps));
form.set("model", model);
@@ -721,7 +726,8 @@ export function Openjutsu() {
<RunView
run={run} job={job} progress={progress} preview={preview} streamError={streamError} now={now}
before={video && same ? { url: video.url, start: whole ? 0 : part.start } : null}
- onCancel={cancel} onReset={reset} onFix={fixResult} onGenerate={video && same && character && !submitting && mode === "simple" ? () => submit("simple") : null}
+ onCancel={cancel} onReset={reset} onFix={fixResult} onGenerate={!video || !same || submitting ? null : mode === "simple" && character ? () => submit("simple") : run.kind === "fit" && ready ? () => submit("object") : null}
+ onUseBox={video && same ? (box) => { setPlacement([{ frame: Math.round((whole ? 0 : part.start) * 24), ...box }]); reset(); } : null}
margin={margin} onMargin={setMargin} onRecheck={video && !submitting ? () => submit("mask") : null}
photos={shown.map((c) => c.url)}
onMontage={video && character && !submitting ? (tracks, labels) => submit("montage", { tracks, labels }) : null} frames={frames}
@@ -1076,6 +1082,7 @@ export function Openjutsu() {
<div className="go">
<div className="row">
{mode === "simple" && <button className="secondary" disabled={!video || submitting} onClick={() => submit("mask")}>Check mask</button>}
+ {mode === "object" && editMode === "add" && <button className="secondary" disabled={!video || !placement.length || submitting} onClick={() => submit("fit")}>Check fit</button>}
{mode === "advanced" && montage ? (<>
<button className="secondary" disabled={!video || !character || submitting} onClick={() => submit("people")}>Find people</button>
<button className="primary" disabled={!video || !character || submitting} onClick={() => submit("montage")}>{submitting ? checkingSetup ? "Checking setup…" : "Uploading…" : "Generate"}</button>
@@ -1122,17 +1129,18 @@ export function Openjutsu() {
);
}
-function RunView({ run, job, progress, preview, streamError, now, before, onCancel, onReset, onFix, onGenerate, margin, onMargin, onRecheck, photos, onMontage, frames, onRetry }: {
+function RunView({ run, job, progress, preview, streamError, now, before, onCancel, onReset, onFix, onGenerate, onUseBox, margin, onMargin, onRecheck, photos, onMontage, frames, onRetry }: {
run: Run; job: PublicJob | null; progress: Progress | null; preview: string; streamError: string; now: number;
before: { url: string; start: number } | null;
onCancel: () => void; onReset: () => void; onFix: (post: Post) => void; onGenerate: (() => void) | null;
+ onUseBox: ((box: NonNullable<FitSummary["needed"]>) => void) | null;
margin: Margin; onMargin: (margin: Margin) => void; onRecheck: (() => void) | null;
photos: string[]; onMontage: ((tracks: string, labels: Label[]) => void) | null; frames: string; onRetry: (() => void) | null;
}) {
const status = job?.status ?? "queued";
const clips = Math.max(run.clips, progress?.clip ?? 1);
const people = run.kind === "people";
- const mask = run.kind === "mask" || people;
+ const mask = run.kind === "mask" || run.kind === "fit" || people;
if (people && status === "succeeded" && job?.data) {
return (
@@ -1152,6 +1160,10 @@ function RunView({ run, job, progress, preview, streamError, now, before, onCanc
);
}
+ if (run.kind === "fit" && status === "succeeded" && job?.video) {
+ return <FitCheck url={job.video.url} data={job.data} onUseBox={onUseBox} onGenerate={onGenerate} onBack={onReset} />;
+ }
+
if (status === "succeeded" && job?.video) {
return (
<section className="panel result">
@@ -1195,7 +1207,7 @@ function RunView({ run, job, progress, preview, streamError, now, before, onCanc
return (
<section className="panel progress">
<div className="row">
- <h2>{status === "queued" ? `Queued${job?.queuePosition ? ` (#${job.queuePosition})` : ""}` : people ? "Finding people" : mask ? "Checking the mask" : color ? "Fixing the colours"
+ <h2>{status === "queued" ? `Queued${job?.queuePosition ? ` (#${job.queuePosition})` : ""}` : people ? "Finding people" : run.kind === "fit" ? "Checking the fit" : mask ? "Checking the mask" : color ? "Fixing the colours"
: face ? (sampling ? `Chunk ${chunk} of up to ${run.clips}` : chunk ? "Pasting the face back" : "Tracking the face")
: progress?.clip ? `Clip ${progress.clip} of ${clips > 1 || run.kind === "montage" ? "about " : ""}${clips}` : "Preparing"}</h2>
<span className="time">{elapsed(now - run.startedAt)}</span>
@@ -1348,6 +1360,40 @@ function MaskCheck({ url, clipSeconds, checked, margin, onMargin, onRecheck, onG
);
}
+// Where the routine of the person the add copies leaves the box, before generating.
+function FitCheck({ url, data, onUseBox, onGenerate, onBack }: {
+ url: string; data: string | null; onUseBox: ((box: NonNullable<FitSummary["needed"]>) => void) | null; onGenerate: (() => void) | null; onBack: () => void;
+}) {
+ const [summary, setSummary] = useState<FitSummary | null>(null);
+ const [error, setError] = useState("");
+
+ useEffect(() => {
+ if (data) fetch(data).then((response) => response.json()).then(setSummary, () => setError("Could not read what the fit check found."));
+ }, [data]);
+
+ const misses = summary?.outside ?? [];
+ return (
+ <section className="panel mask-check">
+ <h2>Check the fit</h2>
+ <video src={url} controls loop autoPlay muted playsInline />
+ <p className="hint legend">
+ <i className="swatch green" /> predicted motion inside the edit area <i className="swatch red" /> outside it: possible clipping. Yellow: edit boundary.
+ </p>
+ {error && <p className="error">{error}</p>}
+ {summary && <p>{!summary.person ? "No one to copy was found in this part, so there is nothing to check the box against."
+ : misses.length ? `${summary.moving ? "Their predicted motion leaves the moving edit area" : "Their routine leaves the box"} at ${misses.map((o) => `${timecode(o.start / 24)}–${timecode(o.end / 24)} (${o.sides.join(", ")})`).join(", ")}.`
+ : summary.moving ? "Their predicted motion stays inside the moving edit area." : "Their whole routine fits inside your box."}</p>}
+ <p className="hint">{summary?.moving ? "The yellow boundary follows the copied routine and is the same moving edit area used for generation. Your box sets placement and scale. H3 can move differently, so this remains a guide." : "The check moves the person nearest your box into it, as tall as the box and standing on its bottom edge. H3 follows a routine loosely, so it is a guide, not a promise. A box much wider than one person can also bring back a copy of the dancer."}</p>
+ <div className="row">
+ {summary?.needed && misses.length > 0 && <button className="primary" disabled={!onUseBox} onClick={() => onUseBox?.(summary.needed!)}>Use a box that fits</button>}
+ <button className={misses.length ? "secondary" : "primary"} disabled={!onGenerate} onClick={() => onGenerate?.()}>Generate</button>
+ <button className="secondary" onClick={onBack}>Back to settings</button>
+ </div>
+ {!onGenerate && <p className="hint">To generate, load the video and part this check used and fill in the Add settings.</p>}
+ </section>
+ );
+}
+
// A montage without review keeps brief or small appearances as they are; say so instead of passing the swap off as complete.
function KeptNote({ url }: { url: string }) {
const [kept, setKept] = useState<Kept[]>([]);
diff --git a/lib/comfy.ts b/lib/comfy.ts
index 25389f9..c49ee0b 100644
--- a/lib/comfy.ts
+++ b/lib/comfy.ts
@@ -7,7 +7,7 @@ import { ComfyLow } from "@comfyorg/sdk/low";
import { MASK_NODES, NODES, cropSource, isLink, pruneMask, pruneSimple, setSection, type ApiGraph, type Margin, type Resolution, type Section, type SimpleSettings } from "./simple.ts";
import { VIDEO, nodeByTitle, pruneAdvanced, type AdvancedSettings } from "./advanced.ts";
import { applyModel, type ModelSettings } from "./model.ts";
-import { OBJECT_PICTURE, OBJECT_VIDEO, pruneObject, type ObjectSettings } from "./object.ts";
+import { OBJECT_PICTURE, OBJECT_VIDEO, pruneFit, pruneObject, type ObjectSettings, type PlacementKeyframe } from "./object.ts";
import { FACE_PICTURE, FACE_VIDEO, pruneFace, type FaceSettings } from "./face.ts";
import { COLOR_VIDEO, pruneColor, type ColorSettings } from "./color.ts";
import { CONTROL_PICTURE, CONTROL_VIDEO, pruneControl, setControlSection, type ControlSettings } from "./control.ts";
@@ -45,7 +45,7 @@ export function rawJobEvents(job: Job, signal: AbortSignal) {
return low().getJobEvents(job.urls.events || job.id, { signal });
}
-export const JOB_KINDS = ["simple", "mask", "advanced", "advanced_maxq", "object", "control", "face", "color", "people", "montage"] as const;
+export const JOB_KINDS = ["simple", "mask", "advanced", "advanced_maxq", "object", "fit", "control", "face", "color", "people", "montage"] as const;
export type JobKind = (typeof JOB_KINDS)[number];
export async function loadGraph(kind: JobKind): Promise<ApiGraph> {
@@ -185,6 +185,16 @@ export async function submitObject(video: StoredFile, section: Section, picture:
return submit(comfy, workflow);
}
+export async function submitFit(video: StoredFile, section: Section, placement: PlacementKeyframe[], resolution: Resolution, action = "") {
+ const comfy = client();
+ const graph = pruneFit(await loadGraph("fit"), placement, action);
+ if (typeof resolution === "object") cropSource(graph, nodeByTitle(graph, OBJECT_VIDEO), resolution);
+ setSection(graph, nodeByTitle(graph, OBJECT_VIDEO), section);
+ const workflow = comfy.workflows.fromJson(graph);
+ workflow.setInput(nodeByTitle(graph, OBJECT_VIDEO), "video", upload(comfy, video));
+ return submit(comfy, workflow);
+}
+
export async function submitControl(video: StoredFile, section: Section, picture: StoredFile, settings: ControlSettings, model: ModelSettings) {
const comfy = client();
const graph = pruneControl(await loadGraph("control"), settings);
diff --git a/lib/object.ts b/lib/object.ts
index a3b8cf1..35eab6a 100644
--- a/lib/object.ts
+++ b/lib/object.ts
@@ -2,7 +2,7 @@
// workflows/object.api.json is exported (workflows/build/oneclick_object_build.js) in change mode with a picture;
// each request prunes it. Nodes are found by title.
import { nodeByTitle } from "./advanced.ts";
-import { setMotionSize, setResolution, type ApiGraph, type MotionSize, type Resolution } from "./simple.ts";
+import { setMotionSize, setResolution, suffixRunName, type ApiGraph, type MotionSize, type Resolution } from "./simple.ts";
export const EDIT_MODES = ["change", "remove", "background", "add"] as const;
export type EditMode = (typeof EDIT_MODES)[number];
@@ -96,6 +96,8 @@ function describer(settings: ObjectSettings) {
};
}
+const copiesRoutine = (action: string) => /\b(?:in[ -]sync|same (?:dance )?routine|synchroni[sz](?:ed|e|ing))\b/i.test(action);
+
export function pruneObject(source: ApiGraph, picture: boolean, settings: ObjectSettings): ApiGraph {
const graph: ApiGraph = structuredClone(source);
const id = (title: string) => nodeByTitle(graph, title);
@@ -165,14 +167,46 @@ export function pruneObject(source: ApiGraph, picture: boolean, settings: Object
plan.change_to = addedObject(settings.changeTo);
plan.keep_outline = false;
plan.action = settings.action ?? "";
- graph.object_placement = { class_type: "H3ObjectPlacementMask", inputs: {
- images: [id("Source clip (frames + audio)"), 0], start_frame: [id("Clip start frame"), 1], placement,
- }, _meta: { title: "Object placement area" } };
+ const moving = copiesRoutine(settings.action ?? "");
+ const drop = ["SAM frames for this clip", "Tracked area", "Tracked area, tight (for the grey shape)", "Turn it grey (shape and movement only)"];
+ if (moving) {
+ graph[id("SAM: what to change")].inputs.text = "person";
+ graph[id("SAM 3.1 track it")].inputs.images = [id("SAM video (1008 px, SAM's own size)"), 0];
+ graph.object_moving_area = { class_type: "H3AddMovingArea", inputs: {
+ images: [id("SAM video (1008 px, SAM's own size)"), 0], track_data: [id("SAM 3.1 track it"), 0], placement, start_frame: 0,
+ }, _meta: { title: "Moving edit area (follows the copied routine)" } };
+ graph.object_placement = { class_type: "MiniMaxH3ContexLoopMaskSlice", inputs: {
+ mask: ["object_moving_area", 0], state: [id("Current Clip"), 0], source_fps: 24,
+ }, _meta: { title: "Moving edit area for this clip" } };
+ suffixRunName(graph, "moving_area_run_name", "_moving_area", "Run name for the moving edit area");
+ } else {
+ graph.object_placement = { class_type: "H3ObjectPlacementMask", inputs: {
+ images: [id("Source clip (frames + audio)"), 0], start_frame: [id("Clip start frame"), 1], placement,
+ }, _meta: { title: "Object placement area" } };
+ drop.push("SAM 3.1", "SAM: what to change", "SAM video (1008 px, SAM's own size)", "SAM 3.1 track it");
+ }
grow.mask = ["object_placement", 0];
- grow.expand = { tight: 0, normal: 2, loose: 4 }[settings.margin];
- for (const title of ["SAM 3.1", "SAM: what to change", "SAM video (1008 px, SAM's own size)", "SAM frames for this clip", "SAM 3.1 track it",
- "Tracked area", "Tracked area, tight (for the grey shape)", "Turn it grey (shape and movement only)"]) delete graph[id(title)];
+ grow.expand = moving ? 0 : { tight: 0, normal: 2, loose: 4 }[settings.margin];
+ for (const title of drop) delete graph[id(title)];
}
setResolution(graph, id("Object Swap Plan (what to change)"), id(OBJECT_VIDEO), settings.resolution);
return graph;
}
+
+// Fit check (workflows/fit.api.json): before an add that copies someone's routine, shows where that routine leaves the box.
+export function pruneFit(source: ApiGraph, placement: PlacementKeyframe[], action = ""): ApiGraph {
+ const graph: ApiGraph = structuredClone(source);
+ const fit = graph[nodeByTitle(graph, "Check the fit")].inputs;
+ fit.placement = JSON.stringify(parsePlacement(placement));
+ fit.moving_area = copiesRoutine(action);
+ return graph;
+}
+
+// What the fit check found, in frames from the start of the part.
+export type FitSummary = {
+ frames: number;
+ moving?: boolean;
+ person: boolean; // someone to copy was found
+ outside: { start: number; end: number; sides: ("left" | "right" | "top" | "bottom")[] }[];
+ needed: { x: number; y: number; width: number; height: number } | null; // the space their whole routine takes, scaled into the box
+};
diff --git a/lib/setup.ts b/lib/setup.ts
index 4e6c1f1..16d2546 100644
--- a/lib/setup.ts
+++ b/lib/setup.ts
@@ -17,6 +17,7 @@ export type SetupRequest =
| { kind: "color"; settings: ColorSettings }
| { kind: "mask"; hatFix: boolean; margin: Margin; target: string }
| { kind: "people"; resolution: Resolution }
+ | { kind: "fit"; resolution: Resolution }
| { kind: "montage"; settings: AdvancedSettings; pictures: number; review: Review | null; model: ModelSettings };
export type SetupResult = {
@@ -31,7 +32,7 @@ export type NodeRegistry = Record<string, NodeInfo>;
const PACKS = [
{ name: "ComfyUI-MiniMaxH3-Context-Loop (0.7.3+)", url: "https://github.com/ethanfel/ComfyUI-MiniMaxH3-Context-Loop", nodes: ["MiniMaxH3GenerationProfile", "MiniMaxH3SourceTimeline", "MiniMaxH3ChainPlanModern", "MiniMaxH3ChainLoopStart", "MiniMaxH3ChainCurrent", "MiniMaxH3ChainContext", "MiniMaxH3ChainSegmentSave", "MiniMaxH3ChainReview", "MiniMaxH3ChainLoopEnd", "MiniMaxH3ChainAssemble", "MiniMaxH3ContexLoopSourceSlice", "MiniMaxH3ContexLoopMaskSlice", "MiniMaxH3ContexMaskedTarget", "MiniMaxH3ContexMaskGridPreview", "MiniMaxH3LoopTrim"] },
- { name: "comfyui-h3-oneclick-swap", url: "https://github.com/D3voz/comfyui-h3-oneclick-swap", nodes: ["H3OneClickSwapPlan", "H3AdvancedSwapPlan", "H3PictureSwapPlan", "H3ObjectSwapPlan", "H3ObjectPlacementMask", "H3PaintedEditArea", "H3SoundtrackOrSilence", "H3PeopleMask", "H3GreyMannequins", "H3ClipShots", "H3FillShots", "H3TrackCrops", "H3TrackRoles", "H3RoleMask", "H3SubjectPrompt", "H3MontagePeople", "H3MontageTracks", "H3MontagePicks", "H3MontagePlan", "H3MontagePass", "H3FaceChunks", "H3FaceJoin"] },
+ { name: "comfyui-h3-oneclick-swap", url: "https://github.com/D3voz/comfyui-h3-oneclick-swap", nodes: ["H3OneClickSwapPlan", "H3AdvancedSwapPlan", "H3PictureSwapPlan", "H3ObjectSwapPlan", "H3ObjectPlacementMask", "H3AddFitCheck", "H3AddMovingArea", "H3PaintedEditArea", "H3SoundtrackOrSilence", "H3PeopleMask", "H3GreyMannequins", "H3ClipShots", "H3FillShots", "H3TrackCrops", "H3TrackRoles", "H3RoleMask", "H3SubjectPrompt", "H3MontagePeople", "H3MontageTracks", "H3MontagePicks", "H3MontagePlan", "H3MontagePass", "H3FaceChunks", "H3FaceJoin"] },
{ name: "ComfyUI-Fantastic-MiniMaxH3-PromptBuilder", url: "https://github.com/Adudeguyman/ComfyUI-Fantastic-MiniMaxH3-PromptBuilder", nodes: ["MiniMaxH3FantasticVideoEditLatent"] },
{ name: "ComfyUI-KJNodes", url: "https://github.com/kijai/ComfyUI-KJNodes", nodes: ["GrowMaskWithBlur", "ImageAndMaskPreview"] },
{ name: "ComfyUI-VideoHelperSuite", url: "https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite", nodes: ["VHS_LoadVideo", "VHS_SelectEveryNthImage", "VHS_VideoCombine"] },
@@ -63,7 +64,7 @@ export function setupGraph(source: ApiGraph, selection: SetupRequest): ApiGraph
else if (selection.kind === "color") graph = pruneColor(source, selection.settings);
else if (selection.kind === "mask") graph = pruneMask(source, selection.hatFix, selection.margin, selection.target);
else if (selection.kind === "montage") graph = pruneMontage(source, selection.pictures, selection.settings, selection.review);
- else if (selection.kind === "people") {
+ else if (selection.kind === "people" || selection.kind === "fit") {
graph = structuredClone(source);
if (typeof selection.resolution === "object") cropSource(graph, PEOPLE_NODES.video, selection.resolution);
} else graph = pruneAdvanced(source, selection.pictures, selection.settings);
diff --git a/test/object.test.ts b/test/object.test.ts
index 9327c25..6d28c32 100644
--- a/test/object.test.ts
+++ b/test/object.test.ts
@@ -3,7 +3,7 @@ import { readFileSync } from "node:fs";
import test from "node:test";
import { nodeByTitle } from "../lib/advanced.ts";
import { applyModel, DEFAULT_MODEL } from "../lib/model.ts";
-import { OBJECT_PICTURE, OBJECT_VIDEO, parsePaint, parsePlacement, placementAt, pruneObject, type ObjectSettings } from "../lib/object.ts";
+import { OBJECT_PICTURE, OBJECT_VIDEO, parsePaint, parsePlacement, placementAt, pruneFit, pruneObject, type ObjectSettings } from "../lib/object.ts";
import { isLink, type ApiGraph } from "../lib/simple.ts";
const graph: ApiGraph = JSON.parse(readFileSync(new URL("../workflows/object.api.json", import.meta.url), "utf8"));
@@ -168,3 +168,45 @@ test("paint rejects malformed and oversized runs at the boundary", () => {
[{ ...valid, runs: [63, 2] }], [{ ...valid, runs: [-1, 2] }], [{ ...valid, runs: [10, 5, 12, 2] }],
[{ ...valid, runs: [10, 1.5] }], [{ ...valid, runs: [10] }], Array(65).fill(valid), " ".repeat(512 * 1024 + 1)]) assert.throws(() => parsePaint(value));
});
+
+test("the fit check gets the placement and tracks people in the part", () => {
+ const fit: ApiGraph = JSON.parse(readFileSync(new URL("../workflows/fit.api.json", import.meta.url), "utf8"));
+ const placement = [{ frame: 0, x: 0.1, y: 0.05, width: 0.2, height: 0.9 }];
+ const pruned = pruneFit(fit, placement);
+ assert.deepEqual(dangling(pruned), []);
+ assert.equal(node(pruned, "Check the fit").inputs.placement, JSON.stringify(placement));
+ assert.equal(node(pruned, "SAM: people").inputs.text, "person");
+ assert.equal(node(fit, "Check the fit").inputs.placement, "", "the source graph is not modified");
+ assert.throws(() => pruneFit(fit, []), /placement area/);
+});
+
+
+test("an in-sync add tracks the full section and slices the moving area per clip", () => {
+ const placement = [{ frame: 0, x: 0.12, y: 0.03, width: 0.21, height: 0.97 }];
+ for (const action of ["she dances in sync with the other dancer", "copy the same routine", "synchronized dancing"]) {
+ const pruned = pruneObject(graph, true, { ...settings("add"), placement, action });
+ assert.deepEqual(dangling(pruned), []);
+ assert.equal(node(pruned, "SAM: what to change").inputs.text, "person");
+ assert.deepEqual(node(pruned, "SAM 3.1 track it").inputs.images, [nodeByTitle(pruned, "SAM video (1008 px, SAM's own size)"), 0]);
+ assert.equal(pruned.object_moving_area.class_type, "H3AddMovingArea");
+ assert.equal(pruned.object_moving_area.inputs.placement, JSON.stringify(placement));
+ assert.equal(pruned.object_moving_area.inputs.start_frame, 0);
+ assert.equal(node(pruned, "Grow mask").inputs.expand, 0);
+ assert.equal(pruned.object_placement.class_type, "MiniMaxH3ContexLoopMaskSlice");
+ assert.deepEqual(pruned.object_placement.inputs.mask, ["object_moving_area", 0]);
+ assert.deepEqual(pruned.object_placement.inputs.state, [nodeByTitle(pruned, "Current Clip"), 0]);
+ assert.deepEqual(node(pruned, "Grow mask").inputs.mask, ["object_placement", 0]);
+ assert.deepEqual(node(pruned, "Reference Conditioning").inputs["ref_images.ref_image_0"], [nodeByTitle(pruned, OBJECT_PICTURE), 0]);
+ assert.equal(pruned.moving_area_run_name.inputs.string_b, "_moving_area");
+ }
+ const custom = pruneObject(graph, true, { ...settings("add"), placement, action: "dance in sync", resolution: { width: 640, height: 960 } });
+ assert.deepEqual(node(custom, "SAM video (1008 px, SAM's own size)").inputs.image, ["custom_size", 0]);
+});
+
+
+test("the fit preview selects the same moving area as synchronized generation", () => {
+ const fit: ApiGraph = JSON.parse(readFileSync(new URL("../workflows/fit.api.json", import.meta.url), "utf8"));
+ const placement = [{ frame: 0, x: 0.12, y: 0.03, width: 0.21, height: 0.97 }];
+ assert.equal(node(pruneFit(fit, placement, "dance in sync"), "Check the fit").inputs.moving_area, true);
+ assert.equal(node(pruneFit(fit, placement, "steam rises"), "Check the fit").inputs.moving_area, false);
+});
diff --git a/workflows/fit.api.json b/workflows/fit.api.json
new file mode 100644
index 0000000..d3fedfb
--- /dev/null
+++ b/workflows/fit.api.json
@@ -0,0 +1,37 @@
+{
+ "1": {
+ "inputs": { "video": "", "force_rate": 24, "custom_width": 0, "custom_height": 0, "frame_load_cap": 0, "skip_first_frames": 0, "select_every_nth": 1, "format": "None" },
+ "class_type": "VHS_LoadVideo",
+ "_meta": { "title": "Source video" }
+ },
+ "2": {
+ "inputs": { "ckpt_name": "sam3.1_multiplex_fp16.safetensors" },
+ "class_type": "CheckpointLoaderSimple",
+ "_meta": { "title": "SAM 3.1" }
+ },
+ "3": {
+ "inputs": { "text": "person", "clip": ["2", 1] },
+ "class_type": "CLIPTextEncode",
+ "_meta": { "title": "SAM: people" }
+ },
+ "4": {
+ "inputs": { "upscale_method": "area", "largest_size": 1008, "image": ["1", 0] },
+ "class_type": "ImageScaleToMaxDimension",
+ "_meta": { "title": "SAM video (1008 px)" }
+ },
+ "5": {
+ "inputs": { "detection_threshold": 0.35, "max_objects": 0, "detect_interval": 1, "images": ["4", 0], "model": ["2", 0], "conditioning": ["3", 0] },
+ "class_type": "SAM3_VideoTrack",
+ "_meta": { "title": "SAM 3.1 track every person" }
+ },
+ "6": {
+ "inputs": { "placement": "", "start_frame": 0, "images": ["4", 0], "track_data": ["5", 0] },
+ "class_type": "H3AddFitCheck",
+ "_meta": { "title": "Check the fit" }
+ },
+ "7": {
+ "inputs": { "frame_rate": 24, "loop_count": 0, "filename_prefix": "openjutsu_fit/fit", "format": "video/h264-mp4", "pix_fmt": "yuv420p", "crf": 19, "save_metadata": false, "trim_to_audio": false, "pingpong": false, "save_output": false, "images": ["6", 0] },
+ "class_type": "VHS_VideoCombine",
+ "_meta": { "title": "Fit preview video" }
+ }
+}