One-click character swap for videos: an open-source, local Higgsfield Genjutsu alternative built on ComfyUI and MiniMax H3, with FLUX.3 Klein relighting.
Add moving-area previews for Add mode (#21)
mixbag0c7fa5faebcc
diff --git a/app/api/jobs/route.ts b/app/api/jobs/route.tsindex c66f7ff..8fc0d8b 100644--- a/app/api/jobs/route.ts+++ b/app/api/jobs/route.ts@@ -1,6 +1,6 @@import { apiAccess, HttpError, IMAGE_SIGNATURE, readFormData, StoredFile, VIDEO_SIGNATURE } from "../../../lib/http.ts";import { NextResponse } from "next/server";-import { listModels, submitAdvanced, submitColor, submitControl, submitFace, submitMask, submitMontage, submitObject, submitPeople, submitSimple } from "../../../lib/comfy.ts";+import { listModels, submitAdvanced, submitColor, submitControl, submitFace, submitFit, submitMask, submitMontage, submitObject, submitPeople, submitSimple } from "../../../lib/comfy.ts";import type { ControlSettings } from "../../../lib/control.ts";import { TRACKS_FILE, type Label } from "../../../lib/montage.ts";import { DEFAULT_TONE, DETAIL, TONE_MODES, TONE_RANGES, type ColorSettings, type ToneMode, type ToneSettings, type ToneValue } from "../../../lib/color.ts";@@ -235,6 +235,11 @@ async function submitForm(form: FormData) {if (mode === "mask") {const { hatFix, margin, target } = parseMask(form);submit = () => submitMask(video, section, hatFix, margin, target);+ } else if (mode === "fit") {+ const placement = parsePlacement(form.get("placement"));+ const resolution = parseResolution(form);+ const action = String(form.get("action") ?? "").replace(/\s+/g, " ").trim().slice(0, 300);+ submit = () => submitFit(video, section, placement, resolution, action);} else if (mode === "people") {const { roles, resolution } = parsePeople(form);submit = () => submitPeople(video, section, roles, resolution);diff --git a/app/globals.css b/app/globals.cssindex 7a2f651..1d864ca 100644--- a/app/globals.css+++ b/app/globals.css@@ -162,6 +162,8 @@ input[type="range"] { accent-color: var(--accent); width: 220px; max-width: 100%.swatch { display: inline-block; width: 12px; height: 12px; border-radius: 3px; }.swatch.cyan { background: rgb(0, 200, 255); }.swatch.orange { background: rgb(255, 140, 0); }+.swatch.green { background: rgb(26, 204, 51); }+.swatch.red { background: rgb(255, 26, 26); }.file-name { min-width: 0; width: min(100%, 420px); }.steps-input { width: 6rem; }diff --git a/components/openjutsu.tsx b/components/openjutsu.tsxindex c5f3782..46753ba 100644--- a/components/openjutsu.tsx+++ b/components/openjutsu.tsx@@ -8,7 +8,7 @@ import { clipCount, clipStarts } from "../lib/clips.ts";import { CLIP_SECONDS, DEFAULT_TARGET, MARGINS, MAX_SEED, MIN_SECTION_SECONDS, MOTION_SIZES, RESOLUTIONS, CUSTOM_SIZE, canvasSize, type Margin, type MotionSize, type Resolution, type Section, type Size } from "../lib/simple.ts";import { MAX_CHARACTERS } from "../lib/advanced.ts";import { DEFAULT_MODEL, MAX_LORAS, STEPS, STRENGTH, type Lora } from "../lib/model.ts";-import type { EditMode, PaintKeyframe, PlacementKeyframe } from "../lib/object.ts";+import type { EditMode, FitSummary, PaintKeyframe, PlacementKeyframe } from "../lib/object.ts";import { CONTROL_MAX_SECONDS } from "../lib/control.ts";import { ObjectPlacement } from "./object-placement.tsx";import { CHUNK_SECONDS, FACE_STYLES, faceChunkCount, type FaceStyle } from "../lib/face.ts";@@ -18,7 +18,7 @@ import { appearances, renderLabels, type Appearance, type Kept, type Label, typetype Character = { file: File; url: string; role: string };type Mode = "simple" | "advanced" | "object" | "post";type Post = "face" | "color";-type Kind = "simple" | "mask" | "advanced" | "advanced_maxq" | "object" | "control" | "face" | "color" | "people" | "montage";+type Kind = "simple" | "mask" | "advanced" | "advanced_maxq" | "object" | "fit" | "control" | "face" | "color" | "people" | "montage";type TonePreset = { name: string; settings: Record<string, unknown> };// frames: the video, part and crop SAM sawtype Run = { id: string; kind: Kind; margin: Margin; clips: number; clipSeconds: number; startedAt: number; frames: string };@@ -33,7 +33,7 @@ const HISTORY_KEY = "openjutsu.history";const HISTORY_LENGTH = 20;const KIND_LABELS: Record<Kind, string> = {- simple: "Person swap", mask: "Mask check", advanced: "Advanced swap", advanced_maxq: "Advanced swap", object: "Object swap",+ simple: "Person swap", mask: "Mask check", advanced: "Advanced swap", advanced_maxq: "Advanced swap", object: "Object swap", fit: "Fit check",control: "ControlNet", face: "Fix faces", color: "Fix colours", people: "Find people", montage: "Montage",};const STATUS_LABELS: Record<string, string> = { succeeded: "Done", failed: "Failed", canceled: "Canceled", expired: "Expired" };@@ -481,7 +481,7 @@ export function Openjutsu() {}async function submit(kind: Kind, review?: Review) {- if (!video || (kind !== "mask" && kind !== "people" && !ready)) return;+ if (!video || (kind === "fit" ? !placement.length : kind !== "mask" && kind !== "people" && !ready)) return;setSubmitting(true);setCheckingSetup(true);setSetup(null);@@ -521,6 +521,11 @@ export function Openjutsu() {form.set("seed", String(seed));form.set("clip_seconds", String(clipSeconds));}+ if (kind === "fit") {+ form.set("action", action);+ form.set("placement", JSON.stringify(placement.map((p) => ({ ...p, frame: p.frame - Math.round((whole ? 0 : part.start) * 24) }))));+ form.set("resolution", resolutionField);+ }if (kind === "people") {for (const c of shown) form.append("role", c.role);form.set("resolution", resolutionField);@@ -567,7 +572,7 @@ export function Openjutsu() {form.set("seed", String(seed));form.set("clip_seconds", String(clipSeconds));}- if (kind !== "mask" && kind !== "color" && kind !== "people") {+ if (kind !== "mask" && kind !== "color" && kind !== "people" && kind !== "fit") {form.set("motion", String(motion));form.set("steps", String(steps));form.set("model", model);@@ -721,7 +726,8 @@ export function Openjutsu() {<RunViewrun={run} job={job} progress={progress} preview={preview} streamError={streamError} now={now}before={video && same ? { url: video.url, start: whole ? 0 : part.start } : null}- onCancel={cancel} onReset={reset} onFix={fixResult} onGenerate={video && same && character && !submitting && mode === "simple" ? () => submit("simple") : null}+ onCancel={cancel} onReset={reset} onFix={fixResult} onGenerate={!video || !same || submitting ? null : mode === "simple" && character ? () => submit("simple") : run.kind === "fit" && ready ? () => submit("object") : null}+ onUseBox={video && same ? (box) => { setPlacement([{ frame: Math.round((whole ? 0 : part.start) * 24), ...box }]); reset(); } : null}margin={margin} onMargin={setMargin} onRecheck={video && !submitting ? () => submit("mask") : null}photos={shown.map((c) => c.url)}onMontage={video && character && !submitting ? (tracks, labels) => submit("montage", { tracks, labels }) : null} frames={frames}@@ -1076,6 +1082,7 @@ export function Openjutsu() {<div className="go"><div className="row">{mode === "simple" && <button className="secondary" disabled={!video || submitting} onClick={() => submit("mask")}>Check mask</button>}+ {mode === "object" && editMode === "add" && <button className="secondary" disabled={!video || !placement.length || submitting} onClick={() => submit("fit")}>Check fit</button>}{mode === "advanced" && montage ? (<><button className="secondary" disabled={!video || !character || submitting} onClick={() => submit("people")}>Find people</button><button className="primary" disabled={!video || !character || submitting} onClick={() => submit("montage")}>{submitting ? checkingSetup ? "Checking setup…" : "Uploading…" : "Generate"}</button>@@ -1122,17 +1129,18 @@ export function Openjutsu() {);}-function RunView({ run, job, progress, preview, streamError, now, before, onCancel, onReset, onFix, onGenerate, margin, onMargin, onRecheck, photos, onMontage, frames, onRetry }: {+function RunView({ run, job, progress, preview, streamError, now, before, onCancel, onReset, onFix, onGenerate, onUseBox, margin, onMargin, onRecheck, photos, onMontage, frames, onRetry }: {run: Run; job: PublicJob | null; progress: Progress | null; preview: string; streamError: string; now: number;before: { url: string; start: number } | null;onCancel: () => void; onReset: () => void; onFix: (post: Post) => void; onGenerate: (() => void) | null;+ onUseBox: ((box: NonNullable<FitSummary["needed"]>) => void) | null;margin: Margin; onMargin: (margin: Margin) => void; onRecheck: (() => void) | null;photos: string[]; onMontage: ((tracks: string, labels: Label[]) => void) | null; frames: string; onRetry: (() => void) | null;}) {const status = job?.status ?? "queued";const clips = Math.max(run.clips, progress?.clip ?? 1);const people = run.kind === "people";- const mask = run.kind === "mask" || people;+ const mask = run.kind === "mask" || run.kind === "fit" || people;if (people && status === "succeeded" && job?.data) {return (@@ -1152,6 +1160,10 @@ function RunView({ run, job, progress, preview, streamError, now, before, onCanc);}+ if (run.kind === "fit" && status === "succeeded" && job?.video) {+ return <FitCheck url={job.video.url} data={job.data} onUseBox={onUseBox} onGenerate={onGenerate} onBack={onReset} />;+ }+if (status === "succeeded" && job?.video) {return (<section className="panel result">@@ -1195,7 +1207,7 @@ function RunView({ run, job, progress, preview, streamError, now, before, onCancreturn (<section className="panel progress"><div className="row">- <h2>{status === "queued" ? `Queued${job?.queuePosition ? ` (#${job.queuePosition})` : ""}` : people ? "Finding people" : mask ? "Checking the mask" : color ? "Fixing the colours"+ <h2>{status === "queued" ? `Queued${job?.queuePosition ? ` (#${job.queuePosition})` : ""}` : people ? "Finding people" : run.kind === "fit" ? "Checking the fit" : mask ? "Checking the mask" : color ? "Fixing the colours": face ? (sampling ? `Chunk ${chunk} of up to ${run.clips}` : chunk ? "Pasting the face back" : "Tracking the face"): progress?.clip ? `Clip ${progress.clip} of ${clips > 1 || run.kind === "montage" ? "about " : ""}${clips}` : "Preparing"}</h2><span className="time">{elapsed(now - run.startedAt)}</span>@@ -1348,6 +1360,40 @@ function MaskCheck({ url, clipSeconds, checked, margin, onMargin, onRecheck, onG);}+// Where the routine of the person the add copies leaves the box, before generating.+function FitCheck({ url, data, onUseBox, onGenerate, onBack }: {+ url: string; data: string | null; onUseBox: ((box: NonNullable<FitSummary["needed"]>) => void) | null; onGenerate: (() => void) | null; onBack: () => void;+}) {+ const [summary, setSummary] = useState<FitSummary | null>(null);+ const [error, setError] = useState("");++ useEffect(() => {+ if (data) fetch(data).then((response) => response.json()).then(setSummary, () => setError("Could not read what the fit check found."));+ }, [data]);++ const misses = summary?.outside ?? [];+ return (+ <section className="panel mask-check">+ <h2>Check the fit</h2>+ <video src={url} controls loop autoPlay muted playsInline />+ <p className="hint legend">+ <i className="swatch green" /> predicted motion inside the edit area <i className="swatch red" /> outside it: possible clipping. Yellow: edit boundary.+ </p>+ {error && <p className="error">{error}</p>}+ {summary && <p>{!summary.person ? "No one to copy was found in this part, so there is nothing to check the box against."+ : misses.length ? `${summary.moving ? "Their predicted motion leaves the moving edit area" : "Their routine leaves the box"} at ${misses.map((o) => `${timecode(o.start / 24)}–${timecode(o.end / 24)} (${o.sides.join(", ")})`).join(", ")}.`+ : summary.moving ? "Their predicted motion stays inside the moving edit area." : "Their whole routine fits inside your box."}</p>}+ <p className="hint">{summary?.moving ? "The yellow boundary follows the copied routine and is the same moving edit area used for generation. Your box sets placement and scale. H3 can move differently, so this remains a guide." : "The check moves the person nearest your box into it, as tall as the box and standing on its bottom edge. H3 follows a routine loosely, so it is a guide, not a promise. A box much wider than one person can also bring back a copy of the dancer."}</p>+ <div className="row">+ {summary?.needed && misses.length > 0 && <button className="primary" disabled={!onUseBox} onClick={() => onUseBox?.(summary.needed!)}>Use a box that fits</button>}+ <button className={misses.length ? "secondary" : "primary"} disabled={!onGenerate} onClick={() => onGenerate?.()}>Generate</button>+ <button className="secondary" onClick={onBack}>Back to settings</button>+ </div>+ {!onGenerate && <p className="hint">To generate, load the video and part this check used and fill in the Add settings.</p>}+ </section>+ );+}+// A montage without review keeps brief or small appearances as they are; say so instead of passing the swap off as complete.function KeptNote({ url }: { url: string }) {const [kept, setKept] = useState<Kept[]>([]);diff --git a/lib/comfy.ts b/lib/comfy.tsindex 25389f9..c49ee0b 100644--- a/lib/comfy.ts+++ b/lib/comfy.ts@@ -7,7 +7,7 @@ import { ComfyLow } from "@comfyorg/sdk/low";import { MASK_NODES, NODES, cropSource, isLink, pruneMask, pruneSimple, setSection, type ApiGraph, type Margin, type Resolution, type Section, type SimpleSettings } from "./simple.ts";import { VIDEO, nodeByTitle, pruneAdvanced, type AdvancedSettings } from "./advanced.ts";import { applyModel, type ModelSettings } from "./model.ts";-import { OBJECT_PICTURE, OBJECT_VIDEO, pruneObject, type ObjectSettings } from "./object.ts";+import { OBJECT_PICTURE, OBJECT_VIDEO, pruneFit, pruneObject, type ObjectSettings, type PlacementKeyframe } from "./object.ts";import { FACE_PICTURE, FACE_VIDEO, pruneFace, type FaceSettings } from "./face.ts";import { COLOR_VIDEO, pruneColor, type ColorSettings } from "./color.ts";import { CONTROL_PICTURE, CONTROL_VIDEO, pruneControl, setControlSection, type ControlSettings } from "./control.ts";@@ -45,7 +45,7 @@ export function rawJobEvents(job: Job, signal: AbortSignal) {return low().getJobEvents(job.urls.events || job.id, { signal });}-export const JOB_KINDS = ["simple", "mask", "advanced", "advanced_maxq", "object", "control", "face", "color", "people", "montage"] as const;+export const JOB_KINDS = ["simple", "mask", "advanced", "advanced_maxq", "object", "fit", "control", "face", "color", "people", "montage"] as const;export type JobKind = (typeof JOB_KINDS)[number];export async function loadGraph(kind: JobKind): Promise<ApiGraph> {@@ -185,6 +185,16 @@ export async function submitObject(video: StoredFile, section: Section, picture:return submit(comfy, workflow);}+export async function submitFit(video: StoredFile, section: Section, placement: PlacementKeyframe[], resolution: Resolution, action = "") {+ const comfy = client();+ const graph = pruneFit(await loadGraph("fit"), placement, action);+ if (typeof resolution === "object") cropSource(graph, nodeByTitle(graph, OBJECT_VIDEO), resolution);+ setSection(graph, nodeByTitle(graph, OBJECT_VIDEO), section);+ const workflow = comfy.workflows.fromJson(graph);+ workflow.setInput(nodeByTitle(graph, OBJECT_VIDEO), "video", upload(comfy, video));+ return submit(comfy, workflow);+}+export async function submitControl(video: StoredFile, section: Section, picture: StoredFile, settings: ControlSettings, model: ModelSettings) {const comfy = client();const graph = pruneControl(await loadGraph("control"), settings);diff --git a/lib/object.ts b/lib/object.tsindex a3b8cf1..35eab6a 100644--- a/lib/object.ts+++ b/lib/object.ts@@ -2,7 +2,7 @@// workflows/object.api.json is exported (workflows/build/oneclick_object_build.js) in change mode with a picture;// each request prunes it. Nodes are found by title.import { nodeByTitle } from "./advanced.ts";-import { setMotionSize, setResolution, type ApiGraph, type MotionSize, type Resolution } from "./simple.ts";+import { setMotionSize, setResolution, suffixRunName, type ApiGraph, type MotionSize, type Resolution } from "./simple.ts";export const EDIT_MODES = ["change", "remove", "background", "add"] as const;export type EditMode = (typeof EDIT_MODES)[number];@@ -96,6 +96,8 @@ function describer(settings: ObjectSettings) {};}+const copiesRoutine = (action: string) => /\b(?:in[ -]sync|same (?:dance )?routine|synchroni[sz](?:ed|e|ing))\b/i.test(action);+export function pruneObject(source: ApiGraph, picture: boolean, settings: ObjectSettings): ApiGraph {const graph: ApiGraph = structuredClone(source);const id = (title: string) => nodeByTitle(graph, title);@@ -165,14 +167,46 @@ export function pruneObject(source: ApiGraph, picture: boolean, settings: Objectplan.change_to = addedObject(settings.changeTo);plan.keep_outline = false;plan.action = settings.action ?? "";- graph.object_placement = { class_type: "H3ObjectPlacementMask", inputs: {- images: [id("Source clip (frames + audio)"), 0], start_frame: [id("Clip start frame"), 1], placement,- }, _meta: { title: "Object placement area" } };+ const moving = copiesRoutine(settings.action ?? "");+ const drop = ["SAM frames for this clip", "Tracked area", "Tracked area, tight (for the grey shape)", "Turn it grey (shape and movement only)"];+ if (moving) {+ graph[id("SAM: what to change")].inputs.text = "person";+ graph[id("SAM 3.1 track it")].inputs.images = [id("SAM video (1008 px, SAM's own size)"), 0];+ graph.object_moving_area = { class_type: "H3AddMovingArea", inputs: {+ images: [id("SAM video (1008 px, SAM's own size)"), 0], track_data: [id("SAM 3.1 track it"), 0], placement, start_frame: 0,+ }, _meta: { title: "Moving edit area (follows the copied routine)" } };+ graph.object_placement = { class_type: "MiniMaxH3ContexLoopMaskSlice", inputs: {+ mask: ["object_moving_area", 0], state: [id("Current Clip"), 0], source_fps: 24,+ }, _meta: { title: "Moving edit area for this clip" } };+ suffixRunName(graph, "moving_area_run_name", "_moving_area", "Run name for the moving edit area");+ } else {+ graph.object_placement = { class_type: "H3ObjectPlacementMask", inputs: {+ images: [id("Source clip (frames + audio)"), 0], start_frame: [id("Clip start frame"), 1], placement,+ }, _meta: { title: "Object placement area" } };+ drop.push("SAM 3.1", "SAM: what to change", "SAM video (1008 px, SAM's own size)", "SAM 3.1 track it");+ }grow.mask = ["object_placement", 0];- grow.expand = { tight: 0, normal: 2, loose: 4 }[settings.margin];- for (const title of ["SAM 3.1", "SAM: what to change", "SAM video (1008 px, SAM's own size)", "SAM frames for this clip", "SAM 3.1 track it",- "Tracked area", "Tracked area, tight (for the grey shape)", "Turn it grey (shape and movement only)"]) delete graph[id(title)];+ grow.expand = moving ? 0 : { tight: 0, normal: 2, loose: 4 }[settings.margin];+ for (const title of drop) delete graph[id(title)];}setResolution(graph, id("Object Swap Plan (what to change)"), id(OBJECT_VIDEO), settings.resolution);return graph;}++// Fit check (workflows/fit.api.json): before an add that copies someone's routine, shows where that routine leaves the box.+export function pruneFit(source: ApiGraph, placement: PlacementKeyframe[], action = ""): ApiGraph {+ const graph: ApiGraph = structuredClone(source);+ const fit = graph[nodeByTitle(graph, "Check the fit")].inputs;+ fit.placement = JSON.stringify(parsePlacement(placement));+ fit.moving_area = copiesRoutine(action);+ return graph;+}++// What the fit check found, in frames from the start of the part.+export type FitSummary = {+ frames: number;+ moving?: boolean;+ person: boolean; // someone to copy was found+ outside: { start: number; end: number; sides: ("left" | "right" | "top" | "bottom")[] }[];+ needed: { x: number; y: number; width: number; height: number } | null; // the space their whole routine takes, scaled into the box+};diff --git a/lib/setup.ts b/lib/setup.tsindex 4e6c1f1..16d2546 100644--- a/lib/setup.ts+++ b/lib/setup.ts@@ -17,6 +17,7 @@ export type SetupRequest =| { kind: "color"; settings: ColorSettings }| { kind: "mask"; hatFix: boolean; margin: Margin; target: string }| { kind: "people"; resolution: Resolution }+ | { kind: "fit"; resolution: Resolution }| { kind: "montage"; settings: AdvancedSettings; pictures: number; review: Review | null; model: ModelSettings };export type SetupResult = {@@ -31,7 +32,7 @@ export type NodeRegistry = Record<string, NodeInfo>;const PACKS = [{ name: "ComfyUI-MiniMaxH3-Context-Loop (0.7.3+)", url: "https://github.com/ethanfel/ComfyUI-MiniMaxH3-Context-Loop", nodes: ["MiniMaxH3GenerationProfile", "MiniMaxH3SourceTimeline", "MiniMaxH3ChainPlanModern", "MiniMaxH3ChainLoopStart", "MiniMaxH3ChainCurrent", "MiniMaxH3ChainContext", "MiniMaxH3ChainSegmentSave", "MiniMaxH3ChainReview", "MiniMaxH3ChainLoopEnd", "MiniMaxH3ChainAssemble", "MiniMaxH3ContexLoopSourceSlice", "MiniMaxH3ContexLoopMaskSlice", "MiniMaxH3ContexMaskedTarget", "MiniMaxH3ContexMaskGridPreview", "MiniMaxH3LoopTrim"] },- { name: "comfyui-h3-oneclick-swap", url: "https://github.com/D3voz/comfyui-h3-oneclick-swap", nodes: ["H3OneClickSwapPlan", "H3AdvancedSwapPlan", "H3PictureSwapPlan", "H3ObjectSwapPlan", "H3ObjectPlacementMask", "H3PaintedEditArea", "H3SoundtrackOrSilence", "H3PeopleMask", "H3GreyMannequins", "H3ClipShots", "H3FillShots", "H3TrackCrops", "H3TrackRoles", "H3RoleMask", "H3SubjectPrompt", "H3MontagePeople", "H3MontageTracks", "H3MontagePicks", "H3MontagePlan", "H3MontagePass", "H3FaceChunks", "H3FaceJoin"] },+ { name: "comfyui-h3-oneclick-swap", url: "https://github.com/D3voz/comfyui-h3-oneclick-swap", nodes: ["H3OneClickSwapPlan", "H3AdvancedSwapPlan", "H3PictureSwapPlan", "H3ObjectSwapPlan", "H3ObjectPlacementMask", "H3AddFitCheck", "H3AddMovingArea", "H3PaintedEditArea", "H3SoundtrackOrSilence", "H3PeopleMask", "H3GreyMannequins", "H3ClipShots", "H3FillShots", "H3TrackCrops", "H3TrackRoles", "H3RoleMask", "H3SubjectPrompt", "H3MontagePeople", "H3MontageTracks", "H3MontagePicks", "H3MontagePlan", "H3MontagePass", "H3FaceChunks", "H3FaceJoin"] },{ name: "ComfyUI-Fantastic-MiniMaxH3-PromptBuilder", url: "https://github.com/Adudeguyman/ComfyUI-Fantastic-MiniMaxH3-PromptBuilder", nodes: ["MiniMaxH3FantasticVideoEditLatent"] },{ name: "ComfyUI-KJNodes", url: "https://github.com/kijai/ComfyUI-KJNodes", nodes: ["GrowMaskWithBlur", "ImageAndMaskPreview"] },{ name: "ComfyUI-VideoHelperSuite", url: "https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite", nodes: ["VHS_LoadVideo", "VHS_SelectEveryNthImage", "VHS_VideoCombine"] },@@ -63,7 +64,7 @@ export function setupGraph(source: ApiGraph, selection: SetupRequest): ApiGraphelse if (selection.kind === "color") graph = pruneColor(source, selection.settings);else if (selection.kind === "mask") graph = pruneMask(source, selection.hatFix, selection.margin, selection.target);else if (selection.kind === "montage") graph = pruneMontage(source, selection.pictures, selection.settings, selection.review);- else if (selection.kind === "people") {+ else if (selection.kind === "people" || selection.kind === "fit") {graph = structuredClone(source);if (typeof selection.resolution === "object") cropSource(graph, PEOPLE_NODES.video, selection.resolution);} else graph = pruneAdvanced(source, selection.pictures, selection.settings);diff --git a/test/object.test.ts b/test/object.test.tsindex 9327c25..6d28c32 100644--- a/test/object.test.ts+++ b/test/object.test.ts@@ -3,7 +3,7 @@ import { readFileSync } from "node:fs";import test from "node:test";import { nodeByTitle } from "../lib/advanced.ts";import { applyModel, DEFAULT_MODEL } from "../lib/model.ts";-import { OBJECT_PICTURE, OBJECT_VIDEO, parsePaint, parsePlacement, placementAt, pruneObject, type ObjectSettings } from "../lib/object.ts";+import { OBJECT_PICTURE, OBJECT_VIDEO, parsePaint, parsePlacement, placementAt, pruneFit, pruneObject, type ObjectSettings } from "../lib/object.ts";import { isLink, type ApiGraph } from "../lib/simple.ts";const graph: ApiGraph = JSON.parse(readFileSync(new URL("../workflows/object.api.json", import.meta.url), "utf8"));@@ -168,3 +168,45 @@ test("paint rejects malformed and oversized runs at the boundary", () => {[{ ...valid, runs: [63, 2] }], [{ ...valid, runs: [-1, 2] }], [{ ...valid, runs: [10, 5, 12, 2] }],[{ ...valid, runs: [10, 1.5] }], [{ ...valid, runs: [10] }], Array(65).fill(valid), " ".repeat(512 * 1024 + 1)]) assert.throws(() => parsePaint(value));});++test("the fit check gets the placement and tracks people in the part", () => {+ const fit: ApiGraph = JSON.parse(readFileSync(new URL("../workflows/fit.api.json", import.meta.url), "utf8"));+ const placement = [{ frame: 0, x: 0.1, y: 0.05, width: 0.2, height: 0.9 }];+ const pruned = pruneFit(fit, placement);+ assert.deepEqual(dangling(pruned), []);+ assert.equal(node(pruned, "Check the fit").inputs.placement, JSON.stringify(placement));+ assert.equal(node(pruned, "SAM: people").inputs.text, "person");+ assert.equal(node(fit, "Check the fit").inputs.placement, "", "the source graph is not modified");+ assert.throws(() => pruneFit(fit, []), /placement area/);+});+++test("an in-sync add tracks the full section and slices the moving area per clip", () => {+ const placement = [{ frame: 0, x: 0.12, y: 0.03, width: 0.21, height: 0.97 }];+ for (const action of ["she dances in sync with the other dancer", "copy the same routine", "synchronized dancing"]) {+ const pruned = pruneObject(graph, true, { ...settings("add"), placement, action });+ assert.deepEqual(dangling(pruned), []);+ assert.equal(node(pruned, "SAM: what to change").inputs.text, "person");+ assert.deepEqual(node(pruned, "SAM 3.1 track it").inputs.images, [nodeByTitle(pruned, "SAM video (1008 px, SAM's own size)"), 0]);+ assert.equal(pruned.object_moving_area.class_type, "H3AddMovingArea");+ assert.equal(pruned.object_moving_area.inputs.placement, JSON.stringify(placement));+ assert.equal(pruned.object_moving_area.inputs.start_frame, 0);+ assert.equal(node(pruned, "Grow mask").inputs.expand, 0);+ assert.equal(pruned.object_placement.class_type, "MiniMaxH3ContexLoopMaskSlice");+ assert.deepEqual(pruned.object_placement.inputs.mask, ["object_moving_area", 0]);+ assert.deepEqual(pruned.object_placement.inputs.state, [nodeByTitle(pruned, "Current Clip"), 0]);+ assert.deepEqual(node(pruned, "Grow mask").inputs.mask, ["object_placement", 0]);+ assert.deepEqual(node(pruned, "Reference Conditioning").inputs["ref_images.ref_image_0"], [nodeByTitle(pruned, OBJECT_PICTURE), 0]);+ assert.equal(pruned.moving_area_run_name.inputs.string_b, "_moving_area");+ }+ const custom = pruneObject(graph, true, { ...settings("add"), placement, action: "dance in sync", resolution: { width: 640, height: 960 } });+ assert.deepEqual(node(custom, "SAM video (1008 px, SAM's own size)").inputs.image, ["custom_size", 0]);+});+++test("the fit preview selects the same moving area as synchronized generation", () => {+ const fit: ApiGraph = JSON.parse(readFileSync(new URL("../workflows/fit.api.json", import.meta.url), "utf8"));+ const placement = [{ frame: 0, x: 0.12, y: 0.03, width: 0.21, height: 0.97 }];+ assert.equal(node(pruneFit(fit, placement, "dance in sync"), "Check the fit").inputs.moving_area, true);+ assert.equal(node(pruneFit(fit, placement, "steam rises"), "Check the fit").inputs.moving_area, false);+});diff --git a/workflows/fit.api.json b/workflows/fit.api.jsonnew file mode 100644index 0000000..d3fedfb--- /dev/null+++ b/workflows/fit.api.json@@ -0,0 +1,37 @@+{+ "1": {+ "inputs": { "video": "", "force_rate": 24, "custom_width": 0, "custom_height": 0, "frame_load_cap": 0, "skip_first_frames": 0, "select_every_nth": 1, "format": "None" },+ "class_type": "VHS_LoadVideo",+ "_meta": { "title": "Source video" }+ },+ "2": {+ "inputs": { "ckpt_name": "sam3.1_multiplex_fp16.safetensors" },+ "class_type": "CheckpointLoaderSimple",+ "_meta": { "title": "SAM 3.1" }+ },+ "3": {+ "inputs": { "text": "person", "clip": ["2", 1] },+ "class_type": "CLIPTextEncode",+ "_meta": { "title": "SAM: people" }+ },+ "4": {+ "inputs": { "upscale_method": "area", "largest_size": 1008, "image": ["1", 0] },+ "class_type": "ImageScaleToMaxDimension",+ "_meta": { "title": "SAM video (1008 px)" }+ },+ "5": {+ "inputs": { "detection_threshold": 0.35, "max_objects": 0, "detect_interval": 1, "images": ["4", 0], "model": ["2", 0], "conditioning": ["3", 0] },+ "class_type": "SAM3_VideoTrack",+ "_meta": { "title": "SAM 3.1 track every person" }+ },+ "6": {+ "inputs": { "placement": "", "start_frame": 0, "images": ["4", 0], "track_data": ["5", 0] },+ "class_type": "H3AddFitCheck",+ "_meta": { "title": "Check the fit" }+ },+ "7": {+ "inputs": { "frame_rate": 24, "loop_count": 0, "filename_prefix": "openjutsu_fit/fit", "format": "video/h264-mp4", "pix_fmt": "yuv420p", "crf": 19, "save_metadata": false, "trim_to_audio": false, "pingpong": false, "save_output": false, "images": ["6", 0] },+ "class_type": "VHS_VideoCombine",+ "_meta": { "title": "Fit preview video" }+ }+}