Skip to content

One-click character swap for videos: an open-source, local Higgsfield Genjutsu alternative built on ComfyUI and MiniMax H3, with FLUX.3 Klein relighting.

Add What to replace for Simple mode and Check mask

D3voz6cabb034941e
diff --git a/README.md b/README.md
index b28d51a..d824eef 100644
--- a/README.md
+++ b/README.md
@@ -73,6 +73,7 @@ A 6 s clip at 480p takes about 6–13 minutes on a 4060 Ti, most of it in H3 sam
- Only the main person is regenerated; anyone else in the video stays as they are.
- **Who do they replace?** names the person for the prompt by role (`the singer`), never by clothes: clothing words leak into the result. Leave it blank and the describer names them.
+- **What to replace** (Simple mode and Check mask) is the text SAM tracks: `person` by default, an animal such as `cat`, or a short description (`black and white cat`) when the biggest match is the wrong one. The biggest match in the frame is replaced.
- **Hat fix** also masks the hat, so it is replaced too (about 45 s more per clip).
- **Check mask** runs only SAM and the mask (about a minute per 3 s of video) and shows a preview you can scrub frame by frame: cyan is the area H3 regenerates, orange the tracked person, and on the right the grey mannequin H3 copies the motion from. Anything outside the cyan area stays as it is. If the character is cut off or too much background changes, set **Mask margin** (Tight, Normal, Loose) and check again; a re-check takes seconds.
- Change the **seed** (under Advanced) for a new take. Running the same inputs again resumes from the last finished clip.
diff --git a/app/api/jobs/route.ts b/app/api/jobs/route.ts
index 8a5cf08..a12cdc4 100644
--- a/app/api/jobs/route.ts
+++ b/app/api/jobs/route.ts
@@ -2,7 +2,7 @@ import { NextResponse } from "next/server";
import { listModels, submitAdvanced, submitMask, submitSimple } from "../../../lib/comfy.ts";
import { MAX_CHARACTERS } from "../../../lib/advanced.ts";
import { MAX_LORAS, STEPS, STRENGTH, type ModelSettings } from "../../../lib/model.ts";
-import { CLIP_SECONDS, MARGINS, MAX_SEED, RESOLUTIONS, type Margin, type Resolution, type SimpleSettings } from "../../../lib/simple.ts";
+import { CLIP_SECONDS, DEFAULT_TARGET, MARGINS, MAX_SEED, RESOLUTIONS, type Margin, type Resolution, type SimpleSettings } from "../../../lib/simple.ts";
export const runtime = "nodejs";
@@ -16,7 +16,8 @@ function parseVideo(form: FormData) {
function parseMask(form: FormData) {
const margin = String(form.get("margin") ?? "normal");
if (!Object.hasOwn(MARGINS, margin)) throw new Error("Choose a supported mask margin.");
- return { hatFix: form.get("hat_fix") === "true", margin: margin as Margin };
+ const target = String(form.get("target") ?? "").replace(/\s+/g, " ").trim().slice(0, 100) || DEFAULT_TARGET;
+ return { hatFix: form.get("hat_fix") === "true", margin: margin as Margin, target };
}
function parsePicture(picture: FormDataEntryValue | null) {
@@ -76,8 +77,8 @@ export async function POST(request: Request) {
const mode = form.get("mode");
const video = parseVideo(form);
if (mode === "mask") {
- const { hatFix, margin } = parseMask(form);
- submit = () => submitMask(video, hatFix, margin);
+ const { hatFix, margin, target } = parseMask(form);
+ submit = () => submitMask(video, hatFix, margin, target);
} else if (mode === "simple") {
const { picture, settings } = parseSimple(form);
const model = parseModel(form, await listModels());
diff --git a/app/globals.css b/app/globals.css
index a761504..e65e22e 100644
--- a/app/globals.css
+++ b/app/globals.css
@@ -78,10 +78,10 @@ video { width: 100%; border-radius: 12px; background: #000; max-height: 420px; }
.character { position: relative; background: var(--panel-2); border-radius: 14px; overflow: hidden; display: grid; }
.character img { width: 100%; aspect-ratio: 3 / 4; object-fit: cover; display: block; }
.character label { display: grid; gap: 4px; padding: 10px; font-size: 0.82em; color: var(--muted); }
-.character input, .seed input, .file-name, .steps-input, .lora input {
+.character input, .seed input, .file-name, .steps-input, .target-input, .lora input {
background: var(--bg); border: 1px solid var(--line); border-radius: 8px; padding: 7px 9px; width: 100%;
}
-.character input:focus, .seed input:focus, .file-name:focus, .steps-input:focus, .lora input:focus { outline: none; border-color: var(--accent); }
+.character input:focus, .seed input:focus, .file-name:focus, .steps-input:focus, .target-input:focus, .lora input:focus { outline: none; border-color: var(--accent); }
.remove {
position: absolute; top: 8px; right: 8px; width: 28px; height: 28px; border-radius: 50%; border: 0;
background: #000a; font-size: 18px; line-height: 1;
@@ -145,6 +145,7 @@ input[type="range"] { accent-color: var(--accent); width: 220px; max-width: 100%
.file-name { min-width: 0; width: min(100%, 420px); }
.steps-input { width: 6rem; }
+.target-input { width: 12rem; }
.setting.loras { align-items: flex-start; }
.lora-list { display: flex; flex-direction: column; align-items: flex-start; gap: 8px; flex: 1 1 100%; }
.lora { display: grid; grid-template-columns: minmax(0, 1fr) 4.5rem auto; gap: 8px; width: 100%; }
diff --git a/components/openjutsu.tsx b/components/openjutsu.tsx
index 97b1756..0e19de1 100644
--- a/components/openjutsu.tsx
+++ b/components/openjutsu.tsx
@@ -4,7 +4,7 @@ import { useEffect, useRef, useState, type DragEvent, type ReactNode } from "rea
import type { PublicJob } from "../lib/comfy.ts";
import type { NodeInfo } from "../lib/progress.ts";
import { clipCount, clipStarts } from "../lib/clips.ts";
-import { CLIP_SECONDS, MARGINS, MAX_SEED, RESOLUTIONS, type Margin, type Resolution } from "../lib/simple.ts";
+import { CLIP_SECONDS, DEFAULT_TARGET, MARGINS, MAX_SEED, RESOLUTIONS, type Margin, type Resolution } from "../lib/simple.ts";
import { MAX_CHARACTERS } from "../lib/advanced.ts";
import { DEFAULT_MODEL, MAX_LORAS, STEPS, STRENGTH, type Lora } from "../lib/model.ts";
@@ -105,6 +105,7 @@ export function Openjutsu() {
const [resolution, setResolution] = useState<Resolution>("736p");
const [hatFix, setHatFix] = useState(false);
const [margin, setMargin] = useState<Margin>("normal");
+ const [target, setTarget] = useState(DEFAULT_TARGET);
const [seed, setSeed] = useState(0);
const [clipSeconds, setClipSeconds] = useState(CLIP_SECONDS.default);
const [steps, setSteps] = useState(DEFAULT_MODEL.steps);
@@ -205,6 +206,7 @@ export function Openjutsu() {
form.set("video", video.file);
form.set("hat_fix", String(hatFix));
form.set("margin", margin);
+ form.set("target", target);
if (kind === "simple" && character) {
form.set("picture", character.file);
form.set("role", character.role);
@@ -367,6 +369,12 @@ export function Openjutsu() {
))}
</div>
</div>
+ {mode === "simple" && (
+ <div className="setting">
+ <span>What to replace <small>What SAM tracks, in English: person, cat, dog, or a short description such as “black and white cat”. The biggest match is replaced.</small></span>
+ <input className="target-input" aria-label="What to replace" placeholder={DEFAULT_TARGET} value={target} onChange={(event) => setTarget(event.target.value)} />
+ </div>
+ )}
<label className="setting toggle">
<span>Hat fix <small>Also replaces hats{mode === "advanced" ? " worn by the people being replaced" : ""} (about 45 s more per clip).</small></span>
<input type="checkbox" checked={hatFix} onChange={(event) => setHatFix(event.target.checked)} />
diff --git a/lib/comfy.ts b/lib/comfy.ts
index bc132dd..00092ff 100644
--- a/lib/comfy.ts
+++ b/lib/comfy.ts
@@ -89,9 +89,9 @@ export async function submitAdvanced(video: File, pictures: File[], settings: Ad
return publicJob(await comfy.submit(workflow));
}
-export async function submitMask(video: File, hatFix: boolean, margin: Margin) {
+export async function submitMask(video: File, hatFix: boolean, margin: Margin, target: string) {
const comfy = client();
- const workflow = comfy.workflows.fromJson(pruneMask(await loadGraph("mask"), hatFix, margin));
+ const workflow = comfy.workflows.fromJson(pruneMask(await loadGraph("mask"), hatFix, margin, target));
workflow.setInput(MASK_NODES.video, "video", await upload(comfy, video));
return publicJob(await comfy.submit(workflow));
}
diff --git a/lib/simple.ts b/lib/simple.ts
index 654bc47..2aa074e 100644
--- a/lib/simple.ts
+++ b/lib/simple.ts
@@ -15,22 +15,26 @@ export const MAX_SEED = 0xffffffff;
export const MARGINS = { tight: 20, normal: 40, loose: 60 } as const;
export type Margin = keyof typeof MARGINS;
+export const DEFAULT_TARGET = "person";
+
export type SimpleSettings = {
role: string; // who the character replaces, "" to let the describer name the main person
resolution: Resolution;
hatFix: boolean;
margin: Margin;
+ target: string; // what SAM tracks; the biggest one is replaced ("person", "cat", ...)
seed: number;
clipSeconds: number;
};
-type HatNodes = { hat: string[]; growMask: string; peopleMask: string };
+type MaskNodes = { target: string; hat: string[]; growMask: string; peopleMask: string };
// Node ids in workflows/simple.api.json.
export const NODES = {
picture: "2",
video: "5",
plan: "11",
+ target: "24",
loopStart: "28",
peopleMask: "39",
growMask: "42",
@@ -42,6 +46,7 @@ export const NODES = {
// Node ids in workflows/mask.api.json.
export const MASK_NODES = {
video: "1",
+ target: "3",
peopleMask: "7",
growMask: "9",
hat: ["10", "11", "12", "13", "19", "20"],
@@ -51,8 +56,9 @@ export function isLink(value: unknown): value is [string, number] {
return Array.isArray(value) && value.length === 2 && typeof value[0] === "string" && typeof value[1] === "number";
}
-function maskSettings(source: ApiGraph, hatFix: boolean, margin: Margin, nodes: HatNodes): ApiGraph {
+function maskSettings(source: ApiGraph, hatFix: boolean, margin: Margin, target: string, nodes: MaskNodes): ApiGraph {
const graph: ApiGraph = structuredClone(source);
+ graph[nodes.target].inputs.text = target;
graph[nodes.growMask].inputs.expand = MARGINS[margin];
if (!hatFix) {
for (const id of nodes.hat) delete graph[id];
@@ -67,7 +73,7 @@ function maskSettings(source: ApiGraph, hatFix: boolean, margin: Margin, nodes:
}
export function pruneSimple(source: ApiGraph, settings: SimpleSettings): ApiGraph {
- const graph = maskSettings(source, settings.hatFix, settings.margin, NODES);
+ const graph = maskSettings(source, settings.hatFix, settings.margin, settings.target, NODES);
const plan = graph[NODES.plan].inputs;
plan.resolution = settings.resolution;
plan.seed = settings.seed;
@@ -76,6 +82,6 @@ export function pruneSimple(source: ApiGraph, settings: SimpleSettings): ApiGrap
return graph;
}
-export function pruneMask(source: ApiGraph, hatFix: boolean, margin: Margin): ApiGraph {
- return maskSettings(source, hatFix, margin, MASK_NODES);
+export function pruneMask(source: ApiGraph, hatFix: boolean, margin: Margin, target: string): ApiGraph {
+ return maskSettings(source, hatFix, margin, target, MASK_NODES);
}
diff --git a/test/simple.test.ts b/test/simple.test.ts
index c813365..2501ee6 100644
--- a/test/simple.test.ts
+++ b/test/simple.test.ts
@@ -9,7 +9,7 @@ const load = (name: string): ApiGraph => JSON.parse(readFileSync(new URL(`../wor
const graph = load("simple");
const mask = load("mask");
const HAT_TYPES = ["CLIPTextEncode", "SAM3_VideoTrack", "SAM3_TrackToMask", "GrowMaskWithBlur", "MaskComposite", "MaskComposite"];
-const settings = (hatFix: boolean): SimpleSettings => ({ role: " the singer ", resolution: "480p", hatFix, margin: "loose", seed: 7, clipSeconds: 6 });
+const settings = (hatFix: boolean): SimpleSettings => ({ role: " the singer ", resolution: "480p", hatFix, margin: "loose", target: "cat", seed: 7, clipSeconds: 6 });
test("the node ids match the export", () => {
const types = { picture: "LoadImage", video: "VHS_LoadVideo", plan: "H3OneClickSwapPlan", loopStart: "MiniMaxH3ChainLoopStart", peopleMask: "H3PeopleMask",
@@ -18,6 +18,7 @@ test("the node ids match the export", () => {
assert.deepEqual(NODES.hat.map((id) => graph[id].class_type), HAT_TYPES);
assert.deepEqual(MASK_NODES.hat.map((id) => mask[id].class_type).sort(), [...HAT_TYPES].sort());
assert.equal(mask[MASK_NODES.video].class_type, "VHS_LoadVideo");
+ for (const [g, id] of [[graph, NODES.target], [mask, MASK_NODES.target]] as const) assert.deepEqual([g[id].class_type, g[id].inputs.text], ["CLIPTextEncode", "person"]);
});
test("without hat fix the hat nodes go and the grow mask reads the people mask", () => {
@@ -27,6 +28,7 @@ test("without hat fix the hat nodes go and the grow mask reads the people mask",
const plan = pruned[NODES.plan].inputs;
assert.deepEqual([plan.resolution, plan.seed, plan.clip_seconds, plan.roles], ["480p", 7, 6, "the singer"]);
assert.equal(pruned[NODES.growMask].inputs.expand, 60);
+ assert.equal(pruned[NODES.target].inputs.text, "cat");
assert.ok(graph[NODES.hat[0]], "the source graph is not modified");
});
@@ -54,11 +56,12 @@ test("clip count follows the Swap Plan grid", () => {
});
test("the mask check drops the hat nodes without hat fix", () => {
- const pruned = pruneMask(mask, false, "tight");
+ const pruned = pruneMask(mask, false, "tight", "dog");
assert.equal(pruned[MASK_NODES.growMask].inputs.expand, 20);
+ assert.equal(pruned[MASK_NODES.target].inputs.text, "dog");
for (const id of MASK_NODES.hat) assert.equal(pruned[id], undefined);
assert.deepEqual(pruned[MASK_NODES.growMask].inputs.mask, [MASK_NODES.peopleMask, 0]);
- assert.equal(Object.keys(pruneMask(mask, true, "normal")).length, Object.keys(mask).length);
+ assert.equal(Object.keys(pruneMask(mask, true, "normal", "person")).length, Object.keys(mask).length);
});
test("clip starts follow the Swap Plan's 39-frame overlap", () => {