Skip to content

One-click character swap for videos: an open-source, local Higgsfield Genjutsu alternative built on ComfyUI and MiniMax H3, with FLUX.3 Klein relighting.

Add a mask margin setting

D3vozb1af671d38ad
diff --git a/app/api/jobs/route.ts b/app/api/jobs/route.ts
index 2695660..6b0154e 100644
--- a/app/api/jobs/route.ts
+++ b/app/api/jobs/route.ts
@@ -1,6 +1,6 @@
import { NextResponse } from "next/server";
import { submitMask, submitSimple } from "../../../lib/comfy.ts";
-import { CLIP_SECONDS, MAX_SEED, RESOLUTIONS, type Resolution, type SimpleSettings } from "../../../lib/simple.ts";
+import { CLIP_SECONDS, MARGINS, MAX_SEED, RESOLUTIONS, type Margin, type Resolution, type SimpleSettings } from "../../../lib/simple.ts";
export const runtime = "nodejs";
@@ -11,6 +11,12 @@ function parseVideo(form: FormData) {
return video;
}
+function parseMask(form: FormData) {
+ const margin = String(form.get("margin") ?? "normal");
+ if (!Object.hasOwn(MARGINS, margin)) throw new Error("Choose a supported mask margin.");
+ return { hatFix: form.get("hat_fix") === "true", margin: margin as Margin };
+}
+
function parseSimple(form: FormData) {
const picture = form.get("picture");
if (!(picture instanceof File) || picture.size === 0) throw new Error("Add a character photo.");
@@ -25,7 +31,7 @@ function parseSimple(form: FormData) {
throw new Error(`Clip length must be ${CLIP_SECONDS.min}-${CLIP_SECONDS.max} s.`);
}
- const settings: SimpleSettings = { role: String(form.get("role") ?? "").slice(0, 200), resolution: resolution as Resolution, hatFix: form.get("hat_fix") === "true", seed, clipSeconds };
+ const settings: SimpleSettings = { role: String(form.get("role") ?? "").slice(0, 200), resolution: resolution as Resolution, ...parseMask(form), seed, clipSeconds };
return { picture, settings };
}
@@ -36,8 +42,8 @@ export async function POST(request: Request) {
const mode = form.get("mode");
const video = parseVideo(form);
if (mode === "mask") {
- const hatFix = form.get("hat_fix") === "true";
- submit = () => submitMask(video, hatFix);
+ const { hatFix, margin } = parseMask(form);
+ submit = () => submitMask(video, hatFix, margin);
} else if (mode === "simple") {
const { picture, settings } = parseSimple(form);
submit = () => submitSimple(video, picture, settings);
diff --git a/components/openjutsu.tsx b/components/openjutsu.tsx
index 45c3b51..8030be6 100644
--- a/components/openjutsu.tsx
+++ b/components/openjutsu.tsx
@@ -4,11 +4,11 @@ import { useEffect, useRef, useState, type DragEvent, type ReactNode } from "rea
import type { PublicJob } from "../lib/comfy.ts";
import type { NodeInfo } from "../lib/progress.ts";
import { clipCount, clipStarts } from "../lib/clips.ts";
-import { CLIP_SECONDS, MAX_SEED, RESOLUTIONS, type Resolution } from "../lib/simple.ts";
+import { CLIP_SECONDS, MARGINS, MAX_SEED, RESOLUTIONS, type Margin, type Resolution } from "../lib/simple.ts";
type Character = { file: File; url: string; role: string };
type Kind = "simple" | "mask";
-type Run = { id: string; kind: Kind; clips: number; clipSeconds: number; startedAt: number };
+type Run = { id: string; kind: Kind; margin: Margin; clips: number; clipSeconds: number; startedAt: number };
type Progress = NodeInfo & { step: number | null; steps: number | null };
const TERMINAL = new Set(["succeeded", "failed", "canceled", "expired"]);
@@ -52,6 +52,16 @@ function clipShare(progress: Progress) {
return 0.05;
}
+function MarginPicker({ value, onChange }: { value: Margin; onChange: (margin: Margin) => void }) {
+ return (
+ <div className="segmented">
+ {(Object.keys(MARGINS) as Margin[]).map((m) => (
+ <button key={m} className={m === value ? "on" : ""} onClick={() => onChange(m)}>{m[0].toUpperCase() + m.slice(1)}</button>
+ ))}
+ </div>
+ );
+}
+
function DropZone({ accept, onFiles, multiple, children, className }: {
accept: string; onFiles: (files: File[]) => void; multiple?: boolean; children: ReactNode; className?: string;
}) {
@@ -75,6 +85,7 @@ export function Openjutsu() {
const [character, setCharacter] = useState<Character | null>(null);
const [resolution, setResolution] = useState<Resolution>("736p");
const [hatFix, setHatFix] = useState(false);
+ const [margin, setMargin] = useState<Margin>("normal");
const [seed, setSeed] = useState(0);
const [clipSeconds, setClipSeconds] = useState(CLIP_SECONDS.default);
const [submitting, setSubmitting] = useState(false);
@@ -152,6 +163,7 @@ export function Openjutsu() {
form.set("mode", kind);
form.set("video", video.file);
form.set("hat_fix", String(hatFix));
+ form.set("margin", margin);
if (kind === "simple" && character) {
form.set("picture", character.file);
form.set("role", character.role);
@@ -163,7 +175,7 @@ export function Openjutsu() {
const response = await fetch("/api/jobs", { method: "POST", body: form });
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? "Could not start the job.");
- const next: Run = { id: body.id, kind, clips: Math.max(clips, 1), clipSeconds, startedAt: Date.now() };
+ const next: Run = { id: body.id, kind, margin, clips: Math.max(clips, 1), clipSeconds, startedAt: Date.now() };
writeRun(next);
setJob(body);
setProgress(null);
@@ -218,6 +230,7 @@ export function Openjutsu() {
<RunView
run={run} job={job} progress={progress} preview={preview} streamError={streamError} now={now}
onCancel={cancel} onReset={reset} onGenerate={video && character && !submitting ? () => submit("simple") : null}
+ margin={margin} onMargin={setMargin} onRecheck={video && !submitting ? () => submit("mask") : null}
/>
) : (
<>
@@ -286,6 +299,10 @@ export function Openjutsu() {
<input type="checkbox" checked={hatFix} onChange={(event) => setHatFix(event.target.checked)} />
<i aria-hidden />
</label>
+ <div className="setting">
+ <span>Mask margin <small>Room around the person for a differently built character. Check mask shows it.</small></span>
+ <MarginPicker value={margin} onChange={setMargin} />
+ </div>
<details>
<summary>Advanced</summary>
<div className="setting">
@@ -318,16 +335,22 @@ export function Openjutsu() {
);
}
-function RunView({ run, job, progress, preview, streamError, now, onCancel, onReset, onGenerate }: {
+function RunView({ run, job, progress, preview, streamError, now, onCancel, onReset, onGenerate, margin, onMargin, onRecheck }: {
run: Run; job: PublicJob | null; progress: Progress | null; preview: string; streamError: string; now: number;
onCancel: () => void; onReset: () => void; onGenerate: (() => void) | null;
+ margin: Margin; onMargin: (margin: Margin) => void; onRecheck: (() => void) | null;
}) {
const status = job?.status ?? "queued";
const clips = Math.max(run.clips, progress?.clip ?? 1);
const mask = run.kind === "mask";
if (mask && status === "succeeded" && job?.video) {
- return <MaskCheck url={job.video.url} clipSeconds={run.clipSeconds} onGenerate={onGenerate} onBack={onReset} />;
+ return (
+ <MaskCheck
+ url={job.video.url} clipSeconds={run.clipSeconds} checked={run.margin ?? "normal"} margin={margin}
+ onMargin={onMargin} onRecheck={onRecheck} onGenerate={onGenerate} onBack={onReset}
+ />
+ );
}
if (status === "succeeded" && job?.video) {
@@ -377,8 +400,9 @@ function RunView({ run, job, progress, preview, streamError, now, onCancel, onRe
);
}
-function MaskCheck({ url, clipSeconds, onGenerate, onBack }: {
- url: string; clipSeconds: number; onGenerate: (() => void) | null; onBack: () => void;
+function MaskCheck({ url, clipSeconds, checked, margin, onMargin, onRecheck, onGenerate, onBack }: {
+ url: string; clipSeconds: number; checked: Margin; margin: Margin; onMargin: (margin: Margin) => void;
+ onRecheck: (() => void) | null; onGenerate: (() => void) | null; onBack: () => void;
}) {
const player = useRef<HTMLVideoElement>(null);
const [frames, setFrames] = useState(0);
@@ -427,8 +451,15 @@ function MaskCheck({ url, clipSeconds, onGenerate, onBack }: {
Anything outside the cyan area stays as it is, so the new character should fit inside it.
</p>
<p className="hint">The check tracks the whole video at once and the render tracks each clip, so they can differ slightly.</p>
+ <div className="setting">
+ <span>Mask margin <small>{margin === checked ? `Showing ${checked}.` : `Showing ${checked}; check again to see ${margin}.`} Tight keeps more background, loose gives a bigger character more room.</small></span>
+ <MarginPicker value={margin} onChange={onMargin} />
+ </div>
<div className="row">
- <button className="primary" disabled={!onGenerate} onClick={() => onGenerate?.()}>Looks good, generate</button>
+ {margin !== checked && <button className="primary" disabled={!onRecheck} onClick={() => onRecheck?.()}>Check again</button>}
+ <button className={margin === checked ? "primary" : "secondary"} disabled={!onGenerate} onClick={() => onGenerate?.()}>
+ {margin === checked ? "Looks good, generate" : `Generate with ${margin}`}
+ </button>
<button className="secondary" onClick={onBack}>Back to settings</button>
</div>
{!onGenerate && <p className="hint">Add a character photo in the settings to generate.</p>}
diff --git a/lib/comfy.ts b/lib/comfy.ts
index 990b1a7..ca6fef8 100644
--- a/lib/comfy.ts
+++ b/lib/comfy.ts
@@ -2,7 +2,7 @@ import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { Comfy, resolveBaseUrl, resolveCredentials, type Job } from "@comfyorg/sdk";
import { ComfyLow } from "@comfyorg/sdk/low";
-import { MASK_NODES, NODES, pruneMask, pruneSimple, type ApiGraph, type SimpleSettings } from "./simple.ts";
+import { MASK_NODES, NODES, pruneMask, pruneSimple, type ApiGraph, type Margin, type SimpleSettings } from "./simple.ts";
export type PublicJob = {
id: string;
@@ -69,9 +69,9 @@ export async function submitSimple(video: File, picture: File, settings: SimpleS
return publicJob(await comfy.submit(workflow));
}
-export async function submitMask(video: File, hatFix: boolean) {
+export async function submitMask(video: File, hatFix: boolean, margin: Margin) {
const comfy = client();
- const workflow = comfy.workflows.fromJson(pruneMask(await loadGraph("mask"), hatFix));
+ const workflow = comfy.workflows.fromJson(pruneMask(await loadGraph("mask"), hatFix, margin));
workflow.setInput(MASK_NODES.video, "video", await upload(comfy, video));
return publicJob(await comfy.submit(workflow));
}
diff --git a/lib/simple.ts b/lib/simple.ts
index c49707c..654bc47 100644
--- a/lib/simple.ts
+++ b/lib/simple.ts
@@ -11,10 +11,15 @@ export type Resolution = (typeof RESOLUTIONS)[number];
export const CLIP_SECONDS = { min: 2.5, max: 15, default: 6 };
export const MAX_SEED = 0xffffffff;
+// Edit-mask grow in pixels at SAM's 1008 px canvas. Motion comes from the tight mannequin either way.
+export const MARGINS = { tight: 20, normal: 40, loose: 60 } as const;
+export type Margin = keyof typeof MARGINS;
+
export type SimpleSettings = {
role: string; // who the character replaces, "" to let the describer name the main person
resolution: Resolution;
hatFix: boolean;
+ margin: Margin;
seed: number;
clipSeconds: number;
};
@@ -46,8 +51,9 @@ export function isLink(value: unknown): value is [string, number] {
return Array.isArray(value) && value.length === 2 && typeof value[0] === "string" && typeof value[1] === "number";
}
-function withoutHats(source: ApiGraph, hatFix: boolean, nodes: HatNodes): ApiGraph {
+function maskSettings(source: ApiGraph, hatFix: boolean, margin: Margin, nodes: HatNodes): ApiGraph {
const graph: ApiGraph = structuredClone(source);
+ graph[nodes.growMask].inputs.expand = MARGINS[margin];
if (!hatFix) {
for (const id of nodes.hat) delete graph[id];
graph[nodes.growMask].inputs.mask = [nodes.peopleMask, 0];
@@ -61,7 +67,7 @@ function withoutHats(source: ApiGraph, hatFix: boolean, nodes: HatNodes): ApiGra
}
export function pruneSimple(source: ApiGraph, settings: SimpleSettings): ApiGraph {
- const graph = withoutHats(source, settings.hatFix, NODES);
+ const graph = maskSettings(source, settings.hatFix, settings.margin, NODES);
const plan = graph[NODES.plan].inputs;
plan.resolution = settings.resolution;
plan.seed = settings.seed;
@@ -70,6 +76,6 @@ export function pruneSimple(source: ApiGraph, settings: SimpleSettings): ApiGrap
return graph;
}
-export function pruneMask(source: ApiGraph, hatFix: boolean): ApiGraph {
- return withoutHats(source, hatFix, MASK_NODES);
+export function pruneMask(source: ApiGraph, hatFix: boolean, margin: Margin): ApiGraph {
+ return maskSettings(source, hatFix, margin, MASK_NODES);
}
diff --git a/test/simple.test.ts b/test/simple.test.ts
index a435830..c813365 100644
--- a/test/simple.test.ts
+++ b/test/simple.test.ts
@@ -9,7 +9,7 @@ const load = (name: string): ApiGraph => JSON.parse(readFileSync(new URL(`../wor
const graph = load("simple");
const mask = load("mask");
const HAT_TYPES = ["CLIPTextEncode", "SAM3_VideoTrack", "SAM3_TrackToMask", "GrowMaskWithBlur", "MaskComposite", "MaskComposite"];
-const settings = (hatFix: boolean): SimpleSettings => ({ role: " the singer ", resolution: "480p", hatFix, seed: 7, clipSeconds: 6 });
+const settings = (hatFix: boolean): SimpleSettings => ({ role: " the singer ", resolution: "480p", hatFix, margin: "loose", seed: 7, clipSeconds: 6 });
test("the node ids match the export", () => {
const types = { picture: "LoadImage", video: "VHS_LoadVideo", plan: "H3OneClickSwapPlan", loopStart: "MiniMaxH3ChainLoopStart", peopleMask: "H3PeopleMask",
@@ -26,6 +26,7 @@ test("without hat fix the hat nodes go and the grow mask reads the people mask",
assert.deepEqual(pruned[NODES.growMask].inputs.mask, [NODES.peopleMask, 0]);
const plan = pruned[NODES.plan].inputs;
assert.deepEqual([plan.resolution, plan.seed, plan.clip_seconds, plan.roles], ["480p", 7, 6, "the singer"]);
+ assert.equal(pruned[NODES.growMask].inputs.expand, 60);
assert.ok(graph[NODES.hat[0]], "the source graph is not modified");
});
@@ -53,10 +54,11 @@ test("clip count follows the Swap Plan grid", () => {
});
test("the mask check drops the hat nodes without hat fix", () => {
- const pruned = pruneMask(mask, false);
+ const pruned = pruneMask(mask, false, "tight");
+ assert.equal(pruned[MASK_NODES.growMask].inputs.expand, 20);
for (const id of MASK_NODES.hat) assert.equal(pruned[id], undefined);
assert.deepEqual(pruned[MASK_NODES.growMask].inputs.mask, [MASK_NODES.peopleMask, 0]);
- assert.equal(Object.keys(pruneMask(mask, true)).length, Object.keys(mask).length);
+ assert.equal(Object.keys(pruneMask(mask, true, "normal")).length, Object.keys(mask).length);
});
test("clip starts follow the Swap Plan's 39-frame overlap", () => {