One-click character swap for videos: an open-source, local Higgsfield Genjutsu alternative built on ComfyUI and MiniMax H3, with FLUX.3 Klein relighting.
Add a mask margin setting
D3vozb1af671d38ad
diff --git a/app/api/jobs/route.ts b/app/api/jobs/route.tsindex 2695660..6b0154e 100644--- a/app/api/jobs/route.ts+++ b/app/api/jobs/route.ts@@ -1,6 +1,6 @@import { NextResponse } from "next/server";import { submitMask, submitSimple } from "../../../lib/comfy.ts";-import { CLIP_SECONDS, MAX_SEED, RESOLUTIONS, type Resolution, type SimpleSettings } from "../../../lib/simple.ts";+import { CLIP_SECONDS, MARGINS, MAX_SEED, RESOLUTIONS, type Margin, type Resolution, type SimpleSettings } from "../../../lib/simple.ts";export const runtime = "nodejs";@@ -11,6 +11,12 @@ function parseVideo(form: FormData) {return video;}+function parseMask(form: FormData) {+ const margin = String(form.get("margin") ?? "normal");+ if (!Object.hasOwn(MARGINS, margin)) throw new Error("Choose a supported mask margin.");+ return { hatFix: form.get("hat_fix") === "true", margin: margin as Margin };+}+function parseSimple(form: FormData) {const picture = form.get("picture");if (!(picture instanceof File) || picture.size === 0) throw new Error("Add a character photo.");@@ -25,7 +31,7 @@ function parseSimple(form: FormData) {throw new Error(`Clip length must be ${CLIP_SECONDS.min}-${CLIP_SECONDS.max} s.`);}- const settings: SimpleSettings = { role: String(form.get("role") ?? "").slice(0, 200), resolution: resolution as Resolution, hatFix: form.get("hat_fix") === "true", seed, clipSeconds };+ const settings: SimpleSettings = { role: String(form.get("role") ?? "").slice(0, 200), resolution: resolution as Resolution, ...parseMask(form), seed, clipSeconds };return { picture, settings };}@@ -36,8 +42,8 @@ export async function POST(request: Request) {const mode = form.get("mode");const video = parseVideo(form);if (mode === "mask") {- const hatFix = form.get("hat_fix") === "true";- submit = () => submitMask(video, hatFix);+ const { hatFix, margin } = parseMask(form);+ submit = () => submitMask(video, hatFix, margin);} else if (mode === "simple") {const { picture, settings } = parseSimple(form);submit = () => submitSimple(video, picture, settings);diff --git a/components/openjutsu.tsx b/components/openjutsu.tsxindex 45c3b51..8030be6 100644--- a/components/openjutsu.tsx+++ b/components/openjutsu.tsx@@ -4,11 +4,11 @@ import { useEffect, useRef, useState, type DragEvent, type ReactNode } from "reaimport type { PublicJob } from "../lib/comfy.ts";import type { NodeInfo } from "../lib/progress.ts";import { clipCount, clipStarts } from "../lib/clips.ts";-import { CLIP_SECONDS, MAX_SEED, RESOLUTIONS, type Resolution } from "../lib/simple.ts";+import { CLIP_SECONDS, MARGINS, MAX_SEED, RESOLUTIONS, type Margin, type Resolution } from "../lib/simple.ts";type Character = { file: File; url: string; role: string };type Kind = "simple" | "mask";-type Run = { id: string; kind: Kind; clips: number; clipSeconds: number; startedAt: number };+type Run = { id: string; kind: Kind; margin: Margin; clips: number; clipSeconds: number; startedAt: number };type Progress = NodeInfo & { step: number | null; steps: number | null };const TERMINAL = new Set(["succeeded", "failed", "canceled", "expired"]);@@ -52,6 +52,16 @@ function clipShare(progress: Progress) {return 0.05;}+function MarginPicker({ value, onChange }: { value: Margin; onChange: (margin: Margin) => void }) {+ return (+ <div className="segmented">+ {(Object.keys(MARGINS) as Margin[]).map((m) => (+ <button key={m} className={m === value ? "on" : ""} onClick={() => onChange(m)}>{m[0].toUpperCase() + m.slice(1)}</button>+ ))}+ </div>+ );+}+function DropZone({ accept, onFiles, multiple, children, className }: {accept: string; onFiles: (files: File[]) => void; multiple?: boolean; children: ReactNode; className?: string;}) {@@ -75,6 +85,7 @@ export function Openjutsu() {const [character, setCharacter] = useState<Character | null>(null);const [resolution, setResolution] = useState<Resolution>("736p");const [hatFix, setHatFix] = useState(false);+ const [margin, setMargin] = useState<Margin>("normal");const [seed, setSeed] = useState(0);const [clipSeconds, setClipSeconds] = useState(CLIP_SECONDS.default);const [submitting, setSubmitting] = useState(false);@@ -152,6 +163,7 @@ export function Openjutsu() {form.set("mode", kind);form.set("video", video.file);form.set("hat_fix", String(hatFix));+ form.set("margin", margin);if (kind === "simple" && character) {form.set("picture", character.file);form.set("role", character.role);@@ -163,7 +175,7 @@ export function Openjutsu() {const response = await fetch("/api/jobs", { method: "POST", body: form });const body = await response.json();if (!response.ok) throw new Error(body.error ?? "Could not start the job.");- const next: Run = { id: body.id, kind, clips: Math.max(clips, 1), clipSeconds, startedAt: Date.now() };+ const next: Run = { id: body.id, kind, margin, clips: Math.max(clips, 1), clipSeconds, startedAt: Date.now() };writeRun(next);setJob(body);setProgress(null);@@ -218,6 +230,7 @@ export function Openjutsu() {<RunViewrun={run} job={job} progress={progress} preview={preview} streamError={streamError} now={now}onCancel={cancel} onReset={reset} onGenerate={video && character && !submitting ? () => submit("simple") : null}+ margin={margin} onMargin={setMargin} onRecheck={video && !submitting ? () => submit("mask") : null}/>) : (<>@@ -286,6 +299,10 @@ export function Openjutsu() {<input type="checkbox" checked={hatFix} onChange={(event) => setHatFix(event.target.checked)} /><i aria-hidden /></label>+ <div className="setting">+ <span>Mask margin <small>Room around the person for a differently built character. Check mask shows it.</small></span>+ <MarginPicker value={margin} onChange={setMargin} />+ </div><details><summary>Advanced</summary><div className="setting">@@ -318,16 +335,22 @@ export function Openjutsu() {);}-function RunView({ run, job, progress, preview, streamError, now, onCancel, onReset, onGenerate }: {+function RunView({ run, job, progress, preview, streamError, now, onCancel, onReset, onGenerate, margin, onMargin, onRecheck }: {run: Run; job: PublicJob | null; progress: Progress | null; preview: string; streamError: string; now: number;onCancel: () => void; onReset: () => void; onGenerate: (() => void) | null;+ margin: Margin; onMargin: (margin: Margin) => void; onRecheck: (() => void) | null;}) {const status = job?.status ?? "queued";const clips = Math.max(run.clips, progress?.clip ?? 1);const mask = run.kind === "mask";if (mask && status === "succeeded" && job?.video) {- return <MaskCheck url={job.video.url} clipSeconds={run.clipSeconds} onGenerate={onGenerate} onBack={onReset} />;+ return (+ <MaskCheck+ url={job.video.url} clipSeconds={run.clipSeconds} checked={run.margin ?? "normal"} margin={margin}+ onMargin={onMargin} onRecheck={onRecheck} onGenerate={onGenerate} onBack={onReset}+ />+ );}if (status === "succeeded" && job?.video) {@@ -377,8 +400,9 @@ function RunView({ run, job, progress, preview, streamError, now, onCancel, onRe);}-function MaskCheck({ url, clipSeconds, onGenerate, onBack }: {- url: string; clipSeconds: number; onGenerate: (() => void) | null; onBack: () => void;+function MaskCheck({ url, clipSeconds, checked, margin, onMargin, onRecheck, onGenerate, onBack }: {+ url: string; clipSeconds: number; checked: Margin; margin: Margin; onMargin: (margin: Margin) => void;+ onRecheck: (() => void) | null; onGenerate: (() => void) | null; onBack: () => void;}) {const player = useRef<HTMLVideoElement>(null);const [frames, setFrames] = useState(0);@@ -427,8 +451,15 @@ function MaskCheck({ url, clipSeconds, onGenerate, onBack }: {Anything outside the cyan area stays as it is, so the new character should fit inside it.</p><p className="hint">The check tracks the whole video at once and the render tracks each clip, so they can differ slightly.</p>+ <div className="setting">+ <span>Mask margin <small>{margin === checked ? `Showing ${checked}.` : `Showing ${checked}; check again to see ${margin}.`} Tight keeps more background, loose gives a bigger character more room.</small></span>+ <MarginPicker value={margin} onChange={onMargin} />+ </div><div className="row">- <button className="primary" disabled={!onGenerate} onClick={() => onGenerate?.()}>Looks good, generate</button>+ {margin !== checked && <button className="primary" disabled={!onRecheck} onClick={() => onRecheck?.()}>Check again</button>}+ <button className={margin === checked ? "primary" : "secondary"} disabled={!onGenerate} onClick={() => onGenerate?.()}>+ {margin === checked ? "Looks good, generate" : `Generate with ${margin}`}+ </button><button className="secondary" onClick={onBack}>Back to settings</button></div>{!onGenerate && <p className="hint">Add a character photo in the settings to generate.</p>}diff --git a/lib/comfy.ts b/lib/comfy.tsindex 990b1a7..ca6fef8 100644--- a/lib/comfy.ts+++ b/lib/comfy.ts@@ -2,7 +2,7 @@ import { readFile } from "node:fs/promises";import { join } from "node:path";import { Comfy, resolveBaseUrl, resolveCredentials, type Job } from "@comfyorg/sdk";import { ComfyLow } from "@comfyorg/sdk/low";-import { MASK_NODES, NODES, pruneMask, pruneSimple, type ApiGraph, type SimpleSettings } from "./simple.ts";+import { MASK_NODES, NODES, pruneMask, pruneSimple, type ApiGraph, type Margin, type SimpleSettings } from "./simple.ts";export type PublicJob = {id: string;@@ -69,9 +69,9 @@ export async function submitSimple(video: File, picture: File, settings: SimpleSreturn publicJob(await comfy.submit(workflow));}-export async function submitMask(video: File, hatFix: boolean) {+export async function submitMask(video: File, hatFix: boolean, margin: Margin) {const comfy = client();- const workflow = comfy.workflows.fromJson(pruneMask(await loadGraph("mask"), hatFix));+ const workflow = comfy.workflows.fromJson(pruneMask(await loadGraph("mask"), hatFix, margin));workflow.setInput(MASK_NODES.video, "video", await upload(comfy, video));return publicJob(await comfy.submit(workflow));}diff --git a/lib/simple.ts b/lib/simple.tsindex c49707c..654bc47 100644--- a/lib/simple.ts+++ b/lib/simple.ts@@ -11,10 +11,15 @@ export type Resolution = (typeof RESOLUTIONS)[number];export const CLIP_SECONDS = { min: 2.5, max: 15, default: 6 };export const MAX_SEED = 0xffffffff;+// Edit-mask grow in pixels at SAM's 1008 px canvas. Motion comes from the tight mannequin either way.+export const MARGINS = { tight: 20, normal: 40, loose: 60 } as const;+export type Margin = keyof typeof MARGINS;+export type SimpleSettings = {role: string; // who the character replaces, "" to let the describer name the main personresolution: Resolution;hatFix: boolean;+ margin: Margin;seed: number;clipSeconds: number;};@@ -46,8 +51,9 @@ export function isLink(value: unknown): value is [string, number] {return Array.isArray(value) && value.length === 2 && typeof value[0] === "string" && typeof value[1] === "number";}-function withoutHats(source: ApiGraph, hatFix: boolean, nodes: HatNodes): ApiGraph {+function maskSettings(source: ApiGraph, hatFix: boolean, margin: Margin, nodes: HatNodes): ApiGraph {const graph: ApiGraph = structuredClone(source);+ graph[nodes.growMask].inputs.expand = MARGINS[margin];if (!hatFix) {for (const id of nodes.hat) delete graph[id];graph[nodes.growMask].inputs.mask = [nodes.peopleMask, 0];@@ -61,7 +67,7 @@ function withoutHats(source: ApiGraph, hatFix: boolean, nodes: HatNodes): ApiGra}export function pruneSimple(source: ApiGraph, settings: SimpleSettings): ApiGraph {- const graph = withoutHats(source, settings.hatFix, NODES);+ const graph = maskSettings(source, settings.hatFix, settings.margin, NODES);const plan = graph[NODES.plan].inputs;plan.resolution = settings.resolution;plan.seed = settings.seed;@@ -70,6 +76,6 @@ export function pruneSimple(source: ApiGraph, settings: SimpleSettings): ApiGrapreturn graph;}-export function pruneMask(source: ApiGraph, hatFix: boolean): ApiGraph {- return withoutHats(source, hatFix, MASK_NODES);+export function pruneMask(source: ApiGraph, hatFix: boolean, margin: Margin): ApiGraph {+ return maskSettings(source, hatFix, margin, MASK_NODES);}diff --git a/test/simple.test.ts b/test/simple.test.tsindex a435830..c813365 100644--- a/test/simple.test.ts+++ b/test/simple.test.ts@@ -9,7 +9,7 @@ const load = (name: string): ApiGraph => JSON.parse(readFileSync(new URL(`../worconst graph = load("simple");const mask = load("mask");const HAT_TYPES = ["CLIPTextEncode", "SAM3_VideoTrack", "SAM3_TrackToMask", "GrowMaskWithBlur", "MaskComposite", "MaskComposite"];-const settings = (hatFix: boolean): SimpleSettings => ({ role: " the singer ", resolution: "480p", hatFix, seed: 7, clipSeconds: 6 });+const settings = (hatFix: boolean): SimpleSettings => ({ role: " the singer ", resolution: "480p", hatFix, margin: "loose", seed: 7, clipSeconds: 6 });test("the node ids match the export", () => {const types = { picture: "LoadImage", video: "VHS_LoadVideo", plan: "H3OneClickSwapPlan", loopStart: "MiniMaxH3ChainLoopStart", peopleMask: "H3PeopleMask",@@ -26,6 +26,7 @@ test("without hat fix the hat nodes go and the grow mask reads the people mask",assert.deepEqual(pruned[NODES.growMask].inputs.mask, [NODES.peopleMask, 0]);const plan = pruned[NODES.plan].inputs;assert.deepEqual([plan.resolution, plan.seed, plan.clip_seconds, plan.roles], ["480p", 7, 6, "the singer"]);+ assert.equal(pruned[NODES.growMask].inputs.expand, 60);assert.ok(graph[NODES.hat[0]], "the source graph is not modified");});@@ -53,10 +54,11 @@ test("clip count follows the Swap Plan grid", () => {});test("the mask check drops the hat nodes without hat fix", () => {- const pruned = pruneMask(mask, false);+ const pruned = pruneMask(mask, false, "tight");+ assert.equal(pruned[MASK_NODES.growMask].inputs.expand, 20);for (const id of MASK_NODES.hat) assert.equal(pruned[id], undefined);assert.deepEqual(pruned[MASK_NODES.growMask].inputs.mask, [MASK_NODES.peopleMask, 0]);- assert.equal(Object.keys(pruneMask(mask, true)).length, Object.keys(mask).length);+ assert.equal(Object.keys(pruneMask(mask, true, "normal")).length, Object.keys(mask).length);});test("clip starts follow the Swap Plan's 39-frame overlap", () => {