Skip to content

One-click character swap for videos: an open-source, local Higgsfield Genjutsu alternative built on ComfyUI and MiniMax H3, with FLUX.3 Klein relighting.

Add Montage tab to Advanced motion transfer

D3voz99bb05bb15ab
diff --git a/README.md b/README.md
index b686447..d96aaef 100644
--- a/README.md
+++ b/README.md
@@ -77,6 +77,8 @@ On Windows, double-click `start_openjutsu.bat` (ComfyUI must be running, or star
A 6 s clip at 480p takes about 6–13 minutes on a 4060 Ti, most of it in H3 sampling. 736p is the default; 1080p is about 2.5× slower than 736p and needs much more memory. Custom sets an exact width × height from 256 to 2048 px a side (rounded to 32); the video is scaled and centre-cropped to that shape first. It needs the `short_edge` input of the H3 Swap Plan nodes in comfyui-h3-oneclick-swap.
+Advanced motion transfer has a **Montage** tab for edited clips where different people take turns within one shot without a hard cut (whip pans, selfie handovers, travel montages). The shot list and H3 are told when each person is on screen, so nobody turns into someone else. It needs the `timing` input of the pack's Clip Shots and Fill Shots nodes. Write roles as "who - looks" ("the dancer - white polo, jeans"): only the part before " - " reaches H3, and clothes named there are drawn on the new person.
+
## Tips
- Only the main person is regenerated; anyone else in the video stays as they are.
diff --git a/app/api/jobs/route.ts b/app/api/jobs/route.ts
index dbb295c..f429224 100644
--- a/app/api/jobs/route.ts
+++ b/app/api/jobs/route.ts
@@ -137,7 +137,7 @@ function parseAdvanced(form: FormData) {
const maxQuality = form.get("max_quality") === "true";
if (maxQuality && pictures.length !== 2) throw new Error("One pass per person needs exactly two characters.");
const roles = form.getAll("role").map((role) => String(role).slice(0, 200));
- return { pictures, maxQuality, settings: { roles, noLipSync: form.get("no_lip_sync") === "true", ...parseRun(form), ...parseMask(form) } };
+ return { pictures, maxQuality, settings: { roles, noLipSync: form.get("no_lip_sync") === "true", montage: form.get("montage") === "true", ...parseRun(form), ...parseMask(form) } };
}
export async function POST(request: Request) {
diff --git a/components/openjutsu.tsx b/components/openjutsu.tsx
index 97ede29..c88533a 100644
--- a/components/openjutsu.tsx
+++ b/components/openjutsu.tsx
@@ -141,6 +141,7 @@ export function Openjutsu() {
const player = useRef<HTMLVideoElement>(null);
const [characters, setCharacters] = useState<Character[]>([]);
const [maxQuality, setMaxQuality] = useState(false);
+ const [montage, setMontage] = useState(false);
const [resolution, setResolution] = useState<Resolution>("736p");
const [customSize, setCustomSize] = useState<Size>({ width: 1248, height: 736 });
const [hatFix, setHatFix] = useState(false);
@@ -347,6 +348,7 @@ export function Openjutsu() {
form.append("role", c.role);
}
form.set("max_quality", String(kind === "advanced_maxq"));
+ form.set("montage", String(montage));
form.set("resolution", resolutionField);
form.set("seed", String(seed));
form.set("clip_seconds", String(clipSeconds));
@@ -501,6 +503,19 @@ export function Openjutsu() {
))}
</div>
)}
+ {mode === "advanced" && (<>
+ <div className="segmented">
+ <button className={montage ? "" : "on"} onClick={() => setMontage(false)}>Standard</button>
+ <button className={montage ? "on" : ""} onClick={() => setMontage(true)}>Montage</button>
+ </div>
+ {montage && (
+ <p className="hint">
+ For edited clips where different people take turns within one shot, with no hard cut between them: whip pans,
+ selfie handovers, travel and &ldquo;vibe&rdquo; montages. H3 is told when each person is on screen, so nobody turns
+ into someone else. Use Standard for everything else.
+ </p>
+ )}
+ </>)}
{colorFix ? (<>
<label className="setting toggle">
<span>Detail contrast <small>Softens H3&apos;s crunchy fine detail and over-sharpened skin, without blurring edges.</small></span>
@@ -573,7 +588,7 @@ export function Openjutsu() {
<input
value={c.role}
maxLength={200}
- placeholder={i === 0 ? "e.g. the singer" : "e.g. the drummer"}
+ placeholder={i === 0 ? "e.g. the singer - red dress" : "e.g. the drummer - black cap"}
onChange={(event) => setRole(i, event.target.value)}
/>
</label>}
@@ -600,7 +615,7 @@ export function Openjutsu() {
<p className="hint">
The character replaces the main person in the video, the one who fills the most of the frame. Everyone else stays.
Mirrors and reflections do not work yet.
- To name them for the prompt, use a role (&ldquo;the singer&rdquo;), not clothes.
+ To name them for the prompt, use a role (&ldquo;the singer&rdquo;), not clothes; their look can go after &ldquo; - &rdquo; (&ldquo;the singer - red dress&rdquo;).
</p>
) : (
<p className="hint">
diff --git a/lib/advanced.ts b/lib/advanced.ts
index f5533e9..0821c7b 100644
--- a/lib/advanced.ts
+++ b/lib/advanced.ts
@@ -2,7 +2,7 @@
// workflows/advanced.api.json (one pass) and advanced_maxq.api.json (one pass per person, 2 characters) are exported
// with every picture, the hat fix and No lip-sync enabled; each request prunes what it does not use. Nodes are found by title, as
// the builder (ComfyUI user/default/oneclick_advanced_build.js) names them.
-import { MARGINS, dropNoLipSync, isLink, setMotionSize, setResolution, type ApiGraph, type Margin, type MotionSize, type Resolution } from "./simple.ts";
+import { MARGINS, dropNoLipSync, isLink, setMotionSize, setResolution, suffixRunName, type ApiGraph, type Margin, type MotionSize, type Resolution } from "./simple.ts";
export const MAX_CHARACTERS = 4;
@@ -12,6 +12,7 @@ export type AdvancedSettings = {
motion: MotionSize;
hatFix: boolean;
noLipSync: boolean;
+ montage: boolean; // different people take turns within one shot (whip pans, selfie handovers)
margin: Margin;
seed: number;
clipSeconds: number;
@@ -69,6 +70,11 @@ export function pruneAdvanced(source: ApiGraph, pictures: number, settings: Adva
}
if (!settings.noLipSync) dropNoLipSync(graph);
setMotionSize(graph, settings.motion);
+ // Montage: the shot list and H3 are told when each person is on screen, so nobody turns into another subject within a shot.
+ if (settings.montage) {
+ for (const node of Object.values(graph)) if (node.class_type === "H3ClipShots" || node.class_type === "H3FillShots") node.inputs.timing = true;
+ suffixRunName(graph, "montage_run_name", "_montage", "Run name for Montage");
+ }
for (const node of Object.values(graph)) {
if (node.class_type === "GrowMaskWithBlur" && / grow mask /.test(node._meta?.title ?? "")) node.inputs.expand = MARGINS[settings.margin];
diff --git a/lib/simple.ts b/lib/simple.ts
index f4f8885..8e355f7 100644
--- a/lib/simple.ts
+++ b/lib/simple.ts
@@ -142,10 +142,15 @@ export function setMotionSize(graph: ApiGraph, size: MotionSize) {
if (!nodes.length) throw new Error("Expected a motion reference resize in the workflow.");
for (const node of nodes) node.inputs.largest_size = size;
if (size === MOTION_SIZES[0]) return;
+ suffixRunName(graph, "motion_run_name", `_motion${size}`, "Run name for this motion reference size");
+}
+
+// Settings the Swap Plan's run name does not know get their own run, so a rerun never resumes from clips made without them.
+export function suffixRunName(graph: ApiGraph, id: string, suffix: string, title: string) {
const plans = Object.values(graph).filter((node) => node.class_type === "MiniMaxH3ChainPlanModern");
if (plans.length !== 1) throw new Error(`Expected one Production Plan node in the workflow, found ${plans.length}.`);
- graph.motion_run_name = { class_type: "StringConcatenate", inputs: { string_a: plans[0].inputs.run_name, string_b: `_motion${size}`, delimiter: "" }, _meta: { title: "Run name for this motion reference size" } };
- plans[0].inputs.run_name = ["motion_run_name", 0];
+ graph[id] = { class_type: "StringConcatenate", inputs: { string_a: plans[0].inputs.run_name, string_b: suffix, delimiter: "" }, _meta: { title } };
+ plans[0].inputs.run_name = [id, 0];
}
export function pruneSimple(source: ApiGraph, settings: SimpleSettings): ApiGraph {
diff --git a/test/advanced.test.ts b/test/advanced.test.ts
index 9e7566c..d08daa2 100644
--- a/test/advanced.test.ts
+++ b/test/advanced.test.ts
@@ -8,7 +8,7 @@ import { isLink, type ApiGraph } from "../lib/simple.ts";
const load = (name: string): ApiGraph => JSON.parse(readFileSync(new URL(`../workflows/${name}.api.json`, import.meta.url), "utf8"));
const graph = load("advanced");
const maxq = load("advanced_maxq");
-const settings = (hatFix: boolean, roles: string[] = [], noLipSync = true): AdvancedSettings => ({ roles, resolution: "480p", motion: 512, hatFix, noLipSync, margin: "tight", seed: 3, clipSeconds: 6 });
+const settings = (hatFix: boolean, roles: string[] = [], noLipSync = true): AdvancedSettings => ({ roles, resolution: "480p", motion: 512, hatFix, noLipSync, montage: false, margin: "tight", seed: 3, clipSeconds: 6 });
const titles = (g: ApiGraph) => Object.values(g).map((node) => node._meta?.title ?? "");
const dangling = (g: ApiGraph) => Object.entries(g).flatMap(([id, node]) =>
Object.entries(node.inputs).filter(([, v]) => isLink(v) && !g[v[0]]).map(([name]) => `${id}.${name}`));
@@ -94,6 +94,19 @@ test("the motion reference size reaches both passes", () => {
assert.deepEqual(sizes(pruneAdvanced(maxq, 2, { ...settings(false), motion: 1376 })), [1376, 1376]);
});
+test("Montage turns on the on-screen timing in both passes and gets its own run", () => {
+ const timing = (g: typeof graph) => Object.values(g).filter((node) => node.class_type === "H3ClipShots" || node.class_type === "H3FillShots").map((node) => node.inputs.timing);
+ for (const source of [graph, maxq]) {
+ const off = pruneAdvanced(source, 2, { ...settings(false), motion: 768 });
+ const on = pruneAdvanced(source, 2, { ...settings(false), motion: 768, montage: true });
+ assert.ok(timing(off).every((v) => v === undefined));
+ assert.ok(timing(on).length >= 2 && timing(on).every((v) => v === true));
+ assert.equal(off.montage_run_name, undefined);
+ assert.deepEqual(on.montage_run_name.inputs, { string_a: ["motion_run_name", 0], string_b: "_montage", delimiter: "" });
+ assert.ok(Object.values(on).some((node) => node.class_type === "MiniMaxH3ChainPlanModern" && isLink(node.inputs.run_name) && node.inputs.run_name[0] === "montage_run_name"));
+ }
+});
+
test("picture counts outside the export are refused", () => {
assert.throws(() => pruneAdvanced(graph, 0, settings(true)));
assert.throws(() => pruneAdvanced(maxq, 3, settings(true)));