Skip to content

One-click character swap for videos: an open-source, local Higgsfield Genjutsu alternative built on ComfyUI and MiniMax H3, with FLUX.3 Klein relighting.

Add a keep-the-crop framing option to frame relight

D3vozea3e91333f49
diff --git a/app/api/picture/route.ts b/app/api/picture/route.ts
index 8a67a82..0882a68 100644
--- a/app/api/picture/route.ts
+++ b/app/api/picture/route.ts
@@ -120,6 +120,8 @@ export async function POST(request: Request) {
const lit = typeof light === "string" ? light.trim() : "", ownPerson = form.get("relight") === "frame";
if (!(video instanceof StoredFile) || !video.size) throw new HttpError(400, "Choose a video.");
if (ownPerson && !lit) throw new HttpError(400, "Choose a light for the video's own person.");
+ const denoise = Number(form.get("denoise") ?? 1);
+ if (!(denoise > 0 && denoise <= 1)) throw new HttpError(400, "Use a denoise above 0 and up to 1.");
const character = !ownPerson && picture instanceof StoredFile && picture.size ? picture : null;
if (!ownPerson && !character) throw new HttpError(400, "Choose a character picture.");
const start = Number(form.get("start") ?? 0), seed = Number(form.get("seed") ?? 0);
@@ -139,7 +141,7 @@ export async function POST(request: Request) {
if (character) graph["4"].inputs.image = await uploadFile(await openAsBlob(character.path), pictureExtension!.toLowerCase());
graph["19"].inputs.seed = seed;
graph["21"]._meta = { title: PREPARED };
- if (ownPerson) applyFrameRelight(graph, lit);
+ if (ownPerson) applyFrameRelight(graph, lit, denoise);
else if (lit) applyRelight(graph, lit);
}
const result = await (await comfy("/prompt", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ prompt: graph }) })).json();
diff --git a/components/from-picture.tsx b/components/from-picture.tsx
index 5676a16..ccecd4a 100644
--- a/components/from-picture.tsx
+++ b/components/from-picture.tsx
@@ -27,6 +27,8 @@ export function FromPicture({ initialVideo, initialPicture, start: initialStart
const [relightPrompt, setRelightPrompt] = useState(RELIGHT_PROMPT);
const [ownPerson, setOwnPerson] = useState(false);
const [framePrompt, setFramePrompt] = useState(FRAME_RELIGHT_PROMPT);
+ const [keepCrop, setKeepCrop] = useState(false);
+ const [denoise, setDenoise] = useState(0.9);
const [job, setJob] = useState<Job | null>(null);
const [prepared, setPrepared] = useState("");
const [preview, setPreview] = useState<{ url: string; frame: string } | null>(null);
@@ -132,7 +134,7 @@ export function FromPicture({ initialVideo, initialPicture, start: initialStart
const form = new FormData();
form.set("video", video); form.set("start", String(start)); form.set("seed", String(nextSeed));
form.set("width", String(size[0])); form.set("height", String(size[1])); form.set("prompt", ownPerson ? framePrompt : relight ? relightPrompt : prompt);
- if (ownPerson) form.set("relight", "frame"); else if (picture) form.set("picture", picture);
+ if (ownPerson) { form.set("relight", "frame"); form.set("denoise", String(keepCrop ? denoise : 1)); } else if (picture) form.set("picture", picture);
if (relight) form.set("light", light);
const response = await fetch("/api/picture", { method: "POST", body: form });
const data = await response.json();
@@ -210,6 +212,14 @@ export function FromPicture({ initialVideo, initialPicture, start: initialStart
{LIGHT_LOOKS[look] && <figure style={{ margin: 0, display: "grid", gap: 6 }}><img className="picture-reference" src={LIGHT_LOOKS[look].image} alt={`${LIGHT_LOOKS[look].name} example`} style={{ width: 200 }} /><small>Example on one portrait. Your picture and seed will differ.</small></figure>}
{relight && !ownPerson && <p className="hint">Poses and relights the character in one FLUX pass at 3 MP, about 2 minutes. FLUX sees only image 1; image 2 reaches it as a written pose, so nothing of the video's person leaks in. The light usually changes the background too. Results vary a lot by seed; try two or three.</p>}
{look === LIGHT_LOOKS.length && <label>Your light<textarea className="report" rows={3} value={ownLight} disabled={busy} onChange={(event) => { invalidate(); setOwnLight(event.target.value); }} /></label>}
+ {ownPerson && <label>Framing<select className="file-name" value={keepCrop ? "keep" : "free"} disabled={busy} onChange={(event) => { invalidate(); setKeepCrop(event.target.value === "keep"); }}>
+ <option value="free">FLUX may recompose · strongest light, may zoom out</option>
+ <option value="keep">Keep the frame's crop · starts from the frame</option>
+ </select></label>}
+ {ownPerson && keepCrop && <label className="setting">
+ <span>Denoise <small>{denoise.toFixed(2)}. Lower keeps more of the frame; near 1 FLUX may zoom out again.</small></span>
+ <input type="range" min={0.8} max={0.99} step={0.01} value={denoise} disabled={busy} onChange={(event) => { invalidate(); setDenoise(Number(event.target.value)); }} />
+ </label>}
{ownPerson
? <label>FLUX relight prompt · {"{look}"} is the light<textarea className="report" rows={4} value={framePrompt} disabled={busy} onChange={(event) => { invalidate(); setFramePrompt(event.target.value); }} /></label>
: relight
diff --git a/lib/picture.ts b/lib/picture.ts
index ba4f999..5dac98d 100644
--- a/lib/picture.ts
+++ b/lib/picture.ts
@@ -136,11 +136,17 @@ export function applyRelight(graph: ApiGraph, light: string) {
}
// Relights the video's own person: FLUX edits the raw frame at 3 MP and keeps its person, pose and setting.
-export function applyFrameRelight(graph: ApiGraph, light: string) {
+// Below denoise 1 sampling starts from the frame itself, which keeps its crop; at 1 FLUX may zoom out to fit the light.
+export function applyFrameRelight(graph: ApiGraph, light: string, denoise: number) {
relightRequest(graph, light);
for (const id of ["4", "5", "7"]) delete graph[id];
graph["8"].inputs.image = ["2", 0];
const { positive, negative, vae } = graph["15"].inputs;
graph["15"] = { class_type: "ReferenceMultiLatent", inputs: { positive, negative, vae, resolution: "resize", "resolution.megapixels": 3, "images.image0": ["2", 0] }, _meta: { title: "Raw frame only, 3 MP" } };
graph["21"].inputs.filename_prefix = "openjutsu_picture/flux3/relit_video_frame";
+ graph["19"].inputs.denoise = denoise;
+ if (denoise === 1) return;
+ graph["relight_frame"] = { class_type: "ImageScaleToTotalPixels", inputs: { image: ["2", 0], upscale_method: "lanczos", megapixels: 3, resolution_steps: 16 }, _meta: { title: "Raw frame at 3 MP" } };
+ graph["relight_start"] = { class_type: "VAEEncode", inputs: { pixels: ["relight_frame", 0], vae: ["12", 0] }, _meta: { title: "Start from the frame" } };
+ graph["19"].inputs.latent_image = ["relight_start", 0];
}
diff --git a/test/picture-pipeline.test.ts b/test/picture-pipeline.test.ts
index b335861..b648611 100644
--- a/test/picture-pipeline.test.ts
+++ b/test/picture-pipeline.test.ts
@@ -61,7 +61,7 @@ test("relight gives FLUX only the character picture at 3 MP on the frame's shape
test("relighting the video's own person gives FLUX and the upsampler only the raw frame", () => {
const graph = load("prepare");
graph["6"].inputs.value = FRAME_RELIGHT_PROMPT;
- applyFrameRelight(graph, LIGHT_LOOKS[7].light);
+ applyFrameRelight(graph, LIGHT_LOOKS[7].light, 1);
assert.ok(String(graph["6"].inputs.value).endsWith(LIGHT_LOOKS[7].light));
assert.equal(graph["4"], undefined);
assert.deepEqual(graph["8"].inputs.image, ["2", 0]);
@@ -69,12 +69,17 @@ test("relighting the video's own person gives FLUX and the upsampler only the ra
assert.equal(graph["15"].inputs["images.image1"], undefined);
assert.equal(graph["15"].inputs["resolution.megapixels"], 3);
assert.deepEqual(graph["19"].inputs.latent_image, ["15", 2]);
+ const kept = load("prepare");
+ applyFrameRelight(kept, LIGHT_LOOKS[0].light, 0.9);
+ assert.equal(kept["19"].inputs.denoise, 0.9);
+ assert.deepEqual(kept["19"].inputs.latent_image, ["relight_start", 0]);
+ assert.deepEqual(kept["relight_frame"].inputs.image, ["2", 0]);
});
test("picture pipelines have valid links and no disconnected processing nodes", () => {
const relit = load("prepare"), frame = load("prepare");
applyRelight(relit, LIGHT_LOOKS[0].light);
- applyFrameRelight(frame, LIGHT_LOOKS[0].light);
+ applyFrameRelight(frame, LIGHT_LOOKS[0].light, 0.9);
for (const [stage, graph] of [...["prepare", "motion", "solar"].map((stage) => [stage, load(stage)] as const), ["relight", relit] as const, ["frame relight", frame] as const]) {
const reached = new Set<string>();
function visit(id: string) {
diff --git a/workflows/picture/README.md b/workflows/picture/README.md
index 19bbb35..7ab1316 100644
--- a/workflows/picture/README.md
+++ b/workflows/picture/README.md
@@ -46,7 +46,7 @@ The image-aware prompt upsampler sees both references and expands that request i
**Relight** (off by default) poses and relights the character in the same FLUX pass; a second edit on top of the prepared picture degrades it. Pick one of nine light looks or write your own. The app then sends the prepare workflow with these changes (`applyRelight` in `lib/picture.ts`): FLUX gets only image 1, resized to 3 MP; image 2 reaches it only through the upsampler's written pose; the canvas is image 2's shape at 3 MP; the request's `{look}` is replaced by the light. The light usually changes the background too, and results vary a lot by seed. Relit pictures save as `relit_frame_*.png`.
-**Relight the video's own person** turns the character picture off. FLUX gets only the raw frame at 3 MP, the upsampler sees the same frame, and the request asks for the same person, pose, framing and setting under the chosen light. The relit frame is saved as `relit_video_frame_*` and renders like any prepared picture.
+**Relight the video's own person** turns the character picture off. FLUX gets only the raw frame at 3 MP, the upsampler sees the same frame, and the request asks for the same person, pose, framing and setting under the chosen light. **Framing** decides how free FLUX is: by default it may recompose the shot (strongest light, but it can zoom out to fit a light above the head); **Keep the frame's crop** starts sampling from the frame at denoise 0.9, which keeps the crop but can leave the old image showing through, such as see-through hair under Cold beam. The relit frame is saved as `relit_video_frame_*` and renders like any prepared picture.
Preparation defaults to a 768×1376 canvas and saves under `ComfyUI/output/openjutsu_picture/flux3/prepared_frame_*.png`. Compare the prepared picture against the raw first-frame preview before using it for video; a successful execution does not guarantee that the requested pose was followed.