Skip to content

One-click character swap for videos: an open-source, local Higgsfield Genjutsu alternative built on ComfyUI and MiniMax H3, with FLUX.3 Klein relighting.

11.9 KB
Raw Blame
import { randomUUID } from "node:crypto";
import { openAsBlob } from "node:fs";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { apiAccess, HttpError, IMAGE_SIGNATURE, readFormData, StoredFile, validateJobId, VIDEO_SIGNATURE } from "../../../lib/http.ts";
import { canvasSize, isLink, MAX_SEED, type ApiGraph } from "../../../lib/simple.ts";
import { listModels } from "../../../lib/comfy.ts";
import { applyFrameRelight, applyPictureVideoSettings, applyRelight, DEFAULT_PICTURE_VIDEO, parsePictureVideoSettings, pictureLoras, solarLoraFiles, type PictureVideoSettings } from "../../../lib/picture.ts";

export const runtime = "nodejs";
const PREPARED = "OpenJutsu · prepared picture";
const RENDERED = "OpenJutsu · picture motion";
const FILE_CHECKS = {
  video: { signature: VIDEO_SIGNATURE, error: "Choose an MP4, MOV, MKV, WebM or AVI video." },
  picture: { signature: IMAGE_SIGNATURE, error: "Choose a PNG, JPEG or WebP character picture." },
};
type OutputFile = { filename: string; subfolder?: string; type?: string };
type History = { prompt?: [number, string, ApiGraph]; status?: { status_str?: string; completed?: boolean }; outputs?: Record<string, { images?: OutputFile[]; gifs?: OutputFile[] }> };

async function comfy(path: string, init?: RequestInit) {
  const base = (process.env.COMFYUI_URL?.trim() || "http://127.0.0.1:8188").replace(/\/+$/, "");
  const response = await fetch(`${base}${path}`, { ...init, cache: "no-store", redirect: "error", signal: AbortSignal.timeout(30_000) });
  if (!response.ok) {
    const data = await response.json().catch(() => null);
    const details = Object.values(data?.node_errors ?? {}).flatMap((node) => (node as { errors?: { message: string; details?: string }[] }).errors ?? [])
      .map((error) => `${error.message}${error.details ? `: ${error.details}` : ""}`).join("; ");
    throw new HttpError(502, details || data?.error?.message || `ComfyUI answered with HTTP ${response.status}. Check its console.`);
  }
  return response;
}

async function template(stage: "prepare" | "motion" | "solar"): Promise<ApiGraph> {
  return JSON.parse(await readFile(join(process.cwd(), "workflows", "picture", `from_picture_${stage}_flux3.api.json`), "utf8"));
}

function view(file: OutputFile) {
  return `/view?${new URLSearchParams({ filename: file.filename, subfolder: file.subfolder ?? "", type: file.type ?? "output" })}`;
}

async function uploadFile(blob: Blob, extension: string) {
  const form = new FormData();
  form.set("image", blob, `openjutsu_picture_${randomUUID()}${extension}`);
  form.set("type", "input");
  const asset = await (await comfy("/upload/image", { method: "POST", body: form })).json();
  return asset.subfolder ? `${asset.subfolder}/${asset.name}` : asset.name;
}

async function history(id: string) {
  validateJobId(id);
  const data = await (await comfy(`/history/${encodeURIComponent(id)}`)).json();
  const item: History | undefined = data[id];
  if (item) {
    const graph = item.prompt?.[2];
    const prepared = graph?.["21"]?.class_type === "SaveImage" && graph["21"]._meta?.title === PREPARED;
    const rendered = graph?.["63"]?.class_type === "VHS_VideoCombine" && graph["63"]._meta?.title === RENDERED;
    if (!prepared && !rendered) throw new HttpError(404, "This is not an OpenJutsu picture job.");
    if (item.status?.status_str === "error") throw new HttpError(422, "Generation failed. Check the ComfyUI console for missing models or memory errors.");
  }
  return item;
}

function failure(error: unknown) {
  return Response.json({ error: error instanceof HttpError ? error.message : "Could not reach ComfyUI. Start it locally and try again." }, { status: error instanceof HttpError ? error.status : 502 });
}

let uploading = false;

export async function POST(request: Request) {
  const denied = apiAccess(request);
  if (denied) return denied;
  if (uploading) return Response.json({ error: "Another picture request is in progress. Try again shortly." }, { status: 429 });
  uploading = true;
  let upload: Awaited<ReturnType<typeof readFormData>> | undefined;
  try {
    let graph: ApiGraph;
    if (request.headers.get("Content-Type")?.startsWith("application/json")) {
      const body = await request.json();
      if (typeof body.prepared !== "string") throw new HttpError(400, "Prepare a picture first.");
      const item = await history(body.prepared);
      const file = item?.outputs?.["21"]?.images?.[0];
      const source = item?.prompt?.[2];
      if (!file || !source || source["21"]?._meta?.title !== PREPARED || source["1"]?.class_type !== "VHS_LoadVideo") throw new HttpError(409, "The prepared picture is not ready. Generate it first.");
      let settings: PictureVideoSettings;
      try { settings = parsePictureVideoSettings(body.settings ?? { ...DEFAULT_PICTURE_VIDEO, seed: source["19"].inputs.seed }); }
      catch (error) { throw new HttpError(400, (error as Error).message); }
      const available = await listModels();
      if (!available.models.includes(settings.model)) throw new HttpError(400, "Choose a video model ComfyUI has.");
      if (settings.renderer === "solar") {
        const installed = solarLoraFiles(settings.solarRank, available.loras);
        const defaults = solarLoraFiles(settings.solarRank);
        for (const key of ["solarFourStep", "solarControl"] as const) if (settings[key] === defaults[key]) settings[key] = installed[key];
        const nodes = await (await comfy("/object_info/SolarRendererLoraLoader")).json();
        if (!nodes.SolarRendererLoraLoader) throw new HttpError(400, "Install the SolarRenderer custom nodes and restart ComfyUI: https://github.com/Enderfga/SolarRenderer");
      }
      if (pictureLoras(settings).some((name) => !available.loras.includes(name))) throw new HttpError(400, "Choose LoRA files installed in ComfyUI/models/loras (subfolders are supported).");
      const [clipInfo, vaeInfo] = await Promise.all([comfy("/object_info/CLIPLoader").then((response) => response.json()), comfy("/object_info/VAELoader").then((response) => response.json())]);
      for (const [name, choices, label] of [
        [settings.textEncoder, clipInfo.CLIPLoader.input.required.clip_name[0], "H3 text encoder"],
        [settings.videoVae, vaeInfo.VAELoader.input.required.vae_name[0], "video VAE"],
        ...(settings.renderer === "h3" ? [[settings.audioVae, vaeInfo.VAELoader.input.required.vae_name[0], "audio VAE"] as const] : []),
      ] as const) if (!choices.includes(name)) throw new HttpError(400, `Choose a ${label} ComfyUI has.`);
      const picture = await (await comfy(view(file))).blob();
      graph = await template(settings.renderer === "solar" ? "solar" : "motion");
      graph["20"] = { class_type: "LoadImage", inputs: { image: await uploadFile(picture, ".png") } };
      graph["1"].inputs.video = source["1"].inputs.video;
      graph["1"].inputs.skip_first_frames = source["1"].inputs.skip_first_frames;
      applyPictureVideoSettings(graph, settings, Number(source["3"].inputs.width), Number(source["3"].inputs.height));
      graph["63"]._meta = { title: RENDERED };
      const keep = new Set<string>();
      function visit(id: string) {
        if (keep.has(id)) return;
        keep.add(id);
        for (const value of Object.values(graph[id].inputs)) if (isLink(value)) visit(value[0]);
      }
      visit("63");
      graph = Object.fromEntries(Object.entries(graph).filter(([id]) => keep.has(id)));
    } else {
      upload = await readFormData(request, FILE_CHECKS);
      const form = upload.form, video = form.get("video"), picture = form.get("picture"), light = form.get("light");
      const lit = typeof light === "string" ? light.trim() : "", ownPerson = form.get("relight") === "frame";
      if (!(video instanceof StoredFile) || !video.size) throw new HttpError(400, "Choose a video.");
      if (ownPerson && !lit) throw new HttpError(400, "Choose a light for the video's own person.");
      const denoise = Number(form.get("denoise") ?? 1);
      if (!(denoise > 0 && denoise <= 1)) throw new HttpError(400, "Use a denoise above 0 and up to 1.");
      const character = !ownPerson && picture instanceof StoredFile && picture.size ? picture : null;
      if (!ownPerson && !character) throw new HttpError(400, "Choose a character picture.");
      const start = Number(form.get("start") ?? 0), seed = Number(form.get("seed") ?? 0);
      const width = Number(form.get("width")), height = Number(form.get("height"));
      if (!Number.isFinite(start) || start < 0 || !Number.isInteger(seed) || seed < 0 || seed > MAX_SEED) throw new HttpError(400, "Use a start time of 0 or later and a valid whole-number seed.");
      if (![width, height].every((side) => Number.isFinite(side) && side > 0)) throw new HttpError(400, "Wait for the video's dimensions to load.");
      const extension = video.name.match(/\.(mp4|mov|mkv|webm|avi)$/i)?.[0];
      const pictureExtension = character?.name.match(/\.(png|jpe?g|webp)$/i)?.[0];
      if (!extension || character && !pictureExtension) throw new HttpError(400, "Choose a supported video and PNG, JPEG or WebP picture.");
      graph = await template("prepare");
      const prompt = form.get("prompt");
      if (typeof prompt === "string" && prompt.trim()) graph["6"].inputs.value = prompt;
      const [editWidth, editHeight] = canvasSize(width, height, "768p");
      for (const id of ["3", "5"]) Object.assign(graph[id].inputs, { width: editWidth, height: editHeight });
      graph["1"].inputs.video = await uploadFile(await openAsBlob(video.path), extension.toLowerCase());
      graph["1"].inputs.skip_first_frames = Math.round(start * 24);
      if (character) graph["4"].inputs.image = await uploadFile(await openAsBlob(character.path), pictureExtension!.toLowerCase());
      graph["19"].inputs.seed = seed;
      graph["21"]._meta = { title: PREPARED };
      if (ownPerson) applyFrameRelight(graph, lit, denoise);
      else if (lit) applyRelight(graph, lit);
    }
    const result = await (await comfy("/prompt", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ prompt: graph }) })).json();
    return Response.json({ id: result.prompt_id }, { status: 202 });
  } catch (error) {
    return failure(error);
  } finally {
    await upload?.dispose();
    uploading = false;
  }
}

export async function GET(request: Request) {
  const denied = apiAccess(request);
  if (denied) return denied;
  try {
    const url = new URL(request.url), id = url.searchParams.get("id") ?? "";
    const item = await history(id);
    if (!item) {
      const queue = await (await comfy("/queue")).json();
      const waiting = [...queue.queue_running, ...queue.queue_pending].some((entry: unknown[]) => entry[1] === id);
      if (!waiting) throw new HttpError(404, "This job was removed or ComfyUI restarted. Generate the picture again.");
      return Response.json({ status: "processing" });
    }
    const isVideo = item.prompt?.[2]?.["63"]?._meta?.title === RENDERED;
    const media = url.searchParams.get("media");
    const output = item.outputs?.[isVideo ? "63" : media === "frame" ? "22" : "21"];
    const file = isVideo ? output?.gifs?.[0] : output?.images?.[0];
    if (!file) throw new HttpError(422, "ComfyUI returned no result. Check its console.");
    const base = `/api/picture?id=${encodeURIComponent(id)}`;
    if (!media) return Response.json({ status: "ready", url: `${base}&media=result`, ...(!isVideo ? { frame: `${base}&media=frame`, width: item.prompt![2]["3"].inputs.width, height: item.prompt![2]["3"].inputs.height } : {}) });
    const response = await comfy(view(file), { headers: request.headers.has("Range") ? { Range: request.headers.get("Range")! } : {} });
    return new Response(response.body, { status: response.status, headers: {
      "Content-Type": isVideo ? "video/mp4" : "image/png", "X-Content-Type-Options": "nosniff", "Cache-Control": "private, no-store",
      ...Object.fromEntries(["Content-Length", "Content-Range", "Accept-Ranges"].flatMap((key) => response.headers.has(key) ? [[key, response.headers.get(key)!]] : [])),
    } });
  } catch (error) { return failure(error); }
}