Skip to content

One-click character swap for videos: an open-source, local Higgsfield Genjutsu alternative built on ComfyUI and MiniMax H3, with FLUX.3 Klein relighting.

15.0 KB
Raw Blame
import { HttpError, validateJobId, type StoredFile } from "./http.ts";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import { Asset, Comfy, COMFY_CLOUD_BASE_URL, NotFound, resolveCredentials, type Job, type Workflow } from "@comfyorg/sdk";
import { ComfyLow } from "@comfyorg/sdk/low";
import { MASK_NODES, NODES, cropSource, isLink, pruneMask, pruneSimple, setSection, type ApiGraph, type Margin, type Resolution, type Section, type SimpleSettings } from "./simple.ts";
import { VIDEO, nodeByTitle, pruneAdvanced, type AdvancedSettings } from "./advanced.ts";
import { applyDescriber, applyModel, type ModelFiles, type ModelSettings } from "./model.ts";
import { OBJECT_PICTURE, OBJECT_VIDEO, pruneFit, pruneObject, type ObjectSettings, type PlacementKeyframe } from "./object.ts";
import { FACE_PICTURE, FACE_VIDEO, pruneFace, type FaceSettings } from "./face.ts";
import { COLOR_VIDEO, pruneColor, type ColorSettings } from "./color.ts";
import { PEOPLE_NODES, peopleRoles, pruneMontage, type Review } from "./montage.ts";

export type PublicJob = {
  id: string;
  status: string;
  queuePosition: number | null;
  error: string | null;
  video: { id: string; name: string; url: string } | null;
  data: string | null; // JSON: what Find people found, or who a montage kept as they are
};

export const TERMINAL = new Set(["succeeded", "failed", "canceled", "expired"]);

// undici clones any request it may redirect, teeing the body into a copy nobody reads, so a whole upload would wait in
// memory. Uploads are never redirected.
const fetchUploads: typeof fetch = (input, init) => fetch(input, init?.body instanceof FormData ? { ...init, redirect: "error" } : init);

// COMFY_BASE_URL and the optional COMFY_API_KEY come from .env.local; the SDK reads both.
export function client() {
  return new Comfy({ clientInfo: "openjutsu", fetch: fetchUploads });
}

// The same deployment as client(); the SDK's resolveBaseUrl() is the Comfy Router's, not COMFY_BASE_URL.
function low() {
  return new ComfyLow(process.env.COMFY_BASE_URL?.trim().replace(/\/+$/, "") || COMFY_CLOUD_BASE_URL, resolveCredentials(), { clientInfo: "openjutsu" });
}

// job.events() drops progress frames whose value is lower than the last one, but the proxy
// reports each node's own step fraction, so the next node's progress would be hidden. The raw
// stream has every frame; the browser's EventSource reconnects when it ends.
export function rawJobEvents(job: Job, signal: AbortSignal) {
  return low().getJobEvents(job.urls.events || job.id, { signal });
}

export const JOB_KINDS = ["simple", "mask", "advanced", "advanced_maxq", "object", "fit", "face", "color", "people", "montage"] as const;
export type JobKind = (typeof JOB_KINDS)[number];

export async function loadGraph(kind: JobKind): Promise<ApiGraph> {
  const graph: ApiGraph = JSON.parse(await readFile(join(process.cwd(), "workflows", `${kind}.api.json`), "utf8"));
  silenceWhenSoundless(graph);
  return graph;
}

// VHS fails as soon as anything reads the soundtrack of a video that has none, so every reader gets silence instead.
function silenceWhenSoundless(graph: ApiGraph) {
  for (const id of Object.keys(graph).filter((id) => graph[id].class_type === "VHS_LoadVideo")) {
    const soundtrack = `${id}_soundtrack`;
    let read = false;
    for (const node of Object.values(graph)) {
      for (const [name, value] of Object.entries(node.inputs)) {
        if (isLink(value) && value[0] === id && value[1] === 2) {
          node.inputs[name] = [soundtrack, 0];
          read = true;
        }
      }
    }
    if (read) graph[soundtrack] = { class_type: "H3SoundtrackOrSilence", inputs: { audio: [id, 2], video_info: [id, 3] }, _meta: { title: "Soundtrack (silence if none)" } };
  }
}

function jobErrorMessage(error: unknown) {
  const message = error && typeof error === "object" && typeof Reflect.get(error, "message") === "string"
    ? (Reflect.get(error, "message") as string)
    : typeof error === "string" ? error : "";
  const missingNode = message.match(/Node ['"]([^'"]+)['"] not found/i)?.[1];
  if (missingNode) return `ComfyUI is missing the "${missingNode}" node. Install the custom nodes listed in the README and restart ComfyUI.`;
  return message || "ComfyUI could not run this workflow. Check the ComfyUI console.";
}

export function publicJob(job: Job): PublicJob {
  const video = job.outputs.find((output) => output.contentType.startsWith("video/"));
  const data = job.outputs.find((output) => output.contentType === "application/json");
  return {
    id: job.id,
    status: job.status,
    queuePosition: job.queuePosition,
    error: job.error ? jobErrorMessage(job.error) : null,
    video: video ? { id: video.id, name: video.name, url: `/api/jobs/${job.id}/outputs/${video.id}` } : null,
    data: data ? `/api/jobs/${job.id}/outputs/${data.id}` : null,
  };
}

// The SDK streams the temp file from disk and names the upload after it.
function upload(comfy: Comfy, file: StoredFile) {
  return comfy.assets.fromFile(file.path);
}

// Inputs this server uploaded, with the jobs still using each. The proxy hands identical bytes the same asset id,
// so an input is deleted only once no job here uses it, and only if this server uploaded it rather than reused it.
const inputs = ((globalThis as { [key: symbol]: unknown })[Symbol.for("openjutsu.inputs")] ??= new Map()) as Map<string, { jobs: number; owned: boolean }>;

function hold(asset: Asset) {
  const entry = inputs.get(asset.id!) ?? { jobs: 0, owned: false };
  inputs.set(asset.id!, { jobs: entry.jobs + 1, owned: entry.owned || asset.createdNew === true });
}

async function release(comfy: Comfy, ids: string[]) {
  for (const id of ids) {
    const entry = inputs.get(id);
    if (!entry) continue;
    if (entry.jobs > 1) {
      inputs.set(id, { ...entry, jobs: entry.jobs - 1 });
      continue;
    }
    inputs.delete(id);
    if (entry.owned) await comfy.assets.delete(id).catch(() => {});
  }
}

// Retry and every other run upload their inputs again from the page, so nothing reuses them after a terminal status.
async function releaseWhenDone(comfy: Comfy, job: Job, ids: string[]) {
  for (;;) {
    try {
      await job.wait();
      break;
    } catch (error) {
      // The proxy forgets jobs when it restarts.
      if (error instanceof NotFound) break;
      await sleep(30_000);
    }
  }
  await release(comfy, ids);
}

async function submit(comfy: Comfy, workflow: Workflow) {
  const assets = Object.values(workflow.json).flatMap((node) => Object.values((node as { inputs?: object }).inputs ?? {})).filter((value) => value instanceof Asset);
  const ids: string[] = [];
  try {
    for (const asset of assets) {
      ids.push(await asset.commit());
      hold(asset);
    }
    const job = await comfy.submit(workflow);
    void releaseWhenDone(comfy, job, ids);
    return publicJob(job);
  } catch (error) {
    await release(comfy, ids);
    throw error;
  }
}

export async function submitSimple(video: StoredFile, section: Section, picture: StoredFile, settings: SimpleSettings, model: ModelSettings) {
  const comfy = client();
  const graph = pruneSimple(await loadGraph("simple"), settings);
  applyModel(graph, model);
  setSection(graph, NODES.video, section);
  const workflow = comfy.workflows.fromJson(graph);
  workflow.setInput(NODES.video, "video", upload(comfy, video));
  workflow.setInput(NODES.picture, "image", upload(comfy, picture));
  return submit(comfy, workflow);
}

export async function submitAdvanced(video: StoredFile, section: Section, pictures: StoredFile[], settings: AdvancedSettings, maxQuality: boolean, model: ModelSettings) {
  const comfy = client();
  const graph = pruneAdvanced(await loadGraph(maxQuality ? "advanced_maxq" : "advanced"), pictures.length, settings);
  applyModel(graph, model);
  setSection(graph, nodeByTitle(graph, VIDEO), section);
  const workflow = comfy.workflows.fromJson(graph);
  workflow.setInput(nodeByTitle(graph, VIDEO), "video", upload(comfy, video));
  for (const [i, picture] of pictures.entries()) workflow.setInput(nodeByTitle(graph, `Picture ${i + 1}`), "image", upload(comfy, picture));
  return submit(comfy, workflow);
}

export async function submitObject(video: StoredFile, section: Section, picture: StoredFile | null, settings: ObjectSettings, model: ModelSettings) {
  const comfy = client();
  const graph = pruneObject(await loadGraph("object"), !!picture, settings);
  applyModel(graph, model);
  setSection(graph, nodeByTitle(graph, OBJECT_VIDEO), section);
  const workflow = comfy.workflows.fromJson(graph);
  workflow.setInput(nodeByTitle(graph, OBJECT_VIDEO), "video", upload(comfy, video));
  if (picture && settings.mode !== "remove") workflow.setInput(nodeByTitle(graph, OBJECT_PICTURE), "image", upload(comfy, picture));
  return submit(comfy, workflow);
}

export async function submitFit(video: StoredFile, section: Section, placement: PlacementKeyframe[], resolution: Resolution, action = "") {
  const comfy = client();
  const graph = pruneFit(await loadGraph("fit"), placement, action);
  if (typeof resolution === "object") cropSource(graph, nodeByTitle(graph, OBJECT_VIDEO), resolution);
  setSection(graph, nodeByTitle(graph, OBJECT_VIDEO), section);
  const workflow = comfy.workflows.fromJson(graph);
  workflow.setInput(nodeByTitle(graph, OBJECT_VIDEO), "video", upload(comfy, video));
  return submit(comfy, workflow);
}

export async function submitFace(video: StoredFile, section: Section, picture: StoredFile, settings: FaceSettings, model: ModelSettings) {
  const comfy = client();
  const graph = pruneFace(await loadGraph("face"), settings);
  applyModel(graph, model);
  setSection(graph, nodeByTitle(graph, FACE_VIDEO), section);
  const workflow = comfy.workflows.fromJson(graph);
  workflow.setInput(nodeByTitle(graph, FACE_VIDEO), "video", upload(comfy, video));
  workflow.setInput(nodeByTitle(graph, FACE_PICTURE), "image", upload(comfy, picture));
  return submit(comfy, workflow);
}

export async function submitColor(video: StoredFile, section: Section, settings: ColorSettings) {
  const comfy = client();
  const graph = pruneColor(await loadGraph("color"), settings);
  setSection(graph, nodeByTitle(graph, COLOR_VIDEO), section);
  const workflow = comfy.workflows.fromJson(graph);
  workflow.setInput(nodeByTitle(graph, COLOR_VIDEO), "video", upload(comfy, video));
  return submit(comfy, workflow);
}

export async function submitMask(video: StoredFile, section: Section, hatFix: boolean, margin: Margin, target: string) {
  const comfy = client();
  const graph = pruneMask(await loadGraph("mask"), hatFix, margin, target);
  setSection(graph, MASK_NODES.video, section);
  const workflow = comfy.workflows.fromJson(graph);
  workflow.setInput(MASK_NODES.video, "video", upload(comfy, video));
  return submit(comfy, workflow);
}

export async function submitMontage(video: StoredFile, section: Section, pictures: StoredFile[], settings: AdvancedSettings, review: Review | null, model: ModelSettings) {
  const comfy = client();
  const graph = pruneMontage(await loadGraph("montage"), pictures.length, settings, review);
  applyModel(graph, model);
  setSection(graph, nodeByTitle(graph, VIDEO), section);
  const workflow = comfy.workflows.fromJson(graph);
  workflow.setInput(nodeByTitle(graph, VIDEO), "video", upload(comfy, video));
  for (const [i, picture] of pictures.entries()) workflow.setInput(nodeByTitle(graph, `Picture ${i + 1}`), "image", upload(comfy, picture));
  return submit(comfy, workflow);
}

export async function submitPeople(video: StoredFile, section: Section, roles: string[], resolution: Resolution, describer?: string) {
  const comfy = client();
  const graph = await loadGraph("people");
  applyDescriber(graph, describer);
  graph[PEOPLE_NODES.crops].inputs.people = peopleRoles(roles);
  if (typeof resolution === "object") cropSource(graph, PEOPLE_NODES.video, resolution);
  setSection(graph, PEOPLE_NODES.video, section);
  const workflow = comfy.workflows.fromJson(graph);
  workflow.setInput(PEOPLE_NODES.video, "video", upload(comfy, video));
  return submit(comfy, workflow);
}

// COMFYUI_URL points at ComfyUI itself, not the proxy: the proxy lists no nodes, models or presets.
export async function comfyUi(path: string, timeout: number) {
  const url = (process.env.COMFYUI_URL?.trim() || "http://127.0.0.1:8188").replace(/\/+$/, "");
  const response = await fetch(`${url}${path}`, { cache: "no-store", signal: AbortSignal.timeout(timeout) });
  if (!response.ok) throw new Error(`ComfyUI answered ${path} with HTTP ${response.status}.`);
  return response.json();
}

export const MODELS_UNAVAILABLE = "Could not read the model lists from ComfyUI. Make sure it is running and COMFYUI_URL points to it, then try again.";

// The pickers read ComfyUI's own node info; when that fails, the page falls back to typed file names.
export async function listModels(): Promise<ModelFiles> {
  const options = async (node: string, input: string) => {
    const names = (await comfyUi(`/object_info/${node}`, 5000))?.[node]?.input?.required?.[input]?.[0];
    if (!Array.isArray(names) || names.some((name) => typeof name !== "string")) throw new Error(`ComfyUI sent no ${input} list.`);
    return names as string[];
  };
  try {
    const [models, loras, textEncoders, vaes] = await Promise.all([
      options("UNETLoader", "unet_name"), options("LoraLoaderModelOnly", "lora_name"),
      options("CLIPLoader", "clip_name"), options("VAELoader", "vae_name"),
    ]);
    return { models, loras, textEncoders, vaes };
  } catch (error) {
    console.warn(`${MODELS_UNAVAILABLE} ${error instanceof Error ? error.message : ""}`.trim());
    throw new HttpError(502, MODELS_UNAVAILABLE);
  }
}

// Orion4D's saved DCTL Tone Mapper presets, read from ComfyUI like the model lists; empty when it does not answer.
export async function listTonePresets(): Promise<{ name: string; settings: Record<string, unknown> }[]> {
  const get = (path: string) => comfyUi(`/orion4d_fxmax/${path}`, 5000);
  try {
    const { presets } = await get("presets/color_fx_dctl_tone_mapper");
    return await Promise.all((presets as string[]).map(async (name) => ({ name, settings: (await get(`preset/color_fx_dctl_tone_mapper/${encodeURIComponent(name)}`)).settings })));
  } catch {
    return [];
  }
}

export function getJob(id: string) {
  validateJobId(id);
  return client().jobs.get(id);
}

export async function getOutput(jobId: string, outputId: string, range: string | null) {
  const job = await getJob(jobId);
  const output = job.outputs.find((candidate) => candidate.id === outputId);
  // A running job lists no outputs yet; its mask previews arrive as live output events, so any other id must name an
  // asset this job produced. Uploaded inputs have no producing job.
  const asset = output ? null : await low().getAsset(outputId).catch(() => null);
  if (!output && (!asset || asset.job_id !== job.id)) throw new HttpError(404, "This job has no such output.");
  const { url } = output ? await output.getDownloadUrl() : await low().getAssetContentUrl(outputId);
  return { name: output?.name ?? asset?.file_path ?? outputId, response: await fetch(url, { headers: range ? { Range: range } : {} }) };
}