The open creative studio on ComfyUI: images, video, 3D and audio on open weights, on your own GPU or the cloud. Camera motions, effects, apps, Cinema, Layers, Image to 3D and an AI agent.
TypeScript ai-videocomfyuicreative-toolsgenerative-aiideogramimage-generationimage-to-3dimage-to-videoltx-videominimax-h3nextjsopen-sourceopen-weightsqwen-imagetext-to-imagevideo-generationwan
import fs from "node:fs";
import net from "node:net";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { GpuWatch, othersOf, ownOf, parseApps, parseGpu } from "../bin/lib/worker/gpu.mjs";
import { cleanJob, vetGraph } from "../bin/lib/worker/checks.mjs";
import { comfyArgs, OwnComfy, PortBusy, portFree } from "../bin/lib/worker/comfy.mjs";
import { parseFor, syncedFolder } from "../bin/lib/worker/config.mjs";
import { worker } from "../bin/lib/worker/index.mjs";
import { localPause } from "../bin/lib/worker/run.mjs";
import { inventoryHash as workerHash, missingModelDirs } from "../bin/lib/worker/inventory.mjs";
import { tempDir } from "./helpers/temp-dir";
const MIN = 60_000;
const PORT = 3367;
describe("the GPU check (check 11)", () => {
it("reads nvidia-smi, [N/A] included", () => {
expect(parseGpu("6144, 61\n")).toEqual({ usedMb: 6144, tempC: 61, busyPct: null });
// How busy it is: its utilization, kept from 0 to 100 ([N/A] is unknown).
expect(parseGpu("6144, 61, 97\n")).toEqual({ usedMb: 6144, tempC: 61, busyPct: 97 });
expect(parseGpu("6144, 61, [N/A]\n")).toEqual({ usedMb: 6144, tempC: 61, busyPct: null });
expect(parseApps("1234, C:\\Program Files\\Blender\\blender.exe, [N/A]\n77, python.exe, 2048\n")).toEqual([
{ pid: 1234, name: "blender.exe", usedMb: null },
{ pid: 77, name: "python.exe", usedMb: 2048 },
]);
});
it("counts others' VRAM as the total minus its own ComfyUI's", () => {
const own = { running: true, mb: 9000, pid: 77 };
expect(othersOf({ usedMb: 9800, apps: [{ pid: 77, name: "python.exe", usedMb: null }] }, own, 1024).busy).toBe(false);
expect(othersOf({ usedMb: 12000, apps: [{ pid: 77, name: "python.exe", usedMb: null }, { pid: 5, name: "blender.exe", usedMb: null }] }, own, 1024)).toMatchObject({ busy: true, who: "blender.exe" });
// Stopped, it holds nothing: a ComfyUI on 8188 that keeps its models loaded keeps the worker paused.
expect(othersOf({ usedMb: 4000, apps: [] }, { running: false }, 1024).busy).toBe(true);
expect(othersOf({ usedMb: 300, apps: [{ pid: 9, name: "game.exe", usedMb: 2000 }] }, { running: false }, 1024)).toMatchObject({ busy: true, who: "game.exe" });
});
it("during its own job, its memory that torch doesn't report (or a slow read) is no other program's: an unsure reading", () => {
// A 20 GB model in ComfyUI's dynamic VRAM: torch reports 2 GB, or nothing when ComfyUI is slow to answer.
const reading = { usedMb: 23_000, apps: [{ pid: 9, name: "ShellHost.exe", usedMb: null }] };
for (const mb of [2000, null]) {
expect(othersOf(reading, { running: true, mb, pid: 77, job: true }, 1024)).toMatchObject({ busy: true, unsure: true });
// Between jobs it still counts (no job to lose): the busy pause, as before.
expect(othersOf(reading, { running: true, mb, pid: 77 }, 1024)).toMatchObject({ busy: true, unsure: false });
}
// nvidia-smi's number for its own process, when it gives one, is its share: sure.
expect(ownOf({ running: true, mb: 2000, pid: 77 }, [{ pid: 77, name: "python", usedMb: 21_500 }])).toEqual({ mb: 21_500, sure: true });
expect(othersOf({ usedMb: 23_000, apps: [{ pid: 77, name: "python", usedMb: 21_500 }] }, { running: true, mb: 2000, pid: 77, job: true }, 1024)).toMatchObject({ busy: true, unsure: false });
expect(othersOf({ usedMb: 22_000, apps: [{ pid: 77, name: "python", usedMb: 21_500 }] }, { running: true, mb: 2000, pid: 77, job: true }, 1024).busy).toBe(false);
// A desktop process is never "loud" on its own, nor named.
expect(othersOf({ usedMb: 2500, apps: [{ pid: 8, name: "msedge.exe", usedMb: 2000 }, { pid: 9, name: "dwm.exe", usedMb: null }] }, { running: false }, 1024)).toMatchObject({ busy: true, who: "another program" });
expect(othersOf({ usedMb: 300, apps: [{ pid: 8, name: "Code.exe", usedMb: 2000 }] }, { running: false }, 1024).busy).toBe(false);
// Another process that shows its own use is sure, job or not.
expect(othersOf({ usedMb: 23_000, apps: [{ pid: 5, name: "game.exe", usedMb: 4000 }] }, { running: true, mb: 2000, pid: 77, job: true }, 1024)).toMatchObject({ busy: true, unsure: false, who: "game.exe" });
});
it("skips unsure readings: they neither pause nor end a pause", () => {
const g = new GpuWatch({ autoResume: true, resumeAfterMs: 10 * MIN, resumeBelowC: 75, busyReadings: 2 });
for (let t = 0; t < 60_000; t += 5000) g.update({ busy: true, unsure: true, who: "another program", job: true }, 60, t);
expect(g.paused).toBe(false);
expect(g.busyInARow).toBe(0);
});
it("during its own job, busy readings pause it only once they lasted the grace time; a game that stays still pauses it", () => {
const opts = { autoResume: true, resumeAfterMs: 10 * MIN, resumeBelowC: 75, busyReadings: 2, graceMs: 20_000 };
const g = new GpuWatch(opts);
for (let t = 0; t <= 15_000; t += 5000) g.update({ busy: true, who: "game", job: true }, 60, t);
expect(g.paused).toBe(false);
g.update({ busy: false, job: true }, 60, 20_000);
g.update({ busy: true, who: "game", job: true }, 60, 25_000);
g.update({ busy: true, who: "game", job: true }, 60, 30_000);
expect(g.paused).toBe(false);
g.update({ busy: true, who: "game", job: true }, 60, 45_000);
expect(g.paused).toBe(true);
expect(g.who).toBe("game");
// With no job, two readings, as before.
const idle = new GpuWatch(opts);
idle.update({ busy: true, who: "game" }, 60, 0);
idle.update({ busy: true, who: "game" }, 60, 5000);
expect(idle.paused).toBe(true);
});
it("a brief pause (the busy readings ended within 30 s) cools down 2 minutes, at most once an hour; one that stays, the whole 10", () => {
const opts = { autoResume: true, resumeAfterMs: 10 * MIN, resumeBelowC: 75, briefMs: 30_000, briefResumeMs: 2 * MIN, briefEveryMs: 60 * MIN };
const g = new GpuWatch(opts);
g.update({ busy: true, who: "x" }, 60, 0);
g.update({ busy: false }, 50, 5000);
g.update({ busy: false }, 50, 5000 + 2 * MIN - 1);
expect(g.paused).toBe(true);
g.update({ busy: false }, 50, 5000 + 2 * MIN);
expect(g.paused).toBe(false);
// A second brief one within the hour: the whole cool-down.
g.update({ busy: true, who: "x" }, 60, 10 * MIN);
g.update({ busy: false }, 50, 10 * MIN + 5000);
g.update({ busy: false }, 50, 10 * MIN + 5000 + 2 * MIN);
expect(g.paused).toBe(true);
g.update({ busy: false }, 50, 20 * MIN + 5000);
expect(g.paused).toBe(false);
// A program that stays past 30 s: the whole cool-down.
const stays = new GpuWatch(opts);
stays.update({ busy: true, who: "x" }, 60, 0);
stays.update({ busy: true, who: "x" }, 60, 40_000);
stays.update({ busy: false }, 50, 45_000);
stays.update({ busy: false }, 50, 45_000 + 2 * MIN);
expect(stays.paused).toBe(true);
stays.update({ busy: false }, 50, 45_000 + 10 * MIN);
expect(stays.paused).toBe(false);
});
it("pauses at once; with the automatic resume off, never resumes by itself", () => {
const g = new GpuWatch({ autoResume: false, resumeAfterMs: 10 * MIN, resumeBelowC: 75 });
g.update({ busy: true, who: "blender.exe" }, 60, 0);
expect(g.paused).toBe(true);
g.update({ busy: false }, 40, 1);
g.update({ busy: false }, 40, 120 * MIN);
expect(g.paused).toBe(true);
g.clear();
expect(g.paused).toBe(false);
});
it("with it on, resumes only after the whole cool-down; one busy reading starts it again; a hot GPU waits", () => {
const g = new GpuWatch({ autoResume: true, resumeAfterMs: 10 * MIN, resumeBelowC: 75 });
g.update({ busy: true, who: "x" }, 60, 0);
g.update({ busy: false }, 50, 1 * MIN);
g.update({ busy: false }, 50, 10 * MIN);
expect(g.paused).toBe(true);
g.update({ busy: true, who: "x" }, 60, 10.5 * MIN);
g.update({ busy: false }, 50, 11 * MIN);
g.update({ busy: false }, 50, 20.9 * MIN);
expect(g.paused).toBe(true);
g.update({ busy: false }, 80, 21 * MIN);
expect(g.paused).toBe(true);
g.update({ busy: false }, 70, 21.1 * MIN);
expect(g.paused).toBe(false);
});
it("a pause by hand always wins over the GPU and its automatic resume", () => {
const free = { paused: false, who: null };
expect(localPause({ paused: true }, free, null, 0)).toEqual({ paused: true, why: "here", until: null });
expect(localPause({ paused: true }, { paused: true, who: "x" }, null, 0)?.why).toBe("here");
expect(localPause({ paused: false }, { paused: true, who: "blender.exe" }, null, 0)?.why).toBe("gpu-busy:blender.exe");
expect(localPause({ paused: true, now: true }, free, null, 0)?.why).toBe("here-now");
expect(localPause({ paused: false }, free, "port-busy", 0)).toEqual({ paused: true, why: "port-busy", until: null });
// A model folder that isn't there yet: after a pause by hand and a busy GPU, before its ComfyUI's trouble.
expect(localPause({ paused: false }, free, "port-busy", 0, ["/mnt/models"])).toEqual({ paused: true, why: "models-not-ready", until: null });
expect(localPause({ paused: false }, { paused: true, who: "game.exe" }, null, 0, ["/mnt/models"])?.why).toBe("gpu-busy:game.exe");
expect(localPause({ paused: true }, free, null, 0, ["/mnt/models"])?.why).toBe("here");
expect(localPause({ paused: false }, free, null, 0, [])).toEqual({ paused: false, why: null, until: null });
});
it("a model folder that can't be read counts as missing; a folder that is there doesn't", () => {
const base = tempDir("fuobe-model-dirs-");
const file = path.join(base, "a-file");
fs.writeFileSync(file, "x");
expect(missingModelDirs([base, path.join(base, "not-yet"), file])).toEqual([path.join(base, "not-yet"), file]);
expect(missingModelDirs(undefined)).toEqual([]);
});
it("--for 2h ends by itself after 2 hours (check 10)", () => {
const until = 2 * 60 * MIN;
expect(parseFor("2h")).toBe(until);
expect(parseFor("2 days")).toBeNull();
expect(localPause({ paused: true, until }, { paused: false }, null, until - 1)?.why).toBe(`here-until:${until}`);
expect(localPause({ paused: true, until }, { paused: false }, null, until)).toBeNull();
});
});
describe("graph checks (check 7)", () => {
const allowed = new Set(["LoadImage", "SaveImage", "KSampler", "CheckpointLoaderSimple"]);
const verified = new Set(["model.safetensors", "checkpoints/model.safetensors"]);
it("refuses an unknown node, a path, an unverified model; passes the rest", () => {
const ok = { "1": { class_type: "CheckpointLoaderSimple", inputs: { ckpt_name: "model.safetensors" } }, "2": { class_type: "SaveImage", inputs: { filename_prefix: "fuobe/x", text: "wait.. a fox" } } };
expect(vetGraph(ok, allowed, verified)).toBeNull();
expect(vetGraph({ "1": { class_type: "ExecuteAnything", inputs: {} } }, allowed, verified)).toBe("unknown-node");
for (const p of ["C:\\Windows\\win.ini", "/etc/passwd", "../../secrets.txt", "\\\\server\\share\\x", "~/x"]) {
expect(vetGraph({ "1": { class_type: "LoadImage", inputs: { image: p } } }, allowed, verified)).toBe("unsafe-path");
}
expect(vetGraph({ "1": { class_type: "CheckpointLoaderSimple", inputs: { ckpt_name: "other.safetensors" } } }, allowed, verified)).toBe("model-unverified");
});
});
describe("its own ComfyUI (check 12)", () => {
it("a port another program holds: refused, and nothing connects to it", async () => {
let connections = 0;
const squatter = net.createServer((s) => {
connections++;
s.destroy();
});
await new Promise<void>((r) => squatter.listen(PORT, "127.0.0.1", r));
try {
expect(await portFree(PORT)).toBe(false);
const dir = tempDir("fuobe-worker-port-");
const c = new OwnComfy({ port: PORT, workDir: dir, comfyDir: dir, comfyCommand: [process.execPath, "-e", "setTimeout(()=>{},1e6)"] });
await expect(c.start()).rejects.toBeInstanceOf(PortBusy);
expect(c.alive()).toBe(false);
expect(connections).toBe(0);
} finally {
await new Promise((r) => squatter.close(r));
}
expect(await portFree(PORT)).toBe(true);
});
it("starts with its own folders, localhost only, no metadata, no browser, only the packs it needs", () => {
const a = comfyArgs({ port: 8190, workDir: "/w" }, ["ComfyUI-VideoHelperSuite"]);
expect(a).toEqual(expect.arrayContaining(["--listen", "127.0.0.1", "--port", "8190", "--disable-auto-launch", "--disable-metadata", "--disable-all-custom-nodes", "--whitelist-custom-nodes", "ComfyUI-VideoHelperSuite"]));
expect(a[a.indexOf("--input-directory") + 1]).toBe(path.join("/w", "input"));
});
});
describe("cleanup (check 13) and setup", () => {
it("after a job, its files leave the worker folder, and its ComfyUI keeps nothing of it: no queue, no prompt, no history", async () => {
const dir = tempDir("fuobe-worker-clean-");
const f = { input: path.join(dir, "input"), output: path.join(dir, "output"), temp: path.join(dir, "temp") };
for (const d of Object.values(f)) fs.mkdirSync(d, { recursive: true });
fs.writeFileSync(path.join(f.input, "fuobe_abcdefgh12.png"), "x");
fs.writeFileSync(path.join(f.input, "fuobe_leftover1234.mp4"), "x");
fs.mkdirSync(path.join(f.output, "fuobe"));
fs.writeFileSync(path.join(f.output, "fuobe", "out_00001_.png"), "x");
fs.writeFileSync(path.join(f.temp, "preview.png"), "x");
const calls: string[] = [];
await cleanJob(f, "http://127.0.0.1:1", (async (url: string, init: RequestInit) => {
calls.push(`${init.method} ${url} ${init.body}`);
return new Response("{}");
}) as typeof fetch);
expect(Object.values(f).flatMap((d) => fs.readdirSync(d))).toEqual([]);
// The whole history goes (a meta-batch's passes included), not only the job's own entry.
expect(calls).toEqual(['POST http://127.0.0.1:1/queue {"clear":true}', "POST http://127.0.0.1:1/interrupt {}", 'POST http://127.0.0.1:1/history {"clear":true}']);
});
it("refuses a synced folder", () => {
expect(syncedFolder("C:\\Users\\me\\OneDrive\\Worker")).toBe(true);
expect(syncedFolder("D:\\FuobeWorker")).toBe(false);
});
// Each setup asks ffmpeg for its version: 60 s, for a busy machine running other tests too.
it("setup again keeps a port pinned by hand and the rest of worker.json, drops the older setups' 8190, and adds no defaults", async () => {
const setupWith = async (saved: Record<string, unknown>) => {
const home = tempDir("fuobe-worker-setup-home-");
const comfyDir = tempDir("fuobe-worker-setup-comfy-");
fs.writeFileSync(path.join(comfyDir, "main.py"), "");
fs.writeFileSync(path.join(home, "worker.json"), JSON.stringify(saved));
const answers = ["https://studio.example", `fuobe_wkr_${"a".repeat(43)}`, comfyDir, path.join(comfyDir, "models"), path.join(tempDir("fuobe-worker-setup-work-"), "w"), "-", "n"];
const io = { env: { FUOBE_WORKER_HOME: home }, home, stdout: { write: () => {} }, stderr: { write: () => {} }, ask: async () => answers.shift() ?? "" };
expect(await worker(["setup"], io)).toBe(0);
return JSON.parse(fs.readFileSync(path.join(home, "worker.json"), "utf8")) as Record<string, unknown>;
};
const kept = await setupWith({ port: 41_000, stallMin: 7, yieldFolder: "C:/turns" });
expect(kept).toMatchObject({ port: 41_000, stallMin: 7, yieldFolder: "C:/turns", site: "https://studio.example", ffmpeg: null, autoResume: false });
for (const key of ["hungMin", "busyVramMb", "localComfyPorts", "allowHttpSite"]) expect(kept).not.toHaveProperty(key);
expect((await setupWith({ port: 8190 })).port).toBeNull();
expect((await setupWith({})).port).toBeNull();
}, 60_000);
it("hashes its inventory the way the studio does", async () => {
const { inventoryHash } = await import("@/server/computer/pool");
const inv = { files: [{ folder: "loras", name: "a.safetensors", size: 7, sha256: null }], nodePacks: [{ name: "P", commit: null }] };
expect(workerHash(inv)).toBe(inventoryHash(inv));
});
});