Skip to content
Python
12.5 KB
Raw Blame
import { test } from "node:test";
import assert from "node:assert/strict";
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { ComfyEvent } from "@comfyorg/sdk";
import { loadRegistry } from "../server/registry.ts";
import { resolveValues } from "../server/runner.ts";
import { History, Meter, pickOutputs, type JobLike } from "../server/history.ts";
import { parseSse, type Frame } from "../server/runner.ts";

const { apps } = loadRegistry();
const byId = (id: string) => apps.find((a) => a.id === id)!;
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3]);

/** A job that runs through queued → running → its end state when wait() is called. */
function fakeJob(end: "succeeded" | "failed", events: ComfyEvent[] = []): JobLike & { canceled: boolean } {
  let status = "queued";
  const started = new Date();
  return {
    id: "job-1",
    canceled: false,
    get status() { return status; },
    get outputs() {
      return status === "succeeded"
        ? [{ name: "ComfyUI_00001_.png", type: "image", nodeId: "9", contentType: "image/png", toBytes: async () => PNG },
           // core SaveVideo reports its mp4 under `images`, so the SDK says "image"
           { name: "video_00001_.mp4", type: "image", nodeId: "68", contentType: "video/mp4", toBytes: async () => PNG },
           { name: "latent.bin", type: "latent", nodeId: "3", contentType: "application/octet-stream", toBytes: async () => new Uint8Array() }]
        : [];
    },
    get error() { return status === "failed" ? { message: "CUDA out of memory", class_type: "KSampler", node_id: "3" } : null; },
    get startedAt() { return status === "queued" ? null : started; },
    get completedAt() { return status === "succeeded" || status === "failed" ? new Date(started.getTime() + 2500) : null; },
    async *events() { for (const e of events) yield e; },
    async wait() { status = "running"; await new Promise((r) => setTimeout(r, 20)); status = end; },
    async cancel() { this.canceled = true; status = "canceled"; },
  };
}

function tempHistory() {
  const dir = mkdtempSync(join(tmpdir(), "ff-history-"));
  return { dir, history: new History(dir), done: () => rmSync(dir, { recursive: true, force: true }) };
}

test("a run is recorded before it is submitted, with its inputs and the seed that was picked", () => {
  const { dir, history, done } = tempHistory();
  try {
    const app = byId("edit");
    const values = resolveValues(app, { image: { bytes: PNG, name: "me.PNG", type: "image/png" }, instruction: "add a hat" });
    const entry = history.start(app, values);
    assert.equal(entry.status, "starting");
    assert.equal(entry.values.instruction, "add a hat");
    assert.equal(typeof entry.values.seed, "number", "the resolved seed is kept, so the run can be repeated");
    assert.equal(entry.values.image, undefined, "files are not copied into the values");
    assert.deepEqual(entry.inputs.map((i) => i.file), ["input-image.png"]);
    assert.deepEqual([...readFileSync(join(dir, entry.id, "input-image.png"))], [...PNG]);
    assert.equal(history.list("edit")[0].inputs[0].url, `/api/history/${entry.id}/files/input-image.png`);
  } finally { done(); }
});

test("following a job stores its media outputs and timings", async () => {
  const { dir, history, done } = tempHistory();
  try {
    const app = byId("z-image");
    const entry = history.start(app, resolveValues(app, { prompt: "a fox" }));
    const progress: ComfyEvent = { kind: "progress", value: 0.5, message: null, nodesDone: 1, nodesTotal: 3, currentNode: "3",
      currentNodeClass: "KSampler", step: 4, steps: 8 };
    const done1 = await history.follow(entry, fakeJob("succeeded", [progress]));
    assert.equal(done1.status, "succeeded");
    assert.equal(done1.job, "job-1");
    assert.deepEqual(done1.outputs.map((o) => [o.file, o.type]), [["out-01.png", "image"], ["out-02.mp4", "video"]],
      "latents are skipped; an mp4 is a video whatever the SDK called it");
    assert.ok(existsSync(join(dir, entry.id, "out-01.png")));
    assert.equal(done1.seconds, 2.5);
    assert.equal(history.get(entry.id)!.progress, null, "live progress is dropped once the run ends");
    assert.equal(JSON.parse(readFileSync(join(dir, entry.id, "meta.json"), "utf-8")).status, "succeeded");
  } finally { done(); }
});

test("the app's declared output nodes come first; the backend also lists what LoadVideo was given", () => {
  const given = { name: "clip.mp4", type: "video", nodeId: "5001" };
  const made = { name: "mixed-media_00001_.mp4", type: "video", nodeId: "4852" };
  const latent = { name: "x.latent", type: "latent", nodeId: "3" };
  assert.deepEqual(pickOutputs([given, made, latent], ["4852"]), [made]);
  assert.deepEqual(pickOutputs([given, made], ["ff_hdr_save", "4852"]), [made], "declared nodes that made nothing are skipped");
  assert.deepEqual(pickOutputs([given, made, latent], []), [given, made], "no declaration: every media output");
  assert.deepEqual(pickOutputs([given, made], ["999"]), [given, made], "a wrong declaration still shows the results");
});

test("the progress bar is ours: nodes finished over the workflow's node count, never backwards, never full", () => {
  const m = new Meter(20, (id) => ({ "3": "KSampler" })[id] ?? null);
  // comfy-api-proxy's first progress_state: the one node touched so far has finished → it says 1/1 = 1.0
  assert.equal(m.feed({ value: 1, nodes_done: 1, nodes_total: 1, step: null, steps: null }).value, 0.05);
  // a per-node frame (nodes_total 0) carries the sampler steps of the running node
  const p = m.feed({ value: 0.5, nodes_done: 0, nodes_total: 0, step: 4, steps: 8, current_node: "3" });
  assert.deepEqual(p, { value: 0.075, step: 4, steps: 8, node: "KSampler" });
  // a lower frame after a drop does not move it back
  assert.equal(m.feed({ value: 0.1, nodes_done: 0, nodes_total: 0, step: 1, steps: 8 }).value, 0.075);
  // all nodes done but the job not over yet: 0.99, not 1
  assert.equal(m.feed({ value: 1, nodes_done: 20, nodes_total: 20 }).value, 0.99);
});

test("a job is followed through the raw event stream when the history has one", async () => {
  const { history, done } = tempHistory();
  const sent: Frame[] = [
    { event: "status", data: { status: "running", queue_position: null } },
    { event: "progress", data: { value: 1, nodes_done: 1, nodes_total: 1 } },
    { event: "progress", data: { value: 0.25, nodes_done: 0, nodes_total: 0, step: 2, steps: 8, current_node: "3" } },
  ];
  const seen: number[] = [];
  let entryId = "";
  const withFrames = new History(history.dir, {
    appInfo: () => ({ outputs: ["9"], nodes: 10, classOf: () => "KSampler" }),
    frames: async function* (jobId) { assert.equal(jobId, "job-1"); for (const f of sent) { yield f; seen.push(withFrames.get(entryId)!.progress?.value ?? -1); await new Promise((r) => setTimeout(r, 1)); } },
  });
  try {
    const app = byId("z-image");
    const entry = withFrames.start(app, resolveValues(app, { prompt: "a fox" }));
    entryId = entry.id;
    const ended = await withFrames.follow(entry, fakeJob("succeeded"));
    assert.equal(ended.status, "succeeded");
    assert.deepEqual(seen, [-1, 0.1, 0.125], "1/1 reads as one node of ten, then the sampler's 2/8 adds a quarter node");
  } finally { done(); }
});

test("server-sent events are read frame by frame, across chunk boundaries", async () => {
  const text = "event: progress\ndata: {\"value\": 0.5}\n\n: comment\nevent: preview\ndata: {\"data_base64\": \"AA==\",\ndata: \"content_type\": \"image/jpeg\"}\n\ndata: tail";
  const chunks = [text.slice(0, 17), text.slice(17, 40), text.slice(40)];
  const body = new ReadableStream<Uint8Array>({ start(c) { for (const ch of chunks) c.enqueue(new TextEncoder().encode(ch)); c.close(); } });
  const frames: Frame[] = [];
  for await (const f of parseSse(body)) frames.push(f);
  assert.deepEqual(frames, [
    { event: "progress", data: { value: 0.5 } },
    { event: "preview", data: { data_base64: "AA==", content_type: "image/jpeg" } },
    { event: "message", data: { text: "tail" } },
  ]);
});

test("a failed job keeps the node and the message", async () => {
  const { history, done } = tempHistory();
  try {
    const app = byId("z-image");
    const entry = await history.follow(history.start(app, resolveValues(app, { prompt: "x" })), fakeJob("failed"));
    assert.equal(entry.status, "failed");
    assert.deepEqual(entry.error, { message: "CUDA out of memory", class_type: "KSampler", node: "3" });
  } finally { done(); }
});

test("a submit that never reached the backend is marked failed with the reason", () => {
  const { history, done } = tempHistory();
  try {
    const app = byId("z-image");
    const entry = history.start(app, resolveValues(app, { prompt: "x" }));
    history.fail(entry, "Set COMFY_BASE_URL to the gateway URL");
    assert.equal(history.get(entry.id)!.status, "failed");
    assert.equal(history.get(entry.id)!.error!.message, "Set COMFY_BASE_URL to the gateway URL");
  } finally { done(); }
});

test("deleting a run that is still being submitted cancels its job once it exists", async () => {
  const { dir, history, done } = tempHistory();
  try {
    const app = byId("z-image");
    const entry = history.start(app, resolveValues(app, { prompt: "x" }));
    assert.ok(history.remove(entry.id));
    assert.ok(!existsSync(join(dir, entry.id)));
    const job = fakeJob("succeeded");
    await history.follow(entry, job);
    assert.ok(job.canceled);
    assert.equal(history.list().length, 0);
  } finally { done(); }
});

test("runs that were going when the server stopped are lost or followed again", async () => {
  const { dir, history, done } = tempHistory();
  try {
    const app = byId("z-image");
    const orphan = history.start(app, resolveValues(app, { prompt: "never submitted" }));
    const pending = history.start(app, resolveValues(app, { prompt: "submitted" }));
    const asFile = JSON.parse(readFileSync(join(dir, pending.id, "meta.json"), "utf-8"));
    asFile.job = "job-1"; asFile.status = "running";
    const { writeFileSync } = await import("node:fs");
    writeFileSync(join(dir, pending.id, "meta.json"), JSON.stringify(asFile));

    const stale = history.start(app, resolveValues(app, { prompt: "submitted yesterday" }));
    const staleFile = JSON.parse(readFileSync(join(dir, stale.id, "meta.json"), "utf-8"));
    staleFile.job = "job-old"; staleFile.status = "running"; staleFile.created = new Date(Date.now() - 864e5).toISOString();
    writeFileSync(join(dir, stale.id, "meta.json"), JSON.stringify(staleFile));

    const restarted = new History(dir);
    const asked: string[] = [];
    restarted.resume(async (id) => { asked.push(id); return fakeJob("succeeded"); });
    await new Promise((r) => setTimeout(r, 100));
    assert.equal(restarted.get(orphan.id)!.status, "lost");
    assert.equal(restarted.get(pending.id)!.status, "succeeded");
    assert.equal(restarted.get(stale.id)!.status, "lost");
    assert.deepEqual(asked, ["job-1"], "an old run is not looked up (that could start a GPU for nothing)");
  } finally { done(); }
});

test("file names are checked before touching the disk", () => {
  const { history, done } = tempHistory();
  try {
    const app = byId("z-image");
    const entry = history.start(app, resolveValues(app, { prompt: "x" }));
    assert.equal(history.file(entry.id, "../meta.json"), null);
    assert.equal(history.file(entry.id, "meta.json"), null);
    assert.equal(history.file("../../etc", "out-01.png"), null);
    assert.equal(history.remove("../x"), false);
  } finally { done(); }
});

test("a poll that cannot reach the backend is retried; a backend answer that the job is gone is not", async () => {
  const { waitPatiently, isTransient } = await import("../server/history.ts");
  let calls = 0;
  const flaky = { async wait() { calls++; if (calls < 3) throw Object.assign(new TypeError("fetch failed"), { cause: { code: "ECONNRESET" } }); } };
  const log: string[] = [];
  const t0 = Date.now();
  await waitPatiently(flaky, 60_000, 60_000, (m) => log.push(m));
  assert.equal(calls, 3);
  assert.equal(log.length, 2);
  assert.ok(Date.now() - t0 >= 2000, "backs off between attempts");

  const gone = { async wait() { throw Object.assign(new Error("job not found"), { httpStatus: 404 }); } };
  await assert.rejects(() => waitPatiently(gone, 60_000, 60_000), /job not found/);
  const proxy = Object.assign(new Error("Bad Gateway"), { httpStatus: 502 });
  assert.equal(isTransient(proxy), true);
  assert.equal(isTransient(new Error("job x not terminal after 10ms (status=running)")), false);
  assert.equal(isTransient(Object.assign(new Error("bad request"), { httpStatus: 400 })), false);
});