Python
// The history: every run started from the web UI, kept on disk so results outlive the backend's
// retention, a closed tab and a server restart.
// out/history/<id>/meta.json what ran: app, values, preset, status, timings, outputs, error
// out/history/<id>/input-<x>.<ext> the files it was given
// out/history/<id>/out-<n>.<ext> what it made
// A run is recorded before it is submitted, because submitting to a cold GPU can take minutes:
// the UI shows it as "starting" right away instead of a spinner on a button.
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
import { extname, join } from "node:path";
import { randomBytes } from "node:crypto";
import type { ComfyEvent } from "@comfyorg/sdk";
import { isFileInput, type App } from "./registry.ts";
import type { FileValue, Frame, Values } from "./runner.ts";
export type Status = "starting" | "queued" | "running" | "succeeded" | "failed" | "canceled" | "expired" | "lost";
const TERMINAL = new Set<Status>(["succeeded", "failed", "canceled", "expired", "lost"]);
const MEDIA = new Set(["image", "video", "audio"]);
export type Entry = {
id: string;
app: string;
status: Status;
job: string | null;
created: string;
started: string | null;
finished: string | null;
/** execution time on the GPU (started → finished), when the backend reports both */
seconds: number | null;
/** wall time from pressing Generate to the result, cold starts and queueing included */
total: number | null;
values: Record<string, unknown>;
preset: string | null;
inputs: { id: string; type: string; file: string; name: string }[];
outputs: { file: string; type: string; name: string; node: string }[];
error: { message: string; class_type?: string | null; node?: string | null } | null;
};
/** The part of the SDK's Job the history uses (tests pass a fake). */
export type JobLike = {
readonly id: string;
readonly status: string;
readonly outputs: readonly { name: string; type: string; nodeId: string; contentType: string; toBytes(): Promise<Uint8Array> }[];
readonly error: { message?: string; class_type?: string | null; node_id?: string | null } | null | undefined;
readonly startedAt: Date | null;
readonly completedAt: Date | null;
readonly progress?: { value: number; step: number | null; steps: number | null; currentNodeClass: string | null } | null;
readonly queuePosition?: number | null;
events(signal?: AbortSignal): AsyncIterable<ComfyEvent>;
wait(timeoutMs?: number, signal?: AbortSignal): Promise<unknown>;
cancel(signal?: AbortSignal): Promise<unknown>;
};
/** What the history needs to know about an app: its declared output nodes, how many nodes its workflow
* has (for the progress bar) and the class of a node id (for the "KSampler 4/8" label). */
export type AppInfo = { outputs: string[]; nodes: number; classOf: (id: string) => string | null };
export type HistoryOptions = {
onContact?: () => void;
appInfo?: (appId: string) => AppInfo | undefined;
/** The raw event stream of a job, read without the SDK (see Meter). Absent: job.events() is used. */
frames?: (jobId: string, signal: AbortSignal) => AsyncIterable<Frame>;
};
type Live = {
job?: JobLike;
cancelRequested?: boolean;
progress: { value: number; step: number | null; steps: number | null; node: string | null } | null;
queue: number | null;
preview: { bytes: Uint8Array; type: string; n: number } | null;
};
/**
* Our own progress figure, because the stream's `value` cannot be shown as it comes. comfy-api-proxy derives it
* from ComfyUI's progress_state, which only lists the nodes touched so far: after the first node finishes and
* before the next starts the frame says 1/1 = 1.0. The SDK's job.events() then drops every later frame as
* "going backwards" (first web runs through the gateway, 23 oct: the bar sat at 100 % for two minutes). So the
* bar is (nodes finished + the running node's step fraction) / the workflow's node count, never backwards and
* never full before the job ends. Per-node `progress` frames (nodes_total 0) carry the sampler steps.
*/
export class Meter {
private done = 0;
private total: number;
private classOf: (id: string) => string | null;
value = 0;
step: number | null = null;
steps: number | null = null;
node: string | null = null;
constructor(expectedNodes: number, classOf: (id: string) => string | null = () => null) {
this.total = Math.max(1, expectedNodes);
this.classOf = classOf;
}
/** Feed one progress snapshot (an SSE frame's data or the SDK's Progress); returns the figure to show. */
feed(d: { value?: unknown; nodes_done?: unknown; nodes_total?: unknown; step?: unknown; steps?: unknown; current_node?: unknown }) {
const n = (x: unknown) => (typeof x === "number" && Number.isFinite(x) ? x : null);
const nodesTotal = n(d.nodes_total) ?? 0;
if (nodesTotal > 0) {
this.done = Math.max(this.done, n(d.nodes_done) ?? 0);
this.total = Math.max(this.total, nodesTotal);
}
const step = n(d.step), steps = n(d.steps);
const inNode = steps && steps > 0 && step !== null ? Math.min(1, Math.max(0, step / steps)) : 0;
this.value = Math.max(this.value, Math.min(0.99, (this.done + inNode) / this.total));
if (steps && steps > 1) { this.step = step; this.steps = steps; } else if (nodesTotal > 0) { this.step = null; this.steps = null; }
const id = typeof d.current_node === "string" ? d.current_node : null;
if (id) this.node = this.classOf(id) ?? this.node;
return { value: this.value, step: this.step, steps: this.steps, node: this.node };
}
}
const ID = /^\d{8}-\d{9}-[0-9a-f]{4}$/;
const EXT: Record<string, string> = {
"image/png": ".png", "image/jpeg": ".jpg", "image/webp": ".webp", "image/gif": ".gif",
"video/mp4": ".mp4", "video/webm": ".webm", "video/quicktime": ".mov",
"audio/mpeg": ".mp3", "audio/wav": ".wav", "audio/x-wav": ".wav", "audio/flac": ".flac", "audio/ogg": ".ogg", "audio/mp4": ".m4a",
};
function newId(now = new Date()): string {
const p = (n: number, w = 2) => String(n).padStart(w, "0");
const day = `${now.getUTCFullYear()}${p(now.getUTCMonth() + 1)}${p(now.getUTCDate())}`;
const time = `${p(now.getUTCHours())}${p(now.getUTCMinutes())}${p(now.getUTCSeconds())}${p(now.getUTCMilliseconds(), 3)}`;
return `${day}-${time}-${randomBytes(2).toString("hex")}`;
}
/** A short lowercase extension we are willing to put on disk. */
function safeExt(name: string | undefined, type: string | undefined): string {
const fromName = extname(name ?? "").toLowerCase();
if (/^\.[a-z0-9]{1,5}$/.test(fromName)) return fromName;
return EXT[(type ?? "").split(";")[0].trim()] ?? ".bin";
}
const VIDEO_EXT = new Set([".mp4", ".webm", ".mov", ".mkv", ".gif"]);
const AUDIO_EXT = new Set([".mp3", ".wav", ".flac", ".ogg", ".m4a", ".opus"]);
/** What the file is, by its extension first: core SaveVideo and CreateVideo report their mp4 under
* `images` (with `animated: true`), so the SDK calls it an image and the web would draw it in an <img>.
* (First web run of morph, 23 oct: a blank card for a good video.) */
function mediaKind(ext: string, sdkType: string): "image" | "video" | "audio" {
if (VIDEO_EXT.has(ext)) return "video";
if (AUDIO_EXT.has(ext)) return "audio";
return sdkType === "video" || sdkType === "audio" ? sdkType : "image";
}
function normStatus(s: string): Status {
if (s === "canceling") return "running";
return (["queued", "running", "succeeded", "failed", "canceled", "expired"] as string[]).includes(s) ? (s as Status) : "running";
}
/** A poll that failed to reach the backend, as opposed to the backend answering that the job is gone or the request is
* wrong: node's fetch says "fetch failed" (connection reset, DNS, TLS), a proxy answers 502/503/504 while the GPU
* container starts, or nothing answers in time. */
export function isTransient(e: unknown): boolean {
const err = e as { message?: string; httpStatus?: number; name?: string; cause?: { code?: string } };
if (typeof err?.httpStatus === "number") return err.httpStatus >= 500 || err.httpStatus === 429;
if (/not terminal after/.test(err?.message ?? "")) return false; // the SDK's own wait timeout
return /fetch failed|ECONN|ETIMEDOUT|EAI_AGAIN|socket hang up|network|timed? ?out|aborted/i.test(`${err?.message ?? ""} ${err?.cause?.code ?? ""}`);
}
const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
/** job.wait() stops at the first poll that fails to reach the backend; a backend waking up (the Modal container's cold
* start) drops one or two, so the wait is picked up again, for up to `patience` of continuous failures, before the
* run is called lost. */
export async function waitPatiently(job: { wait(timeoutMs?: number): Promise<unknown> }, timeoutMs: number, patience = 10 * 60 * 1000,
log: (msg: string) => void = () => {}): Promise<void> {
const deadline = Date.now() + timeoutMs;
let firstFailure: number | null = null;
let failures = 0;
for (;;) {
try {
await job.wait(Math.max(1000, deadline - Date.now()));
return;
} catch (e) {
if (!isTransient(e) || Date.now() >= deadline) throw e;
firstFailure ??= Date.now();
if (Date.now() - firstFailure > patience) throw e;
failures++;
const pause = Math.min(15_000, 2_000 * failures);
log(`poll failed (${(e as Error).message}); trying again in ${pause / 1000}s`);
await sleep(pause);
}
}
}
/** The media outputs worth keeping: those of the app's declared output nodes, in the declared order (the first one
* is the card's thumbnail), and only when none of them produced anything, every media output of the job. The
* backend lists more than the results: core LoadVideo reports the clip it was given (mixed-media's first web run,
* 23 oct: the card showed the input clip, the result was out-02). */
export function pickOutputs<T extends { type: string; nodeId: string }>(outputs: readonly T[], wanted: readonly string[]): T[] {
const media = outputs.filter((o) => MEDIA.has(o.type));
const declared = wanted.flatMap((node) => media.filter((o) => o.nodeId === node));
return declared.length ? declared : media;
}
const secondsBetween = (a: string | Date | null | undefined, b: string | Date | null | undefined) =>
a && b ? Math.max(0, Math.round((new Date(b).getTime() - new Date(a).getTime()) / 100) / 10) : null;
export class History {
private live = new Map<string, Live>();
private removed = new Set<string>();
readonly dir: string;
private onContact: () => void;
private appInfo: (appId: string) => AppInfo | undefined;
private frames?: (jobId: string, signal: AbortSignal) => AsyncIterable<Frame>;
constructor(dir: string, opts: HistoryOptions = {}) {
this.dir = dir;
this.onContact = opts.onContact ?? (() => {});
this.appInfo = opts.appInfo ?? (() => undefined);
this.frames = opts.frames;
mkdirSync(dir, { recursive: true });
}
/** Record a run before it is submitted. `values` are the resolved values (defaults filled, seed picked). */
start(app: App, values: Values, preset?: string): Entry {
const id = newId();
const folder = join(this.dir, id);
mkdirSync(folder);
const inputs: Entry["inputs"] = [];
const shown: Record<string, unknown> = {};
for (const input of app.inputs) {
const v = values[input.id];
if (v === undefined) continue;
if (isFileInput(input)) {
const f = v as FileValue;
const file = `input-${input.id}${safeExt(f.name, f.type)}`;
writeFileSync(join(folder, file), f.bytes);
inputs.push({ id: input.id, type: input.type, file, name: f.name || file });
} else {
shown[input.id] = v;
}
}
const entry: Entry = {
id, app: app.id, status: "starting", job: null, created: new Date().toISOString(), started: null, finished: null,
seconds: null, total: null, values: shown, preset: preset ?? null, inputs, outputs: [], error: null,
};
this.save(entry);
this.live.set(id, { progress: null, queue: null, preview: null });
return entry;
}
/** The submit failed (bad workflow, backend unreachable, missing models): keep the reason with the run. */
fail(entry: Entry, message: string, status: Status = "failed"): void {
entry.status = status;
entry.finished = new Date().toISOString();
entry.total = secondsBetween(entry.created, entry.finished);
entry.error = { message };
this.save(entry);
this.live.delete(entry.id);
}
/** Follow a submitted job to the end, then copy its outputs next to the record. */
async follow(entry: Entry, job: JobLike): Promise<Entry> {
if (this.removed.has(entry.id)) { // deleted while it was being submitted
await job.cancel().catch(() => {});
return entry;
}
const live = this.live.get(entry.id) ?? { progress: null, queue: null, preview: null };
this.live.set(entry.id, live);
live.job = job;
entry.job = job.id;
entry.status = normStatus(job.status);
this.save(entry);
if (live.cancelRequested) await job.cancel().catch(() => {});
const setStatus = (s: Status) => {
if (TERMINAL.has(s) || s === entry.status) return;
entry.status = s;
if (s === "running") entry.started ??= new Date().toISOString();
this.save(entry);
};
const info = this.appInfo(entry.app);
const meter = new Meter(info?.nodes ?? 0, info?.classOf);
const stop = new AbortController();
// live progress from the event stream; the poll in wait() below stays the source of truth
const stream = (async () => {
try {
if (this.frames) {
for await (const f of this.frames(job.id, stop.signal)) {
this.onContact();
if (f.event === "progress") {
live.progress = meter.feed(f.data);
setStatus("running");
} else if (f.event === "preview" && typeof f.data.data_base64 === "string") {
live.preview = { bytes: Buffer.from(f.data.data_base64, "base64"), type: String(f.data.content_type || "image/jpeg"), n: (live.preview?.n ?? 0) + 1 };
} else if (f.event === "status") {
live.queue = typeof f.data.queue_position === "number" ? f.data.queue_position : null;
setStatus(normStatus(String(f.data.status ?? "")));
}
}
} else {
for await (const ev of job.events(stop.signal)) {
this.onContact();
if (ev.kind === "progress") {
live.progress = meter.feed({ nodes_done: ev.nodesDone, nodes_total: ev.nodesTotal, step: ev.step, steps: ev.steps, current_node: ev.currentNode });
if (ev.currentNodeClass) live.progress.node = ev.currentNodeClass;
setStatus("running");
} else if (ev.kind === "preview") {
live.preview = { bytes: ev.data, type: ev.contentType || "image/jpeg", n: (live.preview?.n ?? 0) + 1 };
} else if (ev.kind === "statusChange") {
live.queue = ev.queuePosition;
setStatus(normStatus(ev.status));
}
}
}
} catch {
/* streams drop; polling decides */
}
})();
// the handle's own state moves while wait() polls; mirror it for backends without events
const tick = setInterval(() => {
setStatus(normStatus(job.status));
if (job.queuePosition !== undefined) live.queue = job.queuePosition ?? live.queue;
const p = job.progress;
if (p && (p.step !== null || (p.value > 0 && p.value > meter.value))) {
// a backend whose poll carries a live snapshot (comfy-api-proxy's does not: 0, 0/0)
live.progress = meter.feed({ nodes_done: p.value * (info?.nodes ?? 0), nodes_total: info?.nodes ?? 0, step: p.step, steps: p.steps });
if (p.currentNodeClass) live.progress.node = p.currentNodeClass;
}
}, 1000);
let waitError: unknown = null;
try {
await waitPatiently(job, 6 * 3600 * 1000, 10 * 60 * 1000, (msg) => console.warn(`[history] ${entry.id}: ${msg}`));
this.onContact();
} catch (e) {
waitError = e;
} finally {
clearInterval(tick);
stop.abort();
await stream;
}
entry.started = job.startedAt?.toISOString() ?? entry.started;
entry.finished = (job.completedAt ?? new Date()).toISOString();
// backends that do not report timings: fall back to when this server saw it start running
entry.seconds = secondsBetween(job.startedAt, job.completedAt) ?? (entry.started ? secondsBetween(entry.started, entry.finished) : null);
entry.total = secondsBetween(entry.created, entry.finished);
if (this.removed.has(entry.id)) {
this.live.delete(entry.id);
return entry;
}
if (waitError) {
entry.status = "lost";
entry.error = { message: `lost track of the job: ${(waitError as Error).message}` };
} else {
entry.status = normStatus(job.status);
if (entry.status === "succeeded") {
const folder = join(this.dir, entry.id);
let n = 0;
for (const out of pickOutputs(job.outputs, info?.outputs ?? [])) {
try {
const bytes = await out.toBytes();
const ext = safeExt(out.name, out.contentType);
const file = `out-${String(++n).padStart(2, "0")}${ext}`;
writeFileSync(join(folder, file), bytes);
entry.outputs.push({ file, type: mediaKind(ext, out.type), name: out.name, node: out.nodeId });
} catch (e) {
entry.error = { message: `could not download ${out.name}: ${(e as Error).message}` };
}
}
if (!entry.outputs.length && !entry.error) entry.error = { message: "the job finished without an image, video or audio output" };
} else {
const err = job.error;
entry.error = {
message: (err?.message || entry.status).trim(), class_type: err?.class_type ?? null, node: err?.node_id ?? null,
};
}
}
this.save(entry);
this.live.delete(entry.id);
return entry;
}
/**
* After a restart: recent runs that were still going get followed again (a job is rehydratable
* from its id). Runs that never got a job id, or are too old for the backend to still hold them,
* are marked lost without asking it: asking would start a GPU just to hear the job is gone.
*/
resume(getJob: (id: string) => Promise<JobLike>, maxAgeMs = 60 * 60 * 1000): void {
for (const entry of this.entries()) {
if (TERMINAL.has(entry.status)) continue;
if (!entry.job) {
this.fail(entry, "the server stopped before this run reached the backend", "lost");
continue;
}
if (Date.now() - new Date(entry.started ?? entry.created).getTime() > maxAgeMs) {
this.fail(entry, "the server stopped while this run was going, too long ago to pick it up again", "lost");
continue;
}
this.live.set(entry.id, { progress: null, queue: null, preview: null });
getJob(entry.job)
.then((job) => this.follow(entry, job))
.catch((e) => this.fail(entry, `could not pick this run up again: ${(e as Error).message}`, "lost"));
}
}
async cancel(id: string): Promise<boolean> {
const entry = this.read(id);
if (!entry || TERMINAL.has(entry.status)) return false;
const live = this.live.get(id);
if (!live) return false;
live.cancelRequested = true;
if (live.job) await live.job.cancel().catch(() => {});
return true;
}
remove(id: string): boolean {
if (!ID.test(id) || !existsSync(join(this.dir, id))) return false;
const live = this.live.get(id);
if (live?.job) live.job.cancel().catch(() => {});
this.removed.add(id);
this.live.delete(id);
rmSync(join(this.dir, id), { recursive: true, force: true });
return true;
}
/** Absolute path of a file that belongs to a run, or null. */
file(id: string, name: string): string | null {
if (!ID.test(id) || !/^(input|out)-[a-z0-9_-]+\.[a-z0-9]{1,5}$/.test(name)) return null;
const path = join(this.dir, id, name);
return existsSync(path) ? path : null;
}
preview(id: string): { bytes: Uint8Array; type: string } | null {
return this.live.get(id)?.preview ?? null;
}
list(app?: string, limit = 300) {
return this.entries().filter((e) => !app || e.app === app).slice(0, limit).map((e) => this.view(e));
}
get(id: string) {
const e = this.read(id);
return e ? this.view(e) : null;
}
/** What the UI gets: the record plus URLs and live progress. */
view(e: Entry) {
const base = `/api/history/${e.id}`;
const live = this.live.get(e.id);
return {
...e,
inputs: e.inputs.map((i) => ({ ...i, url: `${base}/files/${i.file}` })),
outputs: e.outputs.map((o) => ({ ...o, url: `${base}/files/${o.file}` })),
progress: live?.progress ?? null,
queue: live?.queue ?? null,
preview: live?.preview ? `${base}/preview?n=${live.preview.n}` : null,
};
}
private entries(): Entry[] {
if (!existsSync(this.dir)) return [];
return readdirSync(this.dir)
.filter((d) => ID.test(d))
.sort()
.reverse()
.map((d) => this.read(d))
.filter((e): e is Entry => e !== null);
}
private read(id: string): Entry | null {
if (!ID.test(id)) return null;
try {
const entry = JSON.parse(readFileSync(join(this.dir, id, "meta.json"), "utf-8")) as Entry;
// entries saved before mediaKind() and pickOutputs() carry the SDK's type and every output the backend
// listed, the given clip first: fix them as they are read (the files stay on disk)
for (const o of entry.outputs ?? []) o.type = mediaKind(extname(o.file).toLowerCase(), o.type);
if (entry.outputs?.length > 1) {
entry.outputs = pickOutputs(entry.outputs.map((o) => ({ ...o, nodeId: o.node })), this.appInfo(entry.app)?.outputs ?? [])
.map(({ nodeId: _, ...o }) => o);
}
return entry;
} catch {
return null;
}
}
private save(entry: Entry): void {
const folder = join(this.dir, entry.id);
if (!existsSync(folder)) return; // deleted while running
const tmp = join(folder, "meta.json.tmp");
writeFileSync(tmp, JSON.stringify(entry, null, 2) + "\n");
renameSync(tmp, join(folder, "meta.json"));
}
}