The open creative studio on ComfyUI: images, video, 3D and audio on open weights, on your own GPU or the cloud. Camera motions, effects, apps, Cinema, Layers, Image to 3D and an AI agent.
TypeScript ai-videocomfyuicreative-toolsgenerative-aiideogramimage-generationimage-to-3dimage-to-videoltx-videominimax-h3nextjsopen-sourceopen-weightsqwen-imagetext-to-imagevideo-generationwan
import "server-only";
import { ENGINE_WORDS } from "@/lib/engine-error";
import crypto from "node:crypto";
import { FINISH_KINDS, type FinishTicket } from "@/lib/finish-ticket";
import { ceilingProblem, cleanCeiling, insideCeiling, lanesInside, type Ceiling } from "@/lib/job-ticket";
import { allLanes, filesBytes, hasFiles, laneLabel, lanesFor, MEDIA_KINDS, memoryFit, minVramBytesFor, needsOf, packKey, partLanes, streamFits, unloadedNodes, type FileNeed, type Inventory, type NodePack, type Needs, type StreamRule } from "@/lib/lanes";
import { fileChoices, hasVariants, pickFiles, swapGraphFiles, type FileChoice } from "@/lib/gpu/model-variants";
import type { FileKey } from "@/lib/gpu/types";
import { POOL_WORDS, RELAY_TIMES, SENT_STATES, type BeatAnswer, type ClaimedTask, type ComputerFacts, type CpuSlot, type WorkerAnswer, type WorkerBeat, type WorkerHolding } from "@/lib/relay";
import { runtimeFileKeys } from "@/lib/runtime-files";
import { cleanPriority } from "@/lib/run-priority";
import { APPS, MODELS } from "@/lib/catalog";
import { laneName, type LaneName } from "@/lib/lane-names";
import { clock, hoursAt, parseHours, type Hours } from "@/lib/worker-hours";
import { CAPACITY_GRACE_MS, type LaneLimit } from "@/lib/capacity";
import { db } from "../db";
import { indexKey, loadIndex } from "../gpu/index-file";
import { hostedRunsOn } from "../hosted-switch";
import { heldView } from "../pool-holds";
import { catalogVersion } from "./catalog-version";
import { cleanFacts, dropProgress, endTask, json, logWorkerJob, noteCold, noteProgress, noteStage, setPoolClocks, setPoolGiveBack, setPoolLost, setPoolOutOfMemory, setTask, stageOf, takeTask, taskById, taskCold, taskProgress, jobNowOf, type JobNow, type TaskRow } from "./tasks";
import { queueSettings, runCeilingMs } from "../queue-settings";
import { createWorker, offerNextToken, revokeWorker, rotateWorker, type WorkerRow } from "./workers";
import { modelLabel } from "./queue-live";
import { effectiveMode, workerModeOf, LOCAL_WORK, lineControls, liveOverrides, modeClassSql, overrideTargets, plainLine, poolMode, sweepControls, taskControl, DEFAULT_MODE, type EffectiveMode, type LineControls, type QueueMode, type TaskControl } from "./queue-controls";
import type { LocalWork, WorkerGpuStats, WorkerStats } from "@/lib/relay";
import { watchingNow } from "./watching";
import { cleanLogLines, lastEventOf, lastOfKind, logWorkerEvent, logWorkerEvents, setEventNames, type NewWorkerEvent, type WorkerEventKind } from "./worker-events";
/**
* Workers (docs/gpu-workers.md): the owner's GPUs on other machines, each running `fuobe worker`, which pulls jobs from
* this studio over HTTPS and runs them on its own ComfyUI. A task for workers belongs to the owner (or, later, an
* account) and names the pools that may take it. This file is the queue's side: the beat, the claim (in one SQLite
* transaction, so two workers never take the same task), putting a silent worker's task back in line, inventory and
* lanes, and the owner's controls. Nothing here runs unless the owner adds a worker.
*/
/**
* The oldest worker program that may take tasks: 2 says which inputs it takes, makes every clip and sound again, checks
* every input against its rules, and keeps its ComfyUI behind the guard. An older one is told to update.
*/
export const MIN_PROTOCOL = 2;
/**
* The oldest worker program that takes an account's job ticket: 3 makes every picture again from its pixels before its
* ComfyUI reads it (the studio sends an account's pictures as they came), and has a CPU slot when worker.json asks.
*/
export const TICKET_PROTOCOL = 3;
/**
* The oldest worker program that checks a job ticket's graph itself (its graphSha, src/lib/ticket-compose.ts): it runs a
* ticket made for another catalog when its own composer makes the same graph from it, and gives the job back at no cost
* otherwise. So a new version of the studio holds back only the people's runs whose graph it changed, and only until
* the worker is updated.
*/
export const GRAPH_PROTOCOL = 4;
/**
* The oldest worker program that runs a workflow with a model variant's files (src/lib/gpu/model-variants.ts): 5 composes
* a job ticket that names a variant with that variant's files, as its own catalog defines them. An older one gets the
* workflow's own files only.
*/
export const VARIANT_PROTOCOL = 5;
/** A worker behind the studio's version for less than this is updating; past it, the owner is shown "Update now". */
export const BEHIND_NOTICE_MS = 3 * 60_000;
/** The owner's "Update now" goes with each beat answer until the worker is on new code, or this long at most. */
export const UPDATE_ASK_MS = 15 * 60_000;
/** A worker behind the studio's version for a short while, on its own way to the new one. */
export const UPDATING_WORDS = "Updating to the new version · back in about a minute";
/** A worker's own pause while a model folder of its isn't there yet: it keeps its last list of files. */
export const MODELS_NOT_READY = "models-not-ready";
/** A task avoids workers that lost or failed it, unless no other worker that can run it beat this recently. */
export const RETRY_WINDOW_MS = 10 * 60_000;
/**
* Lost attempts in all before a task fails (its worker went silent, its ComfyUI went down or lost it: recoveries, which
* keep the task's place). A failed attempt (it stalled, or ran past its run limit) has one more try at the back of the
* line instead (backInLine).
*/
export const MAX_ATTEMPTS = 3;
/** A worker that beat this recently may claim. */
const FRESH_BEAT_MS = 30_000;
/** A job that makes no progress for this long goes back in line (a worker's own worker.json may set another). */
export const STALL_MS = 10 * 60_000;
/**
* A lane's stall limit, which the claim gives the worker: STALL_MS, and never past the lane's run ceiling
* (queue-settings.ts).
*/
export const stallLimitMs = (lane?: string | null) => Math.min(STALL_MS, runCeilingMs(lane));
/**
* The run limit of one attempt: 3 × the lane's p90 on that worker plus its model load, at least 10 minutes, and never
* past the lane's run ceiling (queue-settings.ts; 45 minutes unless the owner sets another). With no p90 yet (a new
* worker, a new lane), the ceiling: a first long job finishes.
*/
export const runLimitMs = (p90?: number | null, lane?: string | null, loadMs?: number | null) => {
const ceiling = runCeilingMs(lane);
if (!p90 || p90 <= 0) return ceiling;
return Math.min(ceiling, Math.max(10 * 60_000, 3 * p90 + Math.max(0, loadMs ?? 0)));
};
/**
* A task's class as a number, in the order workers serve them: people's paid runs (0), then their free runs (1), then
* the owner's own jobs (2). A task from before classes had none: the owner's (no account) is the owner's class.
*/
const CLASS_RANK = "(CASE WHEN class = 'paid' THEN 0 WHEN class = 'owner' OR (class IS NULL AND account IS NULL) THEN 2 ELSE 1 END)";
/** The owner's own jobs' class (classRank): after every person's run. */
const OWNER_CLASS = 2;
export const classRank = (t: Pick<TaskRow, "class" | "account">): number => (t.class === "paid" ? 0 : t.class === "owner" || (!t.class && !t.account) ? 2 : 1);
/**
* The owner's hands-on run (no account, in a person's class): a run the owner made by hand, which waits, runs and makes
* room on a worker's PC as a person's run does. The owner's scripts' runs (a batch, a drill…) stay the owner's class.
*/
export const ownerHandsOn = (t: Pick<TaskRow, "class" | "account">): boolean => !t.account && classRank(t) !== OWNER_CLASS;
/** People's runs (0: an account's, a guest's, the owner's hands-on run) or the owner's own jobs (2), as SQL (classRank). */
const PEOPLE_OR_OWNER = `(CASE WHEN ${CLASS_RANK} = ${OWNER_CLASS} THEN ${OWNER_CLASS} ELSE 0 END)`;
/**
* The window of waiting tasks one look at the line reads, in SQL: people's runs before the owner's own jobs, then by
* rank (a requeued task keeps its own, so it keeps its place), then by when each was made. The order workers serve them
* in is worked out from that window in code (smartLine), and the claim and the place in line read the same order.
*/
const LINE_ORDER = `${PEOPLE_OR_OWNER}, rank_at, created_at, id`;
/** The columns the line's counts read (never a task's graph or files). */
const LINE_COLS = "id, generation_id, principal, pools, only_worker, stop, lane, needs_json, created_at, rank_at, est_ms, tried_json, worker_id, claimed_at, account, class, extra_json, slot, finish_json, priority, failed_once";
/**
* A task a worker took that is still sending its files to its ComfyUI this long after the claim (or after its ComfyUI lost
* it and it began sending them again on the same claim) goes back in line.
*/
export const SENDING_MS = 15 * 60_000;
/** The time a worker has to send a finished job's results up. */
export const SAVING_MS = 15 * 60_000;
/**
* The longest one attempt of a worker's task may last, from its claim: SENDING_MS to send its files, its lane's run
* ceiling twice (a worker sends a job once more after its ComfyUI ran out of memory, with its run limit counted again)
* and SAVING_MS to send its results. The worker's own run limit and stall rule, and the sending clock, end an attempt
* well before; this is only the run's backstop (generations.ts).
*/
export const attemptLimitMs = (lane?: string | null) => SENDING_MS + 2 * runCeilingMs(lane) + SAVING_MS;
/** When the attempt a worker holds now must have ended (none while the task waits in line). */
export function attemptDeadline(t: Pick<TaskRow, "state" | "claimed_at" | "lane">): { at: number; ms: number } | undefined {
if (t.state === "waiting" || !t.claimed_at) return undefined;
const ms = attemptLimitMs(t.lane);
return { at: t.claimed_at + ms, ms };
}
const sha = (s: string) => crypto.createHash("sha256").update(s).digest("hex");
const MAX_RUNNING = 32;
// ---------------------------------------------------------------------------
// The CPU slot (docs/gpu-workers.md): a worker may run one CPU task beside its GPU job
/** A worker's two slots: its GPU's (the jobs that use its GPU) and its CPU slot's (light CPU work, finishing jobs). */
export type WorkerSlot = "gpu" | "cpu";
/** The device a worker's slot takes its tasks under: its GPU's `wkr-<id>`, its CPU slot's `wkr-<id>-cpu`. */
export const slotDevice = (w: Pick<WorkerRow, "id">, slot: WorkerSlot = "gpu") => (slot === "cpu" ? `wkr-${w.id}-cpu` : `wkr-${w.id}`);
/** A worker's CPU slot as its last beat said (null: it has none, and its GPU's slot takes light CPU work as before). */
export function cpuSlotOf(w: Pick<WorkerRow, "cpu_slot_json">): CpuSlot | null {
const s = w.cpu_slot_json ? json<CpuSlot | null>(w.cpu_slot_json, null) : null;
return s && Array.isArray(s.finishes) ? s : null;
}
/** A CPU slot as a beat may say it: the finishing jobs it runs, known kinds only. */
function cleanCpuSlot(v: unknown): CpuSlot | null {
if (!v || typeof v !== "object" || Array.isArray(v)) return null;
const f = (v as { finishes?: unknown }).finishes;
return { finishes: Array.isArray(f) ? FINISH_KINDS.filter((k) => f.includes(k)) : [] };
}
/** A finishing job's kind (src/lib/finish-ticket.ts), or null for any other task. */
const finishKind = (t: Pick<TaskRow, "finish_json">): string | null => (t.finish_json ? (json<FinishTicket | null>(t.finish_json, null)?.kind ?? "?") : null);
/**
* The slot of this worker that takes a task: a finishing job always its CPU slot; light CPU work (a task marked "cpu")
* its CPU slot when it has one, else its GPU's slot, as before; anything else its GPU's.
*/
export function slotFor(t: Pick<TaskRow, "slot" | "finish_json">, w: Pick<WorkerRow, "cpu_slot_json">): WorkerSlot {
if (t.finish_json) return "cpu";
return t.slot === "cpu" && cpuSlotOf(w) ? "cpu" : "gpu";
}
// ---------------------------------------------------------------------------
// What a worker is now
interface Allow {
delayMs: number;
/** The longest clip this worker takes in this lane, in seconds (none: as long as its PC's ceiling allows). */
maxSeconds?: number;
/** Frames larger than this many megapixels... */
bigFrom?: number;
/** ...are taken in clips up to this many seconds (a 720p clip may be shorter than a 480p one). */
bigMaxSeconds?: number;
}
const allowOf = (w: WorkerRow) => json<Record<string, Allow>>(w.allow_json, {});
/** The longest clip a lane's own limits let a worker take at this frame size (undefined: no limit of the lane's own). */
const laneMostSeconds = (a: Allow | undefined, megapixels?: number): number | undefined => {
if (!a) return undefined;
const big = a.bigFrom && a.bigMaxSeconds && (megapixels ?? 0) > a.bigFrom + 1e-9 ? a.bigMaxSeconds : undefined;
const limits = [a.maxSeconds, big].filter((x): x is number => !!x);
return limits.length ? Math.min(...limits) : undefined;
};
/** Is the job longer than this worker takes in its lane at its frame size (the lane's own clip limits)? */
const pastLaneLimit = (allow: Record<string, Allow>, lane: string, size?: { seconds?: number; megapixels?: number } | null) => {
const most = laneMostSeconds(allow[lane], size?.megapixels);
return !!most && (size?.seconds ?? 0) > most + 1e-9;
};
/** The lanes and workflows its files let it run, and the requirements index they were worked out with (none: an older row). */
/** `sets`: the file sets it has whole of each workflow with model variants (lanesFor). Lanes worked out without it: its own files only. */
const capableOf = (w: Pick<WorkerRow, "lanes_json">) => json<{ lanes: string[]; workflows: string[]; index?: string; sets?: Record<string, (string | null)[]> }>(w.lanes_json, { lanes: [], workflows: [] });
const factsOf = (w: WorkerRow) => json<ComputerFacts>(w.facts_json, {});
/** The worker's verified files as its last list said (inventory_json), read once for each list it sends. */
const inventories = new Map<string, { hash: string | null; inv: Inventory }>();
export function inventoryOf(w: Pick<WorkerRow, "id" | "inventory_json" | "inventory_hash">): Inventory {
const kept = inventories.get(w.id);
if (kept && kept.hash === (w.inventory_hash ?? null) && w.inventory_hash) return kept.inv;
const raw = json<Partial<Inventory>>(w.inventory_json, {});
const inv: Inventory = { files: Array.isArray(raw.files) ? raw.files : [], nodePacks: Array.isArray(raw.nodePacks) ? raw.nodePacks : [], ...(Array.isArray(raw.media) && { media: raw.media }) };
inventories.set(w.id, { hash: w.inventory_hash ?? null, inv });
if (inventories.size > 500) inventories.delete(inventories.keys().next().value!);
return inv;
}
const hoursOf = (w: WorkerRow): Hours | null => parseHours(json<unknown>(w.hours_json, null));
const benchOf = (w: WorkerRow) => json<Record<string, { medianMs?: number; p90Ms?: number; loadMs?: number; peakVramBytes?: number }>>(w.bench_json, {});
/** What its PC may ever run, as its last beat said (null: no ceiling of its own). */
export const ceilingOf = (w: Pick<WorkerRow, "ceiling_json">): Ceiling | null => (w.ceiling_json ? cleanCeiling(json<unknown>(w.ceiling_json, {})) : null);
/**
* Its build composes job tickets for this studio's catalog (src/lib/job-ticket.ts). A worker built from another catalog
* (or too old to say) takes the owner's own jobs, and no account's run until it is updated.
*/
export const catalogCurrent = (w: Pick<WorkerRow, "catalog_version">): boolean => {
const v = catalogVersion();
return !!v && w.catalog_version === v;
};
/**
* Takes an account's job ticket: a worker program new enough (TICKET_PROTOCOL: it makes every picture again from its
* pixels) built for this studio's catalog, or one that checks a ticket's graph itself (GRAPH_PROTOCOL) built for any.
*/
export const ticketReady = (w: Pick<WorkerRow, "protocol" | "catalog_version">): boolean =>
(w.protocol ?? 0) >= TICKET_PROTOCOL && (catalogCurrent(w) || ((w.protocol ?? 0) >= GRAPH_PROTOCOL && !!w.catalog_version));
/** How long a worker has been behind the studio's catalog, as its beats said (0: it isn't). */
export const behindFor = (w: Pick<WorkerRow, "behind_since">, now = Date.now()): number => (w.behind_since ? Math.max(0, now - w.behind_since) : 0);
export type WorkerState = "new" | "idle" | "busy" | "draining" | "paused" | "outside-hours" | "gpu-busy" | "comfy-down" | "models-not-ready" | "waiting-for-yield" | "update-worker" | "offline";
/** The state in one word, and in words for the owner ("Paused here until 16:30", "Your GPU is busy (blender.exe)"). */
export function workerState(w: WorkerRow, now = Date.now()): { state: WorkerState; words: string } {
const tz = hoursOf(w) && "tz" in hoursOf(w)! ? (hoursOf(w) as { tz: string }).tz : undefined;
if (!w.seen_at) return { state: "new", words: "Not connected yet" };
if (now - w.seen_at > RELAY_TIMES.poolSilentMs) return { state: "offline", words: "Offline" };
if ((w.protocol ?? 0) < MIN_PROTOCOL) return { state: "paused", words: "Takes no jobs until its worker program is updated" };
const why = w.local_why ?? "";
if (w.paused_local) {
if (why.startsWith("gpu-busy")) return { state: "gpu-busy", words: `Your GPU is busy${why.includes(":") ? ` (${why.split(":").slice(1).join(":")})` : ""}` };
if (why === "comfy-crashes") return { state: "comfy-down", words: "ComfyUI keeps stopping" };
if (why === "port-busy") return { state: "comfy-down", words: "Its ComfyUI port is taken by another program" };
if (why === "self-check") return { state: "comfy-down", words: "Its face check didn't pass after a restart: it checks again within 5 minutes" };
// A model folder isn't there yet (a drive still locked after a restart): it waits for it, with its lanes as they were.
if (why === MODELS_NOT_READY) return { state: "models-not-ready", words: "Waiting for its model folders" };
return { state: "paused", words: w.local_until ? `Paused here until ${clock(w.local_until, tz)}` : "Paused here" };
}
if (w.paused_server) return { state: "paused", words: "Paused on the website" };
const h = hoursAt(hoursOf(w), now);
if (!h.inside) return { state: "outside-hours", words: h.nextChangeAt ? `Outside its hours (back at ${clock(h.nextChangeAt, tz)})` : "No hours set" };
const busy = (db().prepare(`SELECT COUNT(*) AS n FROM relay_tasks WHERE worker_id = ? AND state IN ('sending','queued','running','collecting')`).get(w.id) as { n: number }).n > 0;
if (w.drain) return busy ? { state: "draining", words: "Finishing its job, then pausing" } : { state: "paused", words: "Paused after its last job" };
if (busy) return { state: "busy", words: "Running a job" };
// Studio jobs first: local ComfyUI work on its machine holds the GPU, and the worker waits for it to make room. Not a
// pause: its jobs keep their place. `local_until` is the estimate of the start, when there is one.
if (why.startsWith("waiting-for-yield")) {
if (why.endsWith(":no-answer")) return { state: "waiting-for-yield", words: "Waiting for local ComfyUI work to make room (no answer after 3 minutes)" };
return { state: "waiting-for-yield", words: `Waiting for local ComfyUI work to make room${w.local_until ? ` (starts about ${clock(w.local_until, tz)})` : ""}` };
}
// Built from another catalog while people's runs may come here: it takes the owner's own jobs, and theirs once updated.
if (hostedRunsOn() && (w.protocol ?? 0) < TICKET_PROTOCOL) return { state: "update-worker", words: "Update the worker: it takes only your own jobs until then" };
// Behind the studio's catalog. One that checks a ticket's graph takes people's runs meanwhile (all but those whose
// graph changed); an older one takes only the owner's own jobs. A short while behind, it is on its way to the new
// version by itself; past BEHIND_NOTICE_MS the owner sees how long, and "Update now".
if (hostedRunsOn() && catalogVersion() && !catalogCurrent(w)) {
const graphs = (w.protocol ?? 0) >= GRAPH_PROTOCOL;
const behind = behindFor(w, now);
if (behind < BEHIND_NOTICE_MS) return { state: graphs ? "idle" : "update-worker", words: UPDATING_WORDS };
return { state: "update-worker", words: `Behind the site's version for ${Math.round(behind / 60_000)} min: ${graphs ? "people's runs whose graph changed wait for its update" : "it takes only your own jobs until then"}` };
}
return { state: "idle", words: "Ready" };
}
/** May this worker take a new task now (4.3: paused in neither place, not draining, inside its hours, new enough, heard from)? */
export function eligible(w: WorkerRow, now = Date.now()): boolean {
return (
w.kind === "pool" &&
!w.revoked_at &&
!(w.expires_at && w.expires_at <= now) &&
!w.paused_server &&
!w.paused_local &&
!w.drain &&
(w.protocol ?? 0) >= MIN_PROTOCOL &&
!!w.seen_at &&
now - w.seen_at <= FRESH_BEAT_MS &&
hoursAt(hoursOf(w), now).inside
);
}
// ---------------------------------------------------------------------------
// The beat
const num = (v: unknown) => (typeof v === "number" && Number.isFinite(v) ? v : null);
const text = (v: unknown, max: number) => (typeof v === "string" && v ? v.slice(0, max) : null);
/** A worker's beat as it may be kept: known fields, capped. */
export function cleanBeat(b: unknown): WorkerBeat {
const o = (b && typeof b === "object" ? b : {}) as Record<string, unknown>;
const local = (o.local && typeof o.local === "object" ? o.local : {}) as Record<string, unknown>;
const gpu = (o.gpu && typeof o.gpu === "object" ? o.gpu : {}) as Record<string, unknown>;
const apps = Array.isArray(gpu.otherApps) ? gpu.otherApps.slice(0, 16) : [];
return {
device: String(o.device ?? ""),
protocol: num(o.protocol) ?? 0,
version: text(o.version, 40) ?? "",
...(typeof o.commit === "string" && /^[0-9a-f]{40,64}$/.test(o.commit) && { commit: o.commit }),
...(o.canRotateToken === true && { canRotateToken: true }),
ready: o.ready !== false,
why: text(o.why, 64),
local: { paused: local.paused !== false, why: text(local.why, 96), until: num(local.until) },
gpu: {
otherApps: apps.map((a) => ({ pid: num((a as Record<string, unknown>)?.pid) ?? 0, name: text((a as Record<string, unknown>)?.name, 64) ?? "?", vramBytes: num((a as Record<string, unknown>)?.vramBytes) ?? 0 })),
tempC: num(gpu.tempC),
// How busy its GPU is (0-100); absent from an older worker, null when it can't say.
...((gpu.busyPct === null || num(gpu.busyPct) !== null) && { busyPct: num(gpu.busyPct) === null ? null : Math.round(Math.max(0, Math.min(100, num(gpu.busyPct)!))) }),
},
...(o.facts !== undefined && { facts: cleanFacts(o.facts) }),
...(typeof o.catalogVersion === "string" && /^[0-9a-f]{64}$/.test(o.catalogVersion) && { catalogVersion: o.catalogVersion }),
...(o.ceiling !== undefined && o.ceiling !== null && { ceiling: cleanCeiling(o.ceiling) }),
inventoryHash: text(o.inventoryHash, 80),
warmLane: text(o.warmLane, 16),
running: Array.isArray(o.running) ? (o.running.slice(0, MAX_RUNNING) as WorkerBeat["running"]) : [],
stopped: Array.isArray(o.stopped) ? o.stopped.filter((x): x is string => typeof x === "string").slice(0, 64) : [],
held: Array.isArray(o.held) ? o.held.filter((x): x is string => typeof x === "string").slice(0, 16) : [],
...((num(o.slept) ?? 0) > 0 && { slept: Math.min(num(o.slept)!, MAX_SLEPT_MS) }),
cpuSlot: cleanCpuSlot(o.cpuSlot),
cpuRunning: Array.isArray(o.cpuRunning) ? (o.cpuRunning.slice(0, MAX_RUNNING) as WorkerBeat["running"]) : [],
...(cleanHolding(o.holding) && { holding: cleanHolding(o.holding) }),
// Its own log lines, masked again here (worker-events.ts); an older worker sends none.
...(Array.isArray(o.log) && o.log.length > 0 && { log: cleanLogLines(o.log) }),
// Local work on its PC (counts and codes only); an older worker sends none.
...(o.localWork !== undefined && { localWork: cleanLocalWork(o.localWork) }),
// Its PC's live hardware stats; an older worker sends none.
...(cleanStats(o.stats) && { stats: cleanStats(o.stats)! }),
};
}
const whole = (v: unknown, lo: number, hi: number, digits = 0): number | null => {
const n = num(v);
if (n === null) return null;
const f = 10 ** digits;
return Math.max(lo, Math.min(hi, Math.round(n * f) / f));
};
const sensorName = (v: unknown): string | null => {
if (typeof v !== "string") return null;
const s = v.replace(/[^\w .:#()+/-]+/g, " ").replace(/\s+/g, " ").trim().slice(0, 32).trim();
return s || null;
};
/** A beat's GPU stats as they may be kept: whole numbers in range, null where unknown; else null. */
function cleanGpuStats(v: unknown): WorkerGpuStats | null {
if (!v || typeof v !== "object" || Array.isArray(v)) return null;
const o = v as Record<string, unknown>;
const pstate = typeof o.pstate === "string" && /^P\d{1,2}$/.test(o.pstate) ? o.pstate : null;
return {
busyPct: whole(o.busyPct, 0, 100),
tempC: whole(o.tempC, -50, 150),
powerW: whole(o.powerW, 0, 5000),
powerLimitW: whole(o.powerLimitW, 0, 5000),
...(o.powerSuspect === true && { powerSuspect: true as const }),
coreMHz: whole(o.coreMHz, 0, 20_000),
memMHz: whole(o.memMHz, 0, 50_000),
vramUsedMb: whole(o.vramUsedMb, 0, 1_000_000),
vramTotalMb: whole(o.vramTotalMb, 0, 1_000_000),
fanPct: whole(o.fanPct, 0, 100),
pstate,
};
}
/**
* A beat's live hardware stats as they may be kept (WorkerStats): known fields only, numbers clamped, sensor names short
* and plain, at most 8 board temperatures and fans and 26 drives. Null when it has no reading time.
*/
export function cleanStats(v: unknown): WorkerStats | null {
if (!v || typeof v !== "object" || Array.isArray(v)) return null;
const o = v as Record<string, unknown>;
const at = num(o.at);
if (at === null || at <= 0) return null;
const opt = <K extends string>(k: K, n: number | null) => (n === null ? {} : ({ [k]: n } as Record<K, number>));
const list = (x: unknown) => (Array.isArray(x) ? x : []);
const b = (o.board && typeof o.board === "object" ? o.board : null) as Record<string, unknown> | null;
const temps = list(b?.temps)
.map((t) => ({ name: sensorName((t as Record<string, unknown>)?.name), c: whole((t as Record<string, unknown>)?.c, -50, 150) }))
.filter((t): t is { name: string; c: number } => !!t.name && t.c !== null)
.slice(0, 8);
const fans = list(b?.fans)
.map((f) => ({ name: sensorName((f as Record<string, unknown>)?.name), rpm: whole((f as Record<string, unknown>)?.rpm, 0, 30_000) }))
.filter((f): f is { name: string; rpm: number } => !!f.name && f.rpm !== null)
.slice(0, 8);
const disks = list(o.disks)
.map((d) => d as Record<string, unknown>)
.filter((d) => typeof d?.drive === "string" && /^[A-Z]$/.test(d.drive) && (num(d.freeBytes) ?? -1) >= 0 && (num(d.totalBytes) ?? -1) >= 0)
.map((d) => ({ drive: d.drive as string, freeBytes: Math.floor(num(d.freeBytes)!), totalBytes: Math.floor(num(d.totalBytes)!) }))
.slice(0, 26);
return {
at,
gpu: cleanGpuStats(o.gpu),
cpuPct: whole(o.cpuPct, 0, 100),
cpuCores: whole(o.cpuCores, 0, 4096),
ramUsedMb: whole(o.ramUsedMb, 0, 1e8),
ramTotalMb: whole(o.ramTotalMb, 0, 1e8),
...opt("cpuTempC", whole(o.cpuTempC, -50, 150)),
...opt("cpuPowerW", whole(o.cpuPowerW, 0, 2000)),
...opt("cpuClockMhz", whole(o.cpuClockMhz, 0, 20_000)),
...opt("vcoreV", whole(o.vcoreV, 0, 3, 3)),
...((temps.length > 0 || fans.length > 0) && { board: { temps, fans } }),
...(disks.length > 0 && { disks }),
};
}
/** A worker's live hardware stats as its last beat said, with when the studio heard them (`receivedAt`, its own clock). */
export type WorkerStatsView = WorkerStats & { receivedAt: number };
/** The stats a worker's last beat carried (null: none, or an older worker). The panel judges how fresh they are. */
export function workerStatsOf(w: Pick<WorkerRow, "stats_json">): WorkerStatsView | null {
if (!w.stats_json) return null;
const raw = json<Record<string, unknown> | null>(w.stats_json, null);
const s = cleanStats(raw);
const receivedAt = num(raw?.receivedAt);
return s && receivedAt ? { ...s, receivedAt } : null;
}
/** A beat's `localWork` as it may be kept: a known state, a short code, a count, whether one runs, when; else null. */
export function cleanLocalWork(v: unknown): LocalWork | null {
if (!v || typeof v !== "object" || Array.isArray(v)) return null;
const o = v as Record<string, unknown>;
const states = ["running", "idle", "paused", "yielding", "stopped"] as const;
const state = states.find((s) => s === o.state);
const at = num(o.at);
if (!state || at === null || at <= 0) return null;
const why = typeof o.why === "string" && /^[a-z0-9][a-z0-9:-]{0,47}$/.test(o.why) ? o.why : null;
const waiting = num(o.waiting);
return { state, why, waiting: waiting !== null && waiting >= 0 ? Math.min(Math.floor(waiting), 1_000_000) : null, running: o.running === true, at };
}
/** Local work on a worker's PC counts as there while its last word is this fresh. */
export const LOCAL_WORK_FRESH_MS = 2 * 60_000;
/** Local work on a worker's PC as its last beat said, while fresh (null: none, or too old). */
export function localWorkOf(w: Pick<WorkerRow, "local_work_json" | "seen_at">, now = Date.now()): LocalWork | null {
const l = w.local_work_json ? cleanLocalWork(json<unknown>(w.local_work_json, null)) : null;
return l && now - l.at <= LOCAL_WORK_FRESH_MS ? l : null;
}
/** A beat's `holding` as it may be kept: a short reason in lower-case words, since when, and a task id; else null. */
export function cleanHolding(v: unknown): WorkerHolding | null {
if (!v || typeof v !== "object") return null;
const o = v as Record<string, unknown>;
const why = typeof o.why === "string" && /^[a-z0-9][a-z0-9:-]{0,47}$/.test(o.why) ? o.why : null;
const since = num(o.since);
if (!why || since === null || since <= 0) return null;
const task = typeof o.task === "string" && /^[\w-]{1,64}$/.test(o.task) ? o.task : null;
return { why, since, ...(task && { task }) };
}
/**
* A worker says it is here: its own pause and why, its GPU, its facts, what it runs. The answer: whether it is paused on
* the website, a stop-now, drain, its hours, the lanes it is allowed, and the prompts to take off its ComfyUI.
*/
export function workerBeat(w: WorkerRow, b: WorkerBeat, now = Date.now()): BeatAnswer & { worker: WorkerAnswer; ownerWaitingNow: number } {
const d = db();
const device = `wkr-${w.id}`;
// With the write lock from the start (IMMEDIATE): what it reads of the line is still so when it writes, even while
// another app process on the same database (a deploy's overlap) writes too.
return d.transaction(() => {
// A worker back after going silent finds its task back in line first (its lease no longer works).
sweepPool(now);
// The owner's controls that ran their course end first (a mode or a hold past its time).
sweepControls(now);
// Its saved state and version before this beat: a timeline row only when either changes.
const before = d.prepare("SELECT state, version, commit_sha, behind_since, update_asked_at, update_asked_commit FROM relay_workers WHERE id = ?").get(w.id) as
| Pick<WorkerRow, "state" | "version" | "commit_sha" | "behind_since" | "update_asked_at" | "update_asked_commit">
| undefined;
const facts = b.facts ? { ...b.facts, loras: undefined } : undefined;
// Behind the studio's catalog since when (kept while it stays behind), and the owner's "Update now" until it is on
// new code (another commit, or the studio's catalog again) or UPDATE_ASK_MS went by.
const site = catalogVersion();
const behind = !!site && b.catalogVersion !== site;
const behindSince = behind ? (before?.behind_since ?? now) : null;
const asked = before?.update_asked_at ?? null;
const onNewCode = !!b.commit && !!before?.update_asked_commit && b.commit !== before.update_asked_commit;
const askDone = !!asked && (now - asked > UPDATE_ASK_MS || onNewCode || (!!before?.behind_since && !behind));
d.prepare("UPDATE relay_workers SET commit_sha = ?, behind_since = ?, update_asked_at = ? WHERE id = ?").run(b.commit ?? null, behindSince, askDone ? null : asked, w.id);
d.prepare(
`UPDATE relay_workers SET seen_at = ?, device = ?, protocol = ?, version = ?, paused_local = ?, local_why = ?, local_until = ?,
warm_lane = ?, facts_json = COALESCE(?, facts_json), catalog_version = ?, ceiling_json = ?, cpu_slot_json = ?, holding_json = ?, local_work_json = ?, stats_json = ? WHERE id = ?`,
).run(
now,
device,
b.protocol,
b.version,
b.local.paused ? 1 : 0,
b.local.why,
b.local.until,
b.warmLane,
facts ? JSON.stringify(facts) : null,
b.catalogVersion ?? null,
b.ceiling ? JSON.stringify(b.ceiling) : null,
b.cpuSlot ? JSON.stringify(b.cpuSlot) : null,
// Its own "since" is never later than now (a PC clock ahead of the studio's).
b.holding ? JSON.stringify({ ...b.holding, since: Math.min(b.holding.since, now) }) : null,
// Local work on its PC: its time never later than now either.
b.localWork ? JSON.stringify({ ...b.localWork, at: Math.min(b.localWork.at, now) }) : null,
// Its PC's live stats, with when the studio heard them (none from an older worker: nothing old stays).
b.stats ? JSON.stringify({ ...b.stats, receivedAt: now }) : null,
w.id,
);
// Its PC slept: a job taken back from it for going quiet meanwhile counts against no one.
if (b.slept) forgiveSleep(w, b.slept, now);
// The tasks of both its slots stay alive, with their progress.
for (const r of [...b.running, ...(b.cpuRunning ?? [])]) {
const t = typeof r?.task === "string" ? (d.prepare("SELECT * FROM relay_tasks WHERE id = ? AND worker_id = ?").get(r.task, w.id) as TaskRow | undefined) : undefined;
if (!t || !(SENT_STATES as readonly string[]).includes(t.state)) continue;
setTask(t.id, { device_seen_at: now });
noteProgress(t.id, r as unknown as Record<string, unknown>, now, t.lease_hash);
}
for (const id of b.stopped) d.prepare("UPDATE relay_tasks SET stop = 0 WHERE id = ? AND device = ?").run(id, device);
// A task its CPU slot held that ended with a stop: the CPU slot heard it in its reports and emptied its ComfyUI then.
d.prepare("UPDATE relay_tasks SET stop = 0 WHERE device = ? AND stop = 1 AND state IN ('done','failed','canceled')").run(slotDevice(w, "cpu"));
const stops = (d.prepare("SELECT id, prompt_id, prefix FROM relay_tasks WHERE device = ? AND stop = 1").all(device) as { id: string; prompt_id: string; prefix: string | null }[]).map((s) => ({
task: s.id,
promptId: s.prompt_id,
...(s.prefix && { prefix: s.prefix }),
}));
const fresh = d.prepare("SELECT * FROM relay_workers WHERE id = ?").get(w.id) as WorkerRow;
// On at this beat: what the site can make now counts it until CAPACITY_GRACE_MS after its last such beat.
if (onNow(fresh, now)) d.prepare("UPDATE relay_workers SET available_at = ? WHERE id = ?").run(now, w.id);
// Its lanes from another requirements index (the catalog changed since its list came): worked out again now.
const lanesRefreshed = refreshLanes(fresh);
// Stop now (the website, or the worker paused itself with --now / a busy GPU): its jobs go back in line at no cost,
// and the worker takes their prompts off its ComfyUI.
const stopNow = !!fresh.stop_now || (b.local.paused && !!b.local.why && (b.local.why === "here-now" || b.local.why.startsWith("gpu-busy")));
if (stopNow) {
const held = new Set(b.held ?? []);
// Another program on its GPU stops only its GPU's job: its CPU slot's task needs no GPU, and goes on.
const gpuOnly = !fresh.stop_now && b.local.why !== "here-now";
const sentBack = fresh.stop_now ? "paused now on the website" : b.local.why === "here-now" ? "paused now on its PC" : "another program took its GPU";
for (const t of d.prepare("SELECT * FROM relay_tasks WHERE worker_id = ? AND state IN ('sending','queued','running','collecting')").all(w.id) as TaskRow[]) {
if (gpuOnly && t.device === slotDevice(w, "cpu")) continue;
// Its result is being saved, or the worker keeps the finished results on its disk (the studio may not have heard
// that it finished): let it send them, which needs no GPU, instead of running the job again.
if (t.state === "collecting" || held.has(t.id)) continue;
backInLine(t, { blame: false, sentBack });
stops.push({ task: t.id, promptId: t.prompt_id, ...(t.prefix && { prefix: t.prefix }) });
}
d.prepare("UPDATE relay_workers SET stop_now = 0 WHERE id = ?").run(w.id);
} else {
// People first: the owner's own job on its GPU goes back in line when a person's run waits that this worker can
// take and no other free worker can. No attempt counted, its place among the owner's jobs kept; it runs again
// from the start once no person waits. Never a person's run, never its CPU slot's task.
for (const t of ownerJobsToPreempt(fresh, new Set(b.held ?? []), now)) {
if (!backInLine(t, { blame: false, sentBack: "made room for a person's run" })) continue;
const back = taskById(t.id);
if (back) setTask(t.id, { extra_json: JSON.stringify({ ...json<Record<string, unknown>>(back.extra_json, {}), preempted: now }) });
stops.push({ task: t.id, promptId: t.prompt_id, ...(t.prefix && { prefix: t.prefix }) });
}
}
const state = workerState(fresh, now);
if (state.state !== before?.state) {
d.prepare("UPDATE relay_workers SET state = ? WHERE id = ?").run(state.state, w.id);
logWorkerEvent({ kind: "state", event: state.state, workerId: w.id, words: state.words, state: state.state, detail: { from: before?.state ?? null } }, now);
}
// Best-effort from here: the timeline never fails a beat.
const rows: NewWorkerEvent[] = [];
if (before?.version && b.version && before.version !== b.version) rows.push({ kind: "version", event: "updated", workerId: w.id, words: `Worker program updated to ${b.version}`, detail: { from: before.version, to: b.version } });
else if (before?.commit_sha && b.commit && before.commit_sha !== b.commit) rows.push({ kind: "version", event: "updated", workerId: w.id, words: `Worker program updated to ${b.commit.slice(0, 12)}`, detail: { from: before.commit_sha, to: b.commit } });
// Its PC's clock may run ahead of the studio's: never later than now.
for (const l of b.log ?? []) rows.push({ kind: "log", event: l.level, workerId: w.id, words: l.line, at: Math.min(l.at, now) });
logWorkerEvents(rows, now);
// A worker that says it saves a new token (`canRotateToken` in its beat) gets its next one when half its token's life
// is gone. (Not by its protocol number: a worker may speak a newer protocol without saving tokens.)
const nextToken = b.canRotateToken ? offerNextToken(fresh, now) : null;
const h = hoursAt(hoursOf(fresh), now);
// Pool names are validated at creation (/^[a-z0-9-]{1,32}$/), so they never contain commas. INSTR on the
// CSV avoids the LIKE-wildcard class of risk (%, _) entirely.
const waiting = (d.prepare("SELECT COUNT(*) AS n FROM relay_tasks WHERE pools IS NOT NULL AND state = 'waiting' AND INSTR(',' || pools || ',', ?) > 0").get(`,${fresh.pool},`) as { n: number }).n;
const mode = effectiveMode(fresh, now);
return {
runner: eligible(fresh, now),
heldElsewhere: false,
waiting,
claimableNow: claimableNow(fresh, now),
ownerWaitingNow: ownerWaitingNow(fresh, now),
stops,
worker: {
paused: !!fresh.paused_server,
pauseNow: stopNow,
drain: !!fresh.drain,
autoResume: fresh.auto_resume === null || fresh.auto_resume === undefined ? null : !!fresh.auto_resume,
// The owner's RAM guard switch for this worker (its numbers stay in its worker.json).
ramGuard: !!fresh.ram_guard,
insideHours: h.inside,
nextChangeAt: h.nextChangeAt,
lanes: Object.keys(allowOf(fresh)),
lanesInside: lanesInside(Object.keys(allowOf(fresh)), ceilingOf(fresh)),
minProtocol: MIN_PROTOCOL,
...(catalogVersion() && { catalogVersion: catalogVersion()! }),
// While a model folder isn't there, the studio keeps the list it has (its lanes don't go missing meanwhile).
// Its lanes were just worked out again for a new index: the list again too, for files the old index didn't name.
sendInventory: b.local.why !== MODELS_NOT_READY && (lanesRefreshed || !fresh.inventory_hash || fresh.inventory_hash !== b.inventoryHash),
...(nextToken && { token: nextToken }),
...(fresh.expires_at && { tokenExpiresAt: fresh.expires_at }),
// The owner's mode it follows now: who goes first, and what local work on its PC does.
queue: { mode: mode.mode, localWork: LOCAL_WORK[mode.mode], until: mode.until },
// The owner asked it to update now.
...(fresh.update_asked_at && { updateNow: fresh.update_asked_at }),
},
};
}).immediate();
}
/**
* The owner's own jobs this worker's GPU runs that go back in line for people (workerBeat), in People first and People
* only: while it may take work, a person's run waits that it can take, and no other worker that is free (taking work,
* its GPU's slot empty) can take that run. Not one that is saving its results, nor one whose finished results the
* worker keeps, nor light CPU work.
*/
function ownerJobsToPreempt(w: WorkerRow, held: Set<string>, now: number): TaskRow[] {
// Only where people come before the owner's own jobs (People first, People only); never in Our work first or Balanced.
const mode = effectiveMode(w, now).mode;
if (mode !== "people-first" && mode !== "people-only") return [];
if (!eligible(w, now)) return [];
const d = db();
const mine = (d.prepare("SELECT * FROM relay_tasks WHERE worker_id = ? AND device = ? AND state IN ('sending','queued','running') AND stop = 0").all(w.id, slotDevice(w, "gpu")) as TaskRow[]).filter(
(t) => classRank(t) === OWNER_CLASS && t.slot !== "cpu" && !t.finish_json && !held.has(t.id),
);
if (!mine.length) return [];
const people = gpuClaimableRows(w, now, "people");
if (!people.length) return [];
const others = (d.prepare("SELECT * FROM relay_workers WHERE kind = 'pool' AND revoked_at IS NULL AND id != ?").all(w.id) as WorkerRow[]).filter(
(o) => eligible(o, now) && !d.prepare("SELECT 1 FROM relay_tasks WHERE worker_id = ? AND device = ? AND state IN ('sending','queued','running','collecting')").get(o.id, slotDevice(o, "gpu")),
);
const someoneElse = (t: TaskRow) => others.some((o) => fitsSlot(t, o, now, "gpu") && retryOk(t, o, now) && beforeWindowEnds(t, o, now));
return people.some((t) => !someoneElse(t)) ? mine : [];
}
/**
* People's tasks (an account's, the owner's hands-on run: ownerHandsOn, and any run the owner sent next; never the
* owner's other own jobs) this worker may claim right now: in its pool, in a lane it is
* allowed and able to run, with the files and VRAM they need, past their delays, and ending before its hours do. Its
* own pause doesn't count here (its machine reads this to know whether a person's run waits for it); a pause on the
* website, a drain, an older worker program or being outside its hours does.
*/
export function claimableNow(w: WorkerRow, now = Date.now()): number {
return gpuClaimable(w, now, "people");
}
/**
* The owner's own jobs (a batch, a drill…; never a hands-on run, nor one sent next) this worker may claim right now, by the same rules as
* claimableNow. They never ask local
* ComfyUI work to make room: its machine reads this only to let an idle local ComfyUI free the memory it keeps.
*/
export function ownerWaitingNow(w: WorkerRow, now = Date.now()): number {
return gpuClaimable(w, now, "owner");
}
/** The first job in line this worker may claim right now (people's before the owner's), or null: what a worker that holds back the line would take first. */
export function firstClaimableNow(w: WorkerRow, now = Date.now()): TaskRow | null {
const first = (rows: TaskRow[]) => (rows.length > 1 ? lineOrder(rows, controlsOf(w, now).mode, { now, worker: w, workers: [w] })[0] : rows[0]);
return first(gpuClaimableRows(w, now, "people")) ?? first(gpuClaimableRows(w, now, "owner")) ?? null;
}
function gpuClaimable(w: WorkerRow, now: number, whose: "people" | "owner"): number {
return gpuClaimableRows(w, now, whose).length;
}
function gpuClaimableRows(w: WorkerRow, now: number, whose: "people" | "owner"): TaskRow[] {
if (w.kind !== "pool" || w.revoked_at || w.paused_server || w.drain || (w.protocol ?? 0) < MIN_PROTOCOL || !hoursAt(hoursOf(w), now).inside) return [];
const c = controlsOf(w, now);
// Only its GPU's: work its CPU slot takes never asks local ComfyUI work to make room (left out in the query, so a long
// line of CPU work never hides the GPU's).
// By class: people's runs and the owner's hands-on runs, or the owner's own jobs (a batch, a drill…). A run the owner
// sent next counts as a person's, whatever its class: local work on the PC makes room for it as for a person's run.
const sentNext = JSON.stringify(overrideTargets(c).generations);
const byClass =
whose === "people"
? `(${CLASS_RANK} <> ${OWNER_CLASS} OR generation_id IN (SELECT value FROM json_each(?)))`
: `${CLASS_RANK} = ${OWNER_CLASS} AND generation_id NOT IN (SELECT value FROM json_each(?))`;
const rows = lineRows(LINE_COLS, `pools IS NOT NULL AND state = 'waiting' AND stop = 0 AND ${byClass} AND INSTR(',' || pools || ',', ?) > 0 ${slotWhere(w, "gpu")}`, [sentNext, `,${w.pool},`], c, LOOK_AT);
// A run that failed once counts only when the line is free for its last try (secondTryGate).
const second = secondTryGate(w.pool, slotWhere(w, "gpu"), c, now);
const able = rows.filter((t) => fitsSlot(t, w, now, "gpu") && retryOk(t, w, now) && beforeWindowEnds(t, w, now) && second(t));
// A job left for a free worker that picks first is not this worker's to take for now: its PC makes no room for it.
const left = able.length ? firstPicks(w, now) : null;
const fit = left?.size ? able.filter((t) => !left.has(t.id)) : able;
if (plainLine(c)) return fit;
// Held runs and people, runs pinned to another worker, and the owner's own jobs in People only don't count. In Our work
// first, while local work on its PC runs, people's runs ask it for no room unless the owner sent them next or boosted
// them: the owner's work keeps the GPU, and people's runs start once it is free.
const keepGpu = whose === "people" && c.mode === "ours-first" && localWorkOf(w, now)?.state === "running";
return fit.filter((t) => {
const tc = taskControl(t, classRank(t), c, w.id);
return !tc.out && (!keepGpu || tc.tier < 2);
});
}
/** The controls one look at the line reads: the mode of this worker (or of a pool: `w` its name) and the live overrides. */
function controlsOf(w: Pick<WorkerRow, "pool" | "mode_json"> | string, now: number): LineControls {
return lineControls((typeof w === "string" ? poolMode(w, now) : effectiveMode(w, now)).mode, liveOverrides(now));
}
/** The window's order in SQL under a mode: the mode's group, then rank, when it came and id (LINE_ORDER for people first). */
const lineOrderSql = (mode: QueueMode) => (mode === DEFAULT_MODE ? LINE_ORDER : `${modeClassSql(PEOPLE_OR_OWNER, mode)}, rank_at, created_at, id`);
/**
* The waiting rows one look at the line reads: the first `limit` in the mode's order and, wherever they stand, the rows
* of the runs sent next and of the people boosted (a run sent next is never missed past the window).
*/
function lineRows(cols: string, where: string, args: unknown[], c: LineControls, limit: number): TaskRow[] {
const d = db();
const rows = d.prepare(`SELECT ${cols} FROM relay_tasks WHERE ${where} ORDER BY ${lineOrderSql(c.mode)} LIMIT ?`).all(...args, limit) as TaskRow[];
const { generations, accounts } = overrideTargets(c);
if (!generations.length && !accounts.length) return rows;
const seen = new Set(rows.map((r) => r.id));
const extra = d
.prepare(`SELECT ${cols} FROM relay_tasks WHERE ${where} AND (generation_id IN (SELECT value FROM json_each(?)) OR account IN (SELECT value FROM json_each(?))) ORDER BY ${lineOrderSql(c.mode)} LIMIT ?`)
.all(...args, JSON.stringify(generations), JSON.stringify(accounts), limit) as TaskRow[];
return [...rows, ...extra.filter((r) => !seen.has(r.id))];
}
/**
* Waiting tasks in the order a worker (`look.worker`; null: the line as a whole) serves them under the owner's
* controls: the mode's order (smartLine), then runs sent next first, then boosted people's runs, each group keeping
* its order. The ones it doesn't take now (held, pinned to another worker, the owner's own in People only) are left
* out, except `keep` (the task whose own place is asked).
*/
function controlledOrder(rows: TaskRow[], c: LineControls, look: LineLook, keep?: string): TaskRow[] {
const ordered = lineOrder(rows, c.mode, look);
if (plainLine(c)) return ordered;
const workerId = look.worker?.id ?? null;
const tiers = new Map<string, number>();
const kept = ordered.filter((t) => {
const tc = taskControl(t, classRank(t), c, workerId);
if (tc.out && t.id !== keep) return false;
tiers.set(t.id, tc.out ? 2 : tc.tier);
return true;
});
return kept
.map((t, i) => ({ t, i }))
.sort((a, b) => tiers.get(a.t.id)! - tiers.get(b.t.id)! || a.i - b.i)
.map((x) => x.t);
}
// ---------------------------------------------------------------------------
// Inventory
/**
* A model file's name as a worker reports it: the requirements index's own name for it inside its folder, which may hold
* subfolders ("bigvgan_vocoder/config.json"), with forward slashes. Never a path that climbs out ("..") or starts at a
* root or a drive, never a backslash, an empty part or a control character.
*/
const fileNameOk = (name: string) => name.length <= 256 && !/[\\\0-\x1f]|^\/|^[a-z]:|(^|\/)\.{1,2}(\/|$)|\/\/|\/$/i.test(name);
/**
* The files a worker may report, as "folder/name": the index's (loadIndex keeps one index until its file changes), and the
* run-time files a run may add beyond its workflow's own (src/lib/runtime-files.ts: the LoRA library's pinned files).
*/
const indexFileKeys = new WeakMap<ReturnType<typeof loadIndex>, Set<string>>();
function knownFiles(): Set<string> {
const index = loadIndex();
let keys = indexFileKeys.get(index);
if (!keys) {
keys = new Set([...Object.values(index.files).map((f) => `${f.folder}/${f.file}`), ...runtimeFileKeys()]);
indexFileKeys.set(index, keys);
}
return keys;
}
/**
* The worker's verified files and node packs: what it can run now, and what each other lane lacks. Only the files the
* requirements index names and the run-time files (knownFiles) are kept (by folder and name, subfolders and all), but
* the hash is of the list as the worker sent it, so it equals the hash the worker computes and the studio asks for the
* list again only when it changed. The run-time files it verified decide which runs that add them it may take (fits).
*/
export function saveInventory(w: WorkerRow, raw: unknown): { lanes: string[]; workflows: string[]; missing: { lane: string; label: string; files: string[]; bytes: number }[] } {
const o = (raw && typeof raw === "object" ? raw : {}) as Record<string, unknown>;
const sent = (Array.isArray(o.files) ? o.files : []).slice(0, 20_000).flatMap((f) => {
const x = f as Record<string, unknown>;
const folder = text(x?.folder, 64);
const name = typeof x?.name === "string" && x.name ? x.name : null;
const size = num(x?.size);
const sha256 = typeof x?.sha256 === "string" && /^[0-9a-f]{64}$/.test(x.sha256) ? x.sha256 : null;
return folder && name && size !== null ? [{ folder, name, size, sha256 }] : [];
});
const known = knownFiles();
const files = sent.filter((f) => !/[\\/]|\.\./.test(f.folder) && fileNameOk(f.name) && known.has(`${f.folder}/${f.name}`));
const nodePacks = (Array.isArray(o.nodePacks) ? o.nodePacks : []).slice(0, 500).flatMap((p): NodePack[] => {
const x = p as Record<string, unknown>;
const name = text(x?.name, 128);
if (!name) return [];
// The pack's catalog classes its ComfyUI didn't load (a worker that read them; none said: its folder counts).
const missing = Array.isArray(x.missingClasses) ? x.missingClasses.slice(0, 500).filter((c): c is string => typeof c === "string" && c.length > 0 && c.length <= 128) : undefined;
return [{ name, commit: text(x?.commit, 64), ...(missing && { missingClasses: missing }) }];
});
// The kinds of input it takes. None said: an older worker, which doesn't make clips and sound again, so lanesFor gives
// it pictures only (the list is kept as sent, so its hash stays the one the worker computes).
const media = Array.isArray(o.media) ? MEDIA_KINDS.filter((k) => (o.media as unknown[]).includes(k)) : undefined;
const inv: Inventory = { files, nodePacks, ...(media && { media }) };
const { sets, ...r } = lanesFor(loadIndex(), inv, sha);
const invHash = inventoryHash({ ...inv, files: sent });
db()
.prepare("UPDATE relay_workers SET inventory_json = ?, inventory_hash = ?, lanes_json = ? WHERE id = ?")
.run(JSON.stringify(inv), invHash, JSON.stringify({ lanes: r.lanes, workflows: r.workflows, index: indexKey(), sets }), w.id);
return r;
}
/**
* A worker's lanes worked out with another requirements index than this server's (a deploy changed the catalog after
* its list of files came, or a server with the old index took the list during a deploy's overlap): worked out again at
* once from the list it sent, so a workflow the new index adds, or a lane it changes, counts from this beat on. True when
* they were (the studio then asks for its list again too, for files the old index didn't name). A worker that sent no
* list yet is left as it is.
*/
export function refreshLanes(w: WorkerRow): boolean {
if (!w.inventory_json) return false;
const key = indexKey();
if (capableOf(w).index === key) return false;
const r = lanesFor(loadIndex(), inventoryOf(w), sha);
const lanesJson = JSON.stringify({ lanes: r.lanes, workflows: r.workflows, index: key, sets: r.sets });
db().prepare("UPDATE relay_workers SET lanes_json = ? WHERE id = ?").run(lanesJson, w.id);
w.lanes_json = lanesJson;
return true;
}
/** The hash a worker sends with each beat, so the studio asks for the whole list only when it changed. */
export function inventoryHash(inv: Inventory): string {
const files = [...inv.files].map((f) => `${f.folder}/${f.name}:${f.size}:${f.sha256 ?? ""}`).sort();
const packs = [...inv.nodePacks].map(packKey).sort();
return sha(JSON.stringify({ files, packs, ...(inv.media && { media: [...inv.media].sort() }) }));
}
// ---------------------------------------------------------------------------
// The claim (4.3)
/** Tasks for workers that wait, best first, at most this many looked at per claim. */
const LOOK_AT = 200;
/**
* The tasks a slot of this worker looks at, as SQL: its CPU slot only those marked for one (light CPU work and finishing
* jobs); its GPU's slot never a finishing job, nor light CPU work when it has a CPU slot of its own. In the query, so a
* long line of the other slot's tasks never fills the slot's window of LOOK_AT.
*/
const slotWhere = (w: Pick<WorkerRow, "cpu_slot_json">, slot: WorkerSlot) => (slot === "cpu" ? "AND slot = 'cpu' " : cpuSlotOf(w) ? "AND finish_json IS NULL AND slot IS NULL " : "AND finish_json IS NULL ");
/**
* A worker asks for work for one of its slots (its GPU's, or its CPU slot: `slot`), inside one transaction: first a task
* that slot had and lost track of; then, if the slot holds none and the worker may take work, the best waiting task that
* slot can run (its pool, its allowed lanes and their delays, the files and VRAM it needs, the retry rule, and the end of
* its hours). One task at a time in each slot: a slot that holds a task gets nothing new until that one ends or goes back
* in line, so one worker's token never holds more than one person's GPU job and one CPU task. Sticky lanes (pick), among
* the owner's own jobs only: its warm lane goes first until another lane's best task has waited longer than
* switch_after_ms (and is older). A person's run never waits for a warm lane.
*/
export function workerClaim(w: WorkerRow, running: string[], now = Date.now(), slot: WorkerSlot = "gpu", o: { noStream?: boolean } = {}): ClaimedTask | null {
const d = db();
const device = slotDevice(w, slot);
const noStream = !!o.noStream && slot === "gpu";
return d.transaction((): ClaimedTask | null => {
sweepPool(now);
const fresh = d.prepare("SELECT * FROM relay_workers WHERE id = ?").get(w.id) as WorkerRow;
// A worker with no CPU slot (as its last beat said) has nothing to claim for one.
if (slot === "cpu" && !cpuSlotOf(fresh)) return null;
const held = d.prepare(`SELECT * FROM relay_tasks WHERE worker_id = ? AND device = ? AND state IN ('sending','queued','running','collecting') AND stop = 0 ORDER BY ${LINE_ORDER}`).all(fresh.id, device) as TaskRow[];
const mine = held.find((t) => !running.includes(t.id));
if (mine) return claimed(mine, fresh, device, now);
if (held.length) return null;
if (!eligible(fresh, now)) {
if (slot === "gpu") noteAsked(fresh, "blocked", `Asked for work: not taking work (${blockedWhy(fresh, now)})`, now, noStream ? { noStream: true } : {});
return null;
}
// First pick: its GPU leaves a job for a free worker with a higher pick order that would take it next, for a short
// grace (firstPicks), and takes its own next best.
const left = slot === "gpu" ? firstPicks(fresh, now) : new Map<string, string>();
const leave = new Set(left.keys());
// Its PC's memory is in use (the worker's RAM guard, `noStream`): a preference, never a wait. Its best job as
// always; when that one would stream its weights from this worker's RAM, the jobs that would stream here and that
// another worker takes next (streamTakers) are left for it, and it takes its best of the rest. A streamed job no other
// worker could take now is still its own (marked streamed, so the fallback after running out of memory applies).
const asking = noStream ? withoutStreaming(fresh) : fresh;
const wholeHere = (x: TaskRow) => fitsSlot(x, asking, now, slot);
const first = pick(fresh, now, slot, leave);
const looked = first.looked;
let t = first.task;
if (noStream && t && !wholeHere(t)) {
const taken = streamTakers(fresh, now, leave);
if (taken.has(t.id)) t = pick(fresh, now, slot, leave, (x) => wholeHere(x) || !taken.has(x.id)).task;
}
const streamed = noStream && !!t && !wholeHere(t);
const many = `${looked}${looked >= LOOK_AT ? "+" : ""}`;
const none = left.size ? `Asked for work: ${many} waiting, ${left.size === 1 ? "1 left" : `${left.size} left`} for a worker that picks first` : `Asked for work: ${many} waiting, none it can run now`;
const memory = !noStream ? "" : streamed ? " (its PC's memory is in use: no other worker could take this job now)" : " (its PC's memory is in use: it prefers jobs that fit whole on its card)";
if (slot === "gpu") noteAsked(fresh, t ? "found" : "nothing", `${t ? `Asked for work: took 1 of ${many} waiting` : looked ? none : "Asked for work: nothing waiting"}${memory}`, now, { waiting: looked, ...(left.size && { leftFor: [...new Set(left.values())] }), ...(noStream && { noStream: true }) });
// A job that fits whole on its card is taken as such (a model variant too); one it streams, with its RAM counted.
return t ? claimed(t, noStream && !streamed ? asking : fresh, device, now) : null;
}).immediate();
}
/**
* The jobs other workers take next, now (a claim with `noStream`): each other pool worker that is free now (freeNow, as
* for first pick: may take work, nothing holds it back, its GPU's slot empty), with the job its own claim would take
* next (pick: its pool, lanes and delays, mode, files and memory, the retry rule and its hours), each job counted once.
* A worker whose own last claim asked for no streaming (its RAM guard too) counts only with what fits whole on its card,
* so two such workers never leave a job to each other. A worker busy with a job can't take one now, as for first pick.
*/
function streamTakers(w: WorkerRow, now: number, leave: ReadonlySet<string>): Set<string> {
const taken = new Set(leave);
const others = db().prepare("SELECT * FROM relay_workers WHERE kind = 'pool' AND revoked_at IS NULL AND id != ? AND pool = ? ORDER BY pick_order DESC, id").all(w.id, w.pool) as WorkerRow[];
for (const x of others) {
if (!freeNow(x, now)) continue;
const t = pick(x.asked_no_stream ? withoutStreaming(x) : x, now, "gpu", taken).task;
if (t) taken.add(t.id);
}
for (const id of leave) taken.delete(id);
return taken;
}
/**
* The worker as one claim sees it when its PC's memory is in use (its RAM guard, the claim's `noStream`): no RAM to
* stream weights from, so the memory rule (memoryFit) passes only what fits whole on its card. Nothing is saved.
*/
export function withoutStreaming(w: WorkerRow): WorkerRow {
return { ...w, facts_json: JSON.stringify({ ...factsOf(w), ramBytes: 0 }) };
}
/** Why a worker may not take work now, in a few words (its claim's answer). */
function blockedWhy(w: WorkerRow, now: number): string {
if (w.revoked_at || (w.expires_at && w.expires_at <= now)) return "its token no longer works";
if (w.paused_server) return "paused on the website";
if (w.paused_local) return "paused on its PC";
if (w.drain) return "draining";
if ((w.protocol ?? 0) < MIN_PROTOCOL) return "its worker program is too old";
if (!w.seen_at || now - w.seen_at > FRESH_BEAT_MS) return "no recent beat";
if (!hoursAt(hoursOf(w), now).inside) return "outside its hours";
return "not taking work";
}
/**
* A claim of its GPU's slot and its answer: saved on the worker (asked_at, asked_words) at every claim, and a `looking`
* row when the answer differs from its last one (found, nothing, or blocked for another reason). Best-effort.
*/
function noteAsked(w: WorkerRow, answer: "found" | "nothing" | "blocked", words: string, now: number, detail: Record<string, unknown> = {}): void {
try {
db().prepare("UPDATE relay_workers SET asked_at = ?, asked_words = ?, asked_no_stream = ? WHERE id = ?").run(now, words, detail.noStream === true ? 1 : 0, w.id);
const last = lastOfKind(w.id, "looking");
if (last && last.event === answer && (answer !== "blocked" || last.words === words)) return;
logWorkerEvent({ kind: "looking", event: answer, workerId: w.id, words, detail }, now);
} catch {}
}
function claimed(t: TaskRow, w: WorkerRow, device: string, now: number): ClaimedTask {
const needs = json<Needs | null>(t.needs_json, null);
const bench = t.lane ? benchOf(w)[t.lane] : undefined;
// A workflow with model variants: the files this worker runs it with, chosen when it takes the task from the line and
// kept on the task (extra_json.variant), so a worker that lost track of its task gets the same files again.
const { preempted, variant: lastVariant, streamed: lastStreamed, ...extra } = json<Record<string, unknown>>(t.extra_json, {});
const index = loadIndex();
const variantOf = !t.finish_json && needs?.workflowId && hasVariants(index, needs.workflowId) ? needs.workflowId : null;
const choice = !variantOf
? null
: t.state === "waiting"
? workerFiles(w, variantOf, bytesOf(needs?.runtime))
: (fileChoices(index, variantOf).find((c) => c.variant === (typeof lastVariant === "string" ? lastVariant : null)) ?? null);
const variant = choice?.variant ?? undefined;
// Taken with part of its weights streamed from this worker's RAM (not whole on its card): kept on the task
// (extra_json.streamed), so a run of it that runs out of memory goes to another worker (setPoolOutOfMemory). A worker
// that takes its own task again keeps what its claim said.
const streamed = t.state === "waiting" ? !t.finish_json && memoryWayOf(needs, w) === "stream" : lastStreamed === true;
if (t.state === "waiting") logWorkerJob("claimed", t, w.id, now, {}, { ...(variant && { variant }) });
// A worker that lost track of its own task gets it again with its first claim's time, so the sending clock (SENDING_MS)
// is never wound back by asking again.
// A job ticket's task keeps the time of its claim before this one, and which claim that note is for (`claimFor`, this
// claim's time): a refusal of the ticket on this same claim, before anything ran (another catalog, more than the PC
// allows), puts it back as it was, so a task no worker ever ran keeps its wait limit.
// A run given back for people (workerBeat) is no longer paused once a worker takes it again.
const ticketNote = !!t.ticket_json && t.state === "waiting";
const before =
ticketNote || preempted !== undefined || variant !== lastVariant || streamed !== (lastStreamed === true)
? { extra_json: JSON.stringify({ ...extra, ...(ticketNote && { claimedBefore: t.claimed_at ?? null, claimFor: now }), ...(variant && { variant }), ...(streamed && { streamed: true }) }) }
: {};
const c = takeTask(t, device, now, { worker_id: w.id, ...before, ...(t.state !== "waiting" && t.device === device && t.claimed_at && { claimed_at: t.claimed_at }) });
// A GPU job taken by a worker with another model loaded (or none) loads its model first: it is "getting ready" until
// its first step reports. A light CPU job and a finishing job load none.
if (t.state === "waiting" && t.pools && t.slot !== "cpu" && !t.finish_json && !!t.lane && w.warm_lane !== t.lane) noteCold(t.id, now);
if (t.state === "waiting") noteStage(t.id, now, { slot: device.endsWith("-cpu") ? "cpu" : "gpu" }, true);
// A job given back once for running past its limit gets the whole run ceiling on its next try (a lane's times may
// come from shorter jobs than this one); past that, it fails. Light CPU work has the same limits in either slot. A
// finishing job has its lane's run ceiling (queue-settings.ts, the owner's to set).
const limit = t.finish_json || runLimitedOf(t) ? runCeilingMs(t.lane) : runLimitMs(bench?.p90Ms, t.lane, bench?.loadMs);
const stall = Math.min(stallLimitMs(t.lane), limit);
// A variant's files: the owner's graph gets them in its model slots here; a job ticket names the variant and its files,
// and the worker composes the graph with them, as its own catalog defines the variant (src/lib/ticket-compose.ts).
const swap = choice?.variant ? choice.swap : null;
const fileOf = (k: FileKey): FileNeed[] => {
const f = index.files[k];
return f ? [{ folder: f.folder, name: f.file, size: f.sizeBytes, sha256: f.sha256 ?? null }] : [];
};
return {
...c,
...(swap && c.graph && { graph: swapGraphFiles(c.graph, swap, index) }),
// A job ticket carries its run and stall limits too.
...(c.ticket && { ticket: { ...c.ticket, runLimitMs: limit, stallMs: stall, ...(swap && variant && { variant: { id: variant, files: swap } }) } }),
lane: t.lane ?? undefined,
estMs: estOn(t, w),
runLimitMs: limit,
stallMs: stall,
notAfter: t.not_after,
attempt: t.requeues + 1,
models: choice ? choice.files.flatMap(fileOf) : (needs?.files ?? []),
...(variant && { variant }),
};
}
/**
* The files this worker runs a workflow with (src/lib/gpu/model-variants.ts pickFiles): a model variant made for its
* card's generation (gpuGen in its facts), else the workflow's own files, else another variant, by the file sets it has
* whole (its lanes' `sets`, from the files it verified) and its memory: the VRAM of the set, or streaming its weights
* from RAM (memoryFit: the workflow's streaming need, else the size of the set's files and `extraBytes`, the run-time
* files the run adds). Null when it has none of them, or too little memory. A worker program older than
* VARIANT_PROTOCOL gets the workflow's own files only.
*/
export function workerFiles(w: Pick<WorkerRow, "lanes_json" | "facts_json" | "protocol" | "no_stream_json">, workflowId: string, extraBytes = 0): FileChoice | null {
const cap = capableOf(w);
if (!cap.workflows.includes(workflowId)) return null;
const whole = cap.sets?.[workflowId] ?? [null];
const facts = json<ComputerFacts>(w.facts_json, {});
const index = loadIndex();
const stream = streamRuleOf(w, workflowId);
const choices = fileChoices(index, workflowId).filter((c) => whole.includes(c.variant));
return pickFiles(choices, {
gpuGen: facts.gpuGen ?? null,
has: () => true,
fits: (gb, files) => memoryFit(gb, stream, facts, { fileBytes: filesBytes(index, files), extraBytes }) !== null,
variants: (w.protocol ?? 0) >= VARIANT_PROTOCOL,
});
}
/**
* The workflows this worker no longer takes by streaming its weights from RAM, each with when that was noted: a job of
* one ran out of memory on it that way (noteNoStream). It still takes them with every weight on its card.
*/
export function noStreamOf(w: Pick<WorkerRow, "no_stream_json">): Record<string, number> {
const raw = json<Record<string, unknown>>(w.no_stream_json ?? null, {});
return Object.fromEntries(Object.entries(raw).filter((e): e is [string, number] => typeof e[1] === "number"));
}
/** A workflow's streaming rule on this worker: the catalog's (requirements.json `stream`), or none after it ran out of memory streaming it. */
function streamRuleOf(w: Pick<WorkerRow, "no_stream_json">, workflowId: string): StreamRule {
return noStreamOf(w)[workflowId] !== undefined ? false : loadIndex().workflows[workflowId]?.stream;
}
/** The total size of these files in bytes (a task's own files, or the run-time files it adds: a look's LoRAs). */
const bytesOf = (files: readonly FileNeed[] | undefined): number => (files ?? []).reduce((n, f) => n + (f.size || 0), 0);
/**
* How this worker holds a task's workflow with no model variants: the VRAM the task needs (with the 3% allowance
* already in it: "vram"), or streaming its weights from RAM ("stream": the workflow's streaming need, else the size of
* its files and the run-time files), or not at all (null).
*/
function memoryWay(needs: Needs | null, w: Pick<WorkerRow, "facts_json" | "no_stream_json">, extraBytes = bytesOf(needs?.runtime)): "vram" | "stream" | null {
const facts = json<ComputerFacts>(w.facts_json, {});
if ((needs?.minVramBytes ?? 0) <= (facts.vramBytes ?? 0)) return "vram";
const wf = needs?.workflowId ? loadIndex().workflows[needs.workflowId] : undefined;
if (!needs?.workflowId || !wf) return null;
return streamFits(streamRuleOf(w, needs.workflowId), facts, { fileBytes: bytesOf(needs.files), extraBytes }, wf.vramGb) ? "stream" : null;
}
/** Has this worker the memory for a task's workflow with no model variants (memoryWay)? */
const memoryFor = (needs: Needs | null, w: Pick<WorkerRow, "facts_json" | "no_stream_json">, extraBytes?: number): boolean => memoryWay(needs, w, extraBytes) !== null;
/**
* How this worker would hold a task's workflow: whole on its card ("vram"), streamed from its RAM ("stream"), or not at
* all (null). With model variants, for the files it would run (workerFiles).
*/
function memoryWayOf(needs: Needs | null, w: WorkerRow): "vram" | "stream" | null {
const id = needs?.workflowId;
if (!id || !hasVariants(loadIndex(), id)) return memoryWay(needs, w);
const extraBytes = bytesOf(needs.runtime);
const c = workerFiles(w, id, extraBytes);
return c ? memoryFit(c.vramGb, streamRuleOf(w, id), json<ComputerFacts>(w.facts_json, {}), { fileBytes: filesBytes(loadIndex(), c.files), extraBytes }) : null;
}
/** How long a task should take on this worker: the lane's bench median there, or the task's own estimate. */
function estOn(t: TaskRow, w: WorkerRow): number | null {
return (t.lane && benchOf(w)[t.lane]?.medianMs) || t.est_ms || null;
}
/**
* Why a worker can't run a task (fitWhy, fitSlotWhy), the first reason found: "pool" (not one of the task's pools),
* "elsewhere" (sent to another worker), "stopped", "lane-off" (its lane isn't allowed there), "ceiling" (beyond what
* its PC may ever run), "clip" (longer than the clips its lane takes there), "version" (its worker program or catalog
* doesn't take the job's ticket), "refused" (it refused the job before), "files" (it hasn't the workflow's files),
* "run-files" (it hasn't the files the run adds), "memory" (too little GPU memory, streaming from RAM counted),
* "finish" (no slot of it runs this finishing step), "delay" (its delays haven't passed yet).
*/
export type FitCode = "pool" | "elsewhere" | "stopped" | "lane-off" | "ceiling" | "clip" | "version" | "refused" | "files" | "run-files" | "memory" | "finish" | "delay";
/**
* May this worker run this task: its pool, its lane allowed and capable, its PC's ceiling (the lanes or workflows it
* may ever run, the largest picture, the longest clip, the most results), its build's catalog for a job ticket, its
* files (the run-time files the run adds too), its VRAM, and the delays. Null: it may; else the first reason it may not.
*/
function fitWhy(t: TaskRow, w: WorkerRow, now: number): FitCode | null {
if (!(t.pools ?? "").split(",").includes(w.pool)) return "pool";
if (t.only_worker && t.only_worker !== w.id) return "elsewhere";
if (t.stop) return "stopped";
const allow = allowOf(w);
const lane = t.lane ?? "";
if (!(lane in allow)) return "lane-off";
const cap = capableOf(w);
const needs = json<Needs | null>(t.needs_json, null);
if (ceilingProblem(ceilingOf(w), { workflowId: needs?.workflowId, lane, size: needs?.size })) return "ceiling";
if (pastLaneLimit(allow, lane, needs?.size)) return "clip";
// A ticket made for another catalog than the worker's goes only to a worker that checks the graph itself
// (GRAPH_PROTOCOL), with the ticket's graph fingerprint to check. One that found the graph changed gave the job back
// (refusedBy, below) and isn't offered it again until it is updated.
if (needs?.catalogVersion && needs.catalogVersion !== w.catalog_version && !(needs.graphSha && !!w.catalog_version && (w.protocol ?? 0) >= GRAPH_PROTOCOL)) return "version";
// A ticket's pictures come as they were sent: only a worker that makes them again from their pixels takes it.
if (needs?.catalogVersion && (w.protocol ?? 0) < TICKET_PROTOCOL) return "version";
if (refusedBy(t).includes(refusalKey(w))) return "refused";
if (needs?.workflowId ? !cap.workflows.includes(needs.workflowId) : !cap.lanes.includes(lane)) return "files";
// The files the run adds beyond its workflow's own (a look's LoRAs): only a worker that verified each one.
if (needs?.runtime?.length && !hasFiles(inventoryOf(w), needs.runtime)) return "run-files";
// A workflow with model variants: the files (its own or a variant's) and the memory of the choice this worker would
// run. Either way, streaming from RAM counts as well as the VRAM need (memoryFit).
if (needs?.workflowId && hasVariants(loadIndex(), needs.workflowId)) {
if (!workerFiles(w, needs.workflowId, bytesOf(needs.runtime))) return variantWhy(w, needs.workflowId);
} else if (!memoryFor(needs, w)) return "memory";
if (now - t.created_at < w.claim_after_ms + (allow[lane]?.delayMs ?? 0)) return "delay";
return null;
}
/**
* Why workerFiles has no choice for a workflow with model variants: "memory" when a file set it has would do with all
* the GPU memory there is, else "files". Asked only once the choice failed.
*/
function variantWhy(w: WorkerRow, workflowId: string): FitCode {
const roomy = { ...w, facts_json: JSON.stringify({ ...json<ComputerFacts>(w.facts_json, {}), vramBytes: Number.MAX_SAFE_INTEGER }) };
return workerFiles(roomy, workflowId) ? "memory" : "files";
}
/**
* May this slot of this worker run this task: a finishing job only in a CPU slot that says it runs that kind (in the
* task's pools, for its worker when it names one); light CPU work in its CPU slot when it has one, else in its GPU's
* slot; anything else only in its GPU's slot. Then `fitWhy` as for any task. Null: it may; else the first reason.
*/
function fitSlotWhy(t: TaskRow, w: WorkerRow, now: number, slot: WorkerSlot): FitCode | null {
if (slotFor(t, w) !== slot) return "finish";
const kind = finishKind(t);
if (kind === null) return fitWhy(t, w, now);
if (!(t.pools ?? "").split(",").includes(w.pool)) return "pool";
if (t.only_worker && t.only_worker !== w.id) return "elsewhere";
if (t.stop) return "stopped";
if (refusedBy(t).includes(refusalKey(w))) return "refused";
return cpuSlotOf(w)?.finishes.includes(kind) ? null : "finish";
}
/** May this slot of this worker run this task (fitSlotWhy)? */
function fitsSlot(t: TaskRow, w: WorkerRow, now: number, slot: WorkerSlot): boolean {
return fitSlotWhy(t, w, now, slot) === null;
}
/** Each reason a worker can't run a task, in plain words (a worker of another pool isn't listed). */
export const FIT_WORDS: Record<Exclude<FitCode, "pool">, string> = {
elsewhere: "sent to another worker",
stopped: "stopped",
"lane-off": "not allowed this kind of job",
ceiling: "beyond what its PC may run",
clip: "takes shorter clips of this kind",
version: "needs its worker program updated",
refused: "refused it before",
files: "lacks its model files",
"run-files": "lacks the files this run adds",
memory: "needs more GPU memory",
finish: "doesn't run this finishing step",
delay: "takes it after its delay",
};
/**
* A worker's state in one word for a run's fit: taking jobs ("idle": ready, or behind the site's version, when the fit
* says which jobs it takes), running one ("busy"), not taking any now ("paused": paused in either place, finishing its
* last job before a pause, outside its hours, its GPU or ComfyUI taken or down, local work first), or not heard from
* ("offline").
*/
export type FitState = "idle" | "busy" | "paused" | "offline";
export const fitState = (s: WorkerState): FitState => (s === "idle" || s === "update-worker" ? "idle" : s === "busy" ? "busy" : s === "offline" || s === "new" ? "offline" : "paused");
/** Whether one worker can run a run, why not (`code`, `words`: FIT_WORDS), and its state (`stateCode`, `stateWords`: workerState). */
export interface WorkerFit {
name: string;
can: boolean;
code: Exclude<FitCode, "pool"> | null;
words: string | null;
state: FitState;
stateCode: WorkerState;
stateWords: string;
}
/** Which workers of a run's pools can run it and in what state they are, and the GPU memory it needs in whole GB (null: none named). */
export interface RunFit {
needGb: number | null;
workers: WorkerFit[];
}
/**
* Which workers can run this task, by the claim's own rules (fitSlotWhy, for the slot each would take it in; delays
* aside), and the state each one is in. `workers`: each worker with its workerState, read once for a whole view.
*/
export function runFit(t: TaskRow, workers: readonly { w: WorkerRow; state: { state: WorkerState; words: string } }[], now = Date.now()): RunFit {
const fit: WorkerFit[] = [];
for (const { w, state } of workers) {
const code = fitSlotWhy({ ...t, created_at: 0 }, w, now, slotFor(t, w));
if (code === "pool") continue;
fit.push({ name: w.name, can: code === null, code, words: code ? FIT_WORDS[code] : null, state: fitState(state.state), stateCode: state.state, stateWords: state.words });
}
const need = json<Needs | null>(t.needs_json, null)?.minVramBytes ?? 0;
return { needGb: need > 0 ? Math.round(need / minVramBytesFor(1)) : null, workers: fit };
}
/** The retry rule: a worker that lost or failed this task takes it again only when no other capable worker is about. */
function retryOk(t: TaskRow, w: WorkerRow, now: number): boolean {
const tried = json<string[]>(t.tried_json, []);
if (!tried.includes(w.id)) return true;
const others = db().prepare("SELECT * FROM relay_workers WHERE kind = 'pool' AND revoked_at IS NULL AND id != ? AND seen_at > ?").all(w.id, now - RETRY_WINDOW_MS) as WorkerRow[];
return !others.some((o) => !tried.includes(o.id) && fitsSlot(t, o, now + RETRY_WINDOW_MS, slotFor(t, o)));
}
/** Near the end of its hours, only a task that ends before they do. */
function beforeWindowEnds(t: TaskRow, w: WorkerRow, now: number): boolean {
const h = hoursAt(hoursOf(w), now);
if (h.endsAt === null) return true;
return now + (estOn(t, w) ?? 0) <= h.endsAt;
}
/**
* The best task this worker may run, in the line's order for it (smartLine, with its own estimates). People's runs
* always come first, and a warm lane never makes a person wait: while any person's run waits that this worker can take,
* it takes the best of them. A run that failed once has its last try only when the line is free (secondTryGate). Best
* GPU for the job: a run only this worker can make now goes before the others (onlyHere), so a big GPU doesn't take
* what a smaller one could while a run only it can make waits. Only among the owner's own jobs may one in the lane it
* has loaded go ahead of the best one, while the best one has waited less than the pick allowance (the install's
* setting, capped by this worker's switch_after_ms; 0: strict order). `leave`: the tasks it leaves for a worker that
* picks first (firstPicks); `also`: one more test a task must pass (a claim with `noStream`: workerClaim).
*/
function pick(w: WorkerRow, now: number, slot: WorkerSlot = "gpu", leave?: ReadonlySet<string>, also?: (t: TaskRow) => boolean): { task: TaskRow | null; looked: number } {
const settings = queueSettings(now);
// The owner's controls: this worker's mode, and the runs and people sent first or held.
const c = controlsOf(w, now);
// Each slot looks only at its own tasks.
const rows = lineRows("*", `pools IS NOT NULL AND state = 'waiting' AND stop = 0 AND INSTR(',' || pools || ',', ?) > 0 ${slotWhere(w, slot)}`, [`,${w.pool},`], c, LOOK_AT);
const looked = rows.length;
const second = secondTryGate(w.pool, slotWhere(w, slot), c, now);
const ok = controlledOrder(rows, c, { now, worker: w, workers: [w] }).filter(
(t) => !leave?.has(t.id) && fitsSlot(t, w, now, slot) && retryOk(t, w, now) && beforeWindowEnds(t, w, now) && (!also || also(t)) && second(t),
);
if (!ok.length) return { task: null, looked };
const only = slot === "gpu" ? onlyHere(w, ok, c, now) : null;
if (only) return { task: only, looked };
const best = ok[0];
const allowance = Math.min(w.switch_after_ms, settings.pickAllowanceMs);
// A run the owner sent next, or a boosted person's, never waits for a warm lane either.
const plain = (t: TaskRow) => plainLine(c) || taskControl(t, classRank(t), c, w.id).tier === 2;
if (slot === "cpu" || allowance <= 0 || !w.warm_lane || best.lane === w.warm_lane || classRank(best) !== OWNER_CLASS || !plain(best)) return { task: best, looked };
const warm = ok.find((t) => t.lane === w.warm_lane && classRank(t) === OWNER_CLASS && plain(t));
return { task: warm && (best.rank_at ?? best.created_at) >= now - allowance ? warm : best, looked };
}
// ---------------------------------------------------------------------------
// First pick: a worker with a higher pick order gets a job first, while it is free and can run it
/** A worker with a higher pick order than the one that claims, free now, and the job its own claim would take next. */
export interface FirstPicker {
id: string;
pickOrder: number;
/** That job, and since when it could take it (the job came, and its delays passed). Null: none. */
next: { taskId: string; since: number } | null;
}
/**
* Who gets first pick of a job a worker could take (pure): of the free workers with a pick order above the claiming
* worker's (`mine`), the one with the highest pick order whose next job it is, while that job has been there for it for
* less than `graceMs`. Null: the claiming worker takes it. A grace of 0 leaves no job for anyone.
*/
export function firstPickOf(taskId: string, mine: number, pickers: readonly FirstPicker[], now: number, graceMs: number): string | null {
if (graceMs <= 0) return null;
const by = pickers.filter((p) => p.pickOrder > mine && p.next?.taskId === taskId && now - p.next.since < graceMs);
by.sort((a, b) => b.pickOrder - a.pickOrder || (a.id < b.id ? -1 : 1));
return by[0]?.id ?? null;
}
/**
* Is this worker free to take a job now: it may take work (eligible: heard from within 30 s, paused in neither place,
* not draining, inside its hours), it says nothing holds it back (its beat's `holding`: starting its ComfyUI, local work
* first, an update…), and its GPU's slot holds no job.
*/
function freeNow(x: WorkerRow, now: number): boolean {
if (!eligible(x, now) || x.holding_json || (x.local_why ?? "").startsWith("waiting-for-yield")) return false;
return !db().prepare("SELECT 1 FROM relay_tasks WHERE worker_id = ? AND device = ? AND state IN ('sending','queued','running','collecting')").get(x.id, slotDevice(x, "gpu"));
}
/** When a task became one this worker could take: when it came, plus the worker's claim delay and its lane's delay. */
const claimableSince = (t: TaskRow, x: WorkerRow) => t.created_at + x.claim_after_ms + (allowOf(x)[t.lane ?? ""]?.delayMs ?? 0);
/**
* The tasks this worker's GPU leaves for workers with a higher pick order, each to the worker it is left for
* (firstPickOf): each such worker that is free now (freeNow), highest pick order first, with the job its own claim
* would take next (pick: its pool, lanes and their delays, files, packs and memory, the retry rule, its hours and mode),
* the tasks left for the workers above it left out. Empty at once (one query) when no worker has a higher pick order, or
* the grace is 0.
*/
function firstPicks(w: WorkerRow, now: number): Map<string, string> {
const out = new Map<string, string>();
const graceMs = queueSettings(now).firstPickGraceMs;
if (graceMs <= 0) return out;
const mine = w.pick_order ?? 0;
const above = db().prepare("SELECT * FROM relay_workers WHERE kind = 'pool' AND revoked_at IS NULL AND id != ? AND pick_order > ? ORDER BY pick_order DESC, id").all(w.id, mine) as WorkerRow[];
const pickers: FirstPicker[] = [];
for (const x of above) {
if (!freeNow(x, now)) continue;
const t = pick(x, now, "gpu", new Set(out.keys())).task;
pickers.push({ id: x.id, pickOrder: x.pick_order ?? 0, next: t ? { taskId: t.id, since: claimableSince(t, x) } : null });
const who = t ? firstPickOf(t.id, mine, pickers, now, graceMs) : null;
if (t && who) out.set(t.id, who);
}
return out;
}
// ---------------------------------------------------------------------------
// The smart order: people take turns, in rounds
/** A person's run that has waited this long or longer goes to the front of the line, the oldest first. */
export const AGED_MS = 20 * 60_000;
/**
* Why a waiting task stands where it does in the line (the owner's Queue page reads ORDER_WORDS): it waited 20 minutes
* ("aged"), its person is watching it, has a priority, or has no finished run yet ("first-run"), it is its person's
* fastest in their turn ("turn"), it failed once and waits for its last try ("second-try"), only one GPU on now can make
* it ("only-gpu"), or it is the owner's own job ("owner").
*/
export type OrderWhy = "aged" | "watching" | "priority" | "first-run" | "turn" | "second-try" | "only-gpu" | "owner";
/** The owner's words for why a run stands where it does (never shown to a visitor). */
export const ORDER_WORDS: Record<Exclude<OrderWhy, "owner">, string> = {
watching: "Watching",
priority: "Plan priority",
"first-run": "First run",
turn: "Fastest of theirs",
aged: "Waited 20 min",
"second-try": "Second try when the line is free",
"only-gpu": "Only this GPU can make it",
};
/**
* One look at the line: when, the worker that asks (null: the line as a whole, for a place in line and the owner's
* view), and the pool's workers (a task's estimate on the line as a whole is its lane's bench on the quickest of them).
*/
export interface LineLook {
now: number;
worker: WorkerRow | null;
workers: readonly WorkerRow[];
}
/**
* The group a task is served in under a mode, lower first: people's runs (an account's, a guest's, the owner's
* hands-on run), then the owner's own jobs. Our work first turns the two round; Balanced puts them in one group, where
* the owner takes turns as one more person; People only leaves the owner's own jobs out (queue-controls.ts).
*/
function groupOf(t: Pick<TaskRow, "class" | "account">, mode: QueueMode): number {
const owner = classRank(t) === OWNER_CLASS;
if (mode === "balanced") return 0;
if (mode === "ours-first") return owner ? 0 : 1;
return owner ? (mode === "people-only" ? 2 : 1) : 0;
}
/** Whose turn a task takes: its account's (all guests share one), or the owner's. */
const personOf = (t: Pick<TaskRow, "account" | "principal">) => t.account ?? t.principal;
/** Two tasks in the order they came: by their place (rank: a requeued task keeps its own), then when each came, then id. */
const cameOrder = (a: TaskRow, b: TaskRow) => (a.rank_at ?? a.created_at) - (b.rank_at ?? b.created_at) || a.created_at - b.created_at || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
/** A rough time for a job from its size alone (more pictures, more pixels, more seconds take longer). */
function sizeGuessMs(t: Pick<TaskRow, "needs_json">): number {
const s = json<Needs | null>(t.needs_json, null)?.size;
const mp = Math.max(0.25, s?.megapixels || 1);
const n = Math.max(1, s?.outputs || 1);
return Math.round((s?.seconds ? s.seconds * 20_000 : 8_000) * mp * n);
}
/**
* How long each task should take, for its person's turn: its lane's bench on the worker that asks (on the line as a
* whole: on the quickest of the pool's workers), its own estimate, its lane's typical run, else a guess from its size.
* Ties go to the smaller job (sizeGuessMs).
*/
function estimates(look: LineLook): (t: TaskRow) => number {
const typical = memo(typicalPoolMs);
const kept = new Map<string, number>();
const on = look.worker ? [look.worker] : look.workers;
return (t) => {
let ms = kept.get(t.id);
if (ms === undefined) {
const bench = on.map((w) => (t.lane && benchOf(w as WorkerRow)[t.lane]?.medianMs) || 0).filter((x) => x > 0);
ms = (bench.length ? Math.min(...bench) : 0) || t.est_ms || (t.lane ? typical(t.lane) : null) || sizeGuessMs(t);
kept.set(t.id, ms);
}
return ms;
};
}
/**
* Turns people already had in this round of the line, by person: runs a worker holds for them now, and runs a worker
* took and finished for them since `since` (when the oldest run still waiting came), up to `now`. GPU jobs only; the
* owner's own jobs only with `owner` (Balanced).
*/
function servedTurns(since: number, now: number, owner: boolean): Map<string, number> {
const rows = db()
.prepare(
`SELECT COALESCE(account, principal) AS k, COUNT(*) AS n FROM relay_tasks WHERE pools IS NOT NULL AND slot IS NULL AND finish_json IS NULL
${owner ? "" : `AND ${CLASS_RANK} <> ${OWNER_CLASS}`}
AND (state IN ('sending','queued','running','collecting') OR (state IN ('done','failed') AND claimed_at IS NOT NULL AND finished_at > ? AND finished_at <= ?))
GROUP BY k`,
)
.all(since, now) as { k: string; n: number }[];
return new Map(rows.map((r) => [r.k, r.n]));
}
/** Of these accounts, the ones with a finished run on this install's GPUs (one that ended done). */
function finishedOnce(accounts: string[]): Set<string> {
if (!accounts.length) return new Set();
const principals = JSON.stringify(accounts.map((a) => `acct:${a}`));
const rows = db()
.prepare(
`SELECT principal FROM worker_jobs WHERE event = 'done' AND principal IN (SELECT value FROM json_each(?))
UNION SELECT principal FROM relay_tasks WHERE pools IS NOT NULL AND state = 'done' AND principal IN (SELECT value FROM json_each(?))`,
)
.all(principals, principals) as { principal: string }[];
return new Set(rows.map((r) => r.principal.slice(5)));
}
/**
* The line in the order workers serve it, with why each task stands there (pure but for what it reads of the people:
* who is watching, who has a finished run, the turns they had). By group (groupOf), and in a group of people's runs:
*
* 1. Runs that waited AGED_MS or longer, the oldest first, so no run is ever starved by quicker ones.
* 2. Rounds over people: each person with a run waiting has one turn in each round, and their turn takes their fastest
* run (estimates: on the worker that asks). In a round, a person watching their runs goes first, then by priority
* (higher first), then a person with no finished run yet, then everyone else, and at each level the person who has
* waited longest (their run that has stood in line longest). A person's turns already taken count: the runs a worker
* holds for them now, those it finished since the oldest run still waiting came, and their runs in step 1.
* 3. Runs that failed once, in the order they came: their last try comes only when the line is free (secondTryGate).
*
* The owner's own jobs (a group of their own, unless Balanced) go in the order they came, failed once last.
*/
export function smartLine(rows: readonly TaskRow[], mode: QueueMode, look: LineLook): { task: TaskRow; why: OrderWhy }[] {
const groups = new Map<number, TaskRow[]>();
for (const t of rows) {
const g = groupOf(t, mode);
groups.set(g, [...(groups.get(g) ?? []), t]);
}
const out: { task: TaskRow; why: OrderWhy }[] = [];
for (const g of [...groups.keys()].sort((a, b) => a - b)) {
const list = groups.get(g)!;
const people = mode === "balanced" || list.some((t) => classRank(t) !== OWNER_CLASS);
if (!people) {
out.push(...list.filter((t) => !t.failed_once).sort(cameOrder).map((task) => ({ task, why: "owner" as const })));
out.push(...list.filter((t) => !!t.failed_once).sort(cameOrder).map((task) => ({ task, why: "second-try" as const })));
continue;
}
out.push(...rounds(list, mode === "balanced", look));
}
return out;
}
/** A group of people's runs in its order (smartLine). */
function rounds(list: TaskRow[], owner: boolean, look: LineLook): { task: TaskRow; why: OrderWhy }[] {
const { now } = look;
const failed = list.filter((t) => !!t.failed_once).sort(cameOrder);
const live = list.filter((t) => !t.failed_once);
const aged = live.filter((t) => now - t.created_at >= AGED_MS).sort((a, b) => a.created_at - b.created_at || cameOrder(a, b));
const rest = live.filter((t) => now - t.created_at < AGED_MS);
const byPerson = new Map<string, TaskRow[]>();
for (const t of rest) byPerson.set(personOf(t), [...(byPerson.get(personOf(t)) ?? []), t]);
const turnsOut: { task: TaskRow; why: OrderWhy }[] = [];
if (byPerson.size) {
const est = estimates(look);
const watching = watchingNow(now);
const accounts = [...new Set(rest.map((t) => t.account).filter((a): a is string => !!a && !a.startsWith("guest")))];
const finished = finishedOnce(accounts);
const served = servedTurns(Math.min(...live.map((t) => t.created_at)), now, owner);
const agedTurns = new Map<string, number>();
for (const t of aged) agedTurns.set(personOf(t), (agedTurns.get(personOf(t)) ?? 0) + 1);
const keyed: { task: TaskRow; round: number; watch: number; prio: number; first: number; waited: number; who: string; why: OrderWhy }[] = [];
for (const [who, runs] of byPerson) {
runs.sort((a, b) => est(a) - est(b) || sizeGuessMs(a) - sizeGuessMs(b) || cameOrder(a, b));
const account = runs[0].account;
const watch = !!account && watching.has(account);
const prio = Math.max(...runs.map((t) => cleanPriority(t.priority)));
const first = !!account && !account.startsWith("guest") && !finished.has(account);
const why: OrderWhy = watch ? "watching" : prio > 0 ? "priority" : first ? "first-run" : "turn";
const waited = Math.min(...runs.map((t) => t.rank_at ?? t.created_at));
const had = (served.get(who) ?? 0) + (agedTurns.get(who) ?? 0);
runs.forEach((task, i) => keyed.push({ task, round: had + i, watch: watch ? 0 : 1, prio, first: first ? 0 : 1, waited, who, why }));
}
keyed.sort((a, b) => a.round - b.round || a.watch - b.watch || b.prio - a.prio || a.first - b.first || a.waited - b.waited || (a.who < b.who ? -1 : a.who > b.who ? 1 : 0));
turnsOut.push(...keyed.map((k) => ({ task: k.task, why: k.why })));
}
return [...aged.map((task) => ({ task, why: "aged" as const })), ...turnsOut, ...failed.map((task) => ({ task, why: "second-try" as const }))];
}
/** The waiting tasks in the order workers serve them (smartLine), without the reasons. */
export function lineOrder(rows: readonly TaskRow[], mode: QueueMode = DEFAULT_MODE, look: Partial<LineLook> = {}): TaskRow[] {
return smartLine(rows, mode, { now: look.now ?? Date.now(), worker: look.worker ?? null, workers: look.workers ?? poolWorkers() }).map((x) => x.task);
}
/**
* May a task that failed once have its last try now (the other tasks always may)? Only when no other task waits in
* its pool, for the same slot, in a group the line serves before it or with it (for a person's run: another person's
* run, never the owner's own jobs), that the owner's controls let a worker take, that failed no try, and that a worker of
* the pool on now could run (its delays aside). Read once for each look, and only when a task that failed once is asked
* about.
*/
function secondTryGate(pool: string, slotSql: string, c: LineControls, now: number): (t: Pick<TaskRow, "failed_once" | "class" | "account">) => boolean {
let first: number | undefined;
return (t) => {
if (!t.failed_once) return true;
if (first === undefined) {
first = Infinity;
const rows = lineRows(LINE_COLS, `pools IS NOT NULL AND state = 'waiting' AND stop = 0 AND failed_once = 0 AND INSTR(',' || pools || ',', ?) > 0 ${slotSql}`, [`,${pool},`], c, LOOK_AT);
const on = poolWorkers().filter((x) => x.pool === pool && eligible(x, now));
for (const r of rows) {
const g = groupOf(r, c.mode);
if (g >= first || taskControl(r, classRank(r), c, null).out) continue;
if (on.some((x) => fitsSlot({ ...r, created_at: 0 }, x, now, slotFor(r, x)))) first = g;
}
}
return groupOf(t, c.mode) < first;
};
}
/**
* The first of these tasks (a worker's line, in its order) that only this worker can make now: no other worker of its
* pool that is on now (eligible) could run it, its delays aside. Only in the same tier of the owner's controls and the
* same group of the line as the first task, and never a second try. Null: none, or it is the first task already.
*/
function onlyHere(w: WorkerRow, ok: readonly TaskRow[], c: LineControls, now: number): TaskRow | null {
const best = ok[0];
if (!best) return null;
const others = poolWorkers().filter((x) => x.id !== w.id && x.pool === w.pool && eligible(x, now));
if (!others.length) return null;
const tier = (t: TaskRow) => (plainLine(c) ? 2 : taskControl(t, classRank(t), c, w.id).tier);
const bestTier = tier(best);
const bestGroup = groupOf(best, c.mode);
for (const t of ok) {
if (tier(t) !== bestTier || groupOf(t, c.mode) !== bestGroup) break;
if (t.failed_once) continue;
if (!others.some((x) => fitsSlot({ ...t, created_at: 0 }, x, now, slotFor(t, x)))) return t.id === best.id ? null : t;
}
return null;
}
// ---------------------------------------------------------------------------
// Back in line, and the clocks
/**
* A task goes back in line: its worker let it go (`blame: false`: a pause, at no cost to anyone), lost it (silent,
* its ComfyUI down: the worker is noted in tried_json and it counts as an attempt; a recovery, which keeps its place),
* or its attempt failed (`failed`: it stalled, or ran past its run limit). The third lost attempt fails it. A failed
* attempt sends it to the back of the line (failed_once), where it has one more try once the line is free
* (secondTryGate); a second failed attempt fails it. The old worker's lease stops working, so its late calls get 409.
*
* Only the task as `t` read it: one that has moved on since (another app process put it back in line, or a worker took
* it again with a new lease) stays as it is now, and no attempt is counted. Returns whether it changed.
*/
/**
* Why a task was stopped for good after its tries, in the owner's words. With hosted runs on, the line retries by itself
* until then, so the words say how many tries it had instead of asking for Retry (the account reads one plain sentence
* either way); otherwise the words the owner's own jobs always had.
*/
export function stoppedWords(kind: "silent" | "down" | "stuck" | "other", tries: number): string {
if (!hostedRunsOn()) return OWNER_STOP_WORDS[kind];
const why = { silent: "the GPU stopped answering", down: "ComfyUI kept stopping on the GPUs", stuck: "the GPUs couldn't start it", other: "the GPUs couldn't finish it" }[kind];
return `Stopped after ${tries} ${tries === 1 ? "try" : "tries"}: ${why}.`;
}
/** The owner's words for a task stopped for good, with hosted runs off. */
const OWNER_STOP_WORDS = {
silent: "The GPU running this stopped answering, three times. Press Retry.",
down: "ComfyUI kept stopping on the GPUs. Press Retry.",
stuck: "The GPUs couldn't start this run. Press Retry.",
other: "The GPUs couldn't finish this run. Press Retry.",
};
export function backInLine(t: TaskRow, o: { blame: boolean; failed?: boolean; why?: string; whyKind?: "silent" | "down" | "stuck"; last?: boolean; lostAt?: number; sentBack?: string }): boolean {
const d = db();
return d.transaction((): boolean => {
const now = taskById(t.id);
if (!now || now.state !== t.state || now.worker_id !== t.worker_id || now.lease_hash !== t.lease_hash || now.requeues !== t.requeues) return false;
const tried = json<string[]>(t.tried_json, []);
const blame = o.blame || !!o.failed;
const requeues = t.requeues + (blame ? 1 : 0);
// A failed attempt ends it when one failed before; a lost one when it was the last of MAX_ATTEMPTS.
const spent = o.failed ? !!t.failed_once : requeues >= MAX_ATTEMPTS;
if (blame && (spent || o.last)) {
endTask(t.id, "failed", { error: o.why ?? stoppedWords(o.whyKind ?? "other", requeues), requeues, tried_json: JSON.stringify([...new Set([...tried, t.worker_id ?? ""])].filter(Boolean)) }, true);
return true;
}
const sentBack = o.failed && o.sentBack ? `${o.sentBack} (its last try waits until the line is free)` : o.sentBack;
if (t.worker_id) logWorkerJob("requeued", t, t.worker_id, Date.now(), {}, { ...(sentBack && { why: sentBack }), tries: requeues });
// A live worker that let it go hears the stop in its beat answer; one that went silent finds its lease gone when it
// comes back (409), and takes the prompt off its own ComfyUI. It keeps its rank, so it keeps its place in line, and
// its wait limit no longer applies (a worker took it once).
setTask(t.id, {
state: "waiting",
lease_hash: null,
worker_id: null,
requeues,
sent_at: null,
sending_since: null,
started_at: null,
down_at: null,
down_why: null,
note: "resent",
...(blame && t.worker_id && { tried_json: JSON.stringify([...new Set([...tried, t.worker_id])]) }),
// Failed once: at the back of the line, for its last try (smartLine, secondTryGate).
...(o.failed && { failed_once: 1 }),
// Lost because its worker went quiet: noted, so a sleep the worker reports later takes the blame back (forgiveSleep).
...(o.lostAt && t.worker_id && { extra_json: JSON.stringify({ ...json<Record<string, unknown>>(t.extra_json, {}), lost: { worker: t.worker_id, at: o.lostAt } }) }),
});
d.prepare("UPDATE relay_inputs SET reads = 0 WHERE task_id = ?").run(t.id);
// In line again: nothing of the attempt before shows (no step, no preview).
dropProgress(t.id);
return true;
}).immediate();
}
/**
* The workers' clocks: a task whose worker went silent for 2 minutes (or was revoked), whose ComfyUI was down for 2
* minutes, or that is still sending its files 15 minutes after the claim (or after it began sending them again) goes
* back in line; a task no worker has taken yet that is still waiting past its wait limit fails, not charged. (Once a
* worker took it, the wait limit is over: a task back in line waits in its place until a worker runs it, or its attempts
* run out.)
*/
export function sweepPool(now = Date.now()): void {
const d = db();
// One step with the write lock (IMMEDIATE), on every path (the runner's look at a run too): what it reads is still so
// when it writes, even while another app process on the same database claims or requeues.
d.transaction(() => sweepLine(d, now)).immediate();
}
function sweepLine(d: ReturnType<typeof db>, now: number): void {
const held = d
.prepare(
`SELECT t.*, w.seen_at AS w_seen, w.revoked_at AS w_revoked FROM relay_tasks t LEFT JOIN relay_workers w ON w.id = t.worker_id
WHERE t.pools IS NOT NULL AND t.state IN ('sending','queued','running','collecting')`,
)
.all() as (TaskRow & { w_seen: number | null; w_revoked: number | null })[];
for (const t of held) {
const silent = !t.w_seen || now - t.w_seen > RELAY_TIMES.poolSilentMs || !!t.w_revoked;
const down = !!t.down_at && now - t.down_at > RELAY_TIMES.poolComfyDownMs;
// Sending since its claim, or since its ComfyUI lost it and it began sending again on the same claim.
const sendingSince = t.sending_since ?? t.claimed_at;
const stuck = t.state === "sending" && !!sendingSince && now - sendingSince > SENDING_MS;
if (silent || down || stuck) {
const sentBack = t.w_revoked ? "its token was revoked" : silent ? "went silent" : down ? "its ComfyUI stopped answering" : "still sending its files after 15 min";
backInLine(t, { blame: true, whyKind: silent ? "silent" : down ? "down" : "stuck", sentBack, ...(silent && !t.w_revoked && { lostAt: now }) });
}
}
const late = d.prepare("SELECT id FROM relay_tasks WHERE pools IS NOT NULL AND state = 'waiting' AND claimed_at IS NULL AND not_after IS NOT NULL AND not_after < ?").all(now) as { id: string }[];
for (const t of late) endTask(t.id, "failed", { error: ENGINE_WORDS["long-wait"] });
noteSilentWorkers(now);
}
/**
* Workers silent past the silence limit whose saved state isn't "offline" yet: saved as offline, with a state row (at
* the moment the limit passed). A look at the few rows of relay_workers. Best-effort.
*/
export function noteSilentWorkers(now = Date.now()): void {
try {
const d = db();
const gone = d
.prepare("SELECT id, state, seen_at FROM relay_workers WHERE kind = 'pool' AND revoked_at IS NULL AND seen_at IS NOT NULL AND seen_at < ? AND state != 'offline'")
.all(now - RELAY_TIMES.poolSilentMs) as Pick<WorkerRow, "id" | "state" | "seen_at">[];
for (const w of gone) {
if (d.prepare("UPDATE relay_workers SET state = 'offline' WHERE id = ? AND state = ?").run(w.id, w.state).changes === 0) continue;
logWorkerEvent({ kind: "state", event: "offline", workerId: w.id, words: "Offline", state: "offline", detail: { from: w.state, seenAt: w.seen_at }, at: Math.min(now, w.seen_at! + RELAY_TIMES.poolSilentMs) }, now);
}
} catch (e) {
console.warn(`[fuobe] Couldn't note silent workers: ${(e as Error).message}`);
}
}
/** The longest sleep a worker's beat may report (a week). */
const MAX_SLEPT_MS = 7 * 24 * 60 * 60_000;
/** A job lost this long before the reported sleep began still counts as lost to it (beats come every few seconds). */
const SLEEP_SLACK_MS = 2 * 60_000;
/**
* A worker's PC slept for `sleptMs` until about now: each job the studio took back from it for going quiet during that
* sleep goes on with no blame. The attempt it counted is taken back, and the worker is off the job's tried list, so it
* may take the job again at once. Once a job at most: a worker can't keep a job going forever by saying it slept.
*/
export function forgiveSleep(w: Pick<WorkerRow, "id">, sleptMs: number, now = Date.now()): string[] {
const since = now - Math.min(sleptMs, MAX_SLEPT_MS) - SLEEP_SLACK_MS;
const rows = db().prepare("SELECT * FROM relay_tasks WHERE pools IS NOT NULL AND state IN ('waiting','sending','queued','running','collecting') AND INSTR(extra_json, '\"lost\"') > 0").all() as TaskRow[];
const forgiven: string[] = [];
for (const t of rows) {
const { lost, ...extra } = json<{ lost?: { worker?: string; at?: number }; sleptOnce?: boolean } & Record<string, unknown>>(t.extra_json, {});
if (lost?.worker !== w.id || typeof lost.at !== "number" || lost.at < since || lost.at > now) continue;
if (extra.sleptOnce || t.requeues <= 0) {
setTask(t.id, { extra_json: JSON.stringify(extra) });
continue;
}
const tried = json<string[]>(t.tried_json, []).filter((x) => x !== w.id);
setTask(t.id, { requeues: t.requeues - 1, tried_json: tried.length ? JSON.stringify(tried) : null, extra_json: JSON.stringify({ ...extra, sleptOnce: true }) });
forgiven.push(t.id);
}
return forgiven;
}
setPoolClocks((t, now) => {
sweepPool(now);
return taskById(t.id) ?? t;
});
/** The words of a run that a worker gave back for the last time. */
export const GAVE_BACK_WORDS = "This run couldn't finish.";
/** Workers that refused a task's job ticket without running it (another catalog, or more than their PC allows). */
const refusedBy = (t: Pick<TaskRow, "extra_json">): string[] => json<{ refusedBy?: string[] }>(t.extra_json ?? null, {}).refusedBy ?? [];
/**
* A worker as it was when it refused a ticket: its id, its build's catalog and its PC's ceiling. Once it is updated or
* its ceiling changes, it may be offered the task again.
*/
const refusalKey = (w: Pick<WorkerRow, "id" | "catalog_version" | "ceiling_json">) =>
`${w.id}:${crypto.createHash("sha256").update(`${w.catalog_version ?? ""}\n${w.ceiling_json ?? ""}`).digest("hex").slice(0, 16)}`;
/** A task a worker gave back for running past its run limit (its next try gets the whole run ceiling). */
const runLimitedOf = (t: TaskRow) => json<{ runLimited?: boolean }>(t.extra_json, {}).runLimited === true;
/**
* A worker gives its task back. It stalled, or ran past its run limit: its attempt failed, so it goes to the back of the
* line (backInLine `failed`), and has its last try once no other run waits that a worker could take; a second failed
* attempt fails it (past its run limit a second time, that try had the whole run ceiling: one more would only hold a GPU
* as long again). It takes clips or sound the worker can't make again: a recovery, with its place kept, counted as an
* attempt, so another worker that can run it is preferred and the third time fails it.
*/
setPoolGiveBack((t, why) => {
// A job ticket the worker refused before running it (built from another catalog, or more than its PC allows): back in
// line at no cost, never offered to that worker again. Refused on the claim that took it from the line, before
// anything ran on that claim, it waits as it did before that claim (a task no worker took keeps its wait limit). Once
// a job of it was sent or ran, it keeps its claim's time: its wait limit is over for good. In one locked step with
// going back in line, so a claim another app process makes meanwhile is never changed.
// Ran out of memory here, and its retry here froze too: the same, except that it ran (it counts as an attempt).
if (why === "update-worker" || why === "outside-ceiling" || why === "out-of-memory") {
db()
.transaction(() => {
const w = t.worker_id ? (db().prepare("SELECT id, catalog_version, ceiling_json FROM relay_workers WHERE id = ?").get(t.worker_id) as Pick<WorkerRow, "id" | "catalog_version" | "ceiling_json"> | undefined) : undefined;
const { claimedBefore, claimFor, ...rest } = json<Record<string, unknown>>(t.extra_json, {});
const extra = { ...rest, refusedBy: [...new Set([...refusedBy(t), ...(w ? [refusalKey(w)] : [])])].slice(-64) };
const untouched = typeof claimFor === "number" && claimFor === t.claimed_at && t.state === "sending" && !t.sent_at && !t.started_at;
const sentBack = why === "update-worker" ? "its graph is another on the worker's version (it waits for the worker's update)" : why === "out-of-memory" ? GIVE_BACK_WORDS[why] : "more than its PC runs";
if (!backInLine(t, untouched ? { blame: false, sentBack } : { blame: true, why: GAVE_BACK_WORDS, sentBack })) return;
if (taskById(t.id)?.state !== "waiting") return;
setTask(t.id, { extra_json: JSON.stringify(extra), ...(untouched && { claimed_at: typeof claimedBefore === "number" ? claimedBefore : null }) });
})
.immediate();
return;
}
const sentBack = GIVE_BACK_WORDS[why] ?? "the worker gave it back";
// The worker's own check failed before the run (it pauses itself): back in line with its place kept, at no cost.
if (why === "self-check") return void backInLine(t, { blame: false, sentBack });
if (why === "run-limit") {
if (runLimitedOf(t)) return backInLine(t, { blame: true, failed: true, why: GAVE_BACK_WORDS, last: true, sentBack });
setTask(t.id, { extra_json: JSON.stringify({ ...json<Record<string, unknown>>(t.extra_json, {}), runLimited: true }) });
}
backInLine(t, { blame: true, failed: why === "stalled" || why === "run-limit", why: GAVE_BACK_WORDS, sentBack });
});
/**
* A worker's ComfyUI lost its task twice on one claim (each loss sends it again once): it goes back in line with its
* place kept, and counts as an attempt, so another worker is preferred and the third attempt fails it.
*/
setPoolLost((t) => backInLine(t, { blame: true, sentBack: "its ComfyUI lost it twice" }));
/**
* A worker's job ran out of memory. One it took by streaming its weights from RAM (extra_json.streamed, from its claim):
* the worker no longer takes that workflow by streaming (noteNoStream), and while another worker could run it, the job
* goes back in line with its place kept, at no cost to the run (no attempt counted: this worker can't take it that way
* again, so it can't come back to it). With no other worker for it, or taken with every weight on the card, it fails as
* any failed job does.
*/
setPoolOutOfMemory((t, now) => {
if (!t.worker_id || json<{ streamed?: unknown }>(t.extra_json, {}).streamed !== true) return false;
const needs = json<Needs | null>(t.needs_json, null);
if (!needs?.workflowId) return false;
noteNoStream(t.worker_id, needs.workflowId, now, t);
if (!workersCanRun(needs.workflowId, { size: needs.size, files: needs.runtime, tickets: !!needs.catalogVersion })) return false;
if (!backInLine(t, { blame: false, sentBack: "it ran out of memory with its weights streamed from RAM (another worker takes it)" })) return false;
setTask(t.id, { comfy_why: null });
return true;
});
/** The most workflows a worker's no-streaming list keeps (the oldest go first). */
const NO_STREAM_MAX = 200;
/**
* This worker no longer takes this workflow by streaming its weights from RAM: noted with the time on the worker
* (no_stream_json) and in its timeline. It still takes it with every weight on its card. The owner clears it in
* Settings → Workers (WorkerPatch.clearNoStream). Best-effort.
*/
export function noteNoStream(workerId: string, workflowId: string, now = Date.now(), t?: Pick<TaskRow, "id" | "generation_id" | "lane">): void {
try {
const d = db();
d.transaction(() => {
const w = d.prepare("SELECT no_stream_json FROM relay_workers WHERE id = ?").get(workerId) as Pick<WorkerRow, "no_stream_json"> | undefined;
if (!w) return;
const list = { ...noStreamOf(w), [workflowId]: now };
const kept = Object.entries(list).sort((a, b) => b[1] - a[1]).slice(0, NO_STREAM_MAX);
d.prepare("UPDATE relay_workers SET no_stream_json = ? WHERE id = ?").run(JSON.stringify(Object.fromEntries(kept)), workerId);
})();
logWorkerEvent(
{ kind: "job", event: "no-stream", workerId, words: `Ran out of memory streaming ${workflowId} from RAM: it takes it only with every weight on its card`, taskId: t?.id ?? null, generationId: t?.generation_id ?? null, lane: t?.lane ?? null, detail: { workflowId } },
now,
);
} catch (e) {
console.warn(`[fuobe] Couldn't note a worker's streaming list: ${(e as Error).message}`);
}
}
/** Why a worker gave a job back, in the timeline's words. */
const GIVE_BACK_WORDS: Record<string, string> = {
stalled: "it stalled",
"run-limit": "it ran past its run limit",
media: "the worker can't make its clips or sound again",
"self-check": "the worker's check before the run didn't pass",
"out-of-memory": "it ran out of memory on that worker (another worker takes it)",
};
// ---------------------------------------------------------------------------
// Place in line and the wait (4.4)
/** Tasks looked at for a place in line, at most (a longer line counts as this long). */
const LOOK_AHEAD = 2000;
const poolWorkers = () => db().prepare("SELECT * FROM relay_workers WHERE kind = 'pool' AND revoked_at IS NULL").all() as WorkerRow[];
/** Do two tasks share a pool (the same workers may serve both)? */
const sharePool = (a: Pick<TaskRow, "pools">, b: Pick<TaskRow, "pools">) => (a.pools ?? "").split(",").some((p) => !!p && (b.pools ?? "").split(",").includes(p));
/**
* A lane's typical run on the workers: the median measured time of its last 20 finished jobs (the worker's own
* execution time), else of any lane's, else null before any.
*/
export function typicalPoolMs(lane: string | null): number | null {
const d = db();
const of = (rows: { ms: number }[]) => (rows.length ? rows.map((r) => r.ms).sort((a, b) => a - b)[Math.floor(rows.length / 2)] : null);
const q = (where: string, ...args: unknown[]) =>
d.prepare(`SELECT exec_ms AS ms FROM worker_jobs WHERE event = 'finished' AND exec_ms > 0 ${where} ORDER BY id DESC LIMIT 20`).all(...args) as { ms: number }[];
return (lane ? of(q("AND lane = ?", lane)) : null) ?? of(q(""));
}
/** Each lane's typical run as cardTypicalMs read it last, and when (read again after LANE_TIME_MS). */
const laneTimes = new Map<string, { at: number; ms: number | null }>();
const LANE_TIME_MS = 60_000;
/**
* How long a task's run usually takes, for a person's card (src/lib/run-progress.ts cardProgress): its worker's bench
* for its lane, else its lane's typical run (the middle of its last 20 finished runs). Null when neither says: then
* the card shows no time-based share. Never another lane's times.
*/
export function cardTypicalMs(t: Pick<TaskRow, "lane" | "worker_id">, now = Date.now()): number | null {
if (!t.lane) return null;
const w = t.worker_id ? (db().prepare("SELECT bench_json FROM relay_workers WHERE id = ?").get(t.worker_id) as Pick<WorkerRow, "bench_json"> | undefined) : undefined;
const bench = w ? benchOf(w as WorkerRow)[t.lane]?.medianMs : undefined;
if (bench && bench > 0) return bench;
const kept = laneTimes.get(t.lane);
if (kept && now - kept.at < LANE_TIME_MS) return kept.ms;
const rows = db().prepare("SELECT exec_ms AS ms FROM worker_jobs WHERE event = 'finished' AND exec_ms > 0 AND lane = ? ORDER BY id DESC LIMIT 20").all(t.lane) as { ms: number }[];
const ms = rows.length ? rows.map((r) => r.ms).sort((a, b) => a - b)[Math.floor(rows.length / 2)] : null;
if (laneTimes.size >= 256) laneTimes.delete(laneTimes.keys().next().value!);
laneTimes.set(t.lane, { at: now, ms });
return ms;
}
/** How long a task should take: its lane's bench on the quickest of these workers, its own estimate, or the lane's typical run. */
function estFor(t: Pick<TaskRow, "lane" | "est_ms">, workers: WorkerRow[], typical: (lane: string | null) => number | null): number {
const bench = workers.map((w) => (t.lane && benchOf(w)[t.lane]?.medianMs) || 0).filter((ms) => ms > 0);
return (bench.length ? Math.min(...bench) : 0) || t.est_ms || typical(t.lane) || 60_000;
}
/**
* A place in line: 1 is next. `etaSeconds`: about how long until it is done (null: unknown). `unknown`: no GPU that
* could run it can take it now, so it starts when one is free; `returnAt` (epoch ms): when the first of them is due
* back, when one says.
*/
export interface LinePlace {
place: number;
etaSeconds: number | null;
unknown?: true;
returnAt?: number;
}
/**
* When a worker that can't take work now is due back, when it says (null: it doesn't, or it is on): the start of its
* hours, or the end of a pause on its PC that has one ("Paused here until 16:30"). One that is silent, paused on the
* website, draining or too old for job tickets says nothing of when.
*/
function backAt(w: WorkerRow, now: number): number | null {
if (w.revoked_at || w.paused_server || w.drain || (w.protocol ?? 0) < MIN_PROTOCOL || !w.seen_at || now - w.seen_at > RELAY_TIMES.poolSilentMs) return null;
const until = w.paused_local ? (w.local_until && w.local_until > now ? w.local_until : null) : null;
if (w.paused_local && until === null) return null;
const h = hoursAt(hoursOf(w), until ?? now);
if (h.inside) return until;
return h.nextChangeAt && h.nextChangeAt > now ? h.nextChangeAt : null;
}
/**
* Where a waiting task is in line, and about how long until it is done. Every task ahead of it counts, whatever its lane,
* in the order the workers serve them (LINE_ORDER), when a worker that may run this one may run that one too: one GPU
* serves every lane it has in that order. The wait adds what is left of the jobs those workers are running now, the
* load of its lane when none has it warm, and its own run. No worker able to run it right now (paused, outside its
* hours, silent, built before this studio's update): its place, with no estimate (`unknown`), and the time the first of
* them is due back when one says (`returnAt`: its hours start, or the end of a pause on its PC).
*/
export function placeInLine(t: TaskRow, now = Date.now()): LinePlace {
const d = db();
const all = poolWorkers().filter((w) => sharePool({ pools: w.pool }, t));
// Each worker serves a task in one of its slots (slotFor): a slot's own line is the tasks that slot would take.
const serves = (x: TaskRow, w: WorkerRow) => fitsSlot({ ...x, created_at: 0 }, w, now, slotFor(t, w));
const workers = all.filter((w) => eligible(w, now) && serves(t, w));
const before = lineBefore(t, now);
// Ahead of it: a task the same slots of the same workers would serve first (with none able to run it now, any that
// could run it at all).
const able = workers.length ? workers : all.filter((w) => serves(t, w));
const ahead = before.filter((x) => sharePool(x, t) && (!able.length || able.some((w) => serves(x, w))));
if (!workers.length) {
const back = able.map((w) => backAt(w, now)).filter((at): at is number => at !== null);
return { place: ahead.length + 1, etaSeconds: null, unknown: true, ...(back.length && { returnAt: Math.min(...back) }) };
}
const typical = memo(typicalPoolMs);
const own = estFor(t, workers, typical);
const warm = workers.some((w) => w.warm_lane === t.lane);
const load = warm ? 0 : Math.max(0, ...workers.map((w) => (t.lane && benchOf(w)[t.lane]?.loadMs) || 0));
const sum = ahead.reduce((s, x) => s + estFor(x, workers, typical), 0);
// The jobs these workers' slots hold now: what is left of each.
const devices = new Set(workers.map((w) => slotDevice(w, slotFor(t, w))));
const running = (d.prepare(`SELECT ${LINE_COLS}, device FROM relay_tasks WHERE pools IS NOT NULL AND state IN ('sending','queued','running','collecting') AND worker_id IS NOT NULL`).all() as TaskRow[]).filter((x) => !!x.device && devices.has(x.device));
const left = running.reduce((s, x) => s + Math.max(0, estFor(x, workers, typical) - (now - (x.claimed_at ?? now))), 0);
return { place: ahead.length + 1, etaSeconds: Math.round(((sum + left) / workers.length + load + own) / 1000) };
}
/**
* The waiting tasks served before this one (at most LOOK_AHEAD looked at), in the order of the line as a whole
* (smartLine, with each task's estimate on the quickest worker) under the owner's controls (the pool's mode, runs or
* people sent first or held): those held are left out (they take no turn before it).
*/
function lineBefore(t: TaskRow, now: number): TaskRow[] {
const c = controlsOf((t.pools ?? "pool").split(",")[0] || "pool", now);
const rows = lineRows(LINE_COLS, "pools IS NOT NULL AND state = 'waiting' AND stop = 0 AND id != ?", [t.id], c, LOOK_AHEAD);
const line = controlledOrder([...rows, t], c, { now, worker: null, workers: poolWorkers() }, t.id);
return line.slice(0, line.findIndex((x) => x.id === t.id));
}
/** A one-argument function whose answers are kept for this call's life. */
function memo<A, R>(fn: (a: A) => R): (a: A) => R {
const seen = new Map<A, R>();
return (a) => {
if (!seen.has(a)) seen.set(a, fn(a));
return seen.get(a)!;
};
}
// ---------------------------------------------------------------------------
// Making tasks for workers
/** The catalog's lanes, worked out once for each requirements index (loadIndex keeps one until its file changes). */
const lanesByIndex = new WeakMap<ReturnType<typeof loadIndex>, { lanes: (ReturnType<typeof allLanes>[number] & LaneName)[]; byWorkflow: Map<string, string> }>();
function catalogLanes(): { lanes: (ReturnType<typeof allLanes>[number] & LaneName)[]; byWorkflow: Map<string, string> } {
const index = loadIndex();
let kept = lanesByIndex.get(index);
if (!kept) {
// Each lane by name: who uses its workflows, and its model (its largest file).
const users = workflowUsers();
const lanes = allLanes(index, sha).map((l) => {
const files = l.workflows.flatMap((id) => index.workflows[id]?.files ?? []).map((k) => index.files[k]).filter(Boolean);
const biggest = files.sort((a, b) => b.sizeBytes - a.sizeBytes)[0];
const name = laneName(l.lane, { file: biggest?.file ?? null, apps: l.workflows.flatMap((id) => users.get(id) ?? []), fallback: l.label });
return { ...l, ...name, label: l.lane === "small" ? l.label : name.label };
});
kept = { lanes, byWorkflow: new Map(lanes.flatMap((l) => l.workflows.map((id) => [id, l.lane] as const))) };
lanesByIndex.set(index, kept);
}
return kept;
}
/**
* Who runs each workflow, by name: the catalog's models whose route on ComfyUI is the workflow (a template's variants,
* video-concat-{n}, too), then the apps made on those models.
*/
function workflowUsers(): Map<string, string[]> {
const out = new Map<string, string[]>();
const modelsOf = new Map<string, string[]>();
const add = (wf: string, name: string) => {
const list = out.get(wf) ?? [];
if (!list.includes(name)) list.push(name);
out.set(wf, list);
};
for (const m of MODELS) {
for (const r of m.routes ?? []) {
if (r.provider !== "comfyui" || !r.endpoint) continue;
const ids = r.endpoint.includes("{n}") ? [1, 2, 3, 4].map((n) => r.endpoint.replace("{n}", String(n))) : [r.endpoint];
for (const id of ids) {
add(id, m.name);
modelsOf.set(m.id, [...(modelsOf.get(m.id) ?? []), id]);
}
}
}
for (const a of APPS) for (const r of [a.open, ...(a.openAlso ?? [])]) for (const id of r ? (modelsOf.get(r.modelId) ?? []) : []) add(id, a.name);
return out;
}
/** The lane a workflow's tasks go in (a workflow the catalog doesn't know: the small lane). */
export function poolLane(workflowId: string): string {
return catalogLanes().byWorkflow.get(workflowId) ?? "small";
}
/**
* Is there a worker for this workflow (any workflow: `workflowId` null): one the owner added and didn't revoke, allowed
* the workflow's lane, and able to run it as its last list of files said (its verified files and node packs, the kinds
* of input it takes) with the VRAM it needs, whether it is paused, outside its hours or offline right now (a run then
* waits in line for it). Inside its PC's ceiling: the lanes or workflows that PC may ever run, and, given the job's
* `size`, its largest picture, longest clip and most results. A run that no worker could ever take is refused before
* it is queued, never left to wait for nothing. `tickets`: only a worker that takes job tickets now (ticketReady), as
* an account's run needs; without it, a worker built from another catalog counts too (the owner's own jobs, and
* whether this install's GPUs could ever run it).
*/
export function workersCanRun(workflowId: string | null, o: { size?: Needs["size"]; tickets?: boolean; files?: readonly FileNeed[] } = {}): boolean {
return ableWorkers(workflowId, o).length > 0;
}
/**
* Does a worker hold everything this workflow needs (its files and node packs, as its last list of files said), whether
* or not it is allowed the workflow's lane? Tells a model whose files no GPU here has from one whose lane is switched off.
*/
export function workersHold(workflowId: string): boolean {
return poolWorkers().some((w) => w.pool === "pool" && capableOf(w).workflows.includes(workflowId));
}
/**
* The largest job of a workflow any worker that may and can run it makes (its PC's ceiling), each limit apart: undefined
* where one of them has no limit, and for a workflow none of them runs. Said before Run with a size they don't make.
*/
export function workersMost(workflowId: string, size?: { megapixels?: number }): { maxMegapixels?: number; maxSeconds?: number; maxOutputs?: number } {
const lane = poolLane(workflowId);
// Each worker's PC ceiling, its longest clip narrowed to its lane's own limit (at this frame size) when it has one.
const ceilings = ableWorkers(workflowId).map((w): Ceiling | null => {
const c = ceilingOf(w);
const laneMost = laneMostSeconds(allowOf(w)[lane], size?.megapixels);
if (!laneMost) return c;
return { ...c, maxSeconds: c?.maxSeconds ? Math.min(c.maxSeconds, laneMost) : laneMost };
});
const most = (k: "maxMegapixels" | "maxSeconds" | "maxOutputs"): number | undefined => {
if (!ceilings.length || ceilings.some((c) => !c?.[k])) return undefined;
return Math.max(...ceilings.map((c) => c![k]!));
};
const out: { maxMegapixels?: number; maxSeconds?: number; maxOutputs?: number } = {};
for (const k of ["maxMegapixels", "maxSeconds", "maxOutputs"] as const) {
const v = most(k);
if (v !== undefined) out[k] = v;
}
return out;
}
/**
* The workers that may and can run this workflow (any workflow: null), whatever their state now (workersCanRun). `files`:
* the run-time files a run adds (src/lib/runtime-files.ts), each one the worker verified.
*/
function ableWorkers(workflowId: string | null, o: { size?: Needs["size"]; tickets?: boolean; files?: readonly FileNeed[] } = {}): WorkerRow[] {
const needs = workflowId ? needsOf(loadIndex(), workflowId) : null;
const lane = workflowId ? poolLane(workflowId) : null;
// (Runs made here go to the "pool" pool: poolFields.)
return poolWorkers().filter((w) => {
if (w.pool !== "pool") return false;
if (o.tickets && !ticketReady(w)) return false;
const allow = allowOf(w);
const cap = capableOf(w);
const ceiling = ceilingOf(w);
if (workflowId === null) return cap.workflows.some((id) => poolLane(id) in allow && insideCeiling(ceiling, { workflowId: id, lane: poolLane(id) }));
if (!needs || !(lane! in allow) || !cap.workflows.includes(workflowId) || ceilingProblem(ceiling, { workflowId, lane, size: o.size })) return false;
if (pastLaneLimit(allow, lane!, o.size)) return false;
if (o.files?.length && !hasFiles(inventoryOf(w), o.files)) return false;
if (hasVariants(loadIndex(), workflowId)) return !!workerFiles(w, workflowId, bytesOf(o.files));
return memoryFor(needs, w, bytesOf(o.files));
});
}
/**
* Is a worker on to take work soon: connected and heard from within the silence limit, paused neither on the website
* nor on its PC (a game or another program on its GPU counts as paused), not draining, new enough, and inside its
* hours? A worker that makes room for local ComfyUI work first is on: people's runs keep their place and it takes them
* next.
*/
function onSoon(w: WorkerRow, now: number): boolean {
return (
!w.revoked_at &&
!(w.expires_at && w.expires_at <= now) &&
!w.paused_server &&
!w.paused_local &&
!w.drain &&
(w.protocol ?? 0) >= MIN_PROTOCOL &&
!!w.seen_at &&
now - w.seen_at <= RELAY_TIMES.poolSilentMs &&
hoursAt(hoursOf(w), now).inside
);
}
/**
* Is a worker on now for what the site can make: one of the site's workers, heard from within the silence limit, paused
* neither on the website nor on its PC, not draining, new enough and inside its hours.
*/
function onNow(w: WorkerRow, now: number): boolean {
return w.kind === "pool" && w.pool === "pool" && onSoon(w, now);
}
/**
* Each lane's limits, one per worker counted for what the site can make now (src/lib/capacity.ts): on now, or on at a
* beat within CAPACITY_GRACE_MS, so a short drop (a restart, a power cut) changes nothing. A worker counts in the lanes
* it is allowed, inside its PC's ceiling. A lane no such worker runs is left out.
*/
export function capacityLimits(now = Date.now()): Map<string, LaneLimit[]> {
const out = new Map<string, LaneLimit[]>();
for (const w of poolWorkers()) {
if (w.pool !== "pool" || (w.expires_at && w.expires_at <= now) || (w.protocol ?? 0) < MIN_PROTOCOL) continue;
if (!onNow(w, now) && !(w.available_at && now - w.available_at < CAPACITY_GRACE_MS)) continue;
const allow = allowOf(w);
const ceiling = ceilingOf(w);
for (const lane of lanesInside(Object.keys(allow), ceiling)) {
const a = allow[lane];
const limit: LaneLimit = {
...(a?.maxSeconds && { maxSeconds: a.maxSeconds }),
...(a?.bigFrom && a.bigMaxSeconds && { bigFrom: a.bigFrom, bigMaxSeconds: a.bigMaxSeconds }),
...(ceiling?.maxSeconds && { ceilingSeconds: ceiling.maxSeconds }),
...(ceiling?.maxMegapixels && { ceilingMegapixels: ceiling.maxMegapixels }),
};
out.set(lane, [...(out.get(lane) ?? []), limit]);
}
}
return out;
}
/**
* Is every worker that could run an account's job of this workflow (any workflow: null) away now: paused, draining,
* outside its hours, silent, or never connected? Only workers that take job tickets now count (ticketReady), as an
* account's run goes as one. A run made then is taken all the same and waits in line until one is back: the price check
* says it starts when a GPU is free, and Run stays on. False when no worker could run it at all (workersCanRun refuses
* that) or one is on.
*/
export function workersAway(workflowId: string | null, now = Date.now()): boolean {
const able = ableWorkers(workflowId, { tickets: true });
return able.length > 0 && !able.some((w) => onSoon(w, now));
}
/**
* What the workers allowed a lane have (any lane: `lane` null), for shaping a graph that each of them can run: the least
* VRAM and RAM among those that said, and their device when they all name the same kind. Nothing known: empty.
*/
export function poolFacts(lane: string | null): ComputerFacts {
const facts = poolWorkers()
.filter((w) => lane === null || lane in allowOf(w))
.map((w) => json<ComputerFacts>(w.facts_json, {}));
const least = (k: "vramBytes" | "ramBytes") => {
const v = facts.map((f) => f[k]).filter((n): n is number => typeof n === "number" && n > 0);
return v.length ? Math.min(...v) : undefined;
};
const kinds = new Set(facts.map((f) => /^[a-z]+/i.exec(f.device ?? "")?.[0].toLowerCase()).filter(Boolean));
const vramBytes = least("vramBytes");
const ramBytes = least("ramBytes");
const device = kinds.size === 1 ? facts.find((f) => f.device)?.device : undefined;
return { ...(vramBytes && { vramBytes }), ...(ramBytes && { ramBytes }), ...(device && { device }) };
}
/**
* The line for a new run of this workflow, for a quote or a reserve (the billing service's QueueFacts): how many tasks the
* workers that may run it would serve before it, about how long until it would be done, and how many such workers can
* take work now. (The account's own runs, QueueFacts.mine, are counted by run in hosted.ts.) `rankAt`: the place it
* would take (a run that waited to enter the line at its press time: pool-holds.ts), now by default.
*/
export function lineFacts(accountId: string | null, asked: string | null, now = Date.now(), rankAt = now) {
// A workflow that picks its graph at run time (`…-{n}`) has no lane yet: the whole line counts.
const workflowId = asked && !asked.includes("{") ? asked : null;
const lane = workflowId ? poolLane(workflowId) : null;
const needs = workflowId ? needsOf(loadIndex(), workflowId) : null;
// A new run of a person's, in the free class (a paid run's head start is the billing service's to price, not to quote):
// a job ticket for this studio's catalog, which only a worker built for it takes (its place and wait, and the workers
// online, count those workers only).
const version = catalogVersion();
const probe = { id: "", principal: accountId ? `acct:${accountId}` : "guest", account: accountId ?? "guest", class: "free", pools: "pool", lane, needs_json: needs ? JSON.stringify({ ...needs, ...(version && { catalogVersion: version }) }) : null, rank_at: rankAt, created_at: now, stop: 0, only_worker: null, est_ms: null, slot: workflowId && cpuWorkflow(workflowId) ? "cpu" : null, finish_json: null } as unknown as TaskRow;
const p: LinePlace = lane ? placeInLine(probe, now) : { place: (db().prepare("SELECT COUNT(*) AS n FROM relay_tasks WHERE pools IS NOT NULL AND state = 'waiting'").get() as { n: number }).n + 1, etaSeconds: null };
const online = poolWorkers().filter((w) => eligible(w, now) && (!lane || fitsSlot({ ...probe, created_at: 0 }, w, now, slotFor(probe, w)))).length;
// With none of them on now: when the first is due back, when one says.
return { lane: lane ?? "small", ahead: p.place - 1, etaS: p.etaSeconds, workersOnline: online, nextOpenAt: p.returnAt ?? null };
}
/**
* What a task for workers stores: its lane, its needs, its class, its priority and its place in line. A person's run is
* "free" unless the billing service said "paid", with the priority it gave (none: 0); the owner's own jobs are the
* "owner" class, served after every person's.
*/
export function poolFields(
workflowId: string,
o: { account?: string; class?: "paid" | "free" | "owner"; priority?: number; boostMs?: number; rankAt?: number; waitMs?: number; onlyWorker?: string; size?: Needs["size"]; catalogVersion?: string; graphSha?: boolean; runtime?: FileNeed[] } = {},
now = Date.now(),
) {
const index = loadIndex();
const found = needsOf(index, workflowId);
// How big the job is (a PC's ceiling reads it), and a job ticket's catalog version (only a worker built for it takes it).
// And the files the run adds beyond its workflow's own (a look's LoRAs): only a worker that verified them takes it.
const extra = { ...(o.size && { size: o.size }), ...(o.catalogVersion && { catalogVersion: o.catalogVersion }), ...(o.catalogVersion && o.graphSha && { graphSha: true }), ...(o.runtime?.length && { runtime: o.runtime }) };
const needs = found && { ...found, ...extra };
const lane = poolLane(workflowId);
const cls = o.class ?? (o.account ? ("free" as const) : ("owner" as const));
return {
pools: ["pool"],
// Light CPU work (the requirements index marks the workflow cpu) goes to a worker's CPU slot when it has one.
...(cpuWorkflow(workflowId) && { slot: "cpu" as const }),
account: o.account,
class: cls,
// Its priority within its class (the billing service's), for people's runs only.
...(cls !== "owner" && cleanPriority(o.priority) > 0 && { priority: cleanPriority(o.priority) }),
// A head start is only for people's runs, only inside their class, and never more than a day: the owner's own jobs
// stand in line in the order they came. A later step of a run keeps the run's place (rankAt).
rankAt: o.rankAt ?? now - (cls === "owner" ? 0 : Math.max(0, Math.min(o.boostMs ?? 0, 24 * 60 * 60_000))),
lane,
needs: needs ?? { workflowId, files: [], packs: [], minVramBytes: 0, ...extra },
notAfter: o.waitMs ? now + o.waitMs : undefined,
onlyWorker: o.onlyWorker,
};
}
/** SQL: row `g` was made by the same press of Run as the row asked about (@hold: its billing hold; with none, @batch). */
const SAME_PRESS = "(CASE WHEN @hold IS NOT NULL THEN json_extract(g.request_json, '$.hold') IS @hold ELSE @batch IS NOT NULL AND g.batch_id IS @batch END)";
/**
* Where a row's job waits in line on the workers as part of its run (one press of Run, which made the row and every step
* of its chain at once):
*
* - `class`: the run's ("paid" when the billing service said so, else "free"), and `priority`: the one it gave (0: none).
* - `rankAt`: the run's place in line. The first job any of its rows had on the workers sets it (the row itself, or a
* step of the same run that fed it). A row with no such job yet keeps the time the run was made (less its head start):
* a run's first job waits in the order it came, and so does one sent again after its first send was cut off before
* its job was made (a restart or a deploy: generations.ts sendAgainOnWorkers).
* - `started`: a worker took one of those jobs. The run's wait limit no longer applies then (it applies only before the
* run's first start).
* - `placed`: one of those jobs exists (else `rankAt` is the time the run was made).
* - `later`: the row is a later step of its run (it waits for a step the same press made). It has no wait limit: its run
* was taken in line already, so it waits at the run's place for as long as it takes.
*
* A chain's later step and a finishing job keep the run's place this way, instead of going behind every run sent after
* the run started. Only steps the same press made count (the same billing hold; with none, the same batch): a result
* kept from an earlier run that a step uses gives it neither that run's place nor its start. Null: no such row.
*/
export function runPlaceOf(generationId: string): { class: "paid" | "free"; priority: number; rankAt: number | null; placed: boolean; started: boolean; later: boolean } | null {
const d = db();
const g = d
.prepare(
`SELECT created_at, depends_on, batch_id, json_extract(request_json, '$.hold') AS hold, json_extract(request_json, '$.class') AS cls,
json_extract(request_json, '$.boostMs') AS boost, json_extract(request_json, '$.priority') AS prio FROM generations WHERE id = ?`,
)
.get(generationId) as { created_at: number; depends_on: string | null; batch_id: string | null; hold: unknown; cls: unknown; boost: unknown; prio: unknown } | undefined;
if (!g) return null;
const press = { id: generationId, hold: typeof g.hold === "string" && g.hold ? g.hold : null, batch: g.batch_id };
const jobs = d
.prepare(
`WITH RECURSIVE fed(id) AS (
SELECT @id
UNION
SELECT d.value FROM fed JOIN generations g ON g.id = fed.id,
json_each(CASE WHEN substr(g.depends_on, 1, 1) = '[' AND json_valid(g.depends_on) THEN g.depends_on ELSE json_array(g.depends_on) END) d
WHERE g.depends_on IS NOT NULL AND (g.id = @id OR ${SAME_PRESS})
)
SELECT t.rank_at, t.claimed_at FROM fed JOIN generations g ON g.id = fed.id JOIN relay_tasks t ON t.generation_id = g.id
WHERE (g.id = @id OR ${SAME_PRESS}) AND t.pools IS NOT NULL AND t.rank_at IS NOT NULL ORDER BY t.rank_at ASC`,
)
.all(press) as Pick<TaskRow, "rank_at" | "claimed_at">[];
const boost = Math.max(0, Math.min(Number(g.boost) || 0, 24 * 60 * 60_000));
const later =
!!g.depends_on &&
!!d
.prepare(
`SELECT 1 FROM generations s,
json_each(CASE WHEN substr(s.depends_on, 1, 1) = '[' AND json_valid(s.depends_on) THEN s.depends_on ELSE json_array(s.depends_on) END) d
JOIN generations g ON g.id = d.value
WHERE s.id = @id AND s.depends_on IS NOT NULL AND ${SAME_PRESS} LIMIT 1`,
)
.get(press);
return {
class: g.cls === "paid" ? "paid" : "free",
priority: cleanPriority(g.prio),
rankAt: jobs[0]?.rank_at ?? g.created_at - boost,
placed: jobs.length > 0,
started: jobs.some((t) => t.claimed_at !== null),
later,
};
}
/**
* The class of the owner's job on the workers. "owner" (after people's runs) for a run a script said it made
* (request.source: a batch, a drill, an example, a test), and for every other step of the same press of Run (its batch:
* a join or a first frame a chain adds carries no source of its own). Else "free": a hands-on run, which waits, runs and
* makes room on a worker's PC as a person's run does (ownerHandsOn).
*/
export function ownerRunClass(generationId: string): "free" | "owner" {
const d = db();
const g = d.prepare("SELECT batch_id, created_at, json_extract(request_json, '$.source') AS source FROM generations WHERE id = ?").get(generationId) as
| { batch_id: string | null; created_at: number; source: unknown }
| undefined;
if (!g) return "free";
if (typeof g.source === "string" && g.source !== "hands-on") return "owner";
if (!g.batch_id) return "free";
// A press makes its rows together: the batch's rows are read within two days of this one (by the created_at index).
const tagged = d
.prepare(
`SELECT 1 FROM generations WHERE created_at BETWEEN ? AND ? AND batch_id = ?
AND COALESCE(json_extract(request_json, '$.source'), 'hands-on') <> 'hands-on' LIMIT 1`,
)
.get(g.created_at - 2 * 24 * 60 * 60_000, g.created_at + 2 * 24 * 60 * 60_000, g.batch_id);
return tagged ? "owner" : "free";
}
/**
* Is a workflow light CPU work: the requirements index marks it `cpu` (no model file, no graphics memory, no card its
* nodes need: a join, a trim, adding sound, a Ken Burns move, a filter).
*/
export function cpuWorkflow(workflowId: string): boolean {
return (loadIndex().workflows as Record<string, { cpu?: boolean } | undefined>)[workflowId]?.cpu === true;
}
/** The queue block for GET /api/generations/<id>: the task's place in its lane and the estimated wait. Null when no pool task waits. */
export function queueOfGeneration(generationId: string, now = Date.now()): LinePlace | null {
const t = db().prepare("SELECT * FROM relay_tasks WHERE generation_id = ? AND pools IS NOT NULL AND state = 'waiting' LIMIT 1").get(generationId) as TaskRow | undefined;
return t ? placeInLine(t, now) : null;
}
/**
* Is a run's job with a worker still getting ready: taken by a worker that had another model loaded (or none), with no
* step reported yet (its model loads)?
*/
export function gettingReady(generationId: string): boolean {
const t = db().prepare("SELECT progress_json FROM relay_tasks WHERE generation_id = ? AND pools IS NOT NULL AND state IN ('sending','queued','running') ORDER BY created_at DESC LIMIT 1").get(generationId) as Pick<TaskRow, "progress_json"> | undefined;
return !!t && taskCold(t);
}
/**
* FUOBE_POOL_WAIT_MIN: how long a task for workers waits before it fails. 0 when it isn't set: a run waits in line as
* long as it takes, and starts by itself.
*/
export const poolWaitMs = () => {
const m = Number(process.env.FUOBE_POOL_WAIT_MIN);
return Number.isFinite(m) && m > 0 ? m * 60_000 : 0;
};
/** The waiting words of a task for workers ("Waiting for a GPU · 2 ahead", "No GPU is on now"). */
export function poolWaitWords(t: TaskRow, now = Date.now()): string {
// The owner's own run, given back so that a person's run could start first: it starts again by itself.
if (json<{ preempted?: unknown }>(t.extra_json, {}).preempted !== undefined) return POOL_WORDS.paused;
const p = placeInLine(t, now);
if (p.etaSeconds === null) return "Waiting for a GPU to come on";
return p.place > 1 ? `Waiting for a GPU · ${p.place - 1} ahead` : "Next for a GPU";
}
// ---------------------------------------------------------------------------
// The owner's controls (Settings → Workers, /api/admin/workers)
export interface WorkerView {
id: string;
name: string;
pool: string;
state: WorkerState;
words: string;
pausedServer: boolean;
pausedLocal: boolean;
localWhy: string | null;
drain: boolean;
autoResume: boolean | null;
/**
* Its RAM guard (off by default): with other programs above its line of the PC's memory (its worker.json's numbers),
* it prefers jobs that fit whole on its card and its ComfyUI lets go of its memory before each job and when idle.
*/
ramGuard: boolean;
hours: Hours | null;
allow: Record<string, Allow>;
claimAfterMs: number;
switchAfterMs: number;
/** Its pick order: higher gets first pick of a job that a worker with a lower one could take too, while it is free. 0: none. */
pickOrder: number;
/**
* The workflows it no longer takes by streaming its weights from RAM (a job of one ran out of memory that way), newest
* first, each with when. It still takes them with every weight on its card. The owner clears them (clearNoStream).
*/
noStream: { workflowId: string; at: number }[];
warmLane: string | null;
/** The lanes it can run every workflow of. */
lanes: string[];
/**
* The lanes it can run only some workflows of: how many, of how many. Allowing one is safe (a task goes only to a
* worker that can run its own workflow), so Settings offers them with the count.
*/
partLanes: Record<string, { runnable: number; of: number }>;
/**
* Node packs whose folder it has but whose nodes its ComfyUI didn't load (it said so after its last start): the classes
* missing, and the workflows that need them, which are never sent to it.
*/
unloaded: { pack: string; classes: string[]; workflows: string[] }[];
/** What its PC may ever run (null: no ceiling of its own), and the allowed lanes inside it. */
ceiling: Ceiling | null;
lanesInside: string[];
/** Its CPU slot (light CPU work and the finishing jobs it runs), as its last beat said; null: none. */
cpuSlot: CpuSlot | null;
/** The catalog version its build composes job tickets for, and whether it is this studio's (else: update the worker). */
catalogVersion: string | null;
catalogCurrent: boolean;
facts: ComputerFacts;
bench: ReturnType<typeof benchOf>;
seenAt: number | null;
version: string | null;
lastCountry: string | null;
createdAt: number;
current: { taskId: string; generationId: string; lane: string | null; startedAt: number | null } | null;
/** Its last claim for its GPU's slot and the answer in words (null: none yet). */
asked: { at: number; words: string } | null;
/** Every task it holds now, in both slots, the oldest claim first. */
jobs: ActiveJob[];
/** Its timeline's newest row other than a log line (null: none). */
lastEvent: { at: number; kind: WorkerEventKind; event: string; words: string | null } | null;
/** The owner's mode it follows now (its own, its pool's, or people first), and its own mode as set (null: none). */
mode: EffectiveMode;
ownMode: { mode: QueueMode; until: number | null; by: string; at: number } | null;
/** Local work on its PC, as its last beat said while fresh (null: none). */
localWork: LocalWork | null;
/** Its program against the studio's version: current, updating by itself, or behind long enough for the owner to act. */
update: WorkerUpdate;
/** Its PC's live hardware stats from its last beat, with when the studio heard them (null: none). */
stats: WorkerStatsView | null;
}
/** A worker's program against the studio's version (WorkerView.update). Versions and commits as their first 12 hex. */
export interface WorkerUpdate {
/** "updating": behind for less than BEHIND_NOTICE_MS; "behind": longer (the owner may press Update now). */
state: "current" | "updating" | "behind";
since: number | null;
behindMs: number;
site: string | null;
worker: string | null;
commit: string | null;
/** People's runs it gave back because their graph is another on its version, still waiting. */
peopleHeld: number;
/** People's runs waiting now that it could take. */
peopleWaiting: number;
/** When the owner asked it to update now (null: not asked, or done). */
askedAt: number | null;
words: string;
/** Tell the owner: behind for BEHIND_NOTICE_MS or more. */
needsOwner: boolean;
}
const short = (v: string | null | undefined) => (v ? v.slice(0, 12) : null);
/** A worker's program against the studio's version, for the owner. A silent worker is offline, never behind. */
export function updateOf(w: WorkerRow, now = Date.now()): WorkerUpdate {
const site = catalogVersion();
const live = !!w.seen_at && now - w.seen_at <= RELAY_TIMES.poolSilentMs;
const behind = live && !!site && w.catalog_version !== site;
const behindMs = behind ? behindFor(w, now) : 0;
const state = !behind ? "current" : behindMs < BEHIND_NOTICE_MS ? "updating" : "behind";
const key = refusalKey(w);
const held = behind
? (db().prepare(`SELECT extra_json FROM relay_tasks WHERE pools IS NOT NULL AND state = 'waiting' AND ${CLASS_RANK} <> ${OWNER_CLASS} AND INSTR(extra_json, ?) > 0`).all(key) as Pick<TaskRow, "extra_json">[]).filter((t) => refusedBy(t).includes(key)).length
: 0;
const waiting = live && w.kind === "pool" ? claimableNow(w, now) : 0;
const words =
state === "current"
? `On the site's version${w.commit_sha ? ` (${short(w.commit_sha)})` : ""}`
: state === "updating"
? UPDATING_WORDS
: `Behind the site's version for ${Math.round(behindMs / 60_000)} min${held ? `: ${held} ${held === 1 ? "person's run waits" : "people's runs wait"} for it` : ""}`;
return {
state,
since: behind ? (w.behind_since ?? null) : null,
behindMs,
site: short(site),
worker: short(w.catalog_version),
commit: short(w.commit_sha),
peopleHeld: held,
peopleWaiting: waiting,
askedAt: w.update_asked_at ?? null,
words,
needsOwner: state === "behind",
};
}
/**
* The owner asks a worker to update now: the next beat answer carries it, and the worker looks for new code at once and
* restarts on it after the job in hand (a person's job is never cut off). Until it is on new code, or UPDATE_ASK_MS.
*/
export function askWorkerUpdate(id: string, by = "owner", now = Date.now()): WorkerView | null {
const d = db();
const w = d.prepare("SELECT * FROM relay_workers WHERE id = ? AND kind = 'pool' AND revoked_at IS NULL").get(id) as WorkerRow | undefined;
if (!w) return null;
d.prepare("UPDATE relay_workers SET update_asked_at = ?, update_asked_by = ?, update_asked_commit = ? WHERE id = ?").run(now, by.slice(0, 64), w.commit_sha ?? null, id);
noteAdmin(id, [["update-now", "Update now asked: it looks for new code and restarts after its job"]], by, now);
return workerView(d.prepare("SELECT * FROM relay_workers WHERE id = ?").get(id) as WorkerRow, now);
}
/** A task a worker holds now: where it is, and how long it should take. No prompt, no account's details. */
export interface ActiveJob {
taskId: string;
generationId: string;
slot: WorkerSlot;
lane: string | null;
laneLabel: string | null;
stage: string | null;
state: string;
claimedAt: number | null;
startedAt: number | null;
progress: { value?: number; max?: number; node?: string; nodesDone?: number; position?: number; cold?: boolean } | null;
progressAt: number | null;
/**
* When the worker's watchdog last saw it move (a step, a node, its place, a pass), on this server's clock; null: the
* worker doesn't say (an older one). Past `stallMs` without a move, the worker gives the job back to the line.
*/
movedAt: number | null;
/** How busy the worker's GPU was at its last word (0-100); null: unknown. */
gpuBusy: number | null;
/** What its ComfyUI runs now (its node's class and title, its step or pass); null: nothing said yet. */
doing: JobNow | null;
/** How long the job may go without moving before the worker gives it back (its lane's stall limit). */
stallMs: number;
/** How long a job like it takes on this worker (its lane's bench, the task's estimate, or the lane's typical run). */
typicalMs: number | null;
/**
* About how long until it is done, from typicalMs and its start: more than 0, or null (unknown, or it already ran
* longer than typicalMs: never "0 s left" for a job still running).
*/
etaS: number | null;
owners: "owner" | "account";
feature?: string;
modelName?: string;
/** Its run's operation and kind of result (t2i, i2v…; image, video…): what kind of task it is. */
op?: string;
kind?: string;
}
/** Seconds left from a typical run time and a start: at least 1, or null when there is no typical time or it has passed. */
export function etaOf(typicalMs: number | null, since: number, now: number): number | null {
if (!typicalMs) return null;
const s = Math.round((since + typicalMs - now) / 1000);
return s >= 1 ? s : null;
}
/** A lane's label from the catalog's lanes (worked out once per index). */
export const laneLabelOf = (lane: string | null | undefined): string | null => {
if (!lane) return null;
return catalogLanes().lanes.find((l) => l.lane === lane)?.label ?? (lane === "small" ? "Small jobs" : lane);
};
setEventNames({
lane: laneLabelOf,
run: (generationId) => modelLabel((db().prepare("SELECT model_id FROM generations WHERE id = ?").get(generationId) as { model_id: string } | undefined)?.model_id),
});
type ActiveRow = Pick<TaskRow, "id" | "generation_id" | "lane" | "state" | "device" | "slot" | "finish_json" | "claimed_at" | "started_at" | "progress_json" | "account" | "est_ms">;
function activeJobs(w: WorkerRow, rows: ActiveRow[], now: number, typical: (lane: string | null) => number | null): ActiveJob[] {
const gen = db().prepare("SELECT feature, model_id, op, kind FROM generations WHERE id = ?");
return rows.map((t) => {
const p = taskProgress(t);
const typicalMs = estOn(t as TaskRow, w) ?? typical(t.lane);
const g = gen.get(t.generation_id) as { feature: string | null; model_id: string | null; op: string | null; kind: string | null } | undefined;
const model = modelLabel(g?.model_id);
const progress = p ? Object.fromEntries((["value", "max", "node", "nodesDone", "position", "cold"] as const).filter((k) => p[k] !== undefined).map((k) => [k, p[k]])) : null;
return {
taskId: t.id,
generationId: t.generation_id,
slot: t.device?.endsWith("-cpu") ? "cpu" : "gpu",
lane: t.lane,
laneLabel: laneLabelOf(t.lane),
stage: stageOf(t),
state: t.state,
claimedAt: t.claimed_at,
startedAt: t.started_at,
progress,
progressAt: p?.at ?? null,
movedAt: p?.movedAt ?? null,
gpuBusy: p?.gpuBusy ?? null,
doing: t.state === "running" ? jobNowOf(p) : null,
stallMs: stallLimitMs(t.lane),
typicalMs,
etaS: etaOf(typicalMs, t.started_at ?? now, now),
owners: t.account ? "account" : "owner",
...(g?.feature && { feature: g.feature }),
...(model && { modelName: model }),
...(g?.op && { op: g.op }),
...(g?.kind && { kind: g.kind }),
};
});
}
/** The packs a worker said its ComfyUI didn't load, each with the workflows that need one of those classes. */
export function unloadedPacks(inv: Inventory): WorkerView["unloaded"] {
const index = loadIndex();
const packs = new Map(inv.nodePacks.map((p) => [p.name, p]));
return inv.nodePacks
.filter((p) => p.missingClasses?.length)
.map((p) => ({
pack: p.name,
classes: p.missingClasses!,
workflows: Object.keys(index.workflows)
.filter((id) => !index.workflows[id].isolated && unloadedNodes(index.workflows[id], packs).some((c) => c.startsWith(`${p.name}/`)))
.sort(),
}))
.sort((a, b) => a.pack.localeCompare(b.pack));
}
export function workerView(w: WorkerRow, now = Date.now(), typical: (lane: string | null) => number | null = memo(typicalPoolMs)): WorkerView {
const s = workerState(w, now);
const held = db()
.prepare("SELECT id, generation_id, lane, state, device, slot, finish_json, claimed_at, started_at, progress_json, account, est_ms FROM relay_tasks WHERE worker_id = ? AND state IN ('sending','queued','running','collecting') ORDER BY claimed_at, id")
.all(w.id) as ActiveRow[];
// `current` as before: its GPU's job (else its only job).
const cur = held.find((t) => !t.device?.endsWith("-cpu")) ?? held[0];
return {
id: w.id,
name: w.name,
pool: w.pool,
state: s.state,
words: s.words,
pausedServer: !!w.paused_server,
pausedLocal: !!w.paused_local,
localWhy: w.local_why,
drain: !!w.drain,
autoResume: w.auto_resume === null || w.auto_resume === undefined ? null : !!w.auto_resume,
ramGuard: !!w.ram_guard,
hours: hoursOf(w),
allow: allowOf(w),
claimAfterMs: w.claim_after_ms,
switchAfterMs: w.switch_after_ms,
pickOrder: w.pick_order ?? 0,
noStream: Object.entries(noStreamOf(w)).map(([workflowId, at]) => ({ workflowId, at })).sort((a, b) => b.at - a.at),
warmLane: w.warm_lane,
lanes: capableOf(w).lanes,
partLanes: partLanes(loadIndex(), capableOf(w).workflows, sha),
unloaded: unloadedPacks(inventoryOf(w)),
ceiling: ceilingOf(w),
lanesInside: lanesInside(Object.keys(allowOf(w)), ceilingOf(w)),
cpuSlot: cpuSlotOf(w),
catalogVersion: w.catalog_version ?? null,
catalogCurrent: catalogCurrent(w),
facts: factsOf(w),
bench: benchOf(w),
seenAt: w.seen_at,
version: w.version,
lastCountry: w.last_country,
createdAt: w.created_at,
current: cur ? { taskId: cur.id, generationId: cur.generation_id, lane: cur.lane, startedAt: cur.started_at } : null,
asked: w.asked_at && w.asked_words ? { at: w.asked_at, words: w.asked_words } : null,
jobs: activeJobs(w, held, now, typical),
lastEvent: lastEventOf(w.id),
mode: effectiveMode(w, now),
ownMode: workerModeOf(w),
localWork: localWorkOf(w, now),
update: updateOf(w, now),
stats: workerStatsOf(w),
};
}
export function listWorkers(now = Date.now()): WorkerView[] {
// A worker that went silent shows as offline in its timeline too, even when nothing else swept the line since.
noteSilentWorkers(now);
const typical = memo(typicalPoolMs);
return (db().prepare("SELECT * FROM relay_workers WHERE kind = 'pool' AND revoked_at IS NULL ORDER BY created_at").all() as WorkerRow[]).map((w) => workerView(w, now, typical));
}
/** Add a worker: paused on the website, with no hours and no lanes. Its token is shown once. */
export function addWorker(name: string, pool = "pool"): { id: string; token: string } {
if (!/^[a-z0-9-]{1,32}$/.test(pool)) throw new Error("A pool's name is lower-case letters, digits and hyphens");
const clean = name.trim();
if (!clean) throw new Error("Name the worker");
if (db().prepare("SELECT 1 FROM relay_workers WHERE kind = 'pool' AND revoked_at IS NULL AND name = ?").get(clean)) throw new Error("A worker with that name exists");
return createWorker({ name: clean, kind: "pool", pool });
}
/** The pick order a worker may have (WorkerPatch.pickOrder). */
export const PICK_ORDER_MIN = -100;
export const PICK_ORDER_MAX = 100;
export interface WorkerPatch {
paused?: boolean;
pauseNow?: boolean;
drain?: boolean;
/** true or false sets the automatic resume for this worker; null leaves it to the worker's worker.json. */
autoResume?: boolean | null;
/** Its RAM guard on or off (WorkerView.ramGuard). */
ramGuard?: boolean;
hours?: unknown;
allow?: unknown;
claimAfterMs?: number;
switchAfterMs?: number;
/** A whole number from PICK_ORDER_MIN to PICK_ORDER_MAX: higher picks first (firstPicks). 0: none. */
pickOrder?: number;
/** Clear its no-streaming list (WorkerView.noStream): `true` for all of it, or a list of workflow ids. */
clearNoStream?: true | string[];
rotate?: boolean;
revoke?: boolean;
}
/** A worker's token stops at once, and its tasks go back in line at once (not the worker's fault). */
function revokePoolWorker(id: string, now: number): void {
revokeWorker(id, now);
for (const t of db().prepare("SELECT * FROM relay_tasks WHERE worker_id = ? AND state IN ('sending','queued','running','collecting')").all(id) as TaskRow[]) backInLine(t, { blame: false, sentBack: "its token was revoked" });
}
/**
* Every worker's token stops at once (Settings → Security → Sign everyone out, with every API token), and their tasks go
* back in line with their places kept. People's own paired ComfyUIs keep theirs. Returns how many workers it revoked.
*/
export function revokeAllWorkers(now = Date.now()): number {
const d = db();
return d.transaction(() => {
const ids = (d.prepare("SELECT id FROM relay_workers WHERE kind = 'pool' AND revoked_at IS NULL").all() as { id: string }[]).map((r) => r.id);
for (const id of ids) {
revokePoolWorker(id, now);
noteAdmin(id, [["revoke", "Token revoked: everyone was signed out"]], "owner", now);
}
return ids.length;
})();
}
/** The owner changes a worker. Each change reaches it at its next beat. */
export function patchWorker(id: string, p: WorkerPatch, now = Date.now(), by = "owner"): { view: WorkerView | null; token?: string } {
const d = db();
const w = d.prepare("SELECT * FROM relay_workers WHERE id = ? AND kind = 'pool' AND revoked_at IS NULL").get(id) as WorkerRow | undefined;
if (!w) return { view: null };
if (p.revoke) {
revokePoolWorker(id, now);
noteAdmin(id, [["revoke", "Token revoked"]], by, now);
return { view: null };
}
const set: Record<string, unknown> = {};
if (p.paused !== undefined) {
set.paused_server = p.paused ? 1 : 0;
if (!p.paused) set.drain = 0;
}
if (p.pauseNow) Object.assign(set, { paused_server: 1, stop_now: 1 });
if (p.drain !== undefined) set.drain = p.drain ? 1 : 0;
if (p.autoResume !== undefined) {
if (p.autoResume !== null && typeof p.autoResume !== "boolean") throw new Error("The automatic resume is on, off or the PC's own setting");
set.auto_resume = p.autoResume === null ? null : p.autoResume ? 1 : 0;
}
if (p.ramGuard !== undefined) {
if (typeof p.ramGuard !== "boolean") throw new Error("The RAM guard is on or off");
set.ram_guard = p.ramGuard ? 1 : 0;
}
if (p.hours !== undefined) {
if (p.hours !== null && !parseHours(p.hours)) throw new Error("Hours need a time zone and windows like 01:00–09:00");
set.hours_json = p.hours === null ? null : JSON.stringify(parseHours(p.hours));
}
if (p.allow !== undefined) {
if (!p.allow || typeof p.allow !== "object") throw new Error("Allowed lanes are a list of lanes, each with a delay");
const allow: Record<string, Allow> = {};
for (const [lane, v] of Object.entries(p.allow as Record<string, unknown>)) {
if (!/^[0-9a-f]{12}$|^small$/.test(lane)) throw new Error(`No such lane: ${lane}`);
const delay = Number((v as { delayMs?: unknown })?.delayMs ?? 0);
const most = Number((v as { maxSeconds?: unknown })?.maxSeconds ?? 0);
const bigFrom = Number((v as { bigFrom?: unknown })?.bigFrom ?? 0);
const bigMost = Number((v as { bigMaxSeconds?: unknown })?.bigMaxSeconds ?? 0);
const pos = (n: number) => Number.isFinite(n) && n > 0;
allow[lane] = {
delayMs: Number.isFinite(delay) && delay >= 0 ? Math.min(delay, 24 * 3_600_000) : 0,
...(pos(most) && { maxSeconds: Math.min(most, 86_400) }),
...(pos(bigFrom) && pos(bigMost) && { bigFrom: Math.min(bigFrom, 10_000), bigMaxSeconds: Math.min(bigMost, 86_400) }),
};
}
set.allow_json = JSON.stringify(allow);
}
if (p.claimAfterMs !== undefined) set.claim_after_ms = Math.max(0, Math.min(24 * 3_600_000, Math.round(Number(p.claimAfterMs) || 0)));
if (p.switchAfterMs !== undefined) set.switch_after_ms = Math.max(0, Math.min(24 * 3_600_000, Math.round(Number(p.switchAfterMs) || 0)));
if (p.pickOrder !== undefined) {
if (!Number.isInteger(p.pickOrder) || p.pickOrder < PICK_ORDER_MIN || p.pickOrder > PICK_ORDER_MAX) throw new Error(`The pick order is a whole number from ${PICK_ORDER_MIN} to ${PICK_ORDER_MAX}`);
set.pick_order = p.pickOrder;
}
if (p.clearNoStream !== undefined) {
const ids = p.clearNoStream;
if (ids !== true && !(Array.isArray(ids) && ids.every((x) => typeof x === "string"))) throw new Error("Say which workflows to clear, or all of them");
const left = ids === true ? {} : Object.fromEntries(Object.entries(noStreamOf(w)).filter(([k]) => !ids.includes(k)));
set.no_stream_json = Object.keys(left).length ? JSON.stringify(left) : null;
}
const keys = Object.keys(set);
if (keys.length) d.prepare(`UPDATE relay_workers SET ${keys.map((k) => `${k} = @${k}`).join(", ")} WHERE id = @id`).run({ ...set, id });
const token = p.rotate ? rotateWorker(id) ?? undefined : undefined;
const fresh = d.prepare("SELECT * FROM relay_workers WHERE id = ?").get(id) as WorkerRow;
// The timeline: each action (an edit names its keys, never a value), then the state it leads to when it changed.
const actions: [string, string, Record<string, unknown>?][] = [];
if (p.pauseNow) actions.push(["pause-now", "Paused now on the website: its jobs go back in line"]);
else if (p.paused === true) actions.push(["pause", "Paused on the website"]);
if (p.paused === false) actions.push(["resume", "Resumed on the website"]);
if (p.drain === true) actions.push(["drain", "Set to finish its job, then pause"]);
else if (p.drain === false) actions.push(["undrain", "No longer pausing after its job"]);
const edited = (["autoResume", "ramGuard", "hours", "allow", "claimAfterMs", "switchAfterMs", "pickOrder"] as const).filter((k) => p[k] !== undefined);
if (edited.length) actions.push(["edit", `Settings changed: ${edited.join(", ")}`, { keys: edited }]);
if (p.clearNoStream !== undefined) {
const cleared = p.clearNoStream === true ? Object.keys(noStreamOf(w)) : p.clearNoStream.filter((k) => noStreamOf(w)[k] !== undefined);
actions.push(["clear-no-stream", cleared.length ? `May stream from RAM again: ${cleared.join(", ")}` : "Its list of workflows not to stream was already clear", { workflows: cleared }]);
}
if (token) actions.push(["rotate", "New token made"]);
noteAdmin(id, actions, by, now);
try {
const s = workerState(fresh, now);
if (s.state !== fresh.state && fresh.seen_at) {
d.prepare("UPDATE relay_workers SET state = ? WHERE id = ?").run(s.state, id);
logWorkerEvent({ kind: "state", event: s.state, workerId: id, words: s.words, state: s.state, detail: { from: fresh.state, by } }, now);
}
} catch {}
return { view: workerView(fresh, now), ...(token && { token }) };
}
/** Admin actions in a worker's timeline, with who did them ("owner" or the admin token's name). Best-effort. */
function noteAdmin(id: string, actions: [string, string, Record<string, unknown>?][], by: string, now: number): void {
logWorkerEvents(
actions.map(([event, words, detail]) => ({ kind: "admin", event, workerId: id, words, detail: { by, ...detail } })),
now,
);
}
/**
* The line for workers by lane: waiting, oldest, the estimate, and which workers have it warm. `held`: the runs made but
* waiting to enter the line (pool-holds.ts), by why they wait, and when the oldest was pressed: counts only (no prompt,
* no account).
*/
export function queueView(now = Date.now()) {
const index = loadIndex();
const rows = db().prepare("SELECT lane, COUNT(*) AS n, MIN(created_at) AS oldest FROM relay_tasks WHERE pools IS NOT NULL AND state = 'waiting' GROUP BY lane").all() as { lane: string | null; n: number; oldest: number }[];
const workers = db().prepare("SELECT * FROM relay_workers WHERE kind = 'pool' AND revoked_at IS NULL").all() as WorkerRow[];
return {
held: heldView(),
lanes: rows.map((r) => {
const last = db().prepare("SELECT * FROM relay_tasks WHERE pools IS NOT NULL AND state = 'waiting' AND lane IS ? ORDER BY rank_at DESC LIMIT 1").get(r.lane) as TaskRow | undefined;
return {
lane: r.lane ?? "",
label: r.lane ? laneLabel(index, r.lane, sha) : "",
waiting: r.n,
oldestAt: r.oldest,
etaS: last ? placeInLine(last, now).etaSeconds : null,
warmOn: workers.filter((w) => w.warm_lane === r.lane && eligible(w, now)).map((w) => w.name),
};
}),
};
}
/** Every lane in this studio's catalog, for the owner to allow per worker. */
export const lanesCatalog = () => catalogLanes().lanes.map((l) => ({ lane: l.lane, label: l.label, workflows: l.workflows.length, apps: l.apps, model: l.model, file: l.file, workflowIds: l.workflows }));
/**
* The waiting line of a pool as the owner's Queue page shows it (GET /api/admin/queue): in the order its workers serve
* it under the owner's controls (the pool's mode, runs and people sent first), each with its place (`rank`, 1 first),
* what the controls say of it, and why the line puts it there (`order`: smartLine). The tasks no worker takes now (held,
* the owner's own in People only) follow, with no place. At most `limit` tasks are looked at.
*/
export function controlledLine(
pool = "pool",
now = Date.now(),
limit = LOOK_AHEAD,
): { mode: QueueMode; items: { task: TaskRow; rank: number | null; base: number; control: TaskControl; order: OrderWhy }[] } {
const c = controlsOf(pool, now);
const rows = lineRows("*", "pools IS NOT NULL AND state = 'waiting' AND stop = 0 AND INSTR(',' || pools || ',', ?) > 0", [`,${pool},`], c, limit);
const all = smartLine(rows, c.mode, { now, worker: null, workers: poolWorkers().filter((w) => w.pool === pool) }).map(({ task, why }, i) => ({
task,
i,
base: classRank(task),
control: taskControl(task, classRank(task), c, null),
order: why,
}));
const served = all.filter((x) => !x.control.out).sort((a, b) => a.control.tier - b.control.tier || a.i - b.i);
const out = all.filter((x) => x.control.out);
const item = (x: (typeof all)[number], rank: number | null) => ({ task: x.task, rank, base: x.base, control: x.control, order: x.order });
return { mode: c.mode, items: [...served.map((x, k) => item(x, k + 1)), ...out.map((x) => item(x, null))] };
}