Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
/**
* Suspending computers nobody is using.
*
* A `Sandbox` has `shutdownTime`, an absolute expiry, which is not the question anybody is asking:
* "nobody has touched this for thirty minutes" is. So this asks that one, and it has to survive the
* replica that started it, which is why the work is claimed and leased out of PostgreSQL rather than
* held in a timer.
*
* NOTHING HERE TOUCHES A BROWSER. Deciding whether a computer is idle by dialling it would wake the
* computer, so every idle Bot anything asked about would come back up and the bill would never fall.
* That is the known, invisible way to lose scale-to-zero: everything works, nothing ever suspends.
* Idleness is read from the audit trail, which is a record of what a Bot did rather than a question
* put to the thing that did it.
*/
import { and, inArray, like, sql } from "drizzle-orm";
import {
type ComputerKey,
computerKeyText,
parseComputerKey,
} from "../computer/computer-key";
import type { ComputerProvider } from "../computer/provider";
import type { Database } from "../db/client";
import { auditEvents } from "../db/schema";
import { DEFAULT_MAX_ATTEMPTS, type WorkQueue } from "./queue";
export const CULL_KIND = "computer.suspend";
export type CullerOptions = {
database: Database;
queue: WorkQueue;
provider: ComputerProvider;
/** A computer untouched for this long is idle. */
idleAfterMs: number;
/** Who this replica is, for the lease. */
owner: string;
leaseMs?: number;
/** How many goes an item gets before it stops being offered. */
maxAttempts?: number;
now?: () => Date;
};
export type CullReport = {
considered: number;
/** The computers suspended, as `person:<id>` or `bot:<id>`. */
suspended: string[];
skipped: { computer: string; reason: string }[];
};
/**
* When each computer was last asked to do anything, keyed by `person:<id>` or `bot:<id>`.
*
* The audit trail is the source: every acting call writes a row before the computer is touched, so
* "last used" is already recorded, server-side, and survives a restart. A computer that has never
* acted has no row and reads as idle since it started, which is the answer wanted.
*
* BY COMPUTER, NOT BY BOT. A person's computer carries every Bot they use, so it is in use while any
* of them is: grouping by Bot would put it to sleep under one Bot because another had gone quiet.
* Each row names its computer; a row from before computers were per person names only its Bot, and
* that Bot had a computer of its own, which is what `bot:<id>` means.
*
* Exported for the inactive-computer stop in admin/controls.ts, which asks the same question.
*/
export async function lastUsedByComputer(
database: Database,
computers: string[],
): Promise<Map<string, Date>> {
if (computers.length === 0) return new Map();
/*
* The computer is in the payload rather than in a column, so the grouping key is an expression.
* Written through the query builder rather than as one raw string, because a list parameter has
* to be bound as a list: handed to `= any($1)` as a JavaScript array it arrives as one opaque value
* and the query fails rather than matching nothing, which at least says so.
*/
const computer = sql<string>`coalesce(${auditEvents.payload}->>'computer', 'bot:' || (${auditEvents.payload}->>'bot'))`;
const rows = await database
.select({ computer, last: sql<string>`max(${auditEvents.createdAt})` })
.from(auditEvents)
.where(
and(
like(auditEvents.eventType, "computer.%"),
inArray(computer, computers),
),
)
.groupBy(computer);
return new Map(
rows
.filter((row) => row.computer)
.map((row) => [row.computer, new Date(row.last)] as const),
);
}
/**
* The running computers, one entry each.
*
* The shared computer lists a person's seats on it rather than computers, so a person with three
* Bots there would otherwise be three entries for the one thing that is stopped.
*/
export function runningComputers(
computers: { key: ComputerKey; status: string; startedAt?: string }[],
): { key: ComputerKey; computer: string; startedAt?: string }[] {
const seen = new Map<
string,
{ key: ComputerKey; computer: string; startedAt?: string }
>();
for (const entry of computers) {
if (entry.status !== "running") continue;
const computer = computerKeyText(entry.key);
const held = seen.get(computer);
// The earliest start stands for the computer: it has been up at least that long.
if (
!held ||
(entry.startedAt && (!held.startedAt || entry.startedAt < held.startedAt))
) {
seen.set(computer, {
key: entry.key,
computer,
...(entry.startedAt ? { startedAt: entry.startedAt } : {}),
});
}
}
return [...seen.values()];
}
/**
* Offer every idle computer for suspension.
*
* Offering rather than suspending: the decision and the act are separated so that whichever replica
* runs this does not have to be the one that carries it out, and so a suspension that fails halfway
* is retried by whoever picks the item up next rather than lost with the process that noticed.
*/
export async function offerIdleComputers(
options: CullerOptions,
): Promise<{ offered: string[] }> {
const now = options.now?.() ?? new Date();
const running = runningComputers(await options.provider.list());
const used = await lastUsedByComputer(
options.database,
running.map((computer) => computer.computer),
);
const offered: string[] = [];
for (const computer of running) {
const since =
used.get(computer.computer) ??
(computer.startedAt ? new Date(computer.startedAt) : undefined);
// No row and no start time means nothing is known about it, and suspending on no evidence is
// how somebody's session disappears mid-task. Left alone.
if (!since) continue;
if (now.getTime() - since.getTime() < options.idleAfterMs) continue;
await options.queue.offer({
kind: CULL_KIND,
key: computer.computer,
payload: { computer: computer.computer, idleSince: since.toISOString() },
});
offered.push(computer.computer);
}
return { offered };
}
/**
* Carry out whatever suspensions this replica can claim.
*
* Re-checked at the moment of acting, because the decision was made by another replica at another
* time and a person may have started working in between. Suspending a computer somebody is using is
* worse than leaving an idle one running for another five minutes.
*/
export async function suspendClaimedComputers(
options: CullerOptions,
): Promise<CullReport> {
const leaseMs = options.leaseMs ?? 60_000;
const claimed = await options.queue.claim({
kind: CULL_KIND,
owner: options.owner,
leaseMs,
limit: 20,
...(options.maxAttempts === undefined
? {}
: { maxAttempts: options.maxAttempts }),
});
const report: CullReport = {
considered: claimed.length,
suspended: [],
skipped: [],
};
for (const item of claimed) {
/*
* Whose computer. An item offered before computers were per person carries only its Bot, which
* then had a computer of its own: `bot:<id>`.
*/
const computer =
typeof item.payload.computer === "string"
? item.payload.computer
: typeof item.payload.botId === "string"
? `bot:${item.payload.botId}`
: item.key;
const key = parseComputerKey(computer);
/*
* Renewed before each one, because the batch is twenty and the lease is one.
*
* Twenty suspensions is twenty calls to an API server, and nothing renewed while they ran: on a
* slow cluster the lease expired part-way down the list and another replica claimed the tail
* this one was still working through. A lease nobody renews is a timer, and a timer is what this
* queue exists not to be.
*
* False means it is already somebody else's. Stopping is then the correct answer, not an error:
* the item is being handled, just not here.
*/
if (
!(await options.queue.renew({
kind: CULL_KIND,
key: item.key,
owner: options.owner,
leaseMs,
}))
) {
report.skipped.push({
computer,
reason: "the lease went to another replica",
});
continue;
}
try {
if (!key) throw new Error(`${computer} does not name a computer`);
const now = options.now?.() ?? new Date();
const used = await lastUsedByComputer(options.database, [computer]);
const since = used.get(computer);
if (since && now.getTime() - since.getTime() < options.idleAfterMs) {
// Somebody came back. Drop the item rather than releasing it, because the next sweep will
// offer it again if it goes quiet, and a released one would just be reclaimed and re-checked.
await options.queue.finish({
kind: CULL_KIND,
key: item.key,
owner: options.owner,
});
report.skipped.push({
computer,
reason: "used again before it was suspended",
});
continue;
}
await options.provider.stop({ key });
await options.queue.finish({
kind: CULL_KIND,
key: item.key,
owner: options.owner,
});
report.suspended.push(computer);
} catch (error) {
/*
* Released rather than dropped, and pushed out rather than retried immediately: a cluster that
* refused this once will probably refuse it again in the next second, and a computer left
* running costs money rather than losing anything.
*/
const reason =
error instanceof Error ? error.message : "could not be suspended";
await options.queue.release({
kind: CULL_KIND,
key: item.key,
owner: options.owner,
delayMs: 5 * 60_000,
reason,
});
/*
* Said out loud when it gives up, because otherwise it stops silently.
*
* At the cap the item is no longer claimed, so this loop simply never sees that computer again and
* every sweep looks clean while one computer stays awake indefinitely. The row carries the
* count and the reason for anybody who queries it; this is for the person reading the logs.
*/
if (item.attempts >= (options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS)) {
console.warn(
JSON.stringify({
type: "computer-cull-gave-up",
computer,
attempts: item.attempts,
reason,
}),
);
}
report.skipped.push({ computer, reason });
}
}
return report;
}