Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
/**
* Running one firing of a routine with nobody's browser open.
*
* A turn normally has a person in front of it: their browser runs the agent, sees the reply, and
* tells the server what was said, which is what moves the channel to the top of their roster and
* lights the unread dot. A routine has none of that, so this file does that last step itself —
* `recordActivity` — and it is the only place a headless turn does, which is why there is exactly
* one of those calls per successful firing here and why the tests count them.
*
* TWO LEDGERS, AND THEY ARE NOT THE SAME LEDGER.
*
* The queue's attempt count (`server/src/work/queue.ts`) bounds retries of ONE FIRING: a consumer
* that died mid-turn, a runner that is flapping, a lease that stopped being renewed. It answers
* "how many times have we handed this one due moment out?".
*
* The fatigue rule below counts CONSECUTIVE FIRINGS THAT FAILED, in `routine_runs`, across days.
* It answers "is this routine still worth firing at all?" — a Notion token that expired in March
* fails every night, and no number of retries of any one night's firing will fix it.
*
* Conflating them is how a broken routine either spams a channel — a per-attempt notification
* posting three times for one bad night — or retries for ever, because a rule counting attempts
* within a firing never sees the routine that fails cleanly, once, every single night.
*/
import { HeadlessToolSuspension } from "@openbot/toolkit/headless-tools";
import type { TurnRunner } from "@openbot/toolkit/run-turn";
import { BOT_PAUSED_REASON, isBotPaused } from "../agents/lifecycle";
import type { AgentActor } from "../agents/profile-types";
import type { ChannelStore } from "../channels/routes";
import type { RoutineStore } from "./store";
export type RoutineRunner = { run(routineRunId: string): Promise<void> };
/**
* How much of a failure's reason the channel notification carries.
*
* `recordActivity` caps its preview at 200 code points on its own, so this is not about safety, it
* is about the sentence surviving: the prefix plus this much reason still fits, so the roster shows
* a failure that says what failed rather than a truncated "This routine failed: Error: could not…".
* The whole message is on the run row for anybody who wants all of it.
*/
const MAX_NOTIFIED_REASON = 160;
/** Consecutive failed firings after which a routine stops being fired at all. */
const FATIGUE_LIMIT = 10;
const SWITCHED_OFF =
"This routine has failed ten times in a row, so I have switched it off. Ask me to turn it back on when whatever it needs is working.";
/** Measured in code points, like every other cap in this area, so nothing is cut mid-pair. */
function shorten(reason: string): string {
const codePoints = Array.from(reason);
if (codePoints.length <= MAX_NOTIFIED_REASON) return reason;
return `${codePoints.slice(0, MAX_NOTIFIED_REASON - 1).join("")}…`;
}
function reasonOf(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
export function createRoutineRunner(options: {
routineStore: RoutineStore;
channelStore: ChannelStore;
runTurn: TurnRunner;
}): RoutineRunner {
const { routineStore, channelStore, runTurn } = options;
async function runOnce(routineRunId: string): Promise<void> {
const context = await routineStore.runContext(routineRunId);
/*
* A deleted routine's queued run is nobody's problem. The routine is gone, its runs cascaded
* with it, and there is no row left to finish — so there is nothing to record and nothing to
* say to anybody. The firing is dropped on the floor deliberately.
*/
if (!context) return;
/*
* PAUSED NEVER RUNS, checked here at the moment of running rather than only when the firing was
* offered: a person who switched the routine off after the dispatch, or a Run now that raced a
* pause, has said what they want. Skipped, so the fatigue rule does not count it.
*/
if (context.enabled === false) {
await routineStore.finishRun(
routineRunId,
"skipped",
"the routine is paused",
);
return;
}
const { routineId, ownerUserId, agentId, channelId, instruction } = context;
// Everything below is done AS the owner: their channel, their thread, their grants.
const owner: AgentActor = { id: ownerUserId, role: "user" };
/** One activity record, from the routine's Bot, or a logged miss. Never a throw. */
async function say(text: string): Promise<void> {
try {
await channelStore.recordActivity(owner, channelId, {
text,
agentId,
at: new Date(),
});
} catch (error) {
/*
* A channel that cannot be written to must not turn a recorded outcome into an unrecorded
* one. The run row is the ledger; this call is the courtesy of saying so where a person
* will see it, and losing the courtesy is not worth losing the ledger.
*/
console.error(
JSON.stringify({
type: "routine-activity-unrecorded",
routineId,
routineRunId,
reason: reasonOf(error),
}),
);
}
}
/*
* `get` as the owner already filters soft-deleted channels and non-members, so null covers all
* three ways this can be over: the channel was deleted, the row is gone, or the owner is no
* longer in it. No second check, and no turn — a reply with nowhere to land is model spend for
* nothing.
*
* This is why `routines.channel_id` is not a foreign key: the routine survives its channel so
* the routines page can show it as broken, and the person can point it somewhere else.
*/
const channel = await channelStore.get(owner, channelId);
if (!channel) {
await routineStore.finishRun(
routineRunId,
"skipped",
"the channel is gone",
);
return;
}
// A paused Bot's firing is skipped, not failed, so a long pause never trips the fatigue rule.
if (await isBotPaused(ownerUserId, agentId)) {
await routineStore.finishRun(routineRunId, "skipped", BOT_PAUSED_REASON);
return;
}
/*
* The conversation shown as working while the firing runs, and told it is over once the run
* row has its outcome. The run row is what the conversation reads "a routine is running" from
* (see channels/working.ts), so the signal that ends it comes after `finishRun`, not after the
* turn. Best-effort both ways: a firing is not worth failing over a dot.
*/
await channelStore.signalBusy(channel.threadId, true).catch(() => {});
try {
await fire(channel.threadId);
} finally {
await channelStore.signalBusy(channel.threadId, false).catch(() => {});
}
async function fire(threadId: string): Promise<void> {
let replyText: string;
try {
({ replyText } = await runTurn({
ownerUserId,
routineId,
agentId,
threadId,
instruction,
}));
} catch (error) {
if (error instanceof HeadlessToolSuspension) {
await routineStore.finishRun(
routineRunId,
"waiting",
error.message,
error.waiting,
);
await say(error.message);
return;
}
const reason = reasonOf(error);
await routineStore.finishRun(routineRunId, "failed", reason);
/*
* THE FATIGUE RULE, read after the failure is recorded — the count has to include this
* firing, or the tenth failure in a row reads as the ninth and the routine keeps going.
*
* Its own try/catch, around the read and the switching-off as well as the message: this all
* happens after `finishRun`, and nothing in it is allowed to throw its way out of a firing
* whose failure is already on the row.
*/
try {
const failures = await routineStore.consecutiveFailures(routineId);
if (failures === 1) {
// Only the first failure after a success. A routine that fails every night at three has
// one line in the channel, not one line a night for a month.
await say(`This routine failed: ${shorten(reason)}`);
} else if (failures >= FATIGUE_LIMIT) {
// A dead integration should not burn model spend for ever. Switched off, and said out
// loud, because a routine that goes quiet without explaining itself is worse than one
// that fails.
await routineStore.setEnabled(ownerUserId, routineId, false);
await say(SWITCHED_OFF);
}
// In between, nothing is said. The run rows carry it, and the routines page reads them.
} catch (fatigueError) {
console.error(
JSON.stringify({
type: "routine-fatigue-rule-failed",
routineId,
routineRunId,
reason: reasonOf(fatigueError),
}),
);
}
return;
}
/*
* The reply, then the outcome. `say` cannot throw, and `finishRun` is finish-once, so neither
* order can lose the run row — this one is the browser's order: what was said lands in the
* channel, and then the firing is closed.
*/
await say(replyText);
await routineStore.finishRun(routineRunId, "succeeded");
}
}
return {
async run(routineRunId) {
/*
* `run` never throws. Every outcome a routine can have — a gone channel, a refused model, a
* routine deleted underneath the firing — is recorded on the run row above rather than raised
* at whoever is draining the queue. This backstop is for the ones that are not outcomes at
* all, a store that cannot be reached being the honest example: said out loud, because the
* alternative is a firing that vanishes without a line anywhere.
*/
try {
await runOnce(routineRunId);
} catch (error) {
console.error(
JSON.stringify({
type: "routine-run-crashed",
routineRunId,
reason: reasonOf(error),
}),
);
}
},
};
}