Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

14.3 KB
Raw Blame
/**
 * A Bot's lifecycle as one person runs it: pause and resume, reset, how loudly it may reach them,
 * and where each kind of update goes.
 *
 * PAUSE IS ENFORCED WHERE WORK STARTS, not where it is requested. Every place that starts a Bot's
 * turn without a person typing — a routine firing, a responsibility's event, a hop from another Bot,
 * a follow-up the Bot scheduled for itself, every headless turn — asks {@link isBotPaused} (or
 * {@link guardBotTurn}, which also makes the turn stoppable) immediately before it starts. A queue
 * that was filled before the pause therefore drains into skips rather than into turns.
 *
 * FAILS CLOSED. When the deployment has wired this module and the question cannot be answered, the
 * answer is "paused": a Bot that might have been stopped must not be started on a guess.
 *
 * A process that never wired it (a unit test of some other module) answers "not paused", because
 * there is nobody who could have paused anything.
 */
import { and, eq, inArray, isNotNull, sql } from "drizzle-orm";
import type { Database } from "../db/client";
import { botLifecycle, updateRoutingPreferences } from "../db/schema/lifecycle";

/** The sentence every skipped dispatch records, so the trail and the run rows say the same thing. */
export const BOT_PAUSED_REASON = "The Bot is paused, so it did not run.";

export class BotPausedError extends Error {
  override name = "BotPausedError";
  constructor() {
    super(BOT_PAUSED_REASON);
  }
}

export type BotNotify = "all" | "needs_input" | "none";
export type UpdateKind = "progress" | "decision" | "question";
export type UpdateTransport = "slack" | "teams" | "sms" | "push";
export const UPDATE_KINDS: readonly UpdateKind[] = [
  "progress",
  "decision",
  "question",
];
export const UPDATE_TRANSPORTS: readonly UpdateTransport[] = [
  "slack",
  "teams",
  "sms",
  "push",
];

type RunningTurn = { controller: AbortController; startedAt: number };

/**
 * How long a turn registered by {@link guardBotTurn} is kept after it started.
 *
 * The turn runner never reports that it finished (the guard is one line at its top), so entries are
 * dropped by age: twice the longest a headless turn may run. Aborting one that already finished is
 * harmless, so erring long costs nothing but a map entry.
 */
const TURN_RETENTION_MS = 10 * 60_000;
/** How often running turns are checked against pauses made on other replicas. */
const CROSS_REPLICA_CHECK_MS = 5_000;
/**
 * How many checks in a row may fail before the watcher stops a pair's turns. A turn never STARTS
 * on an unreadable pause, but one slow query must not stop every turn on the replica, for every
 * person and Bot. Three checks is fifteen seconds of a database that stays unreadable.
 */
const UNREADABLE_CHECKS_BEFORE_STOP = 3;

const state: {
  database?: Database;
  running: Map<string, Set<RunningTurn>>;
  /** Consecutive failed pause reads per pair, reset by the next read that answers. */
  unreadable: Map<string, number>;
  watcher?: ReturnType<typeof setInterval>;
} = { running: new Map(), unreadable: new Map() };

const keyOf = (ownerUserId: string, agentId: string) =>
  JSON.stringify([ownerUserId, agentId]);

/** Wire the pause switch to the deployment's database. Called once at start-up. */
export function configureBotLifecycle(options: { database: Database }) {
  state.database = options.database;
}

/** For tests: run the cross-replica check now instead of waiting for its interval. */
export const checkRunningTurnsForTests = () => checkRunningTurns();

/** For tests: forget the wiring and every tracked turn. */
export function resetBotLifecycleForTests() {
  state.database = undefined;
  state.running.clear();
  state.unreadable.clear();
  if (state.watcher) clearInterval(state.watcher);
  state.watcher = undefined;
}

/** Whether this person has paused this Bot, or `undefined` when the read failed (and is logged). */
async function readPause(
  database: Database,
  ownerUserId: string,
  agentId: string,
): Promise<boolean | undefined> {
  try {
    const [row] = await database
      .select({ pausedAt: botLifecycle.pausedAt })
      .from(botLifecycle)
      .where(
        and(
          eq(botLifecycle.ownerUserId, ownerUserId),
          eq(botLifecycle.agentId, agentId),
        ),
      )
      .limit(1);
    return Boolean(row?.pausedAt);
  } catch (error) {
    console.error(
      JSON.stringify({
        type: "bot-pause-check-failed",
        agentId,
        error: error instanceof Error ? error.message : String(error),
      }),
    );
    return undefined;
  }
}

/**
 * Whether this person has paused this Bot.
 *
 * Fails closed once wired: a read that throws answers `true` and says why in the log.
 */
export async function isBotPaused(
  ownerUserId: string,
  agentId: string,
): Promise<boolean> {
  const database = state.database;
  if (!database) return false;
  return (await readPause(database, ownerUserId, agentId)) ?? true;
}

/**
 * The guard at the top of a headless turn: refuse if paused, otherwise hand back a signal that
 * aborts when this person pauses the Bot, on this replica at once and on any other within seconds.
 */
export async function guardBotTurn(input: {
  ownerUserId: string;
  agentId: string;
  signal?: AbortSignal;
}): Promise<AbortSignal> {
  if (await isBotPaused(input.ownerUserId, input.agentId))
    throw new BotPausedError();
  const controller = new AbortController();
  const key = keyOf(input.ownerUserId, input.agentId);
  const now = Date.now();
  const turns = state.running.get(key) ?? new Set<RunningTurn>();
  for (const turn of turns)
    if (now - turn.startedAt > TURN_RETENTION_MS) turns.delete(turn);
  turns.add({ controller, startedAt: now });
  state.running.set(key, turns);
  startWatcher();
  return input.signal
    ? AbortSignal.any([input.signal, controller.signal])
    : controller.signal;
}

/** Stop every turn this replica is running for this person and Bot. Returns how many. */
function stopRunningTurns(key: string): number {
  const turns = state.running.get(key);
  if (!turns) return 0;
  let stopped = 0;
  for (const turn of turns) {
    if (Date.now() - turn.startedAt <= TURN_RETENTION_MS) stopped += 1;
    turn.controller.abort(new BotPausedError());
  }
  state.running.delete(key);
  state.unreadable.delete(key);
  return stopped;
}

/**
 * Pauses made on another replica reach the turns running here through this poll.
 *
 * Only while something is being tracked, and unref'd, so an idle process pays nothing and a
 * one-shot process is never held open by it.
 */
function startWatcher() {
  if (state.watcher) return;
  state.watcher = setInterval(() => {
    void checkRunningTurns();
  }, CROSS_REPLICA_CHECK_MS);
  state.watcher.unref?.();
}

async function checkRunningTurns() {
  const now = Date.now();
  for (const [key, turns] of state.running) {
    for (const turn of turns)
      if (now - turn.startedAt > TURN_RETENTION_MS) turns.delete(turn);
    if (turns.size === 0) {
      state.running.delete(key);
      state.unreadable.delete(key);
    }
  }
  if (state.running.size === 0) {
    if (state.watcher) clearInterval(state.watcher);
    state.watcher = undefined;
    return;
  }
  const database = state.database;
  if (!database) return;
  for (const key of [...state.running.keys()]) {
    const [ownerUserId, agentId] = JSON.parse(key) as [string, string];
    const paused = await readPause(database, ownerUserId, agentId);
    if (paused === undefined) {
      const failures = (state.unreadable.get(key) ?? 0) + 1;
      state.unreadable.set(key, failures);
      if (failures >= UNREADABLE_CHECKS_BEFORE_STOP) stopRunningTurns(key);
      continue;
    }
    state.unreadable.delete(key);
    if (paused) stopRunningTurns(key);
  }
}

export type BotLifecycleState = {
  agentId: string;
  paused: boolean;
  pausedAt: Date | null;
  notify: BotNotify;
};

export type BotLifecycleStore = ReturnType<typeof createBotLifecycleStore>;

/** The per-person rows, read and written. Authorization (may this person reach the Bot) is the caller's. */
export function createBotLifecycleStore(database: Database) {
  const upsert = (
    ownerUserId: string,
    agentId: string,
    values: { pausedAt?: Date | null; notify?: BotNotify },
  ) =>
    database
      .insert(botLifecycle)
      .values({ ownerUserId, agentId, ...values })
      .onConflictDoUpdate({
        target: [botLifecycle.ownerUserId, botLifecycle.agentId],
        set: { ...values, updatedAt: sql`now()` },
      });

  return {
    async get(
      ownerUserId: string,
      agentId: string,
    ): Promise<BotLifecycleState> {
      const [row] = await database
        .select()
        .from(botLifecycle)
        .where(
          and(
            eq(botLifecycle.ownerUserId, ownerUserId),
            eq(botLifecycle.agentId, agentId),
          ),
        )
        .limit(1);
      return {
        agentId,
        paused: Boolean(row?.pausedAt),
        pausedAt: row?.pausedAt ?? null,
        notify: row?.notify ?? "all",
      };
    },

    /** Every Bot this person has paused, for the roster's badges. */
    async pausedFor(ownerUserId: string): Promise<string[]> {
      const rows = await database
        .select({ agentId: botLifecycle.agentId })
        .from(botLifecycle)
        .where(
          and(
            eq(botLifecycle.ownerUserId, ownerUserId),
            isNotNull(botLifecycle.pausedAt),
          ),
        );
      return rows.map((row) => row.agentId);
    },

    async notifyFor(
      ownerUserId: string,
      agentIds: string[],
    ): Promise<Record<string, BotNotify>> {
      if (agentIds.length === 0) return {};
      const rows = await database
        .select({ agentId: botLifecycle.agentId, notify: botLifecycle.notify })
        .from(botLifecycle)
        .where(
          and(
            eq(botLifecycle.ownerUserId, ownerUserId),
            inArray(botLifecycle.agentId, agentIds),
          ),
        );
      return Object.fromEntries(rows.map((row) => [row.agentId, row.notify]));
    },

    /**
     * Pause, and stop what this replica is running for the pair right now. Other replicas stop
     * theirs on their next check. Returns how many running turns were stopped here.
     */
    async pause(ownerUserId: string, agentId: string): Promise<number> {
      await upsert(ownerUserId, agentId, { pausedAt: new Date() });
      return stopRunningTurns(keyOf(ownerUserId, agentId));
    },

    async resume(ownerUserId: string, agentId: string): Promise<void> {
      await upsert(ownerUserId, agentId, { pausedAt: null });
    },

    async setNotify(
      ownerUserId: string,
      agentId: string,
      notify: BotNotify,
    ): Promise<void> {
      await upsert(ownerUserId, agentId, { notify });
    },

    /** Where each kind of update goes. A kind with no row goes everywhere the person is reachable. */
    async routing(
      ownerUserId: string,
    ): Promise<Record<UpdateKind, UpdateTransport[] | "all">> {
      const rows = await database
        .select()
        .from(updateRoutingPreferences)
        .where(eq(updateRoutingPreferences.ownerUserId, ownerUserId));
      const byKind = new Map(rows.map((row) => [row.kind, row.transports]));
      return Object.fromEntries(
        UPDATE_KINDS.map((kind) => [kind, byKind.get(kind) ?? "all"]),
      ) as Record<UpdateKind, UpdateTransport[] | "all">;
    },

    async setRouting(
      ownerUserId: string,
      kind: UpdateKind,
      transports: UpdateTransport[] | "all",
    ): Promise<void> {
      if (transports === "all") {
        await database
          .delete(updateRoutingPreferences)
          .where(
            and(
              eq(updateRoutingPreferences.ownerUserId, ownerUserId),
              eq(updateRoutingPreferences.kind, kind),
            ),
          );
        return;
      }
      const unique = [...new Set(transports)];
      await database
        .insert(updateRoutingPreferences)
        .values({ ownerUserId, kind, transports: unique })
        .onConflictDoUpdate({
          target: [
            updateRoutingPreferences.ownerUserId,
            updateRoutingPreferences.kind,
          ],
          set: { transports: unique, updatedAt: sql`now()` },
        });
    },
  };
}

/** What the delivery router calls its three kinds, as the update kinds a person routes. */
const KIND_OF_DELIVERY: Record<"reply" | "question" | "approval", UpdateKind> =
  {
    reply: "progress",
    approval: "decision",
    question: "question",
  };

export type UpdateRoute = {
  /** Which update kind this delivery is, as the person's preferences name it. */
  kind: UpdateKind;
  /** Whether this delivery may leave the web at all (the per-Bot notification preference). */
  notify: boolean;
  /** Whether a transport may carry this delivery. Call once per binding or device. */
  allows: (transport: UpdateTransport) => boolean;
};

/**
 * The one lookup the delivery router consults before offering an outbox row.
 *
 * Combines the per-Bot notification preference (`none` sends nothing, `needs_input` sends
 * questions and approvals only) with the person's routing by kind. The web inbox and the sidebar
 * badges are not governed by this: they always show everything.
 *
 * FAILS OPEN TO THE DEFAULT, deliberately, and only here: a notification is a courtesy about work
 * that already happened, and a read that fails must not silently swallow a question somebody is
 * waiting to be asked. Unwired, it allows everything, which is the behaviour before this existed.
 */
export async function resolveUpdateRoute(input: {
  ownerUserId: string;
  agentId: string;
  kind: "reply" | "question" | "approval";
}): Promise<UpdateRoute> {
  const kind = KIND_OF_DELIVERY[input.kind];
  const everything: UpdateRoute = { kind, notify: true, allows: () => true };
  const database = state.database;
  if (!database) return everything;
  try {
    const store = createBotLifecycleStore(database);
    const [lifecycle, routing] = await Promise.all([
      store.get(input.ownerUserId, input.agentId),
      store.routing(input.ownerUserId),
    ]);
    const notify =
      lifecycle.notify === "all" ||
      (lifecycle.notify === "needs_input" && kind !== "progress");
    const transports = routing[kind];
    return {
      kind,
      notify,
      allows: (transport) =>
        notify && (transports === "all" || transports.includes(transport)),
    };
  } catch (error) {
    console.error(
      JSON.stringify({
        type: "update-route-lookup-failed",
        error: error instanceof Error ? error.message : String(error),
      }),
    );
    return everything;
  }
}