Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

6.0 KB
Raw Blame
/**
 * What a Bot's screen looked like when it opened a page.
 *
 * Written where the navigation happens, which is the one moment the screen is certainly showing the
 * page that was asked for, and read back when somebody reopens the conversation that asked for it.
 */
import { and, eq, or, sql } from "drizzle-orm";
import type { Database } from "../db/client";
import { computerPageFrame } from "../db/schema";

/**
 * A screenshot, and the ceiling on one.
 *
 * Generous enough for a full page at the sizes a computer runs, small enough that nothing can push
 * megabytes into the store. Refused rather than truncated, because half a PNG is not a smaller
 * picture, it is a broken one.
 */
const MAX_FRAME_BYTES = 4 * 1024 * 1024;

/**
 * How long a turn's screenshot is kept.
 *
 * A month, because reading back a conversation is the thing these exist for and people do that long
 * after the run. Past that the transcript names the page it opened instead, which is the same
 * sentence with less in it rather than a broken one.
 */
export const FRAME_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;

// Small batches: a frame is hundreds of kilobytes, so the audit sweep's five thousand would be a gigabyte a statement.
const PURGE_BATCH = 200;
const MAX_PURGE_BATCHES = 200;

/**
 * How big a base64 string actually is, in bytes.
 *
 * `String.length` counts characters, which is the same number only while every character is ASCII.
 * Base64 is ASCII, so the two agree today; measuring the thing the limit is named for means they
 * cannot quietly stop agreeing.
 */
function byteLength(value: string): number {
  return Buffer.byteLength(value, "utf8");
}

function tooLarge(value: string): boolean {
  return byteLength(value) > MAX_FRAME_BYTES;
}

export type PageFrameStore = {
  save: (frame: {
    computerId: string;
    toolCallId: string;
    url: string;
    title?: string;
    frame: string;
  }) => Promise<void>;
  load: (
    computerId: string,
    toolCallId: string,
  ) => Promise<{ url: string; title: string | null; frame: string } | null>;
  /**
   * Everything kept for one seat, or for every seat on one computer. Returns how many went.
   *
   * Called when a profile is wiped, because "every login the Bot had is gone" is not true while
   * pictures of the signed-in pages are still readable from the transcript. Given a computer
   * (`person:<id>`) it takes every seat filed under it (`person:<id>/<bot>`), which is what wiping a
   * person's whole computer wipes.
   */
  clear: (computerId: string) => Promise<number>;
  /**
   * Frames older than the retention window, across every computer. Returns how many went.
   *
   * A page is a distinct row, so a Bot that browses grows this table for as long as it runs and
   * nothing ever took anything out of it. Screenshots are the largest thing this deployment stores
   * and the least useful once nobody is reading that conversation any more.
   */
  purge: (olderThanMs: number) => Promise<number>;
};

export function createPageFrameStore(database: Database): PageFrameStore {
  return {
    async save(input) {
      if (!input.toolCallId || !input.url) return;
      if (tooLarge(input.frame)) {
        /*
         * Said out loud rather than dropped in silence. A turn with no picture falls back to naming
         * its page, which looks exactly like a turn nobody photographed, so without this line a
         * deployment whose pages are all too big has no way to find that out.
         */
        console.warn(
          `[computer] a frame of ${input.url} was ${byteLength(input.frame)} bytes and was not kept; the limit is ${MAX_FRAME_BYTES}.`,
        );
        return;
      }
      await database
        .insert(computerPageFrame)
        .values({
          computerId: input.computerId,
          toolCallId: input.toolCallId,
          url: input.url,
          ...(input.title ? { title: input.title } : {}),
          frame: input.frame,
        })
        /*
         * Written once. A turn happens once and is then over for good, so a second write for the
         * same turn is either a retry of the same thing or a mistake, and neither should change what
         * a past turn shows. Letting the newer win is what made the record mutable before.
         */
        .onConflictDoNothing();
    },

    async load(computerId, toolCallId) {
      const [row] = await database
        .select({
          url: computerPageFrame.url,
          title: computerPageFrame.title,
          frame: computerPageFrame.frame,
        })
        .from(computerPageFrame)
        /*
         * Both, always. A caller who may reach one Bot must not be able to read another Bot's screen
         * by naming a turn: the Bot in the path is what the route already checked, so it is what this
         * is keyed on too.
         */
        .where(
          and(
            eq(computerPageFrame.computerId, computerId),
            eq(computerPageFrame.toolCallId, toolCallId),
          ),
        );
      return row ?? null;
    },

    async clear(computerId) {
      const gone = await database
        .delete(computerPageFrame)
        .where(
          or(
            eq(computerPageFrame.computerId, computerId),
            sql`starts_with(${computerPageFrame.computerId}, ${`${computerId}/`})`,
          ),
        )
        .returning({ url: computerPageFrame.url });
      return gone.length;
    },

    async purge(olderThanMs) {
      // Batched like the audit sweep: one statement over ninety days of one Bot held its locks for 17s.
      let removed = 0;
      for (let batch = 0; batch < MAX_PURGE_BATCHES; batch += 1) {
        const result = (await database.execute(sql`
          delete from ${computerPageFrame} where ctid in (
            select ctid from ${computerPageFrame}
            where ${computerPageFrame.capturedAt} < now() - make_interval(secs => ${olderThanMs / 1000})
            limit ${PURGE_BATCH}
          )
        `)) as unknown as { count?: number };
        const count = result?.count ?? 0;
        removed += count;
        if (count < PURGE_BATCH) break;
      }
      return removed;
    },
  };
}