Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

8.3 KB
Raw Blame
/**
 * The Recorder in a browser this computer launched: its setup pages closed as they appear, its worker running the
 * Recorder on disk, and its launcher API reached over the browser's DevTools session.
 *
 * Two browsers have it. A person's (`person-browser.ts`), where it is connected from the moment the browser opens, and
 * a Bot's (`profiles.ts`), where it is loaded at every launch and stays disconnected and silent until a person holding
 * the wheel presses Record (`shared/bot-browser.ts`). How it is put in differs (by flag for bundled Chromium, over
 * DevTools for a native window); what is done with it once it is in is the same, and lives here.
 */
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import type { BrowserContext, CDPSession, Page } from "playwright";
import { isRecorderSetupPage, type Recorder } from "./recorder";
import {
  createRecorderLauncher,
  type RecorderLauncher,
} from "./recorder-launcher";

/** The parts of holding the Recorder that can fail on their own, which a person's open names when one does. */
export type RecorderStep = "recorder-files" | "worker" | "recorder-reload";

/** Runs one part, so a caller can name the part that failed. By default the part runs as it is. */
export type RunStep = <T>(
  name: RecorderStep,
  work: () => Promise<T>,
) => Promise<T>;

const asItIs: RunStep = (_name, work) => work();

/**
 * Close the Recorder's setup and connect pages whenever one appears, for the life of the browser.
 *
 * The Recorder opens its setup page when it is installed, and every launch installs it afresh, so
 * this is not a one-off at start. The url a new tab reports can still be the blank one it started
 * on, so each page is checked again as it navigates.
 */
export function closeRecorderSetupPages(
  context: BrowserContext,
  recorderId: string,
) {
  const check = (page: Page) => {
    if (isRecorderSetupPage(page.url(), recorderId))
      void page.close().catch(() => undefined);
  };
  const watch = (page: Page) => {
    check(page);
    page.on("framenavigated", (frame) => {
      if (frame === page.mainFrame()) check(page);
    });
  };
  for (const page of context.pages()) watch(page);
  context.on("page", watch);
}

/**
 * Make a hosted browser's window hold exactly the page the live screen shows, as each tab opens.
 *
 * Playwright sizes the window to the viewport with its own allowance for the browser's toolbars, and
 * that allowance is none when headless, though headless Chromium lays its toolbars out all the same
 * (87 pixels here). The page is told it is 800 high, and a tab that has been laid out again (switched
 * away from and back, say) is then shown in what is left of the window: the live screen lost the
 * bottom of the page, 713 of 800, while the page's own viewport still said 800. Measured on bundled
 * Chromium, headless, with Playwright 1.62.1. `Browser.setContentsSize` grows the window around the
 * contents, so the toolbars come out of the window and not the page, headed or headless. After
 * Playwright's own resize, which it makes as each tab starts and then leaves alone.
 *
 * Settles once the tabs open now are fitted; each tab opened later is fitted as it opens.
 * https://chromedevtools.github.io/devtools-protocol/tot/Browser/#method-setContentsSize
 */
export async function fitWindowsToViewport(
  context: BrowserContext,
  viewport: { width: number; height: number },
  botId: string,
): Promise<void> {
  const fit = async (page: Page) => {
    const session = await context.newCDPSession(page);
    try {
      const { windowId } = await session.send("Browser.getWindowForTarget");
      await session.send("Browser.setContentsSize", { windowId, ...viewport });
    } finally {
      await session.detach().catch(() => undefined);
    }
  };
  const fitting = (page: Page) =>
    fit(page).catch((error: unknown) => {
      // A tab that closed while it was being fitted needs nothing; anything else is said, because
      // the live screen then shows less than the whole page.
      if (page.isClosed()) return;
      console.info(
        JSON.stringify({
          type: "computer-window-not-fitted",
          botId,
          error: error instanceof Error ? error.message : String(error),
        }),
      );
    });
  context.on("page", (page) => void fitting(page));
  await Promise.all(context.pages().map(fitting));
}

/**
 * Make the Recorder's worker run the Recorder on disk, loading it again when it runs an earlier one.
 *
 * WHY. A browser keeps an extension's service worker registered in the profile, its scripts cached
 * in `Service Worker/ScriptCache`, from one run to the next: closing the browser unloads nothing.
 * Loaded at the next start, by `--load-extension` or DevTools `Extensions.loadUnpacked`, the Recorder
 * is installed as if new rather than updated, so nothing unregisters that worker; registering the
 * same script again is, by the service worker rules, the registration already there; and the
 * browser's own update check never runs for an extension's worker. So a profile kept running the
 * Recorder it first cached, whatever had been built since: every Recorder fix was invisible in a
 * browser that had run an earlier one, and an update of OpenBot would never have updated it.
 * Measured on Google Chrome 156 and Playwright's Chromium, for a changed script and for a changed
 * manifest version alike, which makes no difference. Loading the Recorder again while it is loaded
 * is an update, and an update unregisters the worker and registers the one on disk: measured the
 * same way, the new script then runs, and keeps running at later launches.
 *
 * Compared byte for byte, as a service worker's own update check compares, so a browser already
 * running the Recorder on disk is left alone and its worker is not restarted.
 * https://chromium.googlesource.com/chromium/src/+/main/extensions/browser/service_worker/service_worker_task_queue.cc
 * https://chromium.googlesource.com/chromium/src/+/main/content/browser/service_worker/service_worker_register_job.cc
 * https://w3c.github.io/ServiceWorker/#register-algorithm
 * https://chromedevtools.github.io/devtools-protocol/tot/Extensions/#method-loadUnpacked
 */
export async function keepRecorderCurrent(input: {
  session: CDPSession;
  launcher: RecorderLauncher;
  recorder: Recorder;
  recorderId: string;
  botId: string;
  step?: RunStep;
}): Promise<void> {
  const { recorder, launcher } = input;
  const step = input.step ?? asItIs;
  const built = await step("recorder-files", () =>
    readFile(join(recorder.dir, recorder.worker), "utf8"),
  );
  const url = `chrome-extension://${input.recorderId}/${recorder.worker}`;
  // The first call to the worker since the launch: it finds it, and wakes it if it has not started.
  if ((await step("worker", () => launcher.workerSource(url))) === built)
    return;
  console.info(
    JSON.stringify({
      type: "computer-recorder-reloaded",
      botId: input.botId,
      reason:
        "Its service worker ran an earlier build than the Recorder on disk.",
    }),
  );
  await step("recorder-reload", async () => {
    await input.session.send("Extensions.loadUnpacked", { path: recorder.dir });
    if ((await launcher.workerSource(url)) !== built)
      throw new Error(
        "The Recorder in this browser runs an earlier build, and loading the current one did not replace it.",
      );
  });
}

/**
 * Reach the Recorder in a browser that has just started with it in: its setup pages closed for the life of the
 * browser, and its launcher, through which everything is asked of it, over the browser-level DevTools session.
 */
export function reachRecorder(input: {
  /** A browser-level session (`browser.newBrowserCDPSession()`), which alone can reach the worker. */
  session: CDPSession;
  context: BrowserContext;
  /** The Recorder's id in this browser, which says which pages and which worker are its own. */
  recorderId: string;
}): RecorderLauncher {
  closeRecorderSetupPages(input.context, input.recorderId);
  return createRecorderLauncher(input);
}

/** Reach the Recorder, and make its worker run the Recorder on disk before it is used. */
export async function holdRecorder(input: {
  session: CDPSession;
  context: BrowserContext;
  recorder: Recorder;
  recorderId: string;
  botId: string;
  step?: RunStep;
}): Promise<RecorderLauncher> {
  const launcher = reachRecorder(input);
  await keepRecorderCurrent({ ...input, launcher });
  return launcher;
}