Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

24.8 KB
Raw Blame
/**
 * The OpenBot Browser's routes on this computer: open a person's browser with the Recorder in it and
 * connected, and act for the person in it (record, narrate, tabs, navigate, close, clear).
 *
 * Under the computer token like every other route, with the person's browser id in
 * `x-openbot-bot-id`; `index.ts` checks the token, this checks the id. The server is the only caller:
 * it decides who the signed-in person is, mints their Recorder connection, and forwards here. The
 * shapes are the contract in `shared/openbot-browser.ts`, which every slice builds against.
 *
 * Every answer but narration's is the browser's state, so the caller never has to ask twice to show
 * what a change did.
 *
 * Its own module, taking the browsers it acts on, so a test can serve these exact routes over a
 * person's browser launched headless, which the computer's own environment refuses for a native one.
 */
import type { BrowserContext, Page } from "playwright";
import {
  type BrowserOpenRequest,
  type BrowserState,
  browserComputerPaths,
  browserNarrationRequestSchema,
  browserNavigateRequestSchema,
  browserOpenRequestSchema,
  browserRecordingRequestSchema,
  browserTabsRequestSchema,
  isPersonBrowserId,
  type LauncherState,
  type Narration,
} from "../../shared/openbot-browser";
import { createTabBar } from "./browser-tabs";
import { BROWSER_NOT_OPEN, BrowserNotOpenError } from "./person-access";
import {
  NoInstalledBrowserError,
  PersonOpenError,
  type PersonOpenStep,
} from "./person-browser";
import type { OpenPersonBrowser, Profiles } from "./profiles";
import { findRecorder } from "./recorder";
import {
  RecorderUnavailableError,
  SENDING_WAIT_MS,
  untilSent,
} from "./recorder-launcher";

export type BrowserHostOptions = {
  profiles: Profiles;
  /** `OPENBOT_RECORDER_DIR`, read at every open so a rebuilt Recorder is the one the next browser gets. */
  recorderDir: () => string | undefined;
  navigationTimeoutMs: number;
  /**
   * Told while a person's browser is being cleared, once it has closed and before it can open again: the computer
   * forgets the browser it described (`profiles.clear`).
   */
  forget?: (botId: string) => void;
  /** How long a clear waits for a recording it stopped to reach Intelligence: `SENDING_WAIT_MS` unless a test's own. */
  sendingWaitMs?: number;
};

/** What the person is told when their browser did not open for a reason that is not the computer's setup: the reason
 * itself (often a page of Playwright's own text) goes to the computer's log. */
const NOT_OPENED = "Your browser could not be opened. Try again.";

/** How long one tab may take to say whether it is showing, before it is left out of the question. */
const VISIBILITY_MS = 1_000;

/**
 * How long the browser has to say what it is showing before the state route answers that it is not answering.
 *
 * A browser that answers at all answers this in milliseconds: the end-to-end run measured 2 to 19 ms. One whose main
 * thread is stuck never answers DevTools at all, and the route used to wait for it for as long as its caller did:
 * OpenBot gave up after 45 s, after its own server had already closed the person's request with an empty reply at 10.
 * The Browser page asks every second, and then had no browser to show and no sentence, and the sidebar lost its Browser
 * button. Bounded well under OpenBot's own wait for this route (server/src/browser/routes.ts, STATE_TIMEOUT_MS), so the
 * person hears it from here, where it is known that the browser is open and not answering.
 */
export const STATE_ANSWER_MS = 5_000;
/**
 * How long the state waits for the Recorder's own report. Within {@link STATE_ANSWER_MS}: a Recorder still starting, or
 * busy, is left out of the answer rather than making a browser that answers look like one that does not. The page asks
 * again a second later, and a Recorder that has started by then is in that answer.
 */
const RECORDER_REPORT_MS = 3_000;

/** What the state route answers, for OpenBot's log, when the browser did not answer in time. */
export const NOT_ANSWERING = `The browser did not answer within ${STATE_ANSWER_MS / 1000} s.`;

/** The work's value if it settles within `ms`, else nothing; it is left running, and a rejection is the caller's. */
async function within<T>(
  work: Promise<T>,
  ms: number,
): Promise<{ value: T } | undefined> {
  let timer: ReturnType<typeof setTimeout> | undefined;
  const late = new Promise<undefined>((resolve) => {
    timer = setTimeout(() => resolve(undefined), ms);
  });
  try {
    return await Promise.race([work.then((value) => ({ value })), late]);
  } finally {
    clearTimeout(timer);
  }
}

function json(body: unknown, status = 200): Response {
  return new Response(JSON.stringify(body), {
    status,
    headers: { "content-type": "application/json" },
  });
}

/** Enough of a contract schema to check a body with, whichever copy of zod built it. */
export type Schema<T> = {
  safeParse(input: unknown):
    | { success: true; data: T }
    | {
        success: false;
        error: { issues: { path: PropertyKey[]; message: string }[] };
      };
};

/** The first thing wrong with a body, in the words of the field it is about. */
export function parseBody<T>(
  schema: Schema<T>,
  body: unknown,
): { ok: true; value: T } | { ok: false; error: string; response: Response } {
  const parsed = schema.safeParse(body);
  if (parsed.success) return { ok: true, value: parsed.data };
  const issue = parsed.error.issues[0];
  const where = issue?.path.length
    ? `${issue.path.map(String).join(".")}: `
    : "";
  const error = `${where}${issue?.message ?? "That request is not valid."}`;
  return { ok: false, error, response: json({ error }, 400) };
}

const messageOf = (error: unknown): string =>
  error instanceof Error ? error.message : String(error);

/** A person's browser's opens: how many have arrived and not finished, and the newest connection any brought. */
type Opening = {
  /** Settles when the last open to arrive has finished, so the next waits for it. */
  turn: Promise<unknown>;
  pending: number;
  /** None while no open among them brought one: each then leaves the Recorder the connection it holds. */
  newest?: NonNullable<BrowserOpenRequest["connection"]>;
};

export function createBrowserHost(options: BrowserHostOptions) {
  const { profiles } = options;
  const bar = createTabBar({
    navigationTimeoutMs: options.navigationTimeoutMs,
  });

  /**
   * In a native window the person changes tabs in the browser itself, which tells this process
   * nothing, so the tab in front is asked of the pages: the one the browser is showing, and of two
   * windows showing a tab each, the one with the focus. With no window in focus (the person is in
   * OpenBot), the tab already in front stays while it still shows; and a minimised browser, where
   * none shows, keeps the one it had.
   *
   * This reads true only because a native window has no focus emulation (`native-browser.ts`): with
   * it, every tab said it was showing and focused, and the person's own switch was never seen.
   */
  const followNativeTabs = async (browser: OpenPersonBrowser) => {
    if (browser.window !== "native") return;
    const pages = browser.pages();
    const hidden = { showing: false, focused: false };
    const shown = await Promise.all(
      pages.map((page) =>
        Promise.race([
          page
            .evaluate(() => ({
              showing: document.visibilityState === "visible",
              focused: document.hasFocus(),
            }))
            .catch(() => hidden),
          new Promise<typeof hidden>((resolve) =>
            setTimeout(() => resolve(hidden), VISIBILITY_MS),
          ),
        ]),
      ),
    );
    const showing = pages.filter((_, index) => shown[index]?.showing);
    const focused = pages.find(
      (_, index) => shown[index]?.showing && shown[index]?.focused,
    );
    const front =
      focused ??
      (showing.includes(browser.active()) ? undefined : showing.at(-1));
    if (front && front !== browser.active()) browser.activate(front);
  };

  const stateOf = async (
    browser: OpenPersonBrowser,
    recorder?: LauncherState,
    reportWithinMs?: number,
  ): Promise<BrowserState> => {
    // The Recorder's own report, when it answers. A Recorder that is starting or busy leaves it out
    // rather than failing the whole answer: the tabs are still worth showing. Asked beside the tabs,
    // and, where the caller says, for no longer than that.
    const ask = (): Promise<LauncherState | undefined> => {
      const asked = browser.launcher.state().catch(() => undefined);
      return reportWithinMs === undefined
        ? asked
        : within(asked, reportWithinMs).then((answer) => answer?.value);
    };
    const [report, tabs] = await Promise.all([
      recorder ?? ask(),
      bar.tabsOf(browser),
    ]);
    return {
      open: true,
      browser: browser.product,
      window: browser.window,
      ...(report ? { recorder: report } : {}),
      tabs,
    };
  };

  /**
   * The state the state route answers: the browser's own, asked once however often it is asked meanwhile. The page asks
   * every second, and a browser that does not answer would otherwise gather one unanswered question a second for as
   * long as it does not.
   */
  const asking = new WeakMap<BrowserContext, Promise<BrowserState>>();
  /** The browsers last found not answering, so the log says when one stops answering rather than every second. */
  const notAnswering = new WeakSet<BrowserContext>();
  const currentState = (browser: OpenPersonBrowser): Promise<BrowserState> => {
    const pending = asking.get(browser.context);
    if (pending) return pending;
    const asked = (async () => {
      await followNativeTabs(browser);
      return stateOf(browser, undefined, RECORDER_REPORT_MS);
    })();
    asking.set(browser.context, asked);
    void asked
      .finally(() => {
        if (asking.get(browser.context) === asked)
          asking.delete(browser.context);
      })
      .catch(() => undefined);
    return asked;
  };

  const closedState: BrowserState = { open: false, tabs: [] };

  /*
   * OPENS OF ONE BROWSER TAKE TURNS, AND EACH HANDS THE RECORDER THE NEWEST CONNECTION.
   *
   * Every open carries a connection OpenBot minted for it, and minting one revokes the one before:
   * the browser holds one connection, by its install id. Two opens close together, which is what the
   * Browser page sends when it is loaded again while it is still opening, used to run side by side.
   * The first handed the Recorder a connection the second had revoked already, the Recorder refused
   * it as signed out, and the first closed the browser it had just launched, under the second, whose
   * own connect then failed with "The browser is shutting down". Both answered 502 and the browser
   * stayed closed. Measured on Google Chrome 156 with the built Recorder and the real server, after a
   * restart with a newer Recorder, whose reload makes the first open take 2.7 s: two opens 1 s or
   * 2.5 s apart failed both, every time; 3.5 s apart, the first had finished and both opened.
   *
   * So an open waits for the one before it, and connects the Recorder with the newest connection any
   * open has brought, which is the one OpenBot holds: an open that had waited finds the Recorder
   * holding it already, which connects at once. And a browser no open has connected since it
   * launched is closed after a failed connect only when no other open is waiting to connect it.
   */
  const openings = new Map<string, Opening>();
  /** The browsers an open has connected since they launched, by their context, which is theirs alone. */
  const connectedOnce = new WeakSet<BrowserContext>();

  async function open(botId: string, body: unknown): Promise<Response> {
    const arrived = Date.now();
    /*
     * Every open that fails says why in the log, with the step it stopped at: which part of opening
     * (the launch, the Recorder's load or reload, its worker, the microphone, the connect) failed,
     * and how. Without it a failed open said only that the browser was then stopped. The log keeps
     * `reason`, what went wrong in its own words, where the caller is answered with a sentence for
     * the person instead.
     */
    const failed = (
      step: PersonOpenStep | "request",
      status: number,
      error: string,
      reason: string = error,
    ): Response => {
      console.info(
        JSON.stringify({
          type: "computer-browser-open-failed",
          botId,
          step,
          status,
          error: reason,
          ms: Date.now() - arrived,
        }),
      );
      return json({ error }, status);
    };
    const parsed = parseBody(browserOpenRequestSchema, body);
    if (!parsed.ok) return failed("request", 400, parsed.error);

    const brought = parsed.value.connection;
    const opening: Opening = openings.get(botId) ?? {
      turn: Promise.resolve(),
      pending: 0,
    };
    openings.set(botId, opening);
    if (brought) opening.newest = brought;
    opening.pending += 1;
    const done = () => {
      opening.pending -= 1;
      if (opening.pending === 0 && openings.get(botId) === opening)
        openings.delete(botId);
    };
    const run = opening.turn.then(() => openNow(botId, opening, failed));
    opening.turn = run.then(done, done);
    return run;
  }

  async function openNow(
    botId: string,
    opening: Opening,
    failed: (
      step: PersonOpenStep,
      status: number,
      error: string,
      reason?: string,
    ) => Response,
  ): Promise<Response> {
    const found = await findRecorder(options.recorderDir());
    if (!found.ok) return failed("recorder-files", 503, found.error);

    const wasOpen = profiles.person(botId) !== undefined;
    let page: Page;
    try {
      page = await profiles.page(botId, { open: { recorder: found.recorder } });
    } catch (error) {
      if (error instanceof NoInstalledBrowserError)
        return failed("launch", 503, error.message);
      const at = error instanceof PersonOpenError ? error.step : "launch";
      /*
       * A browser this computer cannot run at all (Playwright's Chromium missing from a hosted image,
       * say) is OpenBot's setup: 503, in the computer's own words for whoever runs OpenBot, which the
       * server keeps in its log and never shows the person, as for a missing Recorder. Run 4 saw such
       * a failure reach the person as a 502 with Playwright's text, its install box included.
       */
      if (error instanceof PersonOpenError && error.setup)
        return failed(at, 503, error.message);
      return failed(at, 502, NOT_OPENED, messageOf(error));
    }
    const browser = profiles.person(botId);
    if (!browser) return failed("launch", 409, BROWSER_NOT_OPEN);
    // Already open: the window comes to the front and nothing new opens.
    if (wasOpen && browser.window === "native")
      await bar.putInFront(browser, page);

    /*
     * An open that brought no connection, with none from another open waiting either, hands the Recorder nothing: it
     * keeps the connection it holds, which the state this answers names (`recorder.connectionId`). OpenBot opens so
     * when it holds a connection that stands for this browser, and connects it with a new one only if the Recorder
     * turns out not to hold that one.
     */
    const connection = opening.newest;
    if (!connection) {
      try {
        return json(await stateOf(browser));
      } catch (error) {
        return failed("state", 502, NOT_OPENED, messageOf(error));
      }
    }

    // Who hears the person: the Recorder itself in a native window, the app's page in a hosted one.
    const narration: Narration =
      browser.window === "native" ? "own" : "external";
    let connected: Awaited<ReturnType<typeof browser.launcher.connect>>;
    try {
      // Every open that brings one, not only the first: idempotent, and the connection the server sent
      // last is the one the Recorder should hold.
      connected = await browser.launcher.connect({ connection, narration });
    } catch (error) {
      connected = {
        ok: false,
        error:
          error instanceof Error
            ? error.message
            : "The Recorder did not answer.",
      };
    }
    if (!connected.ok) {
      // A browser no open has connected since it launched is closed again: the person was promised
      // one that is connected. Not while another open waits to connect it, with a connection as new
      // as this one or newer: closing it would fail that open too. One that was connected stays,
      // with the person's tabs.
      if (!connectedOnce.has(browser.context) && opening.pending <= 1)
        await profiles.stop(botId, "its Recorder could not be connected");
      return failed(
        "connect",
        502,
        `The Recorder could not be connected: ${connected.error}`,
      );
    }
    connectedOnce.add(browser.context);
    try {
      return json(await stateOf(browser));
    } catch (error) {
      // The browser went away between the connect and its state: closed by the person, or crashed.
      return failed("state", 502, NOT_OPENED, messageOf(error));
    }
  }

  /**
   * Wipe the person's browser, open or closed (`shared/bot-browser.ts`, CLEARING A BROWSER), and leave it closed: only
   * the server can hand its Recorder a connection, so it is the server that opens it again. A recording running in it
   * is stopped first, and the last of it waited for, because what it has not sent is kept in the storage about to go.
   * Never fails for the Recorder: a browser cleared because it went wrong may well have a Recorder that does not answer,
   * and is cleared all the same.
   */
  async function clear(botId: string): Promise<Response> {
    const browser = profiles.person(botId);
    if (browser) {
      try {
        const report = await browser.launcher.state();
        if (report.phase !== "idle") {
          const stopped = await browser.launcher.record({ action: "stop" });
          const sent = await untilSent(
            browser.launcher,
            stopped.ok ? stopped.state : report,
            options.sendingWaitMs ?? SENDING_WAIT_MS,
          );
          console.info(
            JSON.stringify({
              type: "computer-browser-recording-ended",
              botId,
              reason: "the browser's data was cleared",
              ...(stopped.ok ? {} : { error: stopped.error }),
              ...(sent.stillSending ? { stillSending: true } : {}),
            }),
          );
        }
      } catch (error) {
        console.info(
          JSON.stringify({
            type: "computer-browser-recording-not-ended",
            botId,
            error: messageOf(error),
          }),
        );
      }
    }
    await profiles.clear(botId, () => options.forget?.(botId));
    console.info(JSON.stringify({ type: "computer-browser-cleared", botId }));
    return json(closedState);
  }

  async function handle(
    request: Request,
    pathname: string,
    botId: string,
  ): Promise<Response> {
    if (!isPersonBrowserId(botId))
      return json(
        {
          error:
            "The browser routes are for a person's browser: send its id (person- and 32 hex digits) in x-openbot-bot-id.",
        },
        400,
      );
    const method = pathname === browserComputerPaths.state ? "GET" : "POST";
    if (request.method !== method)
      return json({ error: `Use ${method} for ${pathname}.` }, 405);
    const body =
      method === "POST" ? await request.json().catch(() => null) : null;

    if (pathname === browserComputerPaths.open) return open(botId, body);

    if (pathname === browserComputerPaths.close) {
      await profiles.stop(botId);
      return json(closedState);
    }

    if (pathname === browserComputerPaths.clear) return clear(botId);

    /*
     * OpenBot revoked this browser's connection: the Recorder forgets it now rather than at its next
     * call to OpenBot. A closed browser has no Recorder running to tell, and is not opened to tell
     * it: the next open hands the Recorder whatever connection OpenBot gives it then.
     */
    if (pathname === browserComputerPaths.disconnect) {
      const browser = profiles.person(botId);
      if (!browser) return json(closedState);
      try {
        const result = await browser.launcher.disconnect();
        if (!result.ok) return json({ error: result.error }, 409);
      } catch (error) {
        if (error instanceof RecorderUnavailableError)
          return json({ error: error.message }, 502);
        throw error;
      }
      return json(await stateOf(browser));
    }

    // Validated before asking whether the browser is open, so a malformed call says what is wrong
    // with it rather than that there is nothing to act on.
    const recording =
      pathname === browserComputerPaths.recording
        ? parseBody(browserRecordingRequestSchema, body)
        : undefined;
    const narration =
      pathname === browserComputerPaths.narration
        ? parseBody(browserNarrationRequestSchema, body)
        : undefined;
    const tabs =
      pathname === browserComputerPaths.tabs
        ? parseBody(browserTabsRequestSchema, body)
        : undefined;
    const navigate =
      pathname === browserComputerPaths.navigate
        ? parseBody(browserNavigateRequestSchema, body)
        : undefined;
    for (const parsed of [recording, narration, tabs, navigate])
      if (parsed && !parsed.ok) return parsed.response;

    const browser = profiles.person(botId);
    if (pathname === browserComputerPaths.state) {
      if (!browser) return json(closedState);
      /*
       * A browser that does not answer in time is said to be open and not answering, as a 504 with a sentence, and the
       * question is left running for the next ask to join. There is no rule here that closes or restarts one: a browser
       * that answers again is shown again, and closing it is the person's (`/browser/close`) or the idle sweep's.
       */
      const answered = await within(currentState(browser), STATE_ANSWER_MS);
      if (!answered) {
        if (!notAnswering.has(browser.context)) {
          notAnswering.add(browser.context);
          console.warn(
            JSON.stringify({
              type: "computer-browser-not-answering",
              botId,
              ms: STATE_ANSWER_MS,
            }),
          );
        }
        return json({ error: NOT_ANSWERING }, 504);
      }
      notAnswering.delete(browser.context);
      return json(answered.value);
    }
    if (!browser) return json({ error: BROWSER_NOT_OPEN }, 409);

    try {
      if (recording?.ok) {
        await followNativeTabs(browser);
        // The Recorder records the tab the browser has in front, so a hosted browser's front tab is
        // made the one the live screen shows before it starts.
        if (recording.value.action === "start" && browser.window === "screen")
          await bar.putInFront(browser, browser.active());
        const result = await browser.launcher.record(recording.value);
        if (!result.ok) return json({ error: result.error }, 409);
        return json(await stateOf(browser, result.state));
      }

      if (narration?.ok) {
        /*
         * A phrase, or the person starting or stopping one. The times are already on the server's
         * clock, which corrected them by `sentAt` on the way (see the contract); `sentAt` itself is
         * the server's own and goes no further.
         */
        const said = narration.value;
        const result =
          "text" in said
            ? await browser.launcher.narrate({
                text: said.text,
                startedAt: said.startedAt,
                endedAt: said.endedAt,
              })
            : await browser.launcher.speaking({
                since: said.speaking ? said.since : null,
              });
        if (!result.ok) return json({ error: result.error }, 409);
        return json({ ok: true });
      }

      if (tabs?.ok) {
        const done = await bar.tabs(browser, tabs.value);
        if (!done.ok) return json({ error: done.error }, done.status);
        // A closed tab can take the browser's last window with it.
        const after = profiles.person(botId);
        return json(after ? await stateOf(after) : closedState);
      }

      if (navigate?.ok) {
        await followNativeTabs(browser);
        const done = await bar.navigate(browser, navigate.value);
        if (!done.ok) return json({ error: done.error }, done.status);
        return json(await stateOf(browser));
      }
    } catch (error) {
      if (error instanceof BrowserNotOpenError)
        return json({ error: error.message }, 409);
      if (error instanceof RecorderUnavailableError)
        return json({ error: error.message }, 502);
      throw error;
    }
    return json({ error: "Not found." }, 404);
  }

  return { handle };
}

export type BrowserHost = ReturnType<typeof createBrowserHost>;

/** Whether a path is one of these routes, so the dispatcher hands it over. */
export function isBrowserPath(pathname: string): boolean {
  return (Object.values(browserComputerPaths) as string[]).includes(pathname);
}