Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

9.9 KB
Raw Blame
/**
 * The bar above a browser OpenBot shows: its tabs (new, switch, close) and the tab in front's address, back, forward
 * and reload.
 *
 * Two browsers have it. A person's own (`browser-host.ts`), which is theirs at every moment, and a Bot's
 * (`bot-browser-host.ts`), which a person works only while they hold its wheel. Who may use the bar is each host's to
 * decide; what the bar does, and what it says when a page does not open, is the same in both and lives here.
 *
 * Playwright's own types, and nothing of either host's, so both hand it the browser they have.
 */
import type { BrowserContext, CDPSession, Page } from "playwright";
import type {
  BrowserNavigateRequest,
  BrowserTab,
  BrowserTabsRequest,
  BrowserWindow,
} from "../../shared/openbot-browser";
import { navigationFailure } from "./navigation";
import { addressFrom } from "./request-validation";

/** A browser whose tabs the bar works. */
export type TabbedBrowser = {
  context: BrowserContext;
  /** The browser-level DevTools session, which the tab list reads titles from. */
  session: CDPSession;
  /** `screen`: OpenBot decides which tab is in front and the live screen shows it. `native`: the person's own window. */
  window: BrowserWindow;
  /** The tab in front: the one the live screen shows and the bar acts on. */
  active(): Page;
  /** Put a tab in front. */
  activate(page: Page): void;
  /** The tabs, oldest first, never the Recorder's own pages. */
  pages(): Page[];
};

/** What the bar did: done, or a refusal in a sentence for the person, with the status it is answered with. */
export type BarOutcome =
  | { ok: true }
  | { ok: false; status: 400 | 404 | 502; error: string };

/**
 * Where Chromium's error page lives. A tab whose page did not load shows it, and Playwright reports
 * the error page's own address, `chrome-error://chromewebdata/`, as the tab's. The tab itself still
 * names the address that failed, as a browser's address bar does: DevTools' target for the tab says
 * so. Measured on Playwright 1.62.1's Chromium for a name that does not resolve and a port that
 * refuses.
 */
const ERROR_PAGE = "chrome-error://";
/** How long a failed navigation waits for the tab to show its error page. */
const ERROR_PAGE_MS = 2_000;

const NOT_AN_ADDRESS = "That is not a web address.";

export function createTabBar(options: { navigationTimeoutMs: number }) {
  /** Each tab's DevTools target id, which stays the same for the life of the tab. */
  const tabIds = new WeakMap<Page, string>();
  /** How many documents each tab's main frame has committed, counted from when it was first asked. */
  const commits = new WeakMap<Page, { count: number }>();

  const commitsOf = (page: Page): number => {
    let counted = commits.get(page);
    if (!counted) {
      const counter = { count: 0 };
      page.on("framenavigated", (frame) => {
        if (frame === page.mainFrame()) counter.count += 1;
      });
      commits.set(page, counter);
      counted = counter;
    }
    return counted.count;
  };

  const tabIdOf = async (
    browser: TabbedBrowser,
    page: Page,
  ): Promise<string | undefined> => {
    const known = tabIds.get(page);
    if (known) return known;
    const session = await browser.context.newCDPSession(page);
    try {
      const { targetInfo } = await session.send("Target.getTargetInfo");
      tabIds.set(page, targetInfo.targetId);
      return targetInfo.targetId;
    } finally {
      await session.detach().catch(() => undefined);
    }
  };

  const pageFor = async (
    browser: TabbedBrowser,
    tabId: string,
  ): Promise<Page | undefined> => {
    for (const page of browser.pages()) {
      if ((await tabIdOf(browser, page).catch(() => undefined)) === tabId)
        return page;
    }
    return undefined;
  };

  /** The address a tab shows: its page's, or on an error page the address that failed. */
  const shownUrl = (page: Page, target: { url: string } | undefined): string =>
    page.url().startsWith(ERROR_PAGE) && target?.url ? target.url : page.url();

  /** The tabs as the bar lists them, the one in front marked. */
  const tabsOf = async (browser: TabbedBrowser): Promise<BrowserTab[]> => {
    const { targetInfos } = await browser.session.send("Target.getTargets");
    const targets = new Map(
      targetInfos.map((target) => [target.targetId, target]),
    );
    const active = browser.active();
    const tabs: BrowserTab[] = [];
    for (const page of browser.pages()) {
      const id = await tabIdOf(browser, page).catch(() => undefined);
      if (!id) continue;
      tabs.push({
        id,
        url: shownUrl(page, targets.get(id)),
        title: targets.get(id)?.title ?? "",
        active: page === active,
      });
    }
    return tabs;
  };

  /** One tab's shown address, for a sentence about a page that did not open. */
  const addressOf = async (
    browser: TabbedBrowser,
    page: Page,
  ): Promise<string> => {
    if (!page.url().startsWith(ERROR_PAGE)) return page.url();
    const id = await tabIdOf(browser, page).catch(() => undefined);
    const { targetInfos } = await browser.session
      .send("Target.getTargets")
      .catch(() => ({ targetInfos: [] }));
    return shownUrl(
      page,
      targetInfos.find((target) => target.targetId === id),
    );
  };

  /** Bring a tab to the front: in its window, and so on the live screen and for the Recorder. */
  const putInFront = async (browser: TabbedBrowser, page: Page) => {
    browser.activate(page);
    await page.bringToFront().catch(() => undefined);
  };

  /**
   * A page that did not open, said in a sentence once the tab shows that it did not.
   *
   * Chromium commits its error page a moment after the navigation fails: when `goto` threw, the tab
   * still reported the address it had before (or none, for a new tab), and the error page with the
   * failed address about 100 ms later, measured on Playwright 1.62.1's Chromium and on Chrome. The
   * app asks for the tabs as soon as this answers, and showed the earlier address beside the error
   * page, so this waits for that commit first, briefly, and answers either way. Only for a network
   * error: a navigation that timed out is still loading, and one that became a download or was
   * aborted shows no error page.
   */
  const navigation = async (
    page: Page,
    since: number,
    address: string,
    error: unknown,
  ): Promise<BarOutcome> => {
    const message = error instanceof Error ? error.message : String(error);
    if (/net::ERR_/.test(message) && !/net::ERR_ABORTED/.test(message)) {
      // A document committed since the navigation began, and it is the error page: a reload of a
      // page that had already failed is on one before its own commits.
      const deadline = Date.now() + ERROR_PAGE_MS;
      while (
        Date.now() < deadline &&
        !page.isClosed() &&
        !(commitsOf(page) > since && page.url().startsWith(ERROR_PAGE))
      )
        await new Promise((resolve) => setTimeout(resolve, 25));
    }
    return {
      ok: false,
      status: 502,
      error: navigationFailure(address, error),
    };
  };

  /** Open a tab (at an address, or blank), bring one to the front, or close one. */
  async function tabs(
    browser: TabbedBrowser,
    action: BrowserTabsRequest,
  ): Promise<BarOutcome> {
    if (action.action === "new") {
      const address =
        action.url === undefined ? undefined : addressFrom(action.url);
      if (action.url !== undefined && !address)
        return { ok: false, status: 400, error: NOT_AN_ADDRESS };
      const page = await browser.context.newPage();
      await putInFront(browser, page);
      if (address) {
        const since = commitsOf(page);
        try {
          await page.goto(address, {
            waitUntil: "domcontentloaded",
            timeout: options.navigationTimeoutMs,
          });
        } catch (error) {
          return navigation(page, since, address, error);
        }
      }
      return { ok: true };
    }
    const page = await pageFor(browser, action.tabId);
    if (!page)
      return { ok: false, status: 404, error: "There is no such tab." };
    if (action.action === "activate") {
      await putInFront(browser, page);
      return { ok: true };
    }
    // A browser on the screen always keeps a tab: the live screen needs a page to show, and the
    // person has no window of their own to close instead.
    if (browser.window === "screen" && browser.pages().length === 1)
      await putInFront(browser, await browser.context.newPage());
    await page.close();
    return { ok: true };
  }

  /** Go to an address in the tab in front, or back, forward, or reload it. */
  async function navigate(
    browser: TabbedBrowser,
    go: BrowserNavigateRequest,
  ): Promise<BarOutcome> {
    const page = browser.active();
    const waiting = {
      waitUntil: "domcontentloaded" as const,
      timeout: options.navigationTimeoutMs,
    };
    // The address the person asked for, or for back, forward and reload the page they were on.
    let address = await addressOf(browser, page);
    const since = commitsOf(page);
    try {
      if ("url" in go) {
        const asked = addressFrom(go.url);
        if (!asked) return { ok: false, status: 400, error: NOT_AN_ADDRESS };
        address = asked;
        await page.goto(asked, waiting);
      } else if (go.action === "reload") await page.reload(waiting);
      else {
        /*
         * Back and Forward wait only for the page to commit. A person's native browser keeps
         * the back and forward cache, which brings a page back whole without loading it again,
         * so it never fires the load events a fresh page does and waiting for one never ends.
         */
        const history = { ...waiting, waitUntil: "commit" as const };
        if (go.action === "back") await page.goBack(history);
        else await page.goForward(history);
      }
    } catch (error) {
      return navigation(page, since, address, error);
    }
    return { ok: true };
  }

  return { tabsOf, putInFront, tabs, navigate };
}

export type TabBar = ReturnType<typeof createTabBar>;