Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

17.5 KB
Raw Blame
/**
 * The Recorder's launcher API, called inside its service worker over this process's DevTools pipe.
 *
 * WHY THROUGH THE WORKER. The Recorder takes its connection and its commands only from whoever
 * launched the browser: `globalThis.openbotLauncher` in its service worker, with no message route to
 * it from a web page or another extension. Only this process holds the browser's DevTools pipe, so
 * only this process can call it.
 *
 * FOUND BY RAW TARGETS. Playwright's `serviceWorkers()` did not report the Recorder's worker after a
 * DevTools `Extensions.loadUnpacked` in branded Chrome, so the worker is found the way DevTools
 * itself finds one: `Target.getTargets`, a `service_worker` whose url is the Recorder's.
 *
 * ATTACHED WITHOUT FLATTENING. Playwright's public DevTools sessions attach only to pages and frames,
 * so a session to the worker is opened from the browser session with `Target.attachToTarget` and
 * spoken to with `Target.sendMessageToTarget`. The protocol marks that non-flat mode as one it means
 * to retire in favour of flat sessions (crbug.com/991325), which Playwright does not expose for a
 * worker; `person-browser.test.ts` drives it against real Chromium and Chrome, so a browser that
 * drops it is caught there.
 *
 * WOKEN WHEN ASLEEP. An MV3 service worker stops after about thirty seconds with nothing to do, and
 * a stopped worker has no target to attach to. `ServiceWorker.startWorker` with the extension's scope
 * starts it again; the ServiceWorker domain answers only on a page session, so it is asked through
 * one of the browser's pages. A worker that has just started can be evaluated in before its script
 * has run (crbug.com/341213355), so every call first waits until the launcher is defined.
 * https://chromedevtools.github.io/devtools-protocol/tot/ServiceWorker/#method-startWorker
 * https://developer.chrome.com/docs/extensions/develop/concepts/service-workers/lifecycle
 */
import type { BrowserContext, CDPSession, Page } from "playwright";
import {
  LAUNCHER_GLOBAL,
  type LauncherApi,
  type LauncherResult,
  type LauncherState,
} from "../../shared/openbot-browser";

/** The Recorder did not answer: not loaded, not started, or without the launcher. */
export class RecorderUnavailableError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "RecorderUnavailableError";
  }
}

/**
 * How long ending a recording waits for the last of it to reach Intelligence before the Recorder is let go all the
 * same (`untilSent`): a rejoin and a push, each given the gateway's own timeout (`PHOENIX_TIMEOUT_MS`, 10 s, in
 * `extension/src/offscreen/phoenix-session.ts`; `bot-browser.test.ts` keeps the two in step). Well inside the 45 s
 * the server gives a call to the computer, so Stop, a hand-back and a clear still answer, and the server revokes the
 * person's connection only after.
 */
export const SENDING_WAIT_MS = 2 * 10_000;
/** How often the Recorder is asked whether the last of a recording has gone (`untilSent`). */
const SENDING_POLL_MS = 250;

/**
 * Wait, after a Stop, until the Recorder says the last of the recording it stopped has reached Intelligence
 * (`lastStopped.sending` gone), or `waitMs` runs out. Answers its newest report, and whether it was still sending when
 * the wait ran out.
 *
 * WHY. Stop answers once the recording's outcome is queued; the rest goes to Intelligence after, and until it has gone
 * it is kept in the Recorder's own storage. Whoever lets the Recorder go after a Stop waits on this first: a disconnect
 * keeps what is not sent for the person's next connection in that browser, and clearing the browser deletes that
 * storage outright.
 */
export async function untilSent(
  launcher: Pick<LauncherApi, "state">,
  report: LauncherState | undefined,
  waitMs: number,
): Promise<{ report: LauncherState | undefined; stillSending: boolean }> {
  const recordingId = report?.lastStopped?.recordingId;
  if (!recordingId || !report?.lastStopped?.sending)
    return { report, stillSending: false };
  const deadline = Date.now() + waitMs;
  let latest = report;
  while (Date.now() < deadline) {
    await new Promise((resolve) => setTimeout(resolve, SENDING_POLL_MS));
    // A Recorder too busy to answer is asked again, within the bound.
    const now = await launcher.state().catch(() => undefined);
    if (!now) continue;
    latest = now;
    if (
      now.lastStopped?.recordingId !== recordingId ||
      !now.lastStopped.sending
    )
      return { report: now, stillSending: false };
  }
  return { report: latest, stillSending: true };
}

/** How long a worker that is starting gets to appear and define its launcher. */
const WORKER_START_MS = 10_000;
/** How long one launcher call may take. Starting a recording attaches the debugger to a tab. */
const CALL_MS = 15_000;
/** How long the question "is the launcher there" may take before the session is replaced. */
const READY_MS = 2_000;
const POLL_MS = 100;

type Pending = {
  resolve: (value: unknown) => void;
  reject: (error: Error) => void;
};

type TargetInfo = { targetId: string; type: string; url: string };

/** A DevTools event from inside the worker. */
type WorkerEvent = { method?: string; params?: unknown };

const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));

const WORKER_STOPPED = "The Recorder's worker stopped mid-call.";

export type RecorderLauncher = LauncherApi & {
  /**
   * The source of the script the Recorder's worker is running now, as its JavaScript engine parsed
   * it, or undefined when the worker reports no script at `url`. See `keepRecorderCurrent` in
   * `person-browser.ts` for why it is asked.
   */
  workerSource(url: string): Promise<string | undefined>;
};

export function createRecorderLauncher(input: {
  /** A browser-level session (`browser.newBrowserCDPSession()`), which alone can reach the worker. */
  session: CDPSession;
  context: BrowserContext;
  recorderId: string;
}): RecorderLauncher {
  const { session, context, recorderId } = input;
  const scope = `chrome-extension://${recorderId}/`;
  let attached: { targetId: string; sessionId: string } | undefined;
  let attaching: Promise<string> | undefined;
  let nextId = 0;
  const pending = new Map<number, Pending>();
  const listeners = new Set<(event: WorkerEvent) => void>();

  session.on("Target.receivedMessageFromTarget", (event) => {
    if (event.sessionId !== attached?.sessionId) return;
    let message: {
      id?: number;
      method?: string;
      params?: unknown;
      result?: unknown;
      error?: { message?: string };
    };
    try {
      message = JSON.parse(event.message);
    } catch {
      return;
    }
    if (message.id === undefined) {
      // An event, which only a caller that asked for one listens to.
      for (const listener of listeners) listener(message);
      return;
    }
    const waiting = pending.get(message.id);
    if (!waiting) return;
    pending.delete(message.id);
    if (message.error)
      waiting.reject(
        new RecorderUnavailableError(
          `The Recorder's worker refused a call: ${message.error.message ?? "unknown error"}`,
        ),
      );
    else waiting.resolve(message.result);
  });

  // A worker that stopped takes its session with it. The next call finds or wakes a new one.
  session.on("Target.detachedFromTarget", (event) => {
    if (event.sessionId !== attached?.sessionId) return;
    attached = undefined;
    for (const [id, waiting] of pending) {
      pending.delete(id);
      waiting.reject(new RecorderUnavailableError(WORKER_STOPPED));
    }
  });

  const findWorker = async (): Promise<TargetInfo | undefined> => {
    const { targetInfos } = (await session.send("Target.getTargets")) as {
      targetInfos: TargetInfo[];
    };
    return targetInfos.find(
      (target) =>
        target.type === "service_worker" && target.url.startsWith(scope),
    );
  };

  /** Ask the browser to start the Recorder's worker, through any page it has open. */
  const wake = async (): Promise<void> => {
    const page: Page | undefined = context
      .pages()
      .find((candidate) => !candidate.isClosed());
    if (!page) return;
    const pageSession = await context.newCDPSession(page);
    try {
      await pageSession.send("ServiceWorker.enable");
      await pageSession.send("ServiceWorker.startWorker", { scopeURL: scope });
    } finally {
      await pageSession.detach().catch(() => undefined);
    }
  };

  const workerTarget = async (): Promise<TargetInfo> => {
    const deadline = Date.now() + WORKER_START_MS;
    let lastWake = 0;
    for (;;) {
      const found = await findWorker();
      if (found) return found;
      if (Date.now() > deadline)
        throw new RecorderUnavailableError(
          "The Recorder in this browser did not start.",
        );
      // Right after a load the worker is starting on its own; asked again each second after that,
      // because a wake can land before the extension's registration is ready to start.
      if (Date.now() - lastWake >= 1_000) {
        lastWake = Date.now();
        await wake().catch(() => undefined);
      }
      await sleep(POLL_MS);
    }
  };

  /** Leave a session that no longer answers, so the next call attaches afresh. */
  const drop = () => {
    const stale = attached;
    attached = undefined;
    if (stale)
      void session
        .send("Target.detachFromTarget", { sessionId: stale.sessionId })
        .catch(() => undefined);
  };

  /*
   * A session on the Recorder's running worker, found afresh for every call. A stopped service
   * worker does not end a session attached to it this way (no detach arrives) and a message sent to
   * it then is never answered, so whether the worker is running is asked of the browser each time,
   * and a worker that was gone, even briefly, gets a new session.
   */
  const attach = async (): Promise<string> => {
    const running = await findWorker();
    if (running && attached?.targetId === running.targetId)
      return attached.sessionId;
    drop();
    const target = running ?? (await workerTarget());
    const { sessionId } = (await session.send("Target.attachToTarget", {
      targetId: target.targetId,
      flatten: false,
    })) as { sessionId: string };
    attached = { targetId: target.targetId, sessionId };
    // A worker that started while something held it at its first line runs on from here. Nothing
    // waits on the answer, which the listener above ignores.
    await session.send("Target.sendMessageToTarget", {
      sessionId,
      message: JSON.stringify({
        id: ++nextId,
        method: "Runtime.runIfWaitingForDebugger",
      }),
    });
    return sessionId;
  };

  const sessionId = async (): Promise<string> => {
    attaching ??= attach().finally(() => {
      attaching = undefined;
    });
    return attaching;
  };

  /*
   * One message to the worker and its answer, within `timeoutMs` from start to finish: finding and
   * attaching to the worker, handing the message over, and the answer.
   *
   * NOTHING HERE MAY REJECT WITH NOBODY LISTENING. The timer used to start before the message was
   * handed over and was raced only after, so a browser slow to take the message fired it with
   * nothing awaiting it: an unhandled rejection, which ends a Bun process, and with it every
   * browser on this computer. Seen on a hosted computer while the Recorder recorded a burst of
   * input, when handing over a 2 s readiness check took longer than 2 s. So the whole exchange is
   * raced against the timer from the start, and the answer, which a stopped worker can reject while
   * the message is still being handed over, is marked as handled the moment it exists. Neither can
   * reject unheard, and the call ends in an answer or a refusal, never in the process ending.
   */
  const send = async (
    method: string,
    params: Record<string, unknown>,
    timeoutMs: number,
  ): Promise<unknown> => {
    const id = ++nextId;
    let over = false;
    let timer: ReturnType<typeof setTimeout> | undefined;
    const late = new Promise<never>((_, reject) => {
      timer = setTimeout(() => {
        reject(
          new RecorderUnavailableError("The Recorder did not answer in time."),
        );
      }, timeoutMs);
    });
    const exchange = async (): Promise<unknown> => {
      const target = await sessionId();
      // A call that has already been refused sends nothing: its answer would have nowhere to go.
      if (over) throw new RecorderUnavailableError(WORKER_STOPPED);
      const answer = new Promise<unknown>((resolve, reject) => {
        pending.set(id, { resolve, reject });
      });
      answer.catch(() => undefined);
      try {
        await session.send("Target.sendMessageToTarget", {
          sessionId: target,
          message: JSON.stringify({ id, method, params }),
        });
      } catch {
        // The session went with a worker that stopped before its detach was reported.
        if (attached?.sessionId === target) drop();
        throw new RecorderUnavailableError(WORKER_STOPPED);
      }
      return answer;
    };
    try {
      return await Promise.race([exchange(), late]);
    } finally {
      over = true;
      clearTimeout(timer);
      pending.delete(id);
    }
  };

  const evaluate = async (
    expression: string,
    timeoutMs: number,
  ): Promise<unknown> => {
    const result = (await send(
      "Runtime.evaluate",
      { expression, awaitPromise: true, returnByValue: true },
      timeoutMs,
    )) as {
      result?: { value?: unknown };
      exceptionDetails?: {
        text?: string;
        exception?: { description?: string };
      };
    };
    if (result.exceptionDetails)
      throw new RecorderUnavailableError(
        `The Recorder failed: ${result.exceptionDetails.exception?.description ?? result.exceptionDetails.text ?? "unknown error"}`,
      );
    return result.result?.value;
  };

  const launcher = `globalThis[${JSON.stringify(LAUNCHER_GLOBAL)}]`;

  /**
   * Wait for the launcher, which a worker that has only just started may not have defined yet. A
   * worker that does not answer at all in that time is attached to afresh on the next try.
   */
  const ready = async (): Promise<void> => {
    const deadline = Date.now() + WORKER_START_MS;
    for (;;) {
      const defined = await evaluate(
        `typeof ${launcher}?.state === "function"`,
        READY_MS,
      ).catch((error: unknown) => {
        if (Date.now() > deadline) throw error;
        drop();
        return false;
      });
      if (defined === true) return;
      if (Date.now() > deadline)
        throw new RecorderUnavailableError(
          "The Recorder in this browser does not offer its launcher, so OpenBot cannot connect it.",
        );
      await sleep(POLL_MS);
    }
  };

  const call = async (
    method: keyof LauncherApi,
    argument?: unknown,
    timeoutMs = CALL_MS,
  ): Promise<unknown> => {
    // Once more if the worker stopped between finding it and calling it.
    for (let attempt = 0; ; attempt += 1) {
      try {
        await ready();
        // The argument as JSON, which is a JavaScript expression of the same value.
        return await evaluate(
          `${launcher}.${method}(${argument === undefined ? "" : JSON.stringify(argument)})`,
          timeoutMs,
        );
      } catch (error) {
        if (
          attempt > 0 ||
          !(error instanceof RecorderUnavailableError) ||
          error.message !== WORKER_STOPPED
        )
          throw error;
      }
    }
  };

  const result = <T>(value: unknown): LauncherResult<T> => {
    if (value && typeof value === "object" && "ok" in value)
      return value as LauncherResult<T>;
    throw new RecorderUnavailableError(
      "The Recorder answered with something other than a launcher result.",
    );
  };

  return {
    async connect(argument) {
      return result(await call("connect", argument));
    },
    async record(argument) {
      return result<{ state: LauncherState }>(await call("record", argument));
    },
    async narrate(argument) {
      return result(await call("narrate", argument));
    },
    async speaking(argument) {
      return result(await call("speaking", argument));
    },
    async disconnect() {
      return result(await call("disconnect"));
    },
    /*
     * Read through the worker's own debugger: enabling it reports every script the worker has parsed,
     * each before the answer to the enable, and the source of the one asked for is then read by its
     * id. After `ready`, so the worker's script has run and is among them.
     */
    async workerSource(url) {
      await ready();
      const parsed = new Map<string, string>();
      const listener = (event: WorkerEvent) => {
        if (event.method !== "Debugger.scriptParsed") return;
        const script = event.params as { scriptId?: string; url?: string };
        if (script.url && script.scriptId)
          parsed.set(script.url, script.scriptId);
      };
      listeners.add(listener);
      try {
        await send("Debugger.enable", {}, CALL_MS);
        const scriptId = parsed.get(url);
        if (!scriptId) return undefined;
        const { scriptSource } = (await send(
          "Debugger.getScriptSource",
          { scriptId },
          CALL_MS,
        )) as { scriptSource?: string };
        return scriptSource;
      } finally {
        listeners.delete(listener);
        await send("Debugger.disable", {}, CALL_MS).catch(() => undefined);
      }
    },
    async state() {
      const value = await call("state", undefined, 5_000);
      if (!value || typeof value !== "object" || !("phase" in value))
        throw new RecorderUnavailableError(
          "The Recorder answered with something other than its state.",
        );
      return value as LauncherState;
    },
  };
}