Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

15.8 KB
Raw Blame
import type { BotBrowserApiState } from "@openbot/shared/bot-browser";
import { IconLock, IconPlayerStop } from "@tabler/icons-react";
import { RECORDER_COLLECTS_SHORT } from "@openbot/shared/recorder-disclosure";
import { useMutation, useQuery } from "@tanstack/react-query";
import { type ReactNode, useId } from "react";
import {
  AddressBar,
  ClearBrowserDialog,
  latest,
  shownAddress,
  TabPanel,
  TabStrip,
  useClearBrowserData,
  useOpenedTab,
} from "@/components/browser/browser-bar";
import { NarrationRow } from "@/components/browser/narration";
import {
  type NarrationChoice,
  NarrationChoiceDialog,
  useNarrationChoice,
} from "@/components/browser/narration-choice";
import {
  narrationLanguage,
  RecordingMark,
  RecordingRow,
  StoppedRow,
  useRecordingWith,
} from "@/components/browser/recording";
import { useNarration } from "@/components/browser/use-narration";
import { PageRows } from "@/components/layout/page-shell";
import {
  Item,
  ItemContent,
  ItemDescription,
  ItemMedia,
  ItemTitle,
} from "@/components/ui/item";
import { Separator } from "@/components/ui/separator";
import { currentUserQueryOptions } from "@/lib/auth/queries";
import { useComputerControl } from "@/lib/computers/use-control";
import {
  botBrowserTabsMutationOptions,
  clearBotBrowserMutationOptions,
  navigateBotBrowserMutationOptions,
  sendBotBrowserNarration,
  startBotBrowserRecordingMutationOptions,
  stopBotBrowserRecordingMutationOptions,
} from "@/lib/bot-browser/mutations";
import { queryClient } from "@/query-client";

/*
 * A BOT'S BROWSER, WORKED BY A PERSON (`shared/bot-browser.ts`): the bar above the Bot's screen, and Record under it.
 *
 * Both are the person's own browser's, shared rather than copied (`components/browser`), so a Bot's browser reads as
 * the same browser the Browser page shows. What differs is the wheel: the bar and Record act only for the person who
 * holds it (`mine`, the server's verdict), and only that person is ever recorded.
 */

/**
 * Tabs, back, forward, reload and the address, above the Bot's screen (`children`), which is the tabs' panel with the
 * address. Live for the person holding the wheel; while the Bot (or someone else) holds it, it shows where the Bot is,
 * and nothing in it acts.
 *
 * Until the Bot's browser has first answered (`state` undefined) there is no bar, only the screen: and the screen keeps
 * its place in the tree when the bar arrives, so its live stream is not opened a second time.
 *
 * `onNavigated` is told when an address typed in the bar goes, so the keyboard can go back to the page, as Chrome's
 * address bar gives it to the page it opens (the person's own browser does the same, browser-screen.tsx).
 */
export function BotBrowserBar({
  botId,
  state,
  onNavigated,
  children,
}: {
  botId: string;
  state: BotBrowserApiState | undefined;
  onNavigated?: () => void;
  children: ReactNode;
}) {
  const tabs = useMutation(botBrowserTabsMutationOptions(queryClient, botId));
  const navigate = useMutation(
    navigateBotBrowserMutationOptions(queryClient, botId),
  );
  const clear = useMutation(clearBotBrowserMutationOptions(queryClient, botId));
  const clearing = useClearBrowserData(clear);
  const ids = useId();
  const active = state?.tabs.find((tab) => tab.active);
  const newTab = useOpenedTab(active?.id);
  const live = state ? state.mine && state.open : false;
  // A refusal belongs to the person who asked while they could. Once the wheel has gone, the bar is the Bot's and the
  // sentence about the person's last request is not news.
  const failure = live
    ? latest([
        { error: tabs.error, at: tabs.submittedAt },
        { error: navigate.error, at: navigate.submittedAt },
      ])
    : null;
  return (
    <>
      {state ? (
        <TabStrip
          ids={ids}
          tabs={state.tabs}
          disabled={!live}
          onActivate={(tabId) => tabs.mutate({ action: "activate", tabId })}
          onClose={(tabId, closed) =>
            tabs.mutate(
              { action: "close", tabId },
              closed ? { onSuccess: closed } : undefined,
            )
          }
          onNew={() =>
            tabs.mutate({ action: "new" }, { onSuccess: newTab.opened })
          }
        />
      ) : null}
      {/* Not min-h-0: the screen inside keeps its floor, and the window grows past it rather than clip it. */}
      <TabPanel
        active={active}
        className="flex flex-1 flex-col"
        ids={ids}
        tabs={state !== undefined}
      >
        {state ? (
          <AddressBar
            // A new tab, switching to one, or the wheel changing hands starts the bar again from the tab's own address:
            // an address half typed before the wheel went is not where the browser is when it comes back.
            key={`${active?.id ?? "none"}:${state.mine}`}
            address={shownAddress(active)}
            disabled={!state.open || !active}
            readOnly={!state.mine}
            focus={newTab.focus}
            onFocused={newTab.onFocused}
            onGo={(url) => {
              navigate.mutate({ url });
              // Away to the page, as Chrome's address bar does: the next keys are for the page that opens.
              onNavigated?.();
            }}
            onMove={(action) => navigate.mutate({ action })}
            // Only for the person holding the wheel, so the browser is never wiped under the Bot. Open or closed: its
            // data is there either way, and a running recording of theirs is ended by the clear itself.
            clear={{ disabled: !state.mine, ask: clearing.ask }}
          />
        ) : null}
        {failure ? (
          <p
            className="border-b border-border px-3 py-2 text-destructive text-sm"
            role="alert"
          >
            {failure.message}
          </p>
        ) : null}
        {children}
      </TabPanel>
      {/* Over the Bot's screen, which is a dialog itself. */}
      <ClearBrowserDialog
        clear={clear}
        open={clearing.asking}
        onOpenChange={clearing.setAsking}
        stacked
      />
    </>
  );
}

/**
 * THE PERSON'S RECORDINGS IN A BOT'S BROWSER, as this page saw them run, kept outside any component so closing and
 * opening the Bot's screen again does not forget them.
 *
 * Not who a recording belonged to: the computer says that (`lastStoppedBy`), so a reload says the same. What only this
 * page knows is how one it saw running ended: Stop pressed here, or the computer ending it as the wheel left the person,
 * which the panel says in so many words. One this page never saw running is reported plainly.
 */
type MyRecording = {
  title: string;
  /** Whether the person pressed Stop on it here, rather than the computer ending it as the wheel left them. */
  stoppedHere: boolean;
};
const MY_RECORDINGS = new Map<string, MyRecording>();
/** Per Bot: the person's recording seen running last, until a report names it or another recording starts. */
const LAST_SEEN_RUNNING = new Map<string, string>();
const recordingKey = (botId: string, recordingId: string) =>
  `${botId}\n${recordingId}`;

/** What the panel says when a recording of the person's ended without Stop, while they no longer hold the wheel. */
const ENDED_WITH_THE_WHEEL =
  "The recording ended when control went back to the Bot.";

/**
 * Record and Stop for the Bot's browser, for the person holding the wheel, from the Recorder's report and the server's
 * verdicts in `state`. Narration goes to the Bot's browser's own route.
 */
function useBotRecording(
  botId: string,
  state: BotBrowserApiState,
  choice: NarrationChoice,
  hooks: {
    beforeStop: () => Promise<void>;
    answered: (narrate: boolean) => void;
  },
) {
  const recorder = state.recorder;
  const running = recorder !== undefined && recorder.phase !== "idle";
  const runningId =
    running && state.recordingIsMine ? recorder.recordingId : undefined;
  /*
   * Written during render, as the computer view keeps the turns it has seen: idempotent, and it has to be in hand in
   * the same render that reads it, or the render after a recording ends would not know it was the person's.
   */
  if (runningId) {
    const key = recordingKey(botId, runningId);
    MY_RECORDINGS.set(key, {
      title: recorder?.title ?? "",
      stoppedHere: MY_RECORDINGS.get(key)?.stoppedHere ?? false,
    });
    LAST_SEEN_RUNNING.set(botId, runningId);
  } else if (running) {
    // Someone else's recording: whatever the person's last one was, this is newer.
    LAST_SEEN_RUNNING.delete(botId);
  }

  // The same store the wheel's own button reads, so this says what the row above it says.
  const { someoneElse } = useComputerControl(botId);
  const recording = useRecordingWith({
    recorder,
    waiting: !state.mine
      ? someoneElse
        ? "Someone else has control."
        : "Take control to record."
      : !state.open
        ? "The Bot's browser isn't open."
        : null,
    start: startBotBrowserRecordingMutationOptions(queryClient, botId),
    stop: stopBotBrowserRecordingMutationOptions(queryClient, botId),
    choice,
    ...hooks,
  });

  const last = recording.stopped;
  /*
   * The person's own, by the computer's word, and never a guess: a person not known yet is told nothing rather than
   * told another person's recording went to their Learning space.
   */
  const me = useQuery(currentUserQueryOptions()).data?.id;
  const mine =
    last !== undefined && me !== undefined && state.lastStoppedBy === me;
  const seenEnding = last
    ? MY_RECORDINGS.get(recordingKey(botId, last.recordingId))
    : undefined;
  if (last && LAST_SEEN_RUNNING.get(botId) === last.recordingId)
    LAST_SEEN_RUNNING.delete(botId);
  const seen = running ? undefined : LAST_SEEN_RUNNING.get(botId);
  /**
   * A recording of the person's that ended with no report of it: the Bot's browser restarted under it, and the
   * Recorder that reports now never saw it. It still ended, which is what the person is told.
   */
  const unreported = seen
    ? MY_RECORDINGS.get(recordingKey(botId, seen))
    : undefined;

  return {
    ...recording,
    /** Another person's recording is running: the panel says only that someone is recording. */
    theirs: running && !state.recordingIsMine,
    /** The person's recording Stop ended last, or the computer did as the wheel left them, with what it held. */
    stopped: mine ? last : undefined,
    /** The computer ended it as the wheel left them, while this page watched it run. */
    endedWithoutStop: seenEnding ? !seenEnding.stoppedHere : false,
    unreported,
    /*
     * A Stop the server refused once the recording had already ended (the wheel went back as it was pressed) is not
     * a failure: the panel says the recording ended. Only a refusal while it is still running is shown.
     */
    error:
      recording.startError ?? (runningId ? recording.stopError : null) ?? null,
    async stop() {
      if (runningId) {
        const key = recordingKey(botId, runningId);
        const known = MY_RECORDINGS.get(key);
        if (known) MY_RECORDINGS.set(key, { ...known, stoppedHere: true });
      }
      await recording.stop();
    },
  };
}

/**
 * Record under the Bot's screen: name the task, Record, do it, Stop. Only while the person holds the wheel, so only
 * they are recorded, never the Bot; the recording goes to their own Learning space, as the Recorder's do from any
 * browser.
 *
 * Narration is the person's own choice, the one their account keeps for every browser they record in (the Browser page
 * asks the same question and reads the same answer): asked before their first recording, the microphone closed until
 * they say yes, and their speech turned into text on their own computer.
 */
export function BotRecordPanel(props: {
  botId: string;
  state: BotBrowserApiState;
}) {
  // Where the Bots share one computer, so another Bot could read the person's Recorder connection off the disk.
  if (props.state.recordRefused)
    return (
      <section aria-label="Record a task">
        <h2 className="font-semibold text-sm">Record a task</h2>
        <PageRows className="mt-3">
          <Item size="sm" role="status">
            <ItemMedia variant="icon">
              <IconLock className="text-muted-foreground" />
            </ItemMedia>
            <ItemContent>
              <ItemTitle>Recording isn't available here</ItemTitle>
              <ItemDescription className="line-clamp-none">
                {props.state.recordRefused}
              </ItemDescription>
            </ItemContent>
          </Item>
        </PageRows>
      </section>
    );
  return <RecordPanel {...props} />;
}

function RecordPanel({
  botId,
  state,
}: {
  botId: string;
  state: BotBrowserApiState;
}) {
  const language = narrationLanguage();
  const listening =
    state.recordingIsMine && state.recorder?.phase === "recording";
  const choice = useNarrationChoice();
  const narration = useNarration({
    listening,
    language,
    chosen: choice.narration === "on",
    send: (post) => sendBotBrowserNarration(botId, post),
  });
  const recording = useBotRecording(botId, state, choice, {
    beforeStop: narration.finish,
    // A yes on the narration question is the person's click: narration turns on with it, asking the browser for its
    // speech recognition on their computer while the click still counts.
    answered: (narrate) => {
      if (narrate) narration.setOn(true);
    },
  });
  /** Why a recording of the person's ended that Stop did not end: the wheel going back, as a rule. */
  const ended = state.mine ? "The recording ended." : ENDED_WITH_THE_WHEEL;
  return (
    <section aria-label="Record a task">
      <h2 className="font-semibold text-sm">Record a task</h2>
      <p className="mt-1 text-pretty text-muted-foreground text-sm">
        Do the task in the Bot's browser while you have control. Only what you
        do is recorded, and it goes to your own Learning space.{" "}
        {RECORDER_COLLECTS_SHORT}
      </p>
      <PageRows className="mt-3">
        {recording.theirs ? (
          <Item size="sm">
            <ItemMedia variant="icon">
              <RecordingMark live={false} />
            </ItemMedia>
            <ItemContent>
              <ItemTitle>Someone is recording</ItemTitle>
              <ItemDescription>You can record once they stop.</ItemDescription>
            </ItemContent>
          </Item>
        ) : (
          <RecordingRow
            recording={recording}
            placeholder="Name the task to record"
          />
        )}
        {/* Narration is for the person who can record: their own microphone, while they hold the wheel. */}
        {state.mine && !recording.theirs ? (
          <>
            <Separator />
            <NarrationRow
              narration={narration}
              choice={choice.narration}
              language={language}
              recording={listening}
              onChange={(on) => {
                narration.setOn(on);
                choice.choose(on);
              }}
            />
          </>
        ) : null}
        {recording.unreported ? (
          <>
            <Separator />
            <Item size="sm" role="status">
              <ItemMedia variant="icon">
                <IconPlayerStop className="text-muted-foreground" />
              </ItemMedia>
              <ItemContent>
                <ItemTitle>{recording.unreported.title}</ItemTitle>
                <ItemDescription className="line-clamp-none">
                  {ended}
                </ItemDescription>
              </ItemContent>
            </Item>
          </>
        ) : recording.stopped ? (
          <>
            <Separator />
            <StoppedRow
              stopped={recording.stopped}
              ended={recording.endedWithoutStop ? ended : undefined}
            />
          </>
        ) : null}
      </PageRows>
      {recording.error ? (
        <p className="mt-2 text-destructive text-sm" role="alert">
          {recording.error.message}
        </p>
      ) : null}
      <NarrationChoiceDialog {...recording.question} />
    </section>
  );
}