Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

11.3 KB
Raw Blame
import { installOnDeviceSpeech } from "@openbot/shared/on-device-speech";
import type { LearningSpaceState } from "@openbot/shared/recorder-api";
import { IconAlertTriangle } from "@tabler/icons-react";
import { useCallback, useEffect, useState } from "react";
import { Button } from "@/components/ui/button";
import type {
  PageResponse,
  PageStateResponse,
  PageToBackground,
} from "../../protocol/messages";
import { narrationLanguage } from "../../shared/speech-language";
import { Page } from "../ui/page";
import { ask as askBackground } from "../ui/runtime";
import { desktopSentence } from "./desktop-sentence";
import { Notices } from "./notices";
import { RecordForm } from "./record-form";
import {
  AfterStop,
  afterStopLearningSentence,
  RecordingView,
} from "./recording-view";
import {
  microphoneState as defaultMicrophoneState,
  pageLanguage as defaultPageLanguage,
} from "./voice";

type PopupDeps = {
  ask: typeof askBackground;
  activeTabId: () => Promise<number>;
  pageLanguage: (tabId: number) => Promise<string | null>;
  browserLanguage: () => string;
  microphoneState: () => Promise<PermissionState | null>;
  /** The browser's speech recognition, which downloads a language for on-device recognition on the person's click. */
  Recognition: unknown;
};

/** The active tab; in the e2e variant only, the tab named by RECORDER_E2E_TARGET_TAB (Playwright cannot click the toolbar). */
async function activeTabId(): Promise<number> {
  if (import.meta.env.RECORDER_VARIANT === "e2e") {
    const target = Number(
      new URLSearchParams(location.search).get("RECORDER_E2E_TARGET_TAB"),
    );
    if (Number.isInteger(target) && target > 0) return target;
  }
  const [tab] = await chrome.tabs.query({
    active: true,
    currentWindow: true,
  });
  if (tab?.id === undefined) throw new Error("No active tab");
  return tab.id;
}

const openTab = (url: string) => {
  void chrome.tabs.create({ url });
};

/**
 * The toolbar popup. Signed out: how to connect, plus any recordings kept in this browser (with Discard). Connected:
 * the title and Record (every recording goes into the person's own Learning space); while recording, time, actions,
 * the capture rows, Stop and Start over; after Stop, that the recording went to Intelligence and into their Learning
 * space, and what is still sending or kept.
 */
export function Popup({ deps }: { deps?: Partial<PopupDeps> }) {
  const ask = deps?.ask ?? askBackground;
  const tabId = deps?.activeTabId ?? activeTabId;
  const readPageLanguage = deps?.pageLanguage ?? defaultPageLanguage;
  const browserLanguage = deps?.browserLanguage ?? (() => navigator.language);
  const [readMicrophone] = useState(
    () => deps?.microphoneState ?? defaultMicrophoneState,
  );
  const Recognition =
    deps && "Recognition" in deps
      ? deps.Recognition
      : (globalThis as { SpeechRecognition?: unknown }).SpeechRecognition;
  /** On the person's click, while it still counts: the browser downloads (or unmasks) a language for recognition on
   * their own computer only on one (@openbot/shared/on-device-speech). Not waited for: the Voice row says while it
   * downloads, and voice starts once it is there. */
  const getSpeech = (language: string) =>
    void installOnDeviceSpeech(Recognition, language);
  const [view, setView] = useState<PageStateResponse | null>(null);
  const [learning, setLearning] = useState<LearningSpaceState | null>(null);
  const [desktop, setDesktop] = useState(false);
  const [busy, setBusy] = useState(false);
  const [error, setError] = useState<string | null>(null);
  const [microphone, setMicrophone] = useState<PermissionState | null>(null);

  const refresh = useCallback(async () => {
    const answer = await ask<PageResponse>({ type: "page.state" });
    if (!answer.ok) {
      setError(answer.error);
      return;
    }
    if (!("state" in answer)) return;
    setView(answer);
    if (answer.connection) {
      const space = await ask<PageResponse>({ type: "page.learning" });
      if (space.ok && "learning" in space) setLearning(space.learning);
      else if (!space.ok) setError(space.error);
    } else {
      const probe = await ask<PageResponse>({
        type: "page.desktop.available",
      });
      setDesktop(probe.ok && "available" in probe && probe.available);
    }
  }, [ask]);

  useEffect(() => {
    refresh();
  }, [refresh]);

  // While a recording runs or the last of one is still sending, the state keeps changing under the popup.
  const live =
    view !== null && (view.state.phase !== "idle" || view.state.draining > 0);
  useEffect(() => {
    if (!live) return;
    let alive = true;
    const timer = setInterval(async () => {
      const answer = await ask<PageResponse>({ type: "page.state" });
      if (alive && answer.ok && "state" in answer) setView(answer);
    }, 1000);
    return () => {
      alive = false;
      clearInterval(timer);
    };
  }, [live, ask]);

  // Read when the popup opens: the Record form offers the setup page when the microphone is not allowed.
  useEffect(() => {
    let mounted = true;
    readMicrophone().then((state) => {
      if (mounted) setMicrophone(state);
    });
    return () => {
      mounted = false;
    };
  }, [readMicrophone]);

  const dismiss = async () => {
    await ask({ type: "page.notice.dismiss" });
    await refresh();
  };

  const act = async (message: PageToBackground) => {
    setBusy(true);
    setError(null);
    try {
      const answer = await ask(message);
      if (!answer.ok) setError(answer.error);
      await refresh();
    } catch (cause) {
      setError(cause instanceof Error ? cause.message : String(cause));
    } finally {
      setBusy(false);
    }
  };
  const discardKept = () => act({ type: "page.kept.discard" });

  if (!view) {
    return (
      <Page title="OpenBot Recorder" width="popup">
        {error ? <ErrorLine message={error} /> : null}
      </Page>
    );
  }

  const { connection, policy, state } = view;

  if (!connection) {
    const openAddress = !policy.invalid ? policy.openbotUrl : null;
    return (
      <Page
        title="Connect the Recorder to OpenBot"
        description="Open OpenBot, go to Settings, Browsers, and choose Connect this browser."
        width="popup"
      >
        <Notices notice={view.notice} kept={state.kept} onDismiss={dismiss} />
        {/* Signed out with recordings kept for the person (Decision 19): how to send them, or Discard. */}
        <AfterStop state={state} connected={false} onDiscard={discardKept} />
        {policy.invalid ? (
          <p className="mt-4 text-muted-foreground text-sm">
            Your organization's Recorder policy names an OpenBot address the
            Recorder cannot use. Ask your IT team.
          </p>
        ) : null}
        <div className="mt-4 flex flex-col gap-2">
          {openAddress ? (
            <Button
              size="lg"
              onClick={() => {
                const origin = new URL(openAddress);
                openTab(`${origin.origin}/settings/browsers`);
              }}
            >
              Open {new URL(openAddress).host}
            </Button>
          ) : null}
          {desktop ? (
            <Button
              size="lg"
              variant="secondary"
              onClick={async () => {
                setError(null);
                const answer = await ask<PageResponse>({
                  type: "page.connect.desktop",
                });
                // A good ping does not promise a good handoff: OpenBot Desktop says why it could not connect. Connected,
                // the refresh shows the Record form; there is no Connect page to confirm.
                if (!answer.ok) setError(answer.error);
                else if ("connected" in answer && !answer.connected)
                  setError(desktopSentence(answer.reason));
                await refresh();
              }}
            >
              Connect with OpenBot Desktop
            </Button>
          ) : null}
        </div>
        {error ? <ErrorLine message={error} /> : null}
      </Page>
    );
  }

  // Records at once: website access came with the install, and a recording without the microphone goes on (its Voice
  // row says so and offers the setup page). The popup never opens a tab or asks for a permission here.
  const record = async (title: string) => {
    setBusy(true);
    setError(null);
    try {
      const tab = await tabId();
      const language = narrationLanguage(
        await readPageLanguage(tab),
        browserLanguage(),
      );
      if (view.narration === "on") getSpeech(language);
      const answer = await ask({
        type: "page.record",
        title,
        tabId: tab,
        language,
      });
      if (!answer.ok) setError(answer.error);
      await refresh();
    } catch (cause) {
      setError(cause instanceof Error ? cause.message : String(cause));
    } finally {
      setBusy(false);
    }
  };

  // Only ever on the person's click: the setup page asks for the microphone on a tab they can see.
  const allowMicrophone = () => openTab(chrome.runtime.getURL("setup.html"));

  /** The person's choice to narrate, kept for every recording. On asks for the speech recognition of their own language
   * on this click, and for the microphone where the browser has not allowed it (on the setup page: a popup cannot show
   * the browser's prompt). */
  const chooseNarration = async (on: boolean) => {
    if (on) getSpeech(narrationLanguage(null, browserLanguage()));
    await act({ type: "page.narration", on });
    if (on && microphone !== "granted") allowMicrophone();
  };

  const shownError = error ?? state.error ?? null;
  return (
    <Page
      title={connection.user.name}
      description={new URL(connection.openbotUrl).host}
      width="popup"
    >
      <Notices notice={view.notice} kept={state.kept} onDismiss={dismiss} />
      {state.phase === "idle" ? (
        <AfterStop state={state} connected onDiscard={discardKept} />
      ) : (
        <RecordingView
          state={state}
          busy={busy}
          actions={{
            onStop: () => act({ type: "page.stop" }),
            onRestart: () => act({ type: "page.restart" }),
            onNotice: (choice) => act({ type: "page.notice", choice }),
            voice: {
              onMute: (muted) => act({ type: "page.voice", muted }),
              onAllowMicrophone: allowMicrophone,
              // The language the browser has to download, on this click, then voice on: it starts once it is there.
              onDownload: (language) => {
                getSpeech(language);
                void act({ type: "page.voice", muted: false });
              },
            },
          }}
        />
      )}
      {state.phase === "idle" ? (
        <RecordForm
          learning={learning}
          busy={busy}
          onRecord={record}
          learningShown={afterStopLearningSentence(state, true)}
          microphoneBlocked={microphone === "denied" || microphone === "prompt"}
          onAllowMicrophone={allowMicrophone}
          narration={view.narration}
          onNarration={(on) => void chooseNarration(on)}
        />
      ) : null}
      {shownError ? <ErrorLine message={shownError} /> : null}
    </Page>
  );
}

function ErrorLine({ message }: { message: string }) {
  return (
    <p
      className="mt-4 flex items-start gap-1.5 text-destructive text-sm"
      role="alert"
    >
      <IconAlertTriangle className="mt-0.5 size-4 shrink-0" />
      {message}
    </p>
  );
}