Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

10.3 KB
Raw Blame
import type { LearningSpaceState } from "@openbot/shared/recorder-api";
import {
  IconMicrophone,
  IconMicrophoneOff,
  IconNetwork,
  IconSchool,
} from "@tabler/icons-react";
import { PageRows } from "@/components/layout/page-parts";
import { Button } from "@/components/ui/button";
import {
  Item,
  ItemActions,
  ItemContent,
  ItemDescription,
  ItemFooter,
  ItemMedia,
  ItemTitle,
} from "@/components/ui/item";
import { Separator } from "@/components/ui/separator";
import { Switch } from "@/components/ui/switch";
import type {
  CaptureStatus,
  RecorderState,
  VoiceCapture,
} from "../../protocol/messages";
import { learningSentence } from "../popup/record-form";

const LEADING_STATUS = /^Learning capture is (?:paused|off):\s*/;
const HELD_LINE =
  "Your events are kept in this browser and sent when Intelligence accepts them; it tries again every 5 minutes.";
const PARKED_LINE =
  "Your events are kept in this browser. The Recorder tries again when the browser next starts.";

/** A Task 11 reason without its leading "Learning capture is paused/off:", which the row's title already says, as a
 * sentence of its own (the table's reasons go on in lower case after that phrase). */
export function stripCaptureStatus(reason: string): string {
  const line = reason.replace(LEADING_STATUS, "");
  return line.charAt(0).toUpperCase() + line.slice(1);
}

const NOT_IN_A_SPACE = new Set<LearningSpaceState["state"]>([
  "unassigned",
  "missing",
]);

const INTELLIGENCE_STATUS: Record<CaptureStatus["intelligence"], string> = {
  off: "Off",
  connecting: "Connecting to Intelligence",
  streaming: "Streaming to Intelligence",
  retrying: "Retrying",
  held: "Paused",
  parked: "Off",
};

function plural(count: number, one: string, many: string) {
  return `${count} ${count === 1 ? one : many}`;
}

function describe(status: string, reason: string | undefined) {
  return reason ? `${status}. ${reason}` : status;
}

/** The Learning capture line: a reason that already says "Retrying" stands alone, never "Retrying. ... Retrying; ...". */
function describeIntelligence(
  intelligence: CaptureStatus["intelligence"],
  reason: string | undefined,
) {
  return intelligence === "retrying" && reason && /\bRetrying\b/.test(reason)
    ? reason
    : describe(INTELLIGENCE_STATUS[intelligence], reason);
}

type RowText = { title: string; lines: string[] };

/** A stop that keeps every event: the status as the title, then why, then when it tries again. */
function keptText(parked: boolean, reason: string | undefined): RowText {
  return {
    title: parked ? "Learning capture is off" : "Learning capture is paused",
    lines: [
      ...(reason ? [stripCaptureStatus(reason)] : []),
      parked ? PARKED_LINE : HELD_LINE,
    ],
  };
}

/** What the row says about the recording's Learning space, after the capture status, so neither hides the other. */
function spaceLines(learning: LearningSpaceState | undefined): string[] {
  const sentence = learning ? learningSentence(learning) : null;
  return [
    ...(learning && NOT_IN_A_SPACE.has(learning.state)
      ? ["Not into a Learning space"]
      : []),
    ...(sentence ? [sentence] : []),
  ];
}

/** Learning capture's title and description lines for a running recording's capture status and Learning space. */
function captureText(
  capture: CaptureStatus,
  learning: LearningSpaceState | undefined,
): RowText {
  const status =
    capture.intelligence === "held" || capture.intelligence === "parked"
      ? keptText(capture.intelligence === "parked", capture.intelligenceReason)
      : {
          title: "Learning capture",
          lines: [
            describeIntelligence(
              capture.intelligence,
              capture.intelligenceReason &&
                stripCaptureStatus(capture.intelligenceReason),
            ),
          ],
        };
  const { rejected = 0, dropped = 0 } = capture.events ?? {};
  return {
    title: status.title,
    lines: [
      ...status.lines,
      ...spaceLines(learning),
      ...(rejected > 0
        ? [
            `${plural(rejected, "event was", "events were")} refused by Intelligence`,
          ]
        : []),
      ...(dropped > 0
        ? [`${plural(dropped, "event", "events")} could not be sent`]
        : []),
    ],
  };
}

function LearningRow({ title, lines }: RowText) {
  return (
    <Item size="sm">
      <ItemMedia variant="icon">
        <IconSchool />
      </ItemMedia>
      <ItemContent>
        <ItemTitle>{title}</ItemTitle>
        {lines.map((line) => (
          <ItemDescription key={line} className="line-clamp-none">
            {line}
          </ItemDescription>
        ))}
      </ItemContent>
    </Item>
  );
}

/** The Learning capture row of a running recording. */
function LearningCaptureRow({
  capture,
  learning,
}: {
  capture: CaptureStatus;
  learning?: LearningSpaceState;
}) {
  return <LearningRow {...captureText(capture, learning)} />;
}

/**
 * The Learning capture row after Stop, for events kept by a stop (held or parked). A finished recording's capture status
 * is back to idle (the service worker ignores the document's status once nothing records), so this reads the drains
 * instead; without kept events it says the rest is still sending.
 */
export function SendingLearningRow({
  kept,
  learning,
}: {
  kept: RecorderState["kept"];
  learning?: LearningSpaceState;
}) {
  const status = kept
    ? keptText(kept.parked, kept.reason)
    : {
        title: "Learning capture",
        lines: ["Sending the rest of this recording to Intelligence"],
      };
  return (
    <LearningRow
      title={status.title}
      lines={[...status.lines, ...spaceLines(learning)]}
    />
  );
}

const NETWORK_STATUS: Record<CaptureStatus["network"], string> = {
  off: "Off",
  waiting: "Waiting for the page",
  debugger: "Capturing what each step saves",
  page: "Capturing what each step saves, from inside the page",
  stopped: "Stopped",
};
export type VoiceActions = {
  /** The switch: true mutes the narration, false turns it on again (also the way to try again after a failure). */
  onMute: (muted: boolean) => void;
  /** Opens the setup page, offered when the browser has not let the Recorder use the microphone. */
  onAllowMicrophone: () => void;
  /** Downloads the language the browser has to have on the computer before it can turn speech into text there, on the
   * person's click, and turns voice on: it starts once the language is there. */
  onDownload?: (language: string) => void;
};

/** What the Voice row says under its title: that it listens, or why it does not. */
function voiceLine(voice: VoiceCapture): string {
  switch (voice.state) {
    case "starting":
      // While the browser downloads the language, why voice has not started yet.
      return voice.reason ?? "Starting the microphone";
    case "listening":
      return "Listening";
    case "muted":
      return "Muted. Turn it on to narrate what you do.";
    case "failed":
      return voice.reason ?? "Voice stopped. Recording goes on without it.";
  }
}

/** The person's narration: listening (with the last words heard), muted, or why it stopped, and the switch that mutes
 * it. It sits with the recording's title, above the capture rows, so the live state and the words stay in view in
 * Chrome's 600px popup. */
export function VoiceRow({
  voice,
  actions,
}: {
  voice: VoiceCapture;
  actions?: VoiceActions;
}) {
  const on = voice.state === "starting" || voice.state === "listening";
  const listening = voice.state === "listening";
  return (
    <Item size="sm">
      <ItemMedia variant="icon">
        {on ? <IconMicrophone /> : <IconMicrophoneOff />}
      </ItemMedia>
      <ItemContent>
        <ItemTitle>Voice</ItemTitle>
        <ItemDescription className="line-clamp-none">
          {listening ? (
            <span className="inline-flex items-center gap-1.5">
              {/* Decorative: the word beside it says it is live. */}
              <span
                aria-hidden="true"
                className="size-1.5 shrink-0 animate-pulse rounded-full bg-emerald-500"
              />
              {voiceLine(voice)}
            </span>
          ) : (
            voiceLine(voice)
          )}
        </ItemDescription>
        {/* Still listening, but the browser hands back no words for what it hears. */}
        {listening && voice.reason ? (
          <ItemDescription className="line-clamp-none">
            {voice.reason}
          </ItemDescription>
        ) : null}
        {listening && voice.heard ? (
          <ItemDescription className="text-foreground" aria-live="polite">
            {`“${voice.heard}”`}
          </ItemDescription>
        ) : null}
      </ItemContent>
      {actions ? (
        <ItemActions>
          {/* Binary and immediate: the microphone opens or closes when switched. */}
          <Switch
            aria-label="Voice"
            checked={on}
            onCheckedChange={(checked) => actions.onMute(!checked)}
          />
        </ItemActions>
      ) : null}
      {actions && voice.problem === "microphone" ? (
        <ItemFooter className="justify-end">
          <Button size="sm" variant="ghost" onClick={actions.onAllowMicrophone}>
            Allow microphone
          </Button>
        </ItemFooter>
      ) : null}
      {actions?.onDownload &&
      voice.problem === "speech-pack" &&
      voice.language ? (
        <ItemFooter className="justify-end">
          <Button
            size="sm"
            variant="ghost"
            onClick={() =>
              voice.language && actions.onDownload?.(voice.language)
            }
          >
            Download
          </Button>
        </ItemFooter>
      ) : null}
    </Item>
  );
}

/** Learning capture and Network: what the Recorder is keeping, and why anything is off. */
export function CaptureRows({
  capture,
  learning,
}: {
  capture: CaptureStatus;
  /** The recording's Learning space, so "streaming" never hides one that could not be confirmed. */
  learning?: LearningSpaceState;
}) {
  return (
    <PageRows>
      <LearningCaptureRow capture={capture} learning={learning} />
      <Separator />
      <Item size="sm">
        <ItemMedia variant="icon">
          <IconNetwork />
        </ItemMedia>
        <ItemContent>
          <ItemTitle>Network</ItemTitle>
          <ItemDescription className="line-clamp-none">
            {describe(NETWORK_STATUS[capture.network], capture.networkReason)}
          </ItemDescription>
        </ItemContent>
      </Item>
    </PageRows>
  );
}