Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

12.4 KB
Raw Blame
import type { Recognition } from "@/lib/browser/narration";
import {
  everyLanguage,
  type OnDeviceModel,
  withOnDevice,
} from "./on-device-speech.fixture";

/**
 * WHAT CHROME'S SPEECH RECOGNITION REALLY SENDS, recorded rather than imagined.
 *
 * Captured on 2026-10-21 from Google Chrome 156.0.8078.47 (headless, macOS) running `webkitSpeechRecognition` with
 * `continuous` and `interimResults` on, `lang` en-US, every event logged with its milliseconds since the page started
 * listening. The voice was macOS `say` reading "I am opening the expenses page. [2.5 s pause] Now I add the receipt
 * from the taxi ride.", played into the recognition as a MediaStreamTrack, so no microphone was involved. Each result
 * is [transcript, isFinal, confidence], as `results[i][0]` and `results[i].isFinal` gave them.
 *
 * TWO_PHRASES is the whole run. STOPPED_MID_PHRASE is the first sentence again with `stop()` called before Chrome had
 * finished it, which is what Stop does to a phrase in progress.
 */

export type ChromeEvent =
  | {
      type: "result";
      at: number;
      resultIndex: number;
      results: [transcript: string, isFinal: boolean, confidence: number][];
    }
  | { type: "error"; at: number; error: string }
  | { type: string; at: number };

export const TWO_PHRASES: ChromeEvent[] = [
  { type: "start", at: 210 },
  { type: "audiostart", at: 221 },
  { type: "soundstart", at: 861 },
  { type: "speechstart", at: 861 },
  {
    type: "result",
    at: 1161,
    resultIndex: 0,
    results: [["I am", false, 0.01]],
  },
  {
    type: "result",
    at: 1354,
    resultIndex: 0,
    results: [["I am open", false, 0.01]],
  },
  {
    type: "result",
    at: 1447,
    resultIndex: 0,
    results: [["I am opening", false, 0.01]],
  },
  {
    type: "result",
    at: 1567,
    resultIndex: 0,
    results: [["I am opening the", false, 0.01]],
  },
  {
    type: "result",
    at: 1690,
    resultIndex: 0,
    results: [
      ["I am", false, 0.9],
      [" opening the", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 1863,
    resultIndex: 0,
    results: [
      ["I am", false, 0.9],
      [" opening the EXP", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 2052,
    resultIndex: 0,
    results: [
      ["I am opening", false, 0.9],
      [" the EXP", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 2109,
    resultIndex: 0,
    results: [
      ["I am opening", false, 0.9],
      [" the expens", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 2198,
    resultIndex: 0,
    results: [
      ["I am opening the", false, 0.9],
      [" expens", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 2228,
    resultIndex: 0,
    results: [
      ["I am opening the", false, 0.9],
      [" expenses", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 2495,
    resultIndex: 0,
    results: [
      ["I am opening the", false, 0.9],
      [" expenses page", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 2812,
    resultIndex: 0,
    results: [
      ["I am opening the expenses", false, 0.9],
      [" page", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 3177,
    resultIndex: 0,
    results: [["I am opening the expenses page", true, 0.972]],
  },
  {
    type: "result",
    at: 5379,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 5677,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 5929,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I add", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 6060,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now", false, 0.9],
      [" I add", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 6091,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now", false, 0.9],
      [" I add the", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 6278,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now", false, 0.9],
      [" I add the rece", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 6303,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I", false, 0.9],
      [" add the receipt", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 6434,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I", false, 0.9],
      [" add the receipt from", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 6488,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I add", false, 0.9],
      [" the receipt from", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 6523,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I add", false, 0.9],
      [" the receipt from the", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 6635,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I add the", false, 0.9],
      [" receipt from the", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 6878,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I add the receipt", false, 0.9],
      [" from the tax", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 6991,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I add the receipt from", false, 0.9],
      [" the tax", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 7117,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I add the receipt from the", false, 0.9],
      [" taxi", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 7265,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I add the receipt from the", false, 0.9],
      [" taxi ride", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 7701,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I add the receipt from the taxi", false, 0.9],
      [" ride", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 7919,
    resultIndex: 1,
    results: [
      ["I am opening the expenses page", true, 0.972],
      [" now I add the receipt from the taxi ride", true, 0.969],
    ],
  },
];

export const STOPPED_MID_PHRASE: ChromeEvent[] = [
  { type: "start", at: 212 },
  { type: "audiostart", at: 222 },
  { type: "soundstart", at: 862 },
  { type: "speechstart", at: 862 },
  {
    type: "result",
    at: 1089,
    resultIndex: 0,
    results: [["I am", false, 0.01]],
  },
  {
    type: "result",
    at: 1346,
    resultIndex: 0,
    results: [["I am open", false, 0.01]],
  },
  {
    type: "result",
    at: 1440,
    resultIndex: 0,
    results: [["I am opening", false, 0.01]],
  },
  {
    type: "result",
    at: 1554,
    resultIndex: 0,
    results: [["I am opening the", false, 0.01]],
  },
  {
    type: "result",
    at: 1683,
    resultIndex: 0,
    results: [
      ["I am", false, 0.9],
      [" opening the", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 1861,
    resultIndex: 0,
    results: [
      ["I am", false, 0.9],
      [" opening the EXP", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 2048,
    resultIndex: 0,
    results: [
      ["I am opening", false, 0.9],
      [" the EXP", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 2121,
    resultIndex: 0,
    results: [
      ["I am opening", false, 0.9],
      [" the expens", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 2174,
    resultIndex: 0,
    results: [
      ["I am opening the", false, 0.9],
      [" expens", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 2218,
    resultIndex: 0,
    results: [
      ["I am opening the", false, 0.9],
      [" expenses", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 2496,
    resultIndex: 0,
    results: [
      ["I am opening the", false, 0.9],
      [" expenses page", false, 0.01],
    ],
  },
  { type: "stop", at: 2726 },
  { type: "speechend", at: 2726 },
  { type: "soundend", at: 2727 },
  { type: "audioend", at: 2727 },
  {
    type: "result",
    at: 2764,
    resultIndex: 0,
    results: [
      ["I am opening the expenses", false, 0.9],
      [" page", false, 0.01],
    ],
  },
  {
    type: "result",
    at: 2779,
    resultIndex: 0,
    results: [["I am opening the expenses page", true, 0.972]],
  },
  { type: "end", at: 2779 },
];

/**
 * A recorded result as the `SpeechRecognitionEvent` Chrome dispatched: a result list that is not an array but answers
 * `length`, an index and `item()`, each result the same with its alternatives.
 */
export function speechEvent(
  event: Extract<ChromeEvent, { type: "result" }>,
): SpeechEventShape {
  const results = event.results.map(([transcript, isFinal, confidence]) => {
    const alternative = { transcript, confidence };
    return {
      isFinal,
      length: 1,
      0: alternative,
      item: (index: number) => (index === 0 ? alternative : null),
    };
  });
  return {
    resultIndex: event.resultIndex,
    results: Object.assign(
      {
        length: results.length,
        item: (index: number) => results[index] ?? null,
      },
      results,
    ),
  };
}

export type SpeechEventShape = {
  resultIndex: number;
  results: {
    length: number;
    item(index: number): unknown;
    [index: number]: {
      isFinal: boolean;
      length: number;
      0: { transcript: string; confidence: number };
    };
  };
};

export const isResult = (
  event: ChromeEvent,
): event is Extract<ChromeEvent, { type: "result" }> => event.type === "result";

/**
 * Chrome's speech recognition, as `SpeechRecognition` and `webkitSpeechRecognition`: once listening it sends what Chrome
 * sent in the recorded run up to the moment Stop was pressed, and on stop() what it sent after, the final phrase then
 * the end. One stopped before it heard anything (as strict mode's first, discarded effect is) just ends, as Chrome's
 * does. Shared by every page that narrates a recording: the Browser page and a Bot's screen.
 *
 * `onDevice` is what it can recognise on the computer (on-device-speech.fixture.ts): every language unless a test
 * says, or null for a Chrome with no on-device recognition at all.
 */
export function installChrome(
  onDevice: OnDeviceModel | null = everyLanguage(),
) {
  const recognitions: ChromeRecognition[] = [];
  const stopAt =
    STOPPED_MID_PHRASE.find((event) => event.type === "stop")?.at ?? 0;
  const before = STOPPED_MID_PHRASE.filter(isResult).filter(
    (event) => event.at < stopAt,
  );
  const after = STOPPED_MID_PHRASE.filter(isResult).filter(
    (event) => event.at > stopAt,
  );
  class ChromeRecognition implements Recognition {
    lang = "";
    continuous = false;
    interimResults = false;
    maxAlternatives = 0;
    onstart: Recognition["onstart"] = null;
    onresult: Recognition["onresult"] = null;
    onerror: Recognition["onerror"] = null;
    onend: Recognition["onend"] = null;
    processLocally?: boolean;
    started = false;
    heard = false;
    #listening: ReturnType<typeof setTimeout> | undefined;
    constructor() {
      recognitions.push(this);
    }
    start() {
      this.started = true;
      this.#listening = setTimeout(() => {
        this.onstart?.();
        for (const event of before) this.onresult?.(speechEvent(event));
        this.heard = true;
      }, 0);
    }
    stop() {
      clearTimeout(this.#listening);
      queueMicrotask(() => {
        if (this.heard)
          for (const event of after) this.onresult?.(speechEvent(event));
        this.onend?.();
      });
    }
    abort() {
      clearTimeout(this.#listening);
      queueMicrotask(() => this.onend?.());
    }
  }
  const speech = globalThis as {
    SpeechRecognition?: unknown;
    webkitSpeechRecognition?: unknown;
  };
  // Chrome answers to both names, with the same recognition; its on-device side is on it (on-device-speech.fixture.ts).
  speech.SpeechRecognition = speech.webkitSpeechRecognition = onDevice
    ? withOnDevice(ChromeRecognition, onDevice)
    : ChromeRecognition;
  return recognitions;
}