Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

19.2 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import { installOnDeviceSpeech } from "@openbot/shared/on-device-speech";
import {
  DOWNLOAD_POLL_MS,
  FLUSH_MS,
  type NarrationStatus,
  type Phrase,
  PhraseClock,
  type Recognition,
  type RecognitionResultEvent,
  speechRecognition,
  startNarration,
} from "@/lib/browser/narration";
import {
  type ChromeEvent,
  isResult,
  STOPPED_MID_PHRASE,
  speechEvent,
  TWO_PHRASES,
} from "./browser-narration.fixture";
import {
  everyLanguage,
  type OnDeviceModel,
  onDeviceModel,
  withOnDevice,
} from "./on-device-speech.fixture";

/**
 * Narration on hosted OpenBot: the page transcribes the person with the browser's own speech recognition and stamps
 * each final phrase with when they started and stopped saying it. The rule is pinned against what Chrome 156 really
 * sent in a recorded run (browser-narration.fixture.ts), not against events written by hand.
 */

/** When the page started listening in each replay: an epoch millisecond, as Date.now() gives. */
const START = 1_792_000_000_000;

describe("when a phrase was said", () => {
  test("each phrase runs from its first words coming back to the moment it came back final", async () => {
    const clock = new PhraseClock();
    const phrases = TWO_PHRASES.filter(isResult).flatMap((event) =>
      clock.heard(speechEvent(event), START + event.at),
    );
    expect(phrases).toEqual([
      {
        text: "I am opening the expenses page",
        startedAt: START + 1_161,
        endedAt: START + 3_177,
      },
      {
        // Chrome's final text leads with a space; the phrase is what was said.
        text: "now I add the receipt from the taxi ride",
        startedAt: START + 5_379,
        endedAt: START + 7_919,
      },
    ]);
  });

  test("the second phrase starts at its own first words, not when the first phrase's changing tail used its slot", async () => {
    // In the real run Chrome showed the first sentence's tail at index 1 from 1690 ms, folded it back in at 3177 ms,
    // and only at 5379 ms put the second sentence's first word ("now") there. Keeping index 1's first sighting would
    // start the second phrase 3.7 s before it was said; holding it to the end of the first would still start it 2.2 s
    // early, in the pause, where the person's clicks would read as said during it.
    const clock = new PhraseClock();
    const second = TWO_PHRASES.filter(isResult)
      .flatMap((event) => clock.heard(speechEvent(event), START + event.at))
      .at(1);
    expect(second?.startedAt).toBe(START + 5_379);
    expect(second?.startedAt).not.toBe(START + 1_690);
    expect(second?.startedAt).not.toBe(START + 3_177);
  });

  test("a final phrase Chrome repeats in every later event is sent once", async () => {
    const clock = new PhraseClock();
    const results = TWO_PHRASES.filter(isResult);
    const sent = results.flatMap((event) =>
      clock.heard(speechEvent(event), START + event.at),
    );
    // Every event after 3177 ms repeats the first sentence as final.
    expect(
      results.filter(
        (event) =>
          event.results[0]?.[1] === true &&
          event.at > 3_177 &&
          event.at < 7_919,
      ).length,
    ).toBeGreaterThan(10);
    expect(
      sent.filter((phrase) => phrase.text.startsWith("I am opening")),
    ).toHaveLength(1);
  });

  test("a phrase is never stamped as starting before the one ahead of it ended", async () => {
    const clock = new PhraseClock();
    // A second result shows up while the first is still being finished, then both come back final.
    clock.heard(speechEvent(result(0, [["I am opening", false]])), START + 100);
    clock.heard(
      speechEvent(
        result(0, [
          ["I am opening the page", false],
          [" and", false],
        ]),
      ),
      START + 200,
    );
    const phrases = clock.heard(
      speechEvent(
        result(0, [
          ["I am opening the page", true],
          [" and then the form", true],
        ]),
      ),
      START + 900,
    );
    expect(phrases).toEqual([
      {
        text: "I am opening the page",
        startedAt: START + 100,
        endedAt: START + 900,
      },
      {
        text: "and then the form",
        startedAt: START + 900,
        endedAt: START + 900,
      },
    ]);
  });

  test("a new recognition numbers its results from 0 again and its phrases are timed afresh", async () => {
    const clock = new PhraseClock();
    clock.heard(speechEvent(result(0, [["first", false]])), START + 100);
    clock.heard(speechEvent(result(0, [["first", true]])), START + 500);
    clock.restart();
    clock.heard(speechEvent(result(0, [["second", false]])), START + 4_000);
    expect(
      clock.heard(speechEvent(result(0, [["second", true]])), START + 4_600),
    ).toEqual([
      { text: "second", startedAt: START + 4_000, endedAt: START + 4_600 },
    ]);
  });
});

describe("when a phrase is being said", () => {
  test("is the phrase's start as its first words come back, and nothing once it came back final, in a real run", async () => {
    const s = await setup();
    s.replay(TWO_PHRASES);
    // The Recorder holds the person's steps from the moment each phrase began until the phrase arrives: the moment it is
    // told is the start the phrase is stamped with, so a click made as they began to speak waits for the phrase.
    expect(s.told).toEqual([
      { speaking: START + 1_161 },
      { phrase: "I am opening the expenses page" },
      { speaking: null },
      { speaking: START + 5_379 },
      { phrase: "now I add the receipt from the taxi ride" },
      { speaking: null },
    ]);
    expect(s.phrases.map((phrase) => phrase.startedAt)).toEqual([
      START + 1_161,
      START + 5_379,
    ]);
  });

  test("is told the moment the first words come back, long before the phrase is final", async () => {
    const s = await setup();
    const first = TWO_PHRASES.filter(isResult)[0];
    if (!first) throw new Error("The fixture has no result.");
    s.replay(TWO_PHRASES.filter((event) => event.at <= first.at));
    expect(s.told).toEqual([{ speaking: START + first.at }]);
    expect(s.phrases).toEqual([]);
  });

  test("Stop hands over the phrase being said, then says nothing more is", async () => {
    const s = await setup();
    const stopAt =
      STOPPED_MID_PHRASE.find((event) => event.type === "stop")?.at ?? 0;
    s.replay(STOPPED_MID_PHRASE.filter((event) => event.at < stopAt));
    expect(s.told).toEqual([{ speaking: START + 1_089 }]);
    const stopping = s.narration.stop();
    s.replay(
      STOPPED_MID_PHRASE.filter(
        (event) => event.type !== "stop" && event.at > stopAt,
      ),
    );
    await stopping;
    expect(s.told).toEqual([
      { speaking: START + 1_089 },
      { phrase: "I am opening the expenses page" },
      { speaking: null },
    ]);
  });

  test("ends with no phrase when Chrome returns no words for it, ends listening, or the person mutes", async () => {
    const s = await setup();
    s.at(1_000);
    s.current().onresult?.(speechEvent(result(0, [["", false]])));
    s.at(1_400);
    s.current().onresult?.(speechEvent(result(0, [["", true]])));
    expect(s.told).toEqual([{ speaking: START + 1_000 }, { speaking: null }]);

    // Chrome ends the recognition with a phrase still unfinished: it never comes.
    s.at(2_000);
    s.current().onresult?.(
      speechEvent(
        result(1, [
          ["", true],
          ["um", false],
        ]),
      ),
    );
    s.current().onend?.();
    expect(s.told.slice(2)).toEqual([
      { speaking: START + 2_000 },
      { speaking: null },
    ]);

    // Muted mid-phrase: the phrase is handed over if it comes, and nothing is being said after.
    s.advance(0);
    s.at(70_000);
    s.current().onresult?.(speechEvent(result(0, [["and then", false]])));
    s.narration.mute(true);
    expect(s.told.at(-1)).toEqual({ speaking: START + 70_000 });
    s.current().onend?.();
    expect(s.told.slice(4)).toEqual([
      { speaking: START + 70_000 },
      { speaking: null },
    ]);
    expect(s.phrases).toEqual([]);
  });
});

describe("the browser's speech recognition", () => {
  test("is the standard one, else the webkit one Chrome and Safari answer to, else none (Firefox)", async () => {
    class Standard {}
    class Webkit {}
    expect(speechRecognition({ SpeechRecognition: Standard })).toBe(
      Standard as never,
    );
    expect(speechRecognition({ webkitSpeechRecognition: Webkit })).toBe(
      Webkit as never,
    );
    expect(speechRecognition({})).toBeNull();
  });
});

describe("listening while a recording runs", () => {
  test("listens continuously with interim results, and sends every final phrase of a real run with its timing", async () => {
    const s = await setup();
    const recognition = s.recognitions[0];
    expect(recognition).toMatchObject({
      lang: "en-GB",
      continuous: true,
      interimResults: true,
      // On the person's computer or not at all: without it Chrome streams the audio to its speech service.
      processLocally: true,
      started: true,
    });
    s.replay(TWO_PHRASES);
    expect(s.phrases).toEqual([
      {
        text: "I am opening the expenses page",
        startedAt: START + 1_161,
        endedAt: START + 3_177,
      },
      {
        text: "now I add the receipt from the taxi ride",
        startedAt: START + 5_379,
        endedAt: START + 7_919,
      },
    ]);
    expect(s.statuses).toEqual([{ state: "starting" }, { state: "listening" }]);
  });

  test("Stop hands over the phrase being said before listening ends", async () => {
    const s = await setup();
    const recognition = s.current();
    const beforeStop = STOPPED_MID_PHRASE.filter(
      (event) =>
        event.at <
        (STOPPED_MID_PHRASE.find((each) => each.type === "stop")?.at ?? 0),
    );
    s.replay(beforeStop);
    expect(s.phrases).toEqual([]);
    let stopped = false;
    const stopping = s.narration.stop().then(() => {
      stopped = true;
    });
    expect(recognition.stops).toBe(1);
    expect(recognition.aborts).toBe(0);
    // What Chrome sent after stop() in the real run: the phrase, final, then the end.
    s.replay(
      STOPPED_MID_PHRASE.filter(
        (event) => event.type !== "stop" && !beforeStop.includes(event),
      ),
    );
    await stopping;
    expect(stopped).toBe(true);
    expect(s.phrases).toEqual([
      {
        text: "I am opening the expenses page",
        startedAt: START + 1_089,
        endedAt: START + 2_779,
      },
    ]);
    // Nothing is sent or restarted after it.
    expect(s.recognitions).toHaveLength(1);
  });

  test("Stop does not wait for ever on a speech service that never answers", async () => {
    const s = await setup();
    const stopping = s.narration.stop();
    s.advance(FLUSH_MS);
    await stopping;
    expect(s.current().aborts).toBe(1);
  });

  test("a recognition Chrome ends on its own is replaced, sooner or later depending on how long it lasted", async () => {
    const s = await setup();
    // It ran for a minute and Chrome ended it, as it does after a silence: listening again at once.
    s.at(60_000);
    s.current().onend?.();
    s.advance(0);
    expect(s.recognitions).toHaveLength(2);
    // One that ends at once is not restarted in a tight loop.
    s.current().onend?.();
    s.advance(0);
    expect(s.recognitions).toHaveLength(2);
    s.advance(500);
    expect(s.recognitions).toHaveLength(3);
  });

  test("a refused microphone stops narration, says why, and turning it on again tries again", async () => {
    const s = await setup();
    s.current().onerror?.({ error: "not-allowed" });
    s.current().onend?.();
    s.advance(60_000);
    expect(s.recognitions).toHaveLength(1);
    expect(s.statuses.at(-1)).toEqual({
      state: "failed",
      problem: "not-allowed",
    });
    s.narration.mute(false);
    await s.settle();
    expect(s.recognitions).toHaveLength(2);
    expect(s.current().started).toBe(true);
  });

  test("an unreachable speech service is retried by itself with a growing delay", async () => {
    const s = await setup();
    s.current().onerror?.({ error: "network" });
    s.current().onend?.();
    expect(s.statuses.at(-1)).toEqual({ state: "failed", problem: "network" });
    s.advance(1_999);
    expect(s.recognitions).toHaveLength(1);
    s.advance(1);
    expect(s.recognitions).toHaveLength(2);
  });

  test("muting ends listening after the phrase being said; unmuting listens again", async () => {
    const s = await setup();
    const first = s.current();
    s.narration.mute(true);
    expect(first.stops).toBe(1);
    expect(s.statuses.at(-1)).toEqual({ state: "muted" });
    first.onresult?.(speechEvent(result(0, [["the last words", true]])));
    first.onend?.();
    s.advance(60_000);
    expect(s.phrases.map((phrase) => phrase.text)).toEqual(["the last words"]);
    expect(s.recognitions).toHaveLength(1);
    s.narration.mute(false);
    await s.settle();
    expect(s.recognitions).toHaveLength(2);
  });

  test("a browser that cannot keep speech on the computer never listens, and says why", async () => {
    const s = await setup({ onDevice: null });
    expect(s.recognitions).toHaveLength(0);
    expect(s.statuses.at(-1)).toEqual({
      state: "failed",
      problem: "on-device",
    });
  });

  test("a language the computer cannot recognise on the device never listens", async () => {
    const s = await setup({
      language: "cy-GB",
      onDevice: onDeviceModel({ "en-GB": "installed" }, { unmasked: true }),
    });
    expect(s.recognitions).toHaveLength(0);
    expect(s.statuses.at(-1)).toEqual({ state: "failed", problem: "language" });
  });

  test("a language to be downloaded waits for the person's click, then for the download, then listens", async () => {
    // Chrome answers downloadable for a language the page has not asked for, even when it is on the computer.
    const s = await setup({
      onDevice: onDeviceModel({ "en-GB": "downloadable" }),
    });
    expect(s.recognitions).toHaveLength(0);
    expect(s.statuses.at(-1)).toEqual({
      state: "failed",
      problem: "speech-pack",
    });
    // The person turns narration on: their click asks the browser for it (use-narration.ts), and narration waits for
    // the download.
    const model = s.onDevice as OnDeviceModel;
    model.activation = true;
    void installOnDeviceSpeech(s.Recognition, "en-GB");
    expect(model.installs).toEqual(["en-GB"]);
    s.narration.mute(false);
    await s.settle();
    expect(s.statuses.at(-1)).toEqual({ state: "downloading" });
    expect(s.recognitions).toHaveLength(0);
    s.advance(DOWNLOAD_POLL_MS);
    await s.settle();
    expect(s.recognitions).toHaveLength(0);
    model.finishDownloads();
    s.advance(DOWNLOAD_POLL_MS);
    await s.settle();
    expect(s.recognitions).toHaveLength(1);
    expect(s.current().processLocally).toBe(true);
  });

  test("a browser with no speech recognition says so and never listens", async () => {
    const statuses: NarrationStatus[] = [];
    startNarration(
      {
        language: "en-GB",
        muted: false,
        phrase: () => undefined,
        status: (status) => statuses.push(status),
      },
      { Recognition: null, now: () => START, timers: fakeTimers().timers },
    );
    expect(statuses.at(-1)).toEqual({
      state: "failed",
      problem: "unsupported",
    });
  });
});

function result(
  resultIndex: number,
  results: [string, boolean][],
): Extract<ChromeEvent, { type: "result" }> {
  return {
    type: "result",
    at: 0,
    resultIndex,
    results: results.map(([text, final]) => [text, final, 0.9]),
  };
}

/** setTimeout and clearTimeout on a clock the test moves. */
function fakeTimers() {
  let now = START;
  let next = 0;
  const pending = new Map<number, { at: number; run: () => void }>();
  return {
    now: () => now,
    setNow: (at: number) => {
      now = at;
    },
    advance(ms: number) {
      const until = now + ms;
      for (;;) {
        const due = [...pending.entries()]
          .filter(([, timer]) => timer.at <= until)
          .sort((a, b) => a[1].at - b[1].at)[0];
        if (!due) break;
        pending.delete(due[0]);
        now = due[1].at;
        due[1].run();
      }
      now = until;
    },
    timers: {
      set(run: () => void, ms: number) {
        const id = ++next;
        pending.set(id, { at: now + ms, run });
        return id;
      },
      clear(handle: unknown) {
        pending.delete(handle as number);
      },
    },
  };
}

/** Lets the browser's answers to `available()`, which come on later turns, arrive. */
const settle = async () => {
  for (let turn = 0; turn < 5; turn++) await Promise.resolve();
};

/** A SpeechRecognition double: what the page set on it, and Chrome's own calls back, driven by the test. */
class FakeRecognition implements Recognition {
  lang = "";
  continuous = false;
  interimResults = false;
  maxAlternatives = 0;
  processLocally?: boolean;
  onstart: (() => void) | null = null;
  onresult: ((event: RecognitionResultEvent) => void) | null = null;
  onerror: ((event: { error: string }) => void) | null = null;
  onend: (() => void) | null = null;
  started = false;
  stops = 0;
  aborts = 0;
  start() {
    this.started = true;
  }
  stop() {
    this.stops++;
  }
  abort() {
    this.aborts++;
  }
}

/** The page listening as a recording starts, once the browser said it can keep the language on the computer (an
 * answer it gives on a later turn): `onDevice` is what it can, every language by default; null for a browser with no
 * on-device recognition at all. */
async function setup(
  options: { onDevice?: OnDeviceModel | null; language?: string } = {},
) {
  const time = fakeTimers();
  const recognitions: FakeRecognition[] = [];
  const phrases: Phrase[] = [];
  const statuses: NarrationStatus[] = [];
  /** What the Recorder is told, in order: each phrase, and each change in when the phrase being said began. */
  const told: ({ phrase: string } | { speaking: number | null })[] = [];
  const Chrome = class extends FakeRecognition {
    constructor() {
      super();
      recognitions.push(this);
    }
  };
  const onDevice =
    options.onDevice === undefined ? everyLanguage() : options.onDevice;
  const narration = startNarration(
    {
      language: options.language ?? "en-GB",
      muted: false,
      phrase: (phrase) => {
        phrases.push(phrase);
        told.push({ phrase: phrase.text });
      },
      speaking: (since) => told.push({ speaking: since }),
      status: (status) => statuses.push(status),
    },
    {
      Recognition: onDevice ? withOnDevice(Chrome, onDevice) : Chrome,
      now: time.now,
      timers: time.timers,
    },
  );
  await settle();
  const current = () => {
    const recognition = recognitions.at(-1);
    if (!recognition) throw new Error("Nothing is listening.");
    return recognition;
  };
  return {
    narration,
    onDevice,
    settle,
    Recognition: onDevice ? Chrome : null,
    recognitions,
    phrases,
    statuses,
    told,
    current,
    at: (ms: number) => time.setNow(START + ms),
    advance: time.advance,
    /** Plays recorded events into the current recognition at the moments Chrome sent them. */
    replay(events: ChromeEvent[]) {
      for (const event of events) {
        time.setNow(START + event.at);
        const recognition = current();
        if (isResult(event)) recognition.onresult?.(speechEvent(event));
        else if (event.type === "start") recognition.onstart?.();
        else if (event.type === "end") recognition.onend?.();
      }
    },
  };
}