Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

5.6 KB
Raw Blame
/**
 * ON-DEVICE SPEECH RECOGNITION AS CHROME IMPLEMENTS IT, for the page's speech recognition doubles: the Web Speech API's
 * `available()`, `install()` and `processLocally` (https://webaudio.github.io/web-speech-api/), for one origin, as the
 * specification has them and as Chromium 151 answered `available()` and `start()` when asked on 2026-10-27 (it never
 * answered an `install()` without a click, which the specification refuses):
 * - `available()` with `processLocally` answers `unavailable` for a language the computer cannot recognise, and
 *   `downloadable` for any other until this origin asked for it with `install()` (the specification's download
 *   masking), then `available`, `downloading` or `downloadable` as the language really is.
 * - `install()` that would start a download is refused (`NotAllowedError`) without the person's click (`activation`);
 *   otherwise it unmasks the language and resolves true once it is on the computer (at once when it already was, else
 *   when the test calls `finishDownloads`).
 * - A recognition started with `processLocally` in a language that is not available fails with `language-not-supported`
 *   and never ends.
 */

export type OnDeviceModel = {
  /** Each language this computer can recognise on the device and whether it is downloaded; `*` stands for any other. */
  languages: Map<string, "installed" | "downloadable" | "downloading">;
  /** Languages this origin asked for with install(), so their real state shows; `*` for every one. */
  unmasked: Set<string>;
  /** Whether a call now carries the person's click. */
  activation: boolean;
  /** Every install() asked, in order. */
  installs: string[];
  finishDownloads(): void;
};

export function onDeviceModel(
  languages: Record<string, "installed" | "downloadable" | "downloading">,
  options: { unmasked?: boolean } = {},
): OnDeviceModel {
  const downloads: (() => void)[] = [];
  const model: OnDeviceModel & { downloads: (() => void)[] } = {
    languages: new Map(Object.entries(languages)),
    unmasked: new Set(options.unmasked ? Object.keys(languages) : []),
    activation: false,
    installs: [],
    downloads,
    finishDownloads() {
      for (const [language, state] of model.languages)
        if (state === "downloading") model.languages.set(language, "installed");
      for (const done of downloads.splice(0)) done();
    },
  };
  return model;
}

/** A person who downloaded every language they narrate in. */
export const everyLanguage = () =>
  onDeviceModel({ "*": "installed" }, { unmasked: true });

function stateOf(model: OnDeviceModel, language: string) {
  return model.languages.get(language) ?? model.languages.get("*");
}

/** What `available()` answers for one language. */
export function availability(model: OnDeviceModel, language: string) {
  const state = stateOf(model, language);
  if (state === undefined) return "unavailable";
  if (!model.unmasked.has(language) && !model.unmasked.has("*"))
    return "downloadable";
  return state === "installed" ? "available" : state;
}

type Options = { langs: string[]; processLocally?: boolean };

/**
 * Gives a SpeechRecognition double Chrome's on-device side: the static `available()` and `install()`, and
 * `processLocally` on its instances, which a recognition `start()`ed in a language not on the computer fails.
 */
export function withOnDevice<
  T extends new (
    ...args: never[]
  ) => {
    lang: string;
    start(...args: never[]): void;
    onerror: ((event: { error: string }) => void) | null;
  },
>(Recognition: T, model: OnDeviceModel): T {
  const start = Recognition.prototype.start;
  Object.defineProperty(Recognition.prototype, "processLocally", {
    value: false,
    writable: true,
    configurable: true,
  });
  Recognition.prototype.start = function (
    this: InstanceType<T> & { processLocally?: boolean },
    ...args: never[]
  ) {
    start.apply(this, args);
    if (this.processLocally && availability(model, this.lang) !== "available")
      queueMicrotask(() => this.onerror?.({ error: "language-not-supported" }));
  };
  const downloads = (model as unknown as { downloads: (() => void)[] })
    .downloads;
  Object.assign(Recognition, {
    async available(options: Options) {
      if (options.processLocally !== true)
        return options.langs.length > 0 ? "available" : "unavailable";
      if (options.langs.length === 0) return "unavailable";
      const each = options.langs.map((lang) => availability(model, lang));
      for (const status of [
        "unavailable",
        "downloadable",
        "downloading",
      ] as const)
        if (each.includes(status)) return status;
      return "available";
    },
    async install(options: Options) {
      model.installs.push(...options.langs);
      const wouldDownload = options.langs.some(
        (lang) => availability(model, lang) === "downloadable",
      );
      if (wouldDownload && !model.activation)
        throw Object.assign(new Error("Requires a user gesture."), {
          name: "NotAllowedError",
        });
      // The click is used up by the call.
      model.activation = false;
      if (
        options.langs.some(
          (lang) => availability(model, lang) === "unavailable",
        )
      )
        return false;
      for (const lang of options.langs) {
        model.unmasked.add(lang);
        if (stateOf(model, lang) === "downloadable")
          model.languages.set(lang, "downloading");
      }
      if (
        options.langs.every((lang) => availability(model, lang) === "available")
      )
        return true;
      return new Promise<boolean>((resolve) => {
        downloads.push(() => resolve(true));
      });
    },
  });
  return Recognition;
}