Skip to content

An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.

feat(voice): know whether this browser can transcribe at all — never Firefox, never Brave, and not a fork whose speech service failed before it ever worked — with one recognizer shared by the panel and what listens without it

Shahriarf9f22efef6b7
diff --git a/src/lib/bridge/recognition.ts b/src/lib/bridge/recognition.ts
new file mode 100644
index 0000000..d914861
--- /dev/null
+++ b/src/lib/bridge/recognition.ts
@@ -0,0 +1,67 @@
+interface SpeechAlternativeLike {
+ readonly transcript: string;
+}
+interface SpeechResultLike {
+ readonly isFinal: boolean;
+ readonly length: number;
+ readonly [index: number]: SpeechAlternativeLike;
+}
+interface SpeechResultListLike {
+ readonly length: number;
+ readonly [index: number]: SpeechResultLike;
+}
+export interface SpeechResultEventLike {
+ readonly resultIndex: number;
+ readonly results: SpeechResultListLike;
+}
+export interface SpeechErrorEventLike {
+ readonly error: string;
+ readonly message?: string;
+}
+export interface SpeechRecognitionLike {
+ continuous: boolean;
+ interimResults: boolean;
+ lang: string;
+ maxAlternatives: number;
+ onstart: (() => void) | null;
+ onresult: ((event: SpeechResultEventLike) => void) | null;
+ onerror: ((event: SpeechErrorEventLike) => void) | null;
+ onend: (() => void) | null;
+ start(): void;
+ stop(): void;
+ abort(): void;
+}
+type SpeechRecognitionConstructor = new () => SpeechRecognitionLike;
+
+export function recognitionCtor(): SpeechRecognitionConstructor | null {
+ if (typeof window === 'undefined') return null;
+ const scope = window as unknown as {
+ SpeechRecognition?: SpeechRecognitionConstructor;
+ webkitSpeechRecognition?: SpeechRecognitionConstructor;
+ };
+ return scope.SpeechRecognition ?? scope.webkitSpeechRecognition ?? null;
+}
+
+export function createRecognition(): SpeechRecognitionLike | null {
+ const Ctor = recognitionCtor();
+ if (!Ctor) return null;
+ const speech = new Ctor();
+ speech.continuous = true;
+ speech.interimResults = true;
+ speech.lang = navigator.language || 'en-US';
+ speech.maxAlternatives = 1;
+ return speech;
+}
+
+/** One result event, split into what is still being said and what has settled. */
+export function transcriptOf(event: SpeechResultEventLike): { interim: string; final: string } {
+ let interim = '';
+ let final = '';
+ for (let i = event.resultIndex; i < event.results.length; i++) {
+ const result = event.results[i];
+ const text = result[0]?.transcript ?? '';
+ if (result.isFinal) final += text;
+ else interim += text;
+ }
+ return { interim: interim.trim(), final: final.trim() };
+}
diff --git a/src/lib/bridge/speech-support.test.ts b/src/lib/bridge/speech-support.test.ts
new file mode 100644
index 0000000..b1b3ee8
--- /dev/null
+++ b/src/lib/bridge/speech-support.test.ts
@@ -0,0 +1,54 @@
+import { beforeEach, describe, expect, it } from 'vitest';
+import { fakeBrowser } from 'wxt/testing';
+import { capableOf, handsFreeAllowed, noteSpeechService, readSpeechService } from './speech-support';
+
+const chrome = [{ brand: 'Chromium' }, { brand: 'Google Chrome' }, { brand: 'Not_A Brand' }];
+const brave = [{ brand: 'Brave' }, { brand: 'Chromium' }, { brand: 'Not;A=Brand' }];
+const able = { offscreen: true, recognizer: true, brands: chrome };
+
+beforeEach(() => {
+ fakeBrowser.reset();
+});
+
+describe('capableOf', () => {
+ it('allows a Chromium with somewhere to listen from and something to listen with', () => {
+ expect(capableOf(able)).toBe(true);
+ });
+
+ it('refuses a fork that ships the recognizer with no service behind it', () => {
+ expect(capableOf({ ...able, brands: brave })).toBe(false);
+ });
+
+ it('refuses a browser with no offscreen document, or no recognizer', () => {
+ expect(capableOf({ ...able, offscreen: false })).toBe(false);
+ expect(capableOf({ ...able, recognizer: false })).toBe(false);
+ });
+});
+
+describe('what the speech service has shown', () => {
+ it('starts unknown, which still allows hands-free', async () => {
+ expect(await readSpeechService()).toBeNull();
+ expect(handsFreeAllowed(null, true)).toBe(true);
+ });
+
+ it('hides hands-free once the service fails before it ever worked', async () => {
+ expect(await noteSpeechService('missing')).toBe('missing');
+ expect(handsFreeAllowed(await readSpeechService(), true)).toBe(false);
+ });
+
+ it('keeps hands-free once the service has worked, whatever fails later', async () => {
+ await noteSpeechService('works');
+ expect(await noteSpeechService('missing')).toBe('works');
+ expect(handsFreeAllowed(await readSpeechService(), true)).toBe(true);
+ });
+
+ it('brings hands-free back when a service thought missing answers after all', async () => {
+ await noteSpeechService('missing');
+ expect(await noteSpeechService('works')).toBe('works');
+ });
+
+ it('never allows it where the browser itself cannot listen', async () => {
+ await noteSpeechService('works');
+ expect(handsFreeAllowed(await readSpeechService(), false)).toBe(false);
+ });
+});
diff --git a/src/lib/bridge/speech-support.ts b/src/lib/bridge/speech-support.ts
new file mode 100644
index 0000000..ea5e5b3
--- /dev/null
+++ b/src/lib/bridge/speech-support.ts
@@ -0,0 +1,68 @@
+import { browser } from 'wxt/browser';
+import { recognitionCtor } from './recognition';
+
+export const SPEECH_SERVICE_KEY = 'browsentic/speechService';
+
+/** What the speech service itself has shown: it transcribed something, or it failed before it ever had. */
+export type SpeechService = 'works' | 'missing';
+
+/** Chromium forks that ship the recognizer without the service it talks to, so every attempt fails with `network`. */
+const SERVICELESS_BRANDS = ['Brave'];
+
+type Brands = { brand: string }[];
+
+export interface SpeechPlatform {
+ /** An offscreen document to listen from once the panel is closed. */
+ offscreen: boolean;
+ /** A recognizer to listen with. Unknowable from the service worker, which has no window. */
+ recognizer: boolean;
+ brands: Brands;
+}
+
+export const capableOf = ({ offscreen, recognizer, brands }: SpeechPlatform): boolean =>
+ offscreen && recognizer && !brands.some(({ brand }) => SERVICELESS_BRANDS.includes(brand));
+
+/**
+ * Whether hands-free mode can exist here, from what is knowable without listening. Firefox has
+ * no offscreen document to listen from and no recognizer to listen with; some Chromium forks
+ * have the recognizer and nothing behind it. A fork not named here is settled by the service
+ * itself, through `noteSpeechService`.
+ */
+export function speechCapable(): boolean {
+ if (import.meta.env.FIREFOX) return false;
+ const offscreen = (browser as unknown as { offscreen?: { createDocument?: unknown } }).offscreen;
+ return capableOf({
+ offscreen: typeof offscreen?.createDocument === 'function',
+ recognizer: typeof window === 'undefined' || recognitionCtor() !== null,
+ brands: brandsOfThisBrowser(),
+ });
+}
+
+function brandsOfThisBrowser(): Brands {
+ const agent = navigator as Navigator & { userAgentData?: { brands?: Brands } };
+ return agent.userAgentData?.brands ?? [];
+}
+
+export async function readSpeechService(): Promise<SpeechService | null> {
+ const stored = (await browser.storage.local.get(SPEECH_SERVICE_KEY))[SPEECH_SERVICE_KEY];
+ return stored === 'works' || stored === 'missing' ? stored : null;
+}
+
+/**
+ * Records what the service just did, and answers what is now believed. `works` always wins: a
+ * service that has transcribed once and then fails is out of reach for a moment, not missing.
+ */
+export async function noteSpeechService(outcome: SpeechService): Promise<SpeechService> {
+ const known = await readSpeechService();
+ if (known === 'works' || known === outcome) return known;
+ await browser.storage.local.set({ [SPEECH_SERVICE_KEY]: outcome });
+ return outcome;
+}
+
+export function handsFreeAllowed(service: SpeechService | null, capable = speechCapable()): boolean {
+ return capable && service !== 'missing';
+}
+
+export async function handsFreeSupported(): Promise<boolean> {
+ return handsFreeAllowed(await readSpeechService());
+}
diff --git a/src/lib/bridge/use-speech.ts b/src/lib/bridge/use-speech.ts
index 52f7769..932d795 100644
--- a/src/lib/bridge/use-speech.ts
+++ b/src/lib/bridge/use-speech.ts
@@ -1,50 +1,8 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { browser } from 'wxt/browser';
import { micPermission, openMicPermissionPage, watchMicPermission } from './mic-permission';
-
-interface SpeechAlternativeLike {
- readonly transcript: string;
-}
-interface SpeechResultLike {
- readonly isFinal: boolean;
- readonly length: number;
- readonly [index: number]: SpeechAlternativeLike;
-}
-interface SpeechResultListLike {
- readonly length: number;
- readonly [index: number]: SpeechResultLike;
-}
-interface SpeechResultEventLike {
- readonly resultIndex: number;
- readonly results: SpeechResultListLike;
-}
-interface SpeechErrorEventLike {
- readonly error: string;
- readonly message?: string;
-}
-interface SpeechRecognitionLike {
- continuous: boolean;
- interimResults: boolean;
- lang: string;
- maxAlternatives: number;
- onstart: (() => void) | null;
- onresult: ((event: SpeechResultEventLike) => void) | null;
- onerror: ((event: SpeechErrorEventLike) => void) | null;
- onend: (() => void) | null;
- start(): void;
- stop(): void;
- abort(): void;
-}
-type SpeechRecognitionConstructor = new () => SpeechRecognitionLike;
-
-function recognitionCtor(): SpeechRecognitionConstructor | null {
- if (typeof window === 'undefined') return null;
- const scope = window as unknown as {
- SpeechRecognition?: SpeechRecognitionConstructor;
- webkitSpeechRecognition?: SpeechRecognitionConstructor;
- };
- return scope.SpeechRecognition ?? scope.webkitSpeechRecognition ?? null;
-}
+import { createRecognition, recognitionCtor, transcriptOf, type SpeechRecognitionLike } from './recognition';
+import { SPEECH_SERVICE_KEY, handsFreeAllowed, noteSpeechService, readSpeechService, type SpeechService } from './speech-support';
const RESTART_DELAY_MS = 400;
@@ -82,6 +40,28 @@ export function useVoiceEnabled(): [boolean, (on: boolean) => void] {
return [enabled, set];
}
+/** Hidden until known, so a browser without speech never sees the detach button flash on and off. */
+export function useHandsFreeSupported(): boolean {
+ const [supported, setSupported] = useState(false);
+
+ useEffect(() => {
+ let live = true;
+ void readSpeechService().then((service) => {
+ if (live) setSupported(handsFreeAllowed(service));
+ });
+ const listener = (changes: Record<string, { newValue?: unknown }>) => {
+ if (SPEECH_SERVICE_KEY in changes) setSupported(handsFreeAllowed(changes[SPEECH_SERVICE_KEY].newValue as SpeechService));
+ };
+ browser.storage.local.onChanged.addListener(listener);
+ return () => {
+ live = false;
+ browser.storage.local.onChanged.removeListener(listener);
+ };
+ }, []);
+
+ return supported;
+}
+
export interface UseSpeechOptions {
onFinal: (text: string) => void;
onInterim?: (text: string) => void;
@@ -111,6 +91,7 @@ export function useSpeech({ onFinal, onInterim }: UseSpeechOptions): Speech {
const blocked = useRef(false);
const restartTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const micGrant = useRef<Promise<void> | null>(null);
+ const heardOnce = useRef(false);
const beginRef = useRef<() => void>(() => {});
const ensureMic = useCallback(() => {
@@ -132,8 +113,7 @@ export function useSpeech({ onFinal, onInterim }: UseSpeechOptions): Speech {
}, []);
const begin = useCallback(async () => {
- const Ctor = recognitionCtor();
- if (!Ctor || recognition.current) return;
+ if (!recognitionCtor() || recognition.current) return;
if ((await micPermission()) === 'prompt') {
setListening(false);
setNeedsGrant(true);
@@ -153,27 +133,21 @@ export function useSpeech({ onFinal, onInterim }: UseSpeechOptions): Speech {
}
if (!wanted.current) return;
- const speech = new Ctor();
- speech.continuous = true;
- speech.interimResults = true;
- speech.lang = navigator.language || 'en-US';
- speech.maxAlternatives = 1;
+ const speech = createRecognition();
+ if (!speech) return;
speech.onstart = () => {
setListening(true);
setError(null);
};
speech.onresult = (event) => {
- let interim = '';
- let final = '';
- for (let i = event.resultIndex; i < event.results.length; i++) {
- const result = event.results[i];
- const text = result[0]?.transcript ?? '';
- if (result.isFinal) final += text;
- else interim += text;
+ const { interim, final } = transcriptOf(event);
+ if ((interim || final) && !heardOnce.current) {
+ heardOnce.current = true;
+ void noteSpeechService('works');
}
- if (interim.trim()) handlers.current.onInterim?.(interim.trim());
- if (final.trim()) handlers.current.onFinal(final.trim());
+ if (interim) handlers.current.onInterim?.(interim);
+ if (final) handlers.current.onFinal(final);
};
speech.onerror = (event) => {
if (event.error === 'aborted' || event.error === 'no-speech') return;
@@ -192,6 +166,7 @@ export function useSpeech({ onFinal, onInterim }: UseSpeechOptions): Speech {
}
if (event.error === 'network') {
wanted.current = false;
+ void noteSpeechService('missing');
setError(NO_SPEECH_SERVICE);
return;
}