An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.
TypeScript agentic-browseragentic-browsingagy-cliaiai-agentai-browser-extensionai-toolsbrave-extensionbrowser-agentbrowser-automationbrowser-extensionbrowser-usebrowserskillchrome-extensionclaude-codecodexfirefox-addonmcp-servernanobrowserweb-automation
feat(voice): know whether this browser can transcribe at all — never Firefox, never Brave, and not a fork whose speech service failed before it ever worked — with one recognizer shared by the panel and what listens without it
Shahriarf9f22efef6b7
diff --git a/src/lib/bridge/recognition.ts b/src/lib/bridge/recognition.tsnew file mode 100644index 0000000..d914861--- /dev/null+++ b/src/lib/bridge/recognition.ts@@ -0,0 +1,67 @@+interface SpeechAlternativeLike {+ readonly transcript: string;+}+interface SpeechResultLike {+ readonly isFinal: boolean;+ readonly length: number;+ readonly [index: number]: SpeechAlternativeLike;+}+interface SpeechResultListLike {+ readonly length: number;+ readonly [index: number]: SpeechResultLike;+}+export interface SpeechResultEventLike {+ readonly resultIndex: number;+ readonly results: SpeechResultListLike;+}+export interface SpeechErrorEventLike {+ readonly error: string;+ readonly message?: string;+}+export interface SpeechRecognitionLike {+ continuous: boolean;+ interimResults: boolean;+ lang: string;+ maxAlternatives: number;+ onstart: (() => void) | null;+ onresult: ((event: SpeechResultEventLike) => void) | null;+ onerror: ((event: SpeechErrorEventLike) => void) | null;+ onend: (() => void) | null;+ start(): void;+ stop(): void;+ abort(): void;+}+type SpeechRecognitionConstructor = new () => SpeechRecognitionLike;++export function recognitionCtor(): SpeechRecognitionConstructor | null {+ if (typeof window === 'undefined') return null;+ const scope = window as unknown as {+ SpeechRecognition?: SpeechRecognitionConstructor;+ webkitSpeechRecognition?: SpeechRecognitionConstructor;+ };+ return scope.SpeechRecognition ?? scope.webkitSpeechRecognition ?? null;+}++export function createRecognition(): SpeechRecognitionLike | null {+ const Ctor = recognitionCtor();+ if (!Ctor) return null;+ const speech = new Ctor();+ speech.continuous = true;+ speech.interimResults = true;+ speech.lang = navigator.language || 'en-US';+ speech.maxAlternatives = 1;+ return speech;+}++/** One result event, split into what is still being said and what has settled. */+export function transcriptOf(event: SpeechResultEventLike): { interim: string; final: string } {+ let interim = '';+ let final = '';+ for (let i = event.resultIndex; i < event.results.length; i++) {+ const result = event.results[i];+ const text = result[0]?.transcript ?? '';+ if (result.isFinal) final += text;+ else interim += text;+ }+ return { interim: interim.trim(), final: final.trim() };+}diff --git a/src/lib/bridge/speech-support.test.ts b/src/lib/bridge/speech-support.test.tsnew file mode 100644index 0000000..b1b3ee8--- /dev/null+++ b/src/lib/bridge/speech-support.test.ts@@ -0,0 +1,54 @@+import { beforeEach, describe, expect, it } from 'vitest';+import { fakeBrowser } from 'wxt/testing';+import { capableOf, handsFreeAllowed, noteSpeechService, readSpeechService } from './speech-support';++const chrome = [{ brand: 'Chromium' }, { brand: 'Google Chrome' }, { brand: 'Not_A Brand' }];+const brave = [{ brand: 'Brave' }, { brand: 'Chromium' }, { brand: 'Not;A=Brand' }];+const able = { offscreen: true, recognizer: true, brands: chrome };++beforeEach(() => {+ fakeBrowser.reset();+});++describe('capableOf', () => {+ it('allows a Chromium with somewhere to listen from and something to listen with', () => {+ expect(capableOf(able)).toBe(true);+ });++ it('refuses a fork that ships the recognizer with no service behind it', () => {+ expect(capableOf({ ...able, brands: brave })).toBe(false);+ });++ it('refuses a browser with no offscreen document, or no recognizer', () => {+ expect(capableOf({ ...able, offscreen: false })).toBe(false);+ expect(capableOf({ ...able, recognizer: false })).toBe(false);+ });+});++describe('what the speech service has shown', () => {+ it('starts unknown, which still allows hands-free', async () => {+ expect(await readSpeechService()).toBeNull();+ expect(handsFreeAllowed(null, true)).toBe(true);+ });++ it('hides hands-free once the service fails before it ever worked', async () => {+ expect(await noteSpeechService('missing')).toBe('missing');+ expect(handsFreeAllowed(await readSpeechService(), true)).toBe(false);+ });++ it('keeps hands-free once the service has worked, whatever fails later', async () => {+ await noteSpeechService('works');+ expect(await noteSpeechService('missing')).toBe('works');+ expect(handsFreeAllowed(await readSpeechService(), true)).toBe(true);+ });++ it('brings hands-free back when a service thought missing answers after all', async () => {+ await noteSpeechService('missing');+ expect(await noteSpeechService('works')).toBe('works');+ });++ it('never allows it where the browser itself cannot listen', async () => {+ await noteSpeechService('works');+ expect(handsFreeAllowed(await readSpeechService(), false)).toBe(false);+ });+});diff --git a/src/lib/bridge/speech-support.ts b/src/lib/bridge/speech-support.tsnew file mode 100644index 0000000..ea5e5b3--- /dev/null+++ b/src/lib/bridge/speech-support.ts@@ -0,0 +1,68 @@+import { browser } from 'wxt/browser';+import { recognitionCtor } from './recognition';++export const SPEECH_SERVICE_KEY = 'browsentic/speechService';++/** What the speech service itself has shown: it transcribed something, or it failed before it ever had. */+export type SpeechService = 'works' | 'missing';++/** Chromium forks that ship the recognizer without the service it talks to, so every attempt fails with `network`. */+const SERVICELESS_BRANDS = ['Brave'];++type Brands = { brand: string }[];++export interface SpeechPlatform {+ /** An offscreen document to listen from once the panel is closed. */+ offscreen: boolean;+ /** A recognizer to listen with. Unknowable from the service worker, which has no window. */+ recognizer: boolean;+ brands: Brands;+}++export const capableOf = ({ offscreen, recognizer, brands }: SpeechPlatform): boolean =>+ offscreen && recognizer && !brands.some(({ brand }) => SERVICELESS_BRANDS.includes(brand));++/**+ * Whether hands-free mode can exist here, from what is knowable without listening. Firefox has+ * no offscreen document to listen from and no recognizer to listen with; some Chromium forks+ * have the recognizer and nothing behind it. A fork not named here is settled by the service+ * itself, through `noteSpeechService`.+ */+export function speechCapable(): boolean {+ if (import.meta.env.FIREFOX) return false;+ const offscreen = (browser as unknown as { offscreen?: { createDocument?: unknown } }).offscreen;+ return capableOf({+ offscreen: typeof offscreen?.createDocument === 'function',+ recognizer: typeof window === 'undefined' || recognitionCtor() !== null,+ brands: brandsOfThisBrowser(),+ });+}++function brandsOfThisBrowser(): Brands {+ const agent = navigator as Navigator & { userAgentData?: { brands?: Brands } };+ return agent.userAgentData?.brands ?? [];+}++export async function readSpeechService(): Promise<SpeechService | null> {+ const stored = (await browser.storage.local.get(SPEECH_SERVICE_KEY))[SPEECH_SERVICE_KEY];+ return stored === 'works' || stored === 'missing' ? stored : null;+}++/**+ * Records what the service just did, and answers what is now believed. `works` always wins: a+ * service that has transcribed once and then fails is out of reach for a moment, not missing.+ */+export async function noteSpeechService(outcome: SpeechService): Promise<SpeechService> {+ const known = await readSpeechService();+ if (known === 'works' || known === outcome) return known;+ await browser.storage.local.set({ [SPEECH_SERVICE_KEY]: outcome });+ return outcome;+}++export function handsFreeAllowed(service: SpeechService | null, capable = speechCapable()): boolean {+ return capable && service !== 'missing';+}++export async function handsFreeSupported(): Promise<boolean> {+ return handsFreeAllowed(await readSpeechService());+}diff --git a/src/lib/bridge/use-speech.ts b/src/lib/bridge/use-speech.tsindex 52f7769..932d795 100644--- a/src/lib/bridge/use-speech.ts+++ b/src/lib/bridge/use-speech.ts@@ -1,50 +1,8 @@import { useCallback, useEffect, useMemo, useRef, useState } from 'react';import { browser } from 'wxt/browser';import { micPermission, openMicPermissionPage, watchMicPermission } from './mic-permission';--interface SpeechAlternativeLike {- readonly transcript: string;-}-interface SpeechResultLike {- readonly isFinal: boolean;- readonly length: number;- readonly [index: number]: SpeechAlternativeLike;-}-interface SpeechResultListLike {- readonly length: number;- readonly [index: number]: SpeechResultLike;-}-interface SpeechResultEventLike {- readonly resultIndex: number;- readonly results: SpeechResultListLike;-}-interface SpeechErrorEventLike {- readonly error: string;- readonly message?: string;-}-interface SpeechRecognitionLike {- continuous: boolean;- interimResults: boolean;- lang: string;- maxAlternatives: number;- onstart: (() => void) | null;- onresult: ((event: SpeechResultEventLike) => void) | null;- onerror: ((event: SpeechErrorEventLike) => void) | null;- onend: (() => void) | null;- start(): void;- stop(): void;- abort(): void;-}-type SpeechRecognitionConstructor = new () => SpeechRecognitionLike;--function recognitionCtor(): SpeechRecognitionConstructor | null {- if (typeof window === 'undefined') return null;- const scope = window as unknown as {- SpeechRecognition?: SpeechRecognitionConstructor;- webkitSpeechRecognition?: SpeechRecognitionConstructor;- };- return scope.SpeechRecognition ?? scope.webkitSpeechRecognition ?? null;-}+import { createRecognition, recognitionCtor, transcriptOf, type SpeechRecognitionLike } from './recognition';+import { SPEECH_SERVICE_KEY, handsFreeAllowed, noteSpeechService, readSpeechService, type SpeechService } from './speech-support';const RESTART_DELAY_MS = 400;@@ -82,6 +40,28 @@ export function useVoiceEnabled(): [boolean, (on: boolean) => void] {return [enabled, set];}+/** Hidden until known, so a browser without speech never sees the detach button flash on and off. */+export function useHandsFreeSupported(): boolean {+ const [supported, setSupported] = useState(false);++ useEffect(() => {+ let live = true;+ void readSpeechService().then((service) => {+ if (live) setSupported(handsFreeAllowed(service));+ });+ const listener = (changes: Record<string, { newValue?: unknown }>) => {+ if (SPEECH_SERVICE_KEY in changes) setSupported(handsFreeAllowed(changes[SPEECH_SERVICE_KEY].newValue as SpeechService));+ };+ browser.storage.local.onChanged.addListener(listener);+ return () => {+ live = false;+ browser.storage.local.onChanged.removeListener(listener);+ };+ }, []);++ return supported;+}+export interface UseSpeechOptions {onFinal: (text: string) => void;onInterim?: (text: string) => void;@@ -111,6 +91,7 @@ export function useSpeech({ onFinal, onInterim }: UseSpeechOptions): Speech {const blocked = useRef(false);const restartTimer = useRef<ReturnType<typeof setTimeout> | null>(null);const micGrant = useRef<Promise<void> | null>(null);+ const heardOnce = useRef(false);const beginRef = useRef<() => void>(() => {});const ensureMic = useCallback(() => {@@ -132,8 +113,7 @@ export function useSpeech({ onFinal, onInterim }: UseSpeechOptions): Speech {}, []);const begin = useCallback(async () => {- const Ctor = recognitionCtor();- if (!Ctor || recognition.current) return;+ if (!recognitionCtor() || recognition.current) return;if ((await micPermission()) === 'prompt') {setListening(false);setNeedsGrant(true);@@ -153,27 +133,21 @@ export function useSpeech({ onFinal, onInterim }: UseSpeechOptions): Speech {}if (!wanted.current) return;- const speech = new Ctor();- speech.continuous = true;- speech.interimResults = true;- speech.lang = navigator.language || 'en-US';- speech.maxAlternatives = 1;+ const speech = createRecognition();+ if (!speech) return;speech.onstart = () => {setListening(true);setError(null);};speech.onresult = (event) => {- let interim = '';- let final = '';- for (let i = event.resultIndex; i < event.results.length; i++) {- const result = event.results[i];- const text = result[0]?.transcript ?? '';- if (result.isFinal) final += text;- else interim += text;+ const { interim, final } = transcriptOf(event);+ if ((interim || final) && !heardOnce.current) {+ heardOnce.current = true;+ void noteSpeechService('works');}- if (interim.trim()) handlers.current.onInterim?.(interim.trim());- if (final.trim()) handlers.current.onFinal(final.trim());+ if (interim) handlers.current.onInterim?.(interim);+ if (final) handlers.current.onFinal(final);};speech.onerror = (event) => {if (event.error === 'aborted' || event.error === 'no-speech') return;@@ -192,6 +166,7 @@ export function useSpeech({ onFinal, onInterim }: UseSpeechOptions): Speech {}if (event.error === 'network') {wanted.current = false;+ void noteSpeechService('missing');setError(NO_SPEECH_SERVICE);return;}