Skip to content

An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.

fix(hands-free): never leave the mic open after a release, a tab switch, a navigation or a removed orb, never show a phase the page has not reported, never write speech off for an error that is not a missing service, and show the whole script, every refused turn and each approval once

Shahriar500460860fea
diff --git a/src/extension/entrypoints/dictation/main.ts b/src/extension/entrypoints/dictation/main.ts
index 74dac9a..ee4d42c 100644
--- a/src/extension/entrypoints/dictation/main.ts
+++ b/src/extension/entrypoints/dictation/main.ts
@@ -48,8 +48,10 @@ async function granted(): Promise<boolean> {
return false;
}
+const due = (): boolean => wanted && !speech;
+
async function listen(): Promise<void> {
- if (speech || !wanted || !(await granted())) return;
+ if (!due() || !(await granted()) || !due()) return;
const next = createRecognition();
if (!next) {
report({ op: 'phase', phase: 'no-service' });
@@ -68,7 +70,7 @@ async function listen(): Promise<void> {
next.onerror = (event) => {
if (event.error === 'no-speech') return;
halted = true;
- report({ op: 'phase', phase: PHASE_OF_ERROR[event.error] ?? 'no-service' });
+ report({ op: 'phase', phase: PHASE_OF_ERROR[event.error] ?? 'failed' });
};
next.onend = () => {
speech = null;
@@ -85,7 +87,7 @@ async function listen(): Promise<void> {
next.start();
} catch {
speech = null;
- report({ op: 'phase', phase: 'no-service' });
+ report({ op: 'phase', phase: 'failed' });
}
}
diff --git a/src/lib/bridge/hands-free.test.ts b/src/lib/bridge/hands-free.test.ts
index a579abd..43cfeaa 100644
--- a/src/lib/bridge/hands-free.test.ts
+++ b/src/lib/bridge/hands-free.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
-import { approvalOf, describeOrb, spoken, type OrbWorld } from './hands-free';
+import { approvalOf, describeOrb, type OrbWorld } from './hands-free';
import type { TabSession } from './tab-sessions';
const world: OrbWorld = {
@@ -62,15 +62,14 @@ describe('approvalOf', () => {
expect(ask).toEqual({ toolId: 't', action: 'injectCode', site: undefined, purpose: 'Sum the column', code: 'let a = 1;' });
});
+ it('never cuts the code short, since Allow runs every line of it', () => {
+ const code = `${'let a = 1;\n'.repeat(3_000)}exfiltrate();`;
+ expect(approvalOf({ toolId: 't', action: 'page.runCode', input: { code } }).code).toBe(code);
+ });
+
it('flattens any other input to one short line', () => {
const ask = approvalOf({ toolId: 't', action: 'page.fillInput', input: { selector: '#q', value: 'x'.repeat(400) } });
expect(ask.detail?.length).toBeLessThanOrEqual(180);
expect(ask.detail?.startsWith('selector: #q · value: xxx')).toBe(true);
});
});
-
-describe('spoken', () => {
- it('reads an action name the way a person would say it', () => {
- expect(spoken('page.clickElement')).toBe('Click element');
- });
-});
diff --git a/src/lib/bridge/hands-free.ts b/src/lib/bridge/hands-free.ts
index 7796546..4447297 100644
--- a/src/lib/bridge/hands-free.ts
+++ b/src/lib/bridge/hands-free.ts
@@ -1,6 +1,7 @@
import { browser, type Browser } from 'wxt/browser';
+import { injectContentScript } from '@/lib/actions/client';
import { MAX_STORED_FILE_BYTES } from '@/lib/files/report';
-import { replyOf } from '@/lib/handsfree/caption';
+import { replyOf, spoken } from '@/lib/handsfree/caption';
import {
DICTATION_CHANNEL,
HANDS_FREE_CHANNEL,
@@ -20,28 +21,28 @@ import { pickFocusIn } from './aeye';
import { putBytes } from './file-store';
import { openMicPermissionPage } from './mic-permission';
import { HANDS_FREE_KEY, endHandsFree, readHandsFree, writeHandsFree, type HandsFreeState } from './panel-view';
-import { onPanelPresence, onTurnSettled, runCommand } from './run-port';
+import { attachFile, closePanels, onPanelPresence, onTurnSettled, runCommand } from './run-port';
import { openSidePanel } from './side-panel';
import { TAB_SESSIONS_KEY, readTabSessions, type PendingApproval, type TabAnchor, type TabSession } from './tab-sessions';
import { THEME_KEY, readTheme, type ThemeId } from './theme';
import { SPEECH_SERVICE_KEY, handsFreeSupported, noteSpeechService } from './speech-support';
import { showToast } from './toast';
-import type { DaemonState } from './socket';
+import { DAEMON_STATE_KEY, type DaemonState } from './socket';
export const ORB_POSITION_KEY = 'browsentic/orbPosition';
export const PUSH_TO_TALK_KEY = 'browsentic/pushToTalk';
const DICTATION_KEY = 'browsentic/dictation';
-const DAEMON_KEY = 'browsentic/daemon';
+/** Approvals already announced elsewhere, kept where a revived worker still finds them. */
+const ANNOUNCED_KEY = 'browsentic/approvalsAnnounced';
const DICTATION_PAGE = 'dictation.html';
const HOLD_DICTATION_PAGE = `${DICTATION_PAGE}?hold`;
const NOTIFICATION_PREFIX = 'browsentic-approval:';
+const NO_CONTENT_SCRIPT = 'Receiving end does not exist';
const MAX_DETAIL_CHARS = 180;
-const MAX_CODE_CHARS = 4_000;
/** What every tab was last told. A cache: a revived worker starts it empty and repaints. */
const sent = new Map<number, string>();
const orbTabs = new Set<number>();
-const announced = new Set<string>();
let showing: boolean | null = null;
let listeningTab: number | null = null;
let panelOpen = false;
@@ -96,7 +97,7 @@ export function approvalOf(pending: PendingApproval): OrbApproval {
return {
...base,
purpose: typeof input.purpose === 'string' ? input.purpose : undefined,
- code: input.code.slice(0, MAX_CODE_CHARS),
+ code: input.code,
};
}
return { ...base, detail: flatten(pending.input) };
@@ -115,9 +116,20 @@ function flatten(input: unknown): string | undefined {
return text.length > MAX_DETAIL_CHARS ? `${text.slice(0, MAX_DETAIL_CHARS - 1)}…` : text;
}
-const linkOf = (daemon: DaemonState | null): OrbLink =>
+const linkOf = (daemon: DaemonState | null | undefined): OrbLink =>
!daemon?.paired ? 'off' : daemon.connected ? 'live' : 'pending';
+/** The part of the tab sessions an orb shows. A run rewrites the rest — usage, title, transcript state — on every step. */
+const orbFacts = (sessions: unknown): string =>
+ JSON.stringify(
+ Object.values((sessions ?? {}) as Record<string, TabSession>).map((session) => [
+ session.tabIds,
+ session.currentTabId,
+ session.runId,
+ session.pendingApproval?.toolId,
+ ]),
+ );
+
const sessionOn = (sessions: Record<string, TabSession>, tabId: number): TabSession | null =>
Object.values(sessions).find((session) => session.tabIds.includes(tabId)) ?? null;
@@ -157,6 +169,8 @@ async function paint(): Promise<void> {
!state.muted && link === 'live' && !panelOpen && !frontSession?.runId && !frontSession?.pendingApproval
? frontId
: null;
+ /* The tab in front is told afresh until it shows an orb, and healed if the extension was reloaded under it. */
+ if (frontId != null && !orbTabs.has(frontId)) sent.delete(frontId);
const world: OrbWorld = { theme, position, link, state, phase, pushToTalk, listeningTab: target };
await Promise.all(
@@ -166,24 +180,33 @@ async function paint(): Promise<void> {
const painted = JSON.stringify(view);
if (sent.get(tab.id) === painted) return;
sent.set(tab.id, painted);
- if (await post(tab.id, { channel: HANDS_FREE_CHANNEL, op: 'show', view })) orbTabs.add(tab.id);
+ if (await post(tab.id, { channel: HANDS_FREE_CHANNEL, op: 'show', view }, tab.id === frontId)) orbTabs.add(tab.id);
else orbTabs.delete(tab.id);
}),
);
await aimMic(target != null && orbTabs.has(target) ? target : null, pushToTalk);
- for (const session of Object.values(sessions)) {
- const pending = session.pendingApproval;
- if (!pending || announced.has(pending.toolId)) continue;
- announced.add(pending.toolId);
- if (session.currentTabId !== frontId) void askElsewhere(session, pending, frontId != null);
+ await announce(Object.values(sessions), frontId);
+}
+
+/** Each approval once: raised in a tab nobody is looking at, it is said where they are. */
+async function announce(sessions: TabSession[], frontId: number | null): Promise<void> {
+ const pending = sessions.flatMap((session) => (session.pendingApproval ? [{ session, ask: session.pendingApproval }] : []));
+ const stored = (await browser.storage.session.get(ANNOUNCED_KEY))[ANNOUNCED_KEY];
+ const told = new Set(Array.isArray(stored) ? (stored as string[]) : []);
+ const ids = pending.map(({ ask }) => ask.toolId);
+ if (ids.length !== told.size || ids.some((id) => !told.has(id))) {
+ await browser.storage.session.set({ [ANNOUNCED_KEY]: ids });
+ }
+ for (const { session, ask } of pending) {
+ if (!told.has(ask.toolId) && session.currentTabId !== frontId) void askElsewhere(session, ask, frontId != null);
}
}
async function clearOrbs(): Promise<void> {
await aimMic(null, false);
- announced.clear();
if (showing === false) return;
+ await browser.storage.session.remove(ANNOUNCED_KEY);
const tabs = await browser.tabs.query({});
await Promise.all(tabs.map((tab) => tab.id != null && !tab.discarded && post(tab.id, { channel: HANDS_FREE_CHANNEL, op: 'hide' })));
sent.clear();
@@ -192,9 +215,11 @@ async function clearOrbs(): Promise<void> {
}
async function aimMic(tabId: number | null, hold: boolean): Promise<void> {
+ const moved = listeningTab !== tabId;
listeningTab = tabId;
- if (tabId == null) await closeDictation();
- else await openDictation(hold);
+ /* A hold-to-talk page starts afresh for each tab, so words held down in one never land in another. */
+ if (tabId == null || (moved && hold)) await closeDictation();
+ if (tabId != null) await openDictation(hold);
}
async function dictationUrl(): Promise<URL | null> {
@@ -209,25 +234,30 @@ async function hasDictation(): Promise<boolean> {
return (await dictationUrl()) !== null;
}
-/** One page, in the mode asked for: a page left in the other mode is closed and opened afresh. */
+/**
+ * One page, in the mode asked for: a page left in the other mode is closed and opened afresh.
+ * Its phase is whatever the page itself last reported, so the orb never claims a microphone
+ * the page has not opened. A page that could not be created waits for a tap on the mic.
+ */
async function openDictation(hold: boolean): Promise<void> {
- if (import.meta.env.FIREFOX) return;
const open = await dictationUrl();
- if (open && open.searchParams.has('hold') === hold) return;
- if (open) await closeDictation();
- await browser.storage.session.set({ [DICTATION_KEY]: (hold ? 'held' : 'starting') satisfies DictationPhase });
+ if (open?.searchParams.has('hold') === hold) return;
+ if (!open && (await readPhase()) === 'failed') return;
+ await closeDictation();
await browser.offscreen
.createDocument({
url: hold ? HOLD_DICTATION_PAGE : DICTATION_PAGE,
reasons: [browser.offscreen.Reason.USER_MEDIA],
justification: 'Hands-free mode listens for spoken instructions while the side panel is closed.',
})
- .catch((error) => console.warn('[browsentic] could not start dictation:', error));
+ .catch(async (error) => {
+ console.warn('[browsentic] could not start dictation:', error);
+ await browser.storage.session.set({ [DICTATION_KEY]: 'failed' satisfies DictationPhase });
+ });
}
async function closeDictation(): Promise<void> {
- if (!(await hasDictation())) return;
- await browser.offscreen.closeDocument().catch(() => undefined);
+ if (await hasDictation()) await browser.offscreen.closeDocument().catch(() => undefined);
await browser.storage.session.remove(DICTATION_KEY);
}
@@ -265,7 +295,7 @@ async function sayNoService(): Promise<void> {
/** An approval raised in a tab nobody is looking at: a card on the page they are on, or the OS when the browser is behind. */
async function askElsewhere(session: TabSession, pending: PendingApproval, browserInFront: boolean): Promise<void> {
const title = 'Browsentic needs your OK';
- const body = `${spoken(pending.action)}${pending.site ? ` on ${pending.site}` : ''} — open the tab and tap the mic.`;
+ const body = `The agent wants to ${spoken(pending.action)}${pending.site ? ` on ${pending.site}` : ''} — open the tab and tap the mic.`;
const shown =
browserInFront &&
(await showToast({ toastId: `approval-${pending.toolId}`, tone: 'warn', title, body, tabId: session.currentTabId }));
@@ -280,14 +310,6 @@ async function askElsewhere(session: TabSession, pending: PendingApproval, brows
.catch(() => undefined);
}
-/** `page.clickElement` as a person would say it: “Click element”. */
-export const spoken = (action: string): string =>
- action
- .replace(/^page\./, '')
- .replace(/([a-z])([A-Z])/g, '$1 $2')
- .toLowerCase()
- .replace(/^./, (first) => first.toUpperCase());
-
function largestIcon(): string {
const icons = browser.runtime.getManifest().icons ?? {};
const [largest] = Object.keys(icons)
@@ -304,8 +326,8 @@ async function frontTab(): Promise<Browser.tabs.Tab | null> {
}
async function readDaemon(): Promise<DaemonState | null> {
- const stored = await browser.storage.session.get(DAEMON_KEY);
- return (stored[DAEMON_KEY] as DaemonState | undefined) ?? null;
+ const stored = await browser.storage.session.get(DAEMON_STATE_KEY);
+ return (stored[DAEMON_STATE_KEY] as DaemonState | undefined) ?? null;
}
async function readPhase(): Promise<DictationPhase | null> {
@@ -330,11 +352,14 @@ function asPosition(value: unknown): OrbPosition | null {
/* Answered through sendResponse by the orb, for the reason the toast gives: on Chrome a
returned promise is dropped, and the verdict is what says whether a tab has an orb. */
-async function post(tabId: number, command: OrbCommand): Promise<boolean> {
+async function post(tabId: number, command: OrbCommand, heal = false): Promise<boolean> {
+ const send = async () =>
+ ((await browser.tabs.sendMessage(tabId, command)) as { ok?: boolean } | undefined)?.ok === true;
try {
- return ((await browser.tabs.sendMessage(tabId, command)) as { ok?: boolean } | undefined)?.ok === true;
- } catch {
- return false;
+ return await send();
+ } catch (error) {
+ if (!heal || !String(error).includes(NO_CONTENT_SCRIPT) || !(await injectContentScript(tabId))) return false;
+ return send().catch(() => false);
}
}
@@ -366,8 +391,8 @@ async function attach(request: Extract<OrbRequest, { op: 'attach' }>, tab: TabAn
} catch (error) {
return { ok: false, error: error instanceof Error ? error.message : String(error) };
}
- runCommand({ op: 'attach', file: { id, name, mime, size }, tab });
- return { ok: true, fileId: id };
+ const attached = await attachFile({ id, name, mime, size }, tab);
+ return attached.ok ? { ok: true, fileId: id } : attached;
}
function answer(request: OrbRequest, tab: Browser.tabs.Tab & { id: number }, respond: (reply: unknown) => void): boolean {
@@ -418,7 +443,8 @@ function answer(request: OrbRequest, tab: Browser.tabs.Tab & { id: number }, res
respond({ ok: true });
return false;
case 'talk':
- if (tab.id === listeningTab) {
+ /* Letting go always reaches the page: the key can come up in a tab the mic has already left. */
+ if (!request.on || tab.id === listeningTab) {
const command: DictationCommand = { channel: DICTATION_CHANNEL, op: 'talk', on: request.on === true };
void browser.runtime.sendMessage(command).catch(() => undefined);
}
@@ -460,14 +486,10 @@ export function serveHandsFree(): void {
void syncHandsFree();
});
- onTurnSettled((sessionId, items) => {
+ onTurnSettled((tabId, items) => {
const reply = replyOf(items);
if (!reply) return;
- void (async () => {
- if (!(await readHandsFree())) return;
- const session = (await readTabSessions())[sessionId];
- if (session) await post(session.currentTabId, { channel: HANDS_FREE_CHANNEL, op: 'say', ...reply });
- })();
+ void readHandsFree().then((state) => state && post(tabId, { channel: HANDS_FREE_CHANNEL, op: 'say', ...reply }));
});
browser.notifications?.onClicked.addListener((notificationId) => {
@@ -485,13 +507,26 @@ export function serveHandsFree(): void {
sent.delete(tabId);
orbTabs.delete(tabId);
});
+ /* A page the content script cannot run in never asks for its orb, so the tab the mic is aimed
+ at is asked again once it has loaded, and the mic lets go of it if nothing answers. */
browser.tabs.onUpdated.addListener((tabId, changed) => {
- if (changed.status !== 'loading') return;
+ const leaving = changed.status === 'loading';
+ const landed = changed.status === 'complete' && tabId === listeningTab;
+ if (!leaving && !landed) return;
sent.delete(tabId);
orbTabs.delete(tabId);
+ if (landed) void syncHandsFree();
});
browser.storage.session.onChanged.addListener((changes) => {
- if (HANDS_FREE_KEY in changes || DICTATION_KEY in changes || TAB_SESSIONS_KEY in changes || DAEMON_KEY in changes) {
+ const { [HANDS_FREE_KEY]: handsFree, [TAB_SESSIONS_KEY]: sessions, [DAEMON_STATE_KEY]: daemon } = changes;
+ /* The panel and the orb never share the microphone: opening a panel ends hands-free, and starting it closes every panel. */
+ if (handsFree?.newValue && !handsFree.oldValue) closePanels();
+ if (
+ handsFree ||
+ DICTATION_KEY in changes ||
+ (sessions && orbFacts(sessions.oldValue) !== orbFacts(sessions.newValue)) ||
+ (daemon && linkOf(daemon.oldValue as DaemonState | undefined) !== linkOf(daemon.newValue as DaemonState | undefined))
+ ) {
void syncHandsFree();
}
});
diff --git a/src/lib/bridge/run-port.ts b/src/lib/bridge/run-port.ts
index 48c999b..46ad26e 100644
--- a/src/lib/bridge/run-port.ts
+++ b/src/lib/bridge/run-port.ts
@@ -125,7 +125,7 @@ export type RunMessage =
const ports = new Set<Browser.runtime.Port>();
const panelPorts = new Set<Browser.runtime.Port>();
const panelWatchers = new Set<(open: boolean) => void>();
-const settleWatchers = new Set<(sessionId: string, items: RunItem[]) => void>();
+const settleWatchers = new Set<(tabId: number, items: readonly RunItem[]) => void>();
const buffers = new Map<string, RunItem[]>();
const busy = new Set<string>();
const persistTimers = new Map<string, ReturnType<typeof setTimeout>>();
@@ -257,11 +257,23 @@ export function closePanels(): void {
for (const panel of panelPorts) post(panel, { op: 'close' });
}
-/** Fires once a turn is over — finished, failed or stopped — with the conversation as it now stands. */
-export function onTurnSettled(watch: (sessionId: string, items: RunItem[]) => void): void {
+/**
+ * Fires once a turn is over — finished, failed, stopped, answered in the browser, or refused
+ * before it began — with the tab it belongs to and what its conversation now says.
+ */
+export function onTurnSettled(watch: (tabId: number, items: readonly RunItem[]) => void): void {
settleWatchers.add(watch);
}
+function turnOver(tabId: number, items: readonly RunItem[]): void {
+ for (const watch of settleWatchers) watch(tabId, items);
+}
+
+/** The orb's attach, answered once the file belongs to a conversation, or with why it cannot. */
+export function attachFile(file: NewFile, anchor: TabAnchor): Promise<Attached> {
+ return serialized(() => attach(file, anchor));
+}
+
/** The panel's commands, for a surface in the background's own reach that has no port — the hands-free orb. */
export function runCommand(command: RunCommand): void {
handle(command);
@@ -376,16 +388,19 @@ function handle(command: RunCommand): void {
}
}
+type Attached = { ok: true } | { ok: false; error: string };
+
/** A file belongs to the conversation of the tab it was dropped on, which is started here if need be. */
-async function attach(file: NewFile, anchor: TabAnchor): Promise<void> {
+async function attach(file: NewFile, anchor: TabAnchor): Promise<Attached> {
const ensured = await ensureSessionForTab(anchor);
if (!ensured.ok) {
await removeFile(file.id);
broadcast({ op: 'event', runId: LOCAL_RUN, event: { kind: 'error', code: 'SESSION_LIMIT', message: ensured.message } });
- return;
+ return { ok: false, error: ensured.message };
}
await indexFile(file, ensured.session.sessionId);
await requestAnalysis(file.id);
+ return { ok: true };
}
/** Clears out files whose conversation has left both the open tabs and history. */
@@ -456,11 +471,10 @@ async function settle(sessionId: string): Promise<void> {
await dropDiagnosticsForSession(sessionId);
await patchSession(sessionId, { runId: null, pendingApproval: undefined, handing: undefined });
await syncRunIndicator();
- const settledItems = buffers.get(sessionId) ?? [];
- for (const watch of settleWatchers) watch(sessionId, settledItems);
const session = (await readTabSessions())[sessionId];
if (session) {
+ turnOver(session.currentTabId, buffers.get(sessionId) ?? []);
const tab = await browser.tabs.get(session.currentTabId).catch(() => null);
if (tab?.url) await patchSession(sessionId, { url: tab.url, host: hostOf(tab.url) });
}
@@ -485,11 +499,12 @@ async function instruct(
runId: LOCAL_RUN,
event: { kind: 'error', code: 'SESSION_LIMIT', message: ensured.message },
});
+ turnOver(anchor.tabId, [notice('error', `SESSION_LIMIT: ${ensured.message}`)]);
return;
}
+ const { sessionId } = ensured.session;
if (isContextCommand(text)) {
- const { sessionId } = ensured.session;
const breakdown = await contextBreakdown(ensured.session, await bufferFor(sessionId));
await append(sessionId, { kind: 'user', id: nextId(), text: CONTEXT_COMMAND });
await append(sessionId, { kind: 'context', id: nextId(), breakdown });
@@ -497,11 +512,13 @@ async function instruct(
}
const outcome = await startTurn(ensured.session, text, { agentSkillId, focus, liveTools, fastPath: true });
- if (outcome !== 'busy') return;
- await append(
- ensured.session.sessionId,
- notice('error', 'RUN_IN_PROGRESS: This conversation is still running — stop it before sending another instruction.'),
- );
+ if (outcome === 'busy') {
+ await append(
+ sessionId,
+ notice('error', 'RUN_IN_PROGRESS: This conversation is still running — stop it before sending another instruction.'),
+ );
+ }
+ if (outcome !== 'started') turnOver(anchor.tabId, await bufferFor(sessionId));
}
type TurnOutcome = 'started' | 'local' | 'busy' | 'offline';
diff --git a/src/lib/handsfree/events.ts b/src/lib/handsfree/events.ts
index ccc1f74..9c806f8 100644
--- a/src/lib/handsfree/events.ts
+++ b/src/lib/handsfree/events.ts
@@ -6,7 +6,11 @@ export const DICTATION_CHANNEL = 'browsentic/dictation';
export const AUTO_SEND_MS = 1600;
-/** What the offscreen recognizer reports about itself. `held`: in hold-to-talk, ready and waiting for the key. */
+/**
+ * What the offscreen recognizer reports about itself. `held`: in hold-to-talk, ready and waiting
+ * for the key. `no-service` is only ever `network`, the one error that says the browser has no
+ * speech service; any other stop is `failed`, worth a retry and no verdict on the browser.
+ */
export type DictationPhase =
| 'starting'
| 'listening'
@@ -15,6 +19,7 @@ export type DictationPhase =
| 'blocked'
| 'no-mic'
| 'no-service'
+ | 'failed'
| 'yielded';
/**
@@ -107,6 +112,8 @@ const ORB_REQUESTS = new Set([
'sync', 'submit', 'cancel', 'decide', 'openPanel', 'listen', 'grantMic', 'pick', 'attach', 'detach', 'move',
'pushToTalk', 'talk',
]);
+const DICTATION_REPORTS = new Set(['phase', 'heard']);
+const DICTATION_COMMANDS = new Set(['talk']);
function onChannel(message: unknown, channel: string, ops: Set<string>): boolean {
if (typeof message !== 'object' || message === null) return false;
@@ -121,10 +128,10 @@ export const isOrbRequest = (message: unknown): message is OrbRequest =>
onChannel(message, HANDS_FREE_CHANNEL, ORB_REQUESTS);
export const isDictationReport = (message: unknown): message is DictationReport =>
- onChannel(message, DICTATION_CHANNEL, new Set(['phase', 'heard']));
+ onChannel(message, DICTATION_CHANNEL, DICTATION_REPORTS);
export const isDictationCommand = (message: unknown): message is DictationCommand =>
- onChannel(message, DICTATION_CHANNEL, new Set(['talk']));
+ onChannel(message, DICTATION_CHANNEL, DICTATION_COMMANDS);
/** Lucide paths, copied for the same reason as the rail's: the content script carries no icon package. */
export const ORB_ICONS = {
diff --git a/src/lib/handsfree/host.test.ts b/src/lib/handsfree/host.test.ts
index 43dd211..b5c88ca 100644
--- a/src/lib/handsfree/host.test.ts
+++ b/src/lib/handsfree/host.test.ts
@@ -165,6 +165,15 @@ describe('the hands-free orb', () => {
expect(requests('openPanel')).toHaveLength(1);
});
+ it('says so when recognition stops, and a press tries again', async () => {
+ await show();
+ await show({ voice: 'failed' });
+ expect($('.wrap').dataset.tone).toBe('trouble');
+ expect(shown()).toBe('Speech recognition stopped — tap the mic to try again.');
+ press();
+ expect(requests('listen').at(-1)).toMatchObject({ on: true });
+ });
+
it('turns ember for an approval and opens the request on a press, which a forged click cannot answer', async () => {
await show();
await show({
@@ -340,6 +349,15 @@ describe('the hands-free orb', () => {
expect(requests('talk')).toHaveLength(0);
});
+ it('lets go of the mic when the page takes the orb away mid-hold', async () => {
+ await show(held);
+ key('keydown', 'ControlLeft');
+ vi.advanceTimersByTime(250);
+ document.getElementById('browsentic-hands-free')?.remove();
+ key('keydown', 'ControlLeft', { repeat: true });
+ expect(requests('talk').at(-1)).toMatchObject({ on: false });
+ });
+
it('stays quiet where it is not the tab being listened for', async () => {
await show({ pushToTalk: true, voice: 'paused' });
key('keydown', 'ControlLeft');
diff --git a/src/lib/handsfree/host.ts b/src/lib/handsfree/host.ts
index 7f6b5de..746ff82 100644
--- a/src/lib/handsfree/host.ts
+++ b/src/lib/handsfree/host.ts
@@ -1,9 +1,9 @@
import { browser } from 'wxt/browser';
-import type { FocusedElement } from '@/lib/actions/protocol';
+import { focusName, type FocusedElement } from '@/lib/actions/protocol';
import { MAX_STORED_FILE_BYTES } from '@/lib/files/report';
import { OVERLAY_ATTRIBUTE } from '@/lib/overlay';
import { RAIL_PALETTES, RAIL_TONES } from '@/lib/rail/events';
-import { readingMs, sharedWords, wordsOf } from './caption';
+import { readingMs, sharedWords, spoken, wordsOf } from './caption';
import {
AUTO_SEND_MS,
HANDS_FREE_CHANNEL,
@@ -84,6 +84,7 @@ const VOICE_HINTS: Partial<Record<OrbView['voice'], string>> = {
blocked: 'The microphone is blocked for Browsentic — tap the mic to fix it.',
'no-mic': 'No microphone was found — tap the mic to try again.',
'no-service': 'No speech service answered — tap the mic to try again.',
+ failed: 'Speech recognition stopped — tap the mic to try again.',
yielded: 'Another page took the microphone — tap the mic to listen here.',
};
@@ -95,11 +96,6 @@ function escape(text: string): string {
return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
}
-const spoken = (action: string): string =>
- action.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase();
-
-const focusName = (focus: FocusedElement): string => focus.label?.trim() || focus.role || focus.tag;
-
const request = <T = unknown>(message: Unsent<OrbRequest>): Promise<T | undefined> =>
browser.runtime.sendMessage({ channel: HANDS_FREE_CHANNEL, ...message }).catch(() => undefined) as Promise<T | undefined>;
@@ -966,6 +962,7 @@ function mountOrb(first: OrbView): Orb | null {
}
function remove(): void {
+ holdCancel();
alive = false;
for (const name of Object.keys(timers) as (keyof typeof timers)[]) stop(name);
window.removeEventListener('resize', onResize);