Skip to content

An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.

feat(hands-free): listen through an offscreen page once the side panel is closed — for the one tab in front, never beside a panel, and only while the key is held when hold to talk is on

Shahriar9c633604596e
diff --git a/src/extension/entrypoints/background.ts b/src/extension/entrypoints/background.ts
index f14b0c2..60da0fa 100644
--- a/src/extension/entrypoints/background.ts
+++ b/src/extension/entrypoints/background.ts
@@ -16,6 +16,8 @@ import { closePanels, onPanelPresence, serveRunPorts, serveTabSessions, sweepFil
import { clearStrandedRail, serveRail, setPanelCollapsed, syncRail } from '@/lib/bridge/rail';
import { serveTimers } from '@/lib/bridge/timer';
import { serveToast } from '@/lib/bridge/toast';
+import { serveHandsFree } from '@/lib/bridge/hands-free';
+import { endHandsFree } from '@/lib/bridge/panel-view';
import { closeSidebar, openSidePanel } from '@/lib/bridge/side-panel';
import { isAgentKind } from '@/lib/agents/catalog';
import {
@@ -145,6 +147,7 @@ export default defineBackground(() => {
}
if (message.op === 'panelOpened') {
void setPanelCollapsed(false);
+ void endHandsFree();
sendResponse(success(true));
return;
}
@@ -208,6 +211,7 @@ export default defineBackground(() => {
serveTabSessions();
serveRail();
serveToast();
+ serveHandsFree();
void syncRail();
onWelcome(() => {
diff --git a/src/extension/entrypoints/dictation/index.html b/src/extension/entrypoints/dictation/index.html
new file mode 100644
index 0000000..1d14921
--- /dev/null
+++ b/src/extension/entrypoints/dictation/index.html
@@ -0,0 +1,11 @@
+<!doctype html>
+<html lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="manifest.exclude" content="['firefox']" />
+ <title>Browsentic — hands-free dictation</title>
+ </head>
+ <body>
+ <script type="module" src="./main.ts"></script>
+ </body>
+</html>
diff --git a/src/extension/entrypoints/dictation/main.ts b/src/extension/entrypoints/dictation/main.ts
new file mode 100644
index 0000000..74dac9a
--- /dev/null
+++ b/src/extension/entrypoints/dictation/main.ts
@@ -0,0 +1,115 @@
+import { browser } from 'wxt/browser';
+import { micPermission, watchMicPermission } from '@/lib/bridge/mic-permission';
+import { createRecognition, transcriptOf, type SpeechRecognitionLike } from '@/lib/bridge/recognition';
+import {
+ DICTATION_CHANNEL,
+ isDictationCommand,
+ type DictationPhase,
+ type DictationReport,
+} from '@/lib/handsfree/events';
+
+const RESTART_DELAY_MS = 400;
+
+/*
+ * The offscreen document hands-free mode listens through. It only exists while the orb should
+ * be listening — the background creates it to start and closes it to stop. An `aborted` it did
+ * not ask for is therefore another recognizer taking the microphone, since Chrome runs one at a
+ * time: it yields rather than taking the mic back.
+ *
+ * Opened as `dictation.html?hold` it is hold-to-talk instead: it waits, holding no microphone,
+ * until told to talk, and when told to stop it lets the recognizer finish the words already
+ * spoken rather than cutting them off.
+ */
+const PHASE_OF_ERROR: Record<string, DictationPhase> = {
+ 'not-allowed': 'blocked',
+ 'service-not-allowed': 'blocked',
+ 'audio-capture': 'no-mic',
+ network: 'no-service',
+ aborted: 'yielded',
+};
+
+const holding = new URLSearchParams(location.search).has('hold');
+
+type Unsent<T> = T extends unknown ? Omit<T, 'channel'> : never;
+
+let speech: SpeechRecognitionLike | null = null;
+let wanted = !holding;
+let halted = false;
+let unsettled = '';
+
+function report(message: Unsent<DictationReport>): void {
+ void browser.runtime.sendMessage({ channel: DICTATION_CHANNEL, ...message }).catch(() => undefined);
+}
+
+async function granted(): Promise<boolean> {
+ const permission = await micPermission();
+ if (permission === 'granted') return true;
+ report({ op: 'phase', phase: permission === 'prompt' ? 'needs-mic' : 'blocked' });
+ return false;
+}
+
+async function listen(): Promise<void> {
+ if (speech || !wanted || !(await granted())) return;
+ const next = createRecognition();
+ if (!next) {
+ report({ op: 'phase', phase: 'no-service' });
+ return;
+ }
+
+ halted = false;
+ unsettled = '';
+ next.onstart = () => report({ op: 'phase', phase: 'listening' });
+ next.onresult = (event) => {
+ const { interim, final } = transcriptOf(event);
+ unsettled = interim;
+ if (interim) report({ op: 'heard', text: interim, final: false });
+ if (final) report({ op: 'heard', text: final, final: true });
+ };
+ next.onerror = (event) => {
+ if (event.error === 'no-speech') return;
+ halted = true;
+ report({ op: 'phase', phase: PHASE_OF_ERROR[event.error] ?? 'no-service' });
+ };
+ next.onend = () => {
+ speech = null;
+ if (unsettled && !wanted) report({ op: 'heard', text: unsettled, final: true });
+ unsettled = '';
+ if (halted) return;
+ if (wanted) setTimeout(() => void listen(), RESTART_DELAY_MS);
+ else report({ op: 'phase', phase: 'held' });
+ };
+
+ speech = next;
+ report({ op: 'phase', phase: 'starting' });
+ try {
+ next.start();
+ } catch {
+ speech = null;
+ report({ op: 'phase', phase: 'no-service' });
+ }
+}
+
+async function wait(): Promise<void> {
+ if (await granted()) report({ op: 'phase', phase: 'held' });
+}
+
+function talk(on: boolean): void {
+ wanted = on;
+ if (on) {
+ void listen();
+ return;
+ }
+ if (speech) speech.stop();
+ else void wait();
+}
+
+browser.runtime.onMessage.addListener((message: unknown) => {
+ if (holding && isDictationCommand(message)) talk(message.on);
+});
+
+watchMicPermission((state) => {
+ if (state === 'denied') report({ op: 'phase', phase: 'blocked' });
+ else if (state === 'granted') void (wanted ? listen() : wait());
+});
+
+void (holding ? wait() : listen());
diff --git a/src/extension/entrypoints/mic-permission/App.tsx b/src/extension/entrypoints/mic-permission/App.tsx
index 0a7b915..b0ff784 100644
--- a/src/extension/entrypoints/mic-permission/App.tsx
+++ b/src/extension/entrypoints/mic-permission/App.tsx
@@ -11,7 +11,7 @@ const CLOSE_AFTER_MS = 1500;
const COPY: Record<Phase, { title: string; body: string }> = {
asking: {
title: 'Allow the microphone',
- body: 'Your browser is asking above. Choose Allow, and dictation works in the side panel and the popup from then on.',
+ body: 'Your browser is asking above. Choose Allow, and dictation works in the side panel, the popup and hands-free mode from then on.',
},
granted: {
title: 'The microphone is on',
diff --git a/src/lib/bridge/aeye.ts b/src/lib/bridge/aeye.ts
index 83f8137..17c13bc 100644
--- a/src/lib/bridge/aeye.ts
+++ b/src/lib/bridge/aeye.ts
@@ -17,8 +17,11 @@ interface PickedElement {
export async function pickFocus(): Promise<PickOutcome> {
const [tab] = await browser.tabs.query({ active: true, currentWindow: true });
if (tab?.id == null) return { error: 'No active tab to point at' };
+ return pickFocusIn({ id: tab.id, windowId: tab.windowId });
+}
- const result = await pickInTab({ id: tab.id, windowId: tab.windowId });
+export async function pickFocusIn(tab: { id: number; windowId?: number }): Promise<PickOutcome> {
+ const result = await pickInTab(tab);
if (!result.ok) {
return result.error.code === 'PICK_CANCELLED' ? { cancelled: true } : { error: result.error.message };
}
diff --git a/src/lib/bridge/hands-free.test.ts b/src/lib/bridge/hands-free.test.ts
new file mode 100644
index 0000000..a579abd
--- /dev/null
+++ b/src/lib/bridge/hands-free.test.ts
@@ -0,0 +1,76 @@
+import { describe, expect, it } from 'vitest';
+import { approvalOf, describeOrb, spoken, type OrbWorld } from './hands-free';
+import type { TabSession } from './tab-sessions';
+
+const world: OrbWorld = {
+ theme: 'ember',
+ position: null,
+ link: 'live',
+ state: { muted: false, since: 1 },
+ phase: 'listening',
+ pushToTalk: false,
+ listeningTab: 7,
+};
+
+const session = (patch: Partial<TabSession>): TabSession => ({
+ sessionId: 's',
+ mainTabId: 7,
+ tabIds: [7],
+ currentTabId: 7,
+ windowId: 1,
+ title: 'Page',
+ runId: null,
+ turns: 0,
+ createdAt: 0,
+ lastActivityAt: 0,
+ ...patch,
+});
+
+describe('describeOrb', () => {
+ it('listens only in the tab the microphone is aimed at', () => {
+ expect(describeOrb(world, 7, null).voice).toBe('listening');
+ expect(describeOrb(world, 8, null).voice).toBe('paused');
+ });
+
+ it('carries hold-to-talk to every tab, and the held phase to the one listening', () => {
+ const holding = { ...world, pushToTalk: true, phase: 'held' as const };
+ expect(describeOrb(holding, 7, null)).toMatchObject({ pushToTalk: true, voice: 'held' });
+ expect(describeOrb(holding, 8, null)).toMatchObject({ pushToTalk: true, voice: 'paused' });
+ });
+
+ it('says muted everywhere once muted', () => {
+ expect(describeOrb({ ...world, state: { muted: true, since: 1 } }, 7, null).voice).toBe('muted');
+ expect(describeOrb({ ...world, state: { muted: true, since: 1 } }, 8, null).voice).toBe('muted');
+ });
+
+ it('shows a waiting approval ahead of the run it belongs to', () => {
+ const asking = session({
+ runId: 'r',
+ pendingApproval: { toolId: 't', action: 'page.clickElement', input: { selector: '#go' }, site: 'x.test' },
+ });
+ expect(describeOrb(world, 7, session({ runId: 'r' })).run).toBe('working');
+ expect(describeOrb(world, 7, asking)).toMatchObject({
+ run: 'approval',
+ approval: { toolId: 't', action: 'clickElement', site: 'x.test', detail: 'selector: #go' },
+ });
+ });
+});
+
+describe('approvalOf', () => {
+ it('carries code the agent wrote whole, with its purpose, and no one-line detail', () => {
+ const ask = approvalOf({ toolId: 't', action: 'page.injectCode', input: { purpose: 'Sum the column', code: 'let a = 1;' } });
+ expect(ask).toEqual({ toolId: 't', action: 'injectCode', site: undefined, purpose: 'Sum the column', code: 'let a = 1;' });
+ });
+
+ it('flattens any other input to one short line', () => {
+ const ask = approvalOf({ toolId: 't', action: 'page.fillInput', input: { selector: '#q', value: 'x'.repeat(400) } });
+ expect(ask.detail?.length).toBeLessThanOrEqual(180);
+ expect(ask.detail?.startsWith('selector: #q · value: xxx')).toBe(true);
+ });
+});
+
+describe('spoken', () => {
+ it('reads an action name the way a person would say it', () => {
+ expect(spoken('page.clickElement')).toBe('Click element');
+ });
+});
diff --git a/src/lib/bridge/hands-free.ts b/src/lib/bridge/hands-free.ts
new file mode 100644
index 0000000..7796546
--- /dev/null
+++ b/src/lib/bridge/hands-free.ts
@@ -0,0 +1,505 @@
+import { browser, type Browser } from 'wxt/browser';
+import { MAX_STORED_FILE_BYTES } from '@/lib/files/report';
+import { replyOf } from '@/lib/handsfree/caption';
+import {
+ DICTATION_CHANNEL,
+ HANDS_FREE_CHANNEL,
+ isDictationReport,
+ isOrbRequest,
+ type DictationCommand,
+ type DictationPhase,
+ type DictationReport,
+ type OrbApproval,
+ type OrbCommand,
+ type OrbLink,
+ type OrbPosition,
+ type OrbRequest,
+ type OrbView,
+} from '@/lib/handsfree/events';
+import { pickFocusIn } from './aeye';
+import { putBytes } from './file-store';
+import { openMicPermissionPage } from './mic-permission';
+import { HANDS_FREE_KEY, endHandsFree, readHandsFree, writeHandsFree, type HandsFreeState } from './panel-view';
+import { onPanelPresence, onTurnSettled, runCommand } from './run-port';
+import { openSidePanel } from './side-panel';
+import { TAB_SESSIONS_KEY, readTabSessions, type PendingApproval, type TabAnchor, type TabSession } from './tab-sessions';
+import { THEME_KEY, readTheme, type ThemeId } from './theme';
+import { SPEECH_SERVICE_KEY, handsFreeSupported, noteSpeechService } from './speech-support';
+import { showToast } from './toast';
+import type { DaemonState } from './socket';
+
+export const ORB_POSITION_KEY = 'browsentic/orbPosition';
+export const PUSH_TO_TALK_KEY = 'browsentic/pushToTalk';
+const DICTATION_KEY = 'browsentic/dictation';
+const DAEMON_KEY = 'browsentic/daemon';
+const DICTATION_PAGE = 'dictation.html';
+const HOLD_DICTATION_PAGE = `${DICTATION_PAGE}?hold`;
+const NOTIFICATION_PREFIX = 'browsentic-approval:';
+const MAX_DETAIL_CHARS = 180;
+const MAX_CODE_CHARS = 4_000;
+
+/** What every tab was last told. A cache: a revived worker starts it empty and repaints. */
+const sent = new Map<number, string>();
+const orbTabs = new Set<number>();
+const announced = new Set<string>();
+let showing: boolean | null = null;
+let listeningTab: number | null = null;
+let panelOpen = false;
+let serviceAnswered = false;
+
+let inFlight: Promise<void> | null = null;
+let restated = false;
+
+export function syncHandsFree(): Promise<void> {
+ if (inFlight) {
+ restated = true;
+ return inFlight;
+ }
+ inFlight = (async () => {
+ do {
+ restated = false;
+ await paint().catch((error) => console.warn('[browsentic] hands-free paint failed:', error));
+ } while (restated);
+ inFlight = null;
+ })();
+ return inFlight;
+}
+
+export interface OrbWorld {
+ theme: ThemeId;
+ position: OrbPosition | null;
+ link: OrbLink;
+ state: HandsFreeState;
+ phase: DictationPhase | null;
+ pushToTalk: boolean;
+ listeningTab: number | null;
+}
+
+export function describeOrb(world: OrbWorld, tabId: number, session: TabSession | null): OrbView {
+ const pending = session?.pendingApproval;
+ return {
+ theme: world.theme,
+ since: world.state.since,
+ position: world.position,
+ link: world.link,
+ run: pending ? 'approval' : session?.runId ? 'working' : 'idle',
+ voice: world.state.muted ? 'muted' : tabId === world.listeningTab ? (world.phase ?? 'starting') : 'paused',
+ pushToTalk: world.pushToTalk,
+ ...(pending && { approval: approvalOf(pending) }),
+ };
+}
+
+export function approvalOf(pending: PendingApproval): OrbApproval {
+ const base = { toolId: pending.toolId, action: pending.action.replace(/^page\./, ''), site: pending.site };
+ const input = pending.input as Record<string, unknown> | null;
+ if (typeof input?.code === 'string' && input.code) {
+ return {
+ ...base,
+ purpose: typeof input.purpose === 'string' ? input.purpose : undefined,
+ code: input.code.slice(0, MAX_CODE_CHARS),
+ };
+ }
+ return { ...base, detail: flatten(pending.input) };
+}
+
+function flatten(input: unknown): string | undefined {
+ if (input == null) return undefined;
+ const text =
+ typeof input === 'object'
+ ? Object.entries(input as Record<string, unknown>)
+ .filter(([, value]) => value !== undefined && value !== '')
+ .map(([key, value]) => `${key}: ${typeof value === 'string' ? value : JSON.stringify(value)}`)
+ .join(' · ')
+ : String(input);
+ if (!text) return undefined;
+ return text.length > MAX_DETAIL_CHARS ? `${text.slice(0, MAX_DETAIL_CHARS - 1)}…` : text;
+}
+
+const linkOf = (daemon: DaemonState | null): OrbLink =>
+ !daemon?.paired ? 'off' : daemon.connected ? 'live' : 'pending';
+
+const sessionOn = (sessions: Record<string, TabSession>, tabId: number): TabSession | null =>
+ Object.values(sessions).find((session) => session.tabIds.includes(tabId)) ?? null;
+
+/**
+ * The microphone listens for exactly one tab: the one in front of a focused window, while
+ * its conversation is free to take an instruction and no side panel is open to listen
+ * instead. Every other tab's orb is painted as paused.
+ */
+async function paint(): Promise<void> {
+ const state = await readHandsFree();
+ if (!state) {
+ await clearOrbs();
+ return;
+ }
+ /* The panel never offers hands-free where speech cannot work; this catches the rest — a flag
+ left from before the service was found missing, or written by anything but the panel. */
+ if (!(await handsFreeSupported())) {
+ await endHandsFree();
+ return;
+ }
+ showing = true;
+
+ const [daemon, sessions, theme, position, phase, pushToTalk, front, tabs] = await Promise.all([
+ readDaemon(),
+ readTabSessions(),
+ readTheme(),
+ readPosition(),
+ readPhase(),
+ readPushToTalk(),
+ frontTab(),
+ browser.tabs.query({}),
+ ]);
+ const link = linkOf(daemon);
+ const frontId = front?.id ?? null;
+ const frontSession = frontId == null ? null : sessionOn(sessions, frontId);
+ const target =
+ !state.muted && link === 'live' && !panelOpen && !frontSession?.runId && !frontSession?.pendingApproval
+ ? frontId
+ : null;
+
+ const world: OrbWorld = { theme, position, link, state, phase, pushToTalk, listeningTab: target };
+ await Promise.all(
+ tabs.map(async (tab) => {
+ if (tab.id == null || tab.discarded) return;
+ const view = describeOrb(world, tab.id, sessionOn(sessions, tab.id));
+ const painted = JSON.stringify(view);
+ if (sent.get(tab.id) === painted) return;
+ sent.set(tab.id, painted);
+ if (await post(tab.id, { channel: HANDS_FREE_CHANNEL, op: 'show', view })) orbTabs.add(tab.id);
+ else orbTabs.delete(tab.id);
+ }),
+ );
+
+ await aimMic(target != null && orbTabs.has(target) ? target : null, pushToTalk);
+ for (const session of Object.values(sessions)) {
+ const pending = session.pendingApproval;
+ if (!pending || announced.has(pending.toolId)) continue;
+ announced.add(pending.toolId);
+ if (session.currentTabId !== frontId) void askElsewhere(session, pending, frontId != null);
+ }
+}
+
+async function clearOrbs(): Promise<void> {
+ await aimMic(null, false);
+ announced.clear();
+ if (showing === false) return;
+ const tabs = await browser.tabs.query({});
+ await Promise.all(tabs.map((tab) => tab.id != null && !tab.discarded && post(tab.id, { channel: HANDS_FREE_CHANNEL, op: 'hide' })));
+ sent.clear();
+ orbTabs.clear();
+ showing = false;
+}
+
+async function aimMic(tabId: number | null, hold: boolean): Promise<void> {
+ listeningTab = tabId;
+ if (tabId == null) await closeDictation();
+ else await openDictation(hold);
+}
+
+async function dictationUrl(): Promise<URL | null> {
+ if (import.meta.env.FIREFOX) return null;
+ const [open] = await browser.runtime
+ .getContexts({ contextTypes: [browser.runtime.ContextType.OFFSCREEN_DOCUMENT] })
+ .catch(() => []);
+ return open?.documentUrl ? new URL(open.documentUrl) : null;
+}
+
+async function hasDictation(): Promise<boolean> {
+ return (await dictationUrl()) !== null;
+}
+
+/** One page, in the mode asked for: a page left in the other mode is closed and opened afresh. */
+async function openDictation(hold: boolean): Promise<void> {
+ if (import.meta.env.FIREFOX) return;
+ const open = await dictationUrl();
+ if (open && open.searchParams.has('hold') === hold) return;
+ if (open) await closeDictation();
+ await browser.storage.session.set({ [DICTATION_KEY]: (hold ? 'held' : 'starting') satisfies DictationPhase });
+ await browser.offscreen
+ .createDocument({
+ url: hold ? HOLD_DICTATION_PAGE : DICTATION_PAGE,
+ reasons: [browser.offscreen.Reason.USER_MEDIA],
+ justification: 'Hands-free mode listens for spoken instructions while the side panel is closed.',
+ })
+ .catch((error) => console.warn('[browsentic] could not start dictation:', error));
+}
+
+async function closeDictation(): Promise<void> {
+ if (!(await hasDictation())) return;
+ await browser.offscreen.closeDocument().catch(() => undefined);
+ await browser.storage.session.remove(DICTATION_KEY);
+}
+
+async function hear(report: DictationReport): Promise<void> {
+ if (report.op === 'phase') {
+ if (report.phase === 'no-service' && (await noteSpeechService('missing')) === 'missing') {
+ await sayNoService();
+ return;
+ }
+ if (await hasDictation()) await browser.storage.session.set({ [DICTATION_KEY]: report.phase });
+ return;
+ }
+ if (!serviceAnswered) {
+ serviceAnswered = true;
+ void noteSpeechService('works');
+ }
+ if (listeningTab == null) await syncHandsFree();
+ if (listeningTab == null) return;
+ void post(listeningTab, { channel: HANDS_FREE_CHANNEL, op: 'heard', text: report.text, final: report.final });
+}
+
+/**
+ * The service failed before it ever transcribed a word here, so this browser has none, and the
+ * mic goes with the storage change. The toast says why, since the panel it came from is closed.
+ */
+async function sayNoService(): Promise<void> {
+ await showToast({
+ toastId: 'hands-free-no-speech',
+ tone: 'warn',
+ title: 'Hands-free is not available here',
+ body: 'This browser has no speech service to transcribe with. Open Browsentic from the toolbar to keep going by typing.',
+ });
+}
+
+/** An approval raised in a tab nobody is looking at: a card on the page they are on, or the OS when the browser is behind. */
+async function askElsewhere(session: TabSession, pending: PendingApproval, browserInFront: boolean): Promise<void> {
+ const title = 'Browsentic needs your OK';
+ const body = `${spoken(pending.action)}${pending.site ? ` on ${pending.site}` : ''} — open the tab and tap the mic.`;
+ const shown =
+ browserInFront &&
+ (await showToast({ toastId: `approval-${pending.toolId}`, tone: 'warn', title, body, tabId: session.currentTabId }));
+ if (shown) return;
+ await browser.notifications
+ ?.create(`${NOTIFICATION_PREFIX}${session.currentTabId}:${pending.toolId}`, {
+ type: 'basic',
+ iconUrl: largestIcon(),
+ title,
+ message: body,
+ })
+ .catch(() => undefined);
+}
+
+/** `page.clickElement` as a person would say it: “Click element”. */
+export const spoken = (action: string): string =>
+ action
+ .replace(/^page\./, '')
+ .replace(/([a-z])([A-Z])/g, '$1 $2')
+ .toLowerCase()
+ .replace(/^./, (first) => first.toUpperCase());
+
+function largestIcon(): string {
+ const icons = browser.runtime.getManifest().icons ?? {};
+ const [largest] = Object.keys(icons)
+ .map(Number)
+ .sort((a, b) => b - a);
+ return icons[largest] ?? '';
+}
+
+async function frontTab(): Promise<Browser.tabs.Tab | null> {
+ const window = await browser.windows.getLastFocused({ windowTypes: ['normal'] }).catch(() => null);
+ if (!window?.focused || window.id == null) return null;
+ const [tab] = await browser.tabs.query({ active: true, windowId: window.id }).catch(() => []);
+ return tab ?? null;
+}
+
+async function readDaemon(): Promise<DaemonState | null> {
+ const stored = await browser.storage.session.get(DAEMON_KEY);
+ return (stored[DAEMON_KEY] as DaemonState | undefined) ?? null;
+}
+
+async function readPhase(): Promise<DictationPhase | null> {
+ const stored = await browser.storage.session.get(DICTATION_KEY);
+ return (stored[DICTATION_KEY] as DictationPhase | undefined) ?? null;
+}
+
+async function readPushToTalk(): Promise<boolean> {
+ return (await browser.storage.local.get(PUSH_TO_TALK_KEY))[PUSH_TO_TALK_KEY] === true;
+}
+
+async function readPosition(): Promise<OrbPosition | null> {
+ const stored = await browser.storage.local.get(ORB_POSITION_KEY);
+ return asPosition(stored[ORB_POSITION_KEY]);
+}
+
+function asPosition(value: unknown): OrbPosition | null {
+ const point = value as Partial<OrbPosition> | null;
+ const within = (n: unknown): n is number => typeof n === 'number' && n >= 0 && n <= 1;
+ return point && within(point.x) && within(point.y) ? { x: point.x, y: point.y } : null;
+}
+
+/* Answered through sendResponse by the orb, for the reason the toast gives: on Chrome a
+ returned promise is dropped, and the verdict is what says whether a tab has an orb. */
+async function post(tabId: number, command: OrbCommand): Promise<boolean> {
+ try {
+ return ((await browser.tabs.sendMessage(tabId, command)) as { ok?: boolean } | undefined)?.ok === true;
+ } catch {
+ return false;
+ }
+}
+
+const anchorOf = (tab: Browser.tabs.Tab & { id: number }): TabAnchor => ({
+ tabId: tab.id,
+ url: tab.url,
+ windowId: tab.windowId,
+ title: tab.title,
+});
+
+async function sessionIdFor(tabId: number): Promise<string | null> {
+ return sessionOn(await readTabSessions(), tabId)?.sessionId ?? null;
+}
+
+async function listen(on: boolean): Promise<void> {
+ const state = await readHandsFree();
+ if (!state) return;
+ if (!on) return writeHandsFree({ ...state, muted: true });
+ if (state.muted) return writeHandsFree({ ...state, muted: false });
+ await closeDictation();
+ await syncHandsFree();
+}
+
+async function attach(request: Extract<OrbRequest, { op: 'attach' }>, tab: TabAnchor): Promise<{ ok: boolean; fileId?: string; error?: string }> {
+ const { name, mime, size, content } = request.file;
+ const id = crypto.randomUUID();
+ try {
+ if (content && size <= MAX_STORED_FILE_BYTES) await putBytes({ id, name, mime, content });
+ } catch (error) {
+ return { ok: false, error: error instanceof Error ? error.message : String(error) };
+ }
+ runCommand({ op: 'attach', file: { id, name, mime, size }, tab });
+ return { ok: true, fileId: id };
+}
+
+function answer(request: OrbRequest, tab: Browser.tabs.Tab & { id: number }, respond: (reply: unknown) => void): boolean {
+ switch (request.op) {
+ case 'sync':
+ sent.delete(tab.id);
+ void syncHandsFree();
+ respond({ ok: true });
+ return false;
+ case 'submit':
+ if (request.text.trim()) {
+ runCommand({ op: 'instruct', text: request.text.trim(), tab: anchorOf(tab), focus: request.focus, liveTools: request.liveTools });
+ }
+ respond({ ok: true });
+ return false;
+ case 'cancel':
+ void sessionIdFor(tab.id).then((sessionId) => sessionId && runCommand({ op: 'cancel', sessionId }));
+ respond({ ok: true });
+ return false;
+ case 'decide':
+ void sessionIdFor(tab.id).then(
+ (sessionId) =>
+ sessionId &&
+ runCommand({ op: 'decision', sessionId, toolId: request.toolId, allow: request.allow, remember: request.remember }),
+ );
+ respond({ ok: true });
+ return false;
+ case 'listen':
+ void listen(request.on);
+ respond({ ok: true });
+ return false;
+ case 'grantMic':
+ void openMicPermissionPage();
+ respond({ ok: true });
+ return false;
+ case 'move': {
+ const position = asPosition(request.position);
+ if (position) void browser.storage.local.set({ [ORB_POSITION_KEY]: position });
+ respond({ ok: true });
+ return false;
+ }
+ case 'detach':
+ runCommand({ op: 'detach', fileId: request.fileId });
+ respond({ ok: true });
+ return false;
+ case 'pushToTalk':
+ void browser.storage.local.set({ [PUSH_TO_TALK_KEY]: request.on === true });
+ respond({ ok: true });
+ return false;
+ case 'talk':
+ if (tab.id === listeningTab) {
+ const command: DictationCommand = { channel: DICTATION_CHANNEL, op: 'talk', on: request.on === true };
+ void browser.runtime.sendMessage(command).catch(() => undefined);
+ }
+ respond({ ok: true });
+ return false;
+ case 'pick':
+ pickFocusIn({ id: tab.id, windowId: tab.windowId })
+ .then(respond)
+ .catch((error) => respond({ error: String(error) }));
+ return true;
+ case 'attach':
+ attach(request, anchorOf(tab))
+ .then(respond)
+ .catch((error) => respond({ ok: false, error: String(error) }));
+ return true;
+ case 'openPanel':
+ respond({ ok: true });
+ return false;
+ }
+}
+
+export function serveHandsFree(): void {
+ browser.runtime.onMessage.addListener((message: unknown, sender, sendResponse) => {
+ if (isDictationReport(message)) {
+ if (sender.url && new URL(sender.url).pathname === `/${DICTATION_PAGE}`) void hear(message);
+ return;
+ }
+ if (!isOrbRequest(message)) return;
+ const tab = sender.tab;
+ if (tab?.id == null) return;
+ /* The click that reached us is the only gesture the panel will get — spent before any
+ await, exactly as the rail does. Hands-free ends when the panel reports itself open. */
+ if (message.op === 'openPanel' && tab.windowId != null) void openSidePanel(tab.windowId).catch(() => undefined);
+ return answer(message, tab as Browser.tabs.Tab & { id: number }, sendResponse) || undefined;
+ });
+
+ onPanelPresence((open) => {
+ panelOpen = open;
+ void syncHandsFree();
+ });
+
+ onTurnSettled((sessionId, items) => {
+ const reply = replyOf(items);
+ if (!reply) return;
+ void (async () => {
+ if (!(await readHandsFree())) return;
+ const session = (await readTabSessions())[sessionId];
+ if (session) await post(session.currentTabId, { channel: HANDS_FREE_CHANNEL, op: 'say', ...reply });
+ })();
+ });
+
+ browser.notifications?.onClicked.addListener((notificationId) => {
+ if (!notificationId.startsWith(NOTIFICATION_PREFIX)) return;
+ const tabId = Number(notificationId.slice(NOTIFICATION_PREFIX.length).split(':')[0]);
+ void browser.notifications.clear(notificationId);
+ void browser.tabs.update(tabId, { active: true }).then((tab) => {
+ if (tab?.windowId != null) void browser.windows.update(tab.windowId, { focused: true });
+ }).catch(() => undefined);
+ });
+
+ browser.tabs.onActivated.addListener(() => void syncHandsFree());
+ browser.windows?.onFocusChanged.addListener(() => void syncHandsFree());
+ browser.tabs.onRemoved.addListener((tabId) => {
+ sent.delete(tabId);
+ orbTabs.delete(tabId);
+ });
+ browser.tabs.onUpdated.addListener((tabId, changed) => {
+ if (changed.status !== 'loading') return;
+ sent.delete(tabId);
+ orbTabs.delete(tabId);
+ });
+ browser.storage.session.onChanged.addListener((changes) => {
+ if (HANDS_FREE_KEY in changes || DICTATION_KEY in changes || TAB_SESSIONS_KEY in changes || DAEMON_KEY in changes) {
+ void syncHandsFree();
+ }
+ });
+ browser.storage.local.onChanged.addListener((changes) => {
+ if (THEME_KEY in changes || ORB_POSITION_KEY in changes || SPEECH_SERVICE_KEY in changes || PUSH_TO_TALK_KEY in changes) {
+ void syncHandsFree();
+ }
+ });
+
+ void syncHandsFree();
+}
diff --git a/src/lib/bridge/panel-view.ts b/src/lib/bridge/panel-view.ts
index 1482fe8..9cceeca 100644
--- a/src/lib/bridge/panel-view.ts
+++ b/src/lib/bridge/panel-view.ts
@@ -9,6 +9,34 @@ export async function readPanelCollapsed(): Promise<boolean> {
return stored[PANEL_COLLAPSED_KEY] === true;
}
+/**
+ * Hands-free mode, while it is on. Session storage on purpose: a browser that restarts
+ * comes back with the panel, never with a microphone already listening.
+ */
+export const HANDS_FREE_KEY = 'browsentic/handsFree';
+
+export interface HandsFreeState {
+ muted: boolean;
+ /** When it was switched on, so only the page it was switched on from plays the orb’s entrance. */
+ since: number;
+}
+
+export async function readHandsFree(): Promise<HandsFreeState | null> {
+ const stored = await browser.storage.session.get(HANDS_FREE_KEY);
+ const state = stored[HANDS_FREE_KEY] as HandsFreeState | undefined;
+ return state && typeof state.muted === 'boolean' && typeof state.since === 'number' ? state : null;
+}
+
+export function writeHandsFree(state: HandsFreeState | null): Promise<void> {
+ return state
+ ? browser.storage.session.set({ [HANDS_FREE_KEY]: state })
+ : browser.storage.session.remove(HANDS_FREE_KEY);
+}
+
+export const startHandsFree = (): Promise<void> => writeHandsFree({ muted: false, since: Date.now() });
+
+export const endHandsFree = (): Promise<void> => writeHandsFree(null);
+
export async function readPanelTab(): Promise<PanelTab> {
const stored = await browser.storage.local.get(PANEL_TAB_KEY);
const tab = stored[PANEL_TAB_KEY];
diff --git a/src/lib/bridge/run-port.ts b/src/lib/bridge/run-port.ts
index d220531..48c999b 100644
--- a/src/lib/bridge/run-port.ts
+++ b/src/lib/bridge/run-port.ts
@@ -125,6 +125,7 @@ export type RunMessage =
const ports = new Set<Browser.runtime.Port>();
const panelPorts = new Set<Browser.runtime.Port>();
const panelWatchers = new Set<(open: boolean) => void>();
+const settleWatchers = new Set<(sessionId: string, items: RunItem[]) => void>();
const buffers = new Map<string, RunItem[]>();
const busy = new Set<string>();
const persistTimers = new Map<string, ReturnType<typeof setTimeout>>();
@@ -256,6 +257,16 @@ export function closePanels(): void {
for (const panel of panelPorts) post(panel, { op: 'close' });
}
+/** Fires once a turn is over — finished, failed or stopped — with the conversation as it now stands. */
+export function onTurnSettled(watch: (sessionId: string, items: RunItem[]) => void): void {
+ settleWatchers.add(watch);
+}
+
+/** The panel's commands, for a surface in the background's own reach that has no port — the hands-free orb. */
+export function runCommand(command: RunCommand): void {
+ handle(command);
+}
+
function handle(command: RunCommand): void {
switch (command.op) {
case 'instruct':
@@ -445,6 +456,8 @@ async function settle(sessionId: string): Promise<void> {
await dropDiagnosticsForSession(sessionId);
await patchSession(sessionId, { runId: null, pendingApproval: undefined, handing: undefined });
await syncRunIndicator();
+ const settledItems = buffers.get(sessionId) ?? [];
+ for (const watch of settleWatchers) watch(sessionId, settledItems);
const session = (await readTabSessions())[sessionId];
if (session) {
diff --git a/wxt.config.ts b/wxt.config.ts
index d6ec446..4a3ef82 100644
--- a/wxt.config.ts
+++ b/wxt.config.ts
@@ -26,8 +26,9 @@ export default defineConfig({
permissions: [
'storage', 'unlimitedStorage', 'activeTab', 'contextMenus', 'alarms', 'scripting',
'notifications', 'downloads',
- // Firefox has no sidePanel or debugger permission, and addons.mozilla.org flags each name it does not know.
- ...(browser === 'firefox' ? [] : ['sidePanel', 'debugger']),
+ // Firefox has no sidePanel, debugger or offscreen permission, and addons.mozilla.org flags each name it does not know.
+ // offscreen is the hidden page hands-free mode listens through once the side panel is closed.
+ ...(browser === 'firefox' ? [] : ['sidePanel', 'debugger', 'offscreen']),
],
host_permissions: ['<all_urls>'],
// Release Firefox installs only what addons.mozilla.org has signed, and signing needs a