An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.
TypeScript agentic-browseragentic-browsingagy-cliaiai-agentai-browser-extensionai-toolsbrave-extensionbrowser-agentbrowser-automationbrowser-extensionbrowser-usebrowserskillchrome-extensionclaude-codecodexfirefox-addonmcp-servernanobrowserweb-automation
feat(hands-free): listen through an offscreen page once the side panel is closed — for the one tab in front, never beside a panel, and only while the key is held when hold to talk is on
Shahriar9c633604596e
diff --git a/src/extension/entrypoints/background.ts b/src/extension/entrypoints/background.tsindex f14b0c2..60da0fa 100644--- a/src/extension/entrypoints/background.ts+++ b/src/extension/entrypoints/background.ts@@ -16,6 +16,8 @@ import { closePanels, onPanelPresence, serveRunPorts, serveTabSessions, sweepFilimport { clearStrandedRail, serveRail, setPanelCollapsed, syncRail } from '@/lib/bridge/rail';import { serveTimers } from '@/lib/bridge/timer';import { serveToast } from '@/lib/bridge/toast';+import { serveHandsFree } from '@/lib/bridge/hands-free';+import { endHandsFree } from '@/lib/bridge/panel-view';import { closeSidebar, openSidePanel } from '@/lib/bridge/side-panel';import { isAgentKind } from '@/lib/agents/catalog';import {@@ -145,6 +147,7 @@ export default defineBackground(() => {}if (message.op === 'panelOpened') {void setPanelCollapsed(false);+ void endHandsFree();sendResponse(success(true));return;}@@ -208,6 +211,7 @@ export default defineBackground(() => {serveTabSessions();serveRail();serveToast();+ serveHandsFree();void syncRail();onWelcome(() => {diff --git a/src/extension/entrypoints/dictation/index.html b/src/extension/entrypoints/dictation/index.htmlnew file mode 100644index 0000000..1d14921--- /dev/null+++ b/src/extension/entrypoints/dictation/index.html@@ -0,0 +1,11 @@+<!doctype html>+<html lang="en">+ <head>+ <meta charset="UTF-8" />+ <meta name="manifest.exclude" content="['firefox']" />+ <title>Browsentic — hands-free dictation</title>+ </head>+ <body>+ <script type="module" src="./main.ts"></script>+ </body>+</html>diff --git a/src/extension/entrypoints/dictation/main.ts b/src/extension/entrypoints/dictation/main.tsnew file mode 100644index 0000000..74dac9a--- /dev/null+++ b/src/extension/entrypoints/dictation/main.ts@@ -0,0 +1,115 @@+import { browser } from 'wxt/browser';+import { micPermission, watchMicPermission } from '@/lib/bridge/mic-permission';+import { createRecognition, transcriptOf, type SpeechRecognitionLike } from '@/lib/bridge/recognition';+import {+ DICTATION_CHANNEL,+ isDictationCommand,+ type DictationPhase,+ type DictationReport,+} from '@/lib/handsfree/events';++const RESTART_DELAY_MS = 400;++/*+ * The offscreen document hands-free mode listens through. It only exists while the orb should+ * be listening — the background creates it to start and closes it to stop. An `aborted` it did+ * not ask for is therefore another recognizer taking the microphone, since Chrome runs one at a+ * time: it yields rather than taking the mic back.+ *+ * Opened as `dictation.html?hold` it is hold-to-talk instead: it waits, holding no microphone,+ * until told to talk, and when told to stop it lets the recognizer finish the words already+ * spoken rather than cutting them off.+ */+const PHASE_OF_ERROR: Record<string, DictationPhase> = {+ 'not-allowed': 'blocked',+ 'service-not-allowed': 'blocked',+ 'audio-capture': 'no-mic',+ network: 'no-service',+ aborted: 'yielded',+};++const holding = new URLSearchParams(location.search).has('hold');++type Unsent<T> = T extends unknown ? Omit<T, 'channel'> : never;++let speech: SpeechRecognitionLike | null = null;+let wanted = !holding;+let halted = false;+let unsettled = '';++function report(message: Unsent<DictationReport>): void {+ void browser.runtime.sendMessage({ channel: DICTATION_CHANNEL, ...message }).catch(() => undefined);+}++async function granted(): Promise<boolean> {+ const permission = await micPermission();+ if (permission === 'granted') return true;+ report({ op: 'phase', phase: permission === 'prompt' ? 'needs-mic' : 'blocked' });+ return false;+}++async function listen(): Promise<void> {+ if (speech || !wanted || !(await granted())) return;+ const next = createRecognition();+ if (!next) {+ report({ op: 'phase', phase: 'no-service' });+ return;+ }++ halted = false;+ unsettled = '';+ next.onstart = () => report({ op: 'phase', phase: 'listening' });+ next.onresult = (event) => {+ const { interim, final } = transcriptOf(event);+ unsettled = interim;+ if (interim) report({ op: 'heard', text: interim, final: false });+ if (final) report({ op: 'heard', text: final, final: true });+ };+ next.onerror = (event) => {+ if (event.error === 'no-speech') return;+ halted = true;+ report({ op: 'phase', phase: PHASE_OF_ERROR[event.error] ?? 'no-service' });+ };+ next.onend = () => {+ speech = null;+ if (unsettled && !wanted) report({ op: 'heard', text: unsettled, final: true });+ unsettled = '';+ if (halted) return;+ if (wanted) setTimeout(() => void listen(), RESTART_DELAY_MS);+ else report({ op: 'phase', phase: 'held' });+ };++ speech = next;+ report({ op: 'phase', phase: 'starting' });+ try {+ next.start();+ } catch {+ speech = null;+ report({ op: 'phase', phase: 'no-service' });+ }+}++async function wait(): Promise<void> {+ if (await granted()) report({ op: 'phase', phase: 'held' });+}++function talk(on: boolean): void {+ wanted = on;+ if (on) {+ void listen();+ return;+ }+ if (speech) speech.stop();+ else void wait();+}++browser.runtime.onMessage.addListener((message: unknown) => {+ if (holding && isDictationCommand(message)) talk(message.on);+});++watchMicPermission((state) => {+ if (state === 'denied') report({ op: 'phase', phase: 'blocked' });+ else if (state === 'granted') void (wanted ? listen() : wait());+});++void (holding ? wait() : listen());diff --git a/src/extension/entrypoints/mic-permission/App.tsx b/src/extension/entrypoints/mic-permission/App.tsxindex 0a7b915..b0ff784 100644--- a/src/extension/entrypoints/mic-permission/App.tsx+++ b/src/extension/entrypoints/mic-permission/App.tsx@@ -11,7 +11,7 @@ const CLOSE_AFTER_MS = 1500;const COPY: Record<Phase, { title: string; body: string }> = {asking: {title: 'Allow the microphone',- body: 'Your browser is asking above. Choose Allow, and dictation works in the side panel and the popup from then on.',+ body: 'Your browser is asking above. Choose Allow, and dictation works in the side panel, the popup and hands-free mode from then on.',},granted: {title: 'The microphone is on',diff --git a/src/lib/bridge/aeye.ts b/src/lib/bridge/aeye.tsindex 83f8137..17c13bc 100644--- a/src/lib/bridge/aeye.ts+++ b/src/lib/bridge/aeye.ts@@ -17,8 +17,11 @@ interface PickedElement {export async function pickFocus(): Promise<PickOutcome> {const [tab] = await browser.tabs.query({ active: true, currentWindow: true });if (tab?.id == null) return { error: 'No active tab to point at' };+ return pickFocusIn({ id: tab.id, windowId: tab.windowId });+}- const result = await pickInTab({ id: tab.id, windowId: tab.windowId });+export async function pickFocusIn(tab: { id: number; windowId?: number }): Promise<PickOutcome> {+ const result = await pickInTab(tab);if (!result.ok) {return result.error.code === 'PICK_CANCELLED' ? { cancelled: true } : { error: result.error.message };}diff --git a/src/lib/bridge/hands-free.test.ts b/src/lib/bridge/hands-free.test.tsnew file mode 100644index 0000000..a579abd--- /dev/null+++ b/src/lib/bridge/hands-free.test.ts@@ -0,0 +1,76 @@+import { describe, expect, it } from 'vitest';+import { approvalOf, describeOrb, spoken, type OrbWorld } from './hands-free';+import type { TabSession } from './tab-sessions';++const world: OrbWorld = {+ theme: 'ember',+ position: null,+ link: 'live',+ state: { muted: false, since: 1 },+ phase: 'listening',+ pushToTalk: false,+ listeningTab: 7,+};++const session = (patch: Partial<TabSession>): TabSession => ({+ sessionId: 's',+ mainTabId: 7,+ tabIds: [7],+ currentTabId: 7,+ windowId: 1,+ title: 'Page',+ runId: null,+ turns: 0,+ createdAt: 0,+ lastActivityAt: 0,+ ...patch,+});++describe('describeOrb', () => {+ it('listens only in the tab the microphone is aimed at', () => {+ expect(describeOrb(world, 7, null).voice).toBe('listening');+ expect(describeOrb(world, 8, null).voice).toBe('paused');+ });++ it('carries hold-to-talk to every tab, and the held phase to the one listening', () => {+ const holding = { ...world, pushToTalk: true, phase: 'held' as const };+ expect(describeOrb(holding, 7, null)).toMatchObject({ pushToTalk: true, voice: 'held' });+ expect(describeOrb(holding, 8, null)).toMatchObject({ pushToTalk: true, voice: 'paused' });+ });++ it('says muted everywhere once muted', () => {+ expect(describeOrb({ ...world, state: { muted: true, since: 1 } }, 7, null).voice).toBe('muted');+ expect(describeOrb({ ...world, state: { muted: true, since: 1 } }, 8, null).voice).toBe('muted');+ });++ it('shows a waiting approval ahead of the run it belongs to', () => {+ const asking = session({+ runId: 'r',+ pendingApproval: { toolId: 't', action: 'page.clickElement', input: { selector: '#go' }, site: 'x.test' },+ });+ expect(describeOrb(world, 7, session({ runId: 'r' })).run).toBe('working');+ expect(describeOrb(world, 7, asking)).toMatchObject({+ run: 'approval',+ approval: { toolId: 't', action: 'clickElement', site: 'x.test', detail: 'selector: #go' },+ });+ });+});++describe('approvalOf', () => {+ it('carries code the agent wrote whole, with its purpose, and no one-line detail', () => {+ const ask = approvalOf({ toolId: 't', action: 'page.injectCode', input: { purpose: 'Sum the column', code: 'let a = 1;' } });+ expect(ask).toEqual({ toolId: 't', action: 'injectCode', site: undefined, purpose: 'Sum the column', code: 'let a = 1;' });+ });++ it('flattens any other input to one short line', () => {+ const ask = approvalOf({ toolId: 't', action: 'page.fillInput', input: { selector: '#q', value: 'x'.repeat(400) } });+ expect(ask.detail?.length).toBeLessThanOrEqual(180);+ expect(ask.detail?.startsWith('selector: #q · value: xxx')).toBe(true);+ });+});++describe('spoken', () => {+ it('reads an action name the way a person would say it', () => {+ expect(spoken('page.clickElement')).toBe('Click element');+ });+});diff --git a/src/lib/bridge/hands-free.ts b/src/lib/bridge/hands-free.tsnew file mode 100644index 0000000..7796546--- /dev/null+++ b/src/lib/bridge/hands-free.ts@@ -0,0 +1,505 @@+import { browser, type Browser } from 'wxt/browser';+import { MAX_STORED_FILE_BYTES } from '@/lib/files/report';+import { replyOf } from '@/lib/handsfree/caption';+import {+ DICTATION_CHANNEL,+ HANDS_FREE_CHANNEL,+ isDictationReport,+ isOrbRequest,+ type DictationCommand,+ type DictationPhase,+ type DictationReport,+ type OrbApproval,+ type OrbCommand,+ type OrbLink,+ type OrbPosition,+ type OrbRequest,+ type OrbView,+} from '@/lib/handsfree/events';+import { pickFocusIn } from './aeye';+import { putBytes } from './file-store';+import { openMicPermissionPage } from './mic-permission';+import { HANDS_FREE_KEY, endHandsFree, readHandsFree, writeHandsFree, type HandsFreeState } from './panel-view';+import { onPanelPresence, onTurnSettled, runCommand } from './run-port';+import { openSidePanel } from './side-panel';+import { TAB_SESSIONS_KEY, readTabSessions, type PendingApproval, type TabAnchor, type TabSession } from './tab-sessions';+import { THEME_KEY, readTheme, type ThemeId } from './theme';+import { SPEECH_SERVICE_KEY, handsFreeSupported, noteSpeechService } from './speech-support';+import { showToast } from './toast';+import type { DaemonState } from './socket';++export const ORB_POSITION_KEY = 'browsentic/orbPosition';+export const PUSH_TO_TALK_KEY = 'browsentic/pushToTalk';+const DICTATION_KEY = 'browsentic/dictation';+const DAEMON_KEY = 'browsentic/daemon';+const DICTATION_PAGE = 'dictation.html';+const HOLD_DICTATION_PAGE = `${DICTATION_PAGE}?hold`;+const NOTIFICATION_PREFIX = 'browsentic-approval:';+const MAX_DETAIL_CHARS = 180;+const MAX_CODE_CHARS = 4_000;++/** What every tab was last told. A cache: a revived worker starts it empty and repaints. */+const sent = new Map<number, string>();+const orbTabs = new Set<number>();+const announced = new Set<string>();+let showing: boolean | null = null;+let listeningTab: number | null = null;+let panelOpen = false;+let serviceAnswered = false;++let inFlight: Promise<void> | null = null;+let restated = false;++export function syncHandsFree(): Promise<void> {+ if (inFlight) {+ restated = true;+ return inFlight;+ }+ inFlight = (async () => {+ do {+ restated = false;+ await paint().catch((error) => console.warn('[browsentic] hands-free paint failed:', error));+ } while (restated);+ inFlight = null;+ })();+ return inFlight;+}++export interface OrbWorld {+ theme: ThemeId;+ position: OrbPosition | null;+ link: OrbLink;+ state: HandsFreeState;+ phase: DictationPhase | null;+ pushToTalk: boolean;+ listeningTab: number | null;+}++export function describeOrb(world: OrbWorld, tabId: number, session: TabSession | null): OrbView {+ const pending = session?.pendingApproval;+ return {+ theme: world.theme,+ since: world.state.since,+ position: world.position,+ link: world.link,+ run: pending ? 'approval' : session?.runId ? 'working' : 'idle',+ voice: world.state.muted ? 'muted' : tabId === world.listeningTab ? (world.phase ?? 'starting') : 'paused',+ pushToTalk: world.pushToTalk,+ ...(pending && { approval: approvalOf(pending) }),+ };+}++export function approvalOf(pending: PendingApproval): OrbApproval {+ const base = { toolId: pending.toolId, action: pending.action.replace(/^page\./, ''), site: pending.site };+ const input = pending.input as Record<string, unknown> | null;+ if (typeof input?.code === 'string' && input.code) {+ return {+ ...base,+ purpose: typeof input.purpose === 'string' ? input.purpose : undefined,+ code: input.code.slice(0, MAX_CODE_CHARS),+ };+ }+ return { ...base, detail: flatten(pending.input) };+}++function flatten(input: unknown): string | undefined {+ if (input == null) return undefined;+ const text =+ typeof input === 'object'+ ? Object.entries(input as Record<string, unknown>)+ .filter(([, value]) => value !== undefined && value !== '')+ .map(([key, value]) => `${key}: ${typeof value === 'string' ? value : JSON.stringify(value)}`)+ .join(' · ')+ : String(input);+ if (!text) return undefined;+ return text.length > MAX_DETAIL_CHARS ? `${text.slice(0, MAX_DETAIL_CHARS - 1)}…` : text;+}++const linkOf = (daemon: DaemonState | null): OrbLink =>+ !daemon?.paired ? 'off' : daemon.connected ? 'live' : 'pending';++const sessionOn = (sessions: Record<string, TabSession>, tabId: number): TabSession | null =>+ Object.values(sessions).find((session) => session.tabIds.includes(tabId)) ?? null;++/**+ * The microphone listens for exactly one tab: the one in front of a focused window, while+ * its conversation is free to take an instruction and no side panel is open to listen+ * instead. Every other tab's orb is painted as paused.+ */+async function paint(): Promise<void> {+ const state = await readHandsFree();+ if (!state) {+ await clearOrbs();+ return;+ }+ /* The panel never offers hands-free where speech cannot work; this catches the rest — a flag+ left from before the service was found missing, or written by anything but the panel. */+ if (!(await handsFreeSupported())) {+ await endHandsFree();+ return;+ }+ showing = true;++ const [daemon, sessions, theme, position, phase, pushToTalk, front, tabs] = await Promise.all([+ readDaemon(),+ readTabSessions(),+ readTheme(),+ readPosition(),+ readPhase(),+ readPushToTalk(),+ frontTab(),+ browser.tabs.query({}),+ ]);+ const link = linkOf(daemon);+ const frontId = front?.id ?? null;+ const frontSession = frontId == null ? null : sessionOn(sessions, frontId);+ const target =+ !state.muted && link === 'live' && !panelOpen && !frontSession?.runId && !frontSession?.pendingApproval+ ? frontId+ : null;++ const world: OrbWorld = { theme, position, link, state, phase, pushToTalk, listeningTab: target };+ await Promise.all(+ tabs.map(async (tab) => {+ if (tab.id == null || tab.discarded) return;+ const view = describeOrb(world, tab.id, sessionOn(sessions, tab.id));+ const painted = JSON.stringify(view);+ if (sent.get(tab.id) === painted) return;+ sent.set(tab.id, painted);+ if (await post(tab.id, { channel: HANDS_FREE_CHANNEL, op: 'show', view })) orbTabs.add(tab.id);+ else orbTabs.delete(tab.id);+ }),+ );++ await aimMic(target != null && orbTabs.has(target) ? target : null, pushToTalk);+ for (const session of Object.values(sessions)) {+ const pending = session.pendingApproval;+ if (!pending || announced.has(pending.toolId)) continue;+ announced.add(pending.toolId);+ if (session.currentTabId !== frontId) void askElsewhere(session, pending, frontId != null);+ }+}++async function clearOrbs(): Promise<void> {+ await aimMic(null, false);+ announced.clear();+ if (showing === false) return;+ const tabs = await browser.tabs.query({});+ await Promise.all(tabs.map((tab) => tab.id != null && !tab.discarded && post(tab.id, { channel: HANDS_FREE_CHANNEL, op: 'hide' })));+ sent.clear();+ orbTabs.clear();+ showing = false;+}++async function aimMic(tabId: number | null, hold: boolean): Promise<void> {+ listeningTab = tabId;+ if (tabId == null) await closeDictation();+ else await openDictation(hold);+}++async function dictationUrl(): Promise<URL | null> {+ if (import.meta.env.FIREFOX) return null;+ const [open] = await browser.runtime+ .getContexts({ contextTypes: [browser.runtime.ContextType.OFFSCREEN_DOCUMENT] })+ .catch(() => []);+ return open?.documentUrl ? new URL(open.documentUrl) : null;+}++async function hasDictation(): Promise<boolean> {+ return (await dictationUrl()) !== null;+}++/** One page, in the mode asked for: a page left in the other mode is closed and opened afresh. */+async function openDictation(hold: boolean): Promise<void> {+ if (import.meta.env.FIREFOX) return;+ const open = await dictationUrl();+ if (open && open.searchParams.has('hold') === hold) return;+ if (open) await closeDictation();+ await browser.storage.session.set({ [DICTATION_KEY]: (hold ? 'held' : 'starting') satisfies DictationPhase });+ await browser.offscreen+ .createDocument({+ url: hold ? HOLD_DICTATION_PAGE : DICTATION_PAGE,+ reasons: [browser.offscreen.Reason.USER_MEDIA],+ justification: 'Hands-free mode listens for spoken instructions while the side panel is closed.',+ })+ .catch((error) => console.warn('[browsentic] could not start dictation:', error));+}++async function closeDictation(): Promise<void> {+ if (!(await hasDictation())) return;+ await browser.offscreen.closeDocument().catch(() => undefined);+ await browser.storage.session.remove(DICTATION_KEY);+}++async function hear(report: DictationReport): Promise<void> {+ if (report.op === 'phase') {+ if (report.phase === 'no-service' && (await noteSpeechService('missing')) === 'missing') {+ await sayNoService();+ return;+ }+ if (await hasDictation()) await browser.storage.session.set({ [DICTATION_KEY]: report.phase });+ return;+ }+ if (!serviceAnswered) {+ serviceAnswered = true;+ void noteSpeechService('works');+ }+ if (listeningTab == null) await syncHandsFree();+ if (listeningTab == null) return;+ void post(listeningTab, { channel: HANDS_FREE_CHANNEL, op: 'heard', text: report.text, final: report.final });+}++/**+ * The service failed before it ever transcribed a word here, so this browser has none, and the+ * mic goes with the storage change. The toast says why, since the panel it came from is closed.+ */+async function sayNoService(): Promise<void> {+ await showToast({+ toastId: 'hands-free-no-speech',+ tone: 'warn',+ title: 'Hands-free is not available here',+ body: 'This browser has no speech service to transcribe with. Open Browsentic from the toolbar to keep going by typing.',+ });+}++/** An approval raised in a tab nobody is looking at: a card on the page they are on, or the OS when the browser is behind. */+async function askElsewhere(session: TabSession, pending: PendingApproval, browserInFront: boolean): Promise<void> {+ const title = 'Browsentic needs your OK';+ const body = `${spoken(pending.action)}${pending.site ? ` on ${pending.site}` : ''} — open the tab and tap the mic.`;+ const shown =+ browserInFront &&+ (await showToast({ toastId: `approval-${pending.toolId}`, tone: 'warn', title, body, tabId: session.currentTabId }));+ if (shown) return;+ await browser.notifications+ ?.create(`${NOTIFICATION_PREFIX}${session.currentTabId}:${pending.toolId}`, {+ type: 'basic',+ iconUrl: largestIcon(),+ title,+ message: body,+ })+ .catch(() => undefined);+}++/** `page.clickElement` as a person would say it: “Click element”. */+export const spoken = (action: string): string =>+ action+ .replace(/^page\./, '')+ .replace(/([a-z])([A-Z])/g, '$1 $2')+ .toLowerCase()+ .replace(/^./, (first) => first.toUpperCase());++function largestIcon(): string {+ const icons = browser.runtime.getManifest().icons ?? {};+ const [largest] = Object.keys(icons)+ .map(Number)+ .sort((a, b) => b - a);+ return icons[largest] ?? '';+}++async function frontTab(): Promise<Browser.tabs.Tab | null> {+ const window = await browser.windows.getLastFocused({ windowTypes: ['normal'] }).catch(() => null);+ if (!window?.focused || window.id == null) return null;+ const [tab] = await browser.tabs.query({ active: true, windowId: window.id }).catch(() => []);+ return tab ?? null;+}++async function readDaemon(): Promise<DaemonState | null> {+ const stored = await browser.storage.session.get(DAEMON_KEY);+ return (stored[DAEMON_KEY] as DaemonState | undefined) ?? null;+}++async function readPhase(): Promise<DictationPhase | null> {+ const stored = await browser.storage.session.get(DICTATION_KEY);+ return (stored[DICTATION_KEY] as DictationPhase | undefined) ?? null;+}++async function readPushToTalk(): Promise<boolean> {+ return (await browser.storage.local.get(PUSH_TO_TALK_KEY))[PUSH_TO_TALK_KEY] === true;+}++async function readPosition(): Promise<OrbPosition | null> {+ const stored = await browser.storage.local.get(ORB_POSITION_KEY);+ return asPosition(stored[ORB_POSITION_KEY]);+}++function asPosition(value: unknown): OrbPosition | null {+ const point = value as Partial<OrbPosition> | null;+ const within = (n: unknown): n is number => typeof n === 'number' && n >= 0 && n <= 1;+ return point && within(point.x) && within(point.y) ? { x: point.x, y: point.y } : null;+}++/* Answered through sendResponse by the orb, for the reason the toast gives: on Chrome a+ returned promise is dropped, and the verdict is what says whether a tab has an orb. */+async function post(tabId: number, command: OrbCommand): Promise<boolean> {+ try {+ return ((await browser.tabs.sendMessage(tabId, command)) as { ok?: boolean } | undefined)?.ok === true;+ } catch {+ return false;+ }+}++const anchorOf = (tab: Browser.tabs.Tab & { id: number }): TabAnchor => ({+ tabId: tab.id,+ url: tab.url,+ windowId: tab.windowId,+ title: tab.title,+});++async function sessionIdFor(tabId: number): Promise<string | null> {+ return sessionOn(await readTabSessions(), tabId)?.sessionId ?? null;+}++async function listen(on: boolean): Promise<void> {+ const state = await readHandsFree();+ if (!state) return;+ if (!on) return writeHandsFree({ ...state, muted: true });+ if (state.muted) return writeHandsFree({ ...state, muted: false });+ await closeDictation();+ await syncHandsFree();+}++async function attach(request: Extract<OrbRequest, { op: 'attach' }>, tab: TabAnchor): Promise<{ ok: boolean; fileId?: string; error?: string }> {+ const { name, mime, size, content } = request.file;+ const id = crypto.randomUUID();+ try {+ if (content && size <= MAX_STORED_FILE_BYTES) await putBytes({ id, name, mime, content });+ } catch (error) {+ return { ok: false, error: error instanceof Error ? error.message : String(error) };+ }+ runCommand({ op: 'attach', file: { id, name, mime, size }, tab });+ return { ok: true, fileId: id };+}++function answer(request: OrbRequest, tab: Browser.tabs.Tab & { id: number }, respond: (reply: unknown) => void): boolean {+ switch (request.op) {+ case 'sync':+ sent.delete(tab.id);+ void syncHandsFree();+ respond({ ok: true });+ return false;+ case 'submit':+ if (request.text.trim()) {+ runCommand({ op: 'instruct', text: request.text.trim(), tab: anchorOf(tab), focus: request.focus, liveTools: request.liveTools });+ }+ respond({ ok: true });+ return false;+ case 'cancel':+ void sessionIdFor(tab.id).then((sessionId) => sessionId && runCommand({ op: 'cancel', sessionId }));+ respond({ ok: true });+ return false;+ case 'decide':+ void sessionIdFor(tab.id).then(+ (sessionId) =>+ sessionId &&+ runCommand({ op: 'decision', sessionId, toolId: request.toolId, allow: request.allow, remember: request.remember }),+ );+ respond({ ok: true });+ return false;+ case 'listen':+ void listen(request.on);+ respond({ ok: true });+ return false;+ case 'grantMic':+ void openMicPermissionPage();+ respond({ ok: true });+ return false;+ case 'move': {+ const position = asPosition(request.position);+ if (position) void browser.storage.local.set({ [ORB_POSITION_KEY]: position });+ respond({ ok: true });+ return false;+ }+ case 'detach':+ runCommand({ op: 'detach', fileId: request.fileId });+ respond({ ok: true });+ return false;+ case 'pushToTalk':+ void browser.storage.local.set({ [PUSH_TO_TALK_KEY]: request.on === true });+ respond({ ok: true });+ return false;+ case 'talk':+ if (tab.id === listeningTab) {+ const command: DictationCommand = { channel: DICTATION_CHANNEL, op: 'talk', on: request.on === true };+ void browser.runtime.sendMessage(command).catch(() => undefined);+ }+ respond({ ok: true });+ return false;+ case 'pick':+ pickFocusIn({ id: tab.id, windowId: tab.windowId })+ .then(respond)+ .catch((error) => respond({ error: String(error) }));+ return true;+ case 'attach':+ attach(request, anchorOf(tab))+ .then(respond)+ .catch((error) => respond({ ok: false, error: String(error) }));+ return true;+ case 'openPanel':+ respond({ ok: true });+ return false;+ }+}++export function serveHandsFree(): void {+ browser.runtime.onMessage.addListener((message: unknown, sender, sendResponse) => {+ if (isDictationReport(message)) {+ if (sender.url && new URL(sender.url).pathname === `/${DICTATION_PAGE}`) void hear(message);+ return;+ }+ if (!isOrbRequest(message)) return;+ const tab = sender.tab;+ if (tab?.id == null) return;+ /* The click that reached us is the only gesture the panel will get — spent before any+ await, exactly as the rail does. Hands-free ends when the panel reports itself open. */+ if (message.op === 'openPanel' && tab.windowId != null) void openSidePanel(tab.windowId).catch(() => undefined);+ return answer(message, tab as Browser.tabs.Tab & { id: number }, sendResponse) || undefined;+ });++ onPanelPresence((open) => {+ panelOpen = open;+ void syncHandsFree();+ });++ onTurnSettled((sessionId, items) => {+ const reply = replyOf(items);+ if (!reply) return;+ void (async () => {+ if (!(await readHandsFree())) return;+ const session = (await readTabSessions())[sessionId];+ if (session) await post(session.currentTabId, { channel: HANDS_FREE_CHANNEL, op: 'say', ...reply });+ })();+ });++ browser.notifications?.onClicked.addListener((notificationId) => {+ if (!notificationId.startsWith(NOTIFICATION_PREFIX)) return;+ const tabId = Number(notificationId.slice(NOTIFICATION_PREFIX.length).split(':')[0]);+ void browser.notifications.clear(notificationId);+ void browser.tabs.update(tabId, { active: true }).then((tab) => {+ if (tab?.windowId != null) void browser.windows.update(tab.windowId, { focused: true });+ }).catch(() => undefined);+ });++ browser.tabs.onActivated.addListener(() => void syncHandsFree());+ browser.windows?.onFocusChanged.addListener(() => void syncHandsFree());+ browser.tabs.onRemoved.addListener((tabId) => {+ sent.delete(tabId);+ orbTabs.delete(tabId);+ });+ browser.tabs.onUpdated.addListener((tabId, changed) => {+ if (changed.status !== 'loading') return;+ sent.delete(tabId);+ orbTabs.delete(tabId);+ });+ browser.storage.session.onChanged.addListener((changes) => {+ if (HANDS_FREE_KEY in changes || DICTATION_KEY in changes || TAB_SESSIONS_KEY in changes || DAEMON_KEY in changes) {+ void syncHandsFree();+ }+ });+ browser.storage.local.onChanged.addListener((changes) => {+ if (THEME_KEY in changes || ORB_POSITION_KEY in changes || SPEECH_SERVICE_KEY in changes || PUSH_TO_TALK_KEY in changes) {+ void syncHandsFree();+ }+ });++ void syncHandsFree();+}diff --git a/src/lib/bridge/panel-view.ts b/src/lib/bridge/panel-view.tsindex 1482fe8..9cceeca 100644--- a/src/lib/bridge/panel-view.ts+++ b/src/lib/bridge/panel-view.ts@@ -9,6 +9,34 @@ export async function readPanelCollapsed(): Promise<boolean> {return stored[PANEL_COLLAPSED_KEY] === true;}+/**+ * Hands-free mode, while it is on. Session storage on purpose: a browser that restarts+ * comes back with the panel, never with a microphone already listening.+ */+export const HANDS_FREE_KEY = 'browsentic/handsFree';++export interface HandsFreeState {+ muted: boolean;+ /** When it was switched on, so only the page it was switched on from plays the orb’s entrance. */+ since: number;+}++export async function readHandsFree(): Promise<HandsFreeState | null> {+ const stored = await browser.storage.session.get(HANDS_FREE_KEY);+ const state = stored[HANDS_FREE_KEY] as HandsFreeState | undefined;+ return state && typeof state.muted === 'boolean' && typeof state.since === 'number' ? state : null;+}++export function writeHandsFree(state: HandsFreeState | null): Promise<void> {+ return state+ ? browser.storage.session.set({ [HANDS_FREE_KEY]: state })+ : browser.storage.session.remove(HANDS_FREE_KEY);+}++export const startHandsFree = (): Promise<void> => writeHandsFree({ muted: false, since: Date.now() });++export const endHandsFree = (): Promise<void> => writeHandsFree(null);+export async function readPanelTab(): Promise<PanelTab> {const stored = await browser.storage.local.get(PANEL_TAB_KEY);const tab = stored[PANEL_TAB_KEY];diff --git a/src/lib/bridge/run-port.ts b/src/lib/bridge/run-port.tsindex d220531..48c999b 100644--- a/src/lib/bridge/run-port.ts+++ b/src/lib/bridge/run-port.ts@@ -125,6 +125,7 @@ export type RunMessage =const ports = new Set<Browser.runtime.Port>();const panelPorts = new Set<Browser.runtime.Port>();const panelWatchers = new Set<(open: boolean) => void>();+const settleWatchers = new Set<(sessionId: string, items: RunItem[]) => void>();const buffers = new Map<string, RunItem[]>();const busy = new Set<string>();const persistTimers = new Map<string, ReturnType<typeof setTimeout>>();@@ -256,6 +257,16 @@ export function closePanels(): void {for (const panel of panelPorts) post(panel, { op: 'close' });}+/** Fires once a turn is over — finished, failed or stopped — with the conversation as it now stands. */+export function onTurnSettled(watch: (sessionId: string, items: RunItem[]) => void): void {+ settleWatchers.add(watch);+}++/** The panel's commands, for a surface in the background's own reach that has no port — the hands-free orb. */+export function runCommand(command: RunCommand): void {+ handle(command);+}+function handle(command: RunCommand): void {switch (command.op) {case 'instruct':@@ -445,6 +456,8 @@ async function settle(sessionId: string): Promise<void> {await dropDiagnosticsForSession(sessionId);await patchSession(sessionId, { runId: null, pendingApproval: undefined, handing: undefined });await syncRunIndicator();+ const settledItems = buffers.get(sessionId) ?? [];+ for (const watch of settleWatchers) watch(sessionId, settledItems);const session = (await readTabSessions())[sessionId];if (session) {diff --git a/wxt.config.ts b/wxt.config.tsindex d6ec446..4a3ef82 100644--- a/wxt.config.ts+++ b/wxt.config.ts@@ -26,8 +26,9 @@ export default defineConfig({permissions: ['storage', 'unlimitedStorage', 'activeTab', 'contextMenus', 'alarms', 'scripting','notifications', 'downloads',- // Firefox has no sidePanel or debugger permission, and addons.mozilla.org flags each name it does not know.- ...(browser === 'firefox' ? [] : ['sidePanel', 'debugger']),+ // Firefox has no sidePanel, debugger or offscreen permission, and addons.mozilla.org flags each name it does not know.+ // offscreen is the hidden page hands-free mode listens through once the side panel is closed.+ ...(browser === 'firefox' ? [] : ['sidePanel', 'debugger', 'offscreen']),],host_permissions: ['<all_urls>'],// Release Firefox installs only what addons.mozilla.org has signed, and signing needs a