An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.
TypeScript agentic-browseragentic-browsingagy-cliaiai-agentai-browser-extensionai-toolsbrave-extensionbrowser-agentbrowser-automationbrowser-extensionbrowser-usebrowserskillchrome-extensionclaude-codecodexfirefox-addonmcp-servernanobrowserweb-automation
feat(extension): hands-free from the popup, the right-click menu and a keyboard shortcut — each shown only where speech works — and a shortcut that opens or closes the side panel
Shahriar702e3fd09fad
diff --git a/src/extension/entrypoints/background.ts b/src/extension/entrypoints/background.tsindex ced5c38..bb03e9f 100644--- a/src/extension/entrypoints/background.ts+++ b/src/extension/entrypoints/background.ts@@ -12,14 +12,14 @@ import { nameStoredSession } from '@/lib/bridge/session-store';import { analyzeStoredRecording, resumePendingAnalyses } from '@/lib/bridge/recording-store';import { ingestSample, monitorsForTab, serveMonitor } from '@/lib/bridge/monitor';import { appendEvents, recordingStateFor, serveRecorder } from '@/lib/bridge/recorder';-import { closePanels, onPanelPresence, serveRunPorts, serveTabSessions, sweepFiles } from '@/lib/bridge/run-port';+import { onPanelPresence, serveRunPorts, serveTabSessions, sweepFiles } from '@/lib/bridge/run-port';import { clearStrandedRail, serveRail, setPanelCollapsed, syncRail } from '@/lib/bridge/rail';import { serveTimers } from '@/lib/bridge/timer';import { serveTaskNotices } from '@/lib/bridge/task-notices';import { serveToast } from '@/lib/bridge/toast';import { serveHandsFree } from '@/lib/bridge/hands-free';import { endHandsFree } from '@/lib/bridge/panel-view';-import { closeSidebar, openSidePanel } from '@/lib/bridge/side-panel';+import { serveLaunchers } from '@/lib/bridge/launchers';import { servePreferences } from '@/lib/bridge/preferences';import { isAgentKind } from '@/lib/agents/catalog';import { isPreferenceChange } from '@/lib/settings/preferences';@@ -40,8 +40,6 @@ import {taskOp,} from '@/lib/bridge/socket';-const OPEN_PANEL_MENU = 'open-side-panel';-export default defineBackground(() => {serveDebuggerEvents();@@ -196,33 +194,11 @@ export default defineBackground(() => {});});- const menuReady = browser.contextMenus- .removeAll()- .then(() =>- browser.contextMenus.create({ id: OPEN_PANEL_MENU, title: 'Open Browsentic', contexts: ['all'] }),- );-- let panelOpen = false;onPanelPresence((open) => {- panelOpen = open;- void menuReady.then(() =>- browser.contextMenus.update(OPEN_PANEL_MENU, {- title: open ? 'Close Browsentic' : 'Open Browsentic',- }),- );if (!open) void clearStrandedRail();});- browser.contextMenus.onClicked.addListener((info, tab) => {- if (info.menuItemId !== OPEN_PANEL_MENU || tab?.windowId == null) return;- if (panelOpen) {- if (import.meta.env.FIREFOX) void closeSidebar();- else closePanels();- return;- }- void openSidePanel(tab.windowId);- });-+ serveLaunchers();serveDiagnostics();serveFrameFocus();serveCodeToolkits();diff --git a/src/extension/entrypoints/popup/App.tsx b/src/extension/entrypoints/popup/App.tsxindex 082db89..1aa2e6e 100644--- a/src/extension/entrypoints/popup/App.tsx+++ b/src/extension/entrypoints/popup/App.tsx@@ -1,5 +1,5 @@import { useState } from 'react';-import { Mic, MicOff, PanelRightOpen, Send, SlidersHorizontal } from 'lucide-react';+import { AudioLines, Mic, MicOff, PanelRightOpen, Send, SlidersHorizontal } from 'lucide-react';import { AgentPicker } from '@/extension/components/agent-picker';import { Wordmark } from '@/extension/components/brand';@@ -8,8 +8,10 @@ import { StatusPill, describeStatus, type StatusBlocker } from '@/extension/compimport { Button } from '@/extension/components/ui/button';import { Input } from '@/extension/components/ui/input';import { useDaemonState } from '@/lib/bridge/use-daemon-state';+import { startHandsFree } from '@/lib/bridge/panel-view';import { openSidePanel } from '@/lib/bridge/side-panel';import { useRun } from '@/lib/bridge/use-run';+import { useHandsFreeSupported } from '@/lib/bridge/use-speech';import { useVoiceComposer } from '@/lib/bridge/use-voice-composer';import { cn } from '@/lib/utils';@@ -23,6 +25,7 @@ export default function App() {const daemon = useDaemonState();const run = useRun();const [micOn, setMicOn] = useState(false);+ const handsFree = useHandsFreeSupported();const paired = daemon?.paired ?? false;@@ -34,6 +37,11 @@ export default function App() {}}+ async function goHandsFree() {+ await startHandsFree();+ window.close();+ }+const voice = useVoiceComposer({active: micOn && (daemon?.connected ?? false),onSubmit: (text) => {@@ -145,9 +153,16 @@ export default function App() {</Button></form>- <Button variant="outline" size="sm" className="w-full" onClick={revealPanel}>- <PanelRightOpen /> Open the side panel- </Button>+ <div className={cn('grid w-full gap-2', handsFree && 'grid-cols-2')}>+ <Button variant="outline" size="sm" onClick={revealPanel}>+ <PanelRightOpen /> Open side panel+ </Button>+ {handsFree && (+ <Button variant="outline" size="sm" onClick={goHandsFree}>+ <AudioLines /> Open hands-free+ </Button>+ )}+ </div></main>)}diff --git a/src/lib/bridge/context-menu.test.ts b/src/lib/bridge/context-menu.test.tsnew file mode 100644index 0000000..647aa6c--- /dev/null+++ b/src/lib/bridge/context-menu.test.ts@@ -0,0 +1,47 @@+import { beforeEach, describe, expect, it } from 'vitest';+import { fakeBrowser } from 'wxt/testing';+import { CONTEXT_MENU_KEY, HANDS_FREE_ITEM, PANEL_ITEM, asContextMenuChoice, describeMenu, readContextMenuChoice } from './context-menu';++const both = { panel: true, handsFree: true };+const idle = { choice: both, speech: true, panelOpen: false, handsFree: false };++beforeEach(() => {+ fakeBrowser.reset();+});++describe('describeMenu', () => {+ it('offers the panel and hands-free where speech works', () => {+ expect(describeMenu(idle)).toEqual([+ { id: PANEL_ITEM, title: 'Open Browsentic' },+ { id: HANDS_FREE_ITEM, title: 'Open Browsentic (Hands Free)' },+ ]);+ });++ it('never offers hands-free where speech cannot work, whatever the choice', () => {+ expect(describeMenu({ ...idle, speech: false }).map(({ id }) => id)).toEqual([PANEL_ITEM]);+ });++ it('offers to close whichever surface is showing', () => {+ expect(describeMenu({ ...idle, panelOpen: true })[0].title).toBe('Close Browsentic');+ expect(describeMenu({ ...idle, handsFree: true })[1].title).toBe('Close Browsentic (Hands Free)');+ });++ it('leaves out what was switched off, down to an empty menu', () => {+ expect(describeMenu({ ...idle, choice: { panel: false, handsFree: true } }).map(({ id }) => id)).toEqual([HANDS_FREE_ITEM]);+ expect(describeMenu({ ...idle, choice: { panel: false, handsFree: false } })).toEqual([]);+ });+});++describe('the stored choice', () => {+ it('has both items on until one is switched off', async () => {+ expect(await readContextMenuChoice()).toEqual(both);+ await fakeBrowser.storage.local.set({ [CONTEXT_MENU_KEY]: { panel: false } });+ expect(await readContextMenuChoice()).toEqual({ panel: false, handsFree: true });+ });++ it('reads anything malformed as the default', () => {+ expect(asContextMenuChoice('off')).toEqual(both);+ expect(asContextMenuChoice(null)).toEqual(both);+ expect(asContextMenuChoice({ panel: 0, handsFree: 'no' })).toEqual(both);+ });+});diff --git a/src/lib/bridge/context-menu.ts b/src/lib/bridge/context-menu.tsnew file mode 100644index 0000000..665ff9b--- /dev/null+++ b/src/lib/bridge/context-menu.ts@@ -0,0 +1,40 @@+import { browser } from 'wxt/browser';++export const CONTEXT_MENU_KEY = 'browsentic/contextMenu';+export const PANEL_ITEM = 'open-side-panel';+export const HANDS_FREE_ITEM = 'open-hands-free';++/** Which right-click items this browser shows. Both are on until switched off. */+export interface ContextMenuChoice {+ panel: boolean;+ handsFree: boolean;+}++export function asContextMenuChoice(value: unknown): ContextMenuChoice {+ const stored = (value !== null && typeof value === 'object' ? value : {}) as Partial<Record<keyof ContextMenuChoice, unknown>>;+ return { panel: stored.panel !== false, handsFree: stored.handsFree !== false };+}++export async function readContextMenuChoice(): Promise<ContextMenuChoice> {+ return asContextMenuChoice((await browser.storage.local.get(CONTEXT_MENU_KEY))[CONTEXT_MENU_KEY]);+}++export interface MenuState {+ choice: ContextMenuChoice;+ /** Whether hands-free can exist in this browser at all. */+ speech: boolean;+ panelOpen: boolean;+ handsFree: boolean;+}++export interface MenuItem {+ id: string;+ title: string;+}++/** Each item reads as what a click will do now: the one showing its surface offers to close it. */+export function describeMenu({ choice, speech, panelOpen, handsFree }: MenuState): MenuItem[] {+ const panel = { id: PANEL_ITEM, title: panelOpen ? 'Close Browsentic' : 'Open Browsentic' };+ const mic = { id: HANDS_FREE_ITEM, title: handsFree ? 'Close Browsentic (Hands Free)' : 'Open Browsentic (Hands Free)' };+ return [...(choice.panel ? [panel] : []), ...(choice.handsFree && speech ? [mic] : [])];+}diff --git a/src/lib/bridge/launchers.ts b/src/lib/bridge/launchers.tsnew file mode 100644index 0000000..f5e7d98--- /dev/null+++ b/src/lib/bridge/launchers.ts@@ -0,0 +1,69 @@+import { browser } from 'wxt/browser';+import { HANDS_FREE_SHORTCUT, PANEL_SHORTCUT } from '@/lib/settings/shortcuts';+import { CONTEXT_MENU_KEY, HANDS_FREE_ITEM, PANEL_ITEM, describeMenu, readContextMenuChoice } from './context-menu';+import { HANDS_FREE_KEY, endHandsFree, readHandsFree, startHandsFree } from './panel-view';+import { closePanels, onPanelPresence } from './run-port';+import { closeSidebar, openSidePanel } from './side-panel';+import { SPEECH_SERVICE_KEY, handsFreeSupported } from './speech-support';++let panelOpen = false;+let painting: Promise<void> = Promise.resolve();++/** One paint at a time, so a second change never lands its items between another's removeAll and creates. */+function repaintMenu(): void {+ painting = painting+ .then(paintMenu)+ .catch((error) => console.warn('[browsentic] context menu paint failed:', error));+}++async function paintMenu(): Promise<void> {+ const [choice, speech, handsFree] = await Promise.all([readContextMenuChoice(), handsFreeSupported(), readHandsFree()]);+ await browser.contextMenus.removeAll();+ for (const item of describeMenu({ choice, speech, panelOpen, handsFree: handsFree !== null })) {+ browser.contextMenus.create({ ...item, contexts: ['all'] });+ }+}++/** Called with a user gesture that the first await would spend, so the panel opens before anything is awaited. */+function togglePanel(windowId: number): void {+ if (!panelOpen) {+ void openSidePanel(windowId);+ return;+ }+ if (import.meta.env.FIREFOX) void closeSidebar();+ else closePanels();+}++/** Starting it closes every panel, and the orb takes the microphone once the background paints it. */+async function toggleHandsFree(): Promise<void> {+ if (await readHandsFree()) return endHandsFree();+ if (await handsFreeSupported()) await startHandsFree();+}++/** The ways into Browsentic from outside its own pages: the right-click menu and the keyboard shortcuts. */+export function serveLaunchers(): void {+ onPanelPresence((open) => {+ panelOpen = open;+ repaintMenu();+ });++ browser.contextMenus.onClicked.addListener((info, tab) => {+ if (info.menuItemId === PANEL_ITEM && tab?.windowId != null) togglePanel(tab.windowId);+ if (info.menuItemId === HANDS_FREE_ITEM) void toggleHandsFree();+ });++ browser.commands.onCommand.addListener((command, tab) => {+ if (command === PANEL_SHORTCUT && tab?.windowId != null) togglePanel(tab.windowId);+ if (command === HANDS_FREE_SHORTCUT) void toggleHandsFree();+ });++ browser.storage.session.onChanged.addListener((changes) => {+ const handsFree = changes[HANDS_FREE_KEY];+ if (handsFree && !handsFree.oldValue !== !handsFree.newValue) repaintMenu();+ });+ browser.storage.local.onChanged.addListener((changes) => {+ if (CONTEXT_MENU_KEY in changes || SPEECH_SERVICE_KEY in changes) repaintMenu();+ });++ repaintMenu();+}diff --git a/src/lib/settings/shortcuts.test.ts b/src/lib/settings/shortcuts.test.tsnew file mode 100644index 0000000..3f3b69a--- /dev/null+++ b/src/lib/settings/shortcuts.test.ts@@ -0,0 +1,59 @@+import { describe, expect, it } from 'vitest';+import { HANDS_FREE_SHORTCUT, PANEL_SHORTCUT, keysOf, manifestCommands, shortcutsOf, shortcutsPageFor } from './shortcuts';++describe('manifestCommands', () => {+ it('gives Chromium the popup, the panel and hands-free', () => {+ expect(Object.keys(manifestCommands(false))).toEqual(['_execute_action', PANEL_SHORTCUT, HANDS_FREE_SHORTCUT]);+ });++ it('gives Firefox its own popup and sidebar names, and no hands-free', () => {+ expect(Object.keys(manifestCommands(true))).toEqual(['_execute_browser_action', '_execute_sidebar_action']);+ });+});++describe('keysOf', () => {+ it('splits the Windows and Linux form on +', () => {+ expect(keysOf('Alt+Shift+B')).toEqual(['Alt', 'Shift', 'B']);+ });++ it('splits the Mac symbols one modifier at a time, keeping a named key whole', () => {+ expect(keysOf('⌥⇧B')).toEqual(['⌥', '⇧', 'B']);+ expect(keysOf('⌃⇧Space')).toEqual(['⌃', '⇧', 'Space']);+ });++ it('has no keys for a shortcut nobody set', () => {+ expect(keysOf('')).toEqual([]);+ });+});++describe('shortcutsOf', () => {+ it('orders popup, panel, hands-free and names each the same everywhere', () => {+ const shortcuts = shortcutsOf([+ { name: HANDS_FREE_SHORTCUT, description: 'Start or stop hands-free', shortcut: '' },+ { name: PANEL_SHORTCUT, description: 'Open or close the side panel', shortcut: 'Alt+Shift+B' },+ { name: '_execute_action', description: '', shortcut: '' },+ ]);+ expect(shortcuts.map(({ label }) => label)).toEqual([+ 'Open the popup',+ 'Open or close the side panel',+ 'Start or stop hands-free',+ ]);+ expect(shortcuts[1].keys).toEqual(['Alt', 'Shift', 'B']);+ });++ it('keeps a command it does not know, by its own description, at the end', () => {+ const shortcuts = shortcutsOf([+ { name: 'something-new', description: 'Do something new' },+ { name: PANEL_SHORTCUT, shortcut: '' },+ ]);+ expect(shortcuts.map(({ label }) => label)).toEqual(['Open or close the side panel', 'Do something new']);+ });+});++describe('shortcutsPageFor', () => {+ it('sends Edge to its own scheme and every other Chromium to chrome://', () => {+ expect(shortcutsPageFor('Microsoft Edge')).toBe('edge://extensions/shortcuts');+ expect(shortcutsPageFor('Google Chrome')).toBe('chrome://extensions/shortcuts');+ expect(shortcutsPageFor(undefined)).toBe('chrome://extensions/shortcuts');+ });+});diff --git a/src/lib/settings/shortcuts.ts b/src/lib/settings/shortcuts.tsnew file mode 100644index 0000000..4a42ffa--- /dev/null+++ b/src/lib/settings/shortcuts.ts@@ -0,0 +1,74 @@+/**+ * The keyboard shortcuts the manifest declares. The browser keeps the keys, not Browsentic:+ * it assigns the suggested ones on install unless something else holds them, and the user+ * changes them on the browser's own shortcuts page.+ */++export const PANEL_SHORTCUT = 'toggle-side-panel';+export const HANDS_FREE_SHORTCUT = 'toggle-hands-free';++const POPUP = 'Open the popup';+const PANEL = 'Open or close the side panel';+const HANDS_FREE = 'Start or stop hands-free';++interface ManifestCommand {+ description: string;+ suggested_key?: { default: string };+}++/** Firefox has its own names for the popup and the sidebar, and no hands-free to start. */+export function manifestCommands(firefox: boolean): Record<string, ManifestCommand> {+ if (firefox) {+ return {+ _execute_browser_action: { description: POPUP },+ _execute_sidebar_action: { description: PANEL, suggested_key: { default: 'Alt+Shift+B' } },+ };+ }+ return {+ _execute_action: { description: POPUP },+ [PANEL_SHORTCUT]: { description: PANEL, suggested_key: { default: 'Alt+Shift+B' } },+ [HANDS_FREE_SHORTCUT]: { description: HANDS_FREE, suggested_key: { default: 'Alt+Shift+H' } },+ };+}++const LABELS: Record<string, string> = {+ _execute_action: POPUP,+ _execute_browser_action: POPUP,+ [PANEL_SHORTCUT]: PANEL,+ _execute_sidebar_action: PANEL,+ [HANDS_FREE_SHORTCUT]: HANDS_FREE,+};++const ORDER = Object.keys(LABELS);++export interface Shortcut {+ name: string;+ label: string;+ keys: string[];+}++/** The browser's own list, in the order the settings page shows it, each named the same way everywhere. */+export function shortcutsOf(commands: readonly { name?: string; description?: string; shortcut?: string }[]): Shortcut[] {+ return commands+ .flatMap(({ name, description, shortcut }) =>+ name ? [{ name, label: LABELS[name] ?? description ?? name, keys: keysOf(shortcut ?? '') }] : [],+ )+ .sort((a, b) => rank(a.name) - rank(b.name));+}++const rank = (name: string): number => {+ const index = ORDER.indexOf(name);+ return index === -1 ? ORDER.length : index;+};++/** Chromium on a Mac answers `⌥⇧B`; everywhere else it is `Alt+Shift+B`. */+export function keysOf(shortcut: string): string[] {+ if (!shortcut) return [];+ if (shortcut.includes('+')) return shortcut.split('+');+ const [, modifiers = '', key = ''] = /^([⌃⌥⇧⌘]*)(.*)$/u.exec(shortcut) ?? [];+ return [...modifiers, ...(key ? [key] : [])];+}++/** Edge answers to its own scheme; every other Chromium browser takes chrome://. */+export const shortcutsPageFor = (brand: string | undefined): string =>+ brand === 'Microsoft Edge' ? 'edge://extensions/shortcuts' : 'chrome://extensions/shortcuts';diff --git a/wxt.config.ts b/wxt.config.tsindex e078f34..82eb6a2 100644--- a/wxt.config.ts+++ b/wxt.config.ts@@ -1,5 +1,6 @@import { defineConfig } from 'wxt';import tailwindcss from '@tailwindcss/vite';+import { manifestCommands } from './src/lib/settings/shortcuts';export default defineConfig({modules: ['@wxt-dev/module-react'],@@ -31,6 +32,7 @@ export default defineConfig({...(browser === 'firefox' ? [] : ['sidePanel', 'debugger', 'offscreen']),],host_permissions: ['<all_urls>'],+ commands: manifestCommands(browser === 'firefox'),// Release Firefox installs only what addons.mozilla.org has signed, and signing needs a// permanent id. The update URL is polled daily; the release job publishes updates.json// next to each signed .xpi, and `releases/latest` always resolves to the newest one.