An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.
TypeScript agentic-browseragentic-browsingagy-cliaiai-agentai-browser-extensionai-toolsbrave-extensionbrowser-agentbrowser-automationbrowser-extensionbrowser-usebrowserskillchrome-extensionclaude-codecodexfirefox-addonmcp-servernanobrowserweb-automation
feat(android): a phone conversation is offered only what works on the phone: the Bridge withholds every tool off the phone's list and refuses a call to one, and on the phone tab the panel hides recordings, schedules, live tools, A-Eye, action cues and hands-free, attaches the phone page's context and shows its blocked-site note
Shahriarbec117c95153
diff --git a/src/daemon/agent/service.test.ts b/src/daemon/agent/service.test.tsindex 7e03239..5f77b62 100644--- a/src/daemon/agent/service.test.ts+++ b/src/daemon/agent/service.test.ts@@ -8,6 +8,10 @@ import { profilePath } from '../profile';import { configPath } from './config';import { stubCli } from './runners/fixtures/support';import { AgentSession } from './service';+import { hashManifest } from '@/lib/actions/manifest';+import { describeActions } from '@/lib/actions/registry';+import { PHONE_TOOLS } from '@/lib/phone/features';+import type { PhoneContext } from '@/lib/phone/types';const CODE_TOOLS = ['page.injectCode', 'page.runCode'];@@ -27,13 +31,58 @@ beforeEach(() => {});});-async function offerTo(runId: string, context: { sessionId?: string; liveTools?: boolean }) {+async function offerTo(runId: string, context: { sessionId?: string; liveTools?: boolean; phone?: PhoneContext }) {session.handle({ t: 'instruct', id: runId, text: 'click Sign in', context });const offer = session.offerFor(runId);await vi.waitFor(() => expect(ended).toContain(runId), { timeout: 5_000 });return offer;}+const PHONE: PhoneContext = { serial: 'emulator-5554', model: 'Pixel 8', android: '16', chrome: '150', viewport: { width: 411, height: 675, dpr: 2.625 } };++describe('what a phone run is offered', () => {+ const names = describeActions('chromium').map(({ name }) => name);++ beforeEach(() => {+ session = new AgentSession({+ invoke: async () => failure('EXTENSION_OFFLINE', 'no browser'),+ emit: (runId: string, event: RunEvent) => void (event.kind === 'error' && ended.push(runId)),+ draft: () => {},+ actionNames: () => names,+ });+ });++ test('exactly the phone’s tools, and no reserved ones, even with Live tool on', async () => {+ const offer = await offerTo('p', { sessionId: 's1', phone: PHONE, liveTools: true });+ expect(names.filter((name) => !offer!.withheld.includes(name)).sort()).toEqual([...PHONE_TOOLS].sort());+ expect(offer!.reserved).toEqual([]);+ });++ test('a desktop run beside it is offered what it always was', async () => {+ expect(await offerTo('d', { sessionId: 's2' })).toEqual({ withheld: CODE_TOOLS, reserved: [FOCUS_SHOT_ACTION] });+ });++ test('a phone run leaves the tool list the browser describes, and so its hash, as it was', async () => {+ const before = hashManifest(describeActions('chromium'));+ await offerTo('p', { sessionId: 's1', phone: PHONE });+ expect(hashManifest(describeActions('chromium'))).toBe(before);+ });++ test('a call to a tool off the list is refused before the browser hears of it', async () => {+ const invoked: string[] = [];+ const phoneSession = new AgentSession({+ invoke: async (action) => (invoked.push(action), success({})),+ emit: () => {},+ draft: () => {},+ actionNames: () => names,+ });+ phoneSession.handle({ t: 'instruct', id: 'p2', text: 'look', context: { sessionId: 's3', phone: PHONE } });+ expect(await phoneSession.invokeForRun('p2', 'page.hoverElement', {})).toMatchObject({ ok: false, error: { code: 'NOT_ON_PHONE' } });+ expect(invoked).toEqual([]);+ phoneSession.dispose();+ });+});+describe("what a run's tool list holds", () => {test('nothing, for a run that is not active', () => {expect(session.offerFor('gone')).toBeNull();diff --git a/src/daemon/agent/service.ts b/src/daemon/agent/service.tsindex d1510a6..ac7e61d 100644--- a/src/daemon/agent/service.ts+++ b/src/daemon/agent/service.ts@@ -18,6 +18,7 @@ import { pickElement } from '@/lib/actions/page/pick-element';import { runCode } from '@/lib/actions/page/run-code';import { solveCaptcha } from '@/lib/actions/page/solve-captcha';import { waitForElement } from '@/lib/actions/page/wait-for-element';+import { notOnPhone, phoneOffers } from '@/lib/phone/features';import { FOCUS_SHOT_ACTION, READ_SITEMAP_ACTION } from '@/lib/actions/reserved';import { agentRunToolNames, toolNameFor } from '@/lib/actions/tool-names';import type { FileReport } from '@/lib/files/report';@@ -98,6 +99,8 @@ interface ActiveRun {focusShot?: string;/** The composer's “Live tool” switch, as it stood when this instruction was sent. */liveTools: boolean;+ /** It drives Chrome on an Android phone, which is offered only the tools that work there. */+ phone: boolean;/** Actions the user allowed once in this run that stay allowed for the rest of it. */approved: Set<string>;/** A schedule started it: an approval nobody answers is declined rather than awaited forever. */@@ -199,6 +202,7 @@ export class AgentSession {offerFor(runId: string): { withheld: string[]; reserved: string[] } | null {const run = this.runs.get(runId);if (!run) return null;+ if (run.phone) return { withheld: this.deps.actionNames().filter((name) => !phoneOffers(name)), reserved: [] };const codeListed = run.liveTools || (run.sessionId !== undefined && this.codeListed.has(run.sessionId));return {withheld: codeListed ? [] : [INJECT_ACTION, RUN_CODE_ACTION],@@ -251,6 +255,11 @@ export class AgentSession {return result;}+ if (run.phone && !phoneOffers(action)) {+ emit({ kind: 'toolResult', toolId, ok: false, summary: 'not available on the phone' });+ return failure('NOT_ON_PHONE', notOnPhone(action));+ }+if (!run.liveTools && (action === INJECT_ACTION || action === RUN_CODE_ACTION)) {emit({ kind: 'toolResult', toolId, ok: false, summary: 'live tools are switched off' });return failure(@@ -463,6 +472,7 @@ export class AgentSession {ownedTabIds: [],focusShot: context?.focus?.shot,liveTools: context?.liveTools === true,+ phone: context?.phone !== undefined,approved: new Set(),scheduled: context?.task !== undefined,limits: RUNNERS[config.agent].limits,diff --git a/src/extension/components/composer.tsx b/src/extension/components/composer.tsxindex c9fe4aa..0104023 100644--- a/src/extension/components/composer.tsx+++ b/src/extension/components/composer.tsx@@ -48,6 +48,7 @@ export function Composer({liveTools,handsFree,onToggleLiveTools,+ onPhone = false,scheduling,onToggleScheduling,actionCues,@@ -80,6 +81,8 @@ export function Composer({liveTools: boolean;/** Whether `/hands-free` is offered — never where the browser cannot transcribe speech. */handsFree: boolean;+ /** The panel is showing the phone conversation: what only works in a desktop tab is not offered. */+ onPhone?: boolean;onToggleLiveTools: () => void;scheduling: boolean;onToggleScheduling: () => void;@@ -249,17 +252,19 @@ export function Composer({/><div className="flex items-center gap-0.5 px-1.5 pb-1.5">- <Button- variant={focus || picking ? 'subtle' : 'ghost'}- size="icon-sm"- title="A-Eye — point at an element on the page and send it with your next message"- aria-label="Point at an element with A-Eye"- onClick={onPick}- disabled={!connected || picking}- className={cn(focus || picking ? 'text-ember' : undefined)}- >- <ScanEye className={cn('size-3.5', picking && 'animate-pulse')} />- </Button>+ {!onPhone && (+ <Button+ variant={focus || picking ? 'subtle' : 'ghost'}+ size="icon-sm"+ title="A-Eye — point at an element on the page and send it with your next message"+ aria-label="Point at an element with A-Eye"+ onClick={onPick}+ disabled={!connected || picking}+ className={cn(focus || picking ? 'text-ember' : undefined)}+ >+ <ScanEye className={cn('size-3.5', picking && 'animate-pulse')} />+ </Button>+ )}<Buttonvariant="ghost"size="icon-sm"@@ -280,54 +285,60 @@ export function Composer({><FileUp className="size-3.5" /></Button>- <Button- variant={liveTools ? 'subtle' : 'ghost'}- size="icon-sm"- role="switch"- aria-checked={liveTools}- aria-label={liveTools ? 'Turn live tools off' : 'Turn live tools on'}- title={- liveTools- ? 'Live tool is on — the agent may write a small script for this page and ask you to approve it. Turn it off to keep to the built-in tools.'- : 'Live tool — let the agent write a small script for repetitive work or for something no tool covers. You review and approve the code before it runs.'- }- onClick={onToggleLiveTools}- disabled={!connected}- className={cn(liveTools ? 'text-ember' : undefined)}- >- <Code2 className="size-3.5" />- </Button>- <Button- variant={scheduling ? 'subtle' : 'ghost'}- size="icon-sm"- role="switch"- aria-checked={scheduling}- aria-label={scheduling ? 'Send now instead of scheduling' : 'Schedule this for later'}- title={- scheduling- ? 'Scheduling is on — sending opens the schedule for this message instead of running it now.'- : 'Schedule — run this later, or on repeat, instead of now.'- }- onClick={onToggleScheduling}- disabled={!connected}- >- <CalendarClock className="size-3.5" />- </Button>- <Button- variant={actionCues ? 'subtle' : 'ghost'}- size="icon-sm"- role="switch"- aria-checked={actionCues}- aria-label={actionCues ? 'Hide what the agent does' : 'Show what the agent does'}- title={- actionCues- ? 'Showing what the agent does — a glowing ring marks each element it acts on. Turn it off to hide it.'- : 'Show what the agent does — mark each element it acts on with a glowing ring.'- }- onClick={onToggleActionCues}- >- <SquareDashedMousePointer className="size-3.5" />- </Button>+ {!onPhone && (+ <Button+ variant={liveTools ? 'subtle' : 'ghost'}+ size="icon-sm"+ role="switch"+ aria-checked={liveTools}+ aria-label={liveTools ? 'Turn live tools off' : 'Turn live tools on'}+ title={+ liveTools+ ? 'Live tool is on — the agent may write a small script for this page and ask you to approve it. Turn it off to keep to the built-in tools.'+ : 'Live tool — let the agent write a small script for repetitive work or for something no tool covers. You review and approve the code before it runs.'+ }+ onClick={onToggleLiveTools}+ disabled={!connected}+ className={cn(liveTools ? 'text-ember' : undefined)}+ >+ <Code2 className="size-3.5" />+ </Button>+ )}+ {!onPhone && (+ <Button+ variant={scheduling ? 'subtle' : 'ghost'}+ size="icon-sm"+ role="switch"+ aria-checked={scheduling}+ aria-label={scheduling ? 'Send now instead of scheduling' : 'Schedule this for later'}+ title={+ scheduling+ ? 'Scheduling is on — sending opens the schedule for this message instead of running it now.'+ : 'Schedule — run this later, or on repeat, instead of now.'+ }+ onClick={onToggleScheduling}+ disabled={!connected}+ >+ <CalendarClock className="size-3.5" />+ </Button>+ )}+ {!onPhone && (+ <Button+ variant={actionCues ? 'subtle' : 'ghost'}+ size="icon-sm"+ role="switch"+ aria-checked={actionCues}+ aria-label={actionCues ? 'Hide what the agent does' : 'Show what the agent does'}+ title={+ actionCues+ ? 'Showing what the agent does — a glowing ring marks each element it acts on. Turn it off to hide it.'+ : 'Show what the agent does — mark each element it acts on with a glowing ring.'+ }+ onClick={onToggleActionCues}+ >+ <SquareDashedMousePointer className="size-3.5" />+ </Button>+ )}<Buttonvariant={voiceEnabled && !voice.error ? 'subtle' : 'ghost'}size="icon-sm"diff --git a/src/extension/components/greeting.tsx b/src/extension/components/greeting.tsxindex e38b383..d0f04dd 100644--- a/src/extension/components/greeting.tsx+++ b/src/extension/components/greeting.tsx@@ -22,9 +22,11 @@ export function Greeting({onGo,onFix,onUse,+ hidden = [],}: {blocker?: StatusBlocker;voiceOn: boolean;+ hidden?: PanelTab[];onGo: (tab: PanelTab) => void;onFix: () => void;onUse: (text: string) => void;@@ -70,7 +72,7 @@ export function Greeting({<div className="flex flex-wrap gap-1.5"><Quick icon={BookOpen} label="Teach it this site" onClick={() => onGo('skills')} />- <Quick icon={Clapperboard} label="Record a flow" onClick={() => onGo('recordings')} />+ {!hidden.includes('recordings') && <Quick icon={Clapperboard} label="Record a flow" onClick={() => onGo('recordings')} />}<Quick icon={History} label="Past chats" onClick={() => onGo('history')} /></div></>diff --git a/src/extension/components/panel-nav.tsx b/src/extension/components/panel-nav.tsxindex d347251..e548d96 100644--- a/src/extension/components/panel-nav.tsx+++ b/src/extension/components/panel-nav.tsx@@ -23,10 +23,12 @@ export function PanelNav({tab,counts,onSelect,+ hidden = [],}: {tab: PanelTab;counts: PanelCounts;onSelect: (tab: PanelTab) => void;+ hidden?: PanelTab[];}) {const nav = useRef<HTMLElement>(null);const [fit, setFit] = useState<Fit>('full');@@ -66,7 +68,7 @@ export function PanelNav({aria-label="Panel sections"className="flex shrink-0 items-center gap-0.5 overflow-x-auto border-b border-line px-2 pb-2 [scrollbar-width:none] [&::-webkit-scrollbar]:hidden">- {TABS.map(({ id, label, icon: Icon }) => {+ {TABS.filter(({ id }) => !hidden.includes(id)).map(({ id, label, icon: Icon }) => {const active = tab === id;const count = counts[id] ?? 0;const labelled = fit === 'full' || (fit === 'active' && active);diff --git a/src/extension/entrypoints/popup/App.tsx b/src/extension/entrypoints/popup/App.tsxindex 1aa2e6e..0f4821f 100644--- a/src/extension/entrypoints/popup/App.tsx+++ b/src/extension/entrypoints/popup/App.tsx@@ -9,6 +9,7 @@ import { Button } from '@/extension/components/ui/button';import { Input } from '@/extension/components/ui/input';import { useDaemonState } from '@/lib/bridge/use-daemon-state';import { startHandsFree } from '@/lib/bridge/panel-view';+import { useMirroredPhone } from '@/lib/bridge/phone-client';import { openSidePanel } from '@/lib/bridge/side-panel';import { useRun } from '@/lib/bridge/use-run';import { useHandsFreeSupported } from '@/lib/bridge/use-speech';@@ -25,7 +26,8 @@ export default function App() {const daemon = useDaemonState();const run = useRun();const [micOn, setMicOn] = useState(false);- const handsFree = useHandsFreeSupported();+ const onPhone = useMirroredPhone() !== null;+ const handsFree = useHandsFreeSupported() && !onPhone;const paired = daemon?.paired ?? false;diff --git a/src/extension/entrypoints/sidepanel/App.tsx b/src/extension/entrypoints/sidepanel/App.tsxindex c76a702..e47205e 100644--- a/src/extension/entrypoints/sidepanel/App.tsx+++ b/src/extension/entrypoints/sidepanel/App.tsx@@ -5,7 +5,7 @@ import { browser } from 'wxt/browser';import { invokeInActiveTab } from '@/lib/actions/client';import { isHandsFreeCommand, isRemoveToolsCommand } from '@/lib/bridge/commands';import { getPageInfo } from '@/lib/actions/page/get-page-info';-import { BRIDGE_CHANNEL, type FocusedElement } from '@/lib/actions/protocol';+import { BRIDGE_CHANNEL, failure, type ActionResult, type FocusedElement } from '@/lib/actions/protocol';import { MAX_STORED_FILE_BYTES } from '@/lib/files/report';import { Wordmark } from '@/extension/components/brand';import { BLOCKED_HERE, BlockedSiteNote } from '@/extension/components/blocked-site-note';@@ -17,6 +17,8 @@ import { Greeting } from '@/extension/components/greeting';import { MonitorBar } from '@/extension/components/monitor-bar';import { PanelNav, type PanelTab } from '@/extension/components/panel-nav';import { PhoneToggle } from '@/extension/components/phone-toggle';+import { currentTarget } from '@/lib/bridge/phone';+import { useMirroredPhone } from '@/lib/bridge/phone-client';import { RecordingBar } from '@/extension/components/recording-bar';import { RecordingPanel } from '@/extension/components/recording-panel';import { RunTimeline } from '@/extension/components/run-timeline';@@ -59,6 +61,8 @@ const PINNED_SLACK_PX = 56;const DETACH_MS = 560;const DETACH_STUCK_MS = 1_500;+/** Recordings and schedules run in desktop tabs; the phone conversation has neither. */+const PHONE_HIDDEN_TABS: PanelTab[] = ['recordings', 'tasks'];export default function App() {const daemon = useDaemonState();@@ -66,7 +70,7 @@ export default function App() {const star = useStarNudge(run.completed, run.sessionId, run.running);const [voiceEnabled, setVoiceEnabled] = useVoiceEnabled();const [actionCues, setActionCues] = useActionCues();- const handsFree = useHandsFreeSupported();+ const handsFreeSupported = useHandsFreeSupported();const storedFiles = useStoredFiles();const sessions = useStoredSessions();const recordings = useStoredRecordings();@@ -76,7 +80,10 @@ export default function App() {const [scheduling, setScheduling] = useState(false);const takeSeed = useCallback(() => setTaskSeed(null), []);const tabUrl = useActiveTabUrl();- const blockedHere = useBlockedPattern(tabUrl);+ const phone = useMirroredPhone();+ const onPhone = phone !== null;+ const handsFree = handsFreeSupported && !onPhone;+ const blockedHere = useBlockedPattern(phone ? (currentTarget(phone)?.url ?? '') : tabUrl);const [attachError, setAttachError] = useState<string | null>(null);const [attachedSkill, setAttachedSkill] = useState<AttachedSkill | null>(null);const [focus, setFocus] = useState<FocusedElement | null>(null);@@ -84,6 +91,14 @@ export default function App() {const [picking, setPicking] = useState(false);const [liveTools, setLiveTools] = useState(false);const [tab, setTab] = usePanelTab();+ useEffect(() => {+ if (onPhone && PHONE_HIDDEN_TABS.includes(tab)) setTab('chat');+ }, [onPhone, tab, setTab]);+ useEffect(() => {+ if (!onPhone) return;+ setScheduling(false);+ setFocus(null);+ }, [onPhone]);const [connectionOpen, setConnectionOpen] = useState(false);const [, setCollapsed] = usePanelCollapsed();const viewport = useRef<HTMLDivElement>(null);@@ -248,7 +263,10 @@ export default function App() {setAttachError(BLOCKED_HERE);return;}- const result = await invokeInActiveTab(getPageInfo, {});+ const result: ActionResult<{ document: { title: string; url: string }; selection?: string }> = onPhone+ ? ((await browser.runtime.sendMessage({ channel: BRIDGE_CHANNEL, op: 'invoke', action: getPageInfo.name, input: {} })) ??+ failure('BRIDGE_ERROR', 'The extension did not answer.'))+ : await invokeInActiveTab(getPageInfo, {});if (!result.ok) {voice.setInput(`${voice.input}\n[couldn’t read this page: ${result.error.message}]\n`);return;@@ -358,7 +376,7 @@ export default function App() {{connectionOpen && <ConnectionSheet onClose={() => setConnectionOpen(false)} />}- <PanelNav tab={tab} counts={counts} onSelect={open} />+ <PanelNav tab={tab} counts={counts} onSelect={open} hidden={onPhone ? PHONE_HIDDEN_TABS : []} /><SessionRailsessions={run.sessions}@@ -382,7 +400,7 @@ export default function App() {<span className="shrink-0 font-mono text-[10px] tracking-[0.12em] text-brand uppercase">Watch</span></button>)}- {run.recording && <RecordingBar state={run.recording} onStop={run.stopRecording} />}+ {run.recording && !onPhone && <RecordingBar state={run.recording} onStop={run.stopRecording} />}{run.monitors.map((monitor) => (<MonitorBar key={monitor.monitorId} state={monitor} onStop={() => run.stopMonitor(monitor.monitorId)} />))}@@ -396,6 +414,7 @@ export default function App() {run.items.length === 0 ? (<Greetingblocker={status.blocker}+ hidden={onPhone ? PHONE_HIDDEN_TABS : []}voiceOn={voiceEnabled && voice.supported}onGo={open}onFix={() => setConnectionOpen(true)}@@ -406,7 +425,7 @@ export default function App() {items={run.items}running={run.running}onDecide={run.decide}- onSchedule={(text) => schedule({ text, url: tabUrl })}+ onSchedule={onPhone ? undefined : (text) => schedule({ text, url: tabUrl })}/>)) : tab === 'history' ? (@@ -461,7 +480,7 @@ export default function App() {{starShown && <StarBadge onStar={star.star} onClose={star.close} />}</div>- {showTools && (+ {showTools && !onPhone && (<SavedToolListtools={run.tools}autoRunReady={run.autoRunReady}@@ -474,7 +493,7 @@ export default function App() {{tab === 'chat' && (<footer className="shrink-0 border-t border-line p-3">{blockedHere && <BlockedSiteNote pattern={blockedHere} />}- {run.toolOffer && (+ {run.toolOffer && !onPhone && (<KeepToolPromptoffer={run.toolOffer}autoRunReady={run.autoRunReady}@@ -502,6 +521,7 @@ export default function App() {onToggleActionCues={() => setActionCues(!actionCues)}onAttachSkill={setAttachedSkill}handsFree={handsFree}+ onPhone={onPhone}onCommand={(command) => {if (handsFree && isHandsFreeCommand(command)) {detach();@@ -514,7 +534,7 @@ export default function App() {open('chat');run.send(command);}}- tools={run.tools}+ tools={onPhone ? [] : run.tools}onRunTool={(id) => {open('chat');run.runTool(id);diff --git a/src/lib/bridge/phone-client.ts b/src/lib/bridge/phone-client.tsindex 417301f..a5a24c7 100644--- a/src/lib/bridge/phone-client.ts+++ b/src/lib/bridge/phone-client.ts@@ -2,6 +2,7 @@ import { useEffect, useState } from 'react';import { browser } from 'wxt/browser';import { BRIDGE_CHANNEL, failure, type ActionResult } from '@/lib/actions/protocol';import { PHONE_KEY, readPhone, type PhoneSession } from './phone';+import { useActiveTab } from './use-active-tab';export type PhoneRequest = { op: 'phoneStart'; serial: string; windowId?: number } | { op: 'phoneEnd' } | { op: 'phoneOpenChrome' };@@ -30,3 +31,10 @@ export function usePhoneSession(): PhoneSession | null {return session;}++/** The phone session whose mirror is the active tab, matched by tab id: without the `tabs` permission an extension page's URL is hidden. */+export function useMirroredPhone(): PhoneSession | null {+ const { tabId } = useActiveTab();+ const phone = usePhoneSession();+ return tabId !== undefined && phone?.mirrorTabId === tabId ? phone : null;+}