Skip to content

An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.

feat(android): a phone conversation is offered only what works on the phone: the Bridge withholds every tool off the phone's list and refuses a call to one, and on the phone tab the panel hides recordings, schedules, live tools, A-Eye, action cues and hands-free, attaches the phone page's context and shows its blocked-site note

Shahriarbec117c95153
diff --git a/src/daemon/agent/service.test.ts b/src/daemon/agent/service.test.ts
index 7e03239..5f77b62 100644
--- a/src/daemon/agent/service.test.ts
+++ b/src/daemon/agent/service.test.ts
@@ -8,6 +8,10 @@ import { profilePath } from '../profile';
import { configPath } from './config';
import { stubCli } from './runners/fixtures/support';
import { AgentSession } from './service';
+import { hashManifest } from '@/lib/actions/manifest';
+import { describeActions } from '@/lib/actions/registry';
+import { PHONE_TOOLS } from '@/lib/phone/features';
+import type { PhoneContext } from '@/lib/phone/types';
const CODE_TOOLS = ['page.injectCode', 'page.runCode'];
@@ -27,13 +31,58 @@ beforeEach(() => {
});
});
-async function offerTo(runId: string, context: { sessionId?: string; liveTools?: boolean }) {
+async function offerTo(runId: string, context: { sessionId?: string; liveTools?: boolean; phone?: PhoneContext }) {
session.handle({ t: 'instruct', id: runId, text: 'click Sign in', context });
const offer = session.offerFor(runId);
await vi.waitFor(() => expect(ended).toContain(runId), { timeout: 5_000 });
return offer;
}
+const PHONE: PhoneContext = { serial: 'emulator-5554', model: 'Pixel 8', android: '16', chrome: '150', viewport: { width: 411, height: 675, dpr: 2.625 } };
+
+describe('what a phone run is offered', () => {
+ const names = describeActions('chromium').map(({ name }) => name);
+
+ beforeEach(() => {
+ session = new AgentSession({
+ invoke: async () => failure('EXTENSION_OFFLINE', 'no browser'),
+ emit: (runId: string, event: RunEvent) => void (event.kind === 'error' && ended.push(runId)),
+ draft: () => {},
+ actionNames: () => names,
+ });
+ });
+
+ test('exactly the phone’s tools, and no reserved ones, even with Live tool on', async () => {
+ const offer = await offerTo('p', { sessionId: 's1', phone: PHONE, liveTools: true });
+ expect(names.filter((name) => !offer!.withheld.includes(name)).sort()).toEqual([...PHONE_TOOLS].sort());
+ expect(offer!.reserved).toEqual([]);
+ });
+
+ test('a desktop run beside it is offered what it always was', async () => {
+ expect(await offerTo('d', { sessionId: 's2' })).toEqual({ withheld: CODE_TOOLS, reserved: [FOCUS_SHOT_ACTION] });
+ });
+
+ test('a phone run leaves the tool list the browser describes, and so its hash, as it was', async () => {
+ const before = hashManifest(describeActions('chromium'));
+ await offerTo('p', { sessionId: 's1', phone: PHONE });
+ expect(hashManifest(describeActions('chromium'))).toBe(before);
+ });
+
+ test('a call to a tool off the list is refused before the browser hears of it', async () => {
+ const invoked: string[] = [];
+ const phoneSession = new AgentSession({
+ invoke: async (action) => (invoked.push(action), success({})),
+ emit: () => {},
+ draft: () => {},
+ actionNames: () => names,
+ });
+ phoneSession.handle({ t: 'instruct', id: 'p2', text: 'look', context: { sessionId: 's3', phone: PHONE } });
+ expect(await phoneSession.invokeForRun('p2', 'page.hoverElement', {})).toMatchObject({ ok: false, error: { code: 'NOT_ON_PHONE' } });
+ expect(invoked).toEqual([]);
+ phoneSession.dispose();
+ });
+});
+
describe("what a run's tool list holds", () => {
test('nothing, for a run that is not active', () => {
expect(session.offerFor('gone')).toBeNull();
diff --git a/src/daemon/agent/service.ts b/src/daemon/agent/service.ts
index d1510a6..ac7e61d 100644
--- a/src/daemon/agent/service.ts
+++ b/src/daemon/agent/service.ts
@@ -18,6 +18,7 @@ import { pickElement } from '@/lib/actions/page/pick-element';
import { runCode } from '@/lib/actions/page/run-code';
import { solveCaptcha } from '@/lib/actions/page/solve-captcha';
import { waitForElement } from '@/lib/actions/page/wait-for-element';
+import { notOnPhone, phoneOffers } from '@/lib/phone/features';
import { FOCUS_SHOT_ACTION, READ_SITEMAP_ACTION } from '@/lib/actions/reserved';
import { agentRunToolNames, toolNameFor } from '@/lib/actions/tool-names';
import type { FileReport } from '@/lib/files/report';
@@ -98,6 +99,8 @@ interface ActiveRun {
focusShot?: string;
/** The composer's “Live tool” switch, as it stood when this instruction was sent. */
liveTools: boolean;
+ /** It drives Chrome on an Android phone, which is offered only the tools that work there. */
+ phone: boolean;
/** Actions the user allowed once in this run that stay allowed for the rest of it. */
approved: Set<string>;
/** A schedule started it: an approval nobody answers is declined rather than awaited forever. */
@@ -199,6 +202,7 @@ export class AgentSession {
offerFor(runId: string): { withheld: string[]; reserved: string[] } | null {
const run = this.runs.get(runId);
if (!run) return null;
+ if (run.phone) return { withheld: this.deps.actionNames().filter((name) => !phoneOffers(name)), reserved: [] };
const codeListed = run.liveTools || (run.sessionId !== undefined && this.codeListed.has(run.sessionId));
return {
withheld: codeListed ? [] : [INJECT_ACTION, RUN_CODE_ACTION],
@@ -251,6 +255,11 @@ export class AgentSession {
return result;
}
+ if (run.phone && !phoneOffers(action)) {
+ emit({ kind: 'toolResult', toolId, ok: false, summary: 'not available on the phone' });
+ return failure('NOT_ON_PHONE', notOnPhone(action));
+ }
+
if (!run.liveTools && (action === INJECT_ACTION || action === RUN_CODE_ACTION)) {
emit({ kind: 'toolResult', toolId, ok: false, summary: 'live tools are switched off' });
return failure(
@@ -463,6 +472,7 @@ export class AgentSession {
ownedTabIds: [],
focusShot: context?.focus?.shot,
liveTools: context?.liveTools === true,
+ phone: context?.phone !== undefined,
approved: new Set(),
scheduled: context?.task !== undefined,
limits: RUNNERS[config.agent].limits,
diff --git a/src/extension/components/composer.tsx b/src/extension/components/composer.tsx
index c9fe4aa..0104023 100644
--- a/src/extension/components/composer.tsx
+++ b/src/extension/components/composer.tsx
@@ -48,6 +48,7 @@ export function Composer({
liveTools,
handsFree,
onToggleLiveTools,
+ onPhone = false,
scheduling,
onToggleScheduling,
actionCues,
@@ -80,6 +81,8 @@ export function Composer({
liveTools: boolean;
/** Whether `/hands-free` is offered — never where the browser cannot transcribe speech. */
handsFree: boolean;
+ /** The panel is showing the phone conversation: what only works in a desktop tab is not offered. */
+ onPhone?: boolean;
onToggleLiveTools: () => void;
scheduling: boolean;
onToggleScheduling: () => void;
@@ -249,17 +252,19 @@ export function Composer({
/>
<div className="flex items-center gap-0.5 px-1.5 pb-1.5">
- <Button
- variant={focus || picking ? 'subtle' : 'ghost'}
- size="icon-sm"
- title="A-Eye — point at an element on the page and send it with your next message"
- aria-label="Point at an element with A-Eye"
- onClick={onPick}
- disabled={!connected || picking}
- className={cn(focus || picking ? 'text-ember' : undefined)}
- >
- <ScanEye className={cn('size-3.5', picking && 'animate-pulse')} />
- </Button>
+ {!onPhone && (
+ <Button
+ variant={focus || picking ? 'subtle' : 'ghost'}
+ size="icon-sm"
+ title="A-Eye — point at an element on the page and send it with your next message"
+ aria-label="Point at an element with A-Eye"
+ onClick={onPick}
+ disabled={!connected || picking}
+ className={cn(focus || picking ? 'text-ember' : undefined)}
+ >
+ <ScanEye className={cn('size-3.5', picking && 'animate-pulse')} />
+ </Button>
+ )}
<Button
variant="ghost"
size="icon-sm"
@@ -280,54 +285,60 @@ export function Composer({
>
<FileUp className="size-3.5" />
</Button>
- <Button
- variant={liveTools ? 'subtle' : 'ghost'}
- size="icon-sm"
- role="switch"
- aria-checked={liveTools}
- aria-label={liveTools ? 'Turn live tools off' : 'Turn live tools on'}
- title={
- liveTools
- ? 'Live tool is on — the agent may write a small script for this page and ask you to approve it. Turn it off to keep to the built-in tools.'
- : 'Live tool — let the agent write a small script for repetitive work or for something no tool covers. You review and approve the code before it runs.'
- }
- onClick={onToggleLiveTools}
- disabled={!connected}
- className={cn(liveTools ? 'text-ember' : undefined)}
- >
- <Code2 className="size-3.5" />
- </Button>
- <Button
- variant={scheduling ? 'subtle' : 'ghost'}
- size="icon-sm"
- role="switch"
- aria-checked={scheduling}
- aria-label={scheduling ? 'Send now instead of scheduling' : 'Schedule this for later'}
- title={
- scheduling
- ? 'Scheduling is on — sending opens the schedule for this message instead of running it now.'
- : 'Schedule — run this later, or on repeat, instead of now.'
- }
- onClick={onToggleScheduling}
- disabled={!connected}
- >
- <CalendarClock className="size-3.5" />
- </Button>
- <Button
- variant={actionCues ? 'subtle' : 'ghost'}
- size="icon-sm"
- role="switch"
- aria-checked={actionCues}
- aria-label={actionCues ? 'Hide what the agent does' : 'Show what the agent does'}
- title={
- actionCues
- ? 'Showing what the agent does — a glowing ring marks each element it acts on. Turn it off to hide it.'
- : 'Show what the agent does — mark each element it acts on with a glowing ring.'
- }
- onClick={onToggleActionCues}
- >
- <SquareDashedMousePointer className="size-3.5" />
- </Button>
+ {!onPhone && (
+ <Button
+ variant={liveTools ? 'subtle' : 'ghost'}
+ size="icon-sm"
+ role="switch"
+ aria-checked={liveTools}
+ aria-label={liveTools ? 'Turn live tools off' : 'Turn live tools on'}
+ title={
+ liveTools
+ ? 'Live tool is on — the agent may write a small script for this page and ask you to approve it. Turn it off to keep to the built-in tools.'
+ : 'Live tool — let the agent write a small script for repetitive work or for something no tool covers. You review and approve the code before it runs.'
+ }
+ onClick={onToggleLiveTools}
+ disabled={!connected}
+ className={cn(liveTools ? 'text-ember' : undefined)}
+ >
+ <Code2 className="size-3.5" />
+ </Button>
+ )}
+ {!onPhone && (
+ <Button
+ variant={scheduling ? 'subtle' : 'ghost'}
+ size="icon-sm"
+ role="switch"
+ aria-checked={scheduling}
+ aria-label={scheduling ? 'Send now instead of scheduling' : 'Schedule this for later'}
+ title={
+ scheduling
+ ? 'Scheduling is on — sending opens the schedule for this message instead of running it now.'
+ : 'Schedule — run this later, or on repeat, instead of now.'
+ }
+ onClick={onToggleScheduling}
+ disabled={!connected}
+ >
+ <CalendarClock className="size-3.5" />
+ </Button>
+ )}
+ {!onPhone && (
+ <Button
+ variant={actionCues ? 'subtle' : 'ghost'}
+ size="icon-sm"
+ role="switch"
+ aria-checked={actionCues}
+ aria-label={actionCues ? 'Hide what the agent does' : 'Show what the agent does'}
+ title={
+ actionCues
+ ? 'Showing what the agent does — a glowing ring marks each element it acts on. Turn it off to hide it.'
+ : 'Show what the agent does — mark each element it acts on with a glowing ring.'
+ }
+ onClick={onToggleActionCues}
+ >
+ <SquareDashedMousePointer className="size-3.5" />
+ </Button>
+ )}
<Button
variant={voiceEnabled && !voice.error ? 'subtle' : 'ghost'}
size="icon-sm"
diff --git a/src/extension/components/greeting.tsx b/src/extension/components/greeting.tsx
index e38b383..d0f04dd 100644
--- a/src/extension/components/greeting.tsx
+++ b/src/extension/components/greeting.tsx
@@ -22,9 +22,11 @@ export function Greeting({
onGo,
onFix,
onUse,
+ hidden = [],
}: {
blocker?: StatusBlocker;
voiceOn: boolean;
+ hidden?: PanelTab[];
onGo: (tab: PanelTab) => void;
onFix: () => void;
onUse: (text: string) => void;
@@ -70,7 +72,7 @@ export function Greeting({
<div className="flex flex-wrap gap-1.5">
<Quick icon={BookOpen} label="Teach it this site" onClick={() => onGo('skills')} />
- <Quick icon={Clapperboard} label="Record a flow" onClick={() => onGo('recordings')} />
+ {!hidden.includes('recordings') && <Quick icon={Clapperboard} label="Record a flow" onClick={() => onGo('recordings')} />}
<Quick icon={History} label="Past chats" onClick={() => onGo('history')} />
</div>
</>
diff --git a/src/extension/components/panel-nav.tsx b/src/extension/components/panel-nav.tsx
index d347251..e548d96 100644
--- a/src/extension/components/panel-nav.tsx
+++ b/src/extension/components/panel-nav.tsx
@@ -23,10 +23,12 @@ export function PanelNav({
tab,
counts,
onSelect,
+ hidden = [],
}: {
tab: PanelTab;
counts: PanelCounts;
onSelect: (tab: PanelTab) => void;
+ hidden?: PanelTab[];
}) {
const nav = useRef<HTMLElement>(null);
const [fit, setFit] = useState<Fit>('full');
@@ -66,7 +68,7 @@ export function PanelNav({
aria-label="Panel sections"
className="flex shrink-0 items-center gap-0.5 overflow-x-auto border-b border-line px-2 pb-2 [scrollbar-width:none] [&::-webkit-scrollbar]:hidden"
>
- {TABS.map(({ id, label, icon: Icon }) => {
+ {TABS.filter(({ id }) => !hidden.includes(id)).map(({ id, label, icon: Icon }) => {
const active = tab === id;
const count = counts[id] ?? 0;
const labelled = fit === 'full' || (fit === 'active' && active);
diff --git a/src/extension/entrypoints/popup/App.tsx b/src/extension/entrypoints/popup/App.tsx
index 1aa2e6e..0f4821f 100644
--- a/src/extension/entrypoints/popup/App.tsx
+++ b/src/extension/entrypoints/popup/App.tsx
@@ -9,6 +9,7 @@ import { Button } from '@/extension/components/ui/button';
import { Input } from '@/extension/components/ui/input';
import { useDaemonState } from '@/lib/bridge/use-daemon-state';
import { startHandsFree } from '@/lib/bridge/panel-view';
+import { useMirroredPhone } from '@/lib/bridge/phone-client';
import { openSidePanel } from '@/lib/bridge/side-panel';
import { useRun } from '@/lib/bridge/use-run';
import { useHandsFreeSupported } from '@/lib/bridge/use-speech';
@@ -25,7 +26,8 @@ export default function App() {
const daemon = useDaemonState();
const run = useRun();
const [micOn, setMicOn] = useState(false);
- const handsFree = useHandsFreeSupported();
+ const onPhone = useMirroredPhone() !== null;
+ const handsFree = useHandsFreeSupported() && !onPhone;
const paired = daemon?.paired ?? false;
diff --git a/src/extension/entrypoints/sidepanel/App.tsx b/src/extension/entrypoints/sidepanel/App.tsx
index c76a702..e47205e 100644
--- a/src/extension/entrypoints/sidepanel/App.tsx
+++ b/src/extension/entrypoints/sidepanel/App.tsx
@@ -5,7 +5,7 @@ import { browser } from 'wxt/browser';
import { invokeInActiveTab } from '@/lib/actions/client';
import { isHandsFreeCommand, isRemoveToolsCommand } from '@/lib/bridge/commands';
import { getPageInfo } from '@/lib/actions/page/get-page-info';
-import { BRIDGE_CHANNEL, type FocusedElement } from '@/lib/actions/protocol';
+import { BRIDGE_CHANNEL, failure, type ActionResult, type FocusedElement } from '@/lib/actions/protocol';
import { MAX_STORED_FILE_BYTES } from '@/lib/files/report';
import { Wordmark } from '@/extension/components/brand';
import { BLOCKED_HERE, BlockedSiteNote } from '@/extension/components/blocked-site-note';
@@ -17,6 +17,8 @@ import { Greeting } from '@/extension/components/greeting';
import { MonitorBar } from '@/extension/components/monitor-bar';
import { PanelNav, type PanelTab } from '@/extension/components/panel-nav';
import { PhoneToggle } from '@/extension/components/phone-toggle';
+import { currentTarget } from '@/lib/bridge/phone';
+import { useMirroredPhone } from '@/lib/bridge/phone-client';
import { RecordingBar } from '@/extension/components/recording-bar';
import { RecordingPanel } from '@/extension/components/recording-panel';
import { RunTimeline } from '@/extension/components/run-timeline';
@@ -59,6 +61,8 @@ const PINNED_SLACK_PX = 56;
const DETACH_MS = 560;
const DETACH_STUCK_MS = 1_500;
+/** Recordings and schedules run in desktop tabs; the phone conversation has neither. */
+const PHONE_HIDDEN_TABS: PanelTab[] = ['recordings', 'tasks'];
export default function App() {
const daemon = useDaemonState();
@@ -66,7 +70,7 @@ export default function App() {
const star = useStarNudge(run.completed, run.sessionId, run.running);
const [voiceEnabled, setVoiceEnabled] = useVoiceEnabled();
const [actionCues, setActionCues] = useActionCues();
- const handsFree = useHandsFreeSupported();
+ const handsFreeSupported = useHandsFreeSupported();
const storedFiles = useStoredFiles();
const sessions = useStoredSessions();
const recordings = useStoredRecordings();
@@ -76,7 +80,10 @@ export default function App() {
const [scheduling, setScheduling] = useState(false);
const takeSeed = useCallback(() => setTaskSeed(null), []);
const tabUrl = useActiveTabUrl();
- const blockedHere = useBlockedPattern(tabUrl);
+ const phone = useMirroredPhone();
+ const onPhone = phone !== null;
+ const handsFree = handsFreeSupported && !onPhone;
+ const blockedHere = useBlockedPattern(phone ? (currentTarget(phone)?.url ?? '') : tabUrl);
const [attachError, setAttachError] = useState<string | null>(null);
const [attachedSkill, setAttachedSkill] = useState<AttachedSkill | null>(null);
const [focus, setFocus] = useState<FocusedElement | null>(null);
@@ -84,6 +91,14 @@ export default function App() {
const [picking, setPicking] = useState(false);
const [liveTools, setLiveTools] = useState(false);
const [tab, setTab] = usePanelTab();
+ useEffect(() => {
+ if (onPhone && PHONE_HIDDEN_TABS.includes(tab)) setTab('chat');
+ }, [onPhone, tab, setTab]);
+ useEffect(() => {
+ if (!onPhone) return;
+ setScheduling(false);
+ setFocus(null);
+ }, [onPhone]);
const [connectionOpen, setConnectionOpen] = useState(false);
const [, setCollapsed] = usePanelCollapsed();
const viewport = useRef<HTMLDivElement>(null);
@@ -248,7 +263,10 @@ export default function App() {
setAttachError(BLOCKED_HERE);
return;
}
- const result = await invokeInActiveTab(getPageInfo, {});
+ const result: ActionResult<{ document: { title: string; url: string }; selection?: string }> = onPhone
+ ? ((await browser.runtime.sendMessage({ channel: BRIDGE_CHANNEL, op: 'invoke', action: getPageInfo.name, input: {} })) ??
+ failure('BRIDGE_ERROR', 'The extension did not answer.'))
+ : await invokeInActiveTab(getPageInfo, {});
if (!result.ok) {
voice.setInput(`${voice.input}\n[couldn’t read this page: ${result.error.message}]\n`);
return;
@@ -358,7 +376,7 @@ export default function App() {
{connectionOpen && <ConnectionSheet onClose={() => setConnectionOpen(false)} />}
- <PanelNav tab={tab} counts={counts} onSelect={open} />
+ <PanelNav tab={tab} counts={counts} onSelect={open} hidden={onPhone ? PHONE_HIDDEN_TABS : []} />
<SessionRail
sessions={run.sessions}
@@ -382,7 +400,7 @@ export default function App() {
<span className="shrink-0 font-mono text-[10px] tracking-[0.12em] text-brand uppercase">Watch</span>
</button>
)}
- {run.recording && <RecordingBar state={run.recording} onStop={run.stopRecording} />}
+ {run.recording && !onPhone && <RecordingBar state={run.recording} onStop={run.stopRecording} />}
{run.monitors.map((monitor) => (
<MonitorBar key={monitor.monitorId} state={monitor} onStop={() => run.stopMonitor(monitor.monitorId)} />
))}
@@ -396,6 +414,7 @@ export default function App() {
run.items.length === 0 ? (
<Greeting
blocker={status.blocker}
+ hidden={onPhone ? PHONE_HIDDEN_TABS : []}
voiceOn={voiceEnabled && voice.supported}
onGo={open}
onFix={() => setConnectionOpen(true)}
@@ -406,7 +425,7 @@ export default function App() {
items={run.items}
running={run.running}
onDecide={run.decide}
- onSchedule={(text) => schedule({ text, url: tabUrl })}
+ onSchedule={onPhone ? undefined : (text) => schedule({ text, url: tabUrl })}
/>
)
) : tab === 'history' ? (
@@ -461,7 +480,7 @@ export default function App() {
{starShown && <StarBadge onStar={star.star} onClose={star.close} />}
</div>
- {showTools && (
+ {showTools && !onPhone && (
<SavedToolList
tools={run.tools}
autoRunReady={run.autoRunReady}
@@ -474,7 +493,7 @@ export default function App() {
{tab === 'chat' && (
<footer className="shrink-0 border-t border-line p-3">
{blockedHere && <BlockedSiteNote pattern={blockedHere} />}
- {run.toolOffer && (
+ {run.toolOffer && !onPhone && (
<KeepToolPrompt
offer={run.toolOffer}
autoRunReady={run.autoRunReady}
@@ -502,6 +521,7 @@ export default function App() {
onToggleActionCues={() => setActionCues(!actionCues)}
onAttachSkill={setAttachedSkill}
handsFree={handsFree}
+ onPhone={onPhone}
onCommand={(command) => {
if (handsFree && isHandsFreeCommand(command)) {
detach();
@@ -514,7 +534,7 @@ export default function App() {
open('chat');
run.send(command);
}}
- tools={run.tools}
+ tools={onPhone ? [] : run.tools}
onRunTool={(id) => {
open('chat');
run.runTool(id);
diff --git a/src/lib/bridge/phone-client.ts b/src/lib/bridge/phone-client.ts
index 417301f..a5a24c7 100644
--- a/src/lib/bridge/phone-client.ts
+++ b/src/lib/bridge/phone-client.ts
@@ -2,6 +2,7 @@ import { useEffect, useState } from 'react';
import { browser } from 'wxt/browser';
import { BRIDGE_CHANNEL, failure, type ActionResult } from '@/lib/actions/protocol';
import { PHONE_KEY, readPhone, type PhoneSession } from './phone';
+import { useActiveTab } from './use-active-tab';
export type PhoneRequest = { op: 'phoneStart'; serial: string; windowId?: number } | { op: 'phoneEnd' } | { op: 'phoneOpenChrome' };
@@ -30,3 +31,10 @@ export function usePhoneSession(): PhoneSession | null {
return session;
}
+
+/** The phone session whose mirror is the active tab, matched by tab id: without the `tabs` permission an extension page's URL is hidden. */
+export function useMirroredPhone(): PhoneSession | null {
+ const { tabId } = useActiveTab();
+ const phone = usePhoneSession();
+ return tabId !== undefined && phone?.mirrorTabId === tabId ? phone : null;
+}