Skip to content

An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.

feat(android): A-Eye on the phone: the panel's button and the agent's pick bring the lens up on the phone, a tap on the phone tab's picture picks the element without pressing it, and the pick carries the element's photograph from the phone

Shahriarfeb129cbf481
diff --git a/src/daemon/agent/service.test.ts b/src/daemon/agent/service.test.ts
index acb1e02..fe1cd4b 100644
--- a/src/daemon/agent/service.test.ts
+++ b/src/daemon/agent/service.test.ts
@@ -52,10 +52,10 @@ describe('what a phone run is offered', () => {
});
});
- test('exactly the phone’s tools with Live tool on, and no reserved ones', async () => {
+ test('exactly the phone’s tools with Live tool on, and the tool that shows an A-Eye pick', async () => {
const offer = await offerTo('p', { sessionId: 's1', phone: PHONE, liveTools: true });
expect(names.filter((name) => !offer!.withheld.includes(name)).sort()).toEqual([...PHONE_TOOLS].sort());
- expect(offer!.reserved).toEqual([]);
+ expect(offer!.reserved).toEqual([FOCUS_SHOT_ACTION]);
});
test('the page-code tools only with Live tool on, the same rule as a desktop run', async () => {
diff --git a/src/daemon/agent/service.ts b/src/daemon/agent/service.ts
index 2ec2aa7..c74fd68 100644
--- a/src/daemon/agent/service.ts
+++ b/src/daemon/agent/service.ts
@@ -206,7 +206,7 @@ export class AgentSession {
const codeWithheld = codeListed ? [] : [INJECT_ACTION, RUN_CODE_ACTION];
if (run.phone) {
const offPhone = this.deps.actionNames().filter((name) => !phoneOffers(name));
- return { withheld: [...new Set([...offPhone, ...codeWithheld])], reserved: [] };
+ return { withheld: [...new Set([...offPhone, ...codeWithheld])], reserved: [FOCUS_SHOT_ACTION] };
}
return {
withheld: codeWithheld,
diff --git a/src/extension/components/composer.tsx b/src/extension/components/composer.tsx
index b54c93d..c16d480 100644
--- a/src/extension/components/composer.tsx
+++ b/src/extension/components/composer.tsx
@@ -252,19 +252,17 @@ export function Composer({
/>
<div className="flex items-center gap-0.5 px-1.5 pb-1.5">
- {!onPhone && (
- <Button
- variant={focus || picking ? 'subtle' : 'ghost'}
- size="icon-sm"
- title="A-Eye — point at an element on the page and send it with your next message"
- aria-label="Point at an element with A-Eye"
- onClick={onPick}
- disabled={!connected || picking}
- className={cn(focus || picking ? 'text-ember' : undefined)}
- >
- <ScanEye className={cn('size-3.5', picking && 'animate-pulse')} />
- </Button>
- )}
+ <Button
+ variant={focus || picking ? 'subtle' : 'ghost'}
+ size="icon-sm"
+ title="A-Eye — point at an element on the page and send it with your next message"
+ aria-label="Point at an element with A-Eye"
+ onClick={onPick}
+ disabled={!connected || picking}
+ className={cn(focus || picking ? 'text-ember' : undefined)}
+ >
+ <ScanEye className={cn('size-3.5', picking && 'animate-pulse')} />
+ </Button>
<Button
variant="ghost"
size="icon-sm"
diff --git a/src/lib/bridge/aeye.ts b/src/lib/bridge/aeye.ts
index 751531c..4c25d74 100644
--- a/src/lib/bridge/aeye.ts
+++ b/src/lib/bridge/aeye.ts
@@ -1,5 +1,7 @@
import { browser } from 'wxt/browser';
-import type { FocusedElement } from '@/lib/actions/protocol';
+import { pickElement } from '@/lib/actions/page/pick-element';
+import { BRIDGE_CHANNEL, failure, type ActionResult, type FocusedElement } from '@/lib/actions/protocol';
+import { readPhone } from './phone';
import { pickInTab, type PickShot } from './pick';
export type PickOutcome = { focus: FocusedElement } | { cancelled: true } | { error: string };
@@ -13,15 +15,29 @@ interface PickedElement {
shot?: PickShot;
}
-/** The panel drives A-Eye through the same action the agent calls, so both see one picker. */
+/**
+ * The panel drives A-Eye through the same action the agent calls, so both see one picker. On the
+ * phone tab the pick happens on the phone, which only the background can reach.
+ */
export async function pickFocus(): Promise<PickOutcome> {
const [tab] = await browser.tabs.query({ active: true, currentWindow: true });
if (tab?.id == null) return { error: 'No active tab to point at' };
+ if ((await readPhone())?.mirrorTabId === tab.id) return focusOf(await pickOnPhone());
return pickFocusIn({ id: tab.id, windowId: tab.windowId });
}
+async function pickOnPhone(): Promise<ActionResult> {
+ const answer = (await browser.runtime
+ .sendMessage({ channel: BRIDGE_CHANNEL, op: 'invoke', action: pickElement.name, input: {} })
+ .catch(() => null)) as ActionResult | null;
+ return answer ?? failure('BRIDGE_ERROR', 'The extension did not answer.');
+}
+
export async function pickFocusIn(tab: { id: number; windowId?: number }): Promise<PickOutcome> {
- const result = await pickInTab(tab);
+ return focusOf(await pickInTab(tab));
+}
+
+function focusOf(result: ActionResult): PickOutcome {
if (!result.ok) {
return result.error.code === 'PICK_CANCELLED' ? { cancelled: true } : { error: result.error.message };
}
diff --git a/src/lib/bridge/phone-backend.test.ts b/src/lib/bridge/phone-backend.test.ts
index a6a4bf0..98b315c 100644
--- a/src/lib/bridge/phone-backend.test.ts
+++ b/src/lib/bridge/phone-backend.test.ts
@@ -19,7 +19,8 @@ vi.mock('./socket', () => ({
phone.sent.push({ method, params });
if (method === 'Page.getFrameTree') return success({ frameTree: { frame: { id: 'TOP' } } });
if (method === 'Page.createIsolatedWorld') return success({ executionContextId: ++phone.contexts });
- if (method === 'Page.getLayoutMetrics') return success({ cssVisualViewport: phone.viewport });
+ if (method === 'Page.getLayoutMetrics') return success({ cssVisualViewport: phone.viewport, cssLayoutViewport: { pageX: 0, pageY: 300, clientWidth: 411, clientHeight: 675 } });
+ if (method === 'Page.captureScreenshot') return success({ data: 'SHOT' });
if (method === 'Runtime.evaluate' && String(params?.expression).startsWith('__browsenticPhone.dispatch')) {
if (phone.goneOnce) {
phone.goneOnce = false;
@@ -37,7 +38,7 @@ vi.mock('./socket', () => ({
vi.mock('./phone-mirror', () => ({ attachedSession: async () => 'session-1' }));
-const { pageSideOnPhone, shotClip } = await import('./phone-backend');
+const { pageSideOnPhone, pickClip, shotClip } = await import('./phone-backend');
const SESSION: PhoneSession = {
serial: 'emulator-5554',
@@ -126,3 +127,29 @@ describe('a screenshot on the phone', () => {
expect([shot.width, shot.height]).toEqual([Math.round(411 * (1600 / 5400)), 1600]);
});
});
+
+describe('A-Eye on the phone', () => {
+ const capture = { region: { x: 20, y: 100, w: 200, h: 40 }, viewport: { w: 411, h: 675 }, dpr: 2.625 };
+ const layout = { pageX: 0, pageY: 300, clientWidth: 411, clientHeight: 675 };
+
+ test('the photograph is the element’s box, padded and kept on screen, in document coordinates at device resolution', () => {
+ expect(pickClip({ x: 20, y: 100 }, capture, layout)).toEqual({
+ clip: { x: 4, y: 384, width: 232, height: 72, scale: 1 },
+ width: Math.round(232 * 2.625),
+ height: Math.round(72 * 2.625),
+ });
+ });
+
+ test('a big element is shrunk to the longest side the desktop pick allows', () => {
+ const planned = pickClip({ x: 0, y: 0 }, { ...capture, region: { x: 0, y: 0, w: 411, h: 675 } }, layout)!;
+ expect(Math.max(planned.width, planned.height)).toBe(1200);
+ });
+
+ test('the pick runs in the page with a wait the relay allows, and comes back with the element’s photograph', async () => {
+ phone.answer = () => ({ ok: true, data: { element: { tag: 'button', selector: '#buy' }, content: 'Buy', truncated: false, url: 'https://shop.example/', title: 'Shop', capture } });
+ const picked = await pageSideOnPhone('page.pickElement', { timeoutMs: 300_000 }, SESSION);
+ expect(picked).toMatchObject({ ok: true, data: { element: { selector: '#buy' }, shot: { dataUrl: 'data:image/jpeg;base64,SHOT' } } });
+ expect(String(phone.sent.find(({ params }) => String(params?.expression).includes('page.pickElement'))?.params?.expression)).toContain('"timeoutMs":170000');
+ expect(phone.sent.find(({ method }) => method === 'Page.captureScreenshot')?.params?.clip).toEqual({ x: 4, y: 384, width: 232, height: 72, scale: 1 });
+ });
+});
diff --git a/src/lib/bridge/phone-backend.ts b/src/lib/bridge/phone-backend.ts
index 3a4f8d8..2e328f8 100644
--- a/src/lib/bridge/phone-backend.ts
+++ b/src/lib/bridge/phone-backend.ts
@@ -1,5 +1,6 @@
import { browser } from 'wxt/browser';
import { z } from 'zod';
+import { PICK_DEFAULT_TIMEOUT_MS } from '@/lib/actions/page/pick-element';
import { pressKey } from '@/lib/actions/page/press-key';
import { scrollTo } from '@/lib/actions/page/scroll-to';
import { switchFrame } from '@/lib/actions/page/switch-frame';
@@ -21,6 +22,12 @@ const LOAD_POLL_MS = 150;
const SETTLE_MS = 300;
const SCROLL_PAGE = 0.85;
const MAX_SHOT_SCREENS = 8;
+const PICK_PADDING_CSS_PX = 16;
+const MAX_PICK_SHOT_SIDE = 1200;
+const PICK_SHOT_QUALITY = 85;
+const PAINT_SETTLE_MS = 120;
+/** Under the relay's 180 s cap on one command, so the page's own timeout answers first. */
+const MAX_PICK_WAIT_MS = 170_000;
const GONE = 'The phone disconnected. Reconnect it and switch Android on again.';
type CdpResult = ActionResult<Record<string, unknown>>;
@@ -283,6 +290,58 @@ async function screenshotOn(here: Here, input: unknown): Promise<ActionResult> {
return success({ format: plan.format, width, height, dataUrl: `data:image/${plan.format};base64,${String(shot.data.data)}`, ...(truncated ? { truncated } : {}) });
}
+interface PickCapture {
+ region: { x: number; y: number; w: number; h: number };
+ viewport: { w: number; h: number };
+ dpr: number;
+}
+
+interface LayoutViewport {
+ pageX: number;
+ pageY: number;
+ clientWidth: number;
+ clientHeight: number;
+}
+
+/**
+ * The picked element's photograph: its box padded and kept on screen, in the document coordinates a
+ * phone screenshot clips in, at device resolution up to the longest side the desktop pick allows.
+ */
+export function pickClip(corner: Point, capture: PickCapture, layout: LayoutViewport): { clip: { x: number; y: number; width: number; height: number; scale: number }; width: number; height: number } | null {
+ const x = Math.max(0, corner.x - PICK_PADDING_CSS_PX);
+ const y = Math.max(0, corner.y - PICK_PADDING_CSS_PX);
+ const w = Math.min(layout.clientWidth, corner.x + capture.region.w + PICK_PADDING_CSS_PX) - x;
+ const h = Math.min(layout.clientHeight, corner.y + capture.region.h + PICK_PADDING_CSS_PX) - y;
+ if (w < 1 || h < 1) return null;
+ const dpr = capture.dpr || 1;
+ const perCssPx = Math.min(dpr, MAX_PICK_SHOT_SIDE / Math.max(w, h));
+ return {
+ clip: { x: x + layout.pageX, y: y + layout.pageY, width: w, height: h, scale: perCssPx / dpr },
+ width: Math.max(1, Math.round(w * perCssPx)),
+ height: Math.max(1, Math.round(h * perCssPx)),
+ };
+}
+
+/** A-Eye on the phone: the pick's own lens in the page, where a tap picks what is under it, then the element photographed before anything moves it. */
+async function pickOn(here: Here, input: Record<string, unknown>): Promise<ActionResult> {
+ const timeoutMs = Math.min(Number(input.timeoutMs) || PICK_DEFAULT_TIMEOUT_MS, MAX_PICK_WAIT_MS);
+ const picked = await inPage(here, 'page.pickElement', { ...input, timeoutMs });
+ if (!picked.ok) return picked;
+ const { capture, ...data } = picked.data as { capture?: PickCapture } & Record<string, unknown>;
+ const shot = capture ? await pickShot(here, capture).catch(() => null) : null;
+ return success(shot ? { ...data, shot } : data);
+}
+
+async function pickShot(here: Here, capture: PickCapture): Promise<{ dataUrl: string; width: number; height: number } | null> {
+ const metrics = await here.cdp('Page.getLayoutMetrics');
+ if (!metrics.ok) return null;
+ const planned = pickClip(await inTopDocument(here, { x: capture.region.x, y: capture.region.y }), capture, metrics.data.cssLayoutViewport as LayoutViewport);
+ if (!planned) return null;
+ await pause(PAINT_SETTLE_MS);
+ const shot = await here.cdp('Page.captureScreenshot', { format: 'jpeg', quality: PICK_SHOT_QUALITY, clip: planned.clip, fromSurface: true }, 30_000);
+ return shot.ok ? { dataUrl: `data:image/jpeg;base64,${String(shot.data.data)}`, width: planned.width, height: planned.height } : null;
+}
+
async function switchFrameOn(here: Here, input: unknown): Promise<ActionResult> {
const parsed = switchFrame.input.safeParse(input ?? {});
if (!parsed.success) return failure('INVALID_INPUT', z.prettifyError(parsed.error));
@@ -322,6 +381,7 @@ const TOOLS: Record<string, PhoneTool> = {
'page.scrollTo': scrollOn,
'page.screenshot': screenshotOn,
'page.switchFrame': switchFrameOn,
+ 'page.pickElement': pickOn,
'page.searchSite': (here, input, action) => thenLoaded(here, action, input),
'page.submitForm': (here, input, action) => thenLoaded(here, action, input),
};
diff --git a/src/lib/phone/features.ts b/src/lib/phone/features.ts
index 7cf08bf..6381cfe 100644
--- a/src/lib/phone/features.ts
+++ b/src/lib/phone/features.ts
@@ -27,6 +27,7 @@ export const PHONE_TOOLS: ReadonlySet<string> = new Set([
'page.switchTab',
'page.closeTab',
'page.switchFrame',
+ 'page.pickElement',
'page.injectCode',
'page.runCode',
'page.startTimer',