An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.
TypeScript agentic-browseragentic-browsingagy-cliaiai-agentai-browser-extensionai-toolsbrave-extensionbrowser-agentbrowser-automationbrowser-extensionbrowser-usebrowserskillchrome-extensionclaude-codecodexfirefox-addonmcp-servernanobrowserweb-automation
feat(android): A-Eye on the phone: the panel's button and the agent's pick bring the lens up on the phone, a tap on the phone tab's picture picks the element without pressing it, and the pick carries the element's photograph from the phone
Shahriarfeb129cbf481
diff --git a/src/daemon/agent/service.test.ts b/src/daemon/agent/service.test.tsindex acb1e02..fe1cd4b 100644--- a/src/daemon/agent/service.test.ts+++ b/src/daemon/agent/service.test.ts@@ -52,10 +52,10 @@ describe('what a phone run is offered', () => {});});- test('exactly the phone’s tools with Live tool on, and no reserved ones', async () => {+ test('exactly the phone’s tools with Live tool on, and the tool that shows an A-Eye pick', async () => {const offer = await offerTo('p', { sessionId: 's1', phone: PHONE, liveTools: true });expect(names.filter((name) => !offer!.withheld.includes(name)).sort()).toEqual([...PHONE_TOOLS].sort());- expect(offer!.reserved).toEqual([]);+ expect(offer!.reserved).toEqual([FOCUS_SHOT_ACTION]);});test('the page-code tools only with Live tool on, the same rule as a desktop run', async () => {diff --git a/src/daemon/agent/service.ts b/src/daemon/agent/service.tsindex 2ec2aa7..c74fd68 100644--- a/src/daemon/agent/service.ts+++ b/src/daemon/agent/service.ts@@ -206,7 +206,7 @@ export class AgentSession {const codeWithheld = codeListed ? [] : [INJECT_ACTION, RUN_CODE_ACTION];if (run.phone) {const offPhone = this.deps.actionNames().filter((name) => !phoneOffers(name));- return { withheld: [...new Set([...offPhone, ...codeWithheld])], reserved: [] };+ return { withheld: [...new Set([...offPhone, ...codeWithheld])], reserved: [FOCUS_SHOT_ACTION] };}return {withheld: codeWithheld,diff --git a/src/extension/components/composer.tsx b/src/extension/components/composer.tsxindex b54c93d..c16d480 100644--- a/src/extension/components/composer.tsx+++ b/src/extension/components/composer.tsx@@ -252,19 +252,17 @@ export function Composer({/><div className="flex items-center gap-0.5 px-1.5 pb-1.5">- {!onPhone && (- <Button- variant={focus || picking ? 'subtle' : 'ghost'}- size="icon-sm"- title="A-Eye — point at an element on the page and send it with your next message"- aria-label="Point at an element with A-Eye"- onClick={onPick}- disabled={!connected || picking}- className={cn(focus || picking ? 'text-ember' : undefined)}- >- <ScanEye className={cn('size-3.5', picking && 'animate-pulse')} />- </Button>- )}+ <Button+ variant={focus || picking ? 'subtle' : 'ghost'}+ size="icon-sm"+ title="A-Eye — point at an element on the page and send it with your next message"+ aria-label="Point at an element with A-Eye"+ onClick={onPick}+ disabled={!connected || picking}+ className={cn(focus || picking ? 'text-ember' : undefined)}+ >+ <ScanEye className={cn('size-3.5', picking && 'animate-pulse')} />+ </Button><Buttonvariant="ghost"size="icon-sm"diff --git a/src/lib/bridge/aeye.ts b/src/lib/bridge/aeye.tsindex 751531c..4c25d74 100644--- a/src/lib/bridge/aeye.ts+++ b/src/lib/bridge/aeye.ts@@ -1,5 +1,7 @@import { browser } from 'wxt/browser';-import type { FocusedElement } from '@/lib/actions/protocol';+import { pickElement } from '@/lib/actions/page/pick-element';+import { BRIDGE_CHANNEL, failure, type ActionResult, type FocusedElement } from '@/lib/actions/protocol';+import { readPhone } from './phone';import { pickInTab, type PickShot } from './pick';export type PickOutcome = { focus: FocusedElement } | { cancelled: true } | { error: string };@@ -13,15 +15,29 @@ interface PickedElement {shot?: PickShot;}-/** The panel drives A-Eye through the same action the agent calls, so both see one picker. */+/**+ * The panel drives A-Eye through the same action the agent calls, so both see one picker. On the+ * phone tab the pick happens on the phone, which only the background can reach.+ */export async function pickFocus(): Promise<PickOutcome> {const [tab] = await browser.tabs.query({ active: true, currentWindow: true });if (tab?.id == null) return { error: 'No active tab to point at' };+ if ((await readPhone())?.mirrorTabId === tab.id) return focusOf(await pickOnPhone());return pickFocusIn({ id: tab.id, windowId: tab.windowId });}+async function pickOnPhone(): Promise<ActionResult> {+ const answer = (await browser.runtime+ .sendMessage({ channel: BRIDGE_CHANNEL, op: 'invoke', action: pickElement.name, input: {} })+ .catch(() => null)) as ActionResult | null;+ return answer ?? failure('BRIDGE_ERROR', 'The extension did not answer.');+}+export async function pickFocusIn(tab: { id: number; windowId?: number }): Promise<PickOutcome> {- const result = await pickInTab(tab);+ return focusOf(await pickInTab(tab));+}++function focusOf(result: ActionResult): PickOutcome {if (!result.ok) {return result.error.code === 'PICK_CANCELLED' ? { cancelled: true } : { error: result.error.message };}diff --git a/src/lib/bridge/phone-backend.test.ts b/src/lib/bridge/phone-backend.test.tsindex a6a4bf0..98b315c 100644--- a/src/lib/bridge/phone-backend.test.ts+++ b/src/lib/bridge/phone-backend.test.ts@@ -19,7 +19,8 @@ vi.mock('./socket', () => ({phone.sent.push({ method, params });if (method === 'Page.getFrameTree') return success({ frameTree: { frame: { id: 'TOP' } } });if (method === 'Page.createIsolatedWorld') return success({ executionContextId: ++phone.contexts });- if (method === 'Page.getLayoutMetrics') return success({ cssVisualViewport: phone.viewport });+ if (method === 'Page.getLayoutMetrics') return success({ cssVisualViewport: phone.viewport, cssLayoutViewport: { pageX: 0, pageY: 300, clientWidth: 411, clientHeight: 675 } });+ if (method === 'Page.captureScreenshot') return success({ data: 'SHOT' });if (method === 'Runtime.evaluate' && String(params?.expression).startsWith('__browsenticPhone.dispatch')) {if (phone.goneOnce) {phone.goneOnce = false;@@ -37,7 +38,7 @@ vi.mock('./socket', () => ({vi.mock('./phone-mirror', () => ({ attachedSession: async () => 'session-1' }));-const { pageSideOnPhone, shotClip } = await import('./phone-backend');+const { pageSideOnPhone, pickClip, shotClip } = await import('./phone-backend');const SESSION: PhoneSession = {serial: 'emulator-5554',@@ -126,3 +127,29 @@ describe('a screenshot on the phone', () => {expect([shot.width, shot.height]).toEqual([Math.round(411 * (1600 / 5400)), 1600]);});});++describe('A-Eye on the phone', () => {+ const capture = { region: { x: 20, y: 100, w: 200, h: 40 }, viewport: { w: 411, h: 675 }, dpr: 2.625 };+ const layout = { pageX: 0, pageY: 300, clientWidth: 411, clientHeight: 675 };++ test('the photograph is the element’s box, padded and kept on screen, in document coordinates at device resolution', () => {+ expect(pickClip({ x: 20, y: 100 }, capture, layout)).toEqual({+ clip: { x: 4, y: 384, width: 232, height: 72, scale: 1 },+ width: Math.round(232 * 2.625),+ height: Math.round(72 * 2.625),+ });+ });++ test('a big element is shrunk to the longest side the desktop pick allows', () => {+ const planned = pickClip({ x: 0, y: 0 }, { ...capture, region: { x: 0, y: 0, w: 411, h: 675 } }, layout)!;+ expect(Math.max(planned.width, planned.height)).toBe(1200);+ });++ test('the pick runs in the page with a wait the relay allows, and comes back with the element’s photograph', async () => {+ phone.answer = () => ({ ok: true, data: { element: { tag: 'button', selector: '#buy' }, content: 'Buy', truncated: false, url: 'https://shop.example/', title: 'Shop', capture } });+ const picked = await pageSideOnPhone('page.pickElement', { timeoutMs: 300_000 }, SESSION);+ expect(picked).toMatchObject({ ok: true, data: { element: { selector: '#buy' }, shot: { dataUrl: 'data:image/jpeg;base64,SHOT' } } });+ expect(String(phone.sent.find(({ params }) => String(params?.expression).includes('page.pickElement'))?.params?.expression)).toContain('"timeoutMs":170000');+ expect(phone.sent.find(({ method }) => method === 'Page.captureScreenshot')?.params?.clip).toEqual({ x: 4, y: 384, width: 232, height: 72, scale: 1 });+ });+});diff --git a/src/lib/bridge/phone-backend.ts b/src/lib/bridge/phone-backend.tsindex 3a4f8d8..2e328f8 100644--- a/src/lib/bridge/phone-backend.ts+++ b/src/lib/bridge/phone-backend.ts@@ -1,5 +1,6 @@import { browser } from 'wxt/browser';import { z } from 'zod';+import { PICK_DEFAULT_TIMEOUT_MS } from '@/lib/actions/page/pick-element';import { pressKey } from '@/lib/actions/page/press-key';import { scrollTo } from '@/lib/actions/page/scroll-to';import { switchFrame } from '@/lib/actions/page/switch-frame';@@ -21,6 +22,12 @@ const LOAD_POLL_MS = 150;const SETTLE_MS = 300;const SCROLL_PAGE = 0.85;const MAX_SHOT_SCREENS = 8;+const PICK_PADDING_CSS_PX = 16;+const MAX_PICK_SHOT_SIDE = 1200;+const PICK_SHOT_QUALITY = 85;+const PAINT_SETTLE_MS = 120;+/** Under the relay's 180 s cap on one command, so the page's own timeout answers first. */+const MAX_PICK_WAIT_MS = 170_000;const GONE = 'The phone disconnected. Reconnect it and switch Android on again.';type CdpResult = ActionResult<Record<string, unknown>>;@@ -283,6 +290,58 @@ async function screenshotOn(here: Here, input: unknown): Promise<ActionResult> {return success({ format: plan.format, width, height, dataUrl: `data:image/${plan.format};base64,${String(shot.data.data)}`, ...(truncated ? { truncated } : {}) });}+interface PickCapture {+ region: { x: number; y: number; w: number; h: number };+ viewport: { w: number; h: number };+ dpr: number;+}++interface LayoutViewport {+ pageX: number;+ pageY: number;+ clientWidth: number;+ clientHeight: number;+}++/**+ * The picked element's photograph: its box padded and kept on screen, in the document coordinates a+ * phone screenshot clips in, at device resolution up to the longest side the desktop pick allows.+ */+export function pickClip(corner: Point, capture: PickCapture, layout: LayoutViewport): { clip: { x: number; y: number; width: number; height: number; scale: number }; width: number; height: number } | null {+ const x = Math.max(0, corner.x - PICK_PADDING_CSS_PX);+ const y = Math.max(0, corner.y - PICK_PADDING_CSS_PX);+ const w = Math.min(layout.clientWidth, corner.x + capture.region.w + PICK_PADDING_CSS_PX) - x;+ const h = Math.min(layout.clientHeight, corner.y + capture.region.h + PICK_PADDING_CSS_PX) - y;+ if (w < 1 || h < 1) return null;+ const dpr = capture.dpr || 1;+ const perCssPx = Math.min(dpr, MAX_PICK_SHOT_SIDE / Math.max(w, h));+ return {+ clip: { x: x + layout.pageX, y: y + layout.pageY, width: w, height: h, scale: perCssPx / dpr },+ width: Math.max(1, Math.round(w * perCssPx)),+ height: Math.max(1, Math.round(h * perCssPx)),+ };+}++/** A-Eye on the phone: the pick's own lens in the page, where a tap picks what is under it, then the element photographed before anything moves it. */+async function pickOn(here: Here, input: Record<string, unknown>): Promise<ActionResult> {+ const timeoutMs = Math.min(Number(input.timeoutMs) || PICK_DEFAULT_TIMEOUT_MS, MAX_PICK_WAIT_MS);+ const picked = await inPage(here, 'page.pickElement', { ...input, timeoutMs });+ if (!picked.ok) return picked;+ const { capture, ...data } = picked.data as { capture?: PickCapture } & Record<string, unknown>;+ const shot = capture ? await pickShot(here, capture).catch(() => null) : null;+ return success(shot ? { ...data, shot } : data);+}++async function pickShot(here: Here, capture: PickCapture): Promise<{ dataUrl: string; width: number; height: number } | null> {+ const metrics = await here.cdp('Page.getLayoutMetrics');+ if (!metrics.ok) return null;+ const planned = pickClip(await inTopDocument(here, { x: capture.region.x, y: capture.region.y }), capture, metrics.data.cssLayoutViewport as LayoutViewport);+ if (!planned) return null;+ await pause(PAINT_SETTLE_MS);+ const shot = await here.cdp('Page.captureScreenshot', { format: 'jpeg', quality: PICK_SHOT_QUALITY, clip: planned.clip, fromSurface: true }, 30_000);+ return shot.ok ? { dataUrl: `data:image/jpeg;base64,${String(shot.data.data)}`, width: planned.width, height: planned.height } : null;+}+async function switchFrameOn(here: Here, input: unknown): Promise<ActionResult> {const parsed = switchFrame.input.safeParse(input ?? {});if (!parsed.success) return failure('INVALID_INPUT', z.prettifyError(parsed.error));@@ -322,6 +381,7 @@ const TOOLS: Record<string, PhoneTool> = {'page.scrollTo': scrollOn,'page.screenshot': screenshotOn,'page.switchFrame': switchFrameOn,+ 'page.pickElement': pickOn,'page.searchSite': (here, input, action) => thenLoaded(here, action, input),'page.submitForm': (here, input, action) => thenLoaded(here, action, input),};diff --git a/src/lib/phone/features.ts b/src/lib/phone/features.tsindex 7cf08bf..6381cfe 100644--- a/src/lib/phone/features.ts+++ b/src/lib/phone/features.ts@@ -27,6 +27,7 @@ export const PHONE_TOOLS: ReadonlySet<string> = new Set(['page.switchTab','page.closeTab','page.switchFrame',+ 'page.pickElement','page.injectCode','page.runCode','page.startTimer',