An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.
TypeScript agentic-browseragentic-browsingagy-cliaiai-agentai-browser-extensionai-toolsbrave-extensionbrowser-agentbrowser-automationbrowser-extensionbrowser-usebrowserskillchrome-extensionclaude-codecodexfirefox-addonmcp-servernanobrowserweb-automation
feat(android): the phone tab shows Chrome on the phone live in a phone frame, following its front tab; taps, scrolls and typing on it reach the phone, with back, forward, reload, an address bar and the phone's tabs, and the stream stops while the tab is hidden
Shahriar2ab90deee0ed
diff --git a/src/daemon/android/adb.test.ts b/src/daemon/android/adb.test.tsindex cd7df09..7dd27ff 100644--- a/src/daemon/android/adb.test.ts+++ b/src/daemon/android/adb.test.ts@@ -86,11 +86,11 @@ describe('finding adb', () => {});test.skipIf(process.platform === 'win32')('start-server is judged by its exit, not by output a forked server could hold open', async () => {- const forks = fake('adb-forks', `require('node:child_process').spawn(process.execPath, ['-e', 'setTimeout(() => {}, 3000)'], { stdio: 'inherit', detached: true }).unref();`);+ const forks = fake('adb-forks', `require('node:child_process').spawn(process.execPath, ['-e', 'setTimeout(() => {}, 20000)'], { stdio: 'inherit', detached: true }).unref();`);const started = Date.now();expect(await startServer(forks)).toBe(true);- expect(Date.now() - started).toBeLessThan(2_000);+ expect(Date.now() - started).toBeLessThan(10_000);expect(await startServer(fake('adb-refuses', 'process.exit(1);'))).toBe(false);expect(await startServer(join(dir, 'missing'))).toBe(false);- });+ }, 30_000);});diff --git a/src/daemon/test/protocol-window.test.ts b/src/daemon/test/protocol-window.test.tsindex 2ed4ba9..3d3fb07 100644--- a/src/daemon/test/protocol-window.test.ts+++ b/src/daemon/test/protocol-window.test.ts@@ -84,6 +84,13 @@ describe('the protocol window', () => {});});+async function eventually(done: () => boolean): Promise<void> {+ for (let tries = 0; !done(); tries++) {+ if (tries > 400) throw new Error('timed out waiting');+ await new Promise((resolve) => setTimeout(resolve, 5));+ }+}+const ANDROID_FRAMES: ReadonlySet<SocketFrame['t']> = new Set(['androidInfo', 'phoneOpened', 'cdpResult', 'cdpEvent', 'phoneClosed']);describe('Android across the protocol window', () => {@@ -118,8 +125,7 @@ describe('Android across the protocol window', () => {browser.close();await browser.closed;- await new Promise((resolve) => setTimeout(resolve, 20));- expect(android.watchers).toBe(0);+ await eventually(() => android.watchers === 0);});test('a protocol-23 extension can ask the Bridge to open Chrome on the phone', async () => {@@ -152,7 +158,7 @@ describe('Android across the protocol window', () => {browser.close();await browser.closed;- await new Promise((resolve) => setTimeout(resolve, 20));+ await eventually(() => android.released.length > 0);expect(android.released).toEqual([store.installId]);});@@ -176,8 +182,7 @@ describe('Android across the protocol window', () => {expect(heard).toHaveLength(2);await bridge.android({ watch: true });bridge.close();- await new Promise((resolve) => setTimeout(resolve, 20));- expect(android.watchers).toBe(0);+ await eventually(() => android.watchers === 0);});test('a peek answers only while someone watches', async () => {diff --git a/src/extension/entrypoints/background.ts b/src/extension/entrypoints/background.tsindex 7dfc0e8..b574c22 100644--- a/src/extension/entrypoints/background.ts+++ b/src/extension/entrypoints/background.ts@@ -23,6 +23,7 @@ import { endHandsFree } from '@/lib/bridge/panel-view';import { serveLaunchers } from '@/lib/bridge/launchers';import { servePreferences } from '@/lib/bridge/preferences';import { endPhone, openChromeOnPhone, servePhone, startPhone } from '@/lib/bridge/phone';+import { servePhoneMirror } from '@/lib/bridge/phone-mirror';import { isAgentKind } from '@/lib/agents/catalog';import { isPreferenceChange } from '@/lib/settings/preferences';import {@@ -234,6 +235,7 @@ export default defineBackground(() => {serveHandsFree();servePreferences();servePhone();+ servePhoneMirror();void syncRail();onWelcome(() => {diff --git a/src/extension/entrypoints/phone/App.tsx b/src/extension/entrypoints/phone/App.tsxindex bc4f604..4f26fae 100644--- a/src/extension/entrypoints/phone/App.tsx+++ b/src/extension/entrypoints/phone/App.tsx@@ -1,10 +1,14 @@-import { useEffect, useState } from 'react';+import { useEffect, useRef, useState, type ReactNode } from 'react';import { browser } from 'wxt/browser';-import { Smartphone, Unplug } from 'lucide-react';-import { Wordmark } from '@/extension/components/brand';+import { ArrowLeft, ArrowRight, Layers, Loader2, Plus, RotateCw, Smartphone, Unplug, X } from 'lucide-react';import { Button } from '@/extension/components/ui/button';+import { ANDROID_PROTOCOL } from '@/lib/actions/protocol';import { askPhone, usePhoneSession } from '@/lib/bridge/phone-client';import type { PhoneEndReason, PhoneSession } from '@/lib/bridge/phone';+import { useDaemonState } from '@/lib/bridge/use-daemon-state';+import { addressOf } from '@/lib/phone/mirror';+import { cn } from '@/lib/utils';+import { useMirror, type Mirror } from './use-mirror';const ENDED: Record<PhoneEndReason, string> = {unplugged: 'The phone was disconnected.',@@ -14,72 +18,280 @@ const ENDED: Record<PhoneEndReason, string> = {closed: 'Android was switched off.',};+const CHROME_RETRY_MS = 3_000;+/** The status strip, the address bar and the frame's padding, above and around the screen. */+const FRAME_CHROME_PX = { height: 104, width: 24 };+const CHROME_RETRY_FOR_MS = 15_000;+async function closeThisTab() {const tab = await browser.tabs.getCurrent();if (tab?.id != null) await browser.tabs.remove(tab.id);}-function describe(session: PhoneSession | null): { title: string; body: string } {- const phone = session?.model ?? 'your phone';- if (!session) return { title: 'Android is off', body: 'Switch on Android in the Browsentic side panel to show your phone here.' };- if (session.ended) return { title: `${phone} is not connected`, body: ENDED[session.ended.reason] };- if (session.waitingForChrome) return { title: 'Open Chrome on your phone', body: `Browsentic needs Chrome running on ${phone} to show and drive it.` };- return { title: `${phone} is connected`, body: `${session.targets.length} tab${session.targets.length === 1 ? '' : 's'} open in Chrome on the phone.` };+/** The largest screen of this aspect that fits the space the frame has, so the phone's picture is never stretched. */+function useFit(aspect: number) {+ const area = useRef<HTMLDivElement | null>(null);+ const [space, setSpace] = useState({ width: 0, height: 0 });+ useEffect(() => {+ const element = area.current;+ if (!element) return;+ const observer = new ResizeObserver(([entry]) => setSpace({ width: entry.contentRect.width, height: entry.contentRect.height }));+ observer.observe(element);+ return () => observer.disconnect();+ }, []);+ const height = Math.max(0, Math.min(space.height - FRAME_CHROME_PX.height, (space.width - FRAME_CHROME_PX.width) / aspect));+ return { area, screen: { width: Math.round(height * aspect), height: Math.round(height) } };+}++function useVisible(): boolean {+ const [visible, setVisible] = useState(document.visibilityState === 'visible');+ useEffect(() => {+ const listener = () => setVisible(document.visibilityState === 'visible');+ document.addEventListener('visibilitychange', listener);+ return () => document.removeEventListener('visibilitychange', listener);+ }, []);+ return visible;}export default function App() {const session = usePhoneSession();- const [busy, setBusy] = useState(false);- const [error, setError] = useState<string>();- const { title, body } = describe(session);+ const daemon = useDaemonState();+ const visible = useVisible();+ const live = !!session && !session.ended && !session.waitingForChrome;+ const mirror = useMirror(live && visible);+ const device = daemon?.android?.devices.find((each) => each.serial === session?.serial);+ const asleep = device?.problem?.code === 'SCREEN_OFF';+ const tooOld = !!daemon?.connected && (daemon.protocolVersion ?? 0) < ANDROID_PROTOCOL;useEffect(() => {- document.title = `${session?.model ?? 'Android'} · Browsentic`;+ document.title = `Android · ${session?.model ?? 'Browsentic'}`;}, [session?.model]);- const act = async (request: Parameters<typeof askPhone>[0]) => {+ const wasAsleep = useRef(asleep);+ const { send } = mirror;+ useEffect(() => {+ if (wasAsleep.current && !asleep && live) send({ op: 'restart' });+ wasAsleep.current = asleep;+ }, [asleep, live, send]);++ const overlay = tooOld ? (+ <Notice icon={<Unplug className="size-6" />} title="Update Browsentic Bridge" body="This Bridge predates Android. Run “browsentic update”, then switch Android on again." />+ ) : !session ? (+ <Notice icon={<Smartphone className="size-6" />} title="Android is off" body="Switch on Android in the Browsentic side panel to show your phone here." />+ ) : session.ended ? (+ <Ended session={session} />+ ) : session.waitingForChrome ? (+ <WaitingForChrome session={session} />+ ) : !mirror.image ? (+ <Notice icon={<Loader2 className="size-6 animate-spin" />} title="Connecting" body={`Showing ${session.model ?? 'your phone'}…`} />+ ) : asleep ? (+ <Notice icon={<Smartphone className="size-6" />} title="Wake your phone to see it here" body="Chrome stops drawing while the screen is off." />+ ) : null;++ return (+ <div className="dot-grid flex h-screen flex-col items-center gap-3 overflow-hidden p-4">+ <Header session={session} />+ <PhoneFrame session={session} device={device} mirror={mirror} live={live} overlay={overlay} />+ {mirror.error && <p className="rounded-lg border border-destructive/40 bg-surface px-3 py-1.5 text-xs text-ink">{mirror.error}</p>}+ </div>+ );+}++function Header({ session }: { session: PhoneSession | null }) {+ const daemon = useDaemonState();+ const phones = daemon?.android?.devices.filter((device) => device.state === 'ready') ?? [];+ if (phones.length < 2 || !session) return null;+ return (+ <select+ aria-label="Phone"+ value={session.serial}+ onChange={(event) => void askPhone({ op: 'phoneStart', serial: event.target.value, windowId: session.windowId })}+ className="rounded-full border border-line bg-surface px-3 py-1 text-xs text-ink"+ >+ {phones.map((phone) => (+ <option key={phone.serial} value={phone.serial}>+ {phone.model ?? phone.serial}+ </option>+ ))}+ </select>+ );+}++function PhoneFrame({+ session,+ device,+ mirror,+ live,+ overlay,+}: {+ session: PhoneSession | null;+ device?: { transport: 'usb' | 'wifi'; screen?: { width: number; height: number } };+ mirror: Mirror;+ live: boolean;+ overlay: ReactNode;+}) {+ const [tabsOpen, setTabsOpen] = useState(false);+ const aspect = mirror.image ? mirror.image.width / mirror.image.height : device?.screen ? device.screen.width / device.screen.height : 9 / 19.5;+ const { area, screen } = useFit(aspect);+ return (+ <div ref={area} className="flex min-h-0 w-full flex-1 items-start justify-center">+ <div className="flex flex-col rounded-[2.25rem] border border-line-strong bg-surface p-2.5 shadow-2xl" style={{ width: screen.width + FRAME_CHROME_PX.width }}>+ <div className="flex items-center justify-between px-3 pt-1 pb-2 text-[11px] text-ink-dim">+ <span className="truncate font-medium text-ink">{session?.model ?? 'Android'}</span>+ <span className="flex items-center gap-1.5">+ <span className={cn('glow-dot size-1.5 rounded-full', live ? 'bg-lime' : 'bg-amber')} />+ {device?.transport === 'wifi' ? 'Wi-Fi' : 'USB'}+ </span>+ </div>+ <AddressBar mirror={mirror} live={live} tabs={session?.targets.length ?? 0} onTabs={() => setTabsOpen((open) => !open)} />+ <div className="relative overflow-hidden rounded-[1.5rem] bg-ground" style={screen}>+ <canvas+ ref={mirror.canvas}+ tabIndex={0}+ aria-label="The phone's screen"+ className={cn('block h-full w-full touch-none outline-none', (!mirror.image || overlay) && 'opacity-30')}+ />+ {overlay && <div className="absolute inset-0 flex items-center justify-center p-6">{overlay}</div>}+ {tabsOpen && session && <TabList session={session} mirror={mirror} onClose={() => setTabsOpen(false)} />}+ </div>+ </div>+ </div>+ );+}++function AddressBar({ mirror, live, tabs, onTabs }: { mirror: Mirror; live: boolean; tabs: number; onTabs(): void }) {+ const [typed, setTyped] = useState<string>();+ const shown = typed ?? mirror.page?.url ?? '';+ return (+ <div className="mb-2 flex items-center gap-1 rounded-full bg-ground px-1.5 py-1">+ <Button variant="ghost" size="icon-sm" title="Back" aria-label="Back" disabled={!live || !mirror.page?.canGoBack} onClick={() => mirror.send({ op: 'back' })}>+ <ArrowLeft className="size-3.5" />+ </Button>+ <Button variant="ghost" size="icon-sm" title="Forward" aria-label="Forward" disabled={!live || !mirror.page?.canGoForward} onClick={() => mirror.send({ op: 'forward' })}>+ <ArrowRight className="size-3.5" />+ </Button>+ <Button variant="ghost" size="icon-sm" title="Reload" aria-label="Reload" disabled={!live} onClick={() => mirror.send({ op: 'reload' })}>+ <RotateCw className="size-3.5" />+ </Button>+ <form+ className="min-w-0 flex-1"+ onSubmit={(event) => {+ event.preventDefault();+ if (shown.trim()) mirror.send({ op: 'go', url: addressOf(shown) });+ setTyped(undefined);+ }}+ >+ <input+ aria-label="Address"+ title={mirror.page?.title}+ value={shown}+ disabled={!live}+ onChange={(event) => setTyped(event.target.value)}+ onBlur={() => setTyped(undefined)}+ className="w-full truncate bg-transparent px-2 text-xs text-ink outline-none placeholder:text-ink-faint"+ placeholder="Search or type a URL"+ />+ </form>+ <Button variant="ghost" size="sm" title="Tabs on the phone" aria-label="Tabs on the phone" disabled={!live} onClick={onTabs}>+ <Layers className="size-3.5" />+ {tabs}+ </Button>+ </div>+ );+}++function TabList({ session, mirror, onClose }: { session: PhoneSession; mirror: Mirror; onClose(): void }) {+ return (+ <div className="absolute inset-0 flex flex-col bg-ground/95 p-3 backdrop-blur-sm">+ <div className="mb-2 flex items-center justify-between">+ <span className="text-xs font-medium text-ink">Tabs on the phone</span>+ <span className="flex gap-1">+ <Button variant="ghost" size="sm" onClick={() => (mirror.send({ op: 'newTab' }), onClose())}>+ <Plus className="size-3.5" />+ New tab+ </Button>+ <Button variant="ghost" size="icon-sm" aria-label="Close the list" onClick={onClose}>+ <X className="size-3.5" />+ </Button>+ </span>+ </div>+ <ul className="min-h-0 flex-1 space-y-1 overflow-y-auto">+ {session.targets.map((target) => (+ <li+ key={target.targetId}+ className={cn('flex items-center gap-2 rounded-lg px-2.5 py-2', target.targetId === session.activeTargetId ? 'bg-brand/12' : 'hover:bg-surface')}+ >+ <button type="button" className="min-w-0 flex-1 text-left" onClick={() => (mirror.send({ op: 'switchTab', targetId: target.targetId }), onClose())}>+ <span className="block truncate text-xs text-ink">{target.title || target.url}</span>+ <span className="block truncate text-[11px] text-ink-faint">{target.url}</span>+ </button>+ <Button variant="ghost" size="icon-sm" aria-label={`Close ${target.title || target.url}`} onClick={() => mirror.send({ op: 'closeTab', targetId: target.targetId })}>+ <X className="size-3.5" />+ </Button>+ </li>+ ))}+ </ul>+ </div>+ );+}++function Notice({ icon, title, body, children }: { icon: ReactNode; title: string; body: string; children?: ReactNode }) {+ return (+ <div className="panel-card flex max-w-xs flex-col items-center gap-3 p-6 text-center">+ <span className="flex size-12 items-center justify-center rounded-full border border-brand/50 bg-brand/10 text-brand">{icon}</span>+ <h1 className="text-sm font-medium">{title}</h1>+ <p className="text-xs leading-relaxed text-ink-dim">{body}</p>+ {children}+ </div>+ );+}++function WaitingForChrome({ session }: { session: PhoneSession }) {+ const [busy, setBusy] = useState(false);+ const [error, setError] = useState<string>();+ const open = async () => {setBusy(true);setError(undefined);- const result = await askPhone(request);+ const started = Date.now();+ let result = await askPhone({ op: 'phoneOpenChrome' });+ while (result.ok && result.data.waitingForChrome && Date.now() - started < CHROME_RETRY_FOR_MS) {+ await new Promise((resolve) => setTimeout(resolve, CHROME_RETRY_MS));+ result = await askPhone({ op: 'phoneOpenChrome' });+ }setBusy(false);if (!result.ok) setError(result.error.message);};+ return (+ <Notice icon={<Smartphone className="size-6" />} title="Open Chrome on your phone" body={`Browsentic needs Chrome running on ${session.model ?? 'your phone'} to show and drive it.`}>+ <Button size="sm" disabled={busy} onClick={() => void open()}>+ {busy && <Loader2 className="size-3.5 animate-spin" />}+ Open Chrome on the phone+ </Button>+ {error && <p className="text-xs text-destructive">{error}</p>}+ </Notice>+ );+}+function Ended({ session }: { session: PhoneSession }) {+ const [busy, setBusy] = useState(false);+ const [error, setError] = useState<string>();+ const reconnect = async () => {+ setBusy(true);+ const result = await askPhone({ op: 'phoneStart', serial: session.serial, windowId: session.windowId });+ setBusy(false);+ setError(result.ok ? undefined : result.error.message);+ };return (- <div className="dot-grid flex min-h-screen items-center justify-center p-6">- <main className="panel-card flex w-full max-w-sm flex-col items-center gap-4 p-8 text-center">- <Wordmark />- <span className="glow-brand flex size-16 items-center justify-center rounded-full border border-brand/50 bg-brand/10 text-brand">- {session?.ended ? <Unplug className="size-7" /> : <Smartphone className="size-7" />}- </span>- <h1 className="text-base font-medium">{title}</h1>- <p className="text-xs leading-relaxed text-ink-dim">{body}</p>- {session && !session.ended && session.targets.length > 0 && (- <ul className="w-full space-y-1 text-left text-xs text-ink-dim">- {session.targets.map((target) => (- <li key={target.targetId} className="truncate">- {target.title || target.url}- </li>- ))}- </ul>- )}- {session && !session.ended && session.waitingForChrome && (- <Button size="sm" disabled={busy} onClick={() => void act({ op: 'phoneOpenChrome' })}>- Open Chrome on the phone- </Button>- )}- {session?.ended && (- <div className="flex gap-2">- <Button size="sm" disabled={busy} onClick={() => void act({ op: 'phoneStart', serial: session.serial, windowId: session.windowId })}>- Reconnect- </Button>- <Button size="sm" variant="outline" onClick={() => void closeThisTab()}>- Close tab- </Button>- </div>- )}- {error && <p className="text-xs text-destructive">{error}</p>}- </main>- </div>+ <Notice icon={<Unplug className="size-6" />} title={`${session.model ?? 'The phone'} is not connected`} body={ENDED[session.ended!.reason]}>+ <div className="flex gap-2">+ <Button size="sm" disabled={busy} onClick={() => void reconnect()}>+ Reconnect+ </Button>+ <Button size="sm" variant="outline" onClick={() => void closeThisTab()}>+ Close tab+ </Button>+ </div>+ {error && <p className="text-xs text-destructive">{error}</p>}+ </Notice>);}diff --git a/src/extension/entrypoints/phone/use-mirror.ts b/src/extension/entrypoints/phone/use-mirror.tsnew file mode 100644index 0000000..294c0bd--- /dev/null+++ b/src/extension/entrypoints/phone/use-mirror.ts@@ -0,0 +1,134 @@+import { useCallback, useEffect, useRef, useState } from 'react';+import { browser, type Browser } from 'wxt/browser';+import { PHONE_PORT, isMirrorKey, touchPoint, type FrameMetadata, type MirrorCommand, type MirrorMessage, type PageState, type TouchType } from '@/lib/phone/mirror';++const RECONNECT_MS = 500;+const ERROR_SHOWN_MS = 5_000;++export interface Mirror {+ canvas: React.RefObject<HTMLCanvasElement | null>;+ page: PageState | null;+ image: { width: number; height: number } | null;+ error: string | undefined;+ send(command: MirrorCommand): void;+}++const decode = (data: string) => new Blob([Uint8Array.from(atob(data), (char) => char.charCodeAt(0))], { type: 'image/jpeg' });++/** The page's half of the mirror: frames in and drawn, then acked; touches, wheel and keys out as the phone takes them. */+export function useMirror(watching: boolean): Mirror {+ const canvas = useRef<HTMLCanvasElement | null>(null);+ const port = useRef<Browser.runtime.Port | null>(null);+ const metadata = useRef<FrameMetadata | null>(null);+ const watchingNow = useRef(watching);+ const [page, setPage] = useState<PageState | null>(null);+ const [image, setImage] = useState<{ width: number; height: number } | null>(null);+ const [error, setError] = useState<string>();++ const send = useCallback((command: MirrorCommand) => port.current?.postMessage(command), []);++ useEffect(() => {+ let live = true;+ let retry: ReturnType<typeof setTimeout> | undefined;+ const draw = async (message: Extract<MirrorMessage, { kind: 'frame' }>, from: Browser.runtime.Port) => {+ const bitmap = await createImageBitmap(decode(message.data));+ const target = canvas.current;+ if (target) {+ if (target.width !== bitmap.width || target.height !== bitmap.height) {+ target.width = bitmap.width;+ target.height = bitmap.height;+ setImage({ width: bitmap.width, height: bitmap.height });+ }+ target.getContext('2d')?.drawImage(bitmap, 0, 0);+ }+ bitmap.close();+ metadata.current = message.metadata;+ from.postMessage({ op: 'ack', frame: message.frame } satisfies MirrorCommand);+ };+ const connect = () => {+ const opened = browser.runtime.connect({ name: PHONE_PORT });+ port.current = opened;+ opened.onMessage.addListener((raw) => {+ const message = raw as MirrorMessage;+ if (message.kind === 'frame') void draw(message, opened);+ else if (message.kind === 'page') setPage(message);+ else setError(message.message);+ });+ opened.onDisconnect.addListener(() => {+ if (port.current === opened) port.current = null;+ if (live) retry = setTimeout(connect, RECONNECT_MS);+ });+ if (watchingNow.current) opened.postMessage({ op: 'watch' } satisfies MirrorCommand);+ };+ connect();+ return () => {+ live = false;+ clearTimeout(retry);+ port.current?.disconnect();+ };+ }, []);++ useEffect(() => {+ watchingNow.current = watching;+ send({ op: watching ? 'watch' : 'unwatch' });+ }, [watching, send]);++ useEffect(() => {+ if (!error) return;+ const timer = setTimeout(() => setError(undefined), ERROR_SHOWN_MS);+ return () => clearTimeout(timer);+ }, [error]);++ useEffect(() => {+ const target = canvas.current;+ if (!target) return;+ const at = (event: { clientX: number; clientY: number }) => {+ const frame = metadata.current;+ if (!frame || !target.width) return null;+ const box = target.getBoundingClientRect();+ return touchPoint({ x: (event.clientX - box.left) / box.width, y: (event.clientY - box.top) / box.height }, frame, target);+ };+ const touch = (type: TouchType) => (event: PointerEvent) => {+ if (type !== 'touchStart' && !target.hasPointerCapture(event.pointerId)) return;+ if (type === 'touchStart') {+ target.setPointerCapture(event.pointerId);+ target.focus();+ }+ const point = at(event);+ if (point) send({ op: 'touch', type, ...point });+ event.preventDefault();+ };+ const wheel = (event: WheelEvent) => {+ const point = at(event);+ if (point) send({ op: 'wheel', ...point, deltaX: event.deltaX, deltaY: event.deltaY });+ event.preventDefault();+ };+ const key = (event: KeyboardEvent) => {+ if (event.metaKey || event.ctrlKey || event.altKey) return;+ if (isMirrorKey(event.key)) send({ op: 'key', key: event.key });+ else if (event.key.length === 1) send({ op: 'text', text: event.key });+ else return;+ event.preventDefault();+ };+ const paste = (event: ClipboardEvent) => {+ const text = event.clipboardData?.getData('text/plain');+ if (text) send({ op: 'text', text });+ event.preventDefault();+ };+ const listeners = [+ ['pointerdown', touch('touchStart')],+ ['pointermove', touch('touchMove')],+ ['pointerup', touch('touchEnd')],+ ['pointercancel', touch('touchCancel')],+ ['wheel', wheel],+ ['keydown', key],+ ['paste', paste],+ ] as const;+ for (const [type, listener] of listeners) target.addEventListener(type, listener as EventListener, { passive: false });+ return () => {+ for (const [type, listener] of listeners) target.removeEventListener(type, listener as EventListener);+ };+ }, [send]);++ return { canvas, page, image, error, send };+}diff --git a/src/lib/bridge/phone-mirror.test.ts b/src/lib/bridge/phone-mirror.test.tsnew file mode 100644index 0000000..7a9fe4e--- /dev/null+++ b/src/lib/bridge/phone-mirror.test.ts@@ -0,0 +1,150 @@+import { beforeEach, describe, expect, test, vi } from 'vitest';+import { fakeBrowser } from 'wxt/testing';+import { success } from '@/lib/actions/protocol';+import { BLOCKED_SITES_KEY } from '@/lib/settings/blocked-sites';+import type { CdpEvent } from './socket';+import type { MirrorMessage } from '@/lib/phone/mirror';++const SERIAL = 'emulator-5554';+const FRONT = 'A'.repeat(32);+const BACK = 'B'.repeat(32);++const socket = vi.hoisted(() => ({ sent: [] as { method: string; params?: Record<string, unknown>; sessionId?: string }[], events: [] as ((event: CdpEvent) => void)[] }));++vi.mock('./socket', () => ({+ DAEMON_STATE_KEY: 'browsentic/daemon',+ sendCdp: async (_serial: string, method: string, params?: Record<string, unknown>, sessionId?: string) => {+ socket.sent.push({ method, params, sessionId });+ if (method === 'Target.attachToTarget') return success({ sessionId: `session-${params?.targetId}` });+ if (method === 'Runtime.evaluate') return success({ result: { value: sessionId === `session-${FRONT}` ? 'visible' : 'hidden' } });+ if (method === 'Page.getNavigationHistory') return success({ currentIndex: 1, entries: [{ id: 1, url: 'https://a.example/', title: 'A' }, { id: 2, url: 'https://b.example/', title: 'B' }] });+ return success({});+ },+ onCdpEvent: (listener: (event: CdpEvent) => void) => socket.events.push(listener),+ onPhoneClosed: () => undefined,+ onDaemonClosed: () => undefined,+ closePhone: () => undefined,+ openPhone: async () => success({}),+ launchPhoneChrome: async () => success({}),+}));++type Phone = typeof import('./phone');+let readPhone: Phone['readPhone'];+const connectListeners: ((port: unknown) => void)[] = [];++/** The page's end of the port, as the background sees it. */+function connectPage() {+ const received: MirrorMessage[] = [];+ const messageListeners: ((message: unknown) => void)[] = [];+ const port = {+ name: 'browsentic/phone',+ postMessage: (message: MirrorMessage) => received.push(message),+ onMessage: { addListener: (listener: (message: unknown) => void) => messageListeners.push(listener) },+ onDisconnect: { addListener: () => undefined },+ };+ connectListeners.forEach((listener) => listener(port));+ return { received, say: (message: unknown) => messageListeners.forEach((listener) => listener(message)) };+}++const methods = () => socket.sent.map(({ method }) => method);+const settle = () => new Promise((resolve) => setTimeout(resolve, 20));+const emit = (event: Omit<CdpEvent, 't' | 'serial'>) => socket.events.forEach((listener) => listener({ t: 'cdpEvent', serial: SERIAL, ...event }));++beforeEach(async () => {+ fakeBrowser.reset();+ socket.sent.length = 0;+ socket.events.length = 0;+ connectListeners.length = 0;+ Object.assign(fakeBrowser.runtime, { onConnect: { addListener: (listener: (port: unknown) => void) => connectListeners.push(listener) } });+ vi.resetModules();+ ({ readPhone } = await import('./phone'));+ (await import('./phone-mirror')).servePhoneMirror();+ await fakeBrowser.storage.session.set({+ 'browsentic/phone': {+ serial: SERIAL,+ mirrorTabId: 9,+ windowId: 1,+ openedAt: 0,+ transport: 'usb',+ targets: [+ { targetId: BACK, url: 'https://b.example/', title: 'B' },+ { targetId: FRONT, url: 'https://a.example/', title: 'A' },+ ],+ },+ });+});++describe('the mirror in the background', () => {+ test('watching follows the tab in front, not the first listed, and streams it', async () => {+ const page = connectPage();+ page.say({ op: 'watch' });+ await settle();+ expect((await readPhone())?.activeTargetId).toBe(FRONT);+ expect(socket.sent.find(({ method }) => method === 'Page.startScreencast')).toEqual({+ method: 'Page.startScreencast',+ params: { format: 'jpeg', quality: 60, maxWidth: 720, maxHeight: 1600, everyNthFrame: 1 },+ sessionId: `session-${FRONT}`,+ });+ expect(page.received.find((message) => message.kind === 'page')).toMatchObject({ targetId: FRONT, url: 'https://b.example/', canGoBack: true, canGoForward: false });+ });++ test('a frame goes to the page, and the page’s ack goes to the phone', async () => {+ const page = connectPage();+ page.say({ op: 'watch' });+ await settle();+ emit({ method: 'Page.screencastFrame', params: { data: 'JPEG', sessionId: 7, metadata: { deviceWidth: 411, pageScaleFactor: 1 } }, sessionId: `session-${FRONT}` });+ expect(page.received.find((message) => message.kind === 'frame')).toEqual({ kind: 'frame', data: 'JPEG', frame: 7, metadata: { deviceWidth: 411, pageScaleFactor: 1 } });+ page.say({ op: 'ack', frame: 7 });+ await settle();+ expect(socket.sent.at(-1)).toEqual({ method: 'Page.screencastFrameAck', params: { sessionId: 7 }, sessionId: `session-${FRONT}` });+ });++ test('a finger’s moves while one is on its way collapse into the latest, and the lift still follows', async () => {+ const page = connectPage();+ page.say({ op: 'watch' });+ await settle();+ socket.sent.length = 0;+ page.say({ op: 'touch', type: 'touchStart', x: 10, y: 10 });+ for (let step = 1; step <= 5; step++) page.say({ op: 'touch', type: 'touchMove', x: 10, y: 10 + step * 10 });+ page.say({ op: 'touch', type: 'touchEnd', x: 10, y: 60 });+ await settle();+ expect(socket.sent.map(({ params }) => [params?.type, params?.touchPoints])).toEqual([+ ['touchStart', [{ x: 10, y: 10 }]],+ ['touchMove', [{ x: 10, y: 60 }]],+ ['touchEnd', []],+ ]);+ });++ test('a blocked site typed in the address bar is refused before the phone hears of it', async () => {+ await fakeBrowser.storage.local.set({ [BLOCKED_SITES_KEY]: ['bank.example'] });+ const page = connectPage();+ page.say({ op: 'watch' });+ await settle();+ page.say({ op: 'go', url: 'https://bank.example/login' });+ await settle();+ expect(methods()).not.toContain('Page.navigate');+ expect(page.received.at(-1)).toMatchObject({ kind: 'error' });+ page.say({ op: 'go', url: 'https://ok.example/' });+ await settle();+ expect(socket.sent.at(-1)).toMatchObject({ method: 'Page.navigate', params: { url: 'https://ok.example/' } });+ });++ test('the tab list follows the phone and leaves out the phantom target', async () => {+ connectPage();+ emit({ method: 'Target.targetCreated', params: { targetInfo: { targetId: '1', type: 'page', url: 'https://a.example/', title: 'A' } } });+ emit({ method: 'Target.targetCreated', params: { targetInfo: { targetId: 'C'.repeat(32), type: 'page', url: 'https://c.example/', title: 'C' } } });+ emit({ method: 'Target.targetCreated', params: { targetInfo: { targetId: 'D'.repeat(32), type: 'service_worker', url: 'https://c.example/sw.js', title: '' } } });+ emit({ method: 'Target.targetDestroyed', params: { targetId: BACK } });+ await settle();+ expect((await readPhone())?.targets.map(({ targetId }) => targetId[0])).toEqual(['A', 'C']);+ });++ test('back steps to the previous history entry', async () => {+ const page = connectPage();+ page.say({ op: 'watch' });+ await settle();+ page.say({ op: 'back' });+ await settle();+ expect(socket.sent.at(-1)).toEqual({ method: 'Page.navigateToHistoryEntry', params: { entryId: 1 }, sessionId: `session-${FRONT}` });+ });+});diff --git a/src/lib/bridge/phone-mirror.ts b/src/lib/bridge/phone-mirror.tsnew file mode 100644index 0000000..117eb14--- /dev/null+++ b/src/lib/bridge/phone-mirror.ts@@ -0,0 +1,315 @@+import { browser, type Browser } from 'wxt/browser';+import { PHONE_PORT, keyEvents, type MirrorCommand, type MirrorMessage, type PageState } from '@/lib/phone/mirror';+import type { PhoneTarget } from '@/lib/phone/types';+import { readPhone, updatePhone, type PhoneSession } from './phone';+import { refusalFor } from './site-guard';+import { onCdpEvent, onDaemonClosed, onPhoneClosed, sendCdp, type CdpEvent } from './socket';++type Port = Browser.runtime.Port;+type InputCommand = Extract<MirrorCommand, { op: 'touch' | 'wheel' | 'text' | 'key' }>;++/** Discovery also reports a phantom `"1"` beside a restored tab, and every command sent to it hangs (spike Q3). */+const REAL_TARGET = /^[0-9A-F]{32}$/i;+const CLOSED_TAB_SETTLE_MS = 300;+const ACTIVATE_TRIES = 4;+const ACTIVATE_SETTLE_MS = 250;++interface Followed {+ serial: string;+ targetId: string;+ sessionId: string;+}++const ports = new Set<Port>();+const watchers = new Set<Port>();+let followed: Followed | null = null;+let streaming = false;+let page: PageState | null = null;+let finding: Promise<void> | null = null;+const inputs: InputCommand[] = [];+let pumping = false;++const broadcast = (message: MirrorMessage) => {+ for (const port of ports) port.postMessage(message);+};++const tellError = (message: string) => broadcast({ kind: 'error', message });++interface TargetInfo {+ targetId?: string;+ type?: string;+ url?: string;+ title?: string;+}++const isPageTarget = (info: TargetInfo): info is TargetInfo & { targetId: string } => info.type === 'page' && REAL_TARGET.test(info.targetId ?? '');++async function liveSession(): Promise<PhoneSession | null> {+ const session = await readPhone();+ return session && !session.ended && !session.waitingForChrome ? session : null;+}++function call(method: string, params?: Record<string, unknown>) {+ const held = followed;+ return held ? sendCdp(held.serial, method, params, held.sessionId) : Promise.resolve(null);+}++async function follow(session: PhoneSession, targetId: string): Promise<void> {+ if (followed?.serial === session.serial && followed.targetId === targetId) return;+ await stopStream();+ if (followed) void sendCdp(followed.serial, 'Target.detachFromTarget', { sessionId: followed.sessionId });+ followed = null;+ const attached = await sendCdp(session.serial, 'Target.attachToTarget', { targetId, flatten: true });+ if (!attached.ok) return tellError(attached.error.message);+ followed = { serial: session.serial, targetId, sessionId: String(attached.data.sessionId) };+ await call('Page.enable');+ await updatePhone((held) => ({ ...held, activeTargetId: targetId }));+ await publishPage();+ if (watchers.size) await startStream();+}++async function ensureFollowed(): Promise<void> {+ const session = await liveSession();+ if (!session) return;+ if (followed && followed.serial !== session.serial) followed = null;+ if (followed) return;+ const known = session.targets.find((target) => target.targetId === session.activeTargetId);+ if (known) return follow(session, known.targetId);+ await findFront();+ if (!followed && session.targets[0]) await follow(session, session.targets[0].targetId);+}++async function startStream(): Promise<void> {+ if (!followed) return;+ const wifi = (await readPhone())?.transport === 'wifi';+ streaming = true;+ const started = await call('Page.startScreencast', { format: 'jpeg', quality: wifi ? 40 : 60, maxWidth: wifi ? 540 : 720, maxHeight: 1600, everyNthFrame: 1 });+ if (started && !started.ok) {+ streaming = false;+ tellError(started.error.message);+ }+}++async function stopStream(): Promise<void> {+ if (!followed || !streaming) return;+ streaming = false;+ await call('Page.stopScreencast');+}++async function publishPage(): Promise<void> {+ const held = followed;+ if (!held) return;+ const history = await call('Page.getNavigationHistory');+ if (!history?.ok || followed !== held) return;+ const entries = (history.data.entries as { id: number; url: string; title: string }[] | undefined) ?? [];+ const current = Number(history.data.currentIndex ?? entries.length - 1);+ const entry = entries[current];+ page = {+ targetId: held.targetId,+ url: entry?.url ?? '',+ title: entry?.title ?? '',+ canGoBack: current > 0,+ canGoForward: current < entries.length - 1,+ };+ broadcast({ kind: 'page', ...page });+}++/** A tab switch made on the phone fires no Target event; the tab whose document is visible is the one in front. */+function findFront(): Promise<void> {+ finding ??= (async () => {+ const session = await liveSession();+ if (!session) return;+ for (const target of session.targets) {+ const visible = await isVisible(session.serial, target.targetId);+ if (visible) return follow(session, target.targetId);+ }+ })().finally(() => (finding = null));+ return finding;+}++async function isVisible(serial: string, targetId: string): Promise<boolean> {+ const own = followed?.targetId === targetId ? followed.sessionId : undefined;+ const attached = own ? null : await sendCdp(serial, 'Target.attachToTarget', { targetId, flatten: true });+ const sessionId = own ?? (attached?.ok ? String(attached.data.sessionId) : null);+ if (!sessionId) return false;+ const state = await sendCdp(serial, 'Runtime.evaluate', { expression: 'document.visibilityState', returnByValue: true }, sessionId);+ if (!own) void sendCdp(serial, 'Target.detachFromTarget', { sessionId });+ return state.ok && (state.data.result as { value?: unknown } | undefined)?.value === 'visible';+}++async function trackTargets(event: CdpEvent): Promise<void> {+ const info = (event.params.targetInfo ?? {}) as TargetInfo;+ if (event.method === 'Target.detachedFromTarget' && event.params.sessionId === followed?.sessionId) {+ followed = null;+ streaming = false;+ if (watchers.size) void ensureFollowed();+ return;+ }+ if (event.method === 'Target.targetDestroyed') {+ const targetId = String(event.params.targetId);+ await updatePhone((held) => ({ ...held, targets: held.targets.filter((target) => target.targetId !== targetId) }));+ if (followed?.targetId === targetId) {+ followed = null;+ streaming = false;+ setTimeout(() => void findFront(), CLOSED_TAB_SETTLE_MS);+ }+ return;+ }+ if ((event.method !== 'Target.targetCreated' && event.method !== 'Target.targetInfoChanged') || !isPageTarget(info)) return;+ const target: PhoneTarget = { targetId: info.targetId, url: info.url ?? '', title: info.title ?? '' };+ await updatePhone((held) => ({+ ...held,+ targets: held.targets.some((each) => each.targetId === target.targetId)+ ? held.targets.map((each) => (each.targetId === target.targetId ? target : each))+ : [...held.targets, target],+ }));+ if (page && followed?.targetId === target.targetId) {+ page = { ...page, url: target.url, title: target.title };+ broadcast({ kind: 'page', ...page });+ }+}++function onEvent(event: CdpEvent): void {+ if (event.method.startsWith('Target.') && !event.sessionId) return void trackTargets(event);+ if (!followed || event.sessionId !== followed.sessionId) return;+ if (event.method === 'Page.screencastFrame') {+ const frame = Number(event.params.sessionId);+ if (!watchers.size) return void call('Page.screencastFrameAck', { sessionId: frame });+ broadcast({ kind: 'frame', data: String(event.params.data), metadata: event.params.metadata as never, frame });+ return;+ }+ if (event.method === 'Page.screencastVisibilityChanged' && event.params.visible === false) return void findFront();+ if (event.method === 'Page.frameNavigated' && !(event.params.frame as { parentId?: string } | undefined)?.parentId) return void publishPage();+ if (event.method === 'Page.navigatedWithinDocument') void publishPage();+}++function queueInput(command: InputCommand): void {+ const last = inputs.at(-1);+ if (last?.op === 'touch' && command.op === 'touch' && last.type === 'touchMove' && command.type === 'touchMove') inputs[inputs.length - 1] = command;+ else if (last?.op === 'wheel' && command.op === 'wheel') {+ inputs[inputs.length - 1] = { ...command, deltaX: last.deltaX + command.deltaX, deltaY: last.deltaY + command.deltaY };+ } else inputs.push(command);+ void pump();+}++async function pump(): Promise<void> {+ if (pumping) return;+ pumping = true;+ try {+ for (let next = inputs.shift(); next; next = inputs.shift()) await dispatchInput(next);+ } finally {+ pumping = false;+ }+}++async function dispatchInput(command: InputCommand): Promise<void> {+ if (command.op === 'touch') {+ const lifted = command.type === 'touchEnd' || command.type === 'touchCancel';+ await call('Input.dispatchTouchEvent', { type: command.type, touchPoints: lifted ? [] : [{ x: command.x, y: command.y }] });+ } else if (command.op === 'wheel') {+ await call('Input.dispatchMouseEvent', { type: 'mouseWheel', x: command.x, y: command.y, deltaX: command.deltaX, deltaY: command.deltaY });+ } else if (command.op === 'text') {+ await call('Input.insertText', { text: command.text });+ } else {+ for (const event of keyEvents(command.key)) await call('Input.dispatchKeyEvent', event);+ }+}++/** Chrome on Android can ignore an activation that lands just after a tab opened, so it is asked again until the tab is in front. */+async function bringToFront(serial: string, targetId: string): Promise<void> {+ for (let attempt = 0; attempt < ACTIVATE_TRIES; attempt++) {+ await sendCdp(serial, 'Target.activateTarget', { targetId });+ await new Promise((resolve) => setTimeout(resolve, ACTIVATE_SETTLE_MS));+ if (await isVisible(serial, targetId)) return;+ }+}++async function goInHistory(step: -1 | 1): Promise<void> {+ const history = await call('Page.getNavigationHistory');+ if (!history?.ok) return;+ const entries = (history.data.entries as { id: number }[] | undefined) ?? [];+ const entry = entries[Number(history.data.currentIndex) + step];+ if (entry) await call('Page.navigateToHistoryEntry', { entryId: entry.id });+}++async function handle(port: Port, command: MirrorCommand): Promise<void> {+ switch (command.op) {+ case 'watch':+ watchers.add(port);+ if (!followed) return ensureFollowed();+ if (!streaming) return startStream();+ if (page) port.postMessage({ kind: 'page', ...page });+ return;+ case 'unwatch':+ watchers.delete(port);+ if (!watchers.size) await stopStream();+ return;+ case 'restart':+ await stopStream();+ if (watchers.size) await startStream();+ return;+ case 'ack':+ await call('Page.screencastFrameAck', { sessionId: command.frame });+ return;+ case 'touch':+ case 'wheel':+ case 'text':+ case 'key':+ return queueInput(command);+ case 'go': {+ const refused = await refusalFor(command.url);+ if (refused) return tellError(refused.ok ? '' : refused.error.message);+ const navigated = await call('Page.navigate', { url: command.url });+ if (navigated && !navigated.ok) tellError(navigated.error.message);+ return;+ }+ case 'back':+ return goInHistory(-1);+ case 'forward':+ return goInHistory(1);+ case 'reload':+ await call('Page.reload');+ return;+ case 'switchTab': {+ const session = await liveSession();+ if (!session) return;+ await bringToFront(session.serial, command.targetId);+ return follow(session, command.targetId);+ }+ case 'newTab': {+ const session = await liveSession();+ if (!session) return;+ const created = await sendCdp(session.serial, 'Target.createTarget', { url: 'about:blank' });+ if (!created.ok) return tellError(created.error.message);+ return follow(session, String(created.data.targetId));+ }+ case 'closeTab': {+ const session = await liveSession();+ if (session) await sendCdp(session.serial, 'Target.closeTarget', { targetId: command.targetId });+ return;+ }+ }+}++function forget(): void {+ followed = null;+ streaming = false;+ page = null;+ inputs.length = 0;+}++export function servePhoneMirror(): void {+ browser.runtime.onConnect.addListener((port) => {+ if (port.name !== PHONE_PORT) return;+ ports.add(port);+ if (page) port.postMessage({ kind: 'page', ...page });+ port.onMessage.addListener((message) => void handle(port, message as MirrorCommand));+ port.onDisconnect.addListener(() => {+ ports.delete(port);+ if (watchers.delete(port) && !watchers.size) void stopStream();+ });+ });+ onCdpEvent(onEvent);+ onPhoneClosed(forget);+ onDaemonClosed(forget);+}diff --git a/src/lib/bridge/phone.test.ts b/src/lib/bridge/phone.test.tsindex 41f35cd..6caedf7 100644--- a/src/lib/bridge/phone.test.ts+++ b/src/lib/bridge/phone.test.ts@@ -53,7 +53,7 @@ describe('the phone session', () => {const started = await startPhone(SERIAL);expect(started.ok).toBe(true);const session = await readPhone();- expect(session).toMatchObject({ serial: SERIAL, model: 'Pixel 8', targets: OPENED.targets, activeTargetId: OPENED.targets[0].targetId, waitingForChrome: false });+ expect(session).toMatchObject({ serial: SERIAL, model: 'Pixel 8', transport: 'usb', targets: OPENED.targets, waitingForChrome: false });const mirror = await fakeBrowser.tabs.get(session!.mirrorTabId);expect(mirror.url).toMatch(/phone\.html$/);expect(notify).not.toHaveBeenCalled();diff --git a/src/lib/bridge/phone.ts b/src/lib/bridge/phone.tsindex 9858229..02498b9 100644--- a/src/lib/bridge/phone.ts+++ b/src/lib/bridge/phone.ts@@ -1,6 +1,6 @@import { browser } from 'wxt/browser';import { failure, success, type ActionResult } from '@/lib/actions/protocol';-import type { PhoneClosedReason, PhoneOpened, PhoneTarget } from '@/lib/phone/types';+import type { AndroidDevice, PhoneClosedReason, PhoneOpened, PhoneTarget } from '@/lib/phone/types';import { DAEMON_STATE_KEY, closePhone, launchPhoneChrome, onDaemonClosed, onPhoneClosed, openPhone, type DaemonState } from './socket';export const PHONE_KEY = 'browsentic/phone';@@ -16,10 +16,12 @@ export type PhoneEndReason = PhoneClosedReason | 'bridge-gone';export interface PhoneSession {serial: string;model?: string;+ transport?: AndroidDevice['transport'];mirrorTabId: number;windowId: number;openedAt: number;targets: PhoneTarget[];+ /** The tab the mirror follows: the one in front on the phone, found once the mirror starts. */activeTargetId?: string;/** Chrome was not open on the phone; the mirror waits, and the notification offers to open it. */waitingForChrome?: boolean;@@ -45,24 +47,39 @@ export async function readPhone(): Promise<PhoneSession | null> {const writePhone = (session: PhoneSession | null): Promise<void> =>session ? browser.storage.session.set({ [PHONE_KEY]: session }) : browser.storage.session.remove(PHONE_KEY);+/** A change to the live session, made in turn with every other; nothing happens when there is none. */+export function updatePhone(change: (session: PhoneSession) => PhoneSession): Promise<PhoneSession | null> {+ return locked(async () => {+ const held = await readPhone();+ if (!held || held.ended) return held;+ const next = change(held);+ await writePhone(next);+ return next;+ });+}+const opening = (opened: ActionResult<PhoneOpened>) =>opened.ok- ? { targets: opened.data.targets, activeTargetId: opened.data.targets[0]?.targetId, waitingForChrome: false }+ ? { targets: opened.data.targets, activeTargetId: undefined, waitingForChrome: false }: { targets: [], activeTargetId: undefined, waitingForChrome: true };-async function modelOf(serial: string): Promise<string | undefined> {+async function deviceOf(serial: string): Promise<AndroidDevice | undefined> {const stored = await browser.storage.session.get(DAEMON_STATE_KEY);- return (stored[DAEMON_STATE_KEY] as DaemonState | undefined)?.android?.devices.find((device) => device.serial === serial)?.model;+ return (stored[DAEMON_STATE_KEY] as DaemonState | undefined)?.android?.devices.find((device) => device.serial === serial);}-/** Starts the session and opens its mirror beside the tab the user is on. A second start shows the mirror already open. */+/**+ * Starts the session and opens its mirror beside the tab the user is on. A second start for the same+ * phone shows the mirror already open; one for another phone moves the session, and its tab, to it.+ */export function startPhone(serial: string, windowId?: number): Promise<ActionResult<PhoneSession>> {return locked(async () => {const held = await readPhone();- if (held && !held.ended) {+ if (held && !held.ended && held.serial === serial) {await showTab(held.mirrorTabId, held.windowId);return success(held);}+ if (held && !held.ended) closePhone(held.serial);const opened = await openPhone(serial);if (!opened.ok && opened.error.code !== 'CHROME_NOT_RUNNING') return opened;@@ -79,7 +96,7 @@ export function startPhone(serial: string, windowId?: number): Promise<ActionReswindowId: mirror.windowId ?? windowId ?? browser.windows.WINDOW_ID_CURRENT,openedAt: Date.now(),...opening(opened),- model: (opened.ok ? opened.data.device.model : undefined) ?? (await modelOf(serial)),+ ...(await describedPhone(serial, opened)),};await writePhone(session);if (session.waitingForChrome) await askToOpenChrome(session);@@ -140,6 +157,11 @@ export function servePhone(): void {});}+async function describedPhone(serial: string, opened: ActionResult<PhoneOpened>): Promise<Pick<PhoneSession, 'model' | 'transport'>> {+ const device = opened.ok ? opened.data.device : await deviceOf(serial);+ return { model: device?.model, transport: device?.transport };+}+async function openMirror(windowId?: number) {const [current] = await browser.tabs.query(windowId === undefined ? { active: true, lastFocusedWindow: true } : { active: true, windowId });return browser.tabs.create({diff --git a/src/lib/phone/mirror.test.ts b/src/lib/phone/mirror.test.tsnew file mode 100644index 0000000..bafa323--- /dev/null+++ b/src/lib/phone/mirror.test.ts@@ -0,0 +1,49 @@+import { describe, expect, test } from 'vitest';+import { addressOf, isMirrorKey, keyEvents, touchPoint } from './mirror';++describe('a click on the mirror', () => {+ test('lands where spike Q8 measured it: the frame is the visual viewport, scaled by the zoom', () => {+ const image = { width: 720, height: 1182 };+ expect(touchPoint({ x: 0.5, y: 0.5 }, { deviceWidth: 411, pageScaleFactor: 1 }, image)).toEqual({ x: 205.5, y: (411 * 1182) / 720 / 2 });+ expect(touchPoint({ x: 1, y: 0 }, { deviceWidth: 411, pageScaleFactor: 2 }, image)).toEqual({ x: 205.5, y: 0 });+ });++ test('a frame squeezed by the soft keyboard keeps the x scale and shortens the y range', () => {+ const point = touchPoint({ x: 0.25, y: 1 }, { deviceWidth: 411, pageScaleFactor: 1 }, { width: 720, height: 531 });+ expect(point.x).toBeCloseTo(102.75);+ expect(point.y).toBeCloseTo(303.1, 1);+ });+});++describe('keys on the mirror', () => {+ test('Enter submits as keyDown with its text', () => {+ expect(keyEvents('Enter')).toEqual([+ { type: 'keyDown', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13, text: '\r', unmodifiedText: '\r' },+ { type: 'keyUp', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13 },+ ]);+ });++ test('Backspace and the arrows carry their virtual key code, without which nothing edits', () => {+ expect(keyEvents('Backspace')[0]).toMatchObject({ type: 'rawKeyDown', windowsVirtualKeyCode: 8 });+ expect(keyEvents('ArrowDown')[0]).toMatchObject({ type: 'rawKeyDown', code: 'ArrowDown', windowsVirtualKeyCode: 40 });+ });++ test('only the named keys are keys; everything printable is text', () => {+ expect(isMirrorKey('Tab')).toBe(true);+ expect(isMirrorKey('a')).toBe(false);+ expect(isMirrorKey('F5')).toBe(false);+ });+});++describe('the address bar', () => {+ test.each([+ ['https://example.com/a', 'https://example.com/a'],+ ['example.com', 'https://example.com'],+ ['news.ycombinator.com/item?id=1', 'https://news.ycombinator.com/item?id=1'],+ ['localhost:8080/x', 'https://localhost:8080/x'],+ ['about:blank', 'about:blank'],+ ['cheap flights to lisbon', 'https://www.google.com/search?q=cheap%20flights%20to%20lisbon'],+ ])('%s', (typed, url) => {+ expect(addressOf(typed)).toBe(url);+ });+});diff --git a/src/lib/phone/mirror.ts b/src/lib/phone/mirror.tsnew file mode 100644index 0000000..875fc8c--- /dev/null+++ b/src/lib/phone/mirror.ts@@ -0,0 +1,90 @@+export const PHONE_PORT = 'browsentic/phone';++/** `Page.screencastFrame`'s metadata: the phone's visual viewport when the frame was taken. */+export interface FrameMetadata {+ offsetTop: number;+ pageScaleFactor: number;+ deviceWidth: number;+ deviceHeight: number;+ scrollOffsetX: number;+ scrollOffsetY: number;+}++export type TouchType = 'touchStart' | 'touchMove' | 'touchEnd' | 'touchCancel';++export const MIRROR_KEYS = ['Enter', 'Backspace', 'Delete', 'Tab', 'Escape', 'ArrowLeft', 'ArrowRight', 'ArrowUp', 'ArrowDown', 'Home', 'End'] as const;+export type MirrorKey = (typeof MIRROR_KEYS)[number];++export const isMirrorKey = (key: string): key is MirrorKey => (MIRROR_KEYS as readonly string[]).includes(key);++export type MirrorCommand =+ | { op: 'watch' }+ | { op: 'unwatch' }+ | { op: 'restart' }+ | { op: 'ack'; frame: number }+ | { op: 'touch'; type: TouchType; x: number; y: number }+ | { op: 'wheel'; x: number; y: number; deltaX: number; deltaY: number }+ | { op: 'text'; text: string }+ | { op: 'key'; key: MirrorKey }+ | { op: 'go'; url: string }+ | { op: 'back' }+ | { op: 'forward' }+ | { op: 'reload' }+ | { op: 'switchTab'; targetId: string }+ | { op: 'newTab' }+ | { op: 'closeTab'; targetId: string };++export interface PageState {+ targetId: string;+ url: string;+ title: string;+ canGoBack: boolean;+ canGoForward: boolean;+}++export type MirrorMessage =+ | { kind: 'frame'; data: string; metadata: FrameMetadata; frame: number }+ | ({ kind: 'page' } & PageState)+ | { kind: 'error'; message: string };++/**+ * Where a point on the drawn frame lands on the phone, in the CSS px a touch event takes. The frame+ * is the visual viewport alone, so the phone's own top bar plays no part (spike Q8).+ */+export function touchPoint(+ fraction: { x: number; y: number },+ metadata: Pick<FrameMetadata, 'deviceWidth' | 'pageScaleFactor'>,+ image: { width: number; height: number },+): { x: number; y: number } {+ const width = metadata.deviceWidth / (metadata.pageScaleFactor || 1);+ return { x: fraction.x * width, y: fraction.y * width * (image.height / image.width) };+}++/** Key events that edit only with `windowsVirtualKeyCode`, and an Enter that submits only as `keyDown` with its `text` (spike Q7). */+const KEYS: Record<MirrorKey, { code: string; keyCode: number; text?: string }> = {+ Enter: { code: 'Enter', keyCode: 13, text: '\r' },+ Backspace: { code: 'Backspace', keyCode: 8 },+ Delete: { code: 'Delete', keyCode: 46 },+ Tab: { code: 'Tab', keyCode: 9 },+ Escape: { code: 'Escape', keyCode: 27 },+ ArrowLeft: { code: 'ArrowLeft', keyCode: 37 },+ ArrowRight: { code: 'ArrowRight', keyCode: 39 },+ ArrowUp: { code: 'ArrowUp', keyCode: 38 },+ ArrowDown: { code: 'ArrowDown', keyCode: 40 },+ Home: { code: 'Home', keyCode: 36 },+ End: { code: 'End', keyCode: 35 },+};++export function keyEvents(key: MirrorKey): Record<string, unknown>[] {+ const { code, keyCode, text } = KEYS[key];+ const base = { key, code, windowsVirtualKeyCode: keyCode, nativeVirtualKeyCode: keyCode };+ return [{ type: text ? 'keyDown' : 'rawKeyDown', ...base, ...(text ? { text, unmodifiedText: text } : {}) }, { type: 'keyUp', ...base }];+}++/** What the mirror's address bar was given, as a URL: a bare host gets https://, anything else is searched for. */+export function addressOf(typed: string): string {+ const text = typed.trim();+ if (/^[a-z][a-z0-9+.-]*:\/\//i.test(text) || /^(about|chrome|data):/i.test(text)) return text;+ if (/^[^\s/]+\.[^\s/]{2,}(\/\S*)?$/.test(text) || /^localhost(:\d+)?(\/\S*)?$/.test(text)) return `https://${text}`;+ return `https://www.google.com/search?q=${encodeURIComponent(text)}`;+}