Skip to content

An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.

feat(android): the phone tab shows Chrome on the phone live in a phone frame, following its front tab; taps, scrolls and typing on it reach the phone, with back, forward, reload, an address bar and the phone's tabs, and the stream stops while the tab is hidden

Shahriar2ab90deee0ed
diff --git a/src/daemon/android/adb.test.ts b/src/daemon/android/adb.test.ts
index cd7df09..7dd27ff 100644
--- a/src/daemon/android/adb.test.ts
+++ b/src/daemon/android/adb.test.ts
@@ -86,11 +86,11 @@ describe('finding adb', () => {
});
test.skipIf(process.platform === 'win32')('start-server is judged by its exit, not by output a forked server could hold open', async () => {
- const forks = fake('adb-forks', `require('node:child_process').spawn(process.execPath, ['-e', 'setTimeout(() => {}, 3000)'], { stdio: 'inherit', detached: true }).unref();`);
+ const forks = fake('adb-forks', `require('node:child_process').spawn(process.execPath, ['-e', 'setTimeout(() => {}, 20000)'], { stdio: 'inherit', detached: true }).unref();`);
const started = Date.now();
expect(await startServer(forks)).toBe(true);
- expect(Date.now() - started).toBeLessThan(2_000);
+ expect(Date.now() - started).toBeLessThan(10_000);
expect(await startServer(fake('adb-refuses', 'process.exit(1);'))).toBe(false);
expect(await startServer(join(dir, 'missing'))).toBe(false);
- });
+ }, 30_000);
});
diff --git a/src/daemon/test/protocol-window.test.ts b/src/daemon/test/protocol-window.test.ts
index 2ed4ba9..3d3fb07 100644
--- a/src/daemon/test/protocol-window.test.ts
+++ b/src/daemon/test/protocol-window.test.ts
@@ -84,6 +84,13 @@ describe('the protocol window', () => {
});
});
+async function eventually(done: () => boolean): Promise<void> {
+ for (let tries = 0; !done(); tries++) {
+ if (tries > 400) throw new Error('timed out waiting');
+ await new Promise((resolve) => setTimeout(resolve, 5));
+ }
+}
+
const ANDROID_FRAMES: ReadonlySet<SocketFrame['t']> = new Set(['androidInfo', 'phoneOpened', 'cdpResult', 'cdpEvent', 'phoneClosed']);
describe('Android across the protocol window', () => {
@@ -118,8 +125,7 @@ describe('Android across the protocol window', () => {
browser.close();
await browser.closed;
- await new Promise((resolve) => setTimeout(resolve, 20));
- expect(android.watchers).toBe(0);
+ await eventually(() => android.watchers === 0);
});
test('a protocol-23 extension can ask the Bridge to open Chrome on the phone', async () => {
@@ -152,7 +158,7 @@ describe('Android across the protocol window', () => {
browser.close();
await browser.closed;
- await new Promise((resolve) => setTimeout(resolve, 20));
+ await eventually(() => android.released.length > 0);
expect(android.released).toEqual([store.installId]);
});
@@ -176,8 +182,7 @@ describe('Android across the protocol window', () => {
expect(heard).toHaveLength(2);
await bridge.android({ watch: true });
bridge.close();
- await new Promise((resolve) => setTimeout(resolve, 20));
- expect(android.watchers).toBe(0);
+ await eventually(() => android.watchers === 0);
});
test('a peek answers only while someone watches', async () => {
diff --git a/src/extension/entrypoints/background.ts b/src/extension/entrypoints/background.ts
index 7dfc0e8..b574c22 100644
--- a/src/extension/entrypoints/background.ts
+++ b/src/extension/entrypoints/background.ts
@@ -23,6 +23,7 @@ import { endHandsFree } from '@/lib/bridge/panel-view';
import { serveLaunchers } from '@/lib/bridge/launchers';
import { servePreferences } from '@/lib/bridge/preferences';
import { endPhone, openChromeOnPhone, servePhone, startPhone } from '@/lib/bridge/phone';
+import { servePhoneMirror } from '@/lib/bridge/phone-mirror';
import { isAgentKind } from '@/lib/agents/catalog';
import { isPreferenceChange } from '@/lib/settings/preferences';
import {
@@ -234,6 +235,7 @@ export default defineBackground(() => {
serveHandsFree();
servePreferences();
servePhone();
+ servePhoneMirror();
void syncRail();
onWelcome(() => {
diff --git a/src/extension/entrypoints/phone/App.tsx b/src/extension/entrypoints/phone/App.tsx
index bc4f604..4f26fae 100644
--- a/src/extension/entrypoints/phone/App.tsx
+++ b/src/extension/entrypoints/phone/App.tsx
@@ -1,10 +1,14 @@
-import { useEffect, useState } from 'react';
+import { useEffect, useRef, useState, type ReactNode } from 'react';
import { browser } from 'wxt/browser';
-import { Smartphone, Unplug } from 'lucide-react';
-import { Wordmark } from '@/extension/components/brand';
+import { ArrowLeft, ArrowRight, Layers, Loader2, Plus, RotateCw, Smartphone, Unplug, X } from 'lucide-react';
import { Button } from '@/extension/components/ui/button';
+import { ANDROID_PROTOCOL } from '@/lib/actions/protocol';
import { askPhone, usePhoneSession } from '@/lib/bridge/phone-client';
import type { PhoneEndReason, PhoneSession } from '@/lib/bridge/phone';
+import { useDaemonState } from '@/lib/bridge/use-daemon-state';
+import { addressOf } from '@/lib/phone/mirror';
+import { cn } from '@/lib/utils';
+import { useMirror, type Mirror } from './use-mirror';
const ENDED: Record<PhoneEndReason, string> = {
unplugged: 'The phone was disconnected.',
@@ -14,72 +18,280 @@ const ENDED: Record<PhoneEndReason, string> = {
closed: 'Android was switched off.',
};
+const CHROME_RETRY_MS = 3_000;
+/** The status strip, the address bar and the frame's padding, above and around the screen. */
+const FRAME_CHROME_PX = { height: 104, width: 24 };
+const CHROME_RETRY_FOR_MS = 15_000;
+
async function closeThisTab() {
const tab = await browser.tabs.getCurrent();
if (tab?.id != null) await browser.tabs.remove(tab.id);
}
-function describe(session: PhoneSession | null): { title: string; body: string } {
- const phone = session?.model ?? 'your phone';
- if (!session) return { title: 'Android is off', body: 'Switch on Android in the Browsentic side panel to show your phone here.' };
- if (session.ended) return { title: `${phone} is not connected`, body: ENDED[session.ended.reason] };
- if (session.waitingForChrome) return { title: 'Open Chrome on your phone', body: `Browsentic needs Chrome running on ${phone} to show and drive it.` };
- return { title: `${phone} is connected`, body: `${session.targets.length} tab${session.targets.length === 1 ? '' : 's'} open in Chrome on the phone.` };
+/** The largest screen of this aspect that fits the space the frame has, so the phone's picture is never stretched. */
+function useFit(aspect: number) {
+ const area = useRef<HTMLDivElement | null>(null);
+ const [space, setSpace] = useState({ width: 0, height: 0 });
+ useEffect(() => {
+ const element = area.current;
+ if (!element) return;
+ const observer = new ResizeObserver(([entry]) => setSpace({ width: entry.contentRect.width, height: entry.contentRect.height }));
+ observer.observe(element);
+ return () => observer.disconnect();
+ }, []);
+ const height = Math.max(0, Math.min(space.height - FRAME_CHROME_PX.height, (space.width - FRAME_CHROME_PX.width) / aspect));
+ return { area, screen: { width: Math.round(height * aspect), height: Math.round(height) } };
+}
+
+function useVisible(): boolean {
+ const [visible, setVisible] = useState(document.visibilityState === 'visible');
+ useEffect(() => {
+ const listener = () => setVisible(document.visibilityState === 'visible');
+ document.addEventListener('visibilitychange', listener);
+ return () => document.removeEventListener('visibilitychange', listener);
+ }, []);
+ return visible;
}
export default function App() {
const session = usePhoneSession();
- const [busy, setBusy] = useState(false);
- const [error, setError] = useState<string>();
- const { title, body } = describe(session);
+ const daemon = useDaemonState();
+ const visible = useVisible();
+ const live = !!session && !session.ended && !session.waitingForChrome;
+ const mirror = useMirror(live && visible);
+ const device = daemon?.android?.devices.find((each) => each.serial === session?.serial);
+ const asleep = device?.problem?.code === 'SCREEN_OFF';
+ const tooOld = !!daemon?.connected && (daemon.protocolVersion ?? 0) < ANDROID_PROTOCOL;
useEffect(() => {
- document.title = `${session?.model ?? 'Android'} · Browsentic`;
+ document.title = `Android · ${session?.model ?? 'Browsentic'}`;
}, [session?.model]);
- const act = async (request: Parameters<typeof askPhone>[0]) => {
+ const wasAsleep = useRef(asleep);
+ const { send } = mirror;
+ useEffect(() => {
+ if (wasAsleep.current && !asleep && live) send({ op: 'restart' });
+ wasAsleep.current = asleep;
+ }, [asleep, live, send]);
+
+ const overlay = tooOld ? (
+ <Notice icon={<Unplug className="size-6" />} title="Update Browsentic Bridge" body="This Bridge predates Android. Run “browsentic update”, then switch Android on again." />
+ ) : !session ? (
+ <Notice icon={<Smartphone className="size-6" />} title="Android is off" body="Switch on Android in the Browsentic side panel to show your phone here." />
+ ) : session.ended ? (
+ <Ended session={session} />
+ ) : session.waitingForChrome ? (
+ <WaitingForChrome session={session} />
+ ) : !mirror.image ? (
+ <Notice icon={<Loader2 className="size-6 animate-spin" />} title="Connecting" body={`Showing ${session.model ?? 'your phone'}…`} />
+ ) : asleep ? (
+ <Notice icon={<Smartphone className="size-6" />} title="Wake your phone to see it here" body="Chrome stops drawing while the screen is off." />
+ ) : null;
+
+ return (
+ <div className="dot-grid flex h-screen flex-col items-center gap-3 overflow-hidden p-4">
+ <Header session={session} />
+ <PhoneFrame session={session} device={device} mirror={mirror} live={live} overlay={overlay} />
+ {mirror.error && <p className="rounded-lg border border-destructive/40 bg-surface px-3 py-1.5 text-xs text-ink">{mirror.error}</p>}
+ </div>
+ );
+}
+
+function Header({ session }: { session: PhoneSession | null }) {
+ const daemon = useDaemonState();
+ const phones = daemon?.android?.devices.filter((device) => device.state === 'ready') ?? [];
+ if (phones.length < 2 || !session) return null;
+ return (
+ <select
+ aria-label="Phone"
+ value={session.serial}
+ onChange={(event) => void askPhone({ op: 'phoneStart', serial: event.target.value, windowId: session.windowId })}
+ className="rounded-full border border-line bg-surface px-3 py-1 text-xs text-ink"
+ >
+ {phones.map((phone) => (
+ <option key={phone.serial} value={phone.serial}>
+ {phone.model ?? phone.serial}
+ </option>
+ ))}
+ </select>
+ );
+}
+
+function PhoneFrame({
+ session,
+ device,
+ mirror,
+ live,
+ overlay,
+}: {
+ session: PhoneSession | null;
+ device?: { transport: 'usb' | 'wifi'; screen?: { width: number; height: number } };
+ mirror: Mirror;
+ live: boolean;
+ overlay: ReactNode;
+}) {
+ const [tabsOpen, setTabsOpen] = useState(false);
+ const aspect = mirror.image ? mirror.image.width / mirror.image.height : device?.screen ? device.screen.width / device.screen.height : 9 / 19.5;
+ const { area, screen } = useFit(aspect);
+ return (
+ <div ref={area} className="flex min-h-0 w-full flex-1 items-start justify-center">
+ <div className="flex flex-col rounded-[2.25rem] border border-line-strong bg-surface p-2.5 shadow-2xl" style={{ width: screen.width + FRAME_CHROME_PX.width }}>
+ <div className="flex items-center justify-between px-3 pt-1 pb-2 text-[11px] text-ink-dim">
+ <span className="truncate font-medium text-ink">{session?.model ?? 'Android'}</span>
+ <span className="flex items-center gap-1.5">
+ <span className={cn('glow-dot size-1.5 rounded-full', live ? 'bg-lime' : 'bg-amber')} />
+ {device?.transport === 'wifi' ? 'Wi-Fi' : 'USB'}
+ </span>
+ </div>
+ <AddressBar mirror={mirror} live={live} tabs={session?.targets.length ?? 0} onTabs={() => setTabsOpen((open) => !open)} />
+ <div className="relative overflow-hidden rounded-[1.5rem] bg-ground" style={screen}>
+ <canvas
+ ref={mirror.canvas}
+ tabIndex={0}
+ aria-label="The phone's screen"
+ className={cn('block h-full w-full touch-none outline-none', (!mirror.image || overlay) && 'opacity-30')}
+ />
+ {overlay && <div className="absolute inset-0 flex items-center justify-center p-6">{overlay}</div>}
+ {tabsOpen && session && <TabList session={session} mirror={mirror} onClose={() => setTabsOpen(false)} />}
+ </div>
+ </div>
+ </div>
+ );
+}
+
+function AddressBar({ mirror, live, tabs, onTabs }: { mirror: Mirror; live: boolean; tabs: number; onTabs(): void }) {
+ const [typed, setTyped] = useState<string>();
+ const shown = typed ?? mirror.page?.url ?? '';
+ return (
+ <div className="mb-2 flex items-center gap-1 rounded-full bg-ground px-1.5 py-1">
+ <Button variant="ghost" size="icon-sm" title="Back" aria-label="Back" disabled={!live || !mirror.page?.canGoBack} onClick={() => mirror.send({ op: 'back' })}>
+ <ArrowLeft className="size-3.5" />
+ </Button>
+ <Button variant="ghost" size="icon-sm" title="Forward" aria-label="Forward" disabled={!live || !mirror.page?.canGoForward} onClick={() => mirror.send({ op: 'forward' })}>
+ <ArrowRight className="size-3.5" />
+ </Button>
+ <Button variant="ghost" size="icon-sm" title="Reload" aria-label="Reload" disabled={!live} onClick={() => mirror.send({ op: 'reload' })}>
+ <RotateCw className="size-3.5" />
+ </Button>
+ <form
+ className="min-w-0 flex-1"
+ onSubmit={(event) => {
+ event.preventDefault();
+ if (shown.trim()) mirror.send({ op: 'go', url: addressOf(shown) });
+ setTyped(undefined);
+ }}
+ >
+ <input
+ aria-label="Address"
+ title={mirror.page?.title}
+ value={shown}
+ disabled={!live}
+ onChange={(event) => setTyped(event.target.value)}
+ onBlur={() => setTyped(undefined)}
+ className="w-full truncate bg-transparent px-2 text-xs text-ink outline-none placeholder:text-ink-faint"
+ placeholder="Search or type a URL"
+ />
+ </form>
+ <Button variant="ghost" size="sm" title="Tabs on the phone" aria-label="Tabs on the phone" disabled={!live} onClick={onTabs}>
+ <Layers className="size-3.5" />
+ {tabs}
+ </Button>
+ </div>
+ );
+}
+
+function TabList({ session, mirror, onClose }: { session: PhoneSession; mirror: Mirror; onClose(): void }) {
+ return (
+ <div className="absolute inset-0 flex flex-col bg-ground/95 p-3 backdrop-blur-sm">
+ <div className="mb-2 flex items-center justify-between">
+ <span className="text-xs font-medium text-ink">Tabs on the phone</span>
+ <span className="flex gap-1">
+ <Button variant="ghost" size="sm" onClick={() => (mirror.send({ op: 'newTab' }), onClose())}>
+ <Plus className="size-3.5" />
+ New tab
+ </Button>
+ <Button variant="ghost" size="icon-sm" aria-label="Close the list" onClick={onClose}>
+ <X className="size-3.5" />
+ </Button>
+ </span>
+ </div>
+ <ul className="min-h-0 flex-1 space-y-1 overflow-y-auto">
+ {session.targets.map((target) => (
+ <li
+ key={target.targetId}
+ className={cn('flex items-center gap-2 rounded-lg px-2.5 py-2', target.targetId === session.activeTargetId ? 'bg-brand/12' : 'hover:bg-surface')}
+ >
+ <button type="button" className="min-w-0 flex-1 text-left" onClick={() => (mirror.send({ op: 'switchTab', targetId: target.targetId }), onClose())}>
+ <span className="block truncate text-xs text-ink">{target.title || target.url}</span>
+ <span className="block truncate text-[11px] text-ink-faint">{target.url}</span>
+ </button>
+ <Button variant="ghost" size="icon-sm" aria-label={`Close ${target.title || target.url}`} onClick={() => mirror.send({ op: 'closeTab', targetId: target.targetId })}>
+ <X className="size-3.5" />
+ </Button>
+ </li>
+ ))}
+ </ul>
+ </div>
+ );
+}
+
+function Notice({ icon, title, body, children }: { icon: ReactNode; title: string; body: string; children?: ReactNode }) {
+ return (
+ <div className="panel-card flex max-w-xs flex-col items-center gap-3 p-6 text-center">
+ <span className="flex size-12 items-center justify-center rounded-full border border-brand/50 bg-brand/10 text-brand">{icon}</span>
+ <h1 className="text-sm font-medium">{title}</h1>
+ <p className="text-xs leading-relaxed text-ink-dim">{body}</p>
+ {children}
+ </div>
+ );
+}
+
+function WaitingForChrome({ session }: { session: PhoneSession }) {
+ const [busy, setBusy] = useState(false);
+ const [error, setError] = useState<string>();
+ const open = async () => {
setBusy(true);
setError(undefined);
- const result = await askPhone(request);
+ const started = Date.now();
+ let result = await askPhone({ op: 'phoneOpenChrome' });
+ while (result.ok && result.data.waitingForChrome && Date.now() - started < CHROME_RETRY_FOR_MS) {
+ await new Promise((resolve) => setTimeout(resolve, CHROME_RETRY_MS));
+ result = await askPhone({ op: 'phoneOpenChrome' });
+ }
setBusy(false);
if (!result.ok) setError(result.error.message);
};
+ return (
+ <Notice icon={<Smartphone className="size-6" />} title="Open Chrome on your phone" body={`Browsentic needs Chrome running on ${session.model ?? 'your phone'} to show and drive it.`}>
+ <Button size="sm" disabled={busy} onClick={() => void open()}>
+ {busy && <Loader2 className="size-3.5 animate-spin" />}
+ Open Chrome on the phone
+ </Button>
+ {error && <p className="text-xs text-destructive">{error}</p>}
+ </Notice>
+ );
+}
+function Ended({ session }: { session: PhoneSession }) {
+ const [busy, setBusy] = useState(false);
+ const [error, setError] = useState<string>();
+ const reconnect = async () => {
+ setBusy(true);
+ const result = await askPhone({ op: 'phoneStart', serial: session.serial, windowId: session.windowId });
+ setBusy(false);
+ setError(result.ok ? undefined : result.error.message);
+ };
return (
- <div className="dot-grid flex min-h-screen items-center justify-center p-6">
- <main className="panel-card flex w-full max-w-sm flex-col items-center gap-4 p-8 text-center">
- <Wordmark />
- <span className="glow-brand flex size-16 items-center justify-center rounded-full border border-brand/50 bg-brand/10 text-brand">
- {session?.ended ? <Unplug className="size-7" /> : <Smartphone className="size-7" />}
- </span>
- <h1 className="text-base font-medium">{title}</h1>
- <p className="text-xs leading-relaxed text-ink-dim">{body}</p>
- {session && !session.ended && session.targets.length > 0 && (
- <ul className="w-full space-y-1 text-left text-xs text-ink-dim">
- {session.targets.map((target) => (
- <li key={target.targetId} className="truncate">
- {target.title || target.url}
- </li>
- ))}
- </ul>
- )}
- {session && !session.ended && session.waitingForChrome && (
- <Button size="sm" disabled={busy} onClick={() => void act({ op: 'phoneOpenChrome' })}>
- Open Chrome on the phone
- </Button>
- )}
- {session?.ended && (
- <div className="flex gap-2">
- <Button size="sm" disabled={busy} onClick={() => void act({ op: 'phoneStart', serial: session.serial, windowId: session.windowId })}>
- Reconnect
- </Button>
- <Button size="sm" variant="outline" onClick={() => void closeThisTab()}>
- Close tab
- </Button>
- </div>
- )}
- {error && <p className="text-xs text-destructive">{error}</p>}
- </main>
- </div>
+ <Notice icon={<Unplug className="size-6" />} title={`${session.model ?? 'The phone'} is not connected`} body={ENDED[session.ended!.reason]}>
+ <div className="flex gap-2">
+ <Button size="sm" disabled={busy} onClick={() => void reconnect()}>
+ Reconnect
+ </Button>
+ <Button size="sm" variant="outline" onClick={() => void closeThisTab()}>
+ Close tab
+ </Button>
+ </div>
+ {error && <p className="text-xs text-destructive">{error}</p>}
+ </Notice>
);
}
diff --git a/src/extension/entrypoints/phone/use-mirror.ts b/src/extension/entrypoints/phone/use-mirror.ts
new file mode 100644
index 0000000..294c0bd
--- /dev/null
+++ b/src/extension/entrypoints/phone/use-mirror.ts
@@ -0,0 +1,134 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { browser, type Browser } from 'wxt/browser';
+import { PHONE_PORT, isMirrorKey, touchPoint, type FrameMetadata, type MirrorCommand, type MirrorMessage, type PageState, type TouchType } from '@/lib/phone/mirror';
+
+const RECONNECT_MS = 500;
+const ERROR_SHOWN_MS = 5_000;
+
+export interface Mirror {
+ canvas: React.RefObject<HTMLCanvasElement | null>;
+ page: PageState | null;
+ image: { width: number; height: number } | null;
+ error: string | undefined;
+ send(command: MirrorCommand): void;
+}
+
+const decode = (data: string) => new Blob([Uint8Array.from(atob(data), (char) => char.charCodeAt(0))], { type: 'image/jpeg' });
+
+/** The page's half of the mirror: frames in and drawn, then acked; touches, wheel and keys out as the phone takes them. */
+export function useMirror(watching: boolean): Mirror {
+ const canvas = useRef<HTMLCanvasElement | null>(null);
+ const port = useRef<Browser.runtime.Port | null>(null);
+ const metadata = useRef<FrameMetadata | null>(null);
+ const watchingNow = useRef(watching);
+ const [page, setPage] = useState<PageState | null>(null);
+ const [image, setImage] = useState<{ width: number; height: number } | null>(null);
+ const [error, setError] = useState<string>();
+
+ const send = useCallback((command: MirrorCommand) => port.current?.postMessage(command), []);
+
+ useEffect(() => {
+ let live = true;
+ let retry: ReturnType<typeof setTimeout> | undefined;
+ const draw = async (message: Extract<MirrorMessage, { kind: 'frame' }>, from: Browser.runtime.Port) => {
+ const bitmap = await createImageBitmap(decode(message.data));
+ const target = canvas.current;
+ if (target) {
+ if (target.width !== bitmap.width || target.height !== bitmap.height) {
+ target.width = bitmap.width;
+ target.height = bitmap.height;
+ setImage({ width: bitmap.width, height: bitmap.height });
+ }
+ target.getContext('2d')?.drawImage(bitmap, 0, 0);
+ }
+ bitmap.close();
+ metadata.current = message.metadata;
+ from.postMessage({ op: 'ack', frame: message.frame } satisfies MirrorCommand);
+ };
+ const connect = () => {
+ const opened = browser.runtime.connect({ name: PHONE_PORT });
+ port.current = opened;
+ opened.onMessage.addListener((raw) => {
+ const message = raw as MirrorMessage;
+ if (message.kind === 'frame') void draw(message, opened);
+ else if (message.kind === 'page') setPage(message);
+ else setError(message.message);
+ });
+ opened.onDisconnect.addListener(() => {
+ if (port.current === opened) port.current = null;
+ if (live) retry = setTimeout(connect, RECONNECT_MS);
+ });
+ if (watchingNow.current) opened.postMessage({ op: 'watch' } satisfies MirrorCommand);
+ };
+ connect();
+ return () => {
+ live = false;
+ clearTimeout(retry);
+ port.current?.disconnect();
+ };
+ }, []);
+
+ useEffect(() => {
+ watchingNow.current = watching;
+ send({ op: watching ? 'watch' : 'unwatch' });
+ }, [watching, send]);
+
+ useEffect(() => {
+ if (!error) return;
+ const timer = setTimeout(() => setError(undefined), ERROR_SHOWN_MS);
+ return () => clearTimeout(timer);
+ }, [error]);
+
+ useEffect(() => {
+ const target = canvas.current;
+ if (!target) return;
+ const at = (event: { clientX: number; clientY: number }) => {
+ const frame = metadata.current;
+ if (!frame || !target.width) return null;
+ const box = target.getBoundingClientRect();
+ return touchPoint({ x: (event.clientX - box.left) / box.width, y: (event.clientY - box.top) / box.height }, frame, target);
+ };
+ const touch = (type: TouchType) => (event: PointerEvent) => {
+ if (type !== 'touchStart' && !target.hasPointerCapture(event.pointerId)) return;
+ if (type === 'touchStart') {
+ target.setPointerCapture(event.pointerId);
+ target.focus();
+ }
+ const point = at(event);
+ if (point) send({ op: 'touch', type, ...point });
+ event.preventDefault();
+ };
+ const wheel = (event: WheelEvent) => {
+ const point = at(event);
+ if (point) send({ op: 'wheel', ...point, deltaX: event.deltaX, deltaY: event.deltaY });
+ event.preventDefault();
+ };
+ const key = (event: KeyboardEvent) => {
+ if (event.metaKey || event.ctrlKey || event.altKey) return;
+ if (isMirrorKey(event.key)) send({ op: 'key', key: event.key });
+ else if (event.key.length === 1) send({ op: 'text', text: event.key });
+ else return;
+ event.preventDefault();
+ };
+ const paste = (event: ClipboardEvent) => {
+ const text = event.clipboardData?.getData('text/plain');
+ if (text) send({ op: 'text', text });
+ event.preventDefault();
+ };
+ const listeners = [
+ ['pointerdown', touch('touchStart')],
+ ['pointermove', touch('touchMove')],
+ ['pointerup', touch('touchEnd')],
+ ['pointercancel', touch('touchCancel')],
+ ['wheel', wheel],
+ ['keydown', key],
+ ['paste', paste],
+ ] as const;
+ for (const [type, listener] of listeners) target.addEventListener(type, listener as EventListener, { passive: false });
+ return () => {
+ for (const [type, listener] of listeners) target.removeEventListener(type, listener as EventListener);
+ };
+ }, [send]);
+
+ return { canvas, page, image, error, send };
+}
diff --git a/src/lib/bridge/phone-mirror.test.ts b/src/lib/bridge/phone-mirror.test.ts
new file mode 100644
index 0000000..7a9fe4e
--- /dev/null
+++ b/src/lib/bridge/phone-mirror.test.ts
@@ -0,0 +1,150 @@
+import { beforeEach, describe, expect, test, vi } from 'vitest';
+import { fakeBrowser } from 'wxt/testing';
+import { success } from '@/lib/actions/protocol';
+import { BLOCKED_SITES_KEY } from '@/lib/settings/blocked-sites';
+import type { CdpEvent } from './socket';
+import type { MirrorMessage } from '@/lib/phone/mirror';
+
+const SERIAL = 'emulator-5554';
+const FRONT = 'A'.repeat(32);
+const BACK = 'B'.repeat(32);
+
+const socket = vi.hoisted(() => ({ sent: [] as { method: string; params?: Record<string, unknown>; sessionId?: string }[], events: [] as ((event: CdpEvent) => void)[] }));
+
+vi.mock('./socket', () => ({
+ DAEMON_STATE_KEY: 'browsentic/daemon',
+ sendCdp: async (_serial: string, method: string, params?: Record<string, unknown>, sessionId?: string) => {
+ socket.sent.push({ method, params, sessionId });
+ if (method === 'Target.attachToTarget') return success({ sessionId: `session-${params?.targetId}` });
+ if (method === 'Runtime.evaluate') return success({ result: { value: sessionId === `session-${FRONT}` ? 'visible' : 'hidden' } });
+ if (method === 'Page.getNavigationHistory') return success({ currentIndex: 1, entries: [{ id: 1, url: 'https://a.example/', title: 'A' }, { id: 2, url: 'https://b.example/', title: 'B' }] });
+ return success({});
+ },
+ onCdpEvent: (listener: (event: CdpEvent) => void) => socket.events.push(listener),
+ onPhoneClosed: () => undefined,
+ onDaemonClosed: () => undefined,
+ closePhone: () => undefined,
+ openPhone: async () => success({}),
+ launchPhoneChrome: async () => success({}),
+}));
+
+type Phone = typeof import('./phone');
+let readPhone: Phone['readPhone'];
+const connectListeners: ((port: unknown) => void)[] = [];
+
+/** The page's end of the port, as the background sees it. */
+function connectPage() {
+ const received: MirrorMessage[] = [];
+ const messageListeners: ((message: unknown) => void)[] = [];
+ const port = {
+ name: 'browsentic/phone',
+ postMessage: (message: MirrorMessage) => received.push(message),
+ onMessage: { addListener: (listener: (message: unknown) => void) => messageListeners.push(listener) },
+ onDisconnect: { addListener: () => undefined },
+ };
+ connectListeners.forEach((listener) => listener(port));
+ return { received, say: (message: unknown) => messageListeners.forEach((listener) => listener(message)) };
+}
+
+const methods = () => socket.sent.map(({ method }) => method);
+const settle = () => new Promise((resolve) => setTimeout(resolve, 20));
+const emit = (event: Omit<CdpEvent, 't' | 'serial'>) => socket.events.forEach((listener) => listener({ t: 'cdpEvent', serial: SERIAL, ...event }));
+
+beforeEach(async () => {
+ fakeBrowser.reset();
+ socket.sent.length = 0;
+ socket.events.length = 0;
+ connectListeners.length = 0;
+ Object.assign(fakeBrowser.runtime, { onConnect: { addListener: (listener: (port: unknown) => void) => connectListeners.push(listener) } });
+ vi.resetModules();
+ ({ readPhone } = await import('./phone'));
+ (await import('./phone-mirror')).servePhoneMirror();
+ await fakeBrowser.storage.session.set({
+ 'browsentic/phone': {
+ serial: SERIAL,
+ mirrorTabId: 9,
+ windowId: 1,
+ openedAt: 0,
+ transport: 'usb',
+ targets: [
+ { targetId: BACK, url: 'https://b.example/', title: 'B' },
+ { targetId: FRONT, url: 'https://a.example/', title: 'A' },
+ ],
+ },
+ });
+});
+
+describe('the mirror in the background', () => {
+ test('watching follows the tab in front, not the first listed, and streams it', async () => {
+ const page = connectPage();
+ page.say({ op: 'watch' });
+ await settle();
+ expect((await readPhone())?.activeTargetId).toBe(FRONT);
+ expect(socket.sent.find(({ method }) => method === 'Page.startScreencast')).toEqual({
+ method: 'Page.startScreencast',
+ params: { format: 'jpeg', quality: 60, maxWidth: 720, maxHeight: 1600, everyNthFrame: 1 },
+ sessionId: `session-${FRONT}`,
+ });
+ expect(page.received.find((message) => message.kind === 'page')).toMatchObject({ targetId: FRONT, url: 'https://b.example/', canGoBack: true, canGoForward: false });
+ });
+
+ test('a frame goes to the page, and the page’s ack goes to the phone', async () => {
+ const page = connectPage();
+ page.say({ op: 'watch' });
+ await settle();
+ emit({ method: 'Page.screencastFrame', params: { data: 'JPEG', sessionId: 7, metadata: { deviceWidth: 411, pageScaleFactor: 1 } }, sessionId: `session-${FRONT}` });
+ expect(page.received.find((message) => message.kind === 'frame')).toEqual({ kind: 'frame', data: 'JPEG', frame: 7, metadata: { deviceWidth: 411, pageScaleFactor: 1 } });
+ page.say({ op: 'ack', frame: 7 });
+ await settle();
+ expect(socket.sent.at(-1)).toEqual({ method: 'Page.screencastFrameAck', params: { sessionId: 7 }, sessionId: `session-${FRONT}` });
+ });
+
+ test('a finger’s moves while one is on its way collapse into the latest, and the lift still follows', async () => {
+ const page = connectPage();
+ page.say({ op: 'watch' });
+ await settle();
+ socket.sent.length = 0;
+ page.say({ op: 'touch', type: 'touchStart', x: 10, y: 10 });
+ for (let step = 1; step <= 5; step++) page.say({ op: 'touch', type: 'touchMove', x: 10, y: 10 + step * 10 });
+ page.say({ op: 'touch', type: 'touchEnd', x: 10, y: 60 });
+ await settle();
+ expect(socket.sent.map(({ params }) => [params?.type, params?.touchPoints])).toEqual([
+ ['touchStart', [{ x: 10, y: 10 }]],
+ ['touchMove', [{ x: 10, y: 60 }]],
+ ['touchEnd', []],
+ ]);
+ });
+
+ test('a blocked site typed in the address bar is refused before the phone hears of it', async () => {
+ await fakeBrowser.storage.local.set({ [BLOCKED_SITES_KEY]: ['bank.example'] });
+ const page = connectPage();
+ page.say({ op: 'watch' });
+ await settle();
+ page.say({ op: 'go', url: 'https://bank.example/login' });
+ await settle();
+ expect(methods()).not.toContain('Page.navigate');
+ expect(page.received.at(-1)).toMatchObject({ kind: 'error' });
+ page.say({ op: 'go', url: 'https://ok.example/' });
+ await settle();
+ expect(socket.sent.at(-1)).toMatchObject({ method: 'Page.navigate', params: { url: 'https://ok.example/' } });
+ });
+
+ test('the tab list follows the phone and leaves out the phantom target', async () => {
+ connectPage();
+ emit({ method: 'Target.targetCreated', params: { targetInfo: { targetId: '1', type: 'page', url: 'https://a.example/', title: 'A' } } });
+ emit({ method: 'Target.targetCreated', params: { targetInfo: { targetId: 'C'.repeat(32), type: 'page', url: 'https://c.example/', title: 'C' } } });
+ emit({ method: 'Target.targetCreated', params: { targetInfo: { targetId: 'D'.repeat(32), type: 'service_worker', url: 'https://c.example/sw.js', title: '' } } });
+ emit({ method: 'Target.targetDestroyed', params: { targetId: BACK } });
+ await settle();
+ expect((await readPhone())?.targets.map(({ targetId }) => targetId[0])).toEqual(['A', 'C']);
+ });
+
+ test('back steps to the previous history entry', async () => {
+ const page = connectPage();
+ page.say({ op: 'watch' });
+ await settle();
+ page.say({ op: 'back' });
+ await settle();
+ expect(socket.sent.at(-1)).toEqual({ method: 'Page.navigateToHistoryEntry', params: { entryId: 1 }, sessionId: `session-${FRONT}` });
+ });
+});
diff --git a/src/lib/bridge/phone-mirror.ts b/src/lib/bridge/phone-mirror.ts
new file mode 100644
index 0000000..117eb14
--- /dev/null
+++ b/src/lib/bridge/phone-mirror.ts
@@ -0,0 +1,315 @@
+import { browser, type Browser } from 'wxt/browser';
+import { PHONE_PORT, keyEvents, type MirrorCommand, type MirrorMessage, type PageState } from '@/lib/phone/mirror';
+import type { PhoneTarget } from '@/lib/phone/types';
+import { readPhone, updatePhone, type PhoneSession } from './phone';
+import { refusalFor } from './site-guard';
+import { onCdpEvent, onDaemonClosed, onPhoneClosed, sendCdp, type CdpEvent } from './socket';
+
+type Port = Browser.runtime.Port;
+type InputCommand = Extract<MirrorCommand, { op: 'touch' | 'wheel' | 'text' | 'key' }>;
+
+/** Discovery also reports a phantom `"1"` beside a restored tab, and every command sent to it hangs (spike Q3). */
+const REAL_TARGET = /^[0-9A-F]{32}$/i;
+const CLOSED_TAB_SETTLE_MS = 300;
+const ACTIVATE_TRIES = 4;
+const ACTIVATE_SETTLE_MS = 250;
+
+interface Followed {
+ serial: string;
+ targetId: string;
+ sessionId: string;
+}
+
+const ports = new Set<Port>();
+const watchers = new Set<Port>();
+let followed: Followed | null = null;
+let streaming = false;
+let page: PageState | null = null;
+let finding: Promise<void> | null = null;
+const inputs: InputCommand[] = [];
+let pumping = false;
+
+const broadcast = (message: MirrorMessage) => {
+ for (const port of ports) port.postMessage(message);
+};
+
+const tellError = (message: string) => broadcast({ kind: 'error', message });
+
+interface TargetInfo {
+ targetId?: string;
+ type?: string;
+ url?: string;
+ title?: string;
+}
+
+const isPageTarget = (info: TargetInfo): info is TargetInfo & { targetId: string } => info.type === 'page' && REAL_TARGET.test(info.targetId ?? '');
+
+async function liveSession(): Promise<PhoneSession | null> {
+ const session = await readPhone();
+ return session && !session.ended && !session.waitingForChrome ? session : null;
+}
+
+function call(method: string, params?: Record<string, unknown>) {
+ const held = followed;
+ return held ? sendCdp(held.serial, method, params, held.sessionId) : Promise.resolve(null);
+}
+
+async function follow(session: PhoneSession, targetId: string): Promise<void> {
+ if (followed?.serial === session.serial && followed.targetId === targetId) return;
+ await stopStream();
+ if (followed) void sendCdp(followed.serial, 'Target.detachFromTarget', { sessionId: followed.sessionId });
+ followed = null;
+ const attached = await sendCdp(session.serial, 'Target.attachToTarget', { targetId, flatten: true });
+ if (!attached.ok) return tellError(attached.error.message);
+ followed = { serial: session.serial, targetId, sessionId: String(attached.data.sessionId) };
+ await call('Page.enable');
+ await updatePhone((held) => ({ ...held, activeTargetId: targetId }));
+ await publishPage();
+ if (watchers.size) await startStream();
+}
+
+async function ensureFollowed(): Promise<void> {
+ const session = await liveSession();
+ if (!session) return;
+ if (followed && followed.serial !== session.serial) followed = null;
+ if (followed) return;
+ const known = session.targets.find((target) => target.targetId === session.activeTargetId);
+ if (known) return follow(session, known.targetId);
+ await findFront();
+ if (!followed && session.targets[0]) await follow(session, session.targets[0].targetId);
+}
+
+async function startStream(): Promise<void> {
+ if (!followed) return;
+ const wifi = (await readPhone())?.transport === 'wifi';
+ streaming = true;
+ const started = await call('Page.startScreencast', { format: 'jpeg', quality: wifi ? 40 : 60, maxWidth: wifi ? 540 : 720, maxHeight: 1600, everyNthFrame: 1 });
+ if (started && !started.ok) {
+ streaming = false;
+ tellError(started.error.message);
+ }
+}
+
+async function stopStream(): Promise<void> {
+ if (!followed || !streaming) return;
+ streaming = false;
+ await call('Page.stopScreencast');
+}
+
+async function publishPage(): Promise<void> {
+ const held = followed;
+ if (!held) return;
+ const history = await call('Page.getNavigationHistory');
+ if (!history?.ok || followed !== held) return;
+ const entries = (history.data.entries as { id: number; url: string; title: string }[] | undefined) ?? [];
+ const current = Number(history.data.currentIndex ?? entries.length - 1);
+ const entry = entries[current];
+ page = {
+ targetId: held.targetId,
+ url: entry?.url ?? '',
+ title: entry?.title ?? '',
+ canGoBack: current > 0,
+ canGoForward: current < entries.length - 1,
+ };
+ broadcast({ kind: 'page', ...page });
+}
+
+/** A tab switch made on the phone fires no Target event; the tab whose document is visible is the one in front. */
+function findFront(): Promise<void> {
+ finding ??= (async () => {
+ const session = await liveSession();
+ if (!session) return;
+ for (const target of session.targets) {
+ const visible = await isVisible(session.serial, target.targetId);
+ if (visible) return follow(session, target.targetId);
+ }
+ })().finally(() => (finding = null));
+ return finding;
+}
+
+async function isVisible(serial: string, targetId: string): Promise<boolean> {
+ const own = followed?.targetId === targetId ? followed.sessionId : undefined;
+ const attached = own ? null : await sendCdp(serial, 'Target.attachToTarget', { targetId, flatten: true });
+ const sessionId = own ?? (attached?.ok ? String(attached.data.sessionId) : null);
+ if (!sessionId) return false;
+ const state = await sendCdp(serial, 'Runtime.evaluate', { expression: 'document.visibilityState', returnByValue: true }, sessionId);
+ if (!own) void sendCdp(serial, 'Target.detachFromTarget', { sessionId });
+ return state.ok && (state.data.result as { value?: unknown } | undefined)?.value === 'visible';
+}
+
+async function trackTargets(event: CdpEvent): Promise<void> {
+ const info = (event.params.targetInfo ?? {}) as TargetInfo;
+ if (event.method === 'Target.detachedFromTarget' && event.params.sessionId === followed?.sessionId) {
+ followed = null;
+ streaming = false;
+ if (watchers.size) void ensureFollowed();
+ return;
+ }
+ if (event.method === 'Target.targetDestroyed') {
+ const targetId = String(event.params.targetId);
+ await updatePhone((held) => ({ ...held, targets: held.targets.filter((target) => target.targetId !== targetId) }));
+ if (followed?.targetId === targetId) {
+ followed = null;
+ streaming = false;
+ setTimeout(() => void findFront(), CLOSED_TAB_SETTLE_MS);
+ }
+ return;
+ }
+ if ((event.method !== 'Target.targetCreated' && event.method !== 'Target.targetInfoChanged') || !isPageTarget(info)) return;
+ const target: PhoneTarget = { targetId: info.targetId, url: info.url ?? '', title: info.title ?? '' };
+ await updatePhone((held) => ({
+ ...held,
+ targets: held.targets.some((each) => each.targetId === target.targetId)
+ ? held.targets.map((each) => (each.targetId === target.targetId ? target : each))
+ : [...held.targets, target],
+ }));
+ if (page && followed?.targetId === target.targetId) {
+ page = { ...page, url: target.url, title: target.title };
+ broadcast({ kind: 'page', ...page });
+ }
+}
+
+function onEvent(event: CdpEvent): void {
+ if (event.method.startsWith('Target.') && !event.sessionId) return void trackTargets(event);
+ if (!followed || event.sessionId !== followed.sessionId) return;
+ if (event.method === 'Page.screencastFrame') {
+ const frame = Number(event.params.sessionId);
+ if (!watchers.size) return void call('Page.screencastFrameAck', { sessionId: frame });
+ broadcast({ kind: 'frame', data: String(event.params.data), metadata: event.params.metadata as never, frame });
+ return;
+ }
+ if (event.method === 'Page.screencastVisibilityChanged' && event.params.visible === false) return void findFront();
+ if (event.method === 'Page.frameNavigated' && !(event.params.frame as { parentId?: string } | undefined)?.parentId) return void publishPage();
+ if (event.method === 'Page.navigatedWithinDocument') void publishPage();
+}
+
+function queueInput(command: InputCommand): void {
+ const last = inputs.at(-1);
+ if (last?.op === 'touch' && command.op === 'touch' && last.type === 'touchMove' && command.type === 'touchMove') inputs[inputs.length - 1] = command;
+ else if (last?.op === 'wheel' && command.op === 'wheel') {
+ inputs[inputs.length - 1] = { ...command, deltaX: last.deltaX + command.deltaX, deltaY: last.deltaY + command.deltaY };
+ } else inputs.push(command);
+ void pump();
+}
+
+async function pump(): Promise<void> {
+ if (pumping) return;
+ pumping = true;
+ try {
+ for (let next = inputs.shift(); next; next = inputs.shift()) await dispatchInput(next);
+ } finally {
+ pumping = false;
+ }
+}
+
+async function dispatchInput(command: InputCommand): Promise<void> {
+ if (command.op === 'touch') {
+ const lifted = command.type === 'touchEnd' || command.type === 'touchCancel';
+ await call('Input.dispatchTouchEvent', { type: command.type, touchPoints: lifted ? [] : [{ x: command.x, y: command.y }] });
+ } else if (command.op === 'wheel') {
+ await call('Input.dispatchMouseEvent', { type: 'mouseWheel', x: command.x, y: command.y, deltaX: command.deltaX, deltaY: command.deltaY });
+ } else if (command.op === 'text') {
+ await call('Input.insertText', { text: command.text });
+ } else {
+ for (const event of keyEvents(command.key)) await call('Input.dispatchKeyEvent', event);
+ }
+}
+
+/** Chrome on Android can ignore an activation that lands just after a tab opened, so it is asked again until the tab is in front. */
+async function bringToFront(serial: string, targetId: string): Promise<void> {
+ for (let attempt = 0; attempt < ACTIVATE_TRIES; attempt++) {
+ await sendCdp(serial, 'Target.activateTarget', { targetId });
+ await new Promise((resolve) => setTimeout(resolve, ACTIVATE_SETTLE_MS));
+ if (await isVisible(serial, targetId)) return;
+ }
+}
+
+async function goInHistory(step: -1 | 1): Promise<void> {
+ const history = await call('Page.getNavigationHistory');
+ if (!history?.ok) return;
+ const entries = (history.data.entries as { id: number }[] | undefined) ?? [];
+ const entry = entries[Number(history.data.currentIndex) + step];
+ if (entry) await call('Page.navigateToHistoryEntry', { entryId: entry.id });
+}
+
+async function handle(port: Port, command: MirrorCommand): Promise<void> {
+ switch (command.op) {
+ case 'watch':
+ watchers.add(port);
+ if (!followed) return ensureFollowed();
+ if (!streaming) return startStream();
+ if (page) port.postMessage({ kind: 'page', ...page });
+ return;
+ case 'unwatch':
+ watchers.delete(port);
+ if (!watchers.size) await stopStream();
+ return;
+ case 'restart':
+ await stopStream();
+ if (watchers.size) await startStream();
+ return;
+ case 'ack':
+ await call('Page.screencastFrameAck', { sessionId: command.frame });
+ return;
+ case 'touch':
+ case 'wheel':
+ case 'text':
+ case 'key':
+ return queueInput(command);
+ case 'go': {
+ const refused = await refusalFor(command.url);
+ if (refused) return tellError(refused.ok ? '' : refused.error.message);
+ const navigated = await call('Page.navigate', { url: command.url });
+ if (navigated && !navigated.ok) tellError(navigated.error.message);
+ return;
+ }
+ case 'back':
+ return goInHistory(-1);
+ case 'forward':
+ return goInHistory(1);
+ case 'reload':
+ await call('Page.reload');
+ return;
+ case 'switchTab': {
+ const session = await liveSession();
+ if (!session) return;
+ await bringToFront(session.serial, command.targetId);
+ return follow(session, command.targetId);
+ }
+ case 'newTab': {
+ const session = await liveSession();
+ if (!session) return;
+ const created = await sendCdp(session.serial, 'Target.createTarget', { url: 'about:blank' });
+ if (!created.ok) return tellError(created.error.message);
+ return follow(session, String(created.data.targetId));
+ }
+ case 'closeTab': {
+ const session = await liveSession();
+ if (session) await sendCdp(session.serial, 'Target.closeTarget', { targetId: command.targetId });
+ return;
+ }
+ }
+}
+
+function forget(): void {
+ followed = null;
+ streaming = false;
+ page = null;
+ inputs.length = 0;
+}
+
+export function servePhoneMirror(): void {
+ browser.runtime.onConnect.addListener((port) => {
+ if (port.name !== PHONE_PORT) return;
+ ports.add(port);
+ if (page) port.postMessage({ kind: 'page', ...page });
+ port.onMessage.addListener((message) => void handle(port, message as MirrorCommand));
+ port.onDisconnect.addListener(() => {
+ ports.delete(port);
+ if (watchers.delete(port) && !watchers.size) void stopStream();
+ });
+ });
+ onCdpEvent(onEvent);
+ onPhoneClosed(forget);
+ onDaemonClosed(forget);
+}
diff --git a/src/lib/bridge/phone.test.ts b/src/lib/bridge/phone.test.ts
index 41f35cd..6caedf7 100644
--- a/src/lib/bridge/phone.test.ts
+++ b/src/lib/bridge/phone.test.ts
@@ -53,7 +53,7 @@ describe('the phone session', () => {
const started = await startPhone(SERIAL);
expect(started.ok).toBe(true);
const session = await readPhone();
- expect(session).toMatchObject({ serial: SERIAL, model: 'Pixel 8', targets: OPENED.targets, activeTargetId: OPENED.targets[0].targetId, waitingForChrome: false });
+ expect(session).toMatchObject({ serial: SERIAL, model: 'Pixel 8', transport: 'usb', targets: OPENED.targets, waitingForChrome: false });
const mirror = await fakeBrowser.tabs.get(session!.mirrorTabId);
expect(mirror.url).toMatch(/phone\.html$/);
expect(notify).not.toHaveBeenCalled();
diff --git a/src/lib/bridge/phone.ts b/src/lib/bridge/phone.ts
index 9858229..02498b9 100644
--- a/src/lib/bridge/phone.ts
+++ b/src/lib/bridge/phone.ts
@@ -1,6 +1,6 @@
import { browser } from 'wxt/browser';
import { failure, success, type ActionResult } from '@/lib/actions/protocol';
-import type { PhoneClosedReason, PhoneOpened, PhoneTarget } from '@/lib/phone/types';
+import type { AndroidDevice, PhoneClosedReason, PhoneOpened, PhoneTarget } from '@/lib/phone/types';
import { DAEMON_STATE_KEY, closePhone, launchPhoneChrome, onDaemonClosed, onPhoneClosed, openPhone, type DaemonState } from './socket';
export const PHONE_KEY = 'browsentic/phone';
@@ -16,10 +16,12 @@ export type PhoneEndReason = PhoneClosedReason | 'bridge-gone';
export interface PhoneSession {
serial: string;
model?: string;
+ transport?: AndroidDevice['transport'];
mirrorTabId: number;
windowId: number;
openedAt: number;
targets: PhoneTarget[];
+ /** The tab the mirror follows: the one in front on the phone, found once the mirror starts. */
activeTargetId?: string;
/** Chrome was not open on the phone; the mirror waits, and the notification offers to open it. */
waitingForChrome?: boolean;
@@ -45,24 +47,39 @@ export async function readPhone(): Promise<PhoneSession | null> {
const writePhone = (session: PhoneSession | null): Promise<void> =>
session ? browser.storage.session.set({ [PHONE_KEY]: session }) : browser.storage.session.remove(PHONE_KEY);
+/** A change to the live session, made in turn with every other; nothing happens when there is none. */
+export function updatePhone(change: (session: PhoneSession) => PhoneSession): Promise<PhoneSession | null> {
+ return locked(async () => {
+ const held = await readPhone();
+ if (!held || held.ended) return held;
+ const next = change(held);
+ await writePhone(next);
+ return next;
+ });
+}
+
const opening = (opened: ActionResult<PhoneOpened>) =>
opened.ok
- ? { targets: opened.data.targets, activeTargetId: opened.data.targets[0]?.targetId, waitingForChrome: false }
+ ? { targets: opened.data.targets, activeTargetId: undefined, waitingForChrome: false }
: { targets: [], activeTargetId: undefined, waitingForChrome: true };
-async function modelOf(serial: string): Promise<string | undefined> {
+async function deviceOf(serial: string): Promise<AndroidDevice | undefined> {
const stored = await browser.storage.session.get(DAEMON_STATE_KEY);
- return (stored[DAEMON_STATE_KEY] as DaemonState | undefined)?.android?.devices.find((device) => device.serial === serial)?.model;
+ return (stored[DAEMON_STATE_KEY] as DaemonState | undefined)?.android?.devices.find((device) => device.serial === serial);
}
-/** Starts the session and opens its mirror beside the tab the user is on. A second start shows the mirror already open. */
+/**
+ * Starts the session and opens its mirror beside the tab the user is on. A second start for the same
+ * phone shows the mirror already open; one for another phone moves the session, and its tab, to it.
+ */
export function startPhone(serial: string, windowId?: number): Promise<ActionResult<PhoneSession>> {
return locked(async () => {
const held = await readPhone();
- if (held && !held.ended) {
+ if (held && !held.ended && held.serial === serial) {
await showTab(held.mirrorTabId, held.windowId);
return success(held);
}
+ if (held && !held.ended) closePhone(held.serial);
const opened = await openPhone(serial);
if (!opened.ok && opened.error.code !== 'CHROME_NOT_RUNNING') return opened;
@@ -79,7 +96,7 @@ export function startPhone(serial: string, windowId?: number): Promise<ActionRes
windowId: mirror.windowId ?? windowId ?? browser.windows.WINDOW_ID_CURRENT,
openedAt: Date.now(),
...opening(opened),
- model: (opened.ok ? opened.data.device.model : undefined) ?? (await modelOf(serial)),
+ ...(await describedPhone(serial, opened)),
};
await writePhone(session);
if (session.waitingForChrome) await askToOpenChrome(session);
@@ -140,6 +157,11 @@ export function servePhone(): void {
});
}
+async function describedPhone(serial: string, opened: ActionResult<PhoneOpened>): Promise<Pick<PhoneSession, 'model' | 'transport'>> {
+ const device = opened.ok ? opened.data.device : await deviceOf(serial);
+ return { model: device?.model, transport: device?.transport };
+}
+
async function openMirror(windowId?: number) {
const [current] = await browser.tabs.query(windowId === undefined ? { active: true, lastFocusedWindow: true } : { active: true, windowId });
return browser.tabs.create({
diff --git a/src/lib/phone/mirror.test.ts b/src/lib/phone/mirror.test.ts
new file mode 100644
index 0000000..bafa323
--- /dev/null
+++ b/src/lib/phone/mirror.test.ts
@@ -0,0 +1,49 @@
+import { describe, expect, test } from 'vitest';
+import { addressOf, isMirrorKey, keyEvents, touchPoint } from './mirror';
+
+describe('a click on the mirror', () => {
+ test('lands where spike Q8 measured it: the frame is the visual viewport, scaled by the zoom', () => {
+ const image = { width: 720, height: 1182 };
+ expect(touchPoint({ x: 0.5, y: 0.5 }, { deviceWidth: 411, pageScaleFactor: 1 }, image)).toEqual({ x: 205.5, y: (411 * 1182) / 720 / 2 });
+ expect(touchPoint({ x: 1, y: 0 }, { deviceWidth: 411, pageScaleFactor: 2 }, image)).toEqual({ x: 205.5, y: 0 });
+ });
+
+ test('a frame squeezed by the soft keyboard keeps the x scale and shortens the y range', () => {
+ const point = touchPoint({ x: 0.25, y: 1 }, { deviceWidth: 411, pageScaleFactor: 1 }, { width: 720, height: 531 });
+ expect(point.x).toBeCloseTo(102.75);
+ expect(point.y).toBeCloseTo(303.1, 1);
+ });
+});
+
+describe('keys on the mirror', () => {
+ test('Enter submits as keyDown with its text', () => {
+ expect(keyEvents('Enter')).toEqual([
+ { type: 'keyDown', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13, text: '\r', unmodifiedText: '\r' },
+ { type: 'keyUp', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13 },
+ ]);
+ });
+
+ test('Backspace and the arrows carry their virtual key code, without which nothing edits', () => {
+ expect(keyEvents('Backspace')[0]).toMatchObject({ type: 'rawKeyDown', windowsVirtualKeyCode: 8 });
+ expect(keyEvents('ArrowDown')[0]).toMatchObject({ type: 'rawKeyDown', code: 'ArrowDown', windowsVirtualKeyCode: 40 });
+ });
+
+ test('only the named keys are keys; everything printable is text', () => {
+ expect(isMirrorKey('Tab')).toBe(true);
+ expect(isMirrorKey('a')).toBe(false);
+ expect(isMirrorKey('F5')).toBe(false);
+ });
+});
+
+describe('the address bar', () => {
+ test.each([
+ ['https://example.com/a', 'https://example.com/a'],
+ ['example.com', 'https://example.com'],
+ ['news.ycombinator.com/item?id=1', 'https://news.ycombinator.com/item?id=1'],
+ ['localhost:8080/x', 'https://localhost:8080/x'],
+ ['about:blank', 'about:blank'],
+ ['cheap flights to lisbon', 'https://www.google.com/search?q=cheap%20flights%20to%20lisbon'],
+ ])('%s', (typed, url) => {
+ expect(addressOf(typed)).toBe(url);
+ });
+});
diff --git a/src/lib/phone/mirror.ts b/src/lib/phone/mirror.ts
new file mode 100644
index 0000000..875fc8c
--- /dev/null
+++ b/src/lib/phone/mirror.ts
@@ -0,0 +1,90 @@
+export const PHONE_PORT = 'browsentic/phone';
+
+/** `Page.screencastFrame`'s metadata: the phone's visual viewport when the frame was taken. */
+export interface FrameMetadata {
+ offsetTop: number;
+ pageScaleFactor: number;
+ deviceWidth: number;
+ deviceHeight: number;
+ scrollOffsetX: number;
+ scrollOffsetY: number;
+}
+
+export type TouchType = 'touchStart' | 'touchMove' | 'touchEnd' | 'touchCancel';
+
+export const MIRROR_KEYS = ['Enter', 'Backspace', 'Delete', 'Tab', 'Escape', 'ArrowLeft', 'ArrowRight', 'ArrowUp', 'ArrowDown', 'Home', 'End'] as const;
+export type MirrorKey = (typeof MIRROR_KEYS)[number];
+
+export const isMirrorKey = (key: string): key is MirrorKey => (MIRROR_KEYS as readonly string[]).includes(key);
+
+export type MirrorCommand =
+ | { op: 'watch' }
+ | { op: 'unwatch' }
+ | { op: 'restart' }
+ | { op: 'ack'; frame: number }
+ | { op: 'touch'; type: TouchType; x: number; y: number }
+ | { op: 'wheel'; x: number; y: number; deltaX: number; deltaY: number }
+ | { op: 'text'; text: string }
+ | { op: 'key'; key: MirrorKey }
+ | { op: 'go'; url: string }
+ | { op: 'back' }
+ | { op: 'forward' }
+ | { op: 'reload' }
+ | { op: 'switchTab'; targetId: string }
+ | { op: 'newTab' }
+ | { op: 'closeTab'; targetId: string };
+
+export interface PageState {
+ targetId: string;
+ url: string;
+ title: string;
+ canGoBack: boolean;
+ canGoForward: boolean;
+}
+
+export type MirrorMessage =
+ | { kind: 'frame'; data: string; metadata: FrameMetadata; frame: number }
+ | ({ kind: 'page' } & PageState)
+ | { kind: 'error'; message: string };
+
+/**
+ * Where a point on the drawn frame lands on the phone, in the CSS px a touch event takes. The frame
+ * is the visual viewport alone, so the phone's own top bar plays no part (spike Q8).
+ */
+export function touchPoint(
+ fraction: { x: number; y: number },
+ metadata: Pick<FrameMetadata, 'deviceWidth' | 'pageScaleFactor'>,
+ image: { width: number; height: number },
+): { x: number; y: number } {
+ const width = metadata.deviceWidth / (metadata.pageScaleFactor || 1);
+ return { x: fraction.x * width, y: fraction.y * width * (image.height / image.width) };
+}
+
+/** Key events that edit only with `windowsVirtualKeyCode`, and an Enter that submits only as `keyDown` with its `text` (spike Q7). */
+const KEYS: Record<MirrorKey, { code: string; keyCode: number; text?: string }> = {
+ Enter: { code: 'Enter', keyCode: 13, text: '\r' },
+ Backspace: { code: 'Backspace', keyCode: 8 },
+ Delete: { code: 'Delete', keyCode: 46 },
+ Tab: { code: 'Tab', keyCode: 9 },
+ Escape: { code: 'Escape', keyCode: 27 },
+ ArrowLeft: { code: 'ArrowLeft', keyCode: 37 },
+ ArrowRight: { code: 'ArrowRight', keyCode: 39 },
+ ArrowUp: { code: 'ArrowUp', keyCode: 38 },
+ ArrowDown: { code: 'ArrowDown', keyCode: 40 },
+ Home: { code: 'Home', keyCode: 36 },
+ End: { code: 'End', keyCode: 35 },
+};
+
+export function keyEvents(key: MirrorKey): Record<string, unknown>[] {
+ const { code, keyCode, text } = KEYS[key];
+ const base = { key, code, windowsVirtualKeyCode: keyCode, nativeVirtualKeyCode: keyCode };
+ return [{ type: text ? 'keyDown' : 'rawKeyDown', ...base, ...(text ? { text, unmodifiedText: text } : {}) }, { type: 'keyUp', ...base }];
+}
+
+/** What the mirror's address bar was given, as a URL: a bare host gets https://, anything else is searched for. */
+export function addressOf(typed: string): string {
+ const text = typed.trim();
+ if (/^[a-z][a-z0-9+.-]*:\/\//i.test(text) || /^(about|chrome|data):/i.test(text)) return text;
+ if (/^[^\s/]+\.[^\s/]{2,}(\/\S*)?$/.test(text) || /^localhost(:\d+)?(\/\S*)?$/.test(text)) return `https://${text}`;
+ return `https://www.google.com/search?q=${encodeURIComponent(text)}`;
+}