Skip to content

An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.

feat(android): live tools on the phone: approved page code installs in the phone page's main world over DevTools and runs there, kept tools save for the phone page's site and run with / on the phone tab, and Live tool switches the code tools and the scripting skill on for a phone run as it does on desktop; running on every visit stays desktop only

Shahriar4a4e13f5005a
diff --git a/src/daemon/agent/service.test.ts b/src/daemon/agent/service.test.ts
index cb8524f..acb1e02 100644
--- a/src/daemon/agent/service.test.ts
+++ b/src/daemon/agent/service.test.ts
@@ -52,12 +52,17 @@ describe('what a phone run is offered', () => {
});
});
- test('exactly the phone’s tools, and no reserved ones, even with Live tool on', async () => {
+ test('exactly the phone’s tools with Live tool on, and no reserved ones', async () => {
const offer = await offerTo('p', { sessionId: 's1', phone: PHONE, liveTools: true });
expect(names.filter((name) => !offer!.withheld.includes(name)).sort()).toEqual([...PHONE_TOOLS].sort());
expect(offer!.reserved).toEqual([]);
});
+ test('the page-code tools only with Live tool on, the same rule as a desktop run', async () => {
+ const offer = await offerTo('q', { sessionId: 's4', phone: PHONE });
+ expect(names.filter((name) => !offer!.withheld.includes(name)).sort()).toEqual([...PHONE_TOOLS].filter((name) => !CODE_TOOLS.includes(name)).sort());
+ });
+
test('a desktop run beside it is offered what it always was', async () => {
expect(await offerTo('d', { sessionId: 's2' })).toEqual({ withheld: CODE_TOOLS, reserved: [FOCUS_SHOT_ACTION] });
});
diff --git a/src/daemon/agent/service.ts b/src/daemon/agent/service.ts
index 5f5c585..2ec2aa7 100644
--- a/src/daemon/agent/service.ts
+++ b/src/daemon/agent/service.ts
@@ -202,10 +202,14 @@ export class AgentSession {
offerFor(runId: string): { withheld: string[]; reserved: string[] } | null {
const run = this.runs.get(runId);
if (!run) return null;
- if (run.phone) return { withheld: this.deps.actionNames().filter((name) => !phoneOffers(name)), reserved: [] };
const codeListed = run.liveTools || (run.sessionId !== undefined && this.codeListed.has(run.sessionId));
+ const codeWithheld = codeListed ? [] : [INJECT_ACTION, RUN_CODE_ACTION];
+ if (run.phone) {
+ const offPhone = this.deps.actionNames().filter((name) => !phoneOffers(name));
+ return { withheld: [...new Set([...offPhone, ...codeWithheld])], reserved: [] };
+ }
return {
- withheld: codeListed ? [] : [INJECT_ACTION, RUN_CODE_ACTION],
+ withheld: codeWithheld,
// A mapping run never resumes a conversation, so its tool can come and go. A pick
// comes and goes between the messages of one, so the tool that shows it stays.
reserved: [...(run.map ? [SAVE_SITE_MAP_ACTION] : []), FOCUS_SHOT_ACTION],
diff --git a/src/daemon/agent/skills.test.ts b/src/daemon/agent/skills.test.ts
index 234f0c0..92566ac 100644
--- a/src/daemon/agent/skills.test.ts
+++ b/src/daemon/agent/skills.test.ts
@@ -167,8 +167,9 @@ describe('a run on the Android phone', () => {
expect(onPhone('@page-theming make it readable')).toMatchObject({ base: { name: PHONE_SKILL }, text: 'make it readable' });
});
- test("still gets the notes for the phone page's site, and never the scripting skill", () => {
- expect(onPhone('click sign in', { url: 'https://example.com/', liveTools: true })?.overlays.map((overlay) => overlay.name)).toEqual(['example-com']);
+ test("still gets the notes for the phone page's site, and the scripting skill only with Live tool on", () => {
+ expect(onPhone('click sign in', { url: 'https://example.com/' })?.overlays.map((overlay) => overlay.name)).toEqual(['example-com']);
+ expect(onPhone('click sign in', { url: 'https://example.com/', liveTools: true })?.overlays.map((overlay) => overlay.name)).toEqual(['example-com', SCRIPTING_SKILL]);
});
test('a desktop run is never routed to the phone skill, not even with no default', () => {
diff --git a/src/daemon/agent/skills.ts b/src/daemon/agent/skills.ts
index 440d86d..11c7e81 100644
--- a/src/daemon/agent/skills.ts
+++ b/src/daemon/agent/skills.ts
@@ -181,7 +181,7 @@ function overlaysFor(skills: Skill[], context: RunContext | undefined, forced: S
.sort((a, b) => b.match.length - a.match.length)
.map((entry) => entry.skill)
: [];
- const scripting = context?.liveTools && !context.phone
+ const scripting = context?.liveTools
? skills.filter((skill) => skill.name === SCRIPTING_SKILL && !forced.includes(skill))
: [];
return [...forced, ...matched, ...scripting];
diff --git a/src/extension/components/composer.tsx b/src/extension/components/composer.tsx
index 0104023..b54c93d 100644
--- a/src/extension/components/composer.tsx
+++ b/src/extension/components/composer.tsx
@@ -285,25 +285,23 @@ export function Composer({
>
<FileUp className="size-3.5" />
</Button>
- {!onPhone && (
- <Button
- variant={liveTools ? 'subtle' : 'ghost'}
- size="icon-sm"
- role="switch"
- aria-checked={liveTools}
- aria-label={liveTools ? 'Turn live tools off' : 'Turn live tools on'}
- title={
- liveTools
- ? 'Live tool is on — the agent may write a small script for this page and ask you to approve it. Turn it off to keep to the built-in tools.'
- : 'Live tool — let the agent write a small script for repetitive work or for something no tool covers. You review and approve the code before it runs.'
- }
- onClick={onToggleLiveTools}
- disabled={!connected}
- className={cn(liveTools ? 'text-ember' : undefined)}
- >
- <Code2 className="size-3.5" />
- </Button>
- )}
+ <Button
+ variant={liveTools ? 'subtle' : 'ghost'}
+ size="icon-sm"
+ role="switch"
+ aria-checked={liveTools}
+ aria-label={liveTools ? 'Turn live tools off' : 'Turn live tools on'}
+ title={
+ liveTools
+ ? 'Live tool is on — the agent may write a small script for this page and ask you to approve it. Turn it off to keep to the built-in tools.'
+ : 'Live tool — let the agent write a small script for repetitive work or for something no tool covers. You review and approve the code before it runs.'
+ }
+ onClick={onToggleLiveTools}
+ disabled={!connected}
+ className={cn(liveTools ? 'text-ember' : undefined)}
+ >
+ <Code2 className="size-3.5" />
+ </Button>
{!onPhone && (
<Button
variant={scheduling ? 'subtle' : 'ghost'}
diff --git a/src/extension/components/saved-tools.tsx b/src/extension/components/saved-tools.tsx
index 8a8de94..5c9dd73 100644
--- a/src/extension/components/saved-tools.tsx
+++ b/src/extension/components/saved-tools.tsx
@@ -17,11 +17,14 @@ import { displayName, slugify } from '@/lib/skills/saved-tool';
export function KeepToolPrompt({
offer,
autoRunReady,
+ everyVisit = true,
onKeep,
onDismiss,
}: {
offer: ToolOffer;
autoRunReady: boolean;
+ /** Whether running on every visit is offered: never on the phone, where nothing of the extension is in the page. */
+ everyVisit?: boolean;
onKeep: (slug: string, autoRun: boolean) => void;
onDismiss: () => void;
}) {
@@ -62,11 +65,13 @@ export function KeepToolPrompt({
runs with /{displayName({ host: offer.host, segment: offer.segment }, clean)}
</p>
- <label className="mt-2.5 flex items-center gap-2 text-[11px] text-ink-dim">
- <Switch checked={autoRun} onChange={setAutoRun} label={`Also run it on every visit to ${where}`} />
- Also run it on every visit to {where}
- </label>
- {autoRun && !autoRunReady && <AllowUserScripts />}
+ {everyVisit && (
+ <label className="mt-2.5 flex items-center gap-2 text-[11px] text-ink-dim">
+ <Switch checked={autoRun} onChange={setAutoRun} label={`Also run it on every visit to ${where}`} />
+ Also run it on every visit to {where}
+ </label>
+ )}
+ {everyVisit && autoRun && !autoRunReady && <AllowUserScripts />}
<div className="mt-2.5 flex gap-1.5">
<Button variant="ghost" size="sm" className="flex-1" onClick={onDismiss}>
@@ -96,12 +101,15 @@ function AllowUserScripts() {
export function SavedToolList({
tools,
autoRunReady,
+ everyVisit = true,
onAutoRun,
onForget,
onClose,
}: {
tools: SavedToolMeta[];
autoRunReady: boolean;
+ /** Whether each tool's every-visit switch is shown: not on the phone, where it would only govern desktop visits. */
+ everyVisit?: boolean;
onAutoRun: (id: string, on: boolean) => void;
onForget: (id: string) => void;
onClose: () => void;
@@ -123,7 +131,7 @@ export function SavedToolList({
<p className="mt-0.5 text-[11px] leading-relaxed text-ink-dim">
Removing one deletes its code from the browser and its note from the daemon.
</p>
- {!autoRunReady && tools.some((tool) => tool.autoRun) && <AllowUserScripts />}
+ {everyVisit && !autoRunReady && tools.some((tool) => tool.autoRun) && <AllowUserScripts />}
</div>
<button
type="button"
@@ -148,14 +156,16 @@ export function SavedToolList({
<div className="min-w-0 flex-1">
<p className="truncate font-mono text-[11px] text-ink">{tool.name}</p>
<p className="mt-0.5 text-[11px] leading-relaxed text-ink-dim">{tool.description}</p>
- <label className="mt-1.5 flex items-center gap-2 text-[10px] text-ink-faint">
- <Switch
- checked={tool.autoRun === true}
- onChange={(on) => onAutoRun(tool.id, on)}
- label={`Run ${tool.name} on every visit`}
- />
- Every visit
- </label>
+ {everyVisit && (
+ <label className="mt-1.5 flex items-center gap-2 text-[10px] text-ink-faint">
+ <Switch
+ checked={tool.autoRun === true}
+ onChange={(on) => onAutoRun(tool.id, on)}
+ label={`Run ${tool.name} on every visit`}
+ />
+ Every visit
+ </label>
+ )}
</div>
<button
type="button"
diff --git a/src/extension/entrypoints/sidepanel/App.tsx b/src/extension/entrypoints/sidepanel/App.tsx
index e47205e..120197e 100644
--- a/src/extension/entrypoints/sidepanel/App.tsx
+++ b/src/extension/entrypoints/sidepanel/App.tsx
@@ -83,7 +83,8 @@ export default function App() {
const phone = useMirroredPhone();
const onPhone = phone !== null;
const handsFree = handsFreeSupported && !onPhone;
- const blockedHere = useBlockedPattern(phone ? (currentTarget(phone)?.url ?? '') : tabUrl);
+ const pageUrl = phone ? (currentTarget(phone)?.url ?? '') : tabUrl;
+ const blockedHere = useBlockedPattern(pageUrl);
const [attachError, setAttachError] = useState<string | null>(null);
const [attachedSkill, setAttachedSkill] = useState<AttachedSkill | null>(null);
const [focus, setFocus] = useState<FocusedElement | null>(null);
@@ -480,10 +481,11 @@ export default function App() {
{starShown && <StarBadge onStar={star.star} onClose={star.close} />}
</div>
- {showTools && !onPhone && (
+ {showTools && (
<SavedToolList
tools={run.tools}
autoRunReady={run.autoRunReady}
+ everyVisit={!onPhone}
onAutoRun={run.setAutoRun}
onForget={run.forgetTool}
onClose={() => setShowTools(false)}
@@ -493,10 +495,11 @@ export default function App() {
{tab === 'chat' && (
<footer className="shrink-0 border-t border-line p-3">
{blockedHere && <BlockedSiteNote pattern={blockedHere} />}
- {run.toolOffer && !onPhone && (
+ {run.toolOffer && (
<KeepToolPrompt
offer={run.toolOffer}
autoRunReady={run.autoRunReady}
+ everyVisit={!onPhone}
onKeep={run.keepTool}
onDismiss={run.dismissTool}
/>
@@ -509,7 +512,7 @@ export default function App() {
files={files}
attachError={attachError}
catalog={daemon?.skillCatalog}
- tabUrl={tabUrl}
+ tabUrl={pageUrl}
attachedSkill={attachedSkill}
focus={focus}
picking={picking}
@@ -534,7 +537,7 @@ export default function App() {
open('chat');
run.send(command);
}}
- tools={onPhone ? [] : run.tools}
+ tools={run.tools}
onRunTool={(id) => {
open('chat');
run.runTool(id);
diff --git a/src/lib/bridge/code-toolkit.ts b/src/lib/bridge/code-toolkit.ts
index 0e4f289..027e6eb 100644
--- a/src/lib/bridge/code-toolkit.ts
+++ b/src/lib/bridge/code-toolkit.ts
@@ -67,29 +67,53 @@ export function onToolOffer(listener: (offer: ToolOffer) => void): void {
type ToolkitMap = Record<string, StoredToolkit>;
-interface EvaluateReply {
+export interface EvaluateReply {
result?: { value?: unknown };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}
+/**
+ * Where a toolkit lives: a desktop tab, or the tab in front on the phone. `key` names its record,
+ * `evaluate` runs the installer in the page's main world through a debugger, and `call` runs
+ * `page.runCode`'s page side, which reaches the toolkit by DOM events from its own world.
+ */
+export interface ToolkitPlace {
+ key: string;
+ evaluate(source: string): Promise<ActionResult<EvaluateReply>>;
+ call(input: unknown): Promise<ActionResult>;
+ /** The blocked-sites refusal for the page there now, if any. */
+ refusal(): Promise<ActionResult | null>;
+}
+
+export const tabPlace = (tabId: number): ToolkitPlace => ({
+ key: String(tabId),
+ evaluate: (source) =>
+ withDebugger(tabId, FIREFOX_HINT, async (session) => {
+ const context = await mainWorldOf(session);
+ return context ? success(await evaluate(context, source)) : failure('FRAME_UNREACHABLE', FRAME_CONTEXT_HINT);
+ }) as Promise<ActionResult<EvaluateReply>>,
+ call: (input) => invokeInTab(tabId, runCode.name, input),
+ refusal: () => refusalForTab(tabId),
+});
+
async function readToolkits(): Promise<ToolkitMap> {
const stored = await browser.storage.session.get(TOOLKITS_KEY);
return (stored[TOOLKITS_KEY] as ToolkitMap | undefined) ?? {};
}
-async function writeToolkit(tabId: number, toolkit: StoredToolkit | null): Promise<void> {
+async function writeToolkit(key: string, toolkit: StoredToolkit | null): Promise<void> {
const map = await readToolkits();
- if (toolkit) map[String(tabId)] = toolkit;
- else delete map[String(tabId)];
+ if (toolkit) map[key] = toolkit;
+ else delete map[key];
await browser.storage.session.set({ [TOOLKITS_KEY]: map });
}
-async function forgetToolkit(tabId: number): Promise<void> {
- await writeToolkit(tabId, null);
+async function forgetToolkit(key: string): Promise<void> {
+ await writeToolkit(key, null);
}
export function serveCodeToolkits(): void {
- browser.tabs.onRemoved.addListener((tabId) => void forgetToolkit(tabId));
+ browser.tabs.onRemoved.addListener((tabId) => void forgetToolkit(String(tabId)));
}
export function originOf(url: string | undefined): string | undefined {
@@ -102,7 +126,7 @@ export function originOf(url: string | undefined): string | undefined {
}
}
-export async function installToolkit(tabId: number, url: string | undefined, input: unknown): Promise<ActionResult> {
+export async function installToolkit(place: ToolkitPlace, url: string | undefined, input: unknown): Promise<ActionResult> {
const parsed = injectCode.input.safeParse(input ?? {});
if (!parsed.success) return failure('INVALID_INPUT', z.prettifyError(parsed.error));
@@ -121,11 +145,11 @@ export async function installToolkit(tabId: number, url: string | undefined, inp
installedAt: Date.now(),
};
- const installed = await evaluateInstaller(tabId, toolkit);
+ const installed = await evaluateInstaller(place, toolkit);
if (!installed.ok) return installed;
toolkit.entries = installed.data as ToolkitEntry[];
- await writeToolkit(tabId, toolkit);
+ await writeToolkit(place.key, toolkit);
const summary = {
toolkitId: toolkit.id,
@@ -134,7 +158,7 @@ export async function installToolkit(tabId: number, url: string | undefined, inp
functions: toolkit.entries.map((entry) => entry.name),
};
- const called = call ? await runToolkit(tabId, url, { function: call.function, args: call.args }) : null;
+ const called = call ? await runToolkit(place, url, { function: call.function, args: call.args }) : null;
offerToKeep(toolkit, url, call?.function);
if (!called) return success(summary);
@@ -175,8 +199,8 @@ function offerToKeep(toolkit: StoredToolkit, url: string | undefined, called: st
* loop at all — which is also what keeps it out of reach of an MCP client. Blocked sites
* are the exception: they bind the user's own tools too.
*/
-export async function runSavedTool(tabId: number, url: string | undefined, toolId: string): Promise<ActionResult> {
- const refused = await refusalForTab(tabId);
+export async function runSavedTool(place: ToolkitPlace, url: string | undefined, toolId: string): Promise<ActionResult> {
+ const refused = await place.refusal();
if (refused) return refused;
const tool = await getSavedTool(toolId);
if (!tool) return failure('UNKNOWN_TOOL', 'That tool is no longer saved.');
@@ -195,12 +219,12 @@ export async function runSavedTool(tabId: number, url: string | undefined, toolI
entries: [],
installedAt: Date.now(),
};
- const installed = await evaluateInstaller(tabId, staged);
+ const installed = await evaluateInstaller(place, staged);
if (!installed.ok) return installed;
staged.entries = installed.data as ToolkitEntry[];
- await writeToolkit(tabId, staged);
- return invokeInTab(tabId, runCode.name, { function: tool.fn, args: [], timeoutMs: 10_000 });
+ await writeToolkit(place.key, staged);
+ return place.call({ function: tool.fn, args: [], timeoutMs: 10_000 });
}
/**
@@ -208,16 +232,16 @@ export async function runSavedTool(tabId: number, url: string | undefined, toolI
* carried. Saving reads it from here rather than from the panel, so the code makes one
* fewer hop and the panel never has to hold it to hand it back.
*/
-export async function toolkitCode(tabId: number, toolkitId: string): Promise<string | null> {
- const toolkit = (await readToolkits())[String(tabId)];
+export async function toolkitCode(place: Pick<ToolkitPlace, 'key'>, toolkitId: string): Promise<string | null> {
+ const toolkit = (await readToolkits())[place.key];
return toolkit && toolkit.id === toolkitId ? toolkit.code : null;
}
-export async function runToolkit(tabId: number, url: string | undefined, input: unknown): Promise<ActionResult> {
+export async function runToolkit(place: ToolkitPlace, url: string | undefined, input: unknown): Promise<ActionResult> {
const parsed = runCode.input.safeParse(input ?? {});
if (!parsed.success) return failure('INVALID_INPUT', z.prettifyError(parsed.error));
- const toolkit = (await readToolkits())[String(tabId)];
+ const toolkit = (await readToolkits())[place.key];
if (!toolkit) {
return failure(
TOOLKIT_MISSING,
@@ -227,7 +251,7 @@ export async function runToolkit(tabId: number, url: string | undefined, input:
const origin = originOf(url);
if (origin !== toolkit.origin) {
- await forgetToolkit(tabId);
+ await forgetToolkit(place.key);
return failure(
'TOOLKIT_SCOPE',
`That toolkit was approved for ${toolkit.origin}, but this tab is on ${origin ?? 'another page'}. Install it again here if the job continues.`,
@@ -241,29 +265,26 @@ export async function runToolkit(tabId: number, url: string | undefined, input:
);
}
- const called = await invokeInTab(tabId, runCode.name, parsed.data);
+ const called = await place.call(parsed.data);
if (called.ok || called.error.code !== TOOLKIT_MISSING) return called;
- const reinstalled = await evaluateInstaller(tabId, toolkit);
+ const reinstalled = await evaluateInstaller(place, toolkit);
if (!reinstalled.ok) return reinstalled;
- return invokeInTab(tabId, runCode.name, parsed.data);
+ return place.call(parsed.data);
}
-function evaluateInstaller(tabId: number, toolkit: StoredToolkit): Promise<ActionResult> {
- return withDebugger(tabId, FIREFOX_HINT, async (session) => {
- const context = await mainWorldOf(session);
- if (!context) return failure('FRAME_UNREACHABLE', FRAME_CONTEXT_HINT);
- const reply = await evaluate(context, installerSource(toolkit.id, toolkit.code));
- const thrown = reply.exceptionDetails;
- if (thrown) {
- return failure('CODE_ERROR', `The code failed while installing: ${describeThrow(thrown)}`);
- }
- const entries = reply.result?.value;
- if (!Array.isArray(entries)) {
- return failure('CODE_ERROR', 'The code installed but reported no functions.');
- }
- return success(entries as ToolkitEntry[]);
- });
+async function evaluateInstaller(place: ToolkitPlace, toolkit: StoredToolkit): Promise<ActionResult> {
+ const evaluated = await place.evaluate(installerSource(toolkit.id, toolkit.code));
+ if (!evaluated.ok) return evaluated;
+ const thrown = evaluated.data.exceptionDetails;
+ if (thrown) {
+ return failure('CODE_ERROR', `The code failed while installing: ${describeThrow(thrown)}`);
+ }
+ const entries = evaluated.data.result?.value;
+ if (!Array.isArray(entries)) {
+ return failure('CODE_ERROR', 'The code installed but reported no functions.');
+ }
+ return success(entries as ToolkitEntry[]);
}
function evaluate({ session, contextId }: FrameContext, expression: string): Promise<EvaluateReply> {
diff --git a/src/lib/bridge/invoke.ts b/src/lib/bridge/invoke.ts
index 38a8a33..5fa33da 100644
--- a/src/lib/bridge/invoke.ts
+++ b/src/lib/bridge/invoke.ts
@@ -39,7 +39,7 @@ import { failure, success, type ActionResult } from '@/lib/actions/protocol';
import { EXPIRED_MESSAGE, REFUSED_MESSAGE } from '@/lib/secrets';
import { releaseForAction, sealForPage } from '@/lib/bridge/secret-vault';
import { findCaptchaInTab, solveCaptchaInTab } from '@/lib/bridge/captcha';
-import { installToolkit, runToolkit } from '@/lib/bridge/code-toolkit';
+import { installToolkit, runToolkit, tabPlace } from '@/lib/bridge/code-toolkit';
import { callSiteToolInTab, listSiteToolsInTab, pageInfoWithSiteTools } from '@/lib/bridge/site-tools';
import { captureFromPage } from '@/lib/bridge/downloads';
import { focusedUrl, forgetFrameFocus, framePath, TOP_FRAME } from '@/lib/bridge/frame-focus';
@@ -262,8 +262,8 @@ async function routeInTab(
if (action === pickElement.name) return pickInTab({ id: tab.id, windowId: tab.windowId }, input);
if (action === attachFile.name) return attachStoredFile(tab.id, input);
if (action === captureDownload.name) return captureFromPage(tab.id, input);
- if (action === injectCode.name) return installToolkit(tab.id, await focusedUrl(tab.id, tab.url), input);
- if (action === runCode.name) return runToolkit(tab.id, await focusedUrl(tab.id, tab.url), input);
+ if (action === injectCode.name) return installToolkit(tabPlace(tab.id), await focusedUrl(tab.id, tab.url), input);
+ if (action === runCode.name) return runToolkit(tabPlace(tab.id), await focusedUrl(tab.id, tab.url), input);
if (action === getPageInfo.name) return pageInfoWithSiteTools(tab.id, tab.url, input);
if (action === listSiteTools.name) return listSiteToolsInTab(tab.id, await focusedUrl(tab.id, tab.url));
if (action === callSiteTool.name) return callSiteToolInTab(tab.id, await focusedUrl(tab.id, tab.url), input);
diff --git a/src/lib/bridge/phone-backend.ts b/src/lib/bridge/phone-backend.ts
index 4ad2844..3a4f8d8 100644
--- a/src/lib/bridge/phone-backend.ts
+++ b/src/lib/bridge/phone-backend.ts
@@ -7,8 +7,10 @@ import { typeText } from '@/lib/actions/page/type-text';
import { failure, success, type ActionResult } from '@/lib/actions/protocol';
import { PHONE_API } from '@/lib/phone/page-api';
import { drag, keyPress, onScreen, tap, typing, wheel, type InputStep, type Point, type Viewport } from '@/lib/phone/touch';
+import type { EvaluateReply, ToolkitPlace } from './code-toolkit';
import { currentTarget, type PhoneSession } from './phone';
import { attachedSession } from './phone-mirror';
+import { refusalFor } from './site-guard';
import { sendCdp } from './socket';
const BUNDLE = '/phone-page.js';
@@ -324,6 +326,24 @@ const TOOLS: Record<string, PhoneTool> = {
'page.submitForm': (here, input, action) => thenLoaded(here, action, input),
};
+const TOP_DOCUMENT_ONLY = 'Page code on the phone installs in the top document. Call page.switchFrame with no arguments first.';
+
+/** The phone's front tab as the home of approved page code: installed in its top document's main world, called from the bundle's world. */
+export async function phoneToolkitPlace(phone: PhoneSession): Promise<ToolkitPlace | null> {
+ const here = await hereOn(phone);
+ if (!here) return null;
+ return {
+ key: `phone:${here.targetId}`,
+ evaluate: async (source) => {
+ if ((await focusedFrame(here)).path.length) return failure('FRAME_UNREACHABLE', TOP_DOCUMENT_ONLY);
+ const evaluated = await here.cdp('Runtime.evaluate', { expression: source, returnByValue: true, awaitPromise: true });
+ return evaluated.ok ? success(evaluated.data as EvaluateReply) : evaluated;
+ },
+ call: (input) => inPage(here, 'page.runCode', input),
+ refusal: () => refusalFor(here.url),
+ };
+}
+
/** Every page-side tool on the phone: its own code in the page where it can be, real input where the page cannot fake it. */
export async function pageSideOnPhone(action: string, input: unknown, phone: PhoneSession): Promise<ActionResult> {
const here = await hereOn(phone);
diff --git a/src/lib/bridge/phone-invoke.test.ts b/src/lib/bridge/phone-invoke.test.ts
index 2b3cbfa..ea9b8b3 100644
--- a/src/lib/bridge/phone-invoke.test.ts
+++ b/src/lib/bridge/phone-invoke.test.ts
@@ -113,9 +113,9 @@ describe('a tool on the phone', () => {
const route = async () => (await phoneRoute(mirrorTabId, undefined))!;
test('a tool off the list is refused with where it does work', async () => {
- expect(await invokeOnPhone('page.injectCode', {}, await route())).toEqual({
+ expect(await invokeOnPhone('page.captureDownload', {}, await route())).toEqual({
ok: false,
- error: { code: 'NOT_ON_PHONE', message: 'page.injectCode isn’t available on the phone. It works in desktop tabs.' },
+ error: { code: 'NOT_ON_PHONE', message: 'page.captureDownload isn’t available on the phone. It works in desktop tabs.' },
});
});
diff --git a/src/lib/bridge/phone-invoke.ts b/src/lib/bridge/phone-invoke.ts
index af7a39e..b4d701c 100644
--- a/src/lib/bridge/phone-invoke.ts
+++ b/src/lib/bridge/phone-invoke.ts
@@ -2,14 +2,17 @@ import { browser } from 'wxt/browser';
import { z } from 'zod';
import { ActionError } from '@/lib/actions/core';
import { closeTab } from '@/lib/actions/page/close-tab';
+import { injectCode } from '@/lib/actions/page/inject-code';
import { navigate, resolveNavigation, type NavigateInput } from '@/lib/actions/page/navigate';
import { openTab } from '@/lib/actions/page/open-tab';
+import { runCode } from '@/lib/actions/page/run-code';
import { switchTab } from '@/lib/actions/page/switch-tab';
import { failure, success, type ActionResult } from '@/lib/actions/protocol';
import { EXPIRED_MESSAGE, REFUSED_MESSAGE } from '@/lib/secrets';
import { notOnPhone, phoneOffers } from '@/lib/phone/features';
import { currentTarget, isLive, readPhone, targetForNumber, updatePhone, numbered, type PhoneSession } from './phone';
-import { hereOn, pageSideOnPhone, waitForLoad } from './phone-backend';
+import { installToolkit, runToolkit, tabPlace, type ToolkitPlace } from './code-toolkit';
+import { hereOn, pageSideOnPhone, phoneToolkitPlace, waitForLoad } from './phone-backend';
import { attachedSession, bringToFront, followTarget } from './phone-mirror';
import { releaseForAction, sealForPage } from './secret-vault';
import { refusalFor } from './site-guard';
@@ -71,13 +74,31 @@ function destinationsOf(action: string, input: unknown, base?: string): (string
}
}
+const withToolkit =
+ (use: typeof installToolkit): PhoneBackend =>
+ async (_action, input, phone) => {
+ const place = await phoneToolkitPlace(phone);
+ return place ? use(place, currentTarget(phone)?.url, input) : failure('PHONE_GONE', GONE);
+ };
+
const TAB_TOOLS: Record<string, PhoneBackend> = {
[navigate.name]: navigateOnPhone,
[openTab.name]: openPhoneTab,
[switchTab.name]: switchPhoneTab,
[closeTab.name]: closePhoneTab,
+ [injectCode.name]: withToolkit(installToolkit),
+ [runCode.name]: withToolkit(runToolkit),
};
+/** Where `/` runs a saved tool and a kept toolkit is read from: the phone's front tab when the panel is on the phone tab. Null once the phone is gone. */
+export async function toolkitPlaceFor(tabId: number, url: string | undefined): Promise<{ place: ToolkitPlace; url: string | undefined } | null> {
+ const phone = await phoneRoute(tabId, undefined);
+ if (phone === null) return { place: tabPlace(tabId), url };
+ if (phone === 'gone') return null;
+ const place = await phoneToolkitPlace(phone);
+ return place && { place, url: currentTarget(phone)?.url };
+}
+
export async function inCurrentTab(phone: PhoneSession, method: string, params?: Record<string, unknown>) {
const target = currentTarget(phone);
const sessionId = target && (await attachedSession(phone.serial, target.targetId));
diff --git a/src/lib/bridge/phone-toolkit.test.ts b/src/lib/bridge/phone-toolkit.test.ts
new file mode 100644
index 0000000..ee66d83
--- /dev/null
+++ b/src/lib/bridge/phone-toolkit.test.ts
@@ -0,0 +1,89 @@
+import { beforeEach, describe, expect, test, vi } from 'vitest';
+import { fakeBrowser } from 'wxt/testing';
+import { success } from '@/lib/actions/protocol';
+
+const SERIAL = 'emulator-5554';
+const FRONT = 'A'.repeat(32);
+
+const phone = vi.hoisted(() => ({
+ evaluated: [] as { expression: string; contextId?: unknown }[],
+}));
+
+vi.mock('./socket', () => ({
+ DAEMON_STATE_KEY: 'browsentic/daemon',
+ sendCdp: async (_serial: string, method: string, params?: Record<string, unknown>) => {
+ if (method === 'Page.getFrameTree') return success({ frameTree: { frame: { id: 'TOP' } } });
+ if (method === 'Page.createIsolatedWorld') return success({ executionContextId: 7 });
+ if (method !== 'Runtime.evaluate') return success({});
+ const expression = String(params?.expression);
+ phone.evaluated.push({ expression, contextId: params?.contextId });
+ if (expression.includes('__browsenticToolkit')) return success({ result: { value: [{ name: 'addTag', arity: 1 }] } });
+ if (expression.startsWith('__browsenticPhone.dispatch("page.runCode"')) return success({ result: { value: { ok: true, data: { function: 'addTag', returned: 'tagged' } } } });
+ return success({});
+ },
+ closePhone: () => undefined,
+ openPhone: async () => success({}),
+ launchPhoneChrome: async () => success({}),
+ onPhoneClosed: () => undefined,
+ onDaemonClosed: () => undefined,
+}));
+
+vi.mock('./phone-mirror', () => ({
+ attachedSession: async (_serial: string, targetId: string) => `session-${targetId}`,
+ bringToFront: async () => undefined,
+ followTarget: async () => undefined,
+}));
+
+const { invokeOnPhone, toolkitPlaceFor } = await import('./phone-invoke');
+const { readPhone } = await import('./phone');
+
+let mirrorTabId: number;
+
+const phoneAt = async (url: string) => {
+ await fakeBrowser.storage.session.set({
+ 'browsentic/phone': { serial: SERIAL, mirrorTabId, windowId: 1, openedAt: 0, activeTargetId: FRONT, targets: [{ targetId: FRONT, url, title: 'Tags' }] },
+ });
+ return (await readPhone())!;
+};
+
+const install = { purpose: 'Tag every row', code: 'tools.addTag = (name) => name;' };
+const toolkits = async () => (await fakeBrowser.storage.session.get('browsentic/codeToolkits'))['browsentic/codeToolkits'] as Record<string, { origin: string }>;
+
+beforeEach(async () => {
+ fakeBrowser.reset();
+ phone.evaluated.length = 0;
+ vi.stubGlobal('fetch', async () => new Response('/* the bundle */'));
+ mirrorTabId = (await fakeBrowser.tabs.create({ url: 'chrome-extension://abc/phone.html', active: true })).id!;
+});
+
+describe('live tools on the phone', () => {
+ test('approved code installs in the phone page’s main world, and is recorded for the phone’s tab and site', async () => {
+ const installed = await invokeOnPhone('page.injectCode', install, await phoneAt('https://tags.example/list'));
+ expect(installed).toMatchObject({ ok: true, data: { origin: 'https://tags.example', functions: ['addTag'] } });
+ expect(phone.evaluated.find(({ expression }) => expression.includes('__browsenticToolkit'))?.contextId).toBeUndefined();
+ expect(Object.keys(await toolkits())).toEqual([`phone:${FRONT}`]);
+ });
+
+ test('a call runs runCode’s own page side in the bundle’s world, which reaches the toolkit by DOM events', async () => {
+ const at = await phoneAt('https://tags.example/list');
+ await invokeOnPhone('page.injectCode', install, at);
+ const called = await invokeOnPhone('page.runCode', { function: 'addTag', args: ['urgent'] }, at);
+ expect(called).toEqual({ ok: true, data: { function: 'addTag', returned: 'tagged' } });
+ expect(phone.evaluated.find(({ expression }) => expression.startsWith('__browsenticPhone.dispatch("page.runCode"'))?.contextId).toBe(7);
+ });
+
+ test('a call after the phone went to another site is refused, as on desktop', async () => {
+ await invokeOnPhone('page.injectCode', install, await phoneAt('https://tags.example/list'));
+ const elsewhere = await invokeOnPhone('page.runCode', { function: 'addTag', args: ['x'] }, await phoneAt('https://other.example/'));
+ expect(elsewhere).toMatchObject({ ok: false, error: { code: 'TOOLKIT_SCOPE' } });
+ });
+
+ test('the panel on the phone tab saves and runs tools on the phone’s page, and on a desktop tab on that tab', async () => {
+ await phoneAt('https://tags.example/list');
+ const desktopTabId = (await fakeBrowser.tabs.create({ url: 'https://desk.example/', active: false })).id!;
+ const onPhone = await toolkitPlaceFor(mirrorTabId, '');
+ const onDesktop = await toolkitPlaceFor(desktopTabId, 'https://desk.example/');
+ expect([onPhone?.place.key, onPhone?.url]).toEqual([`phone:${FRONT}`, 'https://tags.example/list']);
+ expect([onDesktop?.place.key, onDesktop?.url]).toEqual([String(desktopTabId), 'https://desk.example/']);
+ });
+});
diff --git a/src/lib/bridge/run-port.ts b/src/lib/bridge/run-port.ts
index 96ad28e..d8795c0 100644
--- a/src/lib/bridge/run-port.ts
+++ b/src/lib/bridge/run-port.ts
@@ -18,6 +18,7 @@ import type { SiteMapDraft } from '@/lib/skills/site-map';
import { navigate } from '@/lib/actions/page/navigate';
import { autoRunReady, syncAutoRuns } from './auto-run';
import { onToolOffer, runSavedTool, toolkitCode, type ToolOffer } from './code-toolkit';
+import { toolkitPlaceFor } from './phone-invoke';
import { CONTEXT_COMMAND, isContextCommand, type ContextBreakdown } from './commands';
import { listSavedTools, withoutCode, type SavedToolMeta } from './saved-tools';
import { dropTool, keepTool, switchAutoRun } from './tool-registry';
@@ -370,7 +371,8 @@ function handle(command: RunCommand): void {
case 'keepTool':
void serialized(async () => {
const offer = pendingOffers.get(command.toolkitId);
- const code = offer ? await toolkitCode(command.tabId, command.toolkitId) : null;
+ const at = offer ? await toolkitPlaceFor(command.tabId, undefined) : null;
+ const code = at ? await toolkitCode(at.place, command.toolkitId) : null;
if (offer && code) {
await keepTool({ offer, code, slug: command.slug, autoRun: command.autoRun }).catch(() => undefined);
}
@@ -397,7 +399,8 @@ function handle(command: RunCommand): void {
return;
case 'runTool':
void serialized(async () => {
- const result = await runSavedTool(command.tab.tabId, command.tab.url, command.id);
+ const at = await toolkitPlaceFor(command.tab.tabId, command.tab.url);
+ const result = at ? await runSavedTool(at.place, at.url, command.id) : failure('PHONE_GONE', 'The phone disconnected. Reconnect it and switch Android on again.');
broadcast({ op: 'item', item: toolRunItem(command.id, result) });
});
return;
@@ -621,7 +624,7 @@ async function startTurn(
agentSessionId: session.agentSessionId,
agentSkillId,
focus: hidden ? undefined : focus,
- liveTools: session.phone ? undefined : liveTools,
+ liveTools,
files: await attachedFiles(session),
recordings: session.phone ? undefined : await attachedRecordings(),
task,
diff --git a/src/lib/phone/features.ts b/src/lib/phone/features.ts
index b14ee74..7cf08bf 100644
--- a/src/lib/phone/features.ts
+++ b/src/lib/phone/features.ts
@@ -27,6 +27,8 @@ export const PHONE_TOOLS: ReadonlySet<string> = new Set([
'page.switchTab',
'page.closeTab',
'page.switchFrame',
+ 'page.injectCode',
+ 'page.runCode',
'page.startTimer',
'page.timerStatus',
'page.stopTimer',