Skip to content

armgit / axuse

Public Imported from GitHub
0

Accessibility-first browser & computer use for AI agents — one MCP server for every harness (Codex, Claude Code, OpenCode, Antigravity). Instruments the DOM/accessibility tree via CDP instead of screenshot loops: diffed snapshots, stable element refs, trusted input, typed errors. Zero dependencies

Bug fixes

soloratechadb4e89e5b1b
diff --git a/src/core/actuate.js b/src/core/actuate.js
index fe1186f..e48cd8e 100644
--- a/src/core/actuate.js
+++ b/src/core/actuate.js
@@ -473,6 +473,28 @@ export function settleExpression(budgetMs, debounceMs) {
})()`;
}
+/** In-page wait for the FIRST DOM mutation (then a short quiet window), bounded
+ * by maxMs. Resolves true when the page changed, false when it stayed still.
+ * Attached after the action, so it only ever sees LATE changes (a delayed
+ * render, a debounced handler) — the case a 150 ms settle reports as "no
+ * change" while the page advances a moment later. */
+export function mutationWaitExpression(maxMs, quietMs) {
+ return `(async () => {
+ const mo = new MutationObserver(() => { changed = true; last = Date.now(); });
+ let changed = false, last = 0;
+ mo.observe(document, { subtree: true, childList: true, attributes: true, characterData: true });
+ const t0 = Date.now();
+ try {
+ for (;;) {
+ const now = Date.now();
+ if (changed && now - last >= ${quietMs}) return true;
+ if (now - t0 >= ${maxMs}) return changed;
+ await new Promise((r) => setTimeout(r, 30));
+ }
+ } finally { mo.disconnect(); }
+ })()`;
+}
+
/** Bounded in-page wait-for: expression or selector, single round trip. */
export function waitForExpression(expression, timeoutMs) {
return `(async () => {
diff --git a/src/core/controller.js b/src/core/controller.js
index 7790e80..719561e 100644
--- a/src/core/controller.js
+++ b/src/core/controller.js
@@ -20,7 +20,7 @@ import {
click as doClick, hover as doHover, drag as doDrag, wheel as doWheel,
insertText, humanType, pressCombo, platformShortcut, planPointerJitter, selectAllCombo,
refResolveExpression, selectOptionExpression, elementInfoExpression, fieldKindExpression, findOptionExpression,
- sliderValueExpression, settleExpression, waitForExpression, MOD,
+ sliderValueExpression, settleExpression, mutationWaitExpression, waitForExpression, MOD,
} from "./actuate.js";
const URL_SCHEMES = /^(https?|file|about|data|blob):/i;
@@ -65,6 +65,18 @@ const RECORDED_EVENTS = new Set([
"Log.entryAdded", // error level only, filtered below
]);
+/** Request types whose completion usually re-renders the page. Images, CSS,
+ * fonts, media, sockets and beacons are not worth waiting for. */
+const NET_TRACKED = new Set(["XHR", "Fetch", "Document"]);
+/** A request in flight longer than this is a long-poll/stream, not a pending render. */
+const NET_LONGPOLL_MS = 3000;
+/** Upper bound on the post-action wait for page-triggered requests. */
+const netSettleCapMs = () => Math.max(0, Number(process.env.AXUSE_NET_SETTLE_MS ?? 2000));
+const netSettleEnabled = () => !/^(0|false|off|no)$/i.test(String(process.env.AXUSE_NET_SETTLE ?? "").trim()) && netSettleCapMs() > 0;
+/** How long an action that showed NO change keeps watching for a late one. */
+const lateChangeMs = () => Math.max(0, Number(process.env.AXUSE_LATE_CHANGE_MS ?? 450));
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
export class Controller {
constructor() {
this.browsers = new Map(); // browser_id → {handle, tabs:Set, key}
@@ -262,6 +274,7 @@ export class Controller {
clipped,
refCount: entries.length,
frameCount: frames.length,
+ items: res.items ?? null, // per main-frame line: id of its enclosing item (find mode)
text,
};
}
@@ -330,13 +343,24 @@ export class Controller {
}
/** Post-action auto-observation. Never masks a successful action. */
- async autoObserve(tab, settleMs) {
+ async autoObserve(tab, settleMs, { expectChange = false } = {}) {
try {
if (tab.dialog) return this.dialogNote(tab);
- if (settleMs !== undefined && settleMs > 0) await this.raceDialog(tab, this.settle(tab, 4000, settleMs));
+ if (settleMs !== undefined && settleMs > 0) {
+ await this.raceDialog(tab, this.settle(tab, 4000, settleMs));
+ await this.raceDialog(tab, this.awaitNetworkIdle(tab));
+ }
if (tab.dialog) return this.dialogNote(tab);
- const obs = await this.raceDialog(tab, this.observation(tab, { mode: "diff" }));
+ let obs = await this.raceDialog(tab, this.observation(tab, { mode: "diff" }));
if (tab.dialog) return this.dialogNote(tab);
+ // "No change" after a click is usually "not yet": a delayed render or a
+ // debounced request. Look once more before telling the agent nothing happened.
+ if (expectChange && typeof obs?.text === "string" && obs.text.startsWith("no changes detected") && lateChangeMs() > 0) {
+ if (await this.raceDialog(tab, this.awaitLateChange(tab, lateChangeMs()))) {
+ obs = await this.raceDialog(tab, this.observation(tab, { mode: "diff" }));
+ if (tab.dialog) return this.dialogNote(tab);
+ }
+ }
return obs;
} catch (e) {
return { skipped: e.code ?? "ERROR", hint: "Call ax_observe." };
@@ -406,7 +430,26 @@ export class Controller {
const tab = id ? this.tabs.get(id) : null;
if (tab) activated = await this.ensureVisible(tab).catch(() => null);
}
- const result = await fn.call(this, args);
+ let result;
+ try {
+ result = await fn.call(this, args);
+ } catch (e) {
+ // A stale ref/state means the page moved on (re-render, step change).
+ // Hand back the CURRENT page in the same response so the agent does
+ // not spend a turn on ax_observe just to find out where it is.
+ if (e instanceof AxError && (e.code === Codes.REF_NOT_FOUND || e.code === Codes.STALE_STATE) && e.retry === "reobserve" && !e.facts?.fresh_observation) {
+ const id = args.tab_id ?? this.activeTabId;
+ const tab = id ? this.tabs.get(id) : null;
+ if (tab && !tab.closed && !tab.dialog) {
+ try {
+ const obs = await this.observation(tab, { mode: "full" });
+ e.facts = { ...(e.facts ?? {}), fresh_observation: obs.text, state_id: obs.state_id };
+ e.hint = "The page changed since your last observation. Its current state is included below: act on these refs; no ax_observe needed.";
+ } catch { /* keep the original error */ }
+ }
+ }
+ throw e;
+ }
const meta = { elapsed_ms: Date.now() - started, ...(activated ? { tab_activated: true } : {}) };
if (result && typeof result === "object") {
if (entry.observed === "full") meta.observed = "full";
@@ -619,6 +662,7 @@ export class Controller {
browser.tabs.add(tabId);
this.activeTabId = tabId;
this.attachEventRouting(tab);
+ void this.enableNetTracking(tab);
this.startReaper();
return tab;
}
@@ -835,12 +879,78 @@ export class Controller {
this.recordEvent(tab, "debugger.detached", { reason: msg.params?.reason ?? null });
return;
}
+ if (msg.sessionId === tab.sessionId && tab.net && msg.method?.startsWith("Network.")) {
+ this.trackNet(tab, msg.method, msg.params);
+ return;
+ }
if (msg.sessionId !== tab.sessionId || !RECORDED_EVENTS.has(msg.method)) return;
this.recordEvent(tab, msg.method, msg.params);
});
tab.offEventRouting = off;
}
+ /** In-flight XHR/fetch/document tracking for post-action settling. Passive:
+ * nothing is injected into the page, only CDP Network events are counted.
+ * AXUSE_NET_SETTLE=0 turns it off (then settling is timing-only). */
+ async enableNetTracking(tab) {
+ if (!netSettleEnabled()) return;
+ tab.net = { inflight: new Map(), seq: 0 };
+ try {
+ await tab.browser.handle.conn.send("Network.enable",
+ { maxTotalBufferSize: 1 << 20, maxResourceBufferSize: 1 << 18 }, tab.sessionId);
+ } catch { tab.net = null; }
+ }
+
+ trackNet(tab, method, params) {
+ const net = tab.net;
+ if (method === "Network.requestWillBeSent") {
+ if (NET_TRACKED.has(params?.type)) { net.inflight.set(params.requestId, Date.now()); net.seq++; }
+ } else if (method === "Network.loadingFinished" || method === "Network.loadingFailed") {
+ net.inflight.delete(params?.requestId);
+ }
+ }
+
+ /** True while a page-triggered request that is not a long-poll is pending. */
+ netBusy(tab) {
+ const net = tab.net;
+ if (!net) return false;
+ const now = Date.now();
+ for (const [id, t0] of net.inflight) {
+ if (now - t0 > NET_LONGPOLL_MS) net.inflight.delete(id);
+ else return true;
+ }
+ return false;
+ }
+
+ /** Wait (bounded) until the page's own requests have landed, then give the
+ * render a beat. Returns whether it actually had to wait. */
+ async awaitNetworkIdle(tab, capMs = netSettleCapMs()) {
+ if (!tab.net || !this.netBusy(tab)) return false;
+ const t0 = Date.now();
+ while (this.netBusy(tab) && Date.now() - t0 < capMs) await sleep(30);
+ await sleep(90);
+ return true;
+ }
+
+ /** The action showed no change. Watch briefly for a late one: a DOM
+ * mutation, or the page firing a request (debounced handlers). */
+ async awaitLateChange(tab, maxMs) {
+ const seq0 = tab.net?.seq ?? 0;
+ const mutation = this.evalInTab(tab, mutationWaitExpression(maxMs, 120), { awaitPromise: true }).then((v) => v === true, () => false);
+ const request = (async () => {
+ const t0 = Date.now();
+ while (Date.now() - t0 < maxMs) {
+ if ((tab.net?.seq ?? 0) > seq0) { await this.awaitNetworkIdle(tab); return true; }
+ await sleep(30);
+ }
+ return false;
+ })();
+ return new Promise((resolve) => {
+ let pending = 2;
+ for (const p of [mutation, request]) p.then((v) => { if (v) resolve(true); else if (--pending === 0) resolve(false); });
+ });
+ }
+
async registerFrame(tab, { sessionId, targetInfo }) {
if (tab.frames.has(sessionId)) return;
const conn = tab.browser.handle.conn;
@@ -1090,27 +1200,45 @@ export class Controller {
* little context, refs intact. The snapshot becomes the diff baseline, so
* later actions still return proper diffs.
*/
- async findObservation(tab, needle, { before = 2, after = 3, maxHits = 12 } = {}) {
+ async findObservation(tab, needle, { before = 2, after = 3, maxHits = 12, maxItemLines = 14 } = {}) {
const obs = await this.observation(tab, { mode: "full", maxNodes: 2000, maxChars: 200000 });
const lines = obs.text.split("\n");
const head = lines.slice(0, 3);
const body = lines.slice(3);
+ // Item ownership of each body line (0 = not inside an article/li/row).
+ // A flat list makes the line above a match belong to the PREVIOUS item;
+ // showing the match with its own item keeps its buttons attached to it.
+ const items = tab.lastSnapshot?.items ?? null;
+ const itemOf = (i) => (items && i < items.length ? items[i] : 0);
const n = String(needle).toLowerCase();
const keep = new Set();
let hits = 0;
+ let grouped = false;
body.forEach((l, i) => {
if (hits >= maxHits || !l.toLowerCase().includes(n)) return;
hits++;
- for (let k = Math.max(0, i - before); k <= Math.min(body.length - 1, i + after); k++) keep.add(k);
+ const id = itemOf(i);
+ if (id) {
+ let lo = i, hi = i;
+ while (lo > 0 && itemOf(lo - 1) === id) lo--;
+ while (hi < body.length - 1 && itemOf(hi + 1) === id) hi++;
+ if (hi - lo + 1 > maxItemLines) { lo = Math.max(lo, i - before); hi = Math.min(hi, i + after + 3); }
+ for (let k = lo; k <= hi; k++) keep.add(k);
+ grouped = true;
+ } else {
+ for (let k = Math.max(0, i - before); k <= Math.min(body.length - 1, i + after); k++) keep.add(k);
+ }
});
const out = [];
let last = -2;
for (const i of [...keep].sort((x, y) => x - y)) {
if (i !== last + 1 && out.length) out.push(" …");
+ else if (out.length && itemOf(i) !== itemOf(last)) out.push(" ·");
out.push(body[i]);
last = i;
}
- const text = [...head, `find "${needle}": ${hits ? `${hits} match${hits > 1 ? "es" : ""}` : "no matches"} (of ${body.length} lines)`, ...out].join("\n");
+ const note = grouped ? "; each match is shown with its own item" : "";
+ const text = [...head, `find "${needle}": ${hits ? `${hits} match${hits > 1 ? "es" : ""}` : "no matches"} (of ${body.length} lines${note})`, ...out].join("\n");
return { ...obs, mode: "find", text };
}
@@ -1141,7 +1269,7 @@ export class Controller {
modifiers: this.modifierMask(a.modifiers),
human: !!a.human,
}));
- const observation = await this.autoObserve(tab, a.settle_ms ?? 150);
+ const observation = await this.autoObserve(tab, a.settle_ms ?? 150, { expectChange: true });
const popup = await this.popupReport(tab);
const noChange = typeof observation?.text === "string" && observation.text.startsWith("no changes detected");
return {
@@ -1150,7 +1278,7 @@ export class Controller {
...(pos.frame ? { frame: pos.frame } : {}),
...(pos.covered ? { note: `Target center is covered by a ${pos.hitTag ?? "element"} — the click landed on the overlay. Verify with the observation diff and re-plan if wrong.` } : {}),
...(noChange && !tab.fileChooser && !popup ? {
- note: "No observable change. If the page was still settling (SPA/framework wiring listeners after load), one retry click is reasonable; otherwise the element may be a no-op in this state (e.g. an already-selected tab).",
+ note: "No observable change, even after waiting for the page to settle. Do not repeat this click blindly: it is likely a no-op in this state (e.g. an already-selected tab). If you expected a change, ax_observe to see the real page state; retry once only if the page had just loaded.",
} : {}),
observation,
...(popup ?? {}),
@@ -1363,7 +1491,7 @@ export class Controller {
this.checkDialog(tab);
const combo = a.platform ? platformShortcut(a.combo) : a.combo;
await this.doSent(tab, () => pressCombo(this.ctx(tab), combo));
- const observation = await this.autoObserve(tab, a.settle_ms ?? 150);
+ const observation = await this.autoObserve(tab, a.settle_ms ?? 150, { expectChange: true });
return { action_sent: "sent", ...(combo !== a.combo ? { combo_sent: combo } : {}), observation, ...((await this.popupReport(tab)) ?? {}) };
}
@@ -1465,7 +1593,7 @@ export class Controller {
}
throw new AxError(Codes.COMMAND_REJECTED, `Option "${a.value}" not found. Available: ${(res?.options ?? []).join(", ")}`, { retry: "never" });
}
- return { action_sent: "sent", value: res.value, observation: await this.autoObserve(tab, a.settle_ms ?? 150) };
+ return { action_sent: "sent", value: res.value, observation: await this.autoObserve(tab, a.settle_ms ?? 150, { expectChange: true }) };
}
async $drag(a) {
diff --git a/src/core/screenshot.js b/src/core/screenshot.js
index caa16e2..210e516 100644
--- a/src/core/screenshot.js
+++ b/src/core/screenshot.js
@@ -27,9 +27,18 @@ function approxBytes(b64) {
export async function captureScreenshot(conn, sessionId, { crop, budgetBytes = DEFAULT_BUDGET_BYTES, quality = 80 } = {}) {
const base = { format: "jpeg", quality, captureBeyondViewport: false };
// Viewport geometry in CSS px. Clips are in document coordinates, so the
- // scroll offset is added; scale 1 renders at CSS resolution regardless of
- // devicePixelRatio, keeping image pixels == click coordinates.
+ // scroll offset is added. clip.scale multiplies ON TOP of devicePixelRatio
+ // (measured: a headed Chrome at DPR 1.25 returns a 1295x879 image for a
+ // 1036x703 viewport at scale 1; a DPR-2 retina screen returns 2x), so the
+ // scale is 1/DPR: image pixels == CSS pixels == click coordinates, which is
+ // exactly what the geometry note promises the model.
let vp = { x: 0, y: 0, w: 0, h: 0 };
+ let dpr = 1;
+ try {
+ const r = await conn.send("Runtime.evaluate", { expression: "window.devicePixelRatio", returnByValue: true }, sessionId);
+ const v = Number(r?.result?.value);
+ if (Number.isFinite(v) && v > 0) dpr = v;
+ } catch {}
try {
const m = await conn.send("Page.getLayoutMetrics", {}, sessionId);
const v = m.cssVisualViewport ?? m.visualViewport;
@@ -45,7 +54,7 @@ export async function captureScreenshot(conn, sessionId, { crop, budgetBytes = D
region = { x, y, w, h };
}
if (region.w > 0 && region.h > 0) {
- base.clip = { x: vp.x + region.x, y: vp.y + region.y, width: region.w, height: region.h, scale: 1 };
+ base.clip = { x: vp.x + region.x, y: vp.y + region.y, width: region.w, height: region.h, scale: 1 / dpr };
}
let res = await conn.send("Page.captureScreenshot", base, sessionId);
let bytes = approxBytes(res.data);
diff --git a/src/core/serialize.js b/src/core/serialize.js
index 32b0b4d..78930ba 100644
--- a/src/core/serialize.js
+++ b/src/core/serialize.js
@@ -199,9 +199,15 @@ export function pageSerializer(opts) {
// display:contents wrappers have no box but render their children.
return getComputedStyle(el).display === "contents";
}
+ // Item ownership for ax_observe {find}: every emitted line remembers the
+ // nearest enclosing article / list item / table row, so a match can be shown
+ // with ITS item instead of the neighbouring lines (which, in a flat list,
+ // belong to the previous item and invite clicking the wrong row's button).
+ var itemSeq = 0, curItem = 0, lineItem = [];
function push(line) {
if (chars + line.length > MAX_CHARS) { truncated = true; return; }
lines.push(line);
+ lineItem.push(curItem);
chars += line.length;
}
function emitText(t, depth, kind) {
@@ -238,6 +244,16 @@ export function pageSerializer(opts) {
}
function one(c, depth, quiet) {
+ var tg = c.tagName;
+ var rl = c.getAttribute ? c.getAttribute("role") : null;
+ if (tg === "ARTICLE" || tg === "LI" || tg === "TR" || rl === "article" || rl === "listitem" || rl === "row") {
+ var prevItem = curItem;
+ curItem = ++itemSeq;
+ try { visit(c, depth, quiet); } finally { curItem = prevItem; }
+ } else visit(c, depth, quiet);
+ }
+
+ function visit(c, depth, quiet) {
var tag = c.tagName;
if (SKIP[tag]) return;
if (!visible(c)) return;
@@ -364,18 +380,18 @@ export function pageSerializer(opts) {
// Drop label-echo text: a plain text line that merely repeats the name of
// the adjacent element (label → control, legend → group) costs tokens and
// invites the model to "click the label".
- var out = [];
+ var out = [], outItem = [];
for (var li = 0; li < lines.length; li++) {
var L = lines[li];
- if (typeof L === "string") { out.push(L); continue; }
+ if (typeof L === "string") { out.push(L); outItem.push(lineItem[li]); continue; }
if (L.tl) {
if (L.kind === "text") {
var prev = lines[li - 1], next = lines[li + 1];
var echo = function (o) { return o && typeof o === "object" && o.name && (o.name === L.t || o.name.indexOf(L.t + " ") === 0); };
if (echo(prev) || echo(next)) continue;
}
- out.push(L.s); textLines.push(L.tl);
- } else out.push(L.s);
+ out.push(L.s); outItem.push(lineItem[li]); textLines.push(L.tl);
+ } else { out.push(L.s); outItem.push(lineItem[li]); }
}
lines = out;
return {
@@ -383,6 +399,7 @@ export function pageSerializer(opts) {
url: location.href,
title: clip(document.title),
text: lines.join("\n"),
+ items: outItem,
entries: entries,
textLines: textLines,
truncated: truncated,
diff --git a/src/mcp/server.js b/src/mcp/server.js
index 6400e81..0285c00 100644
--- a/src/mcp/server.js
+++ b/src/mcp/server.js
@@ -13,19 +13,30 @@ import { Controller } from "../core/controller.js";
import { renderResult } from "./render.js";
// MCP server instructions: the usage contract in a few lines (the full
-// version is SKILL.md). Harnesses surface this to the model once.
-const INSTRUCTIONS = [
+// version is SKILL.md). Some harnesses (Codex) copy these into EVERY tool
+// description, so each line is paid for once per tool on every turn: the base
+// is kept minimal and situational guidance is added only when it applies.
+const BASE_INSTRUCTIONS = [
"axuse: browser control via accessibility snapshots with element refs ([ref=e7]).",
- "Workflow: ax_open → act on refs → read the diff each action returns (no need to re-observe). Pass the tab_id ax_open returned on every call: tabs are shared by all callers of this server, and tab_id is required once more than one tab is open.",
- "Backend: ax_open defaults to the user's own signed-in Chrome when the axuse extension is connected (else a separate CDP-launched browser); the result reports backend.",
- "For bounded source lists, use ax_open_many {urls:[...]} for compact title/status/excerpt results; it closes temporary tabs by default.",
- "Forms: ONE ax_fill call with all fields (+submit). Big pages: ax_observe {find:\"text\"}. Infinite feeds/lazy lists: ax_scroll {until:\"text\"}.",
- "New tabs/popups and JS dialogs opened by an action are reported in its result (answer dialogs with ax_dialog).",
- "Screenshots only for pixels (canvas): then click {x,y} page coords.",
- "When a verification widget (Turnstile/reCAPTCHA/hCaptcha/slider/interstitial) blocks a page you were asked to drive, call ax_solve_captcha {provider} or ax_challenge {op:\"solve\"} instead of abandoning the task; report the returned facts verbatim (LICENSE_INVALID / NO_SOLVER are expected outcomes the operator can act on).",
- "Servers: on a display-less host axuse auto-boots a virtual display and keeps Chrome fully headed (anti-bot-safe) — never pass headless for challenge/anti-bot pages; the ax_open result reports the virtual display fact.",
- "Capability plugins contribute extra tools (ax_vault, ax_net, ax_guard, ax_extract, …): ax_plugins lists them with their tool names.",
-].join("\n");
+ "Workflow: ax_open → act on refs → read the diff each action returns (no need to re-observe). Pass the tab_id ax_open returned on every call.",
+ "Forms: ONE ax_fill call with all fields (+submit). Big pages: ax_observe {find:\"text\"}. Lazy feeds: ax_scroll {until:\"text\"}. Source lists: ax_open_many {urls}.",
+ "New tabs and JS dialogs opened by an action are reported in its result (answer dialogs with ax_dialog). Screenshots only for canvas pixels.",
+];
+const SOLVER_INSTRUCTIONS =
+ "When a verification widget (Turnstile/reCAPTCHA/hCaptcha/slider/interstitial) blocks a page you were asked to drive, call ax_solve_captcha {provider} or ax_challenge {op:\"solve\"} instead of abandoning the task; report the returned facts verbatim (LICENSE_INVALID / NO_SOLVER are expected outcomes the operator can act on).";
+const SERVER_INSTRUCTIONS =
+ "Servers: on a display-less host axuse auto-boots a virtual display and keeps Chrome fully headed (anti-bot-safe) — never pass headless for challenge/anti-bot pages.";
+const PLUGIN_INSTRUCTIONS =
+ "Capability plugins contribute extra tools (ax_vault, ax_net, ax_guard, ax_extract, …): ax_plugins lists them with their tool names.";
+
+export function buildInstructions({ solvers = false, plugins = false, server = false } = {}) {
+ return [
+ ...BASE_INSTRUCTIONS,
+ ...(solvers ? [SOLVER_INSTRUCTIONS] : []),
+ ...(server ? [SERVER_INSTRUCTIONS] : []),
+ ...(plugins ? [PLUGIN_INSTRUCTIONS] : []),
+ ].join("\n");
+}
import { AxError } from "../core/errors.js";
import { browserTools, desktopTools, solverTools, pluginTools } from "./manifest.js";
import { resolveDesktopDriver, desktopUnavailable } from "../desktop/index.js";
@@ -35,6 +46,16 @@ import { displayUsable, ensureVirtualDisplay, serverModeEnabled } from "../core/
const VERSION = "0.1.0";
const SUPPORTED_PROTOCOL = "2025-03-26";
+/** Typed error as JSON in the first block (stable for machine consumers); a
+ * fresh page observation, when the controller attached one, rides along as a
+ * second plain-text block (readable refs, not an escaped string). */
+function errorContent(json) {
+ const { fresh_observation: page, ...rest } = json;
+ return page
+ ? [{ type: "text", text: JSON.stringify(rest) }, { type: "text", text: page }]
+ : [{ type: "text", text: JSON.stringify(json) }];
+}
+
/** `desktop` resolves once (the macOS driver is the one async case). */
export async function createServer({ controller = new Controller(), desktop, solvers } = {}) {
// Server mode (display-less Linux host): boot the virtual display EAGERLY,
@@ -62,6 +83,14 @@ export async function createServer({ controller = new Controller(), desktop, sol
}
const tools = [...browserTools(), ...desktopTools(!!driver, axCapable), ...solverTools(registry), ...pluginTools(registry)];
+ // The captcha line only matters once a solver is actually loaded; the
+ // ax_solve_captcha description already carries it for the no-plugin case.
+ const instructions = buildInstructions({
+ solvers: registry.length > 0,
+ plugins: registry.length > 0,
+ server: process.platform === "linux" && serverModeEnabled(),
+ });
+
const byName = new Map(tools.map((t) => [t.name, t]));
async function handleRequest(msg) {
@@ -73,7 +102,7 @@ export async function createServer({ controller = new Controller(), desktop, sol
protocolVersion: typeof params.protocolVersion === "string" ? params.protocolVersion : SUPPORTED_PROTOCOL,
capabilities: { tools: { listChanged: false } },
serverInfo: { name: "axuse", version: VERSION },
- instructions: INSTRUCTIONS,
+ instructions,
});
} case "ping":
return reply(id, {});
@@ -96,8 +125,10 @@ export async function createServer({ controller = new Controller(), desktop, sol
// Structured errors become tool results, not protocol errors. Plugin
// errors (AxError and plugin-typed errors like LICENSE_INVALID) carry
// toJSON(); anything else is an internal string.
- const text = (e instanceof AxError || typeof e.toJSON === "function") ? JSON.stringify(e.toJSON()) : `internal: ${e.message}`;
- return reply(id, { content: [{ type: "text", text }], isError: true });
+ const content = (e instanceof AxError || typeof e.toJSON === "function")
+ ? errorContent(e.toJSON())
+ : [{ type: "text", text: `internal: ${e.message}` }];
+ return reply(id, { content, isError: true });
}
}
diff --git a/test/async-settle.test.js b/test/async-settle.test.js
new file mode 100644
index 0000000..daad171
--- /dev/null
+++ b/test/async-settle.test.js
@@ -0,0 +1,86 @@
+/**
+ * Post-action settling for pages that answer LATE: a click whose result
+ * arrives after a slow fetch, a debounced render, a genuine no-op, and the
+ * stale-ref recovery that hands back the current page. Gated on a findable
+ * browser, like the other live CDP tests.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { locateBrowser } from "../src/core/cdp.js";
+import { Controller } from "../src/core/controller.js";
+
+const hasBrowser = !!locateBrowser();
+
+const PAGE = `<!doctype html><title>Late</title><body><h1>Wizard</h1><div id="stage">
+<button id="next">Next</button> <button id="later">Later</button> <button id="noop">Noop</button>
+</div><script>
+const stage = document.getElementById("stage");
+document.getElementById("next").onclick = async () => {
+ await fetch("/slow"); // ~400 ms server round trip, then a full re-render
+ stage.innerHTML = '<p>Step 2 of 2</p><button id="finish">Finish</button>';
+};
+document.getElementById("later").onclick = () => {
+ setTimeout(() => { stage.insertAdjacentHTML("beforeend", "<p>debounced result</p>"); }, 300);
+};
+</script></body>`;
+
+function startServer() {
+ return new Promise((resolve) => {
+ const srv = createServer((req, res) => {
+ if (req.url === "/slow") return void setTimeout(() => { res.end("ok"); }, 400);
+ res.setHeader("content-type", "text/html");
+ res.end(PAGE);
+ });
+ srv.listen(0, "127.0.0.1", () => resolve(srv));
+ });
+}
+
+const refOf = (text, name) => text.match(new RegExp(`button "${name}"[^\\[]*\\[ref=(e\\d+)\\]`))?.[1];
+
+test("late-answering pages", { skip: !hasBrowser }, async (t) => {
+ const srv = await startServer();
+ const c = new Controller();
+ t.after(() => { c.dispose(); srv.close(); });
+ const url = `http://127.0.0.1:${srv.address().port}/`;
+
+ let tab; // tab_id of the first page, reused by the stale-ref check
+
+ await t.test("a click answered by a slow fetch reports the re-render, not 'no change'", async () => {
+ const opened = await c.execute("open", { url, headless: true, settle_ms: 100 });
+ tab = opened.tab_id;
+ const next = refOf(opened.observation.text, "Next");
+ assert.ok(next, "Next has a ref");
+ const clicked = await c.execute("click", { tab_id: tab, target: { ref: next }, settle_ms: 150 });
+ assert.match(clicked.observation.text, /Step 2 of 2/, "the diff already shows the new step");
+ assert.doesNotMatch(clicked.observation.text, /no changes detected/);
+ assert.equal(clicked.note, undefined, "no misleading no-op note");
+ });
+
+ await t.test("a stale ref returns the current page with the error", async () => {
+ // The Next button is gone; the page is on step 2.
+ const gone = await c.execute("click", { tab_id: tab, target: { ref: "e99" } }).catch((e) => e);
+ assert.equal(gone.code, "REF_NOT_FOUND");
+ assert.equal(gone.retry, "reobserve");
+ assert.match(gone.facts.fresh_observation, /Step 2 of 2/, "error carries the live page");
+ assert.match(gone.facts.fresh_observation, /button "Finish"/);
+ });
+
+ await t.test("a debounced render is caught by the late-change watch", async () => {
+ const opened = await c.execute("open", { url, headless: true, settle_ms: 100 });
+ const later = refOf(opened.observation.text, "Later");
+ const clicked = await c.execute("click", { tab_id: opened.tab_id, target: { ref: later }, settle_ms: 150 });
+ assert.match(clicked.observation.text, /debounced result/);
+ });
+
+ await t.test("a genuine no-op still says so, and does not hang", async () => {
+ const opened = await c.execute("open", { url, headless: true, settle_ms: 100 });
+ const noop = refOf(opened.observation.text, "Noop");
+ const t0 = Date.now();
+ const clicked = await c.execute("click", { tab_id: opened.tab_id, target: { ref: noop }, settle_ms: 150 });
+ const took = Date.now() - t0;
+ assert.match(clicked.observation.text, /no changes detected/);
+ assert.match(clicked.note, /No observable change/);
+ assert.ok(took < 3000, `no-op settled in ${took} ms`);
+ });
+});
diff --git a/test/find-items.test.js b/test/find-items.test.js
new file mode 100644
index 0000000..9d4f53a
--- /dev/null
+++ b/test/find-items.test.js
@@ -0,0 +1,59 @@
+/**
+ * ax_observe {find}: a match is shown with ITS OWN item.
+ *
+ * In a flat snapshot the line just above an author's name is the previous
+ * article's "Like" button. Showing +-N neighbouring lines therefore put the
+ * wrong row's button next to the match, and agents clicked it (benchmark t7:
+ * 2 of 3 runs liked the neighbouring post, which is unrecoverable there).
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { locateBrowser } from "../src/core/cdp.js";
+import { Controller } from "../src/core/controller.js";
+
+const hasBrowser = !!locateBrowser();
+
+const feed = (n) => Array.from({ length: n }, (_, i) =>
+ `<article><b>@user${i + 1}</b><p>Post #${i + 1}: topic</p><button>Like</button></article>`).join("");
+const list = Array.from({ length: 5 }, (_, i) => `<li>Item ${i + 1} <button>Buy ${i + 1}</button></li>`).join("");
+const table = Array.from({ length: 4 }, (_, i) => `<tr><td>Row ${i + 1}</td><td><button>Edit ${i + 1}</button></td></tr>`).join("");
+const PAGE = "data:text/html," + encodeURIComponent(
+ `<body><h1>Team feed</h1>${feed(6)}<ul>${list}</ul><table>${table}</table></body>`);
+
+test("find shows each match with its own item", { skip: !hasBrowser }, async (t) => {
+ const c = new Controller();
+ t.after(() => c.dispose());
+ const opened = await c.execute("open", { url: PAGE, headless: true, settle_ms: 100 });
+ const tab = opened.tab_id;
+ const find = async (needle) => (await c.execute("observe", { tab_id: tab, find: needle })).observation.text;
+
+ await t.test("article: the author's own Like, none of the neighbours'", async () => {
+ const text = await find("@user3");
+ assert.match(text, /@user3/);
+ assert.match(text, /Post #3/);
+ assert.equal((text.match(/button "Like"/g) ?? []).length, 1, "exactly one Like, the match's own");
+ assert.doesNotMatch(text, /Post #2|Post #4|@user2|@user4/, "no neighbouring item leaks in");
+ assert.match(text, /each match is shown with its own item/);
+ });
+
+ await t.test("list item and table row behave the same", async () => {
+ const li = await find("Item 2");
+ assert.match(li, /Buy 2/);
+ assert.doesNotMatch(li, /Buy 1|Buy 3/);
+ const tr = await find("Row 3");
+ assert.match(tr, /Edit 3/);
+ assert.doesNotMatch(tr, /Edit 2|Edit 4/);
+ });
+
+ await t.test("matches outside any item keep the old line context", async () => {
+ const text = await find("Team feed");
+ assert.match(text, /Team feed/);
+ assert.match(text, /@user1/, "context lines after a heading are still shown");
+ });
+
+ await t.test("two matches in adjacent items are separated", async () => {
+ const text = await find("@user");
+ assert.match(text, /@user1/);
+ assert.match(text, /·/, "item boundary marker between adjacent matched items");
+ });
+});
diff --git a/test/screenshot-dpr.test.js b/test/screenshot-dpr.test.js
new file mode 100644
index 0000000..f74dd0d
--- /dev/null
+++ b/test/screenshot-dpr.test.js
@@ -0,0 +1,54 @@
+/**
+ * Screenshot pixels must equal click pixels at ANY devicePixelRatio.
+ *
+ * ax_screenshot tells the model "image WxH = viewport CSS px; pass ax_click
+ * {x,y} in these same pixels". Page.captureScreenshot's clip.scale multiplies
+ * on top of devicePixelRatio, so a headed Chrome on a 125% Windows display (or
+ * any retina Mac at 2x) returned an image 1.25x/2x larger than the note
+ * claimed, and every coordinate the model read off it landed 1.25x/2x off
+ * target. Emulating the ratio makes the headless CI lane reproduce it.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { locateBrowser } from "../src/core/cdp.js";
+import { Controller } from "../src/core/controller.js";
+
+const hasBrowser = !!locateBrowser();
+
+/** Pixel size of a JPEG, read from its SOF marker. */
+function jpegSize(b64) {
+ const b = Buffer.from(b64, "base64");
+ let i = 2;
+ while (i < b.length) {
+ if (b[i] !== 0xff) { i++; continue; }
+ const m = b[i + 1];
+ if (m >= 0xc0 && m <= 0xc3) return { w: b.readUInt16BE(i + 7), h: b.readUInt16BE(i + 5) };
+ i += 2 + b.readUInt16BE(i + 2);
+ }
+ throw new Error("no SOF marker");
+}
+
+const PAGE = "data:text/html," + encodeURIComponent(
+ '<body style="margin:0"><canvas width="500" height="300" style="margin:20px;background:#eee"></canvas></body>');
+
+test("screenshot image pixels == CSS pixels at any devicePixelRatio", { skip: !hasBrowser }, async (t) => {
+ const c = new Controller();
+ t.after(() => c.dispose());
+ const opened = await c.execute("open", { url: PAGE, headless: true, settle_ms: 100 });
+ const tab = opened.tab_id;
+
+ for (const dpr of [1, 1.25, 2, 3]) {
+ await c.cdpSend(tab, "Emulation.setDeviceMetricsOverride", { width: 800, height: 600, deviceScaleFactor: dpr, mobile: false });
+ const seen = await c.execute("evaluate", { tab_id: tab, expression: "devicePixelRatio" });
+ assert.equal(Number(seen.value), dpr, `emulated dpr ${dpr} took effect`);
+
+ const full = await c.execute("screenshot", { tab_id: tab });
+ const { w, h } = jpegSize(full.image.base64);
+ const claimed = full.note.match(/image (\d+)x(\d+)/);
+ assert.deepEqual([w, h], [Number(claimed[1]), Number(claimed[2])], `dpr ${dpr}: image is ${w}x${h}, note claims ${claimed[1]}x${claimed[2]}`);
+ assert.equal(w, 800, `dpr ${dpr}: full-viewport width is CSS px`);
+
+ const crop = await c.execute("screenshot", { tab_id: tab, crop: { x: 20, y: 20, w: 300, h: 200 } });
+ assert.deepEqual(Object.values(jpegSize(crop.image.base64)), [300, 200], `dpr ${dpr}: crop is w x h CSS px`);
+ }
+});