Skip to content

armgit / axuse

Public Imported from GitHub
0

Accessibility-first browser & computer use for AI agents — one MCP server for every harness (Codex, Claude Code, OpenCode, Antigravity). Instruments the DOM/accessibility tree via CDP instead of screenshot loops: diffed snapshots, stable element refs, trusted input, typed errors. Zero dependencies

8.5 KB
Raw Blame
/**
 * MANUAL TCC VALIDATION LANE — macOS, explicit opt-in only.
 *
 * These tests exercise the GRANTED side of the TCC matrix: real screen
 * capture and a real (inert) input event. They never run in normal CI and
 * never run silently: the gate below requires BOTH the env opt-in and a
 * permission state that says granted — otherwise the test FAILS with typed
 * instructions instead of skipping, because in this lane "not granted" is a
 * lane misconfiguration, not an expected state.
 *
 * Required environment (see docs/macos-support.md §manual-tcc-lane):
 *   AXUSE_TEST_MACOS_TCC_MANUAL=1  (this lane's opt-in)
 *   AXUSE_ENABLE_MACOS_CUA=1
 *   A dedicated Mac with:
 *     - Accessibility granted to the host binary (node/Terminal/IDE)
 *     - Screen Recording granted to the same host
 *   Run: node --test test/desktop-tcc-manual.test.js
 */
import { test } from "node:test";
import assert from "node:assert/strict";
import { getDesktopDriver } from "../src/desktop/index.js";

const manual = process.env.AXUSE_TEST_MACOS_TCC_MANUAL === "1";
const isMac = process.platform === "darwin";

test("tcc lane: granted Screen Recording + Accessibility is verified BEFORE anything runs", { skip: !manual || !isMac }, async () => {
  const d = await getDesktopDriver();
  const p = await d.permissions({ refresh: true });
  const problems = [];
  if (p.accessibility !== true) problems.push("Accessibility NOT granted — grant it to the exact binary running this test (System Settings → Privacy & Security → Accessibility).");
  if (p.screen_recording !== true) problems.push("Screen Recording NOT granted — grant it to the exact binary running this test, then RESTART that binary (macOS applies the grant to new processes only).");
  if (problems.length) {
    // Fail closed with the full typed instructions: this lane exists to
    // prove the granted state, so an ungranted state is a failure here.
    assert.fail(`tcc-manual lane misconfigured:\n  - ${problems.join("\n  - ")}`);
  }
});

test("tcc lane: real screenshot succeeds and stays budgeted (granted path)", { skip: !manual || !isMac }, async () => {
  const d = await getDesktopDriver();
  const info = await d.screenInfo({ refresh: true });
  // A 200x120 slice at the top-left of the primary display: enough to prove
  // pixel capture, small enough not to leak meaningful screen content into
  // logs (only size/mime facts are asserted, never the pixels).
  const shot = await d.screenshot({ x: info.x, y: info.y, w: 200, h: 120, budgetBytes: 120_000 });
  assert.equal(shot.mime, "image/jpeg");
  assert.ok(shot.bytes > 500, `shot has real pixel payload (${shot.bytes} bytes)`);
  assert.ok(shot.bytes <= 120_000);
});

test("tcc lane: inert input event round-trips through the CGEvent tap (granted path)", { skip: !manual || !isMac }, async () => {
  const d = await getDesktopDriver();
  // A zero-delta scroll event: a REAL posted event through the HID tap that
  // moves nothing and types nothing — end-to-end proof of the trusted-input
  // path without disturbing the session's owner.
  const res = await d.wheel(0, 0);
  assert.equal(res.action_sent, "sent");
});

test("tcc lane: wheel direction matches the browser contract under this host's scroll preference", { skip: !manual || !isMac }, async () => {
  // Live pin for the natural-scrolling sign mapping (unit-tested; this
  // records the on-host truth for the evidence file). A zero-delta event
  // again: it proves the preference read + event construction, while the
  // unit tests pin the arithmetic.
  const d = await getDesktopDriver();
  const natural = await d._naturalScrolling({ refresh: true });
  const res = await d.wheel(0, 0);
  assert.equal(res.action_sent, "sent");
  // Record the host's preference so the evidence file can correlate unit
  // expectations with reality on a granted machine.
  console.log(`[tcc-manual] natural scrolling = ${natural} (wheel contract: dy>0 = content down)`);
  assert.equal(typeof natural, "boolean");
});

test("tcc lane: recovery — a denied-then-granted transition is seen without host restart", { skip: !manual || !isMac }, async () => {
  // Cache invalidation contract: after invalidateCaches(), the next
  // permissions() must re-probe (recovery visibility). On an already-granted
  // host the values stay true — the assertion is that the re-probe HAPPENED.
  const d = await getDesktopDriver();
  await d.permissions({ refresh: true });
  d.invalidateCaches();
  const again = await d.permissions();
  assert.equal(again.accessibility, true, "re-probe after invalidation still sees the grant");
});

test("tcc lane: AX structure layer — snapshot + read round-trip on a real app", { skip: !manual || !isMac }, async (t) => {
  const d = await getDesktopDriver();
  const p = await d.permissions({ refresh: true });
  if (p.accessibility !== true) assert.fail("AX pin needs Accessibility granted (lane misconfiguration)");
  // Snapshot the frontmost app (the runner's own Terminal/agent host — a
  // guaranteed-present, safe target): structure facts only, no actions.
  const snap = await d.axSnapshot({ front: true, maxNodes: 60, maxDepth: 6 });
  assert.ok(snap.nodes > 3, `expected a real tree, got ${snap.nodes} nodes`);
  assert.ok(snap.text.includes("[ref=ax"), "refs are rendered");
  const ref = snap.text.match(/\[ref=(ax[\d.]+)\]/)?.[1];
  const read = await d.axRead({ ref });
  assert.equal(typeof read.role, "string");
  assert.equal(read.ref, ref);
  console.log(`[tcc-manual] ax pin: ${snap.app} → ${snap.nodes} nodes, ref ${ref} = ${read.role}`);
});

test("tcc lane: LIVE scroll-direction pin — wheel(0,+N) scrolls OUR page down, world-visible truth", { skip: !manual || !isMac }, async (t) => {
  // The natural-scrolling sign mapping is unit-pinned by derivation; this
  // test measures it against reality. It scrolls only axuse's OWN fixture
  // window (launched at a fixed position) and reads window.scrollY back
  // through CDP — an unambiguous, automated verdict for the manual lane.
  const d = await getDesktopDriver();
  const p = await d.permissions({ refresh: true });
  if (p.accessibility !== true) {
    assert.fail("scroll pin needs Accessibility granted (this lane is misconfigured otherwise)");
  }
  const { BrowserHandle } = await import("../src/core/cdp.js");
  const html = "data:text/html,<!doctype html><html><body style='margin:0'><div style='height:4000px;background:linear-gradient(#333,#fff)'><h1 style='position:fixed;top:8px;left:8px' id='s'>0</h1></div><script>window.addEventListener('scroll',()=>document.getElementById('s').textContent=Math.round(scrollY))</script></body></html>";
  const browser = await BrowserHandle.launch({
    headless: false, // real windows receive CGEvent input; headless does not
    extraArgs: ["--window-position=40,60", "--window-size=500,500", "--activate-on-launch"],
  });
  t.after(() => browser.dispose());
  const { targetId, sessionId } = await browser.createTab(html, { autoAttach: false });
  t.after(() => browser.closeTabTarget(targetId));
  const sleep = (ms) => new Promise((ok) => setTimeout(ok, ms));
  await sleep(800); // let the window settle into place
  // Focus OUR window and park the cursor over its content area.
  await d.activate({ pid: browser.proc.pid });
  await sleep(300);
  const info = await d.screenInfo({ refresh: true });
  const cx = info.x + 40 + 250, cy = info.y + 60 + 250; // window center per launch args
  await d.move(cx, cy);
  await sleep(200);
  const before = await browser.conn.send("Runtime.evaluate", { expression: "Math.round(scrollY)", returnByValue: true }, sessionId);
  await d.wheel(0, 120); // contract: content DOWN
  await sleep(600);
  const after = await browser.conn.send("Runtime.evaluate", { expression: "Math.round(scrollY)", returnByValue: true }, sessionId);
  const y0 = Number(before.result.value), y1 = Number(after.result.value);
  console.log(`[tcc-manual] scroll pin: scrollY ${y0} → ${y1} after wheel(0,120)`);
  assert.ok(y1 > y0 + 5, `wheel(0,+120) must scroll content DOWN (scrollY ${y0} → ${y1}). If it went UP, flip the map in _naturalScrolling()/wheel() in src/desktop/macos.js and update the unit expectations.`);
  // And the reverse direction for symmetry.
  await d.wheel(0, -120);
  await sleep(600);
  const back = await browser.conn.send("Runtime.evaluate", { expression: "Math.round(scrollY)", returnByValue: true }, sessionId);
  const y2 = Number(back.result.value);
  console.log(`[tcc-manual] scroll pin: scrollY ${y1} → ${y2} after wheel(0,-120)`);
  assert.ok(y2 < y1 - 5, `wheel(0,-120) must scroll content UP (scrollY ${y1} → ${y2}).`);
});