Skip to content

armgit / axuse

Public Imported from GitHub
0

Accessibility-first browser & computer use for AI agents — one MCP server for every harness (Codex, Claude Code, OpenCode, Antigravity). Instruments the DOM/accessibility tree via CDP instead of screenshot loops: diffed snapshots, stable element refs, trusted input, typed errors. Zero dependencies

8.4 KB
Raw Blame
/**
 * axuse — AT-SPI2 client over the zero-dependency D-Bus transport.
 *
 * The macOS driver gets its platform AX tree from System Events; on Linux
 * the same tree lives at AT-SPI2, exposed over a private D-Bus socket. This
 * module speaks the AT-SPI interface set directly (verified against the
 * live at-spi2 registry on Xorg/GNOME, Ubuntu 22.04):
 *
 *   - objects are (bus name, object path) pairs — children of the desktop
 *     live on each application's own connection name
 *   - Accessible: GetChildren → a(so), GetChildAtIndex(i), GetRoleName,
 *     GetState → (au), GetInterfaces → as, GetApplication → (so);
 *     Name/ChildCount/Description are D-Bus PROPERTIES (the old GetX
 *     methods do not exist on current registries)
 *   - Component.GetExtents(coordType 0) → (iiii) screen-absolute bounds
 *   - Action: nActions property, GetActionName(i), DoAction(i)
 *   - Text: GetCharCount, GetText(start, end); EditableText.SetTextContents
 *   - Value: current/minimum/maximum as variants
 */
import { AxError, Codes } from "../core/errors.js";
import { DbusClient } from "./dbus.js";

const A11Y = "org.a11y.atspi.Accessible";
const REGISTRY_NAME = "org.a11y.atspi.Registry";
const DESKTOP_PATH = "/org/a11y/atspi/accessible/root";

/** AT-SPI state bits (atspi-state.h) — the operationally meaningful ones. */
export const STATE_NAMES = [
  "invalid", "active", "armed", "busy", "checked", "collapsed", "defunct",
  "editable", "enabled", "expandable", "expanded", "focusable", "focused",
  "has-tooltip", "horizontal", "iconified", "modal", "multi-line",
  "multiselectable", "opaque", "pressed", "resizable", "selectable",
  "selected", "sensitive", "showing", "single-line", "stale", "transient",
  "vertical", "visible", "manages-descendants", "indeterminate",
];

function decodeStates(words) {
  const out = [];
  for (let w = 0; w < words.length; w++) {
    let bits = words[w] >>> 0;
    let b = 0;
    while (bits) {
      if (bits & 1) {
        const name = STATE_NAMES[w * 32 + b];
        if (name) out.push(name);
      }
      bits >>>= 1;
      b++;
    }
  }
  return out;
}

export class AtspiClient {
  /** @param {{sessionBusAddress?: string}} [opts] — the session bus to
   *  discover org.a11y.Bus on (defaults to the ambient environment; the
   *  sandbox test passes a private bus explicitly). */
  constructor(opts = {}) {
    this.sessionBusAddress = opts.sessionBusAddress ?? process.env.DBUS_SESSION_BUS_ADDRESS;
    this.bus = null; // the AT-SPI bus connection
    this.sessionBus = null; // temporary connection used to discover it
  }

  /** Discover and connect the AT-SPI bus. */
  async connect() {
    const direct = process.env.AXUSE_ATSPI_BUS;
    let address = direct ?? null;
    if (!address) {
      const sessionAddress = this.sessionBusAddress;
      if (!sessionAddress) {
        throw new AxError(Codes.DESKTOP_DRIVER_UNAVAILABLE,
          "No D-Bus session bus (DBUS_SESSION_BUS_ADDRESS unset) — the AT-SPI structure layer needs the session's accessibility bus.",
          { hint: "Run inside the desktop session; structure-first desktop control needs AT-SPI.", retry: "never" });
      }
      try {
        this.sessionBus = await new DbusClient(sessionAddress).connect();
        const [a] = await this.sessionBus.call("org.a11y.Bus", "/org/a11y/bus", "org.a11y.Bus", "GetAddress", "", []);
        address = String(a);
      } catch (e) {
        try { this.sessionBus?.close(); } catch {}
        this.sessionBus = null;
        throw new AxError(Codes.DESKTOP_DRIVER_UNAVAILABLE,
          `The session bus has no org.a11y.Bus (is at-spi2 running?): ${e.message}`,
          { hint: "Install/enable at-spi2-core; coordinate+vision tools keep working without it.", retry: "never" });
      }
    }
    try {
      this.bus = await new DbusClient(address).connect();
    } catch (e) {
      try { this.sessionBus?.close(); } catch {}
      this.sessionBus = null;
      throw e;
    }
    return this;
  }

  get connected() {
    return !!this.bus?.socket && !this.bus.dead;
  }

  close() {
    try { this.bus?.close(); } catch {}
    try { this.sessionBus?.close(); } catch {}
    this.bus = null;
    this.sessionBus = null;
  }

  /* ------------------------------------------------------ object helpers */

  /** The desktop root (registry-owned). */
  desktop() {
    return [REGISTRY_NAME, DESKTOP_PATH];
  }

  /** Accessible children as (name, path) pairs. */
  async children(ref) {
    const [list] = await this.bus.call(ref[0], ref[1], A11Y, "GetChildren", "", []);
    return list;
  }

  async childCount(ref) {
    const v = await this.bus.prop(ref[0], ref[1], A11Y, "ChildCount");
    return Number(v);
  }

  async name(ref) {
    const v = await this.bus.prop(ref[0], ref[1], A11Y, "Name");
    return String(v ?? "");
  }

  async description(ref) {
    const v = await this.bus.prop(ref[0], ref[1], A11Y, "Description");
    return String(v ?? "");
  }

  async role(ref) {
    // Method on most at-spi2 apps; a few roots only expose the Role
    // property (role enum) — report it honestly as role#N then.
    try {
      const [r] = await this.bus.call(ref[0], ref[1], A11Y, "GetRoleName", "", []);
      return String(r);
    } catch {
      const v = await this.bus.prop(ref[0], ref[1], A11Y, "Role").catch(() => null);
      return v === null ? "unknown" : `role#${Number(v)}`;
    }
  }

  async states(ref) {
    const [words] = await this.bus.call(ref[0], ref[1], A11Y, "GetState", "", []);
    return decodeStates(Array.isArray(words) ? words : [words]);
  }

  async interfaces(ref) {
    const [list] = await this.bus.call(ref[0], ref[1], A11Y, "GetInterfaces", "", []);
    return list;
  }

  async application(ref) {
    const [app] = await this.bus.call(ref[0], ref[1], A11Y, "GetApplication", "", []);
    return app;
  }

  /** Screen-absolute bounds via Component.GetExtents(coordType 0). */
  async bounds(ref) {
    const [tuple] = await this.bus.call(ref[0], ref[1], "org.a11y.atspi.Component", "GetExtents", "u", [0]);
    return { x: tuple[0], y: tuple[1], w: tuple[2], h: tuple[3] };
  }

  /* --------------------------------------------------------------- Action */
  // CAUTION: GetActions is never called internally — GTK's atk-bridge (2.4x)
  // crashes marshalling its a(sss) reply and takes the app's bus connection
  // with it. DoAction(i) and GetActionName(i) are the safe surface.

  async actionCount(ref) {
    try {
      const v = await this.bus.prop(ref[0], ref[1], "org.a11y.atspi.Action", "nActions");
      if (v !== undefined && v !== null) return Number(v);
    } catch {}
    // No nActions property (GTK): a resolvable action name implies ≥1 action.
    try { await this.actionName(ref, 0); return 1; } catch { return 0; }
  }

  async actionName(ref, index) {
    const [n] = await this.bus.call(ref[0], ref[1], "org.a11y.atspi.Action", "GetActionName", "i", [index]);
    return String(n);
  }

  async doAction(ref, index) {
    const [ok] = await this.bus.call(ref[0], ref[1], "org.a11y.atspi.Action", "DoAction", "i", [index]);
    return !!ok;
  }

  async doAction(ref, index) {
    const [ok] = await this.bus.call(ref[0], ref[1], "org.a11y.atspi.Action", "DoAction", "i", [index]);
    return !!ok;
  }

  /* ----------------------------------------------------------------- Text */

  async charCount(ref) {
    const [n] = await this.bus.call(ref[0], ref[1], "org.a11y.atspi.Text", "GetCharCount", "", []);
    return Number(n);
  }

  async text(ref, start = 0, end = -1) {
    const [s] = await this.bus.call(ref[0], ref[1], "org.a11y.atspi.Text", "GetText", "ii", [start, end]);
    return String(s);
  }

  async setTextContents(ref, contents) {
    const [ok] = await this.bus.call(ref[0], ref[1], "org.a11y.atspi.EditableText", "SetTextContents", "s", [String(contents)]);
    return !!ok;
  }

  /* ---------------------------------------------------------------- Value */

  async value(ref) {
    try {
      const [cur] = await this.bus.call(ref[0], ref[1], "org.a11y.atspi.Value", "GetCurrentValue", "", []);
      return cur?.sig === "d" ? cur.value : cur?.value ?? null;
    } catch {
      return null;
    }
  }

  /* ------------------------------------------------------------- registry */

  /** Applications registered on the desktop: [{ref, name}]. */
  async apps() {
    const kids = await this.children(this.desktop());
    const out = [];
    for (const ref of kids) {
      let n = "";
      try { n = await this.name(ref); } catch {}
      out.push({ ref, name: n });
    }
    return out;
  }
}