Skip to content

πŸ‘Ύ Open Computer Use – Open-Source Alternative to Codex Computer Use

21.1 KB
Raw Blame
import assert from "node:assert/strict";
import test from "node:test";
import { EventEmitter } from "node:events";
import { PassThrough } from "node:stream";
import { JsonLinePeer, PersistentJavaScriptSession, WorkerJavaScriptSession, parseListApps, toolDefinitions } from "./open-computer-use-repl.mjs";

function textResult(text, isError = false, image) {
  const content = [{ type: "text", text }];
  if (image) content.push({ type: "image", mimeType: image.mimeType, data: image.data });
  return { content, isError };
}

function mockNative() {
  const calls = [];
  const native = {
    calls,
    async request(method, params) {
      assert.equal(method, "tools/call");
      calls.push(params);
      switch (params.name) {
        case "list_apps": return textResult('[{"id":"com.example.Text","displayName":"Text"}]');
        case "get_app_state": return textResult(`state:${params.arguments.app}`, false, { mimeType: "image/png", data: Buffer.from([0x89,0x50,0x4e,0x47,0x0d,0x0a,0x1a,0x0a]).toString("base64") });
        case "click": return textResult("clicked");
        case "type_text": return params.arguments.text === "fail" ? textResult("typing failed", true) : textResult("typed");
        default: return textResult(params.name);
      }
    },
  };
  return native;
}

test("advertises only js and js_reset", () => {
  assert.deepEqual(toolDefinitions().map(tool => tool.name), ["js", "js_reset"]);
  assert.deepEqual(toolDefinitions()[0].inputSchema.required, ["code"]);
  assert.equal(toolDefinitions()[0].inputSchema.properties.timeout_ms.maximum, 300_000);
});

test("normalizes native app-list text across macOS, Linux, and Windows", () => {
  assert.deepEqual(parseListApps([
    "TextEdit β€” com.apple.TextEdit [frontmost, running, last-used=2026-09-22, uses=42]",
    "Google Chrome β€” com.google.Chrome [last-used=2026-09-21, uses=7]",
  ].join("\n")), [
    { id: "com.apple.TextEdit", displayName: "TextEdit", isRunning: true, lastUsedDate: "2026-09-22", useCount: 42 },
    { id: "com.google.Chrome", displayName: "Google Chrome", lastUsedDate: "2026-09-21", useCount: 7 },
  ]);
  assert.deepEqual(parseListApps("gedit -- gedit [running, pid=42, window=Draft, notes]"), [
    { id: "gedit", displayName: "gedit", isRunning: true },
  ]);
  assert.deepEqual(parseListApps("No running top-level apps are visible to this Linux runtime."), []);
});

test("cua.listApps returns structured apps for the native text protocol", async () => {
  const native = mockNative();
  native.request = async (method, params) => {
    assert.equal(method, "tools/call");
    native.calls.push(params);
    return textResult("TextEdit β€” com.apple.TextEdit [running, last-used=2026-09-22, uses=4]");
  };
  const session = new PersistentJavaScriptSession({ native });
  const result = await session.run(`
    var apps = await cua.listApps({ emit: false });
    nodeRepl.write(JSON.stringify(apps));
  `);
  assert.equal(result.isError, false);
  assert.deepEqual(JSON.parse(result.content.at(-1).text), [
    { id: "com.apple.TextEdit", displayName: "TextEdit", isRunning: true, lastUsedDate: "2026-09-22", useCount: 4 },
  ]);
});

test("top-level await and lexical bindings persist until reset", async () => {
  const session = new PersistentJavaScriptSession({ native: mockNative() });
  let result = await session.run("var answer = await Promise.resolve(41); nodeRepl.write(answer);");
  assert.equal(result.isError, false);
  assert.equal(result.content.at(-1).text, "41");
  result = await session.run("answer += 1; nodeRepl.write(answer);");
  assert.equal(result.content.at(-1).text, "42");
  session.reset();
  result = await session.run("nodeRepl.write(typeof answer);");
  assert.equal(result.content.at(-1).text, "undefined");
});

test("return values are not implicitly serialized into MCP output", async () => {
  const session = new PersistentJavaScriptSession({ native: mockNative() });
  const result = await session.run("({ huge: 'intermediate-only' })");
  assert.equal(result.isError, false);
  assert.equal(result.content.at(-1).text, "(no output)");
});

test("app-bound API composes actions and state in one js call", async () => {
  const native = mockNative();
  const session = new PersistentJavaScriptSession({ native });
  const result = await session.run(`
    var app = await cua.getApp("Text");
    await app.click(7, { clickMethod: "accessibility" });
    await app.typeText("hello");
    await app.getAXState();
  `);
  assert.equal(result.isError, false);
  assert.deepEqual(native.calls.map(call => call.name), ["get_app_state", "click", "type_text", "get_app_state"]);
  assert.equal(native.calls[1].arguments.element_index, 7);
  assert.equal(native.calls[1].arguments.click_method, "accessibility");
  assert.ok(result.content.some(item => item.type === "text" && item.text === "state:Text"));
});

for (const Session of [PersistentJavaScriptSession, WorkerJavaScriptSession]) {
  test(`${Session.name}: selective observation is explicit and legacy calls keep their arguments`, async t => {
    const native = mockNative();
    const session = new Session({ native });
    t.after(() => session.close?.());
    const result = await session.run(`
      var selectiveApp = await cua.getApp("Text", { initialState: false });
      await selectiveApp.getScreenshot({ includeAX: false, emit: false });
      await selectiveApp.getAXState({ includeScreenshot: false, emit: false });
      await selectiveApp.getScreenshot({ emit: false });
      await selectiveApp.getAXState({ emit: false });
    `);
    assert.equal(result.isError, false);
    assert.deepEqual(native.calls.map(call => call.arguments), [
      { app: "Text", include_ax: false, text_limit: 1 },
      { app: "Text", include_screenshot: false },
      { app: "Text", text_limit: 1 },
      { app: "Text" },
    ]);
    const count = native.calls.length;
    for (const code of [
      'await selectiveApp.getScreenshot({ includeAX: "false" })',
      'await selectiveApp.getAXState({ includeScreenshot: null })',
    ]) {
      const invalid = await session.run(code);
      assert.equal(invalid.isError, true);
      assert.match(invalid.content.at(-1).text, /must be a boolean/);
    }
    assert.equal(native.calls.length, count);
    const empty = await session.run('await selectiveApp.getAXStateAndScreenshot({ includeAX: false, includeScreenshot: false })');
    assert.equal(empty.isError, true);
    assert.match(empty.content.at(-1).text, /At least one/);
    assert.equal(native.calls.length, count);
  });
}

test("app-bound API forwards background-operation options", async () => {
  const native = mockNative();
  const session = new PersistentJavaScriptSession({ native });
  const result = await session.run(`
    var backgroundApp = await cua.getApp("Text");
    await backgroundApp.getAXState({ emit: false, textLimit: "max", maxTreeNodes: 42, maxTreeDepth: 7, windowPlacement: "agent_display" });
    await backgroundApp.getScreenshot({ emit: false, windowPlacement: "keep" });
    await backgroundApp.getAXStateAndScreenshot({ emit: false, windowPlacement: "restore" });
    await backgroundApp.typeText("hello", { keyMethod: "sky_key" });
    await backgroundApp.pressKey("cmd+a", { keyMethod: "sky_key" });
    await backgroundApp.pressKey("ctrl+a", { keyMethod: "global" });
  `);

  assert.equal(result.isError, false);
  assert.deepEqual(native.calls[1], {
    name: "get_app_state",
    arguments: {
      app: "Text",
      text_limit: "max",
      max_tree_nodes: 42,
      max_tree_depth: 7,
      window_placement: "agent_display",
    },
  });
  assert.deepEqual(native.calls[2], {
    name: "get_app_state",
    arguments: { app: "Text", window_placement: "keep", text_limit: 1 },
  });
  assert.deepEqual(native.calls[3], {
    name: "get_app_state",
    arguments: { app: "Text", window_placement: "restore" },
  });
  assert.deepEqual(native.calls[4], {
    name: "type_text",
    arguments: { app: "Text", text: "hello", key_method: "sky_key" },
  });
  assert.deepEqual(native.calls[5], {
    name: "press_key",
    arguments: { app: "Text", key: "cmd+a", key_method: "sky_key" },
  });
  assert.deepEqual(native.calls[6], {
    name: "press_key",
    arguments: { app: "Text", key: "ctrl+a", key_method: "global" },
  });
});

test("tool errors are catchable in JavaScript", async () => {
  const session = new PersistentJavaScriptSession({ native: mockNative() });
  const result = await session.run(`
    var badApp = await cua.getApp("Text");
    try { await badApp.typeText("fail"); } catch (error) { nodeRepl.write("caught: " + error.message); }
  `);
  assert.equal(result.isError, false);
  assert.equal(result.content.at(-1).text, "caught: typing failed");
});

test("thrown errors settle the call and keep bindings", async () => {
  const session = new PersistentJavaScriptSession({ native: mockNative() });
  await session.run("var kept = 1;");
  let result = await session.run("nodeRepl.write('before'); null[1];", 1000);
  assert.equal(result.isError, true);
  assert.match(result.content.at(-1).text, /^Error: Cannot read properties of null/);
  result = await session.run("await 1; throw new Error('after await');", 1000);
  assert.equal(result.isError, true);
  assert.equal(result.content.at(-1).text, "Error: after await");
  result = await session.run("nodeRepl.write(kept);");
  assert.equal(result.content.at(-1).text, "1");
});

for (const Session of [PersistentJavaScriptSession, WorkerJavaScriptSession]) {
  test(`${Session.name}: a late throw from a finished call does not reach the next call`, async () => {
    const session = new Session({ native: mockNative() });
    try {
      let result = await session.run(`var kept = 1; setTimeout(() => { throw new Error("late"); }, 20);
        setTimeout(() => nodeRepl.write("late write"), 20);`, 1000);
      assert.equal(result.isError, false);
      result = await session.run(`await new Promise(resolve => setTimeout(resolve, 100)); nodeRepl.write("next " + kept);`, 1000);
      assert.deepEqual(result, { content: [{ type: "text", text: "next 1" }], isError: false });
      result = await session.run(`nodeRepl.write("after");`, 1000);
      assert.deepEqual(result, { content: [{ type: "text", text: "after" }], isError: false });
    } finally {
      await session.close?.();
    }
  });
}

test("screenshots emit binary image content", async () => {
  const session = new PersistentJavaScriptSession({ native: mockNative() });
  const result = await session.run(`
    var imageApp = await cua.getApp("Text");
    var imageBytes = await imageApp.getScreenshot();
    nodeRepl.write(imageBytes.length);
  `);
  assert.equal(result.isError, false);
  assert.ok(result.content.some(item => item.type === "image" && item.mimeType === "image/png"));
  assert.equal(result.content.at(-1).text, "8");
});

test("JsonLinePeer initializes and correlates native MCP responses", async () => {
  const child = new EventEmitter();
  child.stdout = new PassThrough(); child.stderr = new PassThrough(); child.stdin = new PassThrough();
  child.kill = () => {};
  let input = "";
  child.stdin.setEncoding("utf8");
  child.stdin.on("data", chunk => {
    input += chunk;
    for (;;) {
      const newline = input.indexOf("\n"); if (newline < 0) break;
      const request = JSON.parse(input.slice(0, newline)); input = input.slice(newline + 1);
      if (request.id !== undefined) child.stdout.write(`${JSON.stringify({ jsonrpc: "2.0", id: request.id, result: request.method === "initialize" ? { ok: true } : { content: [] } })}\n`);
    }
  });
  const peer = new JsonLinePeer({ child });
  assert.deepEqual(await peer.initialize(), { ok: true });
  assert.deepEqual(await peer.request("tools/call", { name: "list_apps" }), { content: [] });
  peer.closed = true;
});

test("worker session terminates CPU-bound code and recovers with a fresh kernel", async () => {
  const session = new WorkerJavaScriptSession({ native: mockNative() });
  let result = await session.run("while (true) {}", 100);
  assert.equal(result.isError, true);
  assert.match(result.content[0].text, /timed out/);
  result = await session.run("nodeRepl.write(21 * 2);", 5_000);
  assert.equal(result.isError, false);
  assert.equal(result.content.at(-1).text, "42");
  await session.close();
});

test("worker session returns a structured error for invalid source input", async () => {
  const session = new WorkerJavaScriptSession({ native: mockNative() });
  const result = await session.run(undefined, 5_000);
  assert.equal(result.isError, true);
  assert.match(result.content[0].text, /non-empty JavaScript source/);
  const recovered = await session.run("nodeRepl.write('ready');", 5_000);
  assert.equal(recovered.isError, false);
  assert.equal(recovered.content.at(-1).text, "ready");
  await session.close();
});

test("worker session serializes concurrent callers", async () => {
  const session = new WorkerJavaScriptSession({ native: mockNative() });
  const first = session.run("await new Promise(resolve => setTimeout(resolve, 40)); var order = ['first']; nodeRepl.write(order.join(','));", 5_000);
  const second = session.run("order.push('second'); nodeRepl.write(order.join(','));", 5_000);
  assert.equal((await first).content.at(-1).text, "first");
  assert.equal((await second).content.at(-1).text, "first,second");
  await session.close();
});

test("native session end rejects pending work and initializes a fresh child without replay", async () => {
  const source = `
    let buffer = '';
    process.stdin.setEncoding('utf8');
    process.stdin.on('data', chunk => {
      buffer += chunk;
      for (;;) {
        const end = buffer.indexOf('\\n'); if (end < 0) break;
        const request = JSON.parse(buffer.slice(0, end)); buffer = buffer.slice(end + 1);
        if (request.id === undefined || request.method === 'stall') continue;
        process.stdout.write(JSON.stringify({jsonrpc:'2.0', id:request.id, result:{pid:process.pid}}) + '\\n');
      }
    });
  `;
  const peer = new JsonLinePeer({ command: process.execPath, args: ["--input-type=module", "-e", source] });
  try {
    const first = await peer.initialize();
    const old = peer.child;
    const pending = peer.request("stall", {}, 5_000);
    const rejected = assert.rejects(pending, /session ended/);
    peer.endSession();
    await rejected;
    const next = await peer.request("ping", {}, 5_000);
    assert.notEqual(next.pid, first.pid);
    old.emit("exit", 1, null);
    assert.equal(peer.closed, false);
    await assert.rejects(peer.request("stall", {}, 20), /timed out/);
    assert.equal(peer.closed, true);
    const recovered = await peer.request("ping", {}, 5_000);
    assert.notEqual(recovered.pid, next.pid);
  } finally { peer.close(); }
});

test("worker reset and timeout end the native GUI lease", async () => {
  const native = mockNative();
  let ended = 0;
  native.endSession = () => { ended += 1; };
  const session = new WorkerJavaScriptSession({ native });
  try {
    await session.reset();
    assert.equal(ended, 1);
    const result = await session.run("while (true) {}", 100);
    assert.equal(result.isError, true);
    assert.equal(ended, 2);
  } finally { await session.close(); }
});


test("reset drops old worker native requests before a replacement kernel starts", async () => {
  const native = mockNative();
  const session = new WorkerJavaScriptSession({ native });
  try {
    const old = session.worker;
    const resetting = session.reset();
    old.emit("message", { type: "native_request", id: 1, method: "tools/call", params: { name: "click", arguments: {} } });
    await resetting;
    assert.equal(native.calls.length, 0);
    old.emit("message", { type: "native_request", id: 2, method: "tools/call", params: { name: "click", arguments: {} } });
    assert.equal(native.calls.length, 0);
  } finally { await session.close(); }
});

test("lightweight app binding queries and acts across persistent JS calls without a snapshot", async () => {
  const native = mockNative();
  const request = native.request;
  native.request = async (method, params) => {
    if (params.name !== "query") return request(method, params);
    native.calls.push(params);
    return textResult(JSON.stringify({ matches: [{ index: 1000001, title: "Send" }], truncated: false, stop_reason: "complete", visited_nodes: 3, window_id: 12 }));
  };
  const session = new PersistentJavaScriptSession({ native });
  let result = await session.run(`var queriedApp = await cua.getApp("Text", { initialState: false }); var found = await queriedApp.query({ text: "Send", role: "button", exact: true, maxNodes: 50, windowId: 12 }); nodeRepl.write(found);`);
  assert.equal(result.isError, false, result.content.at(-1).text);
  assert.deepEqual(native.calls.map(c => c.name), ["query"]);
  assert.deepEqual(native.calls[0].arguments, { app: "Text", text: "Send", role: "button", exact: true, limit: undefined, max_nodes: 50, window_id: 12 });
  result = await session.run(`await queriedApp.click(found.matches[0].index);`);
  assert.equal(result.isError, false);
  assert.equal(native.calls.at(-1).arguments.element_index, 1000001);
  assert.equal(native.calls.some(c => c.name === "get_app_state"), false);
});

test("query rejects invalid options and preserves native errors", async () => {
  const native = mockNative(), session = new PersistentJavaScriptSession({ native });
  await session.run(`var queriedApp = await cua.getApp("Text", { initialState: false });`);
  for (const options of [`{}`, `{text: "Send", windowId: -1}`, `{text: "Send", windowId: 4294967296}`, `{text: "Send", exact: 1}`, `{text: 3}`, `{role: "button", maxNodes: 1.5}`, `{text: "Send", typo: true}`]) {
    const result = await session.run(`await queriedApp.query(${options});`);
    assert.equal(result.isError, true, options);
  }
  assert.equal(native.calls.length, 0);
  native.request = async () => textResult("query is unavailable on this platform", true);
  const result = await session.run(`await queriedApp.query({text: "Send"});`);
  assert.equal(result.isError, true);
  assert.match(result.content.at(-1).text, /unavailable on this platform/);
});

test("query returns truncation metadata and rejects malformed native responses", async () => {
  const native = mockNative(), session = new PersistentJavaScriptSession({ native });
  native.request = async () => textResult(JSON.stringify({ matches: [], truncated: true, stop_reason: "timeout" }));
  const result = await session.run(`var queriedApp = await cua.getApp("Text", { initialState: false }); nodeRepl.write(JSON.stringify(await queriedApp.query({role: "button"})));`);
  assert.equal(result.isError, false);
  assert.equal(JSON.parse(result.content.at(-1).text).stop_reason, "timeout");
  native.request = async () => textResult("[]");
  const bad = await session.run(`await queriedApp.query({role: "button"});`);
  assert.equal(bad.isError, true);
  assert.match(bad.content.at(-1).text, /Invalid query response/);
});


for (const Session of [PersistentJavaScriptSession, WorkerJavaScriptSession]) {
  test(`${Session.name}: selectText forwards all selection modes without a snapshot`, async t => {
    const native = mockNative(), session = new Session({ native });
    t.after(() => session.close?.());
    let result = await session.run(`var selectionApp = await cua.getApp("Text", { initialState: false }); await selectionApp.selectText(1000001, "δΈ­ζ–‡πŸ™‚", {prefix: "before ", suffix: " after"});`);
    assert.equal(result.isError, false, result.content.at(-1).text);
    assert.deepEqual(native.calls[0], { name: "select_text", arguments: { app: "Text", element_index: 1000001, text: "δΈ­ζ–‡πŸ™‚", prefix: "before ", suffix: " after", selection: undefined } });
    for (const mode of ["text", "cursor_before", "cursor_after"]) {
      result = await session.run(`await selectionApp.selectText("5", "target", {selection: "${mode}"});`);
      assert.equal(result.isError, false, result.content.at(-1).text);
      assert.equal(native.calls.at(-1).arguments.selection, mode);
    }
    assert.equal(native.calls.some(call => call.name === "get_app_state"), false);
  });

  test(`${Session.name}: selectText rejects invalid inputs before native calls`, async t => {
    const native = mockNative(), session = new Session({ native });
    t.after(() => session.close?.());
    await session.run(`var selectionApp = await cua.getApp("Text", { initialState: false });`);
    for (const args of [`-1,"x"`, `1.5,"x"`, `true,"x"`, `"bad","x"`, `[],"x"`, `0,""`, `0,3`, `0,"x",null`, `0,"x",[]`, `0,"x",{prefix:2}`, `0,"x",{suffix:null}`, `0,"x",{selection:"after"}`, `0,"x",{selection:""}`, `0,"x",{typo:true}`]) {
      const result = await session.run(`await selectionApp.selectText(${args});`);
      assert.equal(result.isError, true, args);
    }
    assert.equal(native.calls.length, 0);
  });

  test(`${Session.name}: selectText preserves native verification and platform failures`, async t => {
    const native = mockNative(), session = new Session({ native });
    t.after(() => session.close?.());
    await session.run(`var selectionApp = await cua.getApp("Text", { initialState: false });`);
    for (const message of ["select_text could not verify the requested range", 'unsupportedTool("select_text")']) {
      native.request = async () => textResult(message, true);
      const result = await session.run(`await selectionApp.selectText(0,"target");`);
      assert.equal(result.isError, true);
      assert.ok(result.content.at(-1).text.includes(message), result.content.at(-1).text);
    }
  });
}