Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
8.7 KB
Raw Blame
import assert from "node:assert/strict";
import test from "node:test";
import { runInNewContext } from "node:vm";
import { assertAbsent, locate, TargetNotFoundError, mapKey, parseTarget, readDom, waitForLocated } from "../src/input.ts";
import type { Surface } from "../src/surface.ts";

function surfaceReturning(value: unknown): Surface {
  return {
    handle: { name: "input-test", kind: "electron", hostKind: "test", cdpUrl: "http://127.0.0.1:1" },
    client: {
      async send(method) {
        if (method === "Runtime.evaluate") return { result: { objectId: "global" } };
        if (method === "Runtime.callFunctionOn") return { result: { value } };
        throw new Error(`Unexpected CDP method ${method}.`);
      },
      close() {},
    },
  };
}

test("parseTarget normalizes bare, structured, and regular-expression targets", () => {
  assert.deepEqual(parseTarget("composer"), {
    bare: { kind: "string", value: "composer" },
    nth: 0,
    composer: true,
  });
  assert.deepEqual(parseTarget({ role: "textbox", label: /password/i, nth: 1 }), {
    text: undefined,
    role: "textbox",
    label: { kind: "regexp", value: "password", flags: "i" },
    placeholder: undefined,
    testId: undefined,
    nth: 1,
    composer: false,
  });
  assert.deepEqual(parseTarget({ role: "button", text: /^Model\b/i }), {
    text: { kind: "regexp", value: "^Model\\b", flags: "i" },
    role: "button",
    label: undefined,
    placeholder: undefined,
    testId: undefined,
    nth: 0,
    composer: false,
  });
});

test("mapKey produces CDP key fields and modifier bits", () => {
  assert.deepEqual(mapKey("Enter"), {
    key: "Enter",
    code: "Enter",
    windowsVirtualKeyCode: 13,
    modifiers: 0,
  });
  assert.deepEqual(mapKey("Meta+R"), {
    key: "R",
    code: "KeyR",
    windowsVirtualKeyCode: 82,
    modifiers: 4,
  });
  assert.throws(() => mapKey("Hyper+R"), /Unsupported modifier/);
});

test("locate reports visible button and link names when no target matches", async () => {
  const surface = surfaceReturning({
    notFound: true,
    candidates: ['button "Model · gpt-5"', 'link "Provider docs"'],
  });
  await assert.rejects(
    locate(surface, { role: "button", text: "Missing" }),
    /Visible button\/link candidates: button "Model · gpt-5", link "Provider docs"/,
  );
});

test("locate finds an accessible switch rendered as a span and keeps its click hit test", async () => {
  class ElementFixture {
    tagName = "SPAN";
    children: ElementFixture[] = [];
    parentElement = null;
    innerText = "";
    textContent = "";
    scrolled = false;
    attributes: Record<string, string> = { role: "switch", "aria-label": "Best practices" };
    getAttribute(name: string) { return this.attributes[name] ?? null; }
    hasAttribute(name: string) { return name in this.attributes; }
    closest() { return null; }
    getBoundingClientRect() { return { left: 10, top: 20, x: 10, y: 20, width: 44, height: 20 }; }
    scrollIntoView() { this.scrolled = true; }
    contains(element: unknown) { return element === this; }
  }
  const control = new ElementFixture();
  const surface = surfaceReturning(null);
  surface.client.send = async (method, params) => {
    if (method === "Runtime.evaluate") return { result: { objectId: "global" } };
    assert.equal(method, "Runtime.callFunctionOn");
    assert.ok(params && typeof params.functionDeclaration === "string" && Array.isArray(params.arguments));
    const serialized = params.arguments[0].value;
    const value = runInNewContext(`(${params.functionDeclaration})(${JSON.stringify(serialized)})`, {
      Element: ElementFixture,
      HTMLElement: ElementFixture,
      HTMLInputElement: class extends ElementFixture {},
      HTMLTextAreaElement: class extends ElementFixture {},
      HTMLSelectElement: class extends ElementFixture {},
      HTMLButtonElement: class extends ElementFixture {},
      innerWidth: 1280,
      innerHeight: 900,
      getComputedStyle: () => ({ display: "inline-flex", visibility: "visible", opacity: "1" }),
      document: {
        getElementById: () => null,
        elementFromPoint: () => control,
        querySelectorAll(selector: string) {
          const matches = selector.split(",").some(part => {
            const token = part.trim();
            if (token === control.tagName.toLowerCase()) return true;
            const attribute = /^\[([\w-]+)(?:="([^"]*)")?\]$/.exec(token);
            return attribute ? control.hasAttribute(attribute[1]) && (attribute[2] === undefined || control.getAttribute(attribute[1]) === attribute[2]) : false;
          });
          return matches ? [control] : [];
        },
      },
    });
    return { result: { value } };
  };
  const located = await locate(surface, { role: "switch", label: "Best practices" });
  assert.equal(located.tag, "span");
  assert.equal(located.name, "Best practices");
  assert.equal(located.visible, true);
  assert.equal(located.hitTestOk, true);
  assert.equal(control.scrolled, true);
});

test("waitForLocated identifies the element covering a visible target", async () => {
  const surface = surfaceReturning({
    center: { x: 50, y: 25 },
    rect: { x: 0, y: 0, width: 100, height: 50 },
    tag: "button",
    name: "Run task",
    visible: true,
    hitTestOk: false,
    editable: false,
    value: "",
    text: "Run task",
    covering: { tag: "div", role: "dialog", text: "Blocking overlay" },
  });
  await assert.rejects(
    waitForLocated(surface, "Run task", { mustHitTest: true, timeoutMs: 10 }),
    /visible=true, hitTestOk=false\. Covered by div role="dialog" text="Blocking overlay"/,
  );
});


test("only a successful browser inspection can report a missing target", async () => {
  await assert.rejects(locate(surfaceReturning({ notFound: true }), "Missing"), TargetNotFoundError);
  await assert.rejects(locate(surfaceReturning(null), "Missing"), error =>
    error instanceof Error && !(error instanceof TargetNotFoundError));
  const disconnected = surfaceReturning(null);
  disconnected.client.send = async () => { throw new Error("CDP disconnected"); };
  await assert.rejects(locate(disconnected, "Missing"), /CDP disconnected/);
});


test("absence never turns disconnection or malformed browser results into a pass", async () => {
  await assertAbsent(surfaceReturning({ notFound: true }), "Missing", 10);
  await assert.rejects(assertAbsent(surfaceReturning(null), "Missing", 10), /Could not locate/);
  await assert.rejects(assertAbsent(surfaceReturning({ center: {}, rect: {} }), "Missing", 10), /invalid located-element geometry/);
  const disconnected = surfaceReturning(null);
  disconnected.client.send = async () => { throw new Error("CDP disconnected"); };
  await assert.rejects(assertAbsent(disconnected, "Missing", 10), /CDP disconnected/);
  await assert.rejects(assertAbsent(surfaceReturning(null), "Missing", 0), /positive duration/);
});

test("absence distinguishes a hidden target from a visible target", async () => {
  const target = { center: { x: 1, y: 1 }, rect: { x: 0, y: 0, width: 2, height: 2 }, tag: "div", name: "Error", visible: false, hitTestOk: false, editable: false, value: "", text: "Error", covering: null };
  await assertAbsent(surfaceReturning(target), "Error", 10);
  await assert.rejects(assertAbsent(surfaceReturning({ ...target, visible: true }), "Error", 10), /remained visible/);
});

test("DOM inspection projects geometry and focus without exposing input values", async () => {
  const input = { tagName: "INPUT", textContent: "", value: "private-password", getBoundingClientRect: () => ({ left: 1, right: 101, top: 2, bottom: 22, width: 100, height: 20 }) };
  const surface = surfaceReturning(null);
  surface.client.send = async (method, params) => {
    if (method === "Runtime.evaluate") return { result: { objectId: "global" } };
    assert.equal(method, "Runtime.callFunctionOn");
    assert.ok(params && typeof params.functionDeclaration === "string");
    assert.deepEqual(params.arguments, [{ value: "input" }]);
    const value = runInNewContext(`(${params.functionDeclaration})("input")`, {
      document: { documentElement: { clientWidth: 390, scrollWidth: 400 }, activeElement: input,
        querySelectorAll(selector: string) { assert.equal(selector, "input"); return [input]; } },
    });
    return { result: { value } };
  };
  const result = await readDom(surface, "input");
  assert.equal(result.elements[0]?.focused, true);
  assert.equal(result.elements[0]?.rect.width, 100);
  assert.equal(result.documentWidth > result.viewportWidth, true);
  assert.equal(JSON.stringify(result).includes("private-password"), false);
  assert.equal(input.value, "private-password");
  await assert.rejects(readDom(surfaceReturning(null), "input"), /invalid snapshot/);
  await assert.rejects(readDom(surfaceReturning({ viewportWidth: 390, documentWidth: 390, elements: [{}] }), "input"), /invalid snapshot/);
});