Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

7.0 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import { catalogueEntry, classifyTool } from "../src/plugins/catalogue";

/**
 * What an action does, when the vendor said so and when nobody did.
 *
 * The property under test is the direction of the failure. Exactly two things can earn a read, and
 * both require somebody to have said so: the vendor recorded exactly `read`, or a curated entry
 * advertised the action and a reviewed list declined to call it a write. Everything else — a
 * recorded write, an unrecognised value, a different case, an empty string, a name no server
 * advertised, a server nobody reviewed — is a write. That asymmetry is the point: an action wrongly
 * gated as a write costs a confirmation, and one wrongly waved through as a read costs somebody's
 * mailbox.
 *
 * THE RECORDED VALUE MAY NARROW WHAT A BOT MAY DO AND MAY NEVER WIDEN IT. It arrives from a vendor
 * listing into a plain `text` column with no constraint on its contents, so it outranks nothing that
 * a human reviewed. Both halves are pinned below: a recorded write settles an action the curated
 * list forgot, and a recorded read cannot unsettle one the curated list named.
 *
 * The first block passes a null entry throughout, which is the brokered shape — a Composio app has
 * no curated entry behind it. The second block passes a real one, because a null entry is itself a
 * blanket write and would let the precedence cases below pass without exercising the precedence.
 */
describe("classifyTool with a recorded effect", () => {
  test("a recorded read is a read", () => {
    expect(classifyTool(null, "GMAIL_FETCH_EMAILS", true, "read")).toBe("read");
  });

  test("a recorded write is a write", () => {
    expect(classifyTool(null, "GMAIL_SEND_EMAIL", true, "write")).toBe("write");
  });

  test("no recorded effect is a write, not a read", () => {
    expect(classifyTool(null, "GMAIL_SEND_EMAIL", true, null)).toBe("write");
    expect(classifyTool(null, "GMAIL_SEND_EMAIL", true, undefined)).toBe(
      "write",
    );
    expect(classifyTool(null, "GMAIL_SEND_EMAIL", true, "")).toBe("write");
  });

  test("a value nothing recognises is a write", () => {
    // A future label, a typo, or a column somebody wrote by hand. None is a licence to read.
    expect(classifyTool(null, "GMAIL_SEND_EMAIL", true, "readonly")).toBe(
      "write",
    );
    expect(classifyTool(null, "GMAIL_SEND_EMAIL", true, "destructive")).toBe(
      "write",
    );
    expect(classifyTool(null, "GMAIL_SEND_EMAIL", true, "READ")).toBe("write");
  });

  test("a recorded read cannot rescue an action the server never advertised", () => {
    // The name came from somewhere other than a listing, so no recorded effect is about it.
    expect(classifyTool(null, "GMAIL_INVENTED", false, "read")).toBe("write");
  });
});

describe("a recorded effect against a curated entry", () => {
  const notion = catalogueEntry("notion");

  /*
   * The two names every case below is built on, asserted once.
   *
   * `notion-update-page` has to be ON the write list and `notion-fetch` has to be OFF it, or the
   * expectations stop meaning what they say: a reviewed write whose name drifted off the list would
   * turn the precedence cases into ordinary unlisted-tool cases and they would keep passing.
   */
  test("the entry these cases are about names one of them a write and not the other", () => {
    expect(notion).not.toBeNull();
    expect(notion?.writeTools).toContain("notion-update-page");
    expect(notion?.writeTools).not.toContain("notion-fetch");
  });

  test("a recorded read cannot override a curated entry's write list", () => {
    /*
     * THE CASE THIS WHOLE BLOCK EXISTS FOR. `effect` is vendor-supplied text in a column with no
     * check constraint and no product writer other than the refresh path, so a `read` in it is
     * reachable by a hand edit or a restore. `writeTools` was reviewed by a person. Letting the
     * column win here would buy an action LESS scrutiny than review already gave it, which is the
     * one direction this classifier must never move in.
     */
    expect(classifyTool(notion, "notion-update-page", true, "read")).toBe(
      "write",
    );
    expect(classifyTool(notion, "notion-create-pages", true, "read")).toBe(
      "write",
    );
    expect(classifyTool(notion, "notion-move-pages", true, "read")).toBe(
      "write",
    );
  });

  test("a recorded read still settles an action the write list does not name", () => {
    // The permitted direction, and the reason the column is consulted at all: where review said
    // nothing, the vendor's own label is the better source and is taken at its word.
    expect(classifyTool(notion, "notion-fetch", true, "read")).toBe("read");
  });

  test("a recorded write settles an action the write list forgot", () => {
    // The other permitted direction. The write list is known-incomplete, so a vendor saying an
    // action writes narrows what a Bot may do and is honoured.
    expect(classifyTool(notion, "notion-fetch", true, "write")).toBe("write");
  });

  test("an empty recorded effect is a write, not an absence of opinion", () => {
    /*
     * A `text` column holding the empty string is a value, not a null. Reading it as "nothing was
     * recorded" sends an advertised action that no reviewed list names down the read branch, which
     * is the widening this classifier exists to refuse. Both sides of the write list are pinned so
     * the answer cannot depend on which one the name falls on.
     */
    expect(classifyTool(notion, "notion-fetch", true, "")).toBe("write");
    expect(classifyTool(notion, "notion-update-page", true, "")).toBe("write");
  });

  test("a recorded value nothing recognises is a write", () => {
    // A label a vendor invents later, a typo, or the wrong case. None of them is `read`, so none of
    // them earns a read, whether or not the reviewed list names the action.
    for (const value of ["readonly", "READ", "Read", "destructive", "none"]) {
      expect(classifyTool(notion, "notion-fetch", true, value)).toBe("write");
      expect(classifyTool(notion, "notion-update-page", true, value)).toBe(
        "write",
      );
    }
  });

  test("nothing recorded leaves the curated write list deciding", () => {
    // The behaviour that shipped before the column existed, unchanged for every row written before
    // it. Null and undefined are the column saying nothing, which is not a value.
    for (const value of [null, undefined]) {
      expect(classifyTool(notion, "notion-update-page", true, value)).toBe(
        "write",
      );
      expect(classifyTool(notion, "notion-fetch", true, value)).toBe("read");
    }
  });

  test("a recorded read cannot rescue a name the entry's server never advertised", () => {
    // Checked before either source, so the model-invented name is refused whatever the column says
    // and whatever the reviewed list says.
    expect(classifyTool(notion, "notion-fetch", false, "read")).toBe("write");
    expect(classifyTool(notion, "notion-invented", false, "read")).toBe(
      "write",
    );
  });
});