Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

9.4 KB
Raw Blame
import { afterEach, describe, expect, test } from "bun:test";
import { listTools, useArcadeClient } from "../src/plugins/arcade";
import { createArcadeClient } from "../src/plugins/arcade-adapter";
import type { AccountRef } from "../src/plugins/shared-accounts";

/**
 * AN EMPTY VENDOR STRUCTURE IS NOT A VENDOR ANSWER, AGAINST A REAL HTTP ENGINE.
 *
 * `metadata: { behavior: {} }` and `requirements: { secrets: {} }` are containers Arcade opened and
 * put nothing in. Read as answers they say "not destructive" and "nothing is required" — a
 * permission decision taken on missing data, and the opposite of what an ABSENT `metadata` or an
 * absent `behavior` already answers two lines away in the same function.
 *
 * SERVED OVER A REAL SOCKET, as `arcade-pagination-engine.test.ts` and
 * `arcade-malformed-listing-engine.test.ts` are and for their reason: the rest of the adapter suite
 * stubs the SDK's resource PROTOTYPES and so decides for itself what the adapter is handed, which
 * cannot see anything that happens between the wire and the reader. Here the SDK parses the body
 * and builds its own pages, so what these assertions read is what a deployment would.
 */

const account: AccountRef = {
  holder: "person",
  userId: "u1",
  vendorUserId: "vendor-u1",
};

type Engine = { port: number; stop: () => void };

/** The rows `/v1/tools` serves, set per test. One short page ends the listing. */
let serving: unknown[] = [];

function startEngine(): Engine {
  const server = Bun.serve({
    port: 0,
    fetch(request) {
      const url = new URL(request.url);
      const limit = Number(url.searchParams.get("limit") ?? "100");
      const offset = Number(url.searchParams.get("offset") ?? "0");
      if (url.pathname === "/v1/tools") {
        const page = serving.slice(offset, offset + limit);
        return Response.json({
          items: page,
          offset,
          limit,
          page_count: page.length,
        });
      }
      return new Response("not found", { status: 404 });
    },
  });
  // `Bun.serve().port` is `number | undefined`; refuse loudly rather than coerce, as
  // `arcade-revoke-invariant.test.ts` does.
  const { port } = server;
  if (port === undefined)
    throw new Error("the stub vendor did not bind a port");
  return { port, stop: () => server.stop(true) };
}

let running: Engine | null = null;
const wasBaseURL = process.env.ARCADE_BASE_URL;

function engine(rows: unknown[]): void {
  serving = rows;
  running = startEngine();
  process.env.ARCADE_BASE_URL = `http://localhost:${running.port}`;
}

afterEach(() => {
  running?.stop();
  running = null;
  serving = [];
  useArcadeClient(null);
  if (wasBaseURL === undefined) delete process.env.ARCADE_BASE_URL;
  else process.env.ARCADE_BASE_URL = wasBaseURL;
});

/** One listed action, with whatever `metadata` and `requirements` the case is about. */
const toolRow = (
  name: string,
  metadata: unknown,
  requirements: unknown = {
    authorization: {
      provider_id: "google",
      oauth2: { scopes: [] },
      token_status: "completed",
    },
  },
) => ({
  qualified_name: `Gmail.${name}`,
  description: "",
  toolkit: { name: "Gmail", description: "" },
  input: { parameters: [] },
  metadata,
  requirements,
});

const listed = async () => {
  const { actions } = createArcadeClient("never-dialled");
  useArcadeClient(actions);
  return await listTools({ url: "arcade://Gmail" });
};

describe("an empty behavior object is the vendor saying nothing, not saying safe", () => {
  test("a behavior object stating neither flag is read as the closed verdict", async () => {
    engine([toolRow("EmptyBehavior", { behavior: {} })]);

    expect(await listed()).toMatchObject([
      {
        name: "EmptyBehavior",
        vendorName: "Gmail.EmptyBehavior",
        effect: "write",
        destructive: true,
      },
    ]);
  });

  test("a behavior object carrying only unrelated keys says nothing either", async () => {
    engine([toolRow("Idempotent", { behavior: { idempotent: true } })]);

    expect(await listed()).toMatchObject([
      {
        name: "Idempotent",
        vendorName: "Gmail.Idempotent",
        effect: "write",
        destructive: true,
      },
    ]);
  });

  test("an empty behavior does not count towards classifiedCount", async () => {
    engine([
      toolRow("EmptyBehavior", { behavior: {} }),
      toolRow("Annotated", {
        behavior: { read_only: true, destructive: false },
      }),
    ]);
    const { broker } = createArcadeClient("never-dialled");

    const apps = await broker.listApps();
    expect(apps.length).toBe(1);
    expect(apps[0]?.actionCount).toBe(2);
    // ONE of the two: the empty container is an action Arcade has not classified, which is exactly
    // what the coverage warning exists to name.
    expect(apps[0]?.classifiedCount).toBe(1);
  });

  test("a stated behaviour is still read, in both directions", async () => {
    engine([
      toolRow("Read", { behavior: { read_only: true } }),
      toolRow("Write", { behavior: { destructive: false, read_only: false } }),
      toolRow("Delete", { behavior: { destructive: true } }),
    ]);

    expect(await listed()).toMatchObject([
      {
        name: "Read",
        vendorName: "Gmail.Read",
        effect: "read",
        destructive: false,
      },
      {
        name: "Write",
        vendorName: "Gmail.Write",
        effect: "write",
        destructive: false,
      },
      {
        name: "Delete",
        vendorName: "Gmail.Delete",
        effect: "write",
        destructive: true,
      },
    ]);
  });
});

describe("a secrets requirement that is not a list is unreadable, not satisfied", () => {
  const withSecrets = (secrets: unknown) =>
    toolRow(
      "Send",
      { behavior: { read_only: true } },
      {
        secrets,
        authorization: {
          provider_id: "google",
          oauth2: { scopes: [] },
          token_status: "completed",
        },
      },
    );

  test("an empty secrets object does not answer authorized", async () => {
    engine([withSecrets({})]);
    const { broker } = createArcadeClient("never-dialled");

    expect(await broker.grantsFor({ account, toolkit: "Gmail" })).toEqual([
      {
        action: "Gmail.Send",
        state: "unknown",
        reason:
          "Arcade listed this action's deployment secrets as something other than a list, so whether any is unset could not be read.",
      },
    ]);
  });

  test("no secrets field at all is still no secrets required", async () => {
    engine([toolRow("Send", { behavior: { read_only: true } })]);
    const { broker } = createArcadeClient("never-dialled");

    expect(await broker.grantsFor({ account, toolkit: "Gmail" })).toEqual([
      {
        action: "Gmail.Send",
        state: "authorized",
        scopes: [],
        authorization: "held",
      },
    ]);
  });

  test("a listed secret is still read, met or unset", async () => {
    engine([withSecrets([{ key: "GITHUB_SERVER_URL", met: false }])]);
    const { broker } = createArcadeClient("never-dialled");

    expect(await broker.grantsFor({ account, toolkit: "Gmail" })).toEqual([
      {
        action: "Gmail.Send",
        state: "needs-operator",
        missing: ["GITHUB_SERVER_URL"],
      },
    ]);
  });
});

/**
 * THE DEFECT: EVERY ARCADE ACTION WAS UNCALLABLE BY EVERY MODEL, AND NOTHING SAID SO.
 *
 * Arcade's listing answers `qualified_name` — `Gmail.ListEmails` — and `mcp_tools.name` used to be
 * recorded as exactly that. `toolNameFor` then spliced it into `mcp__arcade-Gmail__Gmail.ListEmails`
 * and the model API refused the request: a tool name may only match `^[a-zA-Z0-9_-]+$`, and the dot
 * does not. Because a request carries EVERY tool together, one granted Arcade action failed the
 * WHOLE call — `Invalid 'tools[47].name'` — so a Bot holding one Arcade tool could not answer
 * anything at all.
 *
 * It survived every test in this tree because no test ever asked whether the name it recorded was a
 * name a model could be offered. These do.
 */
describe("an Arcade action is recorded under a name a model may be offered", () => {
  test("the toolkit prefix is dropped, and the vendor's own spelling is kept beside it", async () => {
    engine([toolRow("ListEmails", { behavior: { read: true } })]);

    expect(await listed()).toMatchObject([
      { name: "ListEmails", vendorName: "Gmail.ListEmails" },
    ]);
  });

  test("every listed name is one the model API accepts", async () => {
    engine([
      toolRow("ListEmails", null),
      toolRow("SendEmail", null),
      toolRow("TrashEmail", null),
    ]);

    /*
     * THE PATTERN ITSELF, not a sample of names somebody thought of. This is the whole of what the
     * model API will take, and `name` is spliced into a tool name unchanged — so a listing that
     * passes here is a request that is not refused for its names.
     */
    for (const tool of await listed()) {
      expect(tool.name).toMatch(/^[a-zA-Z0-9_-]+$/);
    }
  });

  test("a qualified name that is not this toolkit's is left whole", async () => {
    /*
     * ONLY THE VENDOR'S OWN PREFIX IS REMOVED. A listing that qualifies an action with some other
     * name is not something to repair by guessing: the name is kept as it stands and the guard in
     * `storableTools` is what leaves it out and names it, where it has a provenance.
     */
    engine([{ ...toolRow("Other", null), qualified_name: "Elsewhere.Other" }]);

    expect(await listed()).toMatchObject([
      { name: "Elsewhere.Other", vendorName: "Elsewhere.Other" },
    ]);
  });
});