Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

25.1 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import {
  authoriseAgentCall,
  hashCallbackToken,
  looksLikeCallbackToken,
  mintCallbackToken,
  mintRunAssertion,
  readApprovedRunAssertion,
  readRunAssertion,
  sameToken,
} from "../src/agents/callback-token";
import { createApp } from "../src/app";
import { loadConfig } from "../src/config";
import { PluginRefusedError, type PluginStore } from "../src/plugins/store";
import { markUntrusted } from "../src/untrusted-content";
import { testEnvironment } from "./support/environment";

const KEY = "test-encryption-key-not-a-real-one";
const RUN = { botId: "knowledge", actorId: "user_7", runId: "run_1" };

describe("an agent's callback token", () => {
  test("is recognisable, and different every time", () => {
    const first = mintCallbackToken();
    const second = mintCallbackToken();
    expect(looksLikeCallbackToken(first)).toBe(true);
    expect(first).not.toBe(second);
  });

  test("does not accept something that is not one of ours", () => {
    expect(looksLikeCallbackToken("Bearer hunter2")).toBe(false);
    expect(looksLikeCallbackToken("obot_agt_")).toBe(false);
  });

  test("matches only its own hash", () => {
    const token = mintCallbackToken();
    expect(sameToken(hashCallbackToken(token), hashCallbackToken(token))).toBe(
      true,
    );
    expect(
      sameToken(
        hashCallbackToken(token),
        hashCallbackToken(mintCallbackToken()),
      ),
    ).toBe(false);
  });
});

describe("the run assertion", () => {
  test("survives a round trip", () => {
    const signed = mintRunAssertion(RUN, KEY);
    // A run that began with a person is depth zero, which is what an unstated depth means.
    expect(readRunAssertion(signed, KEY)).toEqual({
      ...RUN,
      depth: 0,
      initiator: { kind: "person" },
    });
  });

  test("is refused when signed with another key", () => {
    const signed = mintRunAssertion(RUN, "another-key");
    expect(readRunAssertion(signed, KEY)).toBeNull();
  });

  test("is refused when the Bot is edited", () => {
    // The whole point: an agent must not be able to promote itself to another Bot's grants.
    const signed = mintRunAssertion(RUN, KEY);
    const [value, signature] = signed.split(".");
    const payload = JSON.parse(
      Buffer.from(value ?? "", "base64url").toString("utf8"),
    );
    payload.botId = "risk-analyst";
    const forged = `${Buffer.from(JSON.stringify(payload)).toString("base64url")}.${signature}`;
    expect(readRunAssertion(forged, KEY)).toBeNull();
  });

  test("is refused once it has expired", () => {
    const signed = mintRunAssertion(RUN, KEY, 0);
    // Eleven minutes later: past the ten-minute life of an assertion.
    expect(readRunAssertion(signed, KEY, 11 * 60 * 1000)).toBeNull();
    // Still good a minute in, so the bound is a real window rather than nothing.
    expect(readRunAssertion(signed, KEY, 60 * 1000)).toEqual({
      ...RUN,
      depth: 0,
      initiator: { kind: "person" },
    });
  });

  test("is refused when it is missing, empty or not a string", () => {
    expect(readRunAssertion(undefined, KEY)).toBeNull();
    expect(readRunAssertion("", KEY)).toBeNull();
    expect(readRunAssertion(42, KEY)).toBeNull();
    expect(readRunAssertion("not-signed-at-all", KEY)).toBeNull();
  });
});

describe("who may call a tool back, and as whom", () => {
  const AGENT_A = "agent_a";
  const AGENT_B = "agent_b";
  const tokenA = mintCallbackToken();
  const tokenB = mintCallbackToken();

  /** The two agents this deployment has issued a token to, and nobody else. */
  const lookup = async (hash: string) => {
    if (hash === hashCallbackToken(tokenA)) return { id: AGENT_A };
    if (hash === hashCallbackToken(tokenB)) return { id: AGENT_B };
    return null;
  };

  const runForA = () =>
    mintRunAssertion(
      { botId: AGENT_A, actorId: "visitor_9", runId: "r1" },
      KEY,
    );

  const call = (presented: string, run: unknown, legacyToken?: string) =>
    authoriseAgentCall({
      presented,
      run,
      encryptionKey: KEY,
      lookup,
      ...(legacyToken ? { legacyToken } : {}),
    });

  test("allows an agent to act as the Bot its token was issued for", async () => {
    expect(await call(tokenA, runForA())).toEqual({
      ok: true,
      botId: AGENT_A,
      actorId: "visitor_9",
      initiator: { kind: "person" },
      run: {
        botId: AGENT_A,
        actorId: "visitor_9",
        runId: "r1",
        depth: 0,
        initiator: { kind: "person" },
      },
    });
  });

  test("refuses another agent presenting a valid assertion it did not earn", async () => {
    /*
     * The whole point of the change. One deployment-wide token used to mean any holder could spend any
     * Bot's grants; the token now says who is calling and this says they may not be somebody else.
     */
    expect(await call(tokenB, runForA())).toEqual({
      ok: false,
      status: 403,
      reason: "That token is not for this Bot.",
    });
  });

  test("refuses a token with no assertion at all", async () => {
    expect(await call(tokenA, undefined)).toEqual({
      ok: false,
      status: 401,
      reason: "Not authorised.",
    });
  });

  test("refuses an unknown token", async () => {
    expect(await call(mintCallbackToken(), runForA())).toEqual({
      ok: false,
      status: 401,
      reason: "Not authorised.",
    });
  });

  test("refuses an empty token", async () => {
    expect(await call("", runForA())).toEqual({
      ok: false,
      status: 401,
      reason: "Not authorised.",
    });
  });

  test("says the same thing whichever half was wrong", async () => {
    // A caller told its token was fine but its assertion stale has learned its token is fine.
    const badToken = await call(mintCallbackToken(), runForA());
    const badAssertion = await call(tokenA, "not-signed");
    expect(badToken).toEqual(badAssertion);
  });

  test("accepts the deployment-wide token, and still takes identity from the assertion", async () => {
    // The Bots in the box are configured with it. It authenticates; it does not assert.
    expect(await call("legacy-secret", runForA(), "legacy-secret")).toEqual({
      ok: true,
      botId: AGENT_A,
      actorId: "visitor_9",
      initiator: { kind: "person" },
      run: {
        botId: AGENT_A,
        actorId: "visitor_9",
        runId: "r1",
        depth: 0,
        initiator: { kind: "person" },
      },
    });
  });

  test("refuses the deployment-wide token when it is not configured", async () => {
    expect(await call("legacy-secret", runForA())).toEqual({
      ok: false,
      status: 401,
      reason: "Not authorised.",
    });
  });

  test("refuses the deployment-wide token with no assertion, which is what the old hole was", async () => {
    /*
     * Before this, the Bot and the actor came out of the request body, so this exact call succeeded
     * and could name any Bot and any person.
     */
    expect(await call("legacy-secret", undefined, "legacy-secret")).toEqual({
      ok: false,
      status: 401,
      reason: "Not authorised.",
    });
  });
});

/**
 * A refused callback leaves a row, because it is a boundary being held.
 *
 * The three MCP call outcomes are all written inside `callTool`, which is reached only once the
 * caller has proved which Bot it is. A caller that fails that check never gets there, so this
 * refusal used to leave nothing behind at all: no row, no log, nothing to count.
 *
 * Which made the product's most confusing failure silent. A Bot holding a token the deployment no
 * longer accepts — a secret rotated, a container not recreated with it — has every call refused
 * here, returns nothing to its own model, and the model tells the person "no files were found". A
 * false negative delivered as an answer, with every place somebody would check agreeing that
 * nothing had happened. That is how it was found: by driving Drive and getting "no results" from a
 * Drive that had them.
 *
 * These assert the shape the row must keep. `authoriseAgentCall` is the decision the route acts on,
 * so a verdict that stops being a refusal is the thing that would silently drop the row again.
 */
describe("a callback that cannot prove which Bot it is", () => {
  test("is refused, with a reason and a status to record", async () => {
    const verdict = await authoriseAgentCall({
      presented: "obot_agt_not_a_token_this_deployment_issued",
      run: await mintRunAssertion(RUN, KEY),
      encryptionKey: KEY,
      legacyToken: "the-deployment-secret",
      lookup: async () => null,
    });

    expect(verdict.ok).toBe(false);
    if (verdict.ok) return;
    // Both are written onto the audit row, so both have to exist.
    expect(typeof verdict.reason).toBe("string");
    expect(verdict.reason.length).toBeGreaterThan(0);
    expect(verdict.status).toBeGreaterThanOrEqual(400);
  });

  test("is refused when the token is right and the run assertion is not", async () => {
    // The other half of the pair. A shared secret alone must not be enough to spend a Bot's grants:
    // without a run this deployment signed, there is no statement of who it is acting for.
    const verdict = await authoriseAgentCall({
      presented: "the-deployment-secret",
      run: "not-an-assertion-this-deployment-signed",
      encryptionKey: KEY,
      legacyToken: "the-deployment-secret",
      lookup: async () => null,
    });

    expect(verdict.ok).toBe(false);
  });

  test("carries no Bot or actor to record, which is why the row names neither", async () => {
    /*
     * The row deliberately records no bot and no actor. Both arrive inside the credential that just
     * failed to verify, so writing them down would put an unproven claim in the one place that is
     * supposed to be believed. This holds that there is nothing trustworthy to write.
     */
    const verdict = await authoriseAgentCall({
      presented: "",
      run: undefined,
      encryptionKey: KEY,
      legacyToken: "the-deployment-secret",
      lookup: async () => null,
    });

    expect(verdict.ok).toBe(false);
    expect(verdict).not.toHaveProperty("botId");
    expect(verdict).not.toHaveProperty("actorId");
  });
});

/**
 * How deep a chain of Bots already is travels here because it has to cross a process.
 *
 * A Bot handing work to another is A to B to C, three runs on up to three pods. A counter in a
 * variable stops applying the moment the second hop lands somewhere else, which is also the moment a
 * loop starts costing real money: the cap would go quiet exactly when it was needed. Signed with the
 * rest, so it is the deployment's number rather than one a Bot can edit.
 */
describe("how deep a run is", () => {
  test("survives a round trip", () => {
    const signed = mintRunAssertion({ ...RUN, depth: 2 }, KEY);
    expect(readRunAssertion(signed, KEY)?.depth).toBe(2);
  });

  test("a run that began with a person is zero", () => {
    expect(readRunAssertion(mintRunAssertion(RUN, KEY), KEY)?.depth).toBe(0);
  });

  /*
   * Read as zero rather than refused. The signature has already been checked, so this is a field
   * that predates the feature being absent rather than a caller lying, and the cap refuses on the
   * way out anyway.
   */
  test("a depth that is not a depth reads as zero", () => {
    for (const nonsense of [-1, 1.5, "2", null]) {
      const signed = mintRunAssertion(
        { ...RUN, depth: nonsense as never },
        KEY,
      );
      expect(readRunAssertion(signed, KEY)?.depth).toBe(0);
    }
  });

  test("what started the run survives a round trip", () => {
    for (const initiator of [
      { kind: "person" } as const,
      { kind: "deployment" } as const,
      { kind: "routine", id: "routine_7" } as const,
      { kind: "responsibility", id: "responsibility_7" } as const,
      { kind: "handoff", id: "research-assistant" } as const,
    ]) {
      const signed = mintRunAssertion({ ...RUN, initiator }, KEY);
      expect(readRunAssertion(signed, KEY)?.initiator).toEqual(initiator);
    }
  });

  test("a run that says nothing about what started it reads as a person", () => {
    expect(
      readRunAssertion(mintRunAssertion(RUN, KEY), KEY)?.initiator,
    ).toEqual({ kind: "person" });
  });

  /*
   * The point of putting this inside the signature. A Bot cannot relabel its own run, and a kind
   * this deployment does not write cannot arrive as a string the Audit screen has no branch for.
   */
  test("an initiator that is not one reads as a person rather than being kept", () => {
    for (const nonsense of [
      { kind: "administrator" },
      { kind: "routine" },
      { kind: "handoff", id: "" },
      { kind: "routine", id: 7 },
      "routine",
      null,
    ]) {
      const signed = mintRunAssertion(
        { ...RUN, initiator: nonsense as never },
        KEY,
      );
      expect(readRunAssertion(signed, KEY)?.initiator).toEqual({
        kind: "person",
      });
    }
  });

  test("the conversation survives a round trip, and is absent when there is none", () => {
    expect(
      readRunAssertion(mintRunAssertion({ ...RUN, threadId: "t1" }, KEY), KEY)
        ?.threadId,
    ).toBe("t1");
    expect(readRunAssertion(mintRunAssertion(RUN, KEY), KEY)?.threadId).toBe(
      undefined,
    );
  });
});

/**
 * WHAT THE ROUTE THIS TOKEN GUARDS HANDS BACK WHEN THE CALL FAILS, which nothing exercised at all.
 *
 * `/api/agent-tools/call` is where a Bot running its own loop in its own process calls a tool. What
 * it answers with goes straight into that model's context as the tool result, so this surface is
 * the widest audience any error message in this deployment reaches: a model repeats what it is
 * given, to the person asking and into whatever it writes next.
 *
 * THE IN-PROCESS SIBLING ALREADY DECIDES THIS and decides it the other way. `plugins/tools.ts`
 * wraps the identical `callTool` for a Bot running here, and its catch refuses to relay anything on
 * the `isDeploymentFault` shelf — "a contradiction in this deployment's own tables says nothing to
 * a model". The route below relayed `error.message` with no such question asked, so one of the two
 * doors to one store answered a query failure with `Failed query: … params: …` and the other
 * answered "That tool could not be called." Which door a Bot came through is a deployment topology
 * decision, not a disclosure decision.
 */
describe("the tool-call route a callback token guards", () => {
  /** The deployment-wide token, which authenticates without naming a Bot of its own. */
  const DEPLOYMENT_TOKEN = "deployment-wide-agent-token";

  const config = loadConfig(
    testEnvironment({ AGENT_TOOL_TOKEN: DEPLOYMENT_TOKEN }),
  );

  /**
   * The app with one thing in it: a store whose `callTool` throws what the test is about.
   *
   * `pluginStore` is the fifteenth positional argument, so the gap is spelled rather than guessed —
   * a miscount here would silently hand the store to `componentStore` and leave the route absent.
   */
  function appWhoseToolThrows(thrown: unknown) {
    return createApp(
      config,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      {
        brokerProvider: "composio" as const,
        callTool: async () => {
          throw thrown;
        },
      } as unknown as PluginStore,
    );
  }

  /** What the model is handed, as the route builds it. */
  async function toolResult(thrown: unknown): Promise<string> {
    const response = await appWhoseToolThrows(thrown).request(
      "http://openbot.local/api/agent-tools/call",
      {
        method: "POST",
        headers: {
          "content-type": "application/json",
          "x-openbot-agent-token": DEPLOYMENT_TOKEN,
        },
        body: JSON.stringify({
          name: "mcp__linear__LINEAR_CREATE_ISSUE",
          args: {},
          run: mintRunAssertion(
            { botId: "knowledge", actorId: "usr_7", runId: "run_1" },
            config.keyEncryptionKey,
          ),
        }),
      },
    );
    expect(response.status).toBe(200);
    const body = (await response.json()) as { text: string; isError: boolean };
    expect(body.isError).toBe(true);
    return body.text;
  }

  /**
   * The drizzle shape, spelled the way `plugin-store.integration.test.ts` spells it: the statement
   * and every value bound to it in `message`, the driver's own error on `cause`, and `query` and
   * `params` as own properties — which is what {@link isQueryFailure} recognises it by.
   */
  function queryFailure() {
    return Object.assign(
      new Error(
        'Failed query: select "credential_id" from "mcp_user_credentials" where "server_id" = $1 and "user_id" = $2 params: linear, usr_7',
      ),
      {
        query:
          'select "credential_id" from "mcp_user_credentials" where "server_id" = $1 and "user_id" = $2',
        params: ["linear", "usr_7"],
        cause: new Error("canceling statement due to statement timeout"),
      },
    );
  }

  test("a query of this deployment's own never reaches the model that asked", async () => {
    const text = await toolResult(queryFailure());

    /*
     * Not the statement, and not the values bound to it. On this path those are server ids, user
     * ids and credential ids, and a model handed them can repeat them to the person asking, quote
     * them into a document it writes, or send them to the next tool it calls — which is why this
     * is the worst of the three places this shape has been found leaking.
     */
    expect(text).not.toContain("Failed query");
    expect(text).not.toContain("params:");
    expect(text).not.toContain("mcp_user_credentials");
    expect(text).not.toContain("usr_7");
    // The same thing the in-process door says about the same shelf, which is the property.
    expect(text).toContain("That tool could not be called.");
  });

  /**
   * ONLY A REFUSAL IS MARKED AS ONE, because the marker is what the transcript draws.
   *
   * `chat-transcript.tsx` labels a tool result that starts with `REFUSAL_MARKER` as blocked, and the
   * model reads "Refused." as "not allowed". `callTool` throws `PluginRefusedError` for a boundary
   * holding, and rethrows a vendor that broke after recording `mcp.call_failed`. The in-process door
   * keeps the two apart — "one means 'not allowed', the other means 'it broke'" — and this route
   * put the marker in front of every throw, so on a Bot running its own loop a vendor outage and a
   * database fault were both drawn as a policy refusal. Asked of both doors with the same store.
   */
  test("a throw is marked as a refusal only when it is one, the way the in-process door marks it", async () => {
    const { grantedTools, REFUSAL_MARKER } = await import(
      "../src/plugins/tools"
    );
    const throws: [string, unknown][] = [
      ["vendor failure", new Error("fetch failed")],
      ["deployment fault", queryFailure()],
      [
        "refusal",
        new PluginRefusedError(
          "No Bot holds linear/LINEAR_CREATE_ISSUE, so nothing was called.",
          null,
        ),
      ],
    ];

    const seen: string[] = [];
    for (const [kind, thrown] of throws) {
      const callback = await toolResult(thrown);
      const [tool] = await grantedTools({
        store: {
          callTool: async () => {
            throw thrown;
          },
          listForAgent: async () => ({
            tools: [
              {
                toolName: "mcp__linear__LINEAR_CREATE_ISSUE",
                ref: "linear/LINEAR_CREATE_ISSUE",
                description: "Create an issue.",
                inputSchema: { type: "object" },
              },
            ],
          }),
        } as unknown as PluginStore,
        botId: "knowledge",
        actorId: "usr_7",
      });
      const inProcess = await tool?.execute({});
      seen.push(
        `${kind}: marked ${callback.startsWith(REFUSAL_MARKER)}, same as in-process ${callback === inProcess}`,
      );
    }

    expect(seen).toEqual([
      "vendor failure: marked false, same as in-process true",
      "deployment fault: marked false, same as in-process true",
      "refusal: marked true, same as in-process true",
    ]);
  });

  /**
   * AND THE REFUSAL STILL SPEAKS, because a guard that silences everything is not the fix.
   *
   * A `PluginRefusedError` is this deployment telling a Bot it may not do something, and its
   * sentence is written for whoever reads the answer. Losing it would turn every policy boundary
   * into an unexplained failure, which is what the marker on this route exists to prevent.
   */
  test("a refusal this deployment wrote is still relayed in full", async () => {
    expect(
      await toolResult(
        new PluginRefusedError(
          "No Bot holds linear/LINEAR_CREATE_ISSUE, so nothing was called.",
          null,
        ),
      ),
    ).toContain("No Bot holds linear/LINEAR_CREATE_ISSUE");
  });

  /** And a vendor's own words, which are the useful half of a 403 and are nobody's secret. */
  test("a vendor's own sentence is still relayed", async () => {
    expect(
      await toolResult(new Error("The caller does not have permission.")),
    ).toContain("The caller does not have permission.");
  });

  /**
   * A VENDOR THAT ANSWERED WITH AN ERROR, rather than one that threw, which is how an MCP server says
   * no: `{ isError: true }` and a sentence, resolved and not thrown.
   *
   * The in-process door names that sentence as the vendor's (`plugins/tools.ts`), because handing it
   * over as content already cost a diagnosis: Google's "The caller does not have permission" read as
   * a result, and the model told the person it had no access to their Drive. Neither framework Bot
   * words it on its way through — the LangGraph Bot passes an `isError` answer on untouched, and
   * the Python one reads only `text` — so what this route writes is what the model reads.
   */
  function storeAnswering(result: { text: string; isError: boolean }) {
    return {
      brokerProvider: "composio" as const,
      callTool: async () => ({ ...result, truncated: false }),
      listForAgent: async () => ({
        tools: [
          {
            toolName: "mcp__linear__LINEAR_CREATE_ISSUE",
            ref: "linear/LINEAR_CREATE_ISSUE",
            description: "Create an issue.",
            inputSchema: { type: "object" },
          },
        ],
      }),
    } as unknown as PluginStore;
  }

  /** Both doors' answers to one call against the same store: the callback route's, and the in-process one's. */
  async function bothDoors(store: PluginStore) {
    const response = await createApp(
      config,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      store,
    ).request("http://openbot.local/api/agent-tools/call", {
      method: "POST",
      headers: {
        "content-type": "application/json",
        "x-openbot-agent-token": DEPLOYMENT_TOKEN,
      },
      body: JSON.stringify({
        name: "mcp__linear__LINEAR_CREATE_ISSUE",
        args: {},
        run: mintRunAssertion(
          { botId: "knowledge", actorId: "usr_7", runId: "run_1" },
          config.keyEncryptionKey,
        ),
      }),
    });
    expect(response.status).toBe(200);
    const callback = (await response.json()) as {
      text: string;
      isError: boolean;
    };

    const { grantedTools } = await import("../src/plugins/tools");
    const [tool] = await grantedTools({
      store,
      botId: "knowledge",
      actorId: "usr_7",
    });
    const inProcess = await tool?.execute({});
    return { callback, inProcess };
  }

  test("a vendor's error answer is named as the vendor's, the way the in-process door names it", async () => {
    const { callback, inProcess } = await bothDoors(
      storeAnswering({
        text: "The caller does not have permission.",
        isError: true,
      }),
    );

    expect(callback.isError).toBe(true);
    // One store, one answer: which door a Bot comes through is topology, not what its model is told.
    expect(callback.text).toBe(inProcess);
    // The vendor's words, inside the untrusted-content envelope the model reads outside data in.
    expect(callback.text).toBe(
      `The vendor reported an error:\n${markUntrusted("The caller does not have permission.", "connector error")}`,
    );
  });

  test("a vendor's result that is not an error reaches the model as the vendor wrote it", async () => {
    const { callback, inProcess } = await bothDoors(
      storeAnswering({ text: "Created LIN-42.", isError: false }),
    );

    // As the vendor wrote it, marked as untrusted data rather than rewritten.
    expect(callback).toEqual({
      text: markUntrusted("Created LIN-42.", "connector result"),
      isError: false,
    });
    expect(callback.text).toBe(inProcess);
  });
});

describe("an approved action's stored run", () => {
  test("is read after the live expiry, and still refused with the wrong key", () => {
    const signed = mintRunAssertion({ ...RUN, depth: 2 }, KEY, 0);
    expect(readRunAssertion(signed, KEY)).toBeNull();
    expect(readApprovedRunAssertion(signed, KEY)?.depth).toBe(2);
    expect(readApprovedRunAssertion(signed, "another-key")).toBeNull();
    expect(readApprovedRunAssertion(`${signed}x`, KEY)).toBeNull();
  });
});