Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

11.7 KB
Raw Blame
import { afterEach, describe, expect, test } from "bun:test";
import { useArcadeClient } from "../src/plugins/arcade";
import { createArcadeClient } from "../src/plugins/arcade-adapter";
import { withVendorDeadline } from "../src/plugins/route-groups/context";
import type { AccountRef } from "../src/plugins/shared-accounts";

/**
 * ONE LARGE TOOLKIT MUST BE READABLE INSIDE ITS DEADLINE: THE GRANTS SCREEN AND DISCONNECT.
 *
 * The sibling of `arcade-directory-deadline-engine.test.ts`. That file holds the WHOLE directory to
 * the deadline; this one holds the PER-TOOLKIT action listing — read by `grantsFor` (the grants
 * screen) and by `authProvidersNamedBy` (behind `authProviderOf`, `isConnected` and `revoke`) — to
 * the same deadline, at the size of the largest toolkit Arcade publishes.
 *
 * MEASURED against the live directory (8,320 tools across 129 toolkits): `GithubApi` has **818**
 * actions. Read 100 at a time, one after another, that is 9 sequential pages at ~1.19 s each,
 * ~10.7 s — past the 10 s `VENDOR_CALL_TIMEOUT_MS` every one of those routes is wrapped in. The
 * grants screen for one of the most likely apps anybody adds could not load, and Disconnect for it
 * could not even start.
 *
 * Time is scaled ×10 as in the directory test; the RATIO between page latency and deadline is the
 * live one:
 *
 * | measured live                          | here (÷10)   |
 * |----------------------------------------|--------------|
 * | `GithubApi` 818 actions                | 818 (same)   |
 * | ~1.19 s per page at limit ≤ 100        | 119 ms       |
 * | ~1.8 s per page at limit 250…1000      | 180 ms       |
 * | limit above 1000 REJECTED, empty body  | same         |
 * | `VENDOR_CALL_TIMEOUT_MS` 10,000 ms     | 1,000 ms     |
 *
 * The admin listings and the DELETE on the Disconnect path were not measured separately; they are
 * served at the same small-page latency, which is the conservative reading.
 */

const SCALE = 10;
/** `route-groups/context.ts`'s `VENDOR_CALL_TIMEOUT_MS`, scaled. */
const DEADLINE_MS = 10_000 / SCALE;
/** The measured largest toolkit in Arcade's directory. */
const GITHUB_API = 818;
const TOOLKIT = "GithubApi";

const account: AccountRef = {
  holder: "person",
  userId: "u1",
  vendorUserId: "vendor-u1",
};

const toolRow = (n: number) => ({
  qualified_name: `${TOOLKIT}.Action${n}`,
  description: "",
  toolkit: { name: TOOLKIT, description: "" },
  input: { parameters: [] },
  requirements: {
    authorization: { provider_id: "github", oauth2: { scopes: [] } },
  },
});

type Asked = {
  toolkit: string | null;
  userId: string | null;
  limit: number;
  offset: number;
  /** How many tool-listing requests were in flight when this one arrived, itself included. */
  inFlightAtStart: number;
  /** How many tool-listing requests had already been ANSWERED when this one arrived. */
  answeredBefore: number;
};

type Engine = {
  /** Every request to `/v1/tools`, in arrival order. */
  asked: Asked[];
  maxInFlight: () => number;
  deletes: () => number;
  stop: () => void;
};

type EngineOptions = {
  /** What `total_count` says on every page of the action listing. */
  claimed: number;
  /** How many actions the engine actually serves, which may be fewer — or more — than it claims. */
  served?: number;
  /** Per-request latency in ms at limit ≤ 100, and above it. Defaults are the live ones, ÷10. */
  latency?: { small: number; large: number };
};

const sleep = (ms: number) =>
  new Promise<void>((resolve) => setTimeout(resolve, ms));

function startEngine(options: EngineOptions): Engine {
  const asked: Asked[] = [];
  const latency = options.latency ?? {
    small: 1190 / SCALE,
    large: 1800 / SCALE,
  };
  const served = options.served ?? options.claimed;
  let inFlight = 0;
  let peak = 0;
  let answered = 0;
  let deleted = 0;
  const server = Bun.serve({
    port: 0,
    async fetch(request) {
      const url = new URL(request.url);
      const offset = Number(url.searchParams.get("offset") ?? "0");
      const limit = Number(url.searchParams.get("limit") ?? "25");

      if (url.pathname.startsWith("/v1/admin/user_connections/")) {
        await sleep(latency.small);
        deleted += 1;
        return new Response(null, { status: 204 });
      }
      if (url.pathname === "/v1/admin/user_connections") {
        await sleep(latency.small);
        const items =
          offset === 0
            ? [
                {
                  id: "at_1",
                  connection_id: "ac_1",
                  connection_status: "active",
                  provider_id: "github-openbot",
                  provider_type: "oauth2",
                  provider_user_info: { name: "person one" },
                  scopes: [],
                  user_id: account.vendorUserId,
                },
              ]
            : [];
        return Response.json({
          items,
          offset,
          limit,
          page_count: items.length,
          total_count: 1,
        });
      }
      if (url.pathname === "/v1/admin/auth_providers") {
        await sleep(latency.small);
        const items =
          offset === 0
            ? [{ id: "github-openbot", provider_id: "github", type: "oauth2" }]
            : [];
        return Response.json({
          items,
          offset,
          limit,
          page_count: items.length,
          total_count: 1,
        });
      }
      if (url.pathname !== "/v1/tools")
        return new Response("not found", { status: 404 });

      inFlight += 1;
      peak = Math.max(peak, inFlight);
      asked.push({
        toolkit: url.searchParams.get("toolkit"),
        userId: url.searchParams.get("user_id"),
        limit,
        offset,
        inFlightAtStart: inFlight,
        answeredBefore: answered,
      });
      try {
        // MEASURED: limit=5000 answered an EMPTY response in ~0.17 s. A page size past the
        // vendor's ceiling must fail here exactly as it fails live.
        if (limit > 1000) {
          await sleep(170 / SCALE);
          return new Response("", { status: 200 });
        }
        await sleep(limit <= 100 ? latency.small : latency.large);
        const items = [];
        for (let n = offset; n < Math.min(offset + limit, served); n++)
          items.push(toolRow(n));
        return Response.json({
          items,
          offset,
          limit,
          page_count: items.length,
          total_count: options.claimed,
        });
      } finally {
        inFlight -= 1;
        answered += 1;
      }
    },
  });
  process.env.ARCADE_BASE_URL = `http://localhost:${server.port}`;
  return {
    asked,
    maxInFlight: () => peak,
    deletes: () => deleted,
    stop: () => server.stop(true),
  };
}

let running: Engine | null = null;
const wasBaseURL = process.env.ARCADE_BASE_URL;

function engine(options: EngineOptions): Engine {
  running = startEngine(options);
  return running;
}

afterEach(() => {
  running?.stop();
  running = null;
  useArcadeClient(null);
  if (wasBaseURL === undefined) delete process.env.ARCADE_BASE_URL;
  else process.env.ARCADE_BASE_URL = wasBaseURL;
});

/** Each broker member exactly as its route calls it: under the vendor deadline. */
const underDeadline = <T>(
  run: (broker: ReturnType<typeof createArcadeClient>["broker"]) => Promise<T>,
) =>
  withVendorDeadline(
    () => run(createArcadeClient("never-dialled").broker),
    DEADLINE_MS,
  );

const grants = () =>
  underDeadline((broker) => broker.grantsFor({ account, toolkit: TOOLKIT }));

/** The offsets asked for after page one, in offset order. */
const restOffsets = (e: Engine) =>
  e.asked
    .slice(1)
    .map((one) => one.offset)
    .sort((a, b) => a - b);

describe("the largest Arcade toolkit is read inside the vendor deadline", () => {
  test("the grants screen lists all 818 GithubApi actions before the deadline", async () => {
    const e = engine({ claimed: GITHUB_API });
    const started = performance.now();

    const held = await grants();

    const took = performance.now() - started;
    expect(held.length).toBe(GITHUB_API);
    expect(took).toBeLessThan(DEADLINE_MS);
    // Still nine pages of 100 — the page size did not change, only when the pages are asked for.
    expect(e.asked.length).toBe(9);
    for (const one of e.asked) {
      expect(one.toolkit).toBe(TOOLKIT);
      expect(one.userId).toBe(account.vendorUserId);
      expect(one.limit).toBe(100);
    }
  });

  test("page one is asked for alone, then the other eight pages at once", async () => {
    const e = engine({ claimed: GITHUB_API });

    await grants();

    // Page one first, alone — its `total_count` is what says how many pages follow.
    expect(e.asked[0]).toMatchObject({
      limit: 100,
      offset: 0,
      inFlightAtStart: 1,
      answeredBefore: 0,
    });
    expect(restOffsets(e)).toEqual([100, 200, 300, 400, 500, 600, 700, 800]);
    // ...all asked for AFTER page one answered and BEFORE any of them answered: one wave.
    for (const one of e.asked.slice(1)) expect(one.answeredBefore).toBe(1);
    expect(e.maxInFlight()).toBe(8);
  });

  test("the auth-provider read behind authProviderOf finishes before the deadline too", async () => {
    const e = engine({ claimed: GITHUB_API });

    expect(
      await underDeadline((broker) => broker.authProviderOf(TOOLKIT)),
    ).toBe("github");
    expect(e.asked.length).toBe(9);
    // The catalogue read carries no person: it is the same listing for everybody.
    for (const one of e.asked) expect(one.userId).toBeNull();
    expect(restOffsets(e)).toEqual([100, 200, 300, 400, 500, 600, 700, 800]);
    expect(e.maxInFlight()).toBe(8);
  });

  test("Disconnect on GithubApi — the action listing, then the connection lifecycle — fits the deadline", async () => {
    const e = engine({ claimed: GITHUB_API });

    expect(
      await underDeadline((broker) =>
        broker.revoke({ account, toolkit: TOOLKIT }),
      ),
    ).toBe(true);
    expect(e.deletes()).toBe(1);
  });

  test("a bigger toolkit is still fetched at most eight pages at a time", async () => {
    // 2,000 actions: 20 pages after the first, so the bound — not the toolkit — decides the peak.
    const e = engine({ claimed: 2000, latency: { small: 5, large: 20 } });

    expect((await grants()).length).toBe(2000);
    expect(e.asked.length).toBe(21);
    expect(e.maxInFlight()).toBe(8);
  });
});

describe("fetching a toolkit ahead does not weaken the incompleteness check", () => {
  test("a toolkit that claims 818 and serves 750 is refused, on the grants screen and on Disconnect", async () => {
    const e = engine({ claimed: GITHUB_API, served: 750 });

    await expect(grants()).rejects.toThrow(
      /listed 750 of 818 actions for GithubApi, so what this account holds there could not be read in full/,
    );
    await expect(
      underDeadline((broker) => broker.revoke({ account, toolkit: TOOLKIT })),
    ).rejects.toThrow(
      /listed 750 of 818 actions for GithubApi, so the auth provider/,
    );
    expect(e.deletes()).toBe(0);
  });

  test("a short page in the MIDDLE ends the listing, and the pages fetched past it are not counted", async () => {
    // The page at offset 400 holds 50 rows; the four pages fetched ahead past it hold nothing.
    engine({ claimed: GITHUB_API, served: 450 });

    await expect(grants()).rejects.toThrow(/listed 450 of 818 actions/);
  });

  test("a total that UNDER-reports is read on past, sequentially, to the real end", async () => {
    // Claims 300, serves 818: the prefetch covers offsets 100…300, all full, so the listing must
    // go on asking — a short page ends it, never the total.
    const e = engine({
      claimed: 300,
      served: GITHUB_API,
      latency: { small: 5, large: 20 },
    });

    expect((await grants()).length).toBe(GITHUB_API);
    expect(e.asked.map((one) => one.offset).sort((a, b) => a - b)).toEqual([
      0, 100, 200, 300, 400, 500, 600, 700, 800,
    ]);
  });
});