Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

13.6 KB
Raw Blame
import {
  afterAll,
  afterEach,
  beforeAll,
  beforeEach,
  expect,
  test,
} from "bun:test";
import { GlobalRegistrator } from "@happy-dom/global-registrator";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import {
  createMemoryHistory,
  createRootRoute,
  createRouter,
  RouterProvider,
} from "@tanstack/react-router";
import {
  cleanup,
  fireEvent,
  render,
  waitFor,
  within,
} from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import type { ReactNode } from "react";
import { InlineApproval } from "@/components/approvals/inline-approval";
import { ApprovalSettings } from "@/components/approvals/settings";
import { toVisibleChatItems } from "@/components/channels/chat-messages";
import {
  groupToolSteps,
  ToolActivity,
} from "@/components/channels/tool-activity";

beforeAll(() => GlobalRegistrator.register());
afterEach(cleanup);
afterAll(() => GlobalRegistrator.unregister());

const originalFetch = global.fetch;
let role: "user" | "admin" = "user";
const sent: { path: string; method: string; body: unknown }[] = [];

const inbox = {
  enabled: false,
  requests: [
    {
      id: "request-1",
      status: "pending",
      createdAt: new Date().toISOString(),
      action: {
        botId: "Shopper",
        toolRef: "computer_click",
        effect: "write",
        scope: "shop.example",
        threadId: "thread",
        args: { element: "Pay now" },
        target: {},
        toolCallId: "pay-call",
        policy: {
          behaviour: "hand_off",
          source: "safety",
          reason:
            "Paying, purchasing or moving money is always done by the person.",
        },
      },
    },
  ],
  rules: [
    {
      id: "mine",
      botId: "*",
      toolRef: "mcp/gmail/*",
      effect: "*",
      scope: "*",
      behaviour: "allow",
    },
  ],
  teamRules: [
    {
      id: "locked",
      botId: "*",
      toolRef: "mcp/drive/share",
      effect: "*",
      scope: "*",
      behaviour: "ask",
    },
  ],
  preferences: { enabled: false, autoReview: false, hostCommands: "allow" },
  team: {
    enforceAutoReview: true,
    customRulesEnabled: true,
    hostCommandsCap: "ask",
  },
  hostCommands: "ask",
  questions: [],
};

/** Per-test changes to the inbox the server returns. */
let inboxOverride: Record<string, unknown> = {};
/** A refusal the server gives the next time a rule is added, or none. */
let refuseRule: string | null = null;

beforeEach(() => {
  sent.length = 0;
  inboxOverride = {};
  refuseRule = null;
  role = "user";
  global.fetch = Object.assign(
    async (path: Parameters<typeof fetch>[0], init?: RequestInit) => {
      const url = String(path);
      if (init?.method && init.method !== "GET") {
        sent.push({
          path: url,
          method: init.method,
          body: init.body ? JSON.parse(String(init.body)) : undefined,
        });
        if (url === "/api/approvals/rules" && refuseRule) {
          const error = refuseRule;
          refuseRule = null;
          return Response.json({ error }, { status: 400 });
        }
        return Response.json({ ok: true });
      }
      if (url === "/api/approvals")
        return Response.json({ ...inbox, ...inboxOverride });
      if (url.startsWith("/api/agents"))
        return Response.json({ agents: [{ id: "shopper-bot", name: "Shop" }] });
      if (url === "/api/me")
        return Response.json({
          user: { id: "me", email: "[email protected]", role },
        });
      return new Response(null, { status: 404 });
    },
    { preconnect: originalFetch.preconnect },
  );
});
afterEach(() => {
  global.fetch = originalFetch;
});

/** A screen in a router of one route, for the links it holds. */
function draw(screen: ReactNode = <ApprovalSettings />) {
  const router = createRouter({
    history: createMemoryHistory({ initialEntries: ["/"] }),
    routeTree: createRootRoute({ component: () => screen }),
  });
  return render(
    <QueryClientProvider
      client={
        new QueryClient({ defaultOptions: { queries: { retry: false } } })
      }
    >
      <RouterProvider router={router as never} />
    </QueryClientProvider>,
  );
}

test("team rules are locked for members, enforced auto-review is shown as required, and the host cap is named", async () => {
  const view = draw(<ApprovalSettings />);
  await view.findByText("Ask before taking action (team rule)");
  const mine = view.getByRole("combobox", {
    name: "Behaviour for mcp/gmail/*",
  });
  expect(mine.textContent).toContain("Take action without asking");
  expect(view.getByText("Locked")).toBeTruthy();
  const review = view.getByRole("switch", { name: "Auto-review" });
  expect(review.getAttribute("aria-checked")).toBe("true");
  expect(view.getByText(/An admin requires this/)).toBeTruthy();
  expect(
    view.getByText(/An admin limits this to "Ask every time"/),
  ).toBeTruthy();
  expect(view.queryByText("Team settings")).toBeNull();
});

test("a member changes a saved rule in place", async () => {
  const view = draw(<ApprovalSettings />);
  const mine = await view.findByRole("combobox", {
    name: "Behaviour for mcp/gmail/*",
  });
  const user = userEvent.setup({ document });
  await user.click(mine);
  await user.click(
    await view.findByRole("option", { name: "Hand off to you" }),
  );
  await waitFor(() =>
    expect(sent).toContainEqual({
      path: "/api/approvals/rules/mine",
      method: "PATCH",
      body: { behaviour: "hand_off" },
    }),
  );
  expect(
    view.queryByRole("combobox", { name: "Behaviour for mcp/drive/share" }),
  ).toBeNull();
});

test("a member adds a rule with one of the four behaviours", async () => {
  const view = draw(<ApprovalSettings />);
  await view.findByText("Add a rule");
  const user = userEvent.setup({ document });
  await user.click(view.getByRole("button", { name: /Add a rule/ }));
  await user.type(
    await view.findByRole("textbox", { name: "Tool or app" }),
    "mcp/slack/*",
  );
  // Rules here apply to every Bot, so there is no Bot to name.
  expect(view.queryByRole("textbox", { name: "Bot" })).toBeNull();
  await user.click(view.getByRole("combobox", { name: "Behaviour" }));
  await user.click(
    await view.findByRole("option", { name: "Take action if pre-approved" }),
  );
  await user.click(view.getByRole("button", { name: "Save rule" }));
  await waitFor(() =>
    expect(sent).toContainEqual({
      path: "/api/approvals/rules",
      method: "POST",
      body: {
        botId: "*",
        toolRef: "mcp/slack/*",
        effect: "*",
        scope: "*",
        behaviour: "pre_approved",
      },
    }),
  );
});

test("a rule that is not saved keeps the dialog open, with what was typed and why", async () => {
  refuseRule = "That tool is not one a Bot can use.";
  const view = draw(<ApprovalSettings />);
  await view.findByText("Add a rule");
  const user = userEvent.setup({ document });
  await user.click(view.getByRole("button", { name: /Add a rule/ }));
  await user.type(
    await view.findByRole("textbox", { name: "Tool or app" }),
    "mcp/nowhere/*",
  );
  await user.click(view.getByRole("button", { name: "Save rule" }));
  const dialog = await view.findByRole("dialog");
  expect((await within(dialog).findByRole("alert")).textContent).toContain(
    "That tool is not one a Bot can use.",
  );
  expect(
    (
      within(dialog).getByRole("textbox", {
        name: "Tool or app",
      }) as HTMLInputElement
    ).value,
  ).toBe("mcp/nowhere/*");
  // Saved the second time, the dialog closes.
  await user.click(within(dialog).getByRole("button", { name: "Save rule" }));
  await waitFor(() => expect(view.queryByRole("dialog")).toBeNull());
  expect(
    sent.filter((request) => request.path === "/api/approvals/rules"),
  ).toHaveLength(2);
});

test("two rules for one tool are told apart by the rest of what they match, never cut short", async () => {
  inboxOverride = {
    rules: [
      {
        id: "anywhere",
        botId: "*",
        toolRef: "computer_click",
        effect: "*",
        scope: "*",
        behaviour: "ask",
      },
      {
        id: "one-site",
        botId: "*",
        toolRef: "computer_click",
        effect: "write",
        scope: "checkout.a-very-long-shop-name.example",
        behaviour: "allow",
      },
    ],
  };
  const view = draw(<ApprovalSettings />);
  const match = await view.findByText(
    "write on checkout.a-very-long-shop-name.example",
  );
  expect(match.className).toContain("line-clamp-none");
  // The tool stays the row's title.
  expect(view.getAllByText("computer_click")).toHaveLength(2);
});

test("with personal rules switched off by the team, they are shown as not applying and none can be added", async () => {
  inboxOverride = {
    team: { ...inbox.team, customRulesEnabled: false },
  };
  const view = draw(<ApprovalSettings />);
  expect(
    await view.findByText(
      /An admin has turned off personal rules\. Yours are kept but don't apply\./,
    ),
  ).toBeTruthy();
  expect(view.queryByText("Add a rule")).toBeNull();
});

test("an administrator is told where team settings are", async () => {
  role = "admin";
  const view = draw(<ApprovalSettings />);
  const link = await view.findByRole("link", { name: "Admin → Approvals" });
  expect(link.getAttribute("href")).toBe("/admin/approvals");
});

test("a member is not pointed at Admin", async () => {
  const view = draw(<ApprovalSettings />);
  await view.findByText("Rules for every Bot");
  expect(view.queryByRole("link", { name: "Admin → Approvals" })).toBeNull();
});

test("rules for every Bot point to where rules for a single Bot are", async () => {
  inboxOverride = {
    rules: [
      ...inbox.rules,
      {
        id: "one-bot",
        botId: "shopper-bot",
        toolRef: "computer_click",
        effect: "*",
        scope: "*",
        behaviour: "ask",
      },
    ],
  };
  const view = draw(<ApprovalSettings />);
  const link = await view.findByRole("link", { name: "that Bot's page" });
  expect(link.getAttribute("href")).toBe("/bots");
  // A rule naming one Bot lives on that Bot's page, not here.
  expect(
    view.queryByRole("combobox", { name: "Behaviour for computer_click" }),
  ).toBeNull();
});

test("a rule naming no Bot this person can open is listed so it can be removed", async () => {
  inboxOverride = {
    rules: [
      ...inbox.rules,
      {
        id: "orphan",
        botId: "Shopper",
        toolRef: "computer_click",
        effect: "*",
        scope: "*",
        behaviour: "allow",
      },
    ],
  };
  const view = draw(<ApprovalSettings />);
  expect(await view.findByText("Rules for other Bots")).toBeTruthy();
  expect(
    view.getByRole("combobox", { name: "Behaviour for computer_click" }),
  ).toBeTruthy();
});

test("a pending change is drawn in the conversation where the action was, and decided there", async () => {
  const view = render(
    <QueryClientProvider
      client={
        new QueryClient({ defaultOptions: { queries: { retry: false } } })
      }
    >
      <InlineApproval toolCallId="pay-call" fallback={<p>Clicked on page</p>} />
      <InlineApproval
        toolCallId="other-call"
        fallback={<p>Filled in field</p>}
      />
    </QueryClientProvider>,
  );
  await view.findByRole("region", { name: "Waiting for your approval" });
  expect(view.getByText(/This needs you: click on shop.example/)).toBeTruthy();
  // Another call's line is left to its own renderer.
  expect(view.getByText("Filled in field")).toBeTruthy();
  expect(view.queryByText("Clicked on page")).toBeNull();
  fireEvent.click(view.getByRole("button", { name: "I did it myself" }));
  await waitFor(() =>
    expect(sent).toContainEqual({
      path: "/api/approvals/request-1/decision",
      method: "POST",
      body: { decision: "handled" },
    }),
  );
});

test("in the real transcript shape a waiting click is folded into browser activity, so its card is drawn beside the group", async () => {
  // What the channel transcript builds from the conversation: a navigate that finished and the
  // click the chat stopped on, unanswered, grouped into one browser-activity line.
  const items = groupToolSteps(
    toVisibleChatItems([
      { id: "u1", role: "user", content: "Order a pizza on httpbin" },
      {
        id: "a1",
        role: "assistant",
        content: "",
        toolCalls: [
          {
            id: "nav-call",
            type: "function",
            function: {
              name: "computer_navigate",
              arguments: '{"url":"https://httpbin.org/forms/post"}',
            },
          },
          {
            id: "pay-call",
            type: "function",
            function: {
              name: "computer_click",
              arguments: '{"ref":"e7","snapshotId":3}',
            },
          },
        ],
      },
      {
        id: "t1",
        role: "tool",
        toolCallId: "nav-call",
        content: '{"ok":true,"url":"https://httpbin.org/forms/post"}',
      },
    ] as never),
    // Browser steps fold whatever the app draws, so nothing else needs naming here.
    () => false,
  );
  const group = items.find((item) => item.kind === "tools");
  if (group?.kind !== "tools") throw new Error("no tool group");
  const view = render(
    <QueryClientProvider
      client={
        new QueryClient({ defaultOptions: { queries: { retry: false } } })
      }
    >
      {/* ToolActivity mounts the waiting cards beside its folded line itself. */}
      <ToolActivity group={group} active={false} />
    </QueryClientProvider>,
  );
  await view.findByRole("region", { name: "Waiting for your approval" });
  // Only the waiting call gets a card; the finished navigate does not.
  expect(
    view.getAllByRole("region", { name: "Waiting for your approval" }),
  ).toHaveLength(1);
  expect(view.getByRole("button", { name: "I did it myself" })).toBeTruthy();
});