Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { readFileSync } from "node:fs";
import {
RESEARCH_APP_NAMES,
researchRoleOf,
researchRoleOfToolName,
RESEARCH_TOOLS as SHARED_RESEARCH_TOOLS,
} from "@openbot/shared/research";
import type { GrantedTool } from "@openbot/toolkit/granted-tool";
import { parse } from "yaml";
import { z } from "zod";
import {
CATALOGUE,
catalogueEntry,
classifyTool,
researchRoleFor,
resolveServerUrl,
serverCredentialKind,
tokenHeaderFor,
} from "../src/plugins/catalogue";
import { listTools } from "../src/plugins/mcp";
import { selectTools } from "../src/plugins/selection";
import { grantedToolGuidance, holdsWebSearch } from "../src/plugins/tools";
const tool = (server: string, name: string): GrantedTool => ({
name: `mcp__${server}__${name}`,
ref: `${server}/${name}`,
description: "Public web",
parameters: z.object({}),
execute: async () => "unused",
});
const RESEARCH_TOOLS: Record<string, [string, string]> = {
parallel: ["web_search", "web_fetch"],
"parallel-authenticated": ["web_search", "web_fetch"],
exa: ["web_search_exa", "web_fetch_exa"],
"exa-authenticated": ["web_search_exa", "web_fetch_exa"],
firecrawl: ["firecrawl_search", "firecrawl_scrape"],
"firecrawl-authenticated": ["firecrawl_search", "firecrawl_scrape"],
};
const pair = (server = "parallel") =>
(RESEARCH_TOOLS[server] ?? []).map((name) => tool(server, name));
describe("whether a Bot holds a search tool", () => {
test("any provider's search tool counts, from its ref or the name a model sees", () => {
for (const [server, [search, fetch]] of Object.entries(RESEARCH_TOOLS)) {
expect(holdsWebSearch([tool(server, search)])).toBe(true);
expect(holdsWebSearch([tool(server, fetch)])).toBe(false);
expect(researchRoleOfToolName(`mcp__${server}__${search}`)).toBe(
"search",
);
expect(researchRoleOfToolName(`mcp__${server}__${fetch}`)).toBe("fetch");
}
expect(holdsWebSearch([])).toBe(false);
expect(holdsWebSearch([tool("google-drive", "search_files")])).toBe(false);
expect(researchRoleOfToolName("web_search")).toBeNull();
expect(
researchRoleOfToolName("mcp__google-drive__search_files"),
).toBeNull();
});
test("the search apps a Bot without one offers are the catalogue's, by a name the card resolves", () => {
// The card resolves an app by key or title. A renamed title would leave a Bot offering a card
// for an app this deployment has no name for.
const research = CATALOGUE.filter((entry) =>
Object.hasOwn(SHARED_RESEARCH_TOOLS, entry.key),
);
const named = research.filter((entry) =>
RESEARCH_APP_NAMES.includes(entry.title),
);
expect(named.map((entry) => entry.title).sort()).toEqual(
[...RESEARCH_APP_NAMES].sort(),
);
// Every research vendor is offered, so the choice stays the administrator's.
expect(new Set(named.map((entry) => entry.vendor))).toEqual(
new Set(research.map((entry) => entry.vendor)),
);
});
});
describe("research providers in the catalogue", () => {
test("each is pinned to its vendor's official endpoint", () => {
const urls: Record<string, string> = {
parallel: "https://search.parallel.ai/mcp",
"parallel-authenticated": "https://search.parallel.ai/mcp",
exa: "https://mcp.exa.ai/mcp",
"exa-authenticated": "https://mcp.exa.ai/mcp",
firecrawl: "https://mcp.firecrawl.dev/v2/mcp",
"firecrawl-authenticated": "https://mcp.firecrawl.dev/v2/mcp",
};
for (const [key, url] of Object.entries(urls)) {
expect(resolveServerUrl(key, "https://attacker.example")?.url).toBe(url);
}
});
test("names its search and fetch tools, and both read", () => {
for (const [key, [search, fetch]] of Object.entries(RESEARCH_TOOLS)) {
const entry = catalogueEntry(key)!;
expect(SHARED_RESEARCH_TOOLS[key]).toMatchObject({ search, fetch });
expect(classifyTool(entry, search, true)).toBe("read");
expect(classifyTool(entry, fetch, true)).toBe("read");
expect(classifyTool(entry, "not_advertised", false)).toBe("write");
expect(classifyTool(entry, search, true, "write")).toBe("write");
expect(researchRoleFor(`${key}/${search}`)).toBe("search");
expect(researchRoleFor(`${key}/${fetch}`)).toBe("fetch");
}
expect(researchRoleFor("notion/notion-search")).toBeNull();
// The app labels research from this table, so every key must be a real catalogue entry.
for (const key of Object.keys(SHARED_RESEARCH_TOOLS))
expect(catalogueEntry(key)).not.toBeNull();
// Instructions are built from these, so a tool missing its ref must not throw.
expect(researchRoleFor(undefined as never)).toBeNull();
expect(
grantedToolGuidance([{ name: "mcp__drive__search" } as GrantedTool]),
).toContain("drive: search");
});
test("a tool outside an entry's reads is a write, even one the vendor called a read", () => {
const firecrawl = catalogueEntry("firecrawl")!;
// Published as a family, so no write list could have named it.
expect(classifyTool(firecrawl, "firecrawl_monitor_delete", true)).toBe(
"write",
);
expect(
classifyTool(firecrawl, "firecrawl_monitor_delete", true, "read"),
).toBe("write");
expect(classifyTool(firecrawl, "firecrawl_crawl", true)).toBe("write");
expect(classifyTool(firecrawl, "firecrawl_map", true)).toBe("read");
expect(classifyTool(firecrawl, "firecrawl_parse", true)).toBe("read");
// Spends credits on a multi-step run.
expect(classifyTool(catalogueEntry("exa")!, "agent_run", true)).toBe(
"write",
);
});
test("says whose credential each uses, and where Exa's key goes", () => {
expect(serverCredentialKind(catalogueEntry("parallel")!)).toBeNull();
expect(serverCredentialKind(catalogueEntry("exa")!)).toBeNull();
expect(serverCredentialKind(catalogueEntry("firecrawl")!)).toBeNull();
for (const key of [
"parallel-authenticated",
"exa-authenticated",
"firecrawl-authenticated",
])
expect(serverCredentialKind(catalogueEntry(key)!)).toBe("mcp");
expect(tokenHeaderFor(catalogueEntry("exa-authenticated"))).toBe(
"x-api-key",
);
expect(
tokenHeaderFor(catalogueEntry("firecrawl-authenticated")),
).toBeUndefined();
expect(tokenHeaderFor(catalogueEntry("parallel"))).toBeUndefined();
expect(tokenHeaderFor(null)).toBeUndefined();
});
});
describe("what a Bot is told about research", () => {
test("a provider is described only while both of its tools are held", () => {
for (const server of Object.keys(RESEARCH_TOOLS)) {
const guidance = grantedToolGuidance(pair(server));
expect(guidance).toContain("For public-web research you can use:");
expect(guidance).toContain("do not connect a person's private account");
// Described, not preferred: a Bot holding another search tool is not told to pass it over.
expect(guidance).not.toContain("by default");
// Searches unprompted for facts that change, and only a little.
expect(guidance).toContain(
"Search before answering when the answer depends on facts that change",
);
expect(guidance).toContain("a search or two");
expect(grantedToolGuidance(pair(server).slice(0, 1))).not.toContain(
"public-web research",
);
}
expect(grantedToolGuidance([], ["exa"])).not.toContain(
"public-web research",
);
expect(grantedToolGuidance([])).toBe("");
});
test("names each provider's own tools and adds only its own notes", () => {
const exa = grantedToolGuidance(pair("exa"));
expect(exa).toContain(
"- Exa: mcp__exa__web_search_exa finds sources and mcp__exa__web_fetch_exa reads selected pages.",
);
expect(exa).not.toContain("session_id");
expect(exa).toContain("Do not name the provider in your answer");
expect(grantedToolGuidance(pair())).toContain(
"Parallel: Generate one session_id",
);
});
test("a vendor added twice is described once, with its key", () => {
const guidance = grantedToolGuidance([
...pair(),
...pair("parallel-authenticated"),
]);
expect(guidance).toContain(
"- Parallel: mcp__parallel-authenticated__web_search finds sources",
);
expect(guidance).not.toContain("mcp__parallel__web_search");
});
test("two vendors are both described, and the person's choice decides", () => {
const guidance = grantedToolGuidance([...pair(), ...pair("firecrawl")]);
expect(guidance).toContain("- Parallel:");
expect(guidance).toContain("- Firecrawl:");
expect(guidance).toContain("Use the one the person asks for.");
});
});
describe("a skill declaring research by role", () => {
const others = Array.from({ length: 12 }, (_, index) => ({
ref: `slack/tool_${index}`,
}));
const research = {
slug: "research",
title: "Research",
summary: "Research the public web.",
tools: ["@research/search", "@research/fetch"],
};
const slack = {
slug: "slack",
title: "Slack",
summary: "Slack.",
tools: others.map((entry) => entry.ref),
};
const choose = (slug: string) => async () =>
JSON.stringify({ skills: [slug] });
test("brings whichever provider's tools the Bot holds", async () => {
const tools = [
...others,
{ ref: "exa/web_search_exa" },
{ ref: "exa/web_fetch_exa" },
{ ref: "exa/web_search_advanced_exa" },
];
const selection = await selectTools({
tools,
skills: [research, slack],
text: "find recent coverage of this",
choose: choose("research"),
});
const offered = selection.offered.map((entry) => entry.ref);
expect(offered).toContain("exa/web_search_exa");
expect(offered).toContain("exa/web_fetch_exa");
expect(offered).not.toContain("slack/tool_0");
// Not a research role, and no skill declares it, so it is offered as any undeclared tool is.
expect(offered).toContain("exa/web_search_advanced_exa");
});
test("never offers a provider the Bot was not granted", async () => {
const selection = await selectTools({
tools: [...others, { ref: "firecrawl/firecrawl_search" }],
skills: [research, slack],
text: "find recent coverage of this",
choose: choose("research"),
});
const offered = selection.offered.map((entry) => entry.ref);
expect(offered).toEqual(["firecrawl/firecrawl_search"]);
});
test("a role ref is only ever the two roles", () => {
expect(researchRoleOf("@research/search")).toBe("search");
expect(researchRoleOf("@research/fetch")).toBe("fetch");
expect(researchRoleOf("@research/delete")).toBeNull();
expect(researchRoleOf("@research/search/more")).toBeNull();
expect(researchRoleOf("research/search")).toBeNull();
});
test("the shipped Research Desk skill declares roles, not one provider", () => {
const skills = parse(
readFileSync(
new URL("../../examples/fintech/skills.yaml", import.meta.url),
"utf8",
),
).skills;
const skill = skills.find(
(item: { slug: string }) => item.slug === "research-public-web",
);
expect(skill.tools).toEqual(["@research/search", "@research/fetch"]);
const desk = parse(
readFileSync(
new URL(
"../../examples/fintech/catalog/research-desk.yaml",
import.meta.url,
),
"utf8",
),
);
expect(desk.skills).toContain("research-public-web");
});
});
describe("the header a deployment token is sent in", () => {
let server: ReturnType<typeof Bun.serve>;
let seen: Headers | undefined;
beforeAll(() => {
server = Bun.serve({
port: 0,
fetch(request) {
seen ??= request.headers;
return new Response("no", { status: 401 });
},
});
});
afterAll(() => server.stop(true));
const listed = async (connection: {
token: string;
tokenHeader?: string;
}) => {
seen = undefined;
await listTools({
url: `http://127.0.0.1:${server.port}/mcp`,
...connection,
}).catch(() => undefined);
return seen;
};
test("is Authorization as a Bearer token by default", async () => {
const headers = await listed({ token: "secret" });
expect(headers?.get("authorization")).toBe("Bearer secret");
});
test("is the vendor's own header, as stored, when the entry names one", async () => {
const headers = await listed({
token: " secret ",
tokenHeader: "x-api-key",
});
expect(headers?.get("x-api-key")).toBe("secret");
expect(headers?.get("authorization")).toBeNull();
});
});