Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Alexandria connectors: reuse exchangeRetrieve as the whitelist, proxy /exchange/connectors (#5133)

Developers Digest5b355150afc7
diff --git a/apps/api/src/__tests__/routes/exchange-connectors.routes.test.ts b/apps/api/src/__tests__/routes/exchange-connectors.routes.test.ts
new file mode 100644
index 000000000..8bc76b72a
--- /dev/null
+++ b/apps/api/src/__tests__/routes/exchange-connectors.routes.test.ts
@@ -0,0 +1,188 @@
+import express from "express";
+import request from "supertest";
+
+// The real exchangeRouter, authMiddleware and auth.ts; only I/O is mocked.
+const mocks = vi.hoisted(() => ({
+ authChunk: vi.fn(),
+ upstreamFetch: vi.fn(),
+}));
+
+vi.mock("../../db/connection", () => ({ db: {}, dbRr: {}, dbIndex: {} }));
+vi.mock("../../db/rpc", () => ({
+ authCreditUsageChunk: mocks.authChunk,
+ authCreditUsageChunkFromTeam: vi.fn(),
+ getAgentFreeRequestsLeft: vi.fn(),
+}));
+vi.mock("../../services/redis", () => ({
+ getValue: vi.fn().mockResolvedValue(null),
+ setValue: vi.fn(),
+ deleteKey: vi.fn(),
+}));
+vi.mock("../../services/redlock", () => ({
+ redlock: { using: vi.fn(async (_k, _t, _o, fn) => fn({ aborted: false })) },
+}));
+vi.mock("ioredis", () => ({ default: class {} }));
+vi.mock("../../services/rate-limiter", async importOriginal => ({
+ ...(await importOriginal<typeof import("../../services/rate-limiter")>()),
+ getRateLimiter: vi.fn(() => ({ consume: vi.fn() })),
+ getAutumnRateLimiter: vi.fn(() => ({ consume: vi.fn() })),
+ redisRateLimitClient: {
+ set: vi.fn().mockResolvedValue("OK"),
+ get: vi.fn().mockResolvedValue(null),
+ del: vi.fn().mockResolvedValue(1),
+ },
+}));
+vi.mock("../../services/autumn/autumn.service", async importOriginal => {
+ const actual =
+ await importOriginal<
+ typeof import("../../services/autumn/autumn.service")
+ >();
+ return {
+ ...actual,
+ autumnService: {
+ getTeamLimits: vi.fn().mockResolvedValue({
+ concurrency_limit: 2,
+ rate_limit_multiplier: 1,
+ }),
+ },
+ };
+});
+vi.mock("../../services/agent-sponsor", () => ({
+ getAgentSponsorStatus: vi.fn().mockResolvedValue(null),
+}));
+vi.mock("../../services/queue-service", () => ({
+ getRedisConnection: vi.fn(() => ({ sadd: vi.fn() })),
+ getBillingQueue: vi.fn(() => ({ add: vi.fn() })),
+}));
+vi.mock("../../services/logging/log_job", () => ({
+ logRequest: vi.fn(),
+ logProviderScrape: vi.fn(),
+}));
+vi.mock("undici", async importOriginal => ({
+ ...(await importOriginal<typeof import("undici")>()),
+ fetch: mocks.upstreamFetch,
+}));
+
+import { config } from "../../config";
+import { exchangeRouter } from "../../routes/exchange";
+
+const KEY = "33333333-3333-4333-8333-333333333333";
+
+function keyResolvesAs(flags: Record<string, unknown> | null) {
+ mocks.authChunk.mockImplementation(
+ async (_db: unknown, key: string, purpose = "general") =>
+ key === KEY && purpose === "general"
+ ? [
+ {
+ api_key: KEY,
+ api_key_id: 9,
+ api_key_id_text: "9",
+ team_id: "team-connectors",
+ org_id: "org-connectors",
+ is_banned: false,
+ flags,
+ },
+ ]
+ : [],
+ );
+}
+
+const app = express();
+app.use(express.json());
+app.use("/exchange", exchangeRouter);
+
+function upstream(index = 0): { url: string; init: RequestInit } {
+ const [url, init] = mocks.upstreamFetch.mock.calls[index]!;
+ return { url: String(url), init: init as RequestInit };
+}
+
+describe("Alexandria connector routes on the Exchange proxy", () => {
+ const original = {
+ USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,
+ FIRE_EXCHANGE_URL: config.FIRE_EXCHANGE_URL,
+ };
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ config.USE_DB_AUTHENTICATION = true;
+ config.FIRE_EXCHANGE_URL = "https://exchange.test";
+ mocks.upstreamFetch.mockImplementation(
+ async () =>
+ new Response(JSON.stringify({ connectors: [] }), { status: 200 }),
+ );
+ });
+
+ afterEach(() => {
+ Object.assign(config, original);
+ });
+
+ it.each([
+ ["get", "/exchange/connectors", "/v1/connectors"],
+ ["get", "/exchange/connectors/catalog", "/v1/connectors/catalog"],
+ ["post", "/exchange/connectors", "/v1/connectors"],
+ ["get", "/exchange/connectors/mcp-deepwiki", "/v1/connectors/mcp-deepwiki"],
+ [
+ "patch",
+ "/exchange/connectors/mcp-deepwiki",
+ "/v1/connectors/mcp-deepwiki",
+ ],
+ [
+ "delete",
+ "/exchange/connectors/mcp-deepwiki",
+ "/v1/connectors/mcp-deepwiki",
+ ],
+ [
+ "post",
+ "/exchange/connectors/mcp-deepwiki/sync",
+ "/v1/connectors/mcp-deepwiki/sync",
+ ],
+ [
+ "patch",
+ "/exchange/connectors/mcp-deepwiki/tools/read_wiki_structure",
+ "/v1/connectors/mcp-deepwiki/tools/read_wiki_structure",
+ ],
+ ] as const)(
+ "forwards %s %s for a team with exchangeRetrieve, granting connectors",
+ async (method, path, upstreamPath) => {
+ keyResolvesAs({ exchangeRetrieve: true });
+ const body = { catalogId: "deepwiki" };
+ let req = request(app)
+ [method](path)
+ .set("authorization", `Bearer ${KEY}`);
+ if (method !== "get" && method !== "delete") req = req.send(body);
+ const response = await req;
+ expect(response.status).toBe(200);
+ expect(mocks.upstreamFetch).toHaveBeenCalledTimes(1);
+ const { url, init } = upstream();
+ expect(url).toBe(`https://exchange.test${upstreamPath}`);
+ expect(init.method).toBe(method.toUpperCase());
+ expect(init.headers).toMatchObject({
+ "x-exchange-team-id": "team-connectors",
+ "x-exchange-connectors": "true",
+ });
+ if (method === "post" || method === "patch")
+ expect(JSON.parse(String(init.body))).toEqual(body);
+ },
+ );
+
+ it("refuses a team without exchangeRetrieve before reaching Exchange", async () => {
+ keyResolvesAs({});
+ const response = await request(app)
+ .post("/exchange/connectors")
+ .set("authorization", `Bearer ${KEY}`)
+ .send({ catalogId: "deepwiki" });
+ expect(response.status).toBe(403);
+ expect(mocks.upstreamFetch).not.toHaveBeenCalled();
+ });
+
+ it("never forwards a caller's own x-exchange-connectors header", async () => {
+ keyResolvesAs({});
+ // A route open without the flag: the header must still not reach Exchange.
+ await request(app)
+ .get("/exchange/enrichment/preferences")
+ .set("authorization", `Bearer ${KEY}`)
+ .set("x-exchange-connectors", "true");
+ expect(mocks.upstreamFetch).toHaveBeenCalledTimes(1);
+ expect(upstream().init.headers).not.toHaveProperty("x-exchange-connectors");
+ });
+});
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index b871c7e52..2fcba9cf2 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -756,6 +756,7 @@ export async function scrapeController(
? await discoverTools(
{
teamId: req.auth.team_id,
+ connectors: req.acuc?.flags?.exchangeRetrieve === true,
toolDetail: req.body.toolDetail,
urls: [
...new Set(
diff --git a/apps/api/src/routes/exchange.ts b/apps/api/src/routes/exchange.ts
index 88106a186..39a15d9fb 100644
--- a/apps/api/src/routes/exchange.ts
+++ b/apps/api/src/routes/exchange.ts
@@ -89,6 +89,10 @@ function exchangeProxy(
),
...(hasBody ? { "content-type": "application/json" } : {}),
"x-exchange-team-id": authedReq.auth.team_id,
+ // Built here, never forwarded from the caller: the connectors whitelist is this flag.
+ ...(authedReq.acuc?.flags?.exchangeRetrieve
+ ? { "x-exchange-connectors": "true" }
+ : {}),
},
body: hasBody ? JSON.stringify(req.body ?? {}) : undefined,
signal: AbortSignal.timeout(timeout),
@@ -224,6 +228,31 @@ exchangeRouter.post(
wrap(exchangeProxy(ANALYTICS_TIMEOUT_MS, { requiresRetrieveFlag: false })),
);
+// A team's own MCP connectors (Alexandria connectors): the catalogue, and adding, listing,
+// syncing, changing and removing its connections. Behind the exchangeRetrieve flag, the same
+// flag that tells Exchange the team may use them. Connecting and syncing reach the team's server
+// and list its tools before answering, so writes get the longer timeout.
+exchangeRouter.get(
+ "/connectors{/*path}",
+ authMiddleware(RateLimiterMode.Labs),
+ wrap(exchangeProxy(ANALYTICS_TIMEOUT_MS)),
+);
+exchangeRouter.post(
+ "/connectors{/*path}",
+ authMiddleware(RateLimiterMode.Labs),
+ wrap(exchangeProxy(RETRIEVE_TIMEOUT_MS)),
+);
+exchangeRouter.patch(
+ "/connectors{/*path}",
+ authMiddleware(RateLimiterMode.Labs),
+ wrap(exchangeProxy(RETRIEVE_TIMEOUT_MS)),
+);
+exchangeRouter.delete(
+ "/connectors{/*path}",
+ authMiddleware(RateLimiterMode.Labs),
+ wrap(exchangeProxy(ANALYTICS_TIMEOUT_MS)),
+);
+
exchangeRouter.post(
"/retrieve",
authMiddleware(RateLimiterMode.Exchange, { allowAgentManagedKey: true }),
diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.ts
index af1968929..28ff52b80 100644
--- a/apps/api/src/search/alexandria.ts
+++ b/apps/api/src/search/alexandria.ts
@@ -46,6 +46,8 @@ export async function discoverTools(
urls?: string[];
limit: number;
timeoutMs: number;
+ /** The team may use its own MCP connectors: their listed tools join its discovery. */
+ connectors?: boolean;
},
logger: Logger,
): Promise<ToolDiscovery> {
@@ -63,6 +65,7 @@ export async function discoverTools(
path: "/v1/retrieve",
timeoutMs: remaining(),
maximumCredits: 0,
+ connectors: input.connectors === true,
body: {
provider: "firecrawl",
capability: "find-tools",
@@ -147,6 +150,7 @@ export async function discoverTools(
path: "/v1/skills/resolve",
body: { urls: urls.slice(i, i + 100) },
timeoutMs: remaining(),
+ connectors: input.connectors === true,
});
if (result.status !== 200)
throw new Error("Domain discovery unavailable");
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 0a45a5905..b0887fa98 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -262,6 +262,7 @@ export async function executeSearch(
{
toolDetail: options.toolDetail ?? "compact",
teamId,
+ connectors: flags?.exchangeRetrieve === true,
limit,
query: wantsTools ? query : undefined,
urls:
diff --git a/apps/api/src/services/alexandria/client.test.ts b/apps/api/src/services/alexandria/client.test.ts
new file mode 100644
index 000000000..656f1cd26
--- /dev/null
+++ b/apps/api/src/services/alexandria/client.test.ts
@@ -0,0 +1,53 @@
+const mocks = vi.hoisted(() => ({ fetch: vi.fn() }));
+
+vi.mock("undici", async importOriginal => ({
+ ...(await importOriginal<typeof import("undici")>()),
+ fetch: mocks.fetch,
+}));
+
+import { config } from "../../config";
+import { exchangeRequest } from "./client";
+
+describe("exchangeRequest", () => {
+ const original = config.FIRE_EXCHANGE_URL;
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ config.FIRE_EXCHANGE_URL = "https://exchange.test";
+ mocks.fetch.mockImplementation(
+ async () => new Response(JSON.stringify({ success: true })),
+ );
+ });
+
+ afterEach(() => {
+ config.FIRE_EXCHANGE_URL = original;
+ });
+
+ const headersOf = () =>
+ (mocks.fetch.mock.calls[0]![1] as { headers: Record<string, string> })
+ .headers;
+
+ it("grants the team's own MCP connectors only when asked", async () => {
+ await exchangeRequest({
+ teamId: "team",
+ path: "/v1/retrieve",
+ body: {},
+ timeoutMs: 1000,
+ connectors: true,
+ });
+ expect(headersOf()).toMatchObject({
+ "x-exchange-team-id": "team",
+ "x-exchange-connectors": "true",
+ });
+ });
+
+ it("sends no connectors grant by default", async () => {
+ await exchangeRequest({
+ teamId: "team",
+ path: "/v1/retrieve",
+ body: {},
+ timeoutMs: 1000,
+ });
+ expect(headersOf()).not.toHaveProperty("x-exchange-connectors");
+ });
+});
diff --git a/apps/api/src/services/alexandria/client.ts b/apps/api/src/services/alexandria/client.ts
index 44a946d5a..ebea3b135 100644
--- a/apps/api/src/services/alexandria/client.ts
+++ b/apps/api/src/services/alexandria/client.ts
@@ -17,8 +17,14 @@ export async function exchangeRequest(input: {
maximumCredits?: number;
resultAuthorization?: string;
termsIdentity?: { organizationId: string; apiKeyId: string };
+ /**
+ * The team may use its own MCP connectors (Alexandria connectors). Sent for teams with the
+ * `exchangeRetrieve` flag, which is the connectors whitelist; Exchange reads it as the grant.
+ */
+ connectors?: boolean;
}): Promise<ExchangeResponse> {
- if (!config.FIRE_EXCHANGE_URL) throw new Error("Alexandria is not configured");
+ if (!config.FIRE_EXCHANGE_URL)
+ throw new Error("Alexandria is not configured");
const base = config.FIRE_EXCHANGE_URL.replace(/\/+$/, "");
const response = await fetch(base + input.path, {
method: input.body === undefined ? "GET" : "POST",
@@ -38,6 +44,7 @@ export async function exchangeRequest(input: {
}
: {}),
"x-exchange-extended-catalog-access": "true",
+ ...(input.connectors ? { "x-exchange-connectors": "true" } : {}),
...(input.requestId ? { "x-request-id": input.requestId } : {}),
...(input.maximumCredits === undefined
? {}
diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.ts
index b0db2f2b1..82105e972 100644
--- a/apps/api/src/services/alexandria/retrieve.ts
+++ b/apps/api/src/services/alexandria/retrieve.ts
@@ -245,6 +245,7 @@ export async function retrieveProviders(input: {
path: "/v1/retrieve/quote",
body: { requests: input.calls },
timeoutMs: Math.min(10000, remaining()),
+ connectors: input.flags?.exchangeRetrieve === true,
}).catch(error => {
logger.warn("Provider quote unavailable", { error, chargeId: id });
return undefined;
@@ -369,6 +370,8 @@ export async function retrieveProviders(input: {
body: { requests: input.calls },
timeoutMs: remaining(),
requestId: id,
+ // The team's own MCP connectors (mcp-...) resolve only when Exchange is told it may use them.
+ connectors: input.flags?.exchangeRetrieve === true,
// Sent with every execution so Exchange tools that call Firecrawl as the caller
// (the caller pays) can authenticate upstream. Exchange decides which tools read it.
...(input.resultAuthorization