Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Alexandria connectors: reuse exchangeRetrieve as the whitelist, proxy /exchange/connectors (#5133)
Developers Digest5b355150afc7
diff --git a/apps/api/src/__tests__/routes/exchange-connectors.routes.test.ts b/apps/api/src/__tests__/routes/exchange-connectors.routes.test.tsnew file mode 100644index 000000000..8bc76b72a--- /dev/null+++ b/apps/api/src/__tests__/routes/exchange-connectors.routes.test.ts@@ -0,0 +1,188 @@+import express from "express";+import request from "supertest";++// The real exchangeRouter, authMiddleware and auth.ts; only I/O is mocked.+const mocks = vi.hoisted(() => ({+ authChunk: vi.fn(),+ upstreamFetch: vi.fn(),+}));++vi.mock("../../db/connection", () => ({ db: {}, dbRr: {}, dbIndex: {} }));+vi.mock("../../db/rpc", () => ({+ authCreditUsageChunk: mocks.authChunk,+ authCreditUsageChunkFromTeam: vi.fn(),+ getAgentFreeRequestsLeft: vi.fn(),+}));+vi.mock("../../services/redis", () => ({+ getValue: vi.fn().mockResolvedValue(null),+ setValue: vi.fn(),+ deleteKey: vi.fn(),+}));+vi.mock("../../services/redlock", () => ({+ redlock: { using: vi.fn(async (_k, _t, _o, fn) => fn({ aborted: false })) },+}));+vi.mock("ioredis", () => ({ default: class {} }));+vi.mock("../../services/rate-limiter", async importOriginal => ({+ ...(await importOriginal<typeof import("../../services/rate-limiter")>()),+ getRateLimiter: vi.fn(() => ({ consume: vi.fn() })),+ getAutumnRateLimiter: vi.fn(() => ({ consume: vi.fn() })),+ redisRateLimitClient: {+ set: vi.fn().mockResolvedValue("OK"),+ get: vi.fn().mockResolvedValue(null),+ del: vi.fn().mockResolvedValue(1),+ },+}));+vi.mock("../../services/autumn/autumn.service", async importOriginal => {+ const actual =+ await importOriginal<+ typeof import("../../services/autumn/autumn.service")+ >();+ return {+ ...actual,+ autumnService: {+ getTeamLimits: vi.fn().mockResolvedValue({+ concurrency_limit: 2,+ rate_limit_multiplier: 1,+ }),+ },+ };+});+vi.mock("../../services/agent-sponsor", () => ({+ getAgentSponsorStatus: vi.fn().mockResolvedValue(null),+}));+vi.mock("../../services/queue-service", () => ({+ getRedisConnection: vi.fn(() => ({ sadd: vi.fn() })),+ getBillingQueue: vi.fn(() => ({ add: vi.fn() })),+}));+vi.mock("../../services/logging/log_job", () => ({+ logRequest: vi.fn(),+ logProviderScrape: vi.fn(),+}));+vi.mock("undici", async importOriginal => ({+ ...(await importOriginal<typeof import("undici")>()),+ fetch: mocks.upstreamFetch,+}));++import { config } from "../../config";+import { exchangeRouter } from "../../routes/exchange";++const KEY = "33333333-3333-4333-8333-333333333333";++function keyResolvesAs(flags: Record<string, unknown> | null) {+ mocks.authChunk.mockImplementation(+ async (_db: unknown, key: string, purpose = "general") =>+ key === KEY && purpose === "general"+ ? [+ {+ api_key: KEY,+ api_key_id: 9,+ api_key_id_text: "9",+ team_id: "team-connectors",+ org_id: "org-connectors",+ is_banned: false,+ flags,+ },+ ]+ : [],+ );+}++const app = express();+app.use(express.json());+app.use("/exchange", exchangeRouter);++function upstream(index = 0): { url: string; init: RequestInit } {+ const [url, init] = mocks.upstreamFetch.mock.calls[index]!;+ return { url: String(url), init: init as RequestInit };+}++describe("Alexandria connector routes on the Exchange proxy", () => {+ const original = {+ USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,+ FIRE_EXCHANGE_URL: config.FIRE_EXCHANGE_URL,+ };++ beforeEach(() => {+ vi.clearAllMocks();+ config.USE_DB_AUTHENTICATION = true;+ config.FIRE_EXCHANGE_URL = "https://exchange.test";+ mocks.upstreamFetch.mockImplementation(+ async () =>+ new Response(JSON.stringify({ connectors: [] }), { status: 200 }),+ );+ });++ afterEach(() => {+ Object.assign(config, original);+ });++ it.each([+ ["get", "/exchange/connectors", "/v1/connectors"],+ ["get", "/exchange/connectors/catalog", "/v1/connectors/catalog"],+ ["post", "/exchange/connectors", "/v1/connectors"],+ ["get", "/exchange/connectors/mcp-deepwiki", "/v1/connectors/mcp-deepwiki"],+ [+ "patch",+ "/exchange/connectors/mcp-deepwiki",+ "/v1/connectors/mcp-deepwiki",+ ],+ [+ "delete",+ "/exchange/connectors/mcp-deepwiki",+ "/v1/connectors/mcp-deepwiki",+ ],+ [+ "post",+ "/exchange/connectors/mcp-deepwiki/sync",+ "/v1/connectors/mcp-deepwiki/sync",+ ],+ [+ "patch",+ "/exchange/connectors/mcp-deepwiki/tools/read_wiki_structure",+ "/v1/connectors/mcp-deepwiki/tools/read_wiki_structure",+ ],+ ] as const)(+ "forwards %s %s for a team with exchangeRetrieve, granting connectors",+ async (method, path, upstreamPath) => {+ keyResolvesAs({ exchangeRetrieve: true });+ const body = { catalogId: "deepwiki" };+ let req = request(app)+ [method](path)+ .set("authorization", `Bearer ${KEY}`);+ if (method !== "get" && method !== "delete") req = req.send(body);+ const response = await req;+ expect(response.status).toBe(200);+ expect(mocks.upstreamFetch).toHaveBeenCalledTimes(1);+ const { url, init } = upstream();+ expect(url).toBe(`https://exchange.test${upstreamPath}`);+ expect(init.method).toBe(method.toUpperCase());+ expect(init.headers).toMatchObject({+ "x-exchange-team-id": "team-connectors",+ "x-exchange-connectors": "true",+ });+ if (method === "post" || method === "patch")+ expect(JSON.parse(String(init.body))).toEqual(body);+ },+ );++ it("refuses a team without exchangeRetrieve before reaching Exchange", async () => {+ keyResolvesAs({});+ const response = await request(app)+ .post("/exchange/connectors")+ .set("authorization", `Bearer ${KEY}`)+ .send({ catalogId: "deepwiki" });+ expect(response.status).toBe(403);+ expect(mocks.upstreamFetch).not.toHaveBeenCalled();+ });++ it("never forwards a caller's own x-exchange-connectors header", async () => {+ keyResolvesAs({});+ // A route open without the flag: the header must still not reach Exchange.+ await request(app)+ .get("/exchange/enrichment/preferences")+ .set("authorization", `Bearer ${KEY}`)+ .set("x-exchange-connectors", "true");+ expect(mocks.upstreamFetch).toHaveBeenCalledTimes(1);+ expect(upstream().init.headers).not.toHaveProperty("x-exchange-connectors");+ });+});diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex b871c7e52..2fcba9cf2 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -756,6 +756,7 @@ export async function scrapeController(? await discoverTools({teamId: req.auth.team_id,+ connectors: req.acuc?.flags?.exchangeRetrieve === true,toolDetail: req.body.toolDetail,urls: [...new Set(diff --git a/apps/api/src/routes/exchange.ts b/apps/api/src/routes/exchange.tsindex 88106a186..39a15d9fb 100644--- a/apps/api/src/routes/exchange.ts+++ b/apps/api/src/routes/exchange.ts@@ -89,6 +89,10 @@ function exchangeProxy(),...(hasBody ? { "content-type": "application/json" } : {}),"x-exchange-team-id": authedReq.auth.team_id,+ // Built here, never forwarded from the caller: the connectors whitelist is this flag.+ ...(authedReq.acuc?.flags?.exchangeRetrieve+ ? { "x-exchange-connectors": "true" }+ : {}),},body: hasBody ? JSON.stringify(req.body ?? {}) : undefined,signal: AbortSignal.timeout(timeout),@@ -224,6 +228,31 @@ exchangeRouter.post(wrap(exchangeProxy(ANALYTICS_TIMEOUT_MS, { requiresRetrieveFlag: false })),);+// A team's own MCP connectors (Alexandria connectors): the catalogue, and adding, listing,+// syncing, changing and removing its connections. Behind the exchangeRetrieve flag, the same+// flag that tells Exchange the team may use them. Connecting and syncing reach the team's server+// and list its tools before answering, so writes get the longer timeout.+exchangeRouter.get(+ "/connectors{/*path}",+ authMiddleware(RateLimiterMode.Labs),+ wrap(exchangeProxy(ANALYTICS_TIMEOUT_MS)),+);+exchangeRouter.post(+ "/connectors{/*path}",+ authMiddleware(RateLimiterMode.Labs),+ wrap(exchangeProxy(RETRIEVE_TIMEOUT_MS)),+);+exchangeRouter.patch(+ "/connectors{/*path}",+ authMiddleware(RateLimiterMode.Labs),+ wrap(exchangeProxy(RETRIEVE_TIMEOUT_MS)),+);+exchangeRouter.delete(+ "/connectors{/*path}",+ authMiddleware(RateLimiterMode.Labs),+ wrap(exchangeProxy(ANALYTICS_TIMEOUT_MS)),+);+exchangeRouter.post("/retrieve",authMiddleware(RateLimiterMode.Exchange, { allowAgentManagedKey: true }),diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.tsindex af1968929..28ff52b80 100644--- a/apps/api/src/search/alexandria.ts+++ b/apps/api/src/search/alexandria.ts@@ -46,6 +46,8 @@ export async function discoverTools(urls?: string[];limit: number;timeoutMs: number;+ /** The team may use its own MCP connectors: their listed tools join its discovery. */+ connectors?: boolean;},logger: Logger,): Promise<ToolDiscovery> {@@ -63,6 +65,7 @@ export async function discoverTools(path: "/v1/retrieve",timeoutMs: remaining(),maximumCredits: 0,+ connectors: input.connectors === true,body: {provider: "firecrawl",capability: "find-tools",@@ -147,6 +150,7 @@ export async function discoverTools(path: "/v1/skills/resolve",body: { urls: urls.slice(i, i + 100) },timeoutMs: remaining(),+ connectors: input.connectors === true,});if (result.status !== 200)throw new Error("Domain discovery unavailable");diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 0a45a5905..b0887fa98 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -262,6 +262,7 @@ export async function executeSearch({toolDetail: options.toolDetail ?? "compact",teamId,+ connectors: flags?.exchangeRetrieve === true,limit,query: wantsTools ? query : undefined,urls:diff --git a/apps/api/src/services/alexandria/client.test.ts b/apps/api/src/services/alexandria/client.test.tsnew file mode 100644index 000000000..656f1cd26--- /dev/null+++ b/apps/api/src/services/alexandria/client.test.ts@@ -0,0 +1,53 @@+const mocks = vi.hoisted(() => ({ fetch: vi.fn() }));++vi.mock("undici", async importOriginal => ({+ ...(await importOriginal<typeof import("undici")>()),+ fetch: mocks.fetch,+}));++import { config } from "../../config";+import { exchangeRequest } from "./client";++describe("exchangeRequest", () => {+ const original = config.FIRE_EXCHANGE_URL;++ beforeEach(() => {+ vi.clearAllMocks();+ config.FIRE_EXCHANGE_URL = "https://exchange.test";+ mocks.fetch.mockImplementation(+ async () => new Response(JSON.stringify({ success: true })),+ );+ });++ afterEach(() => {+ config.FIRE_EXCHANGE_URL = original;+ });++ const headersOf = () =>+ (mocks.fetch.mock.calls[0]![1] as { headers: Record<string, string> })+ .headers;++ it("grants the team's own MCP connectors only when asked", async () => {+ await exchangeRequest({+ teamId: "team",+ path: "/v1/retrieve",+ body: {},+ timeoutMs: 1000,+ connectors: true,+ });+ expect(headersOf()).toMatchObject({+ "x-exchange-team-id": "team",+ "x-exchange-connectors": "true",+ });+ });++ it("sends no connectors grant by default", async () => {+ await exchangeRequest({+ teamId: "team",+ path: "/v1/retrieve",+ body: {},+ timeoutMs: 1000,+ });+ expect(headersOf()).not.toHaveProperty("x-exchange-connectors");+ });+});diff --git a/apps/api/src/services/alexandria/client.ts b/apps/api/src/services/alexandria/client.tsindex 44a946d5a..ebea3b135 100644--- a/apps/api/src/services/alexandria/client.ts+++ b/apps/api/src/services/alexandria/client.ts@@ -17,8 +17,14 @@ export async function exchangeRequest(input: {maximumCredits?: number;resultAuthorization?: string;termsIdentity?: { organizationId: string; apiKeyId: string };+ /**+ * The team may use its own MCP connectors (Alexandria connectors). Sent for teams with the+ * `exchangeRetrieve` flag, which is the connectors whitelist; Exchange reads it as the grant.+ */+ connectors?: boolean;}): Promise<ExchangeResponse> {- if (!config.FIRE_EXCHANGE_URL) throw new Error("Alexandria is not configured");+ if (!config.FIRE_EXCHANGE_URL)+ throw new Error("Alexandria is not configured");const base = config.FIRE_EXCHANGE_URL.replace(/\/+$/, "");const response = await fetch(base + input.path, {method: input.body === undefined ? "GET" : "POST",@@ -38,6 +44,7 @@ export async function exchangeRequest(input: {}: {}),"x-exchange-extended-catalog-access": "true",+ ...(input.connectors ? { "x-exchange-connectors": "true" } : {}),...(input.requestId ? { "x-request-id": input.requestId } : {}),...(input.maximumCredits === undefined? {}diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsindex b0db2f2b1..82105e972 100644--- a/apps/api/src/services/alexandria/retrieve.ts+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -245,6 +245,7 @@ export async function retrieveProviders(input: {path: "/v1/retrieve/quote",body: { requests: input.calls },timeoutMs: Math.min(10000, remaining()),+ connectors: input.flags?.exchangeRetrieve === true,}).catch(error => {logger.warn("Provider quote unavailable", { error, chargeId: id });return undefined;@@ -369,6 +370,8 @@ export async function retrieveProviders(input: {body: { requests: input.calls },timeoutMs: remaining(),requestId: id,+ // The team's own MCP connectors (mcp-...) resolve only when Exchange is told it may use them.+ connectors: input.flags?.exchangeRetrieve === true,// Sent with every execution so Exchange tools that call Firecrawl as the caller// (the caller pays) can authenticate upstream. Exchange decides which tools read it....(input.resultAuthorization