Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): proxy /labs/search to the labs service (#4165)

Nicolasfccab9d8f373
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 1b8219f40..2d2a600ff 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -32,6 +32,8 @@ const configSchema = z.object({
SUPPORT_AGENT_URL: z.string().url().optional(),
SUPPORT_AGENT_VERCEL_BYPASS_SECRET: z.string().optional(),
RESEARCH_PROXY_URL: z.string().url().optional(),
+ LABS_SEARCH_URL: z.string().url().optional(),
+ LABS_SEARCH_SECRET: z.string().optional(),
// Express
EXPRESS_TRUST_PROXY: z.coerce.number().optional(),
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index b18fde40c..e5a5f5d12 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1306,6 +1306,7 @@ export type TeamFlags = {
searchFeedbackOptOut?: boolean;
researchBeta?: boolean;
enrichBeta?: boolean;
+ labsSearch?: boolean;
professionalProfileCompanyDataBeta?: boolean;
organizationDataSourceAccess?: Record<
string,
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 3b39e6b74..580b9d41c 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -31,6 +31,7 @@ import { v7 as uuidv7 } from "uuid";
import { attachWsProxy } from "./services/agentLivecastWS";
import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";
import { v2Router } from "./routes/v2";
+import { labsRouter } from "./routes/labs";
import { registerMcpActionLogIngestRoute } from "./routes/mcp-action-logs";
import { startMcpActionLogRetentionWorkerIfEnabled } from "./services/mcp/action-logs";
import { db } from "./db/connection";
@@ -131,6 +132,7 @@ app.get("/e2e-test", (_, res) => {
app.use(v0Router);
app.use("/v1", v1Router);
app.use("/v2", v2Router);
+app.use("/labs", labsRouter);
app.use(adminRouter);
const DEFAULT_PORT = config.PORT;
diff --git a/apps/api/src/routes/labs.ts b/apps/api/src/routes/labs.ts
new file mode 100644
index 000000000..93f58ea0c
--- /dev/null
+++ b/apps/api/src/routes/labs.ts
@@ -0,0 +1,200 @@
+import express, { Request, Response } from "express";
+import { Readable } from "node:stream";
+import { Agent, fetch } from "undici";
+import { config } from "../config";
+import { logger as rootLogger } from "../lib/logger";
+import type { RequestWithAuth } from "../controllers/v1/types";
+import { RateLimiterMode } from "../types";
+import { authMiddleware, wrap } from "./shared";
+
+const TIMEOUT_MS = 120_000;
+
+const FORWARDED_REQUEST_HEADERS = ["accept", "x-request-id"];
+const FORWARDED_RESPONSE_HEADERS = ["content-type", "x-request-id"];
+
+const dispatcher = new Agent({
+ connectTimeout: TIMEOUT_MS,
+ headersTimeout: TIMEOUT_MS,
+ bodyTimeout: TIMEOUT_MS,
+});
+
+function labsError(res: Response, status: number, error: string) {
+ return res.status(status).json({ success: false, error });
+}
+
+function upstreamBase(): string | null {
+ if (!config.LABS_SEARCH_URL || !config.LABS_SEARCH_SECRET) return null;
+ return config.LABS_SEARCH_URL.replace(/\/+$/, "");
+}
+
+function callerApiKey(req: Request): string | undefined {
+ const header = req.headers.authorization;
+ if (typeof header !== "string") return undefined;
+ return header.split(" ")[1];
+}
+
+function isMultipart(req: Request): boolean {
+ const contentType = req.headers["content-type"];
+ return (
+ typeof contentType === "string" &&
+ contentType.includes("multipart/form-data")
+ );
+}
+
+function hasJsonBody(req: Request): boolean {
+ return req.method !== "GET" && req.method !== "DELETE";
+}
+
+function upstreamHeaders(req: RequestWithAuth<any, any, any>) {
+ const headers: Record<string, string> = {};
+ for (const h of FORWARDED_REQUEST_HEADERS) {
+ const value = req.headers[h];
+ if (typeof value === "string") headers[h] = value;
+ }
+
+ headers["x-labs-search-secret"] = config.LABS_SEARCH_SECRET!;
+ headers["x-labs-search-team-id"] = req.auth.team_id;
+ const apiKey = callerApiKey(req);
+ if (apiKey) headers["x-labs-search-key"] = apiKey;
+
+ if (isMultipart(req)) {
+ // The body is piped through unparsed, so the multipart boundary and length
+ // have to travel with it.
+ for (const h of ["content-type", "content-length"]) {
+ const value = req.headers[h];
+ if (typeof value === "string") headers[h] = value;
+ }
+ } else if (hasJsonBody(req)) {
+ headers["content-type"] = "application/json";
+ }
+
+ return headers;
+}
+
+function upstreamBody(req: RequestWithAuth<any, any, any>) {
+ if (isMultipart(req)) {
+ return Readable.toWeb(req) as ReadableStream;
+ }
+ if (!hasJsonBody(req)) return undefined;
+ return JSON.stringify(req.body ?? {});
+}
+
+async function labsProxyController(req: Request, res: Response) {
+ const authedReq = req as RequestWithAuth<any, any, any>;
+ const logger = rootLogger.child({
+ module: "api/labs",
+ method: req.method,
+ path: req.path,
+ teamId: authedReq.auth.team_id,
+ });
+
+ const base = upstreamBase();
+ if (!base) {
+ return labsError(res, 503, "This endpoint is not available.");
+ }
+
+ if (!authedReq.acuc?.flags?.labsSearch) {
+ return labsError(res, 403, "This endpoint is not enabled for this team.");
+ }
+
+ try {
+ const upstream = await fetch(base + req.originalUrl, {
+ method: req.method,
+ headers: upstreamHeaders(authedReq),
+ body: upstreamBody(authedReq),
+ duplex: "half",
+ signal: AbortSignal.timeout(TIMEOUT_MS),
+ dispatcher,
+ });
+
+ for (const h of FORWARDED_RESPONSE_HEADERS) {
+ const value = upstream.headers.get(h);
+ if (value) res.setHeader(h, value);
+ }
+
+ const text = await upstream.text();
+ let responseBody: any;
+ try {
+ responseBody = text ? JSON.parse(text) : null;
+ } catch {
+ responseBody = text;
+ }
+
+ if (responseBody === null || typeof responseBody === "string") {
+ return res.status(upstream.status).send(responseBody ?? "");
+ }
+ return res.status(upstream.status).json(responseBody);
+ } catch (error: unknown) {
+ if (error instanceof DOMException && error.name === "TimeoutError") {
+ logger.error("Labs proxy timed out");
+ return labsError(res, 504, "The request timed out.");
+ }
+ logger.error("Labs proxy error", { error });
+ return labsError(res, 502, "The request could not be completed.");
+ }
+}
+
+export const labsRouter = express.Router();
+
+// No credit check or billing here on purpose: the service behind this proxy
+// makes its own calls to the public API with the caller's own key, so usage is
+// billed on those existing paths instead.
+labsRouter.post(
+ "/search",
+ authMiddleware(RateLimiterMode.Search),
+ wrap(labsProxyController),
+);
+
+labsRouter.post(
+ "/search/data/sites",
+ authMiddleware(RateLimiterMode.Search),
+ wrap(labsProxyController),
+);
+
+labsRouter.post(
+ "/search/data/documents",
+ authMiddleware(RateLimiterMode.Search),
+ wrap(labsProxyController),
+);
+
+labsRouter.get(
+ "/search/data",
+ authMiddleware(RateLimiterMode.Search),
+ wrap(labsProxyController),
+);
+
+labsRouter.delete(
+ "/search/data/:sourceId",
+ authMiddleware(RateLimiterMode.Search),
+ wrap(labsProxyController),
+);
+
+labsRouter.get(
+ "/search/providers",
+ authMiddleware(RateLimiterMode.Search),
+ wrap(labsProxyController),
+);
+
+labsRouter.post(
+ "/search/configs",
+ authMiddleware(RateLimiterMode.Search),
+ wrap(labsProxyController),
+);
+
+labsRouter.get(
+ "/search/configs",
+ authMiddleware(RateLimiterMode.Search),
+ wrap(labsProxyController),
+);
+
+labsRouter.patch(
+ "/search/configs/:id",
+ authMiddleware(RateLimiterMode.Search),
+ wrap(labsProxyController),
+);
+
+labsRouter.delete(
+ "/search/configs/:id",
+ authMiddleware(RateLimiterMode.Search),
+ wrap(labsProxyController),
+);