Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import express, { Request, Response } from "express";
import { Readable } from "node:stream";
import { Agent, fetch } from "undici";
import { config } from "../config";
import { logger as rootLogger } from "../lib/logger";
import { isUsOnly } from "../lib/us-only";
import type { RequestWithAuth } from "../controllers/v1/types";
import { RateLimiterMode } from "../types";
import { authMiddleware, wrap } from "./shared";
const TIMEOUT_MS = 120_000;
const FORWARDED_REQUEST_HEADERS = ["accept", "x-request-id"];
const FORWARDED_RESPONSE_HEADERS = ["content-type", "x-request-id"];
const dispatcher = new Agent({
connectTimeout: TIMEOUT_MS,
headersTimeout: TIMEOUT_MS,
bodyTimeout: TIMEOUT_MS,
});
function labsError(res: Response, status: number, error: string) {
return res.status(status).json({ success: false, error });
}
function upstreamBase(): string | null {
if (!config.LABS_SEARCH_URL || !config.LABS_SEARCH_SECRET) return null;
return config.LABS_SEARCH_URL.replace(/\/+$/, "");
}
function callerApiKey(req: Request): string | undefined {
const header = req.headers.authorization;
if (typeof header !== "string") return undefined;
return header.split(" ")[1];
}
function isMultipart(req: Request): boolean {
const contentType = req.headers["content-type"];
return (
typeof contentType === "string" &&
contentType.includes("multipart/form-data")
);
}
function hasJsonBody(req: Request): boolean {
return req.method !== "GET" && req.method !== "DELETE";
}
function upstreamHeaders(req: RequestWithAuth<any, any, any>) {
const headers: Record<string, string> = {};
for (const h of FORWARDED_REQUEST_HEADERS) {
const value = req.headers[h];
if (typeof value === "string") headers[h] = value;
}
headers["x-labs-search-secret"] = config.LABS_SEARCH_SECRET!;
headers["x-labs-search-team-id"] = req.auth.team_id;
const apiKey = callerApiKey(req);
if (apiKey) headers["x-labs-search-key"] = apiKey;
if (isMultipart(req)) {
for (const h of ["content-type", "content-length"]) {
const value = req.headers[h];
if (typeof value === "string") headers[h] = value;
}
} else if (hasJsonBody(req)) {
headers["content-type"] = "application/json";
}
return headers;
}
function upstreamBody(req: RequestWithAuth<any, any, any>) {
if (isMultipart(req)) {
return Readable.toWeb(req) as ReadableStream;
}
if (!hasJsonBody(req)) return undefined;
return JSON.stringify(req.body ?? {});
}
async function labsProxyController(req: Request, res: Response) {
const authedReq = req as RequestWithAuth<any, any, any>;
const logger = rootLogger.child({
module: "api/labs",
method: req.method,
path: req.path,
teamId: authedReq.auth.team_id,
});
const base = upstreamBase();
if (!base) {
return labsError(res, 503, "This endpoint is not available.");
}
if (!authedReq.acuc?.flags?.labsSearch) {
return labsError(res, 403, "This endpoint is not enabled for this team.");
}
// Every labs route is served from a hosted vector index, so none has a US-only route.
if (isUsOnly(authedReq.acuc?.flags)) {
return res.status(403).json({
success: false,
code: "US_ONLY_UNAVAILABLE",
error:
"Labs Search is not available for teams restricted to US-only processing.",
});
}
try {
const upstream = await fetch(base + req.originalUrl, {
method: req.method,
headers: upstreamHeaders(authedReq),
body: upstreamBody(authedReq),
duplex: "half",
signal: AbortSignal.timeout(TIMEOUT_MS),
dispatcher,
});
for (const h of FORWARDED_RESPONSE_HEADERS) {
const value = upstream.headers.get(h);
if (value) res.setHeader(h, value);
}
const text = await upstream.text();
let responseBody: any;
try {
responseBody = text ? JSON.parse(text) : null;
} catch {
responseBody = text;
}
if (responseBody === null || typeof responseBody === "string") {
return res.status(upstream.status).send(responseBody ?? "");
}
return res.status(upstream.status).json(responseBody);
} catch (error: unknown) {
if (error instanceof DOMException && error.name === "TimeoutError") {
logger.error("Labs proxy timed out");
return labsError(res, 504, "The request timed out.");
}
logger.error("Labs proxy error", { error });
return labsError(res, 502, "The request could not be completed.");
}
}
export const labsRouter = express.Router();
labsRouter.post(
"/search",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.post(
"/search/data/sites",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.post(
"/search/data/documents",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.post(
"/search/data/pages",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.get(
"/search/data",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.patch(
"/search/data/:sourceId",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.delete(
"/search/data/:sourceId",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.post(
"/search/data/:sourceId/refresh",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.get(
"/search/providers",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.get(
"/search/packs",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.patch(
"/search/packs/:packId",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.post(
"/search/configs",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.get(
"/search/configs",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.patch(
"/search/configs/:id",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.delete(
"/search/configs/:id",
authMiddleware(RateLimiterMode.Labs),
wrap(labsProxyController),
);
labsRouter.all(/(.*)/, (req: Request, res: Response) => {
rootLogger.warn("Labs path is not proxied by this API", {
module: "api/labs",
method: req.method,
path: req.originalUrl,
});
return res.status(404).json({
success: false,
code: "LABS_ROUTE_NOT_PROXIED",
error:
`${req.method} ${req.baseUrl}${req.path} is not routed by this API. The Labs Search ` +
`service may serve it, but this proxy does not forward it yet.`,
});
});