Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import express, { Request, Response } from "express";
import { externalRequestId } from "../../lib/external-request-id";
import { z } from "zod";
import { v7 as uuidv7 } from "uuid";
import { logger as rootLogger } from "../../lib/logger";
import {
fetchGovUpstream,
fetchResearchUpstream,
usOnlySkippedWarning,
usOnlyUpstreamRefusal,
} from "../../lib/research-upstream";
import { isUsOnly } from "../../lib/us-only";
import { chargeKeylessCredits } from "../../lib/keyless";
import {
indexDailyLimitRefusal,
reserveIndexCalls,
type IndexName,
} from "../../lib/index-daily-limit";
import { billTeam } from "../../services/billing/credit_billing";
import { getEffectiveSearchForcedKind } from "../../lib/safe-mode";
import {
logRequest,
logResearchEndpoint,
} from "../../services/logging/log_job";
import type {
ResearchRequestKind,
ResearchTableName,
} from "../../services/logging/log_job";
import type { RequestWithAuth } from "../v1/types";
import { checkCreditsMiddleware, wrap } from "../../routes/shared";
import { SEARCH_CREDITS_FEATURE_ID } from "../../services/autumn/autumn.service";
import { integrationSchema, resolveIntegration } from "../../utils/integration";
import { requestOrigin } from "../../lib/request-origin";
const SEARCH_CREDITS_PER_TEN_RESULTS = 2;
// Billed developer searches are refused up front when the team can't cover
// the smallest search charge, the same gate /v2/search applies.
const searchCreditGate = () =>
checkCreditsMiddleware(
SEARCH_CREDITS_PER_TEN_RESULTS,
SEARCH_CREDITS_FEATURE_ID,
{ mcpCredits: true },
);
const ZDR_SEARCH_CREDITS_PER_TEN_RESULTS = 10;
const DEVELOPER_SEARCH_TIMEOUT_MS = 15_000;
const GOV_SEARCH_TIMEOUT_MS = 15_000;
const PAPER_SEARCH_TIMEOUT_MS = 30_000;
const PAPER_INSPECT_TIMEOUT_MS = 5_000;
const SIMILAR_PAPERS_TIMEOUT_MS = 10_000;
const PAPER_READ_TIMEOUT_MS = 120_000;
// The paper endpoints, all served by the Research Index and limited by its
// daily budget.
const RESEARCH_INDEX_KINDS = new Set<ResearchRequestKind>([
"research_paper_search",
"research_related_papers",
"research_paper_read",
"research_paper_inspect",
]);
const FORWARDED_REQUEST_HEADERS = ["accept", "x-request-id"];
const FORWARDED_RESPONSE_HEADERS = ["content-type", "x-request-id"];
const multiString = z
.union([z.string(), z.array(z.string())])
.optional()
.transform(value => {
if (!value) return undefined;
return Array.isArray(value) ? value : [value];
});
const kSchema = (max: number) =>
z.coerce.number().int().positive().max(max).optional();
const dateSchema = z
.string()
.regex(/^\d{4}-\d{2}-\d{2}$/)
.optional();
const commonQuery = {
origin: z.string().optional(),
integration: integrationSchema.optional(),
};
const searchPapersSchema = z.strictObject({
query: z.string().min(1),
k: kSchema(500),
authors: multiString,
categories: multiString,
from: dateSchema,
to: dateSchema,
sort: z.enum(["relevance", "date_desc"]).optional(),
...commonQuery,
});
const paperSchema = z
.strictObject({
query: z.string().min(1).optional(),
k: kSchema(50),
...commonQuery,
})
.refine(value => value.query !== undefined || value.k === undefined, {
message: "k is only valid when query is present",
path: ["k"],
});
const similarPapersSchema = z.strictObject({
intent: z.string().min(1),
mode: z.enum(["similar", "citers", "references"]).optional(),
k: kSchema(500),
rerank: z
.enum(["true", "false"])
.optional()
.transform(value => (value === undefined ? undefined : value === "true")),
anchor: multiString,
...commonQuery,
});
const starsSchema = z.coerce.number().int().nonnegative().optional();
const booleanFlag = z
.union([z.boolean(), z.enum(["true", "false"])])
.optional()
.transform(value =>
value === undefined ? undefined : value === true || value === "true",
);
const developerSearchSchema = z.strictObject({
query: z.string().min(1),
k: kSchema(100),
types: multiString,
repos: multiString,
sources: multiString,
passages: kSchema(5),
language: z.string().min(1).optional(),
topic: multiString,
license: z.string().min(1).optional(),
min_stars: starsSchema,
max_stars: starsSchema,
archived: booleanFlag,
fork: booleanFlag,
skills: z.enum(["only"]).optional(),
...commonQuery,
});
const DEVELOPER_SEARCH_QUERY_KEYS = [
"query",
"k",
"types",
"repos",
"sources",
"passages",
"language",
"topic",
"license",
"min_stars",
"max_stars",
"archived",
"fork",
"skills",
];
const govSource = z.enum(["web", "gov"]);
const govCite = z.string().trim().min(1);
const govCites = z.array(govCite).min(1).max(32);
const govSearchSchema = z.strictObject({
query: z.string().min(1),
k: kSchema(100),
jurisdiction: z
.string()
.trim()
.toLowerCase()
.regex(/^us(-[a-z]{2})?$/, {
message:
'jurisdiction must be "us" for federal, or "us-" plus a two-letter state or territory code, such as "us-ca" or "us-dc"',
})
.optional(),
court: z.string().trim().min(1).optional(),
courtType: z
.enum([
"federal-supreme",
"federal-appellate",
"federal-district",
"federal-bankruptcy",
"federal-special",
"military",
"state-supreme",
"state-appellate",
"state-trial",
"state-special",
"state-attorney-general",
])
.optional(),
cites: z
.unknown()
.transform((value, ctx) => {
const parsed = (Array.isArray(value) ? govCites : govCite).safeParse(
value,
);
if (!parsed.success) {
ctx.issues.push(
...parsed.error.issues.map(
issue => ({ ...issue, input: value }) as z.core.$ZodRawIssue,
),
);
return z.NEVER;
}
const cites = [...new Set([parsed.data].flat())];
return cites.length === 1 ? cites[0] : cites;
})
.optional(),
domain: z.string().trim().min(1).optional(),
sources: z
.union([govSource, z.array(govSource).min(1)])
.optional()
.transform(value =>
value === undefined ? undefined : [...new Set([value].flat())],
),
...commonQuery,
});
type ResearchEndpointConfig = {
kind: ResearchRequestKind;
table: ResearchTableName;
action: string;
targetHint: (
params: Record<string, any>,
req: RequestWithAuth<any, any, any>,
) => string;
timeoutMs?: number;
billAs: "scrape" | "search";
} & (
| {
upstreamPath: (
params: Record<string, any>,
req: RequestWithAuth<any, any, any>,
) => string;
}
| {
fetchUpstream: (
params: Record<string, any>,
headers: Record<string, string>,
) => ReturnType<typeof fetchResearchUpstream>;
}
);
type ResearchController = (req: Request, res: Response) => Promise<any>;
type ResearchQueryParams = Record<string, any> & {
origin?: string;
integration?: string;
};
const LEGACY_SNAKE_CASE_ALIASES: Record<string, string> = {
paperId: "paper_id",
primaryId: "primary_id",
createdDate: "created_date",
updateDate: "update_date",
articleRank: "article_rank",
seedOverlap: "seed_overlap",
poolSize: "pool_size",
resultType: "result_type",
pageType: "page_type",
segmentCount: "segment_count",
readmeUrl: "readme_url",
contentMd: "content_md",
};
function addLegacySnakeCaseAliases<T>(value: T): T {
if (Array.isArray(value)) {
return value.map(addLegacySnakeCaseAliases) as T;
}
if (!value || typeof value !== "object") {
return value;
}
const source = value as Record<string, unknown>;
const object: Record<string, unknown> = {};
for (const [key, item] of Object.entries(source)) {
object[key] = addLegacySnakeCaseAliases(item);
}
for (const [camelKey, snakeKey] of Object.entries(
LEGACY_SNAKE_CASE_ALIASES,
)) {
if (Object.prototype.hasOwnProperty.call(source, camelKey)) {
object[snakeKey] = object[camelKey];
}
}
return object as T;
}
const GOV_RESULT_KEYS: Record<string, string> = {
court_type: "courtType",
cited_as: "citedAs",
updated_at: "updatedAt",
};
function camelCaseGovResults(body: any) {
const gov = body?.data?.gov;
if (!Array.isArray(gov)) return body;
body.data.gov = gov.map(item =>
Object.fromEntries(
Object.entries(item).map(([key, value]) => [
GOV_RESULT_KEYS[key] ?? key,
value,
]),
),
);
return body;
}
function resultCount(body: any): number {
if (Array.isArray(body?.results)) return body.results.length;
const group = body?.data?.web ?? body?.data?.gov;
return Array.isArray(group) ? group.length : 0;
}
function creditsFor(
config: ResearchEndpointConfig,
body: any,
req: RequestWithAuth<any, any, any>,
) {
// The Research Index and the Government Index are free. Developer/code
// search remains billable.
if (config.kind === "gov_search" || RESEARCH_INDEX_KINDS.has(config.kind))
return 0;
if (config.billAs === "scrape") return 1;
const forcedKind = getEffectiveSearchForcedKind(req.acuc?.flags, undefined);
const perTen =
forcedKind === "zdr"
? ZDR_SEARCH_CREDITS_PER_TEN_RESULTS
: SEARCH_CREDITS_PER_TEN_RESULTS;
return Math.ceil(resultCount(body) / 10) * perTen;
}
function researchError(
res: Response,
status: number,
error: string,
details?: unknown,
) {
return res.status(status).json({
success: false,
error,
...(details === undefined ? {} : { details }),
});
}
function isResearchTimeoutError(error: unknown): boolean {
return (
(error instanceof DOMException && error.name === "TimeoutError") ||
(error instanceof Error &&
(error.name === "ConnectTimeoutError" ||
(error.cause instanceof Error &&
error.cause.name === "ConnectTimeoutError")))
);
}
async function fetchForRequest(
req: RequestWithAuth<any, any, any>,
endpoint: ResearchEndpointConfig,
params: Record<string, any>,
queryKeys: string[],
zeroDataRetention: boolean,
) {
const headers: Record<string, string> = {};
for (const h of FORWARDED_REQUEST_HEADERS) {
const v = req.headers[h];
if (typeof v === "string") headers[h] = v;
}
headers["firecrawl-team-id"] = req.auth.team_id;
headers["firecrawl-zero-data-retention"] = String(zeroDataRetention);
if (isUsOnly(req.acuc?.flags)) headers["firecrawl-us-only"] = "true";
if ("fetchUpstream" in endpoint) {
return endpoint.fetchUpstream(params, headers);
}
return fetchResearchUpstream({
path: endpoint.upstreamPath(params, req),
params,
queryKeys,
headers,
timeoutMs: endpoint.timeoutMs,
});
}
function createResearchController(
schema: z.ZodTypeAny,
queryKeys: string[],
endpoint: ResearchEndpointConfig,
options: { legacy?: boolean } = {},
): ResearchController {
return async (req, res: Response) => {
const authedReq = req as RequestWithAuth<any, any, any>;
const zeroDataRetention =
getEffectiveSearchForcedKind(authedReq.acuc?.flags, undefined) !== null;
const started = Date.now();
const jobId = uuidv7();
const logger = rootLogger.child({
module: "api/v2/research",
method: endpoint.action,
jobId,
teamId: authedReq.auth.team_id,
zeroDataRetention,
});
const source = req.method === "POST" ? (req.body ?? {}) : req.query;
const parsed = schema.safeParse(source);
if (!parsed.success) {
logger.warn("Invalid research query", { error: parsed.error.issues });
// The outer finally below never runs for validation failures, but a
// malformed keyless request must still leave the canonical keyless/usage
// trail — ID-enumeration abuse produces plenty of 400s.
chargeKeylessCredits(authedReq.auth.team_id, 0).catch(() => {});
return researchError(
res,
400,
"Invalid query parameters",
parsed.error.issues,
);
}
const params = parsed.data as ResearchQueryParams;
const targetHint = endpoint.targetHint(params, authedReq);
await logRequest({
id: jobId,
kind: endpoint.kind,
api_version: "v2",
external_request_id: externalRequestId(authedReq),
team_id: authedReq.auth.team_id,
origin: requestOrigin(params, req),
integration: resolveIntegration(params.integration, req) ?? null,
target_hint: targetHint,
zeroDataRetention,
api_key_id: authedReq.acuc?.api_key_id ?? null,
});
const index: IndexName | null =
endpoint.kind === "gov_search"
? "gov"
: RESEARCH_INDEX_KINDS.has(endpoint.kind)
? "research"
: null;
let releaseIndexCall: (() => Promise<void>) | undefined;
if (index) {
const indexBudget = await reserveIndexCalls(
index,
authedReq.auth.team_id,
authedReq.acuc?.rate_limit_multiplier ?? 1,
1,
);
if (!indexBudget.ok) {
chargeKeylessCredits(authedReq.auth.team_id, 0).catch(() => {});
res.setHeader("Retry-After", String(indexBudget.retryAfterSeconds));
return res.status(429).json({
success: false,
code: "INDEX_DAILY_LIMIT",
...indexDailyLimitRefusal(index, authedReq.auth.team_id, indexBudget),
});
}
releaseIndexCall = indexBudget.release;
}
let statusCode = 500;
let responseBody: any = null;
let error: string | undefined;
let credits = 0;
try {
const upstream = await fetchForRequest(
authedReq,
endpoint,
params,
queryKeys,
zeroDataRetention,
);
if (!upstream) {
if (releaseIndexCall) await releaseIndexCall();
statusCode = 404;
error = "Research service is not configured";
return res.status(404).end();
}
statusCode = upstream.status;
for (const h of FORWARDED_RESPONSE_HEADERS) {
const value = upstream.headers.get(h);
if (value) res.setHeader(h, value);
}
if (isUsOnly(authedReq.acuc?.flags) && upstream.status === 403) {
const refusal = await usOnlyUpstreamRefusal(
upstream,
"This endpoint is not available for teams restricted to US-only processing.",
);
error = refusal.message;
res.setHeader("content-type", "application/json");
return res.status(403).json({
success: false,
code: "US_ONLY_UNAVAILABLE",
error: refusal.message,
});
}
const text = await upstream.text();
try {
responseBody = text ? JSON.parse(text) : null;
} catch {
responseBody = text;
}
if (upstream.ok) {
credits = creditsFor(endpoint, responseBody, authedReq);
if (credits > 0) {
billTeam(
authedReq.auth.team_id,
authedReq.acuc?.org_id ?? null,
credits,
authedReq.acuc?.api_key_id ?? null,
{
endpoint: endpoint.billAs === "scrape" ? "scrape" : "search",
jobId,
chargeId: jobId,
hostedMcp: authedReq.auth.hostedMcp,
},
).catch(billingError => {
logger.error("Failed to bill research request", {
error: billingError,
credits,
});
});
}
} else {
error =
typeof responseBody === "object" && responseBody !== null
? (responseBody.detail ?? responseBody.error ?? responseBody.title)
: undefined;
}
if (
endpoint.kind === "gov_search" &&
statusCode === 400 &&
typeof responseBody?.message === "string"
) {
error = responseBody.message;
return researchError(res, 400, responseBody.message);
}
if (responseBody === null || typeof responseBody === "string") {
return res.status(statusCode).send(responseBody ?? "");
}
const response = !upstream.ok
? responseBody
: options.legacy
? addLegacySnakeCaseAliases(responseBody)
: endpoint.kind === "gov_search"
? camelCaseGovResults(responseBody)
: responseBody;
const usOnlyWarning =
upstream.ok && isUsOnly(authedReq.acuc?.flags)
? usOnlySkippedWarning(upstream.headers)
: undefined;
if (usOnlyWarning && !Array.isArray(response)) {
response.warning =
typeof response.warning === "string" && response.warning.length > 0
? `${response.warning} ${usOnlyWarning}`
: usOnlyWarning;
}
return res.status(statusCode).json(response);
} catch (err: unknown) {
if (isResearchTimeoutError(err)) {
statusCode = 504;
error = "Research service timed out";
return res.status(504).end();
}
statusCode = 502;
error = "Research proxy error";
logger.error("Research proxy error", { error: err });
return res.status(502).end();
} finally {
const timeTaken = (Date.now() - started) / 1000;
// No-op for keyed teams. For keyless callers: billable credits are added
// to the per-IP daily budget and land as a `keyless_credit_usage` row;
// zero-credit outcomes emit the canonical `keyless/usage` log line
// instead (the durable zero-credit row waits on the firecrawl-db
// migration). It runs on every outcome, not just billable successes: the
// paper endpoints cost 0 credits and ID enumeration mostly produces
// upstream misses, so gating this on `credits > 0` left those requests
// with no IP recorded anywhere. `credits` is still 0 on every non-2xx
// path, so nothing extra is charged.
chargeKeylessCredits(authedReq.auth.team_id, credits).catch(() => {});
logResearchEndpoint({
table: endpoint.table,
id: jobId,
request_id: jobId,
team_id: authedReq.auth.team_id,
target: targetHint,
options: params,
response: responseBody,
num_results: resultCount(responseBody),
time_taken: timeTaken,
credits_cost: statusCode >= 200 && statusCode < 300 ? credits : 0,
is_successful: statusCode >= 200 && statusCode < 300,
error,
zeroDataRetention,
}).catch(logError => {
logger.warn("Research endpoint log failed", { error: logError });
});
}
};
}
function paperId(req: Request) {
return (req.params.id as string[]).filter(Boolean).join("/");
}
export function createResearchRouter(options: { legacy?: boolean } = {}) {
const router = express.Router();
if (options.legacy) {
router.use((req, _res, next) => {
const authedReq = req as RequestWithAuth<any, any, any>;
rootLogger.warn("Legacy research endpoint used", {
module: "api/v2/research",
teamId: authedReq.auth?.team_id,
method: req.method,
path: req.originalUrl,
requestId: req.headers["x-request-id"],
zeroDataRetention:
getEffectiveSearchForcedKind(authedReq.acuc?.flags, undefined) !==
null,
});
next();
});
}
router.get(
"/papers",
wrap(
createResearchController(
searchPapersSchema,
["query", "k", "authors", "categories", "from", "to", "sort"],
{
kind: "research_paper_search",
table: "research_paper_searches",
action: "searchPapers",
targetHint: params => String(params.query),
upstreamPath: () => "/v2/research/papers",
timeoutMs: PAPER_SEARCH_TIMEOUT_MS,
billAs: "search",
},
options,
),
),
);
router.get(
"/papers/*id/similar",
wrap(
createResearchController(
similarPapersSchema,
["intent", "mode", "k", "rerank", "anchor"],
{
kind: "research_related_papers",
table: "research_related_papers",
action: "similarPapers",
targetHint: (params, req) =>
`${paperId(req)}: ${String(params.intent)}`,
upstreamPath: (_params, req) =>
`/v2/research/papers/${encodeURIComponent(paperId(req))}/similar`,
timeoutMs: SIMILAR_PAPERS_TIMEOUT_MS,
billAs: "search",
},
options,
),
),
);
router.get(
"/papers/*id",
wrap(async (req: Request, res: Response) => {
const authedReq = req as RequestWithAuth<any, any, any>;
const parsed = paperSchema.safeParse(req.query);
const isRead = parsed.success && parsed.data.query !== undefined;
const controller = createResearchController(
paperSchema,
["query", "k"],
{
kind: isRead ? "research_paper_read" : "research_paper_inspect",
table: isRead ? "research_paper_reads" : "research_paper_inspects",
action: isRead ? "readPaper" : "inspectPaper",
targetHint: (_params, request) => paperId(request),
upstreamPath: (_params, request) =>
`/v2/research/papers/${encodeURIComponent(paperId(request))}`,
timeoutMs: isRead ? PAPER_READ_TIMEOUT_MS : PAPER_INSPECT_TIMEOUT_MS,
billAs: "scrape",
},
options,
);
return controller(authedReq, res);
}),
);
return router;
}
export function createDeveloperRouter(options: { root?: boolean } = {}) {
const router = express.Router();
const controller = wrap(
createResearchController(
developerSearchSchema,
DEVELOPER_SEARCH_QUERY_KEYS,
{
kind: "code_search",
table: "code_searches",
action: "searchDeveloper",
targetHint: params => String(params.query),
upstreamPath: () => "/v2/code/search",
timeoutMs: DEVELOPER_SEARCH_TIMEOUT_MS,
billAs: "search",
},
),
);
const gate = searchCreditGate();
if (options.root) {
router.get("/", gate, controller);
router.post("/", gate, controller);
} else {
router.get("/search", gate, controller);
router.post("/search", gate, controller);
}
return router;
}
export function createGovRouter() {
const router = express.Router();
const controller = wrap(
createResearchController(govSearchSchema, [], {
kind: "gov_search",
table: "gov_searches",
action: "searchGov",
targetHint: params => String(params.query),
billAs: "search",
fetchUpstream: (params, headers) =>
fetchGovUpstream({
query: params.query,
k: params.k,
jurisdiction: params.jurisdiction,
court: params.court,
courtType: params.courtType,
cites: params.cites,
domain: params.domain,
sources: params.sources,
headers,
timeoutMs: GOV_SEARCH_TIMEOUT_MS,
}),
}),
);
router.get("/", controller);
router.post("/", controller);
return router;
}