Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

22.4 KB
Raw Blame
import express, { Request, Response } from "express";
import { externalRequestId } from "../../lib/external-request-id";
import { z } from "zod";
import { v7 as uuidv7 } from "uuid";
import { logger as rootLogger } from "../../lib/logger";
import {
  fetchGovUpstream,
  fetchResearchUpstream,
  usOnlySkippedWarning,
  usOnlyUpstreamRefusal,
} from "../../lib/research-upstream";
import { isUsOnly } from "../../lib/us-only";
import { chargeKeylessCredits } from "../../lib/keyless";
import {
  indexDailyLimitRefusal,
  reserveIndexCalls,
  type IndexName,
} from "../../lib/index-daily-limit";
import { billTeam } from "../../services/billing/credit_billing";
import { getEffectiveSearchForcedKind } from "../../lib/safe-mode";
import {
  logRequest,
  logResearchEndpoint,
} from "../../services/logging/log_job";
import type {
  ResearchRequestKind,
  ResearchTableName,
} from "../../services/logging/log_job";
import type { RequestWithAuth } from "../v1/types";
import { checkCreditsMiddleware, wrap } from "../../routes/shared";
import { SEARCH_CREDITS_FEATURE_ID } from "../../services/autumn/autumn.service";
import { integrationSchema, resolveIntegration } from "../../utils/integration";
import { requestOrigin } from "../../lib/request-origin";

const SEARCH_CREDITS_PER_TEN_RESULTS = 2;
// Billed developer searches are refused up front when the team can't cover
// the smallest search charge, the same gate /v2/search applies.
const searchCreditGate = () =>
  checkCreditsMiddleware(
    SEARCH_CREDITS_PER_TEN_RESULTS,
    SEARCH_CREDITS_FEATURE_ID,
    { mcpCredits: true },
  );
const ZDR_SEARCH_CREDITS_PER_TEN_RESULTS = 10;

const DEVELOPER_SEARCH_TIMEOUT_MS = 15_000;
const GOV_SEARCH_TIMEOUT_MS = 15_000;
const PAPER_SEARCH_TIMEOUT_MS = 30_000;
const PAPER_INSPECT_TIMEOUT_MS = 5_000;
const SIMILAR_PAPERS_TIMEOUT_MS = 10_000;
const PAPER_READ_TIMEOUT_MS = 120_000;

// The paper endpoints, all served by the Research Index and limited by its
// daily budget.
const RESEARCH_INDEX_KINDS = new Set<ResearchRequestKind>([
  "research_paper_search",
  "research_related_papers",
  "research_paper_read",
  "research_paper_inspect",
]);

const FORWARDED_REQUEST_HEADERS = ["accept", "x-request-id"];
const FORWARDED_RESPONSE_HEADERS = ["content-type", "x-request-id"];

const multiString = z
  .union([z.string(), z.array(z.string())])
  .optional()
  .transform(value => {
    if (!value) return undefined;
    return Array.isArray(value) ? value : [value];
  });

const kSchema = (max: number) =>
  z.coerce.number().int().positive().max(max).optional();

const dateSchema = z
  .string()
  .regex(/^\d{4}-\d{2}-\d{2}$/)
  .optional();

const commonQuery = {
  origin: z.string().optional(),
  integration: integrationSchema.optional(),
};

const searchPapersSchema = z.strictObject({
  query: z.string().min(1),
  k: kSchema(500),
  authors: multiString,
  categories: multiString,
  from: dateSchema,
  to: dateSchema,
  sort: z.enum(["relevance", "date_desc"]).optional(),
  ...commonQuery,
});

const paperSchema = z
  .strictObject({
    query: z.string().min(1).optional(),
    k: kSchema(50),
    ...commonQuery,
  })
  .refine(value => value.query !== undefined || value.k === undefined, {
    message: "k is only valid when query is present",
    path: ["k"],
  });

const similarPapersSchema = z.strictObject({
  intent: z.string().min(1),
  mode: z.enum(["similar", "citers", "references"]).optional(),
  k: kSchema(500),
  rerank: z
    .enum(["true", "false"])
    .optional()
    .transform(value => (value === undefined ? undefined : value === "true")),
  anchor: multiString,
  ...commonQuery,
});

const starsSchema = z.coerce.number().int().nonnegative().optional();

const booleanFlag = z
  .union([z.boolean(), z.enum(["true", "false"])])
  .optional()
  .transform(value =>
    value === undefined ? undefined : value === true || value === "true",
  );

const developerSearchSchema = z.strictObject({
  query: z.string().min(1),
  k: kSchema(100),
  types: multiString,
  repos: multiString,
  sources: multiString,
  passages: kSchema(5),
  language: z.string().min(1).optional(),
  topic: multiString,
  license: z.string().min(1).optional(),
  min_stars: starsSchema,
  max_stars: starsSchema,
  archived: booleanFlag,
  fork: booleanFlag,
  skills: z.enum(["only"]).optional(),
  ...commonQuery,
});

const DEVELOPER_SEARCH_QUERY_KEYS = [
  "query",
  "k",
  "types",
  "repos",
  "sources",
  "passages",
  "language",
  "topic",
  "license",
  "min_stars",
  "max_stars",
  "archived",
  "fork",
  "skills",
];

const govSource = z.enum(["web", "gov"]);

const govCite = z.string().trim().min(1);

const govCites = z.array(govCite).min(1).max(32);

const govSearchSchema = z.strictObject({
  query: z.string().min(1),
  k: kSchema(100),
  jurisdiction: z
    .string()
    .trim()
    .toLowerCase()
    .regex(/^us(-[a-z]{2})?$/, {
      message:
        'jurisdiction must be "us" for federal, or "us-" plus a two-letter state or territory code, such as "us-ca" or "us-dc"',
    })
    .optional(),
  court: z.string().trim().min(1).optional(),
  courtType: z
    .enum([
      "federal-supreme",
      "federal-appellate",
      "federal-district",
      "federal-bankruptcy",
      "federal-special",
      "military",
      "state-supreme",
      "state-appellate",
      "state-trial",
      "state-special",
      "state-attorney-general",
    ])
    .optional(),
  cites: z
    .unknown()
    .transform((value, ctx) => {
      const parsed = (Array.isArray(value) ? govCites : govCite).safeParse(
        value,
      );
      if (!parsed.success) {
        ctx.issues.push(
          ...parsed.error.issues.map(
            issue => ({ ...issue, input: value }) as z.core.$ZodRawIssue,
          ),
        );
        return z.NEVER;
      }
      const cites = [...new Set([parsed.data].flat())];
      return cites.length === 1 ? cites[0] : cites;
    })
    .optional(),
  domain: z.string().trim().min(1).optional(),
  sources: z
    .union([govSource, z.array(govSource).min(1)])
    .optional()
    .transform(value =>
      value === undefined ? undefined : [...new Set([value].flat())],
    ),
  ...commonQuery,
});

type ResearchEndpointConfig = {
  kind: ResearchRequestKind;
  table: ResearchTableName;
  action: string;
  targetHint: (
    params: Record<string, any>,
    req: RequestWithAuth<any, any, any>,
  ) => string;
  timeoutMs?: number;
  billAs: "scrape" | "search";
} & (
  | {
      upstreamPath: (
        params: Record<string, any>,
        req: RequestWithAuth<any, any, any>,
      ) => string;
    }
  | {
      fetchUpstream: (
        params: Record<string, any>,
        headers: Record<string, string>,
      ) => ReturnType<typeof fetchResearchUpstream>;
    }
);

type ResearchController = (req: Request, res: Response) => Promise<any>;
type ResearchQueryParams = Record<string, any> & {
  origin?: string;
  integration?: string;
};

const LEGACY_SNAKE_CASE_ALIASES: Record<string, string> = {
  paperId: "paper_id",
  primaryId: "primary_id",
  createdDate: "created_date",
  updateDate: "update_date",
  articleRank: "article_rank",
  seedOverlap: "seed_overlap",
  poolSize: "pool_size",
  resultType: "result_type",
  pageType: "page_type",
  segmentCount: "segment_count",
  readmeUrl: "readme_url",
  contentMd: "content_md",
};

function addLegacySnakeCaseAliases<T>(value: T): T {
  if (Array.isArray(value)) {
    return value.map(addLegacySnakeCaseAliases) as T;
  }

  if (!value || typeof value !== "object") {
    return value;
  }

  const source = value as Record<string, unknown>;
  const object: Record<string, unknown> = {};

  for (const [key, item] of Object.entries(source)) {
    object[key] = addLegacySnakeCaseAliases(item);
  }

  for (const [camelKey, snakeKey] of Object.entries(
    LEGACY_SNAKE_CASE_ALIASES,
  )) {
    if (Object.prototype.hasOwnProperty.call(source, camelKey)) {
      object[snakeKey] = object[camelKey];
    }
  }

  return object as T;
}

const GOV_RESULT_KEYS: Record<string, string> = {
  court_type: "courtType",
  cited_as: "citedAs",
  updated_at: "updatedAt",
};

function camelCaseGovResults(body: any) {
  const gov = body?.data?.gov;
  if (!Array.isArray(gov)) return body;
  body.data.gov = gov.map(item =>
    Object.fromEntries(
      Object.entries(item).map(([key, value]) => [
        GOV_RESULT_KEYS[key] ?? key,
        value,
      ]),
    ),
  );
  return body;
}

function resultCount(body: any): number {
  if (Array.isArray(body?.results)) return body.results.length;
  const group = body?.data?.web ?? body?.data?.gov;
  return Array.isArray(group) ? group.length : 0;
}

function creditsFor(
  config: ResearchEndpointConfig,
  body: any,
  req: RequestWithAuth<any, any, any>,
) {
  // The Research Index and the Government Index are free. Developer/code
  // search remains billable.
  if (config.kind === "gov_search" || RESEARCH_INDEX_KINDS.has(config.kind))
    return 0;

  if (config.billAs === "scrape") return 1;
  const forcedKind = getEffectiveSearchForcedKind(req.acuc?.flags, undefined);
  const perTen =
    forcedKind === "zdr"
      ? ZDR_SEARCH_CREDITS_PER_TEN_RESULTS
      : SEARCH_CREDITS_PER_TEN_RESULTS;
  return Math.ceil(resultCount(body) / 10) * perTen;
}

function researchError(
  res: Response,
  status: number,
  error: string,
  details?: unknown,
) {
  return res.status(status).json({
    success: false,
    error,
    ...(details === undefined ? {} : { details }),
  });
}

function isResearchTimeoutError(error: unknown): boolean {
  return (
    (error instanceof DOMException && error.name === "TimeoutError") ||
    (error instanceof Error &&
      (error.name === "ConnectTimeoutError" ||
        (error.cause instanceof Error &&
          error.cause.name === "ConnectTimeoutError")))
  );
}

async function fetchForRequest(
  req: RequestWithAuth<any, any, any>,
  endpoint: ResearchEndpointConfig,
  params: Record<string, any>,
  queryKeys: string[],
  zeroDataRetention: boolean,
) {
  const headers: Record<string, string> = {};
  for (const h of FORWARDED_REQUEST_HEADERS) {
    const v = req.headers[h];
    if (typeof v === "string") headers[h] = v;
  }
  headers["firecrawl-team-id"] = req.auth.team_id;
  headers["firecrawl-zero-data-retention"] = String(zeroDataRetention);
  if (isUsOnly(req.acuc?.flags)) headers["firecrawl-us-only"] = "true";

  if ("fetchUpstream" in endpoint) {
    return endpoint.fetchUpstream(params, headers);
  }

  return fetchResearchUpstream({
    path: endpoint.upstreamPath(params, req),
    params,
    queryKeys,
    headers,
    timeoutMs: endpoint.timeoutMs,
  });
}

function createResearchController(
  schema: z.ZodTypeAny,
  queryKeys: string[],
  endpoint: ResearchEndpointConfig,
  options: { legacy?: boolean } = {},
): ResearchController {
  return async (req, res: Response) => {
    const authedReq = req as RequestWithAuth<any, any, any>;
    const zeroDataRetention =
      getEffectiveSearchForcedKind(authedReq.acuc?.flags, undefined) !== null;

    const started = Date.now();
    const jobId = uuidv7();
    const logger = rootLogger.child({
      module: "api/v2/research",
      method: endpoint.action,
      jobId,
      teamId: authedReq.auth.team_id,
      zeroDataRetention,
    });

    const source = req.method === "POST" ? (req.body ?? {}) : req.query;
    const parsed = schema.safeParse(source);
    if (!parsed.success) {
      logger.warn("Invalid research query", { error: parsed.error.issues });
      // The outer finally below never runs for validation failures, but a
      // malformed keyless request must still leave the canonical keyless/usage
      // trail — ID-enumeration abuse produces plenty of 400s.
      chargeKeylessCredits(authedReq.auth.team_id, 0).catch(() => {});
      return researchError(
        res,
        400,
        "Invalid query parameters",
        parsed.error.issues,
      );
    }

    const params = parsed.data as ResearchQueryParams;
    const targetHint = endpoint.targetHint(params, authedReq);
    await logRequest({
      id: jobId,
      kind: endpoint.kind,
      api_version: "v2",
      external_request_id: externalRequestId(authedReq),
      team_id: authedReq.auth.team_id,
      origin: requestOrigin(params, req),
      integration: resolveIntegration(params.integration, req) ?? null,
      target_hint: targetHint,
      zeroDataRetention,
      api_key_id: authedReq.acuc?.api_key_id ?? null,
    });

    const index: IndexName | null =
      endpoint.kind === "gov_search"
        ? "gov"
        : RESEARCH_INDEX_KINDS.has(endpoint.kind)
          ? "research"
          : null;
    let releaseIndexCall: (() => Promise<void>) | undefined;
    if (index) {
      const indexBudget = await reserveIndexCalls(
        index,
        authedReq.auth.team_id,
        authedReq.acuc?.rate_limit_multiplier ?? 1,
        1,
      );
      if (!indexBudget.ok) {
        chargeKeylessCredits(authedReq.auth.team_id, 0).catch(() => {});
        res.setHeader("Retry-After", String(indexBudget.retryAfterSeconds));
        return res.status(429).json({
          success: false,
          code: "INDEX_DAILY_LIMIT",
          ...indexDailyLimitRefusal(index, authedReq.auth.team_id, indexBudget),
        });
      }
      releaseIndexCall = indexBudget.release;
    }

    let statusCode = 500;
    let responseBody: any = null;
    let error: string | undefined;
    let credits = 0;

    try {
      const upstream = await fetchForRequest(
        authedReq,
        endpoint,
        params,
        queryKeys,
        zeroDataRetention,
      );
      if (!upstream) {
        if (releaseIndexCall) await releaseIndexCall();
        statusCode = 404;
        error = "Research service is not configured";
        return res.status(404).end();
      }

      statusCode = upstream.status;
      for (const h of FORWARDED_RESPONSE_HEADERS) {
        const value = upstream.headers.get(h);
        if (value) res.setHeader(h, value);
      }

      if (isUsOnly(authedReq.acuc?.flags) && upstream.status === 403) {
        const refusal = await usOnlyUpstreamRefusal(
          upstream,
          "This endpoint is not available for teams restricted to US-only processing.",
        );
        error = refusal.message;
        res.setHeader("content-type", "application/json");
        return res.status(403).json({
          success: false,
          code: "US_ONLY_UNAVAILABLE",
          error: refusal.message,
        });
      }

      const text = await upstream.text();
      try {
        responseBody = text ? JSON.parse(text) : null;
      } catch {
        responseBody = text;
      }

      if (upstream.ok) {
        credits = creditsFor(endpoint, responseBody, authedReq);
        if (credits > 0) {
          billTeam(
            authedReq.auth.team_id,
            authedReq.acuc?.org_id ?? null,
            credits,
            authedReq.acuc?.api_key_id ?? null,
            {
              endpoint: endpoint.billAs === "scrape" ? "scrape" : "search",
              jobId,
              chargeId: jobId,
              hostedMcp: authedReq.auth.hostedMcp,
            },
          ).catch(billingError => {
            logger.error("Failed to bill research request", {
              error: billingError,
              credits,
            });
          });
        }
      } else {
        error =
          typeof responseBody === "object" && responseBody !== null
            ? (responseBody.detail ?? responseBody.error ?? responseBody.title)
            : undefined;
      }

      if (
        endpoint.kind === "gov_search" &&
        statusCode === 400 &&
        typeof responseBody?.message === "string"
      ) {
        error = responseBody.message;
        return researchError(res, 400, responseBody.message);
      }

      if (responseBody === null || typeof responseBody === "string") {
        return res.status(statusCode).send(responseBody ?? "");
      }
      const response = !upstream.ok
        ? responseBody
        : options.legacy
          ? addLegacySnakeCaseAliases(responseBody)
          : endpoint.kind === "gov_search"
            ? camelCaseGovResults(responseBody)
            : responseBody;
      const usOnlyWarning =
        upstream.ok && isUsOnly(authedReq.acuc?.flags)
          ? usOnlySkippedWarning(upstream.headers)
          : undefined;
      if (usOnlyWarning && !Array.isArray(response)) {
        response.warning =
          typeof response.warning === "string" && response.warning.length > 0
            ? `${response.warning} ${usOnlyWarning}`
            : usOnlyWarning;
      }
      return res.status(statusCode).json(response);
    } catch (err: unknown) {
      if (isResearchTimeoutError(err)) {
        statusCode = 504;
        error = "Research service timed out";
        return res.status(504).end();
      }
      statusCode = 502;
      error = "Research proxy error";
      logger.error("Research proxy error", { error: err });
      return res.status(502).end();
    } finally {
      const timeTaken = (Date.now() - started) / 1000;

      // No-op for keyed teams. For keyless callers: billable credits are added
      // to the per-IP daily budget and land as a `keyless_credit_usage` row;
      // zero-credit outcomes emit the canonical `keyless/usage` log line
      // instead (the durable zero-credit row waits on the firecrawl-db
      // migration). It runs on every outcome, not just billable successes: the
      // paper endpoints cost 0 credits and ID enumeration mostly produces
      // upstream misses, so gating this on `credits > 0` left those requests
      // with no IP recorded anywhere. `credits` is still 0 on every non-2xx
      // path, so nothing extra is charged.
      chargeKeylessCredits(authedReq.auth.team_id, credits).catch(() => {});

      logResearchEndpoint({
        table: endpoint.table,
        id: jobId,
        request_id: jobId,
        team_id: authedReq.auth.team_id,
        target: targetHint,
        options: params,
        response: responseBody,
        num_results: resultCount(responseBody),
        time_taken: timeTaken,
        credits_cost: statusCode >= 200 && statusCode < 300 ? credits : 0,
        is_successful: statusCode >= 200 && statusCode < 300,
        error,
        zeroDataRetention,
      }).catch(logError => {
        logger.warn("Research endpoint log failed", { error: logError });
      });
    }
  };
}

function paperId(req: Request) {
  return (req.params.id as string[]).filter(Boolean).join("/");
}

export function createResearchRouter(options: { legacy?: boolean } = {}) {
  const router = express.Router();

  if (options.legacy) {
    router.use((req, _res, next) => {
      const authedReq = req as RequestWithAuth<any, any, any>;
      rootLogger.warn("Legacy research endpoint used", {
        module: "api/v2/research",
        teamId: authedReq.auth?.team_id,
        method: req.method,
        path: req.originalUrl,
        requestId: req.headers["x-request-id"],
        zeroDataRetention:
          getEffectiveSearchForcedKind(authedReq.acuc?.flags, undefined) !==
          null,
      });
      next();
    });
  }

  router.get(
    "/papers",
    wrap(
      createResearchController(
        searchPapersSchema,
        ["query", "k", "authors", "categories", "from", "to", "sort"],
        {
          kind: "research_paper_search",
          table: "research_paper_searches",
          action: "searchPapers",
          targetHint: params => String(params.query),
          upstreamPath: () => "/v2/research/papers",
          timeoutMs: PAPER_SEARCH_TIMEOUT_MS,
          billAs: "search",
        },
        options,
      ),
    ),
  );

  router.get(
    "/papers/*id/similar",
    wrap(
      createResearchController(
        similarPapersSchema,
        ["intent", "mode", "k", "rerank", "anchor"],
        {
          kind: "research_related_papers",
          table: "research_related_papers",
          action: "similarPapers",
          targetHint: (params, req) =>
            `${paperId(req)}: ${String(params.intent)}`,
          upstreamPath: (_params, req) =>
            `/v2/research/papers/${encodeURIComponent(paperId(req))}/similar`,
          timeoutMs: SIMILAR_PAPERS_TIMEOUT_MS,
          billAs: "search",
        },
        options,
      ),
    ),
  );

  router.get(
    "/papers/*id",
    wrap(async (req: Request, res: Response) => {
      const authedReq = req as RequestWithAuth<any, any, any>;
      const parsed = paperSchema.safeParse(req.query);
      const isRead = parsed.success && parsed.data.query !== undefined;
      const controller = createResearchController(
        paperSchema,
        ["query", "k"],
        {
          kind: isRead ? "research_paper_read" : "research_paper_inspect",
          table: isRead ? "research_paper_reads" : "research_paper_inspects",
          action: isRead ? "readPaper" : "inspectPaper",
          targetHint: (_params, request) => paperId(request),
          upstreamPath: (_params, request) =>
            `/v2/research/papers/${encodeURIComponent(paperId(request))}`,
          timeoutMs: isRead ? PAPER_READ_TIMEOUT_MS : PAPER_INSPECT_TIMEOUT_MS,
          billAs: "scrape",
        },
        options,
      );
      return controller(authedReq, res);
    }),
  );

  return router;
}

export function createDeveloperRouter(options: { root?: boolean } = {}) {
  const router = express.Router();

  const controller = wrap(
    createResearchController(
      developerSearchSchema,
      DEVELOPER_SEARCH_QUERY_KEYS,
      {
        kind: "code_search",
        table: "code_searches",
        action: "searchDeveloper",
        targetHint: params => String(params.query),
        upstreamPath: () => "/v2/code/search",
        timeoutMs: DEVELOPER_SEARCH_TIMEOUT_MS,
        billAs: "search",
      },
    ),
  );

  const gate = searchCreditGate();
  if (options.root) {
    router.get("/", gate, controller);
    router.post("/", gate, controller);
  } else {
    router.get("/search", gate, controller);
    router.post("/search", gate, controller);
  }

  return router;
}

export function createGovRouter() {
  const router = express.Router();

  const controller = wrap(
    createResearchController(govSearchSchema, [], {
      kind: "gov_search",
      table: "gov_searches",
      action: "searchGov",
      targetHint: params => String(params.query),
      billAs: "search",
      fetchUpstream: (params, headers) =>
        fetchGovUpstream({
          query: params.query,
          k: params.k,
          jurisdiction: params.jurisdiction,
          court: params.court,
          courtType: params.courtType,
          cites: params.cites,
          domain: params.domain,
          sources: params.sources,
          headers,
          timeoutMs: GOV_SEARCH_TIMEOUT_MS,
        }),
    }),
  );

  router.get("/", controller);
  router.post("/", controller);

  return router;
}