Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

25.8 KB
Raw Blame
import { NextFunction, Request, Response } from "express";
// import { crawlStatusController } from "../../src/controllers/v1/crawl-status";
import {
  isAgentExtractModelValid,
  RequestWithAuth,
  RequestWithMaybeAuth,
  RequestWithMaybeACUC,
} from "../controllers/v1/types";
import { RateLimiterMode } from "../types";
import { authenticateUser } from "../controllers/auth";
import { applyAgentAuthDiscoveryHeader } from "../lib/agent-auth-discovery";
import { createIdempotencyKey } from "../services/idempotency/create";
import { validateIdempotencyKey } from "../services/idempotency/validate";
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
import { logger } from "../lib/logger";
import { logAuthDenied } from "../lib/auth-denied-log";
import {
  httpRequestDurationSeconds,
  getRoutePattern,
} from "../lib/http-metrics";
import { UNSUPPORTED_SITE_MESSAGE } from "../lib/strings";
import * as geoip from "geoip-country";
import { isSelfHosted } from "../lib/deployment";
import { validate as isUuid } from "uuid";

import { config } from "../config";
import { CREDIT_CHECK_EXEMPT_TEAM_ID } from "../controllers/v2/agent-credit-gate";
import {
  autumnService,
  CREDITS_FEATURE_ID,
} from "../services/autumn/autumn.service";
import { getTeamBalance } from "../services/autumn/usage";
import {
  checkHostedMcpCredits,
  hostedMcpCreditsRefusal,
} from "../services/autumn/hosted-mcp-gate";
import { getApiKeySpendLimit } from "../lib/api-key-spend-limit";
import {
  ThirdPartyDataTermsRequiredError,
  ThirdPartyDataUnsupportedOptionError,
} from "../lib/exchange";
import { getExchangeAccessForRequestBody } from "../lib/exchange-request";
import { isFreeCatalogueSearch } from "../search/alexandria";
import { wantsGovCategory } from "../search/gov";
import { getScrapeZDR, getThreatProtection } from "../lib/zdr-helpers";
import { getSafeMode, isLockdownZeroDataRetention } from "../lib/safe-mode";
import {
  agentInteropStatus,
  isAgentInteropSecretValid,
} from "../lib/agent-interop";
import { inferIntegration } from "../lib/infer-integration";
import { resolveIntegration } from "../utils/integration";

export function checkCreditsMiddleware(
  _minimum?: number,
  featureId: string = CREDITS_FEATURE_ID,
  // Set on routes that bill a hosted MCP call to MCP credits.
  { mcpCredits = false }: { mcpCredits?: boolean } = {},
): (req: RequestWithAuth, res: Response, next: NextFunction) => void {
  return (req, res, next) => {
    let minimum = _minimum;
    (async () => {
      if (
        config.AGENT_INTEROP_SECRET &&
        req.body &&
        (req.body as any).__agentInterop &&
        (req.body as any).__agentInterop.auth &&
        isAgentInteropSecretValid((req.body as any).__agentInterop.auth) &&
        (req.body as any).__agentInterop.shouldBill === false
      ) {
        return next();
      }

      // Agent-provisioned key enforcement: check sponsor status and 50-credit cap
      if (req.acuc?._agentSponsor) {
        const sponsor = req.acuc._agentSponsor;

        if (sponsor.status === "blocked") {
          logAuthDenied(req, 403, "agent_key_blocked", req.acuc);
          return res.status(403).json({
            success: false,
            error: "This API key has been blocked by the account holder.",
          });
        }

        if (sponsor.status === "pending") {
          const deadline = new Date(sponsor.verification_deadline);
          if (deadline < new Date()) {
            logAuthDenied(req, 403, "agent_key_verification_expired", req.acuc);
            return res.status(403).json({
              success: false,
              error: "sponsor_verification_expired",
              message:
                "Sponsor verification has expired. The account holder needs to log in to confirm.",
              login_url: "https://firecrawl.dev/signin",
            });
          }

          // Enforce 50-credit cap for unverified agent keys. Autumn is the
          // source of truth for credit usage: getTeamBalance().usage is the
          // team's credits used this period. If Autumn is unavailable we fail
          // open (skip the cap), matching the Autumn-outage behavior of the
          // main credit check below.
          const UNVERIFIED_CREDIT_LIMIT = 50;
          let unverifiedCreditsUsed: number | null = null;
          try {
            const balance = await getTeamBalance(req.auth.team_id);
            unverifiedCreditsUsed = balance?.usage ?? 0;
          } catch (balanceError) {
            logger.warn(
              "Failed to fetch Autumn balance for unverified agent-key cap; failing open",
              { error: balanceError, teamId: req.auth.team_id },
            );
          }
          if (
            unverifiedCreditsUsed !== null &&
            unverifiedCreditsUsed >= UNVERIFIED_CREDIT_LIMIT
          ) {
            return res.status(402).json({
              success: false,
              error: "unverified_credit_limit_reached",
              message:
                "This agent key has used its 50 unverified credits. Ask the account holder to confirm the key to unlock full access.",
              credit_limit: UNVERIFIED_CREDIT_LIMIT,
              credits_used: unverifiedCreditsUsed,
              sponsor_status: "pending",
              login_url: "https://firecrawl.dev/signin",
              upgrade_url: "https://firecrawl.dev/pricing",
            });
          }

          // Force index-only mode for all pre-confirmation agent requests
          (req as any).agentIndexOnly = true;
        }
        // If verified, fall through to normal credit check (key is now on real account)
      }

      // Catalogue discovery and Government Index results are free; provider
      // execution reserves its own credits. A gov search still pays to scrape
      // its results, and for threat scans where threat protection or Safe Mode
      // can turn them on.
      const sources = (req.body as any)?.sources;
      const categories = (req.body as any)?.categories;
      const formats = (req.body as any)?.scrapeOptions?.formats;
      const freeGovSearch =
        wantsGovCategory(categories) &&
        !(Array.isArray(formats) && formats.length > 0) &&
        getThreatProtection(req.acuc?.flags) === "disabled" &&
        !getSafeMode(req.acuc?.flags);
      const freeSearch =
        req.path === "/search" &&
        (isFreeCatalogueSearch(sources, categories) || freeGovSearch);
      if (
        (req.path === "/scrape" &&
          (req.body as any)?.alexandria !== undefined) ||
        freeSearch
      ) {
        req.account = { remainingCredits: Infinity };
        return next();
      }

      if (!minimum && req.body) {
        minimum = Number(
          (req.body as any)?.limit ?? (req.body as any)?.urls?.length ?? 1,
        );
        if (isNaN(minimum) || !isFinite(minimum) || minimum <= 0) {
          minimum = undefined;
        }
      }

      // The agent controller gates credits itself, after it has consumed a
      // free request: only a consumed one may skip the check.
      if (req.path.startsWith("/agent")) {
        return next();
      }

      const requestedCredits = minimum ?? 1;

      // No org means no billable identity to gate: keyless and preview teams
      // carry none, and neither does the DB-authentication bypass. `checkCredits`
      // answered null for exactly those, so take its fail-open branch here
      // rather than hand the service an org it would have to go and find.
      const orgId = req.auth.org_id;
      if (!orgId) {
        req.account = { remainingCredits: Infinity };
        return next();
      }

      const check = {
        teamId: req.auth.team_id,
        // The ACUC already carries the org, so the credit check does not have
        // to read `teams.org_id` for it.
        orgId,
        value: requestedCredits,
        properties: {
          source: "checkCreditsMiddleware",
          path: req.path,
          apiKeyId: req.acuc?.api_key_id ?? null,
        },
        featureId,
      };
      // Gated on the balance its charge will be tracked on.
      const gate: Awaited<ReturnType<typeof checkHostedMcpCredits>> =
        mcpCredits && req.auth.hostedMcp
          ? await checkHostedMcpCredits(check)
          : {
              result: await autumnService.checkCredits(check),
              featureId,
              onTeamCredits: false,
            };
      if (gate.onTeamCredits) req.auth.hostedMcp = false;
      const autumnResult = gate.result;
      const gateFeatureId = gate.featureId;

      // Autumn is the source of truth for credits. If it's unavailable
      // (returns null), fail open — matches the behavior in browser.ts /
      // scrape-browser.ts and avoids turning an Autumn outage into a
      // customer outage.
      if (autumnResult === null) {
        req.account = { remainingCredits: Infinity };
        return next();
      }

      const success = autumnResult.allowed;
      const onGateFeature = {
        remaining: autumnResult.remaining,
        featureId: gateFeatureId,
        onTeam: false,
      };
      // A hosted call both balances refuse tries the team clamp first when the
      // team covers more, then MCP credits.
      const clamps =
        gate.teamRefusal && gate.teamRefusal.remaining > autumnResult.remaining
          ? [
              {
                remaining: gate.teamRefusal.remaining,
                featureId,
                onTeam: true,
              },
              onGateFeature,
            ]
          : [onGateFeature];
      // When Autumn allows the request (including overage), don't let a
      // small remaining balance clamp downstream limits (e.g. crawl).
      const remainingCredits = success ? Infinity : clamps[0].remaining;
      req.account = { remainingCredits };
      if (!success) {
        const requestedLimit = Number((req.body as any)?.limit);
        for (const clampOn of clamps) {
          const clampedLimit = Math.min(requestedLimit, clampOn.remaining);
          if (
            !_minimum &&
            req.body &&
            (req.body as any).limit !== undefined &&
            Number.isFinite(clampedLimit) &&
            clampedLimit > 0
          ) {
            // `remaining` is the team's credit balance, and a per-API-key spend
            // limit never lowers it, so a key over its own cap still reports a
            // healthy balance here. Re-check the clamped limit so only a genuinely
            // low team balance gets shrunk to fit; a per-key denial falls through
            // to the 402 below. A null re-check keeps the fail-open behavior.
            const clampedResult = await autumnService.checkCredits({
              teamId: req.auth.team_id,
              orgId,
              value: clampedLimit,
              properties: {
                source: "checkCreditsMiddleware:clamp",
                path: req.path,
                apiKeyId: req.acuc?.api_key_id ?? null,
              },
              featureId: clampOn.featureId,
            });

            if (clampedResult === null || clampedResult.allowed) {
              if (clampOn.onTeam) req.auth.hostedMcp = false;
              logger.warn("Adjusting limit to remaining credits", {
                teamId: req.auth.team_id,
                remainingCredits: clampOn.remaining,
                clampedLimit,
              });
              (req.body as any).limit = clampedLimit;
              return next();
            }
          }
        }

        const currencyName = req.acuc?.is_extract ? "tokens" : "credits";
        logger.error(
          `Insufficient ${currencyName}: ${JSON.stringify({ team_id: req.auth.team_id, minimum, remainingCredits })}`,
          {
            teamId: req.auth.team_id,
            minimum,
            remainingCredits,
            path: req.path,
          },
        );
        // A Gateway account's credits come from the platform that created
        // it, so its user has no plan of their own to upgrade.
        if (
          !res.headersSent &&
          req.auth.team_id !== CREDIT_CHECK_EXEMPT_TEAM_ID &&
          autumnResult.gatewayProvisioned === true
        ) {
          return res.status(402).json({
            success: false,
            code: "platform_managed_credits",
            error: `Insufficient ${currencyName} to perform this request. This account's ${currencyName} are managed by the platform that created it, so contact that platform for more ${currencyName}.`,
          });
        }
        if (
          !res.headersSent &&
          req.auth.team_id !== CREDIT_CHECK_EXEMPT_TEAM_ID &&
          currencyName === "tokens"
        ) {
          return res.status(402).json({
            success: false,
            error:
              "Insufficient tokens to perform this request. For more tokens, you can upgrade your plan at https://www.firecrawl.dev/extract#pricing.",
          });
        }
        if (
          !res.headersSent &&
          req.auth.team_id !== CREDIT_CHECK_EXEMPT_TEAM_ID
        ) {
          // Name the limit that refused the request: the balance alone can't
          // explain a refusal that leaves credits on the dashboard.
          const resetsAt = autumnResult.resetsAt
            ? new Date(autumnResult.resetsAt).toISOString()
            : undefined;
          if (gateFeatureId !== featureId) {
            return res.status(402).json({
              success: false,
              code: "mcp_credits_exhausted",
              requested_credits: requestedCredits,
              ...(resetsAt ? { resets_at: resetsAt } : {}),
              error: hostedMcpCreditsRefusal(autumnResult.resetsAt),
            });
          }

          const apiKeyId = req.acuc?.api_key_id;
          const keyLimit =
            apiKeyId && autumnResult.remaining >= requestedCredits
              ? await getApiKeySpendLimit(apiKeyId)
              : null;
          if (keyLimit) {
            return res.status(402).json({
              success: false,
              code: "api_key_spend_limit",
              requested_credits: requestedCredits,
              remaining_credits: autumnResult.remaining,
              limit: {
                type: "api_key",
                credits: keyLimit.credits,
                interval: keyLimit.interval,
              },
              error:
                `This API key has reached its spend limit of ${keyLimit.credits} credits per ${keyLimit.interval}, while your team still has ${autumnResult.remaining} credits. ` +
                (requestedCredits > keyLimit.credits
                  ? `This request can use up to ${requestedCredits} credits, more than the key's whole limit, so lower the request's limit or raise the key's limit on the API Keys page at https://www.firecrawl.dev/app/api-keys.`
                  : `Raise or remove the key's limit on the API Keys page at https://www.firecrawl.dev/app/api-keys, use another key, or wait for the limit to reset next ${keyLimit.interval}.`),
            });
          }

          return res.status(402).json({
            success: false,
            code: "insufficient_credits",
            requested_credits: requestedCredits,
            remaining_credits: autumnResult.remaining,
            ...(resetsAt ? { resets_at: resetsAt } : {}),
            error: `Insufficient credits to perform this request: it can use up to ${requestedCredits} credits and your team has ${autumnResult.remaining} left${resetsAt ? `, with plan credits resetting at ${resetsAt}` : ""}. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing or try changing the request limit to a lower value.`,
          });
        }
      }
      next();
    })().catch(err => next(err));
  };
}

export function authMiddleware(
  rateLimiterMode: RateLimiterMode,
  options: {
    allowKeyless?: boolean | ((req: RequestWithMaybeAuth) => boolean);
    allowAgentManagedKey?: boolean;
    hostedMcpRateLimit?: boolean;
  } = {},
): (req: RequestWithMaybeAuth, res: Response, next: NextFunction) => void {
  return (req, res, next) => {
    (async () => {
      const allowKeyless =
        typeof options.allowKeyless === "function"
          ? options.allowKeyless(req)
          : options.allowKeyless;

      let currentRateLimiterMode = rateLimiterMode;
      if (
        currentRateLimiterMode === RateLimiterMode.Extract &&
        isAgentExtractModelValid((req.body as any)?.agent?.model)
      ) {
        currentRateLimiterMode = RateLimiterMode.ExtractAgentPreview;
      }

      // if (currentRateLimiterMode === RateLimiterMode.Scrape && isAgentExtractModelValid((req.body as any)?.agent?.model)) {
      //   currentRateLimiterMode = RateLimiterMode.ScrapeAgentPreview;
      // }

      const auth = await authenticateUser(req, res, currentRateLimiterMode, {
        ...options,
        allowKeyless,
      });

      if (!auth.success) {
        if (!res.headersSent) {
          if (auth.status === 401 || auth.agentAuthDiscovery) {
            applyAgentAuthDiscoveryHeader(res);
          }
          if (auth.retryAfterSeconds) {
            res.setHeader("Retry-After", String(auth.retryAfterSeconds));
          }
          return res.status(auth.status).json({
            success: false,
            error: auth.error,
            ...(auth.keylessReason ? { reason: auth.keylessReason } : {}),
            ...(auth.retryAfterSeconds
              ? { retry_after_seconds: auth.retryAfterSeconds }
              : {}),
            ...(auth.signupUrl ? { signup_url: auth.signupUrl } : {}),
          });
        } else {
          return;
        }
      }

      const { team_id, org_id, chunk } = auth;
      const agentInterop = agentInteropStatus(req);
      // An agent run started by a hosted MCP call makes its own calls under
      // the interop secret, and says so in the interop block. Only a trusted
      // block may say it: the run itself was verified when it started.
      const hostedMcp =
        auth.hostedMcp ||
        (agentInterop === "trusted" &&
          (req.body as any)?.__agentInterop?.hostedMcp === true);

      req.auth = {
        team_id,
        org_id,
        agentInterop,
        ...(hostedMcp ? { hostedMcp: true } : {}),
      };
      req.acuc = chunk ?? undefined;
      next();
    })().catch(err => next(err));
  };
}

/**
 * Sets the body `integration` via resolveIntegration, then infers a still
 * missing one. An invalid header is ignored.
 */
export function inferIntegrationMiddleware(
  req: Request,
  _res: Response,
  next: NextFunction,
) {
  const body = req.body;
  if (!body || typeof body !== "object" || Array.isArray(body)) {
    return next();
  }

  const resolved = resolveIntegration(body.integration, req);
  if (resolved !== undefined) {
    body.integration = resolved;
  } else if (config.INFER_INTEGRATION_ENABLED) {
    const inferred = inferIntegration({
      origin: req.get("origin"),
      referer: req.get("referer"),
      userAgent: req.get("user-agent"),
      bodyOrigin: body.origin,
    });
    if (inferred) body.integration = inferred;
  }
  next();
}

export function idempotencyMiddleware(
  req: Request,
  res: Response,
  next: NextFunction,
) {
  (async () => {
    if (req.headers["x-idempotency-key"]) {
      const isIdempotencyValid = await validateIdempotencyKey(req);
      if (!isIdempotencyValid) {
        if (!res.headersSent) {
          return res
            .status(409)
            .json({ success: false, error: "Idempotency key already used" });
        }
      }
      createIdempotencyKey(req);
    }
    next();
  })().catch(err => next(err));
}
export function blocklistMiddleware(
  req: RequestWithMaybeACUC<any, any, any>,
  res: Response,
  next: NextFunction,
) {
  return blocklistGate(req, res, next, { exchange: false });
}

/**
 * Blocklist gate for single-URL scrape-shaped routes (scrape, crawl), where a
 * blocklisted URL still passes when the exchange engine can serve it, asks
 * for the provider's terms when unaccepted terms are all that stands in the
 * way, and names the request option that keeps a provider out. Unblocked
 * URLs never consult the Exchange here: engine selection decides for them,
 * identically on every route, and scrapes them normally when terms are
 * missing. v2 batch scrape runs the same check per URL in its controller;
 * map, search and monitors keep plain blocklist behavior.
 */
export function scrapeBlocklistMiddleware(
  req: RequestWithMaybeACUC<any, any, any>,
  res: Response,
  next: NextFunction,
) {
  return blocklistGate(req, res, next, { exchange: true });
}

function blocklistGate(
  req: RequestWithMaybeACUC<any, any, any>,
  res: Response,
  next: NextFunction,
  options: { exchange: boolean },
) {
  (async () => {
    if (
      typeof req.body.url !== "string" ||
      !isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {
        team_id: req.acuc?.team_id ?? null,
        org_id: req.acuc?.org_id ?? null,
        origin: typeof req.body.origin === "string" ? req.body.origin : null,
      })
    ) {
      return next();
    }

    if (options.exchange) {
      // Safe Mode lockdown implies ZDR, which keeps the Exchange out.
      const zeroDataRetention =
        getScrapeZDR(req.acuc?.flags) === "forced" ||
        req.body?.zeroDataRetention === true ||
        isLockdownZeroDataRetention(req.acuc?.flags, req.body?.safeMode);
      const exchangeAccess = await getExchangeAccessForRequestBody({
        body: req.body,
        flags: req.acuc?.flags ?? null,
        url: req.body.url,
        blocked: true,
        zeroDataRetention,
        teamId: req.acuc?.team_id ?? null,
        orgId: req.acuc?.org_id ?? null,
      });

      if (exchangeAccess.allowed) {
        return next();
      }

      if (exchangeAccess.termsRequired && !res.headersSent) {
        return res
          .status(403)
          .json(
            new ThirdPartyDataTermsRequiredError(
              exchangeAccess.terms,
            ).response(),
          );
      }

      if (
        !exchangeAccess.termsRequired &&
        exchangeAccess.unsupportedOption !== undefined &&
        !res.headersSent
      ) {
        return res
          .status(400)
          .json(
            new ThirdPartyDataUnsupportedOptionError(
              exchangeAccess.unsupportedOption,
            ).response(),
          );
      }
    }

    if (!res.headersSent) {
      return res.status(403).json({
        success: false,
        error: UNSUPPORTED_SITE_MESSAGE,
      });
    }
    next();
  })().catch(err => next(err));
}

export function countryCheck(
  req: RequestWithAuth<any, any, any>,
  res: Response,
  next: NextFunction,
) {
  if (req.acuc?.flags?.skipCountryCheck) {
    return next();
  }

  if (!req.ip) {
    logger.warn("IP address not found, unable to check country");
    return next();
  }

  const country = geoip.lookup(req.ip);
  if (!country || !country.country) {
    logger.warn("IP address country data not found", { ip: req.ip });
    return next();
  }

  if (config.RESTRICTED_COUNTRIES?.includes(country.country)) {
    logger.warn("Denied access to restricted country", {
      ip: req.ip,
      country: country.country,
      teamId: req.auth.team_id,
    });
    return res.status(403).json({
      success: false,
      error: isSelfHosted()
        ? "Use of headers, actions, the agent, and browser/interact sessions is not allowed by default in your country. Please check your server configuration."
        : "Use of headers, actions, the agent, and browser/interact sessions is not allowed by default in your country. Please contact us at [email protected]",
    });
  }

  next();
}

export function restrictedOptionsCountryCheck(
  req: RequestWithAuth<any, any, any>,
  res: Response,
  next: NextFunction,
) {
  const usesRestrictedOptions =
    req.body &&
    (req.body.actions ||
      (req.body.headers &&
        typeof req.body.headers === "object" &&
        Object.keys(req.body.headers).length > 0) ||
      req.body.agent ||
      req.body.jsonOptions?.agent ||
      req.body.extract?.agent ||
      req.body.scrapeOptions?.actions ||
      (req.body.scrapeOptions?.headers &&
        typeof req.body.scrapeOptions.headers === "object" &&
        Object.keys(req.body.scrapeOptions.headers).length > 0) ||
      req.body.scrapeOptions?.agent ||
      req.body.scrapeOptions?.jsonOptions?.agent ||
      req.body.scrapeOptions?.extract?.agent);

  if (!usesRestrictedOptions) {
    return next();
  }

  countryCheck(req, res, next);
}

export function isValidJobId(jobId: string | undefined): jobId is string {
  return typeof jobId === "string" && isUuid(jobId);
}

export function validateJobIdParam(
  req: Request<{ jobId?: string }>,
  res: Response,
  next: NextFunction,
) {
  if (!isValidJobId(req.params.jobId)) {
    return res.status(400).json({
      success: false,
      error: "Invalid job ID format. Job ID must be a valid UUID.",
    });
  }

  next();
}

export function requestTimingMiddleware(version: string) {
  return (req: Request, res: Response, next: NextFunction) => {
    const startTime = new Date().getTime();

    // Attach timing data to request
    (req as any).requestTiming = {
      startTime,
      version,
    };

    // Override res.json to log timing when response is sent
    const originalJson = res.json.bind(res);
    res.json = function (body: any) {
      const requestTime = new Date().getTime() - startTime;

      const durationSeconds = requestTime / 1000;
      const route = getRoutePattern(req);
      const status = String(res.statusCode);

      httpRequestDurationSeconds
        .labels(version, req.method, route, status)
        .observe(durationSeconds);

      // Only log for successful responses to avoid duplicate error logs
      if (body?.success !== false) {
        logger.info(`${version} request completed`, {
          version,
          path: req.path,
          method: req.method,
          startTime,
          requestTime,
          statusCode: res.statusCode,
        });
      }

      return originalJson(body);
    };

    next();
  };
}

export function wrap(
  controller: (req: Request, res: Response) => Promise<any>,
): (req: Request, res: Response, next: NextFunction) => any {
  return (req, res, next) => {
    controller(req, res).catch(err => next(err));
  };
}