Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

42.3 KB
Raw Blame
import { RateLimiterRedis } from "rate-limiter-flexible";
import { isValidUuid } from "../lib/owner-id";
import { config } from "../config";
import { logger } from "../lib/logger";
import { timingSafeSecretEqual } from "../lib/secret-compare";
import { parseApi } from "../lib/parseApi";
import { withAuth } from "../lib/withAuth";
import { getAgentSponsorStatus } from "../services/agent-sponsor";
import {
  getRateLimiter,
  getAutumnRateLimiter,
  getHostedMcpConcurrencyLimit,
  getHostedMcpRateLimiter,
  getRateLimitOverride,
  HOBBY_RATE_LIMIT_MULTIPLIER,
} from "../services/rate-limiter";
import { isTrustedAgentInteropRequest } from "../lib/agent-interop";
import {
  consumeKeylessRequest,
  isKeylessConfigured,
  keylessExhaustionTelemetry,
  isKeylessIpEligible,
  keylessLimitPrompt,
  keylessSignupUrlForIp,
  keylessTeamId,
  keylessWorldIdIdentity,
  normalizeKeylessIpv4,
  reportKeylessPromptShown,
} from "../lib/keyless";
import { keylessSignupSurface } from "../lib/keyless-signup-link";
import { consumeIndexRateLimit } from "../lib/index-daily-limit";
import { isKeylessIpSuspicious } from "../lib/spur";
import { verifyWorldIdCredential } from "../lib/world-id";
import { keylessAuthTotal } from "../lib/keyless-metrics";
import { checkIpRestriction } from "../lib/ip-restriction";
import { checkKeyEndpointRestriction } from "../lib/key-restriction";
import { type AuthDenialReason, logAuthDenied } from "../lib/auth-denied-log";
import { deleteKey, getValue, setValue } from "../services/redis";
import { redlock } from "../services/redlock";
import { db, dbRr } from "../db/connection";
import {
  authCreditUsageChunk,
  authCreditUsageChunkFromTeam,
  AuthCreditUsageChunkRow,
} from "../db/rpc";
import { AuthResponse, RateLimiterMode } from "../types";
import {
  AuthCreditUsageChunk,
  AuthCreditUsageChunkFromTeam,
  TeamFlags,
} from "./v1/types";
import {
  FIRECRAWL_REST_RESOURCE,
  OAuthIntrospectionUnavailableError,
  resolveOAuthToken as resolveOAuthTokenWithIntrospection,
} from "../services/oauth-token-introspection";
import type { OAuthIntrospectionResponse } from "../services/oauth-token-introspection";
import { verifyMcpDelegatedCredential } from "../lib/mcp-delegated-credential";
import {
  looksLikeJwt,
  VercelJwksUnavailableError,
} from "../lib/vercel-marketplace-oidc";
import {
  resolveVercelOidcToken,
  type VercelOidcResolution,
} from "../services/vercel-marketplace-oidc";
import {
  autumnService,
  DEFAULT_TEAM_LIMITS,
  type TeamLimits,
} from "../services/autumn/autumn.service";
import { ReplyError } from "ioredis";
import { resolveIntegration } from "../utils/integration";

const ACUC_TTL_SECONDS = 600;
// A chunk built on the fail-open limits is cached only briefly, so a
// transient Autumn error is retried within a minute.
const ACUC_FAIL_OPEN_TTL_SECONDS = 60;
// When Autumn can't answer, fail open rather than throttle or gate real teams
// on the low defaults. Generous but bounded: the concurrency queue cap still
// applies.
const FAIL_OPEN_LIMITS: TeamLimits = {
  concurrency_limit: 200,
  rate_limit_multiplier: 2500,
  is_paid_plan: true,
};

/** The team's effective limits, and how long a chunk carrying them may live. */
async function readTeamLimits(
  teamId: string,
  orgId: string | null,
): Promise<{ limits: TeamLimits; ttl: number }> {
  try {
    return {
      limits: await autumnService.getTeamLimits(teamId, orgId),
      ttl: ACUC_TTL_SECONDS,
    };
  } catch (error) {
    logger.error("Autumn limits unavailable, failing open", { teamId, error });
    return { limits: FAIL_OPEN_LIMITS, ttl: ACUC_FAIL_OPEN_TTL_SECONDS };
  }
}

/** A cached chunk, with live limits when it predates them (not written back). */
async function withLimits<T extends AuthCreditUsageChunkFromTeam>(
  chunk: T,
): Promise<T> {
  if (
    chunk.concurrency_limit !== undefined &&
    chunk.rate_limit_multiplier !== undefined &&
    chunk.is_paid_plan !== undefined
  ) {
    return chunk;
  }
  return {
    ...chunk,
    ...(await readTeamLimits(chunk.team_id, chunk.org_id)).limits,
  };
}

function normalizedApiIsUuid(potentialUuid: string): boolean {
  // Check if the string is a valid UUID
  return isValidUuid(potentialUuid);
}

async function setCachedACUC(
  api_key: string,
  is_extract: boolean,
  acuc:
    | AuthCreditUsageChunk
    | null
    | ((acuc: AuthCreditUsageChunk) => AuthCreditUsageChunk | null),
  credentialPurpose: "general" | "hosted_mcp_oauth" = "general",
  ttl = ACUC_TTL_SECONDS,
) {
  const cacheKeyACUC = `acuc_${credentialPurpose}_${api_key}_${is_extract ? "extract" : "scrape"}`;
  const redLockKey = `lock_${cacheKeyACUC}`;

  try {
    await redlock.using([redLockKey], 10000, {}, async signal => {
      if (typeof acuc === "function") {
        acuc = acuc(JSON.parse((await getValue(cacheKeyACUC)) ?? "null"));

        if (acuc === null) {
          if (signal.aborted) {
            throw signal.error;
          }

          return;
        }
      }

      if (signal.aborted) {
        throw signal.error;
      }

      await setValue(cacheKeyACUC, JSON.stringify(acuc), ttl, true);
    });
  } catch (error) {
    logger.error("Error updating cached ACUC", {
      error,
    });
  }
}

const mockPreviewACUC: (
  team_id: string,
  is_extract: boolean,
) => AuthCreditUsageChunk = (team_id, is_extract) => ({
  api_key: "preview",
  api_key_id: 0,
  team_id,
  org_id: "preview",
  flags: null,
  is_banned: false,
  is_extract,
  ...DEFAULT_TEAM_LIMITS,
});

const mockACUC: () => AuthCreditUsageChunk = () => ({
  api_key: "bypass",
  api_key_id: 0,
  team_id: "bypass",
  org_id: "bypass",
  flags: null,
  is_banned: false,
  is_extract: false,
  ...DEFAULT_TEAM_LIMITS,
});

/**
 * Resolve an OAuth access token (fco_…) to the underlying API key via
 * the introspection endpoint. Results are cached in Redis for the
 * remaining token TTL (up to 5 minutes).
 */
async function resolveOAuthToken(
  token: string,
): Promise<OAuthIntrospectionResponse | null> {
  const introspectUrl = config.OAUTH_INTROSPECT_URL;
  const introspectSecret = config.OAUTH_INTROSPECT_SECRET;

  if (!introspectUrl || !introspectSecret) {
    logger.warn(
      "OAuth introspection not configured (OAUTH_INTROSPECT_URL / OAUTH_INTROSPECT_SECRET)",
    );
    throw new OAuthIntrospectionUnavailableError(
      "OAuth introspection is not configured",
    );
  }

  return resolveOAuthTokenWithIntrospection(token, {
    introspectUrl,
    introspectSecret,
    expectedResource: FIRECRAWL_REST_RESOURCE,
  });
}

async function getACUC(
  api_key: string,
  cacheOnly = false,
  useCache = true,
  mode?: RateLimiterMode,
  credentialPurpose: "general" | "hosted_mcp_oauth" = "general",
): Promise<AuthCreditUsageChunk | null> {
  // Managed OAuth credentials are connection-scoped and can be created or
  // revoked immediately before this request. Never serve them from Redis or a
  // lagging read replica: both authorization and revocation must observe the
  // primary database.
  const requiresPrimaryRead = credentialPurpose === "hosted_mcp_oauth";
  if (requiresPrimaryRead) useCache = false;

  let isExtract =
    mode === RateLimiterMode.Extract ||
    mode === RateLimiterMode.ExtractStatus ||
    mode === RateLimiterMode.ExtractAgentPreview;

  if (timingSafeSecretEqual(api_key, config.PREVIEW_TOKEN)) {
    const acuc = mockPreviewACUC(api_key, isExtract);
    acuc.is_extract = isExtract;
    return acuc;
  }

  if (config.USE_DB_AUTHENTICATION !== true) {
    const acuc = mockACUC();
    acuc.is_extract = isExtract;
    return acuc;
  }

  const cacheKeyACUC = `acuc_${credentialPurpose}_${api_key}_${isExtract ? "extract" : "scrape"}`;

  if (useCache) {
    let cachedACUC: string | null;
    try {
      cachedACUC = await getValue(cacheKeyACUC);
    } catch (error) {
      if (
        error instanceof ReplyError ||
        (error instanceof Error &&
          (error as any).address &&
          (error as any).code &&
          error.name === "Error") ||
        (error instanceof Error && error.name === "MaxRetriesPerRequestError")
      ) {
        logger.warn(
          "Reading ACUC out of cache redis failed, treating as miss",
          {
            error,
          },
        );
        cachedACUC = null;
      } else {
        throw error;
      }
    }
    if (cachedACUC !== null) {
      try {
        return await withLimits(JSON.parse(cachedACUC));
      } catch (error) {
        logger.warn("Ignoring malformed ACUC cache entry", {
          error,
        });
        void deleteKey(cacheKeyACUC).catch(deleteError => {
          logger.warn("Failed to delete malformed ACUC cache entry", {
            error: deleteError,
          });
        });
      }
    }
  }

  if (!cacheOnly) {
    let data: AuthCreditUsageChunkRow[] = [];
    let retries = 0;
    const maxRetries = 5;
    while (retries < maxRetries) {
      // General-purpose reads prefer the replica: the result is Redis-cached
      // for 10 minutes, so sub-second replication lag is irrelevant. Fall back
      // to the primary on replica error. hosted_mcp_oauth must stay on the
      // primary — revocation has to observe fresh state.
      const database = requiresPrimaryRead ? db : retries === 0 ? dbRr : db;
      try {
        data = await authCreditUsageChunk(database, api_key, credentialPurpose);
        break;
      } catch (error) {
        logger.warn(
          `Failed to retrieve authentication and credit usage data after ${retries}, trying again...`,
          { error },
        );
        retries++;
        if (retries === maxRetries) {
          throw new Error(
            "Failed to retrieve authentication and credit usage data after 3 attempts: " +
              JSON.stringify(error),
          );
        }

        // Wait for a short time before retrying
        await new Promise(resolve => setTimeout(resolve, 200));
      }
    }

    const chunk: AuthCreditUsageChunk | null =
      data.length === 0
        ? null
        : data[0].team_id === null
          ? null
          : (data[0] as any);

    if (chunk === null) return null;

    // NOTE: Should we cache null chunks? - mogery
    if (useCache) {
      const { limits, ttl } = await readTeamLimits(chunk.team_id, chunk.org_id);
      const built = { ...chunk, is_extract: isExtract, ...limits };
      setCachedACUC(api_key, isExtract, built, credentialPurpose, ttl);
      return built;
    }

    // An uncached chunk (a hosted MCP credential) takes the team ACUC's
    // limits, so authenticating it per request never costs an Autumn read.
    const team = await getACUCTeam(chunk.team_id).catch(() => null);
    const limits =
      team ?? (await readTeamLimits(chunk.team_id, chunk.org_id)).limits;
    return {
      ...chunk,
      is_extract: isExtract,
      concurrency_limit: limits.concurrency_limit,
      rate_limit_multiplier: limits.rate_limit_multiplier,
      is_paid_plan: limits.is_paid_plan,
    };
  } else {
    return null;
  }
}

async function setCachedACUCTeam(
  team_id: string,
  is_extract: boolean,
  acuc:
    | AuthCreditUsageChunkFromTeam
    | null
    | ((
        acuc: AuthCreditUsageChunkFromTeam,
      ) => AuthCreditUsageChunkFromTeam | null),
  ttl = ACUC_TTL_SECONDS,
) {
  const cacheKeyACUC = `acuc_team_${team_id}_${is_extract ? "extract" : "scrape"}`;
  const redLockKey = `lock_${cacheKeyACUC}`;

  try {
    await redlock.using([redLockKey], 10000, {}, async signal => {
      if (typeof acuc === "function") {
        acuc = acuc(JSON.parse((await getValue(cacheKeyACUC)) ?? "null"));

        if (acuc === null) {
          if (signal.aborted) {
            throw signal.error;
          }

          return;
        }
      }

      if (signal.aborted) {
        throw signal.error;
      }

      await setValue(cacheKeyACUC, JSON.stringify(acuc), ttl, true);
    });
  } catch (error) {
    logger.error("Error updating cached ACUC", {
      cacheKey: cacheKeyACUC,
      error,
    });
  }
}

export async function getACUCTeam(
  team_id: string,
  cacheOnly = false,
  useCache = true,
  mode?: RateLimiterMode,
): Promise<AuthCreditUsageChunkFromTeam | null> {
  let isExtract =
    mode === RateLimiterMode.Extract ||
    mode === RateLimiterMode.ExtractStatus ||
    mode === RateLimiterMode.ExtractAgentPreview;

  if (team_id.startsWith("preview")) {
    const acuc = mockPreviewACUC(team_id, isExtract);
    return acuc;
  }

  if (config.USE_DB_AUTHENTICATION !== true) {
    const acuc = mockACUC();
    acuc.is_extract = isExtract;
    return acuc;
  }

  const cacheKeyACUC = `acuc_team_${team_id}_${isExtract ? "extract" : "scrape"}`;

  if (useCache) {
    let cachedACUC: string | null;
    try {
      cachedACUC = await getValue(cacheKeyACUC);
    } catch (error) {
      if (
        error instanceof ReplyError ||
        (error instanceof Error &&
          (error as any).address &&
          (error as any).code &&
          error.name === "Error") ||
        (error instanceof Error && error.name === "MaxRetriesPerRequestError")
      ) {
        logger.warn(
          "Reading ACUC out of cache redis failed, treating as miss",
          {
            cacheKey: cacheKeyACUC,
            error,
          },
        );
        cachedACUC = null;
      } else {
        throw error;
      }
    }
    if (cachedACUC !== null) {
      // A corrupt entry is a miss, not a failure: callers that fall back to a
      // null org on a throw would otherwise take the high fail-open limits.
      try {
        return await withLimits(JSON.parse(cachedACUC));
      } catch (error) {
        logger.warn("Ignoring malformed ACUC cache entry", {
          cacheKey: cacheKeyACUC,
          error,
        });
        void deleteKey(cacheKeyACUC).catch(deleteError => {
          logger.warn("Failed to delete malformed ACUC cache entry", {
            cacheKey: cacheKeyACUC,
            error: deleteError,
          });
        });
      }
    }
  }

  if (!cacheOnly) {
    let data: AuthCreditUsageChunkRow[] = [];
    let retries = 0;
    const maxRetries = 5;

    while (retries < maxRetries) {
      // Prefer the replica (10-minute Redis cache makes lag irrelevant); fall
      // back to the primary on replica error.
      const database = retries === 0 ? dbRr : db;
      try {
        data = await authCreditUsageChunkFromTeam(database, team_id);
        break;
      } catch (error) {
        logger.warn(
          `Failed to retrieve authentication and credit usage data after ${retries}, trying again...`,
          { error },
        );
        retries++;
        if (retries === maxRetries) {
          throw new Error(
            "Failed to retrieve authentication and credit usage data after 3 attempts: " +
              JSON.stringify(error),
          );
        }

        // Wait for a short time before retrying
        await new Promise(resolve => setTimeout(resolve, 200));
      }
    }

    const chunk: AuthCreditUsageChunk | null =
      data.length === 0
        ? null
        : data[0].team_id === null
          ? null
          : (data[0] as any);

    if (chunk === null) return null;

    const { limits, ttl } = await readTeamLimits(chunk.team_id, chunk.org_id);
    const built = { ...chunk, ...limits };

    // NOTE: Should we cache null chunks? - mogery
    if (useCache) {
      setCachedACUCTeam(team_id, isExtract, built, ttl);
    }

    return { ...built, is_extract: isExtract };
  } else {
    return null;
  }
}

/**
 * The flags of the team behind a request's plain API key, read without rate
 * limiting or the other auth checks. Null without a known API key.
 */
export async function getApiKeyTeamFlags(req: {
  headers: { authorization?: string };
}): Promise<TeamFlags | null> {
  const token = req.headers.authorization?.split(" ")[1];
  if (!token) return null;
  const normalizedApi = parseApi(token);
  if (!normalizedApiIsUuid(normalizedApi)) return null;
  const chunk = await getACUC(normalizedApi);
  return chunk?.flags ?? null;
}

export async function clearACUC(api_key: string): Promise<void> {
  // Clear both purpose-qualified keys and the legacy unqualified keys during
  // rollout so credential-purpose isolation cannot leave stale auth entries.
  const modes = [true, false];
  const credentialPurposes = ["general", "hosted_mcp_oauth"] as const;
  await Promise.all(
    modes.flatMap(mode => [
      deleteKey(`acuc_${api_key}_${mode ? "extract" : "scrape"}`),
      ...credentialPurposes.map(credentialPurpose =>
        deleteKey(
          `acuc_${credentialPurpose}_${api_key}_${mode ? "extract" : "scrape"}`,
        ),
      ),
    ]),
  );

  // Also clear the base cache key
  await deleteKey(`acuc_${api_key}`);
}

export async function clearACUCTeam(team_id: string): Promise<void> {
  // Delete cache for all rate limiter modes
  const modes = [true, false];
  await Promise.all(
    modes.map(async mode => {
      const cacheKey = `acuc_team_${team_id}_${mode ? "extract" : "scrape"}`;
      await deleteKey(cacheKey);
    }),
  );

  // Also clear the base cache key
  await deleteKey(`acuc_team_${team_id}`);
}

function denied(
  req,
  reason: AuthDenialReason,
  response: Extract<AuthResponse, { success: false }>,
  key?: AuthCreditUsageChunk,
): AuthResponse {
  logAuthDenied(req, response.status, reason, key);
  return response;
}

// Both prompts end the sentence after the URL with a space, so a copied link
// never picks up the period.
function keylessEndpointNotAvailableMessage(signupUrl: string): string {
  return `This endpoint is not supported by the keyless free tier. Sign up for a free API key at ${signupUrl} for more endpoints, more usage, and higher rate limits.

Then authenticate with:
Authorization: Bearer YOUR_API_KEY`;
}

function keylessSuspiciousIpMessage(signupUrl: string): string {
  return `Unfortunately, your IP address looks like a VPN or proxy, so Firecrawl can't be used without an API key from here. Sign up with Google or GitHub at ${signupUrl} for a free API key with 1000 credits and higher rate limits. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
}

/**
 * The real client IP for keyless. A trusted proxy (e.g. the hosted MCP) may
 * forward the end-user's IP via x-firecrawl-keyless-ip, authenticated with a
 * shared secret — without the secret the header is ignored, so direct callers
 * can't spoof their IP to dodge the per-IP cap.
 */
export function keylessClientIp(req): string {
  let ip = req.ip ?? req.socket?.remoteAddress ?? "unknown";
  if (
    timingSafeSecretEqual(
      req.headers["x-firecrawl-keyless-secret"],
      config.KEYLESS_PROXY_SECRET,
    )
  ) {
    const forwarded = req.headers["x-firecrawl-keyless-ip"];
    if (typeof forwarded === "string" && forwarded.trim()) {
      ip = forwarded.trim();
    }
  }
  return ip;
}

/**
 * Keyless free tier: official MCP/CLI/SDK clients can call scrape, search, and
 * interact with no API key. `origin`/`integration` are client-set and spoofable,
 * so they're only a soft UX gate — the real abuse controls are the per-IP daily
 * request + credit caps plus the `keyless/consume` canonical log emitted here.
 * Always returns an AuthResponse — handles every no-API-key request.
 */
async function handleKeylessAuth(
  req,
  mode: RateLimiterMode,
  allowKeyless: boolean | undefined,
): Promise<AuthResponse> {
  const unauthorized: AuthResponse = {
    success: false,
    error: "Unauthorized",
    status: 401,
  };

  // The keyless tier is off unless BOTH limits are configured (even to 0). When
  // unconfigured we behave exactly as before — a generic 401 — and don't reveal
  // that the tier exists.
  if (!isKeylessConfigured()) {
    return denied(req, "missing_credentials", unauthorized);
  }

  // Configured, but this endpoint isn't part of the keyless tier: tell the user
  // they need a key (with the signup nudge) rather than a bare "Unauthorized".
  if (!allowKeyless) {
    const ip = keylessClientIp(req);
    const surface = keylessSignupSurface(req);
    const { url, signupRef } = keylessSignupUrlForIp(
      ip,
      surface,
      "unsupported_endpoint",
    );
    reportKeylessPromptShown(
      ip,
      surface,
      "unsupported_endpoint",
      401,
      signupRef,
    );
    return denied(req, "missing_credentials", {
      success: false,
      error: keylessEndpointNotAvailableMessage(url),
      status: 401,
      signupUrl: url,
    });
  }

  const origin = req.body?.origin;
  const integration = resolveIntegration(req.body?.integration, req);
  // No origin/surface gate: any request without an API key may use the free
  // tier on the allowlisted endpoints (the API itself is free). origin and
  // integration are still recorded below for abuse monitoring, and pick the
  // surface of the signup link.
  const signupSurface = keylessSignupSurface(req);

  let ip = keylessClientIp(req);

  // A World ID verified human (a valid `fcwid_` credential) gets their own
  // bucket keyed on their World ID instead of the IP, so the IP checks below
  // don't apply. A missing or invalid credential falls back to the IP bucket.
  const worldIdSubject = verifyWorldIdCredential(
    req.headers["x-firecrawl-world-id"],
  );
  const worldIdIdentity = worldIdSubject
    ? keylessWorldIdIdentity(worldIdSubject)
    : null;

  // Only a valid IPv4 identity gets keyless: IPv6 is too cheap to rotate for a
  // per-IP cap to mean anything, and malformed/forwarded values must not be
  // usable as arbitrary limiter buckets. Anything else falls through to 401.
  if (!worldIdIdentity && !isKeylessIpEligible(ip)) {
    return denied(req, "keyless_ip_ineligible", unauthorized);
  }

  // Canonicalize `::ffff:`-mapped IPv4 so a client gets one Spur cache entry,
  // one quota bucket, and one team id regardless of how the socket reported it.
  ip = normalizeKeylessIpv4(ip);

  // Optional Spur Context check (only when SPUR_API_KEY is set): refuse keyless
  // for IPs fronting anonymizing/rotating infrastructure (VPN/proxy/TOR), the
  // main way the per-IP caps get bypassed. Fails open on any Spur error, and
  // runs before consuming quota so a flagged IP doesn't burn a request slot.
  if (!worldIdIdentity && (await isKeylessIpSuspicious(ip))) {
    keylessAuthTotal.inc({ mode, outcome: "suspicious" });
    const { url, signupRef } = keylessSignupUrlForIp(
      ip,
      signupSurface,
      "suspicious_ip",
    );
    reportKeylessPromptShown(
      ip,
      signupSurface,
      "suspicious_ip",
      403,
      signupRef,
    );
    logger.warn("Keyless request blocked: suspicious IP", {
      canonicalLog: "keyless/consume",
      ip,
      origin: req.body?.origin,
      integration,
      blocked: true,
      reason: "suspicious",
      ...(signupRef ? { signupRef } : {}),
    });
    return denied(req, "keyless_ip_suspicious", {
      success: false,
      error: keylessSuspiciousIpMessage(url),
      status: 403,
      // Tell agents where to find the key/signup flow they now need.
      agentAuthDiscovery: true,
      signupUrl: url,
    });
  }

  // The quota bucket and team id. Logs keep the client IP either way.
  const identity = worldIdIdentity ?? ip;
  const teamId = keylessTeamId(identity);
  const modeLabel =
    mode === RateLimiterMode.Search
      ? "search"
      : mode === RateLimiterMode.Research
        ? "research"
        : mode === RateLimiterMode.DeveloperSearch
          ? "developer"
          : mode === RateLimiterMode.GovSearch
            ? "gov"
            : mode === RateLimiterMode.BrowserExecute
              ? "interact"
              : "scrape";

  let result: Awaited<ReturnType<typeof consumeKeylessRequest>>;
  try {
    result = await consumeKeylessRequest(identity);
  } catch (error) {
    keylessAuthTotal.inc({ mode, outcome: "error" });
    // Limiter store (Redis) unavailable — fail closed with a controlled auth
    // response instead of surfacing a 500, and shed the free traffic while the
    // limiter can't enforce quotas.
    logger.warn("Keyless quota check failed", {
      canonicalLog: "keyless/consume",
      ip,
      mode: modeLabel,
      teamId,
      error,
    });
    return denied(req, "keyless_limiter_unavailable", unauthorized);
  }
  const baseLog = {
    canonicalLog: "keyless/consume",
    ip,
    origin,
    integration,
    mode: modeLabel,
    teamId,
    worldId: worldIdIdentity !== null,
    requestsUsed: result.requestsUsed,
    creditsUsed: result.creditsUsed,
  };

  if (!result.ok) {
    keylessAuthTotal.inc({ mode, outcome: result.reason ?? "error" });
    const prompt = keylessLimitPrompt(
      worldIdIdentity ? null : ip,
      signupSurface,
    );
    logger.warn("Keyless request blocked", {
      ...baseLog,
      blocked: true,
      event: "keyless_exhausted",
      reason: result.reason,
      retryAfterSeconds: result.retryAfterSeconds,
      ...(prompt.signupRef ? { signupRef: prompt.signupRef } : {}),
      ...keylessExhaustionTelemetry(identity),
    });
    return {
      success: false,
      error: prompt.error,
      status: 429,
      signupUrl: prompt.signup_url,
      // Direct API callers receive discovery metadata; MCP maps this in-band.
      agentAuthDiscovery: true,
      keylessReason: result.reason,
      retryAfterSeconds: result.retryAfterSeconds,
    };
  }

  if (modeLabel === "gov" || modeLabel === "research") {
    const perMinute = await consumeIndexRateLimit(
      modeLabel,
      teamId,
      1,
      undefined,
      1,
    );
    if (!perMinute.ok) {
      keylessAuthTotal.inc({ mode, outcome: "index_rate_limit" });
      logger.warn("Keyless request blocked", {
        ...baseLog,
        blocked: true,
        reason: "index_rate_limit",
        retryAfterSeconds: perMinute.retryAfterSeconds,
      });
      return {
        success: false,
        error: perMinute.error,
        status: 429,
        retryAfterSeconds: perMinute.retryAfterSeconds,
      };
    }
  }

  logger.debug("Keyless request consumed", { ...baseLog, blocked: false });
  keylessAuthTotal.inc({ mode, outcome: "allowed" });

  // Tag as a preview team so billing (autumn isPreviewTeam) and GCS persistence
  // are skipped automatically; mockPreviewACUC supplies concurrency 2 + credits.
  // Actual credits consumed are charged to the IP's daily budget after the
  // request completes (see chargeKeylessCredits).
  return {
    success: true,
    team_id: teamId,
    org_id: null,
    chunk: mockPreviewACUC(teamId, false),
  };
}

export async function authenticateUser(
  req,
  res,
  mode: RateLimiterMode,
  options?: AuthenticateOptions,
): Promise<AuthResponse> {
  const bypassChunk = mockACUC();
  bypassChunk.is_extract =
    mode === RateLimiterMode.Extract ||
    mode === RateLimiterMode.ExtractStatus ||
    mode === RateLimiterMode.ExtractAgentPreview;

  return withAuth(supaAuthenticateUser, {
    success: true,
    chunk: bypassChunk,
    team_id: bypassChunk.team_id,
    org_id: null,
  })(req, res, mode, options);
}

/**
 * Builds the rate limiter for an authenticated team from its Autumn rate-limit
 * multiplier. Shared by the OAuth and API-key paths so their limiter setup
 * can't diverge.
 *
 * The org flags carry the optional per-endpoint override, so they are passed
 * on to getAutumnRateLimiter, which stays the only place deciding the final
 * limit. An override makes the multiplier irrelevant, so we skip fetching it
 * from Autumn in that case rather than paying for a value that is discarded.
 */
async function buildAuthenticatedRateLimiter(
  chunk: AuthCreditUsageChunk,
  mode: RateLimiterMode,
  minMultiplier?: number,
): Promise<RateLimiterRedis> {
  return getAutumnRateLimiter(
    mode,
    authenticatedRateLimitMultiplier(chunk, mode, minMultiplier),
    chunk.flags,
  );
}

/**
 * The multiplier an authenticated team's limiter uses for a mode: a neutral 1
 * under an override, otherwise the Autumn multiplier floored at
 * `minMultiplier` (trusted agent traffic passes the hobby multiplier). The
 * floor never applies on top of an override, which already replaces the whole
 * computation.
 */
function authenticatedRateLimitMultiplier(
  chunk: AuthCreditUsageChunk,
  mode: RateLimiterMode,
  minMultiplier?: number,
): number {
  if (
    getRateLimitOverride(mode, chunk.flags?.rateLimitOverrides) !== undefined
  ) {
    return 1;
  }
  return minMultiplier !== undefined
    ? Math.max(chunk.rate_limit_multiplier, minMultiplier)
    : chunk.rate_limit_multiplier;
}

type AuthenticateOptions = {
  allowKeyless?: boolean;
  // Route opt-in: a trusted agent-interop request may authenticate with a
  // hosted_mcp_oauth key, which agent runs started from the hosted MCP carry.
  allowAgentManagedKey?: boolean;
  // Route opt-in: calls the hosted MCP makes get the raised per-minute and
  // concurrency minimums (see getHostedMcpRateLimiter and
  // getHostedMcpConcurrencyLimit) only on routes that set this.
  hostedMcpRateLimit?: boolean;
};

async function supaAuthenticateUser(
  req,
  res,
  mode: RateLimiterMode,
  options?: AuthenticateOptions,
): Promise<AuthResponse> {
  const authHeader =
    req.headers.authorization ??
    (req.headers["sec-websocket-protocol"]
      ? `Bearer ${req.headers["sec-websocket-protocol"]}`
      : null);
  if (!authHeader) {
    return handleKeylessAuth(req, mode, options?.allowKeyless);
  }
  const token = authHeader.split(" ")[1]; // Extract the token from "Bearer <token>"
  if (!token) {
    return denied(req, "malformed_authorization", {
      success: false,
      error: "Unauthorized: Token missing",
      status: 401,
    });
  }

  const incomingIP = (req.headers["x-preview-ip"] ||
    req.headers["x-forwarded-for"] ||
    req.socket.remoteAddress) as string;
  const iptoken = incomingIP + token;

  // An agent run fans one customer request out into ~10 sub-requests against
  // the team's own bucket, so a free team (×1) gets throttled by its own agent.
  // Floor trusted agent traffic at the hobby multiplier; paid plans already
  // meet it and are unchanged.
  const trustedAgentInterop = isTrustedAgentInteropRequest(req);
  const minRateMultiplier = trustedAgentInterop
    ? HOBBY_RATE_LIMIT_MULTIPLIER
    : undefined;

  let rateLimiter: RateLimiterRedis;
  let subscriptionData: { team_id: string } | null = null;
  let normalizedApi: string;

  let teamId: string | null = null;
  let chunk: AuthCreditUsageChunk | null = null;
  // Set only once a hosted MCP delegated credential has verified, so the
  // client never decides it.
  let hostedMcpDelegation = false;
  if (token == "this_is_just_a_preview_token") {
    throw new Error(
      "Unauthenticated Playground calls are temporarily disabled due to abuse. Please sign up.",
    );
  }
  if (timingSafeSecretEqual(token, config.PREVIEW_TOKEN)) {
    if (mode == RateLimiterMode.CrawlStatus) {
      rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus);
    } else if (mode == RateLimiterMode.ExtractStatus) {
      rateLimiter = getRateLimiter(RateLimiterMode.ExtractStatus);
    } else {
      rateLimiter = getRateLimiter(RateLimiterMode.Preview);
    }
    teamId = `preview_${iptoken}`;
  } else if (token.startsWith("fcmcp_")) {
    const delegation = verifyMcpDelegatedCredential(
      token,
      config.MCP_DELEGATED_CREDENTIAL_SECRET,
    );
    if (!delegation) {
      return denied(req, "invalid_mcp_credential", {
        success: false,
        error: "Unauthorized: Invalid token",
        status: 401,
      });
    }

    const resolvedApi = parseApi(delegation.api_key);
    chunk = await getACUC(
      resolvedApi,
      false,
      false,
      RateLimiterMode.Scrape,
      "hosted_mcp_oauth",
    );
    if (chunk === null) {
      return denied(req, "unknown_key", {
        success: false,
        error: "Unauthorized: Invalid token",
        status: 401,
      });
    }

    teamId = chunk.team_id;
    subscriptionData = { team_id: teamId };
    hostedMcpDelegation = true;
    rateLimiter = await buildAuthenticatedRateLimiter(
      chunk,
      mode,
      minRateMultiplier,
    );
  } else if (token.startsWith("fco_")) {
    // OAuth access token — resolve via introspection endpoint
    let introspection: OAuthIntrospectionResponse | null;
    try {
      introspection = await resolveOAuthToken(token);
    } catch (error) {
      if (error instanceof OAuthIntrospectionUnavailableError) {
        return {
          success: false,
          error: "OAuth authentication is temporarily unavailable",
          status: 503,
        };
      }
      throw error;
    }
    if (!introspection) {
      return denied(req, "invalid_oauth_token", {
        success: false,
        error: "Unauthorized: Invalid or expired OAuth token",
        status: 401,
      });
    }
    const credentialPurpose = introspection.credential_purpose ?? "general";
    const isRestGrantKey =
      credentialPurpose === "hosted_mcp_oauth" &&
      introspection.aud === FIRECRAWL_REST_RESOURCE;
    if (credentialPurpose !== "general" && !isRestGrantKey) {
      return denied(req, "oauth_purpose_mismatch", {
        success: false,
        error: "Unauthorized: Invalid token",
        status: 401,
      });
    }

    const resolvedApi = parseApi(introspection.api_key);
    chunk = await getACUC(
      resolvedApi,
      false,
      true,
      RateLimiterMode.Scrape,
      credentialPurpose,
    );

    if (chunk === null) {
      return denied(req, "unknown_key", {
        success: false,
        error: "Unauthorized: Invalid token",
        status: 401,
      });
    }
    if (chunk.team_id !== introspection.team_id) {
      return denied(
        req,
        "oauth_team_mismatch",
        {
          success: false,
          error: "Unauthorized: Invalid token",
          status: 401,
        },
        chunk,
      );
    }

    teamId = chunk.team_id;

    subscriptionData = {
      team_id: teamId,
    };
    rateLimiter = await buildAuthenticatedRateLimiter(
      chunk,
      mode,
      minRateMultiplier,
    );
  } else if (
    config.USE_DB_AUTHENTICATION === true &&
    config.VERCEL_MARKETPLACE_INTEGRATION_ID &&
    looksLikeJwt(token)
  ) {
    // Vercel Marketplace OIDC resource token. It resolves to the API key issued
    // for the resource, so per-key behavior and billing stay on the key path.
    let resolution: VercelOidcResolution;
    try {
      resolution = await resolveVercelOidcToken(
        token,
        config.VERCEL_MARKETPLACE_INTEGRATION_ID,
      );
    } catch (error) {
      if (error instanceof VercelJwksUnavailableError) {
        return {
          success: false,
          error: "Vercel authentication is temporarily unavailable",
          status: 503,
        };
      }
      throw error;
    }
    if (!resolution.ok) {
      return denied(req, "invalid_vercel_oidc_token", {
        success: false,
        error: "Unauthorized: Invalid token",
        status: 401,
      });
    }

    chunk = await getACUC(
      resolution.apiKey,
      false,
      true,
      RateLimiterMode.Scrape,
      "general",
    );
    if (chunk === null) {
      return denied(req, "unknown_key", {
        success: false,
        error: "Unauthorized: Invalid token",
        status: 401,
      });
    }
    if (chunk.team_id !== resolution.teamId) {
      return denied(
        req,
        "vercel_oidc_team_mismatch",
        {
          success: false,
          error: "Unauthorized: Invalid token",
          status: 401,
        },
        chunk,
      );
    }

    teamId = chunk.team_id;
    subscriptionData = { team_id: teamId };
    rateLimiter = await buildAuthenticatedRateLimiter(
      chunk,
      mode,
      minRateMultiplier,
    );
  } else {
    normalizedApi = parseApi(token);
    if (!normalizedApiIsUuid(normalizedApi)) {
      return denied(req, "malformed_key", {
        success: false,
        error: "Unauthorized: Invalid token",
        status: 401,
      });
    }

    chunk = await getACUC(normalizedApi, false, true, RateLimiterMode.Scrape);

    // Agent runs from the hosted MCP hold the grant's hosted_mcp_oauth key,
    // not a general one. Accept it only from the trusted agent service, only
    // on opted-in routes, and only via the uncached primary-read lookup.
    if (
      chunk === null &&
      options?.allowAgentManagedKey === true &&
      trustedAgentInterop
    ) {
      chunk = await getACUC(
        normalizedApi,
        false,
        false,
        RateLimiterMode.Scrape,
        "hosted_mcp_oauth",
      );
    }

    if (chunk === null) {
      return denied(req, "unknown_key", {
        success: false,
        error: "Unauthorized: Invalid token",
        status: 401,
      });
    }

    teamId = chunk.team_id;

    subscriptionData = {
      team_id: teamId,
    };
    rateLimiter = await buildAuthenticatedRateLimiter(
      chunk,
      mode,
      minRateMultiplier,
    );
  }

  // Banned teams are rejected here, where the mcp / OAuth / API-key paths
  // converge. Ban enforcement used to rely on auth_credit_usage_chunk zeroing
  // the rate_limits payload, but authenticated limiting now derives from Autumn
  // and never reads that field, so bans went unenforced. auth_chunk_2 surfaces
  // teams.banned as is_banned and we deny it explicitly.
  if (chunk?.is_banned) {
    return denied(
      req,
      "team_banned",
      {
        success: false,
        error:
          "Unauthorized: This account has been banned. Contact [email protected] if you believe this is a mistake.",
        status: 403,
      },
      chunk,
    );
  }

  if (chunk?.flags?.ipRestriction) {
    const ipCheck = await checkIpRestriction(
      req.ip ?? req.socket?.remoteAddress,
      chunk.team_id,
      chunk.flags,
    );
    if (!ipCheck.allowed) {
      return denied(
        req,
        "ip_restricted",
        { success: false, error: ipCheck.error, status: ipCheck.status },
        chunk,
      );
    }
  }

  if (chunk?.flags?.keyRestriction) {
    // Enforced here rather than in route middleware so every authenticated
    // surface (v0/v1/v2, websocket status) goes through the same gate.
    const endpointCheck = await checkKeyEndpointRestriction(
      req.originalUrl ?? req.url ?? "",
      chunk.api_key_id,
      chunk.flags,
    );
    if (!endpointCheck.allowed) {
      return denied(
        req,
        "endpoint_restricted",
        {
          success: false,
          error: endpointCheck.error,
          status: endpointCheck.status,
        },
        chunk,
      );
    }
  }

  let hostedMcpCall = false;
  if (chunk !== null) {
    // Calls the hosted MCP makes on routes that opt in (every v2 route) get a
    // raised per-minute minimum, and scrape calls a raised concurrency
    // minimum. Two signals prove the hosted MCP made the call: a verified
    // delegated credential (managed OAuth sessions) or the proxy secret it
    // shares with the API (sessions that send a plain API key).
    // `fco_` tokens never qualify: the hosted MCP keeps those for the issuer,
    // so a REST call carrying one comes from the client itself. Origin and
    // integration are client-set, so they never change the limit.
    hostedMcpCall =
      hostedMcpDelegation ||
      (!token.startsWith("fco_") &&
        timingSafeSecretEqual(
          req.headers["x-firecrawl-keyless-secret"],
          config.KEYLESS_PROXY_SECRET,
        ));
    const hostedMcpRaise =
      options?.hostedMcpRateLimit === true && hostedMcpCall;
    // The tiers follow the team's own plan multiplier, not the one trusted
    // agent traffic floors at hobby, so a free team never gets the hobby tier.
    const hostedMcpLimiter = hostedMcpRaise
      ? getHostedMcpRateLimiter(mode, chunk.rate_limit_multiplier, chunk.flags)
      : undefined;
    const hostedMcpConcurrency = hostedMcpRaise
      ? getHostedMcpConcurrencyLimit(
          mode,
          chunk.concurrency_limit,
          chunk.rate_limit_multiplier,
          chunk.flags,
        )
      : undefined;
    if (hostedMcpConcurrency !== undefined) {
      // Only this request's chunk changes; the cached team limits do not.
      chunk = { ...chunk, concurrency_limit: hostedMcpConcurrency };
    }
    if (hostedMcpLimiter) {
      rateLimiter = hostedMcpLimiter;
    }
  }

  const team_endpoint_token = timingSafeSecretEqual(token, config.PREVIEW_TOKEN)
    ? iptoken
    : teamId;

  try {
    await rateLimiter.consume(team_endpoint_token);
  } catch (rateLimiterRes) {
    logger.error(`Rate limit exceeded: ${rateLimiterRes}`, {
      teamId,
      mode,
      rateLimiterRes,
    });

    const secs = Math.ceil(rateLimiterRes.msBeforeNext / 1000) || 1;
    const retryDate = new Date(Date.now() + rateLimiterRes.msBeforeNext);

    // We can only send a rate limit email every 7 days, send notification already has the date in between checking
    // const startDate = new Date();
    // const endDate = new Date();
    // endDate.setDate(endDate.getDate() + 7);

    // await sendNotification(team_id, NotificationType.RATE_LIMIT_REACHED, startDate.toISOString(), endDate.toISOString());

    return {
      success: false,
      error: `Rate limit exceeded. Consumed (req/min): ${rateLimiterRes.consumedPoints}, Remaining (req/min): ${rateLimiterRes.remainingPoints}. Upgrade your plan at https://firecrawl.dev/pricing for increased rate limits or please retry after ${secs}s, resets at ${retryDate}`,
      status: 429,
      retryAfterSeconds: secs,
    };
  }

  if (
    timingSafeSecretEqual(token, config.PREVIEW_TOKEN) &&
    (mode === RateLimiterMode.Scrape ||
      mode === RateLimiterMode.Preview ||
      mode === RateLimiterMode.Map ||
      mode === RateLimiterMode.Crawl ||
      mode === RateLimiterMode.CrawlStatus ||
      mode === RateLimiterMode.Extract ||
      mode === RateLimiterMode.Search ||
      mode === RateLimiterMode.Research)
  ) {
    return {
      success: true,
      team_id: `preview_${iptoken}`,
      org_id: null,
      chunk: null,
    };
    // check the origin of the request and make sure its from firecrawl.dev
    // const origin = req.headers.origin;
    // if (origin && origin.includes("firecrawl.dev")){
    //   return { success: true, team_id: "preview" };
    // }
    // if(config.ENV !== "production") {
    //   return { success: true, team_id: "preview" };
    // }

    // return { success: false, error: "Unauthorized: Invalid token", status: 401 };
  }

  // Check if this is an agent-provisioned key and attach sponsor status
  if (chunk && chunk.api_key_id) {
    try {
      const sponsorStatus = await getAgentSponsorStatus({
        apiKeyId: chunk.api_key_id,
      });
      if (sponsorStatus) {
        chunk._agentSponsor = {
          status: sponsorStatus.status,
          verification_deadline: sponsorStatus.verification_deadline,
          email: sponsorStatus.email,
        };
      }
    } catch (err) {
      logger.warn("Failed to check agent sponsor status", {
        error: err,
        api_key_id: chunk.api_key_id,
      });
    }
  }

  return {
    success: true,
    team_id: teamId ?? undefined,
    org_id: chunk?.org_id ?? null,
    chunk,
    ...(hostedMcpCall ? { hostedMcp: true } : {}),
  };
}