Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { RateLimiterRedis } from "rate-limiter-flexible";
import { isValidUuid } from "../lib/owner-id";
import { config } from "../config";
import { logger } from "../lib/logger";
import { timingSafeSecretEqual } from "../lib/secret-compare";
import { parseApi } from "../lib/parseApi";
import { withAuth } from "../lib/withAuth";
import { getAgentSponsorStatus } from "../services/agent-sponsor";
import {
getRateLimiter,
getAutumnRateLimiter,
getHostedMcpConcurrencyLimit,
getHostedMcpRateLimiter,
getRateLimitOverride,
HOBBY_RATE_LIMIT_MULTIPLIER,
} from "../services/rate-limiter";
import { isTrustedAgentInteropRequest } from "../lib/agent-interop";
import {
consumeKeylessRequest,
isKeylessConfigured,
keylessExhaustionTelemetry,
isKeylessIpEligible,
keylessLimitPrompt,
keylessSignupUrlForIp,
keylessTeamId,
keylessWorldIdIdentity,
normalizeKeylessIpv4,
reportKeylessPromptShown,
} from "../lib/keyless";
import { keylessSignupSurface } from "../lib/keyless-signup-link";
import { consumeIndexRateLimit } from "../lib/index-daily-limit";
import { isKeylessIpSuspicious } from "../lib/spur";
import { verifyWorldIdCredential } from "../lib/world-id";
import { keylessAuthTotal } from "../lib/keyless-metrics";
import { checkIpRestriction } from "../lib/ip-restriction";
import { checkKeyEndpointRestriction } from "../lib/key-restriction";
import { type AuthDenialReason, logAuthDenied } from "../lib/auth-denied-log";
import { deleteKey, getValue, setValue } from "../services/redis";
import { redlock } from "../services/redlock";
import { db, dbRr } from "../db/connection";
import {
authCreditUsageChunk,
authCreditUsageChunkFromTeam,
AuthCreditUsageChunkRow,
} from "../db/rpc";
import { AuthResponse, RateLimiterMode } from "../types";
import {
AuthCreditUsageChunk,
AuthCreditUsageChunkFromTeam,
TeamFlags,
} from "./v1/types";
import {
FIRECRAWL_REST_RESOURCE,
OAuthIntrospectionUnavailableError,
resolveOAuthToken as resolveOAuthTokenWithIntrospection,
} from "../services/oauth-token-introspection";
import type { OAuthIntrospectionResponse } from "../services/oauth-token-introspection";
import { verifyMcpDelegatedCredential } from "../lib/mcp-delegated-credential";
import {
looksLikeJwt,
VercelJwksUnavailableError,
} from "../lib/vercel-marketplace-oidc";
import {
resolveVercelOidcToken,
type VercelOidcResolution,
} from "../services/vercel-marketplace-oidc";
import {
autumnService,
DEFAULT_TEAM_LIMITS,
type TeamLimits,
} from "../services/autumn/autumn.service";
import { ReplyError } from "ioredis";
import { resolveIntegration } from "../utils/integration";
const ACUC_TTL_SECONDS = 600;
// A chunk built on the fail-open limits is cached only briefly, so a
// transient Autumn error is retried within a minute.
const ACUC_FAIL_OPEN_TTL_SECONDS = 60;
// When Autumn can't answer, fail open rather than throttle or gate real teams
// on the low defaults. Generous but bounded: the concurrency queue cap still
// applies.
const FAIL_OPEN_LIMITS: TeamLimits = {
concurrency_limit: 200,
rate_limit_multiplier: 2500,
is_paid_plan: true,
};
/** The team's effective limits, and how long a chunk carrying them may live. */
async function readTeamLimits(
teamId: string,
orgId: string | null,
): Promise<{ limits: TeamLimits; ttl: number }> {
try {
return {
limits: await autumnService.getTeamLimits(teamId, orgId),
ttl: ACUC_TTL_SECONDS,
};
} catch (error) {
logger.error("Autumn limits unavailable, failing open", { teamId, error });
return { limits: FAIL_OPEN_LIMITS, ttl: ACUC_FAIL_OPEN_TTL_SECONDS };
}
}
/** A cached chunk, with live limits when it predates them (not written back). */
async function withLimits<T extends AuthCreditUsageChunkFromTeam>(
chunk: T,
): Promise<T> {
if (
chunk.concurrency_limit !== undefined &&
chunk.rate_limit_multiplier !== undefined &&
chunk.is_paid_plan !== undefined
) {
return chunk;
}
return {
...chunk,
...(await readTeamLimits(chunk.team_id, chunk.org_id)).limits,
};
}
function normalizedApiIsUuid(potentialUuid: string): boolean {
// Check if the string is a valid UUID
return isValidUuid(potentialUuid);
}
async function setCachedACUC(
api_key: string,
is_extract: boolean,
acuc:
| AuthCreditUsageChunk
| null
| ((acuc: AuthCreditUsageChunk) => AuthCreditUsageChunk | null),
credentialPurpose: "general" | "hosted_mcp_oauth" = "general",
ttl = ACUC_TTL_SECONDS,
) {
const cacheKeyACUC = `acuc_${credentialPurpose}_${api_key}_${is_extract ? "extract" : "scrape"}`;
const redLockKey = `lock_${cacheKeyACUC}`;
try {
await redlock.using([redLockKey], 10000, {}, async signal => {
if (typeof acuc === "function") {
acuc = acuc(JSON.parse((await getValue(cacheKeyACUC)) ?? "null"));
if (acuc === null) {
if (signal.aborted) {
throw signal.error;
}
return;
}
}
if (signal.aborted) {
throw signal.error;
}
await setValue(cacheKeyACUC, JSON.stringify(acuc), ttl, true);
});
} catch (error) {
logger.error("Error updating cached ACUC", {
error,
});
}
}
const mockPreviewACUC: (
team_id: string,
is_extract: boolean,
) => AuthCreditUsageChunk = (team_id, is_extract) => ({
api_key: "preview",
api_key_id: 0,
team_id,
org_id: "preview",
flags: null,
is_banned: false,
is_extract,
...DEFAULT_TEAM_LIMITS,
});
const mockACUC: () => AuthCreditUsageChunk = () => ({
api_key: "bypass",
api_key_id: 0,
team_id: "bypass",
org_id: "bypass",
flags: null,
is_banned: false,
is_extract: false,
...DEFAULT_TEAM_LIMITS,
});
/**
* Resolve an OAuth access token (fco_…) to the underlying API key via
* the introspection endpoint. Results are cached in Redis for the
* remaining token TTL (up to 5 minutes).
*/
async function resolveOAuthToken(
token: string,
): Promise<OAuthIntrospectionResponse | null> {
const introspectUrl = config.OAUTH_INTROSPECT_URL;
const introspectSecret = config.OAUTH_INTROSPECT_SECRET;
if (!introspectUrl || !introspectSecret) {
logger.warn(
"OAuth introspection not configured (OAUTH_INTROSPECT_URL / OAUTH_INTROSPECT_SECRET)",
);
throw new OAuthIntrospectionUnavailableError(
"OAuth introspection is not configured",
);
}
return resolveOAuthTokenWithIntrospection(token, {
introspectUrl,
introspectSecret,
expectedResource: FIRECRAWL_REST_RESOURCE,
});
}
async function getACUC(
api_key: string,
cacheOnly = false,
useCache = true,
mode?: RateLimiterMode,
credentialPurpose: "general" | "hosted_mcp_oauth" = "general",
): Promise<AuthCreditUsageChunk | null> {
// Managed OAuth credentials are connection-scoped and can be created or
// revoked immediately before this request. Never serve them from Redis or a
// lagging read replica: both authorization and revocation must observe the
// primary database.
const requiresPrimaryRead = credentialPurpose === "hosted_mcp_oauth";
if (requiresPrimaryRead) useCache = false;
let isExtract =
mode === RateLimiterMode.Extract ||
mode === RateLimiterMode.ExtractStatus ||
mode === RateLimiterMode.ExtractAgentPreview;
if (timingSafeSecretEqual(api_key, config.PREVIEW_TOKEN)) {
const acuc = mockPreviewACUC(api_key, isExtract);
acuc.is_extract = isExtract;
return acuc;
}
if (config.USE_DB_AUTHENTICATION !== true) {
const acuc = mockACUC();
acuc.is_extract = isExtract;
return acuc;
}
const cacheKeyACUC = `acuc_${credentialPurpose}_${api_key}_${isExtract ? "extract" : "scrape"}`;
if (useCache) {
let cachedACUC: string | null;
try {
cachedACUC = await getValue(cacheKeyACUC);
} catch (error) {
if (
error instanceof ReplyError ||
(error instanceof Error &&
(error as any).address &&
(error as any).code &&
error.name === "Error") ||
(error instanceof Error && error.name === "MaxRetriesPerRequestError")
) {
logger.warn(
"Reading ACUC out of cache redis failed, treating as miss",
{
error,
},
);
cachedACUC = null;
} else {
throw error;
}
}
if (cachedACUC !== null) {
try {
return await withLimits(JSON.parse(cachedACUC));
} catch (error) {
logger.warn("Ignoring malformed ACUC cache entry", {
error,
});
void deleteKey(cacheKeyACUC).catch(deleteError => {
logger.warn("Failed to delete malformed ACUC cache entry", {
error: deleteError,
});
});
}
}
}
if (!cacheOnly) {
let data: AuthCreditUsageChunkRow[] = [];
let retries = 0;
const maxRetries = 5;
while (retries < maxRetries) {
// General-purpose reads prefer the replica: the result is Redis-cached
// for 10 minutes, so sub-second replication lag is irrelevant. Fall back
// to the primary on replica error. hosted_mcp_oauth must stay on the
// primary — revocation has to observe fresh state.
const database = requiresPrimaryRead ? db : retries === 0 ? dbRr : db;
try {
data = await authCreditUsageChunk(database, api_key, credentialPurpose);
break;
} catch (error) {
logger.warn(
`Failed to retrieve authentication and credit usage data after ${retries}, trying again...`,
{ error },
);
retries++;
if (retries === maxRetries) {
throw new Error(
"Failed to retrieve authentication and credit usage data after 3 attempts: " +
JSON.stringify(error),
);
}
// Wait for a short time before retrying
await new Promise(resolve => setTimeout(resolve, 200));
}
}
const chunk: AuthCreditUsageChunk | null =
data.length === 0
? null
: data[0].team_id === null
? null
: (data[0] as any);
if (chunk === null) return null;
// NOTE: Should we cache null chunks? - mogery
if (useCache) {
const { limits, ttl } = await readTeamLimits(chunk.team_id, chunk.org_id);
const built = { ...chunk, is_extract: isExtract, ...limits };
setCachedACUC(api_key, isExtract, built, credentialPurpose, ttl);
return built;
}
// An uncached chunk (a hosted MCP credential) takes the team ACUC's
// limits, so authenticating it per request never costs an Autumn read.
const team = await getACUCTeam(chunk.team_id).catch(() => null);
const limits =
team ?? (await readTeamLimits(chunk.team_id, chunk.org_id)).limits;
return {
...chunk,
is_extract: isExtract,
concurrency_limit: limits.concurrency_limit,
rate_limit_multiplier: limits.rate_limit_multiplier,
is_paid_plan: limits.is_paid_plan,
};
} else {
return null;
}
}
async function setCachedACUCTeam(
team_id: string,
is_extract: boolean,
acuc:
| AuthCreditUsageChunkFromTeam
| null
| ((
acuc: AuthCreditUsageChunkFromTeam,
) => AuthCreditUsageChunkFromTeam | null),
ttl = ACUC_TTL_SECONDS,
) {
const cacheKeyACUC = `acuc_team_${team_id}_${is_extract ? "extract" : "scrape"}`;
const redLockKey = `lock_${cacheKeyACUC}`;
try {
await redlock.using([redLockKey], 10000, {}, async signal => {
if (typeof acuc === "function") {
acuc = acuc(JSON.parse((await getValue(cacheKeyACUC)) ?? "null"));
if (acuc === null) {
if (signal.aborted) {
throw signal.error;
}
return;
}
}
if (signal.aborted) {
throw signal.error;
}
await setValue(cacheKeyACUC, JSON.stringify(acuc), ttl, true);
});
} catch (error) {
logger.error("Error updating cached ACUC", {
cacheKey: cacheKeyACUC,
error,
});
}
}
export async function getACUCTeam(
team_id: string,
cacheOnly = false,
useCache = true,
mode?: RateLimiterMode,
): Promise<AuthCreditUsageChunkFromTeam | null> {
let isExtract =
mode === RateLimiterMode.Extract ||
mode === RateLimiterMode.ExtractStatus ||
mode === RateLimiterMode.ExtractAgentPreview;
if (team_id.startsWith("preview")) {
const acuc = mockPreviewACUC(team_id, isExtract);
return acuc;
}
if (config.USE_DB_AUTHENTICATION !== true) {
const acuc = mockACUC();
acuc.is_extract = isExtract;
return acuc;
}
const cacheKeyACUC = `acuc_team_${team_id}_${isExtract ? "extract" : "scrape"}`;
if (useCache) {
let cachedACUC: string | null;
try {
cachedACUC = await getValue(cacheKeyACUC);
} catch (error) {
if (
error instanceof ReplyError ||
(error instanceof Error &&
(error as any).address &&
(error as any).code &&
error.name === "Error") ||
(error instanceof Error && error.name === "MaxRetriesPerRequestError")
) {
logger.warn(
"Reading ACUC out of cache redis failed, treating as miss",
{
cacheKey: cacheKeyACUC,
error,
},
);
cachedACUC = null;
} else {
throw error;
}
}
if (cachedACUC !== null) {
// A corrupt entry is a miss, not a failure: callers that fall back to a
// null org on a throw would otherwise take the high fail-open limits.
try {
return await withLimits(JSON.parse(cachedACUC));
} catch (error) {
logger.warn("Ignoring malformed ACUC cache entry", {
cacheKey: cacheKeyACUC,
error,
});
void deleteKey(cacheKeyACUC).catch(deleteError => {
logger.warn("Failed to delete malformed ACUC cache entry", {
cacheKey: cacheKeyACUC,
error: deleteError,
});
});
}
}
}
if (!cacheOnly) {
let data: AuthCreditUsageChunkRow[] = [];
let retries = 0;
const maxRetries = 5;
while (retries < maxRetries) {
// Prefer the replica (10-minute Redis cache makes lag irrelevant); fall
// back to the primary on replica error.
const database = retries === 0 ? dbRr : db;
try {
data = await authCreditUsageChunkFromTeam(database, team_id);
break;
} catch (error) {
logger.warn(
`Failed to retrieve authentication and credit usage data after ${retries}, trying again...`,
{ error },
);
retries++;
if (retries === maxRetries) {
throw new Error(
"Failed to retrieve authentication and credit usage data after 3 attempts: " +
JSON.stringify(error),
);
}
// Wait for a short time before retrying
await new Promise(resolve => setTimeout(resolve, 200));
}
}
const chunk: AuthCreditUsageChunk | null =
data.length === 0
? null
: data[0].team_id === null
? null
: (data[0] as any);
if (chunk === null) return null;
const { limits, ttl } = await readTeamLimits(chunk.team_id, chunk.org_id);
const built = { ...chunk, ...limits };
// NOTE: Should we cache null chunks? - mogery
if (useCache) {
setCachedACUCTeam(team_id, isExtract, built, ttl);
}
return { ...built, is_extract: isExtract };
} else {
return null;
}
}
/**
* The flags of the team behind a request's plain API key, read without rate
* limiting or the other auth checks. Null without a known API key.
*/
export async function getApiKeyTeamFlags(req: {
headers: { authorization?: string };
}): Promise<TeamFlags | null> {
const token = req.headers.authorization?.split(" ")[1];
if (!token) return null;
const normalizedApi = parseApi(token);
if (!normalizedApiIsUuid(normalizedApi)) return null;
const chunk = await getACUC(normalizedApi);
return chunk?.flags ?? null;
}
export async function clearACUC(api_key: string): Promise<void> {
// Clear both purpose-qualified keys and the legacy unqualified keys during
// rollout so credential-purpose isolation cannot leave stale auth entries.
const modes = [true, false];
const credentialPurposes = ["general", "hosted_mcp_oauth"] as const;
await Promise.all(
modes.flatMap(mode => [
deleteKey(`acuc_${api_key}_${mode ? "extract" : "scrape"}`),
...credentialPurposes.map(credentialPurpose =>
deleteKey(
`acuc_${credentialPurpose}_${api_key}_${mode ? "extract" : "scrape"}`,
),
),
]),
);
// Also clear the base cache key
await deleteKey(`acuc_${api_key}`);
}
export async function clearACUCTeam(team_id: string): Promise<void> {
// Delete cache for all rate limiter modes
const modes = [true, false];
await Promise.all(
modes.map(async mode => {
const cacheKey = `acuc_team_${team_id}_${mode ? "extract" : "scrape"}`;
await deleteKey(cacheKey);
}),
);
// Also clear the base cache key
await deleteKey(`acuc_team_${team_id}`);
}
function denied(
req,
reason: AuthDenialReason,
response: Extract<AuthResponse, { success: false }>,
key?: AuthCreditUsageChunk,
): AuthResponse {
logAuthDenied(req, response.status, reason, key);
return response;
}
// Both prompts end the sentence after the URL with a space, so a copied link
// never picks up the period.
function keylessEndpointNotAvailableMessage(signupUrl: string): string {
return `This endpoint is not supported by the keyless free tier. Sign up for a free API key at ${signupUrl} for more endpoints, more usage, and higher rate limits.
Then authenticate with:
Authorization: Bearer YOUR_API_KEY`;
}
function keylessSuspiciousIpMessage(signupUrl: string): string {
return `Unfortunately, your IP address looks like a VPN or proxy, so Firecrawl can't be used without an API key from here. Sign up with Google or GitHub at ${signupUrl} for a free API key with 1000 credits and higher rate limits. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
}
/**
* The real client IP for keyless. A trusted proxy (e.g. the hosted MCP) may
* forward the end-user's IP via x-firecrawl-keyless-ip, authenticated with a
* shared secret — without the secret the header is ignored, so direct callers
* can't spoof their IP to dodge the per-IP cap.
*/
export function keylessClientIp(req): string {
let ip = req.ip ?? req.socket?.remoteAddress ?? "unknown";
if (
timingSafeSecretEqual(
req.headers["x-firecrawl-keyless-secret"],
config.KEYLESS_PROXY_SECRET,
)
) {
const forwarded = req.headers["x-firecrawl-keyless-ip"];
if (typeof forwarded === "string" && forwarded.trim()) {
ip = forwarded.trim();
}
}
return ip;
}
/**
* Keyless free tier: official MCP/CLI/SDK clients can call scrape, search, and
* interact with no API key. `origin`/`integration` are client-set and spoofable,
* so they're only a soft UX gate — the real abuse controls are the per-IP daily
* request + credit caps plus the `keyless/consume` canonical log emitted here.
* Always returns an AuthResponse — handles every no-API-key request.
*/
async function handleKeylessAuth(
req,
mode: RateLimiterMode,
allowKeyless: boolean | undefined,
): Promise<AuthResponse> {
const unauthorized: AuthResponse = {
success: false,
error: "Unauthorized",
status: 401,
};
// The keyless tier is off unless BOTH limits are configured (even to 0). When
// unconfigured we behave exactly as before — a generic 401 — and don't reveal
// that the tier exists.
if (!isKeylessConfigured()) {
return denied(req, "missing_credentials", unauthorized);
}
// Configured, but this endpoint isn't part of the keyless tier: tell the user
// they need a key (with the signup nudge) rather than a bare "Unauthorized".
if (!allowKeyless) {
const ip = keylessClientIp(req);
const surface = keylessSignupSurface(req);
const { url, signupRef } = keylessSignupUrlForIp(
ip,
surface,
"unsupported_endpoint",
);
reportKeylessPromptShown(
ip,
surface,
"unsupported_endpoint",
401,
signupRef,
);
return denied(req, "missing_credentials", {
success: false,
error: keylessEndpointNotAvailableMessage(url),
status: 401,
signupUrl: url,
});
}
const origin = req.body?.origin;
const integration = resolveIntegration(req.body?.integration, req);
// No origin/surface gate: any request without an API key may use the free
// tier on the allowlisted endpoints (the API itself is free). origin and
// integration are still recorded below for abuse monitoring, and pick the
// surface of the signup link.
const signupSurface = keylessSignupSurface(req);
let ip = keylessClientIp(req);
// A World ID verified human (a valid `fcwid_` credential) gets their own
// bucket keyed on their World ID instead of the IP, so the IP checks below
// don't apply. A missing or invalid credential falls back to the IP bucket.
const worldIdSubject = verifyWorldIdCredential(
req.headers["x-firecrawl-world-id"],
);
const worldIdIdentity = worldIdSubject
? keylessWorldIdIdentity(worldIdSubject)
: null;
// Only a valid IPv4 identity gets keyless: IPv6 is too cheap to rotate for a
// per-IP cap to mean anything, and malformed/forwarded values must not be
// usable as arbitrary limiter buckets. Anything else falls through to 401.
if (!worldIdIdentity && !isKeylessIpEligible(ip)) {
return denied(req, "keyless_ip_ineligible", unauthorized);
}
// Canonicalize `::ffff:`-mapped IPv4 so a client gets one Spur cache entry,
// one quota bucket, and one team id regardless of how the socket reported it.
ip = normalizeKeylessIpv4(ip);
// Optional Spur Context check (only when SPUR_API_KEY is set): refuse keyless
// for IPs fronting anonymizing/rotating infrastructure (VPN/proxy/TOR), the
// main way the per-IP caps get bypassed. Fails open on any Spur error, and
// runs before consuming quota so a flagged IP doesn't burn a request slot.
if (!worldIdIdentity && (await isKeylessIpSuspicious(ip))) {
keylessAuthTotal.inc({ mode, outcome: "suspicious" });
const { url, signupRef } = keylessSignupUrlForIp(
ip,
signupSurface,
"suspicious_ip",
);
reportKeylessPromptShown(
ip,
signupSurface,
"suspicious_ip",
403,
signupRef,
);
logger.warn("Keyless request blocked: suspicious IP", {
canonicalLog: "keyless/consume",
ip,
origin: req.body?.origin,
integration,
blocked: true,
reason: "suspicious",
...(signupRef ? { signupRef } : {}),
});
return denied(req, "keyless_ip_suspicious", {
success: false,
error: keylessSuspiciousIpMessage(url),
status: 403,
// Tell agents where to find the key/signup flow they now need.
agentAuthDiscovery: true,
signupUrl: url,
});
}
// The quota bucket and team id. Logs keep the client IP either way.
const identity = worldIdIdentity ?? ip;
const teamId = keylessTeamId(identity);
const modeLabel =
mode === RateLimiterMode.Search
? "search"
: mode === RateLimiterMode.Research
? "research"
: mode === RateLimiterMode.DeveloperSearch
? "developer"
: mode === RateLimiterMode.GovSearch
? "gov"
: mode === RateLimiterMode.BrowserExecute
? "interact"
: "scrape";
let result: Awaited<ReturnType<typeof consumeKeylessRequest>>;
try {
result = await consumeKeylessRequest(identity);
} catch (error) {
keylessAuthTotal.inc({ mode, outcome: "error" });
// Limiter store (Redis) unavailable — fail closed with a controlled auth
// response instead of surfacing a 500, and shed the free traffic while the
// limiter can't enforce quotas.
logger.warn("Keyless quota check failed", {
canonicalLog: "keyless/consume",
ip,
mode: modeLabel,
teamId,
error,
});
return denied(req, "keyless_limiter_unavailable", unauthorized);
}
const baseLog = {
canonicalLog: "keyless/consume",
ip,
origin,
integration,
mode: modeLabel,
teamId,
worldId: worldIdIdentity !== null,
requestsUsed: result.requestsUsed,
creditsUsed: result.creditsUsed,
};
if (!result.ok) {
keylessAuthTotal.inc({ mode, outcome: result.reason ?? "error" });
const prompt = keylessLimitPrompt(
worldIdIdentity ? null : ip,
signupSurface,
);
logger.warn("Keyless request blocked", {
...baseLog,
blocked: true,
event: "keyless_exhausted",
reason: result.reason,
retryAfterSeconds: result.retryAfterSeconds,
...(prompt.signupRef ? { signupRef: prompt.signupRef } : {}),
...keylessExhaustionTelemetry(identity),
});
return {
success: false,
error: prompt.error,
status: 429,
signupUrl: prompt.signup_url,
// Direct API callers receive discovery metadata; MCP maps this in-band.
agentAuthDiscovery: true,
keylessReason: result.reason,
retryAfterSeconds: result.retryAfterSeconds,
};
}
if (modeLabel === "gov" || modeLabel === "research") {
const perMinute = await consumeIndexRateLimit(
modeLabel,
teamId,
1,
undefined,
1,
);
if (!perMinute.ok) {
keylessAuthTotal.inc({ mode, outcome: "index_rate_limit" });
logger.warn("Keyless request blocked", {
...baseLog,
blocked: true,
reason: "index_rate_limit",
retryAfterSeconds: perMinute.retryAfterSeconds,
});
return {
success: false,
error: perMinute.error,
status: 429,
retryAfterSeconds: perMinute.retryAfterSeconds,
};
}
}
logger.debug("Keyless request consumed", { ...baseLog, blocked: false });
keylessAuthTotal.inc({ mode, outcome: "allowed" });
// Tag as a preview team so billing (autumn isPreviewTeam) and GCS persistence
// are skipped automatically; mockPreviewACUC supplies concurrency 2 + credits.
// Actual credits consumed are charged to the IP's daily budget after the
// request completes (see chargeKeylessCredits).
return {
success: true,
team_id: teamId,
org_id: null,
chunk: mockPreviewACUC(teamId, false),
};
}
export async function authenticateUser(
req,
res,
mode: RateLimiterMode,
options?: AuthenticateOptions,
): Promise<AuthResponse> {
const bypassChunk = mockACUC();
bypassChunk.is_extract =
mode === RateLimiterMode.Extract ||
mode === RateLimiterMode.ExtractStatus ||
mode === RateLimiterMode.ExtractAgentPreview;
return withAuth(supaAuthenticateUser, {
success: true,
chunk: bypassChunk,
team_id: bypassChunk.team_id,
org_id: null,
})(req, res, mode, options);
}
/**
* Builds the rate limiter for an authenticated team from its Autumn rate-limit
* multiplier. Shared by the OAuth and API-key paths so their limiter setup
* can't diverge.
*
* The org flags carry the optional per-endpoint override, so they are passed
* on to getAutumnRateLimiter, which stays the only place deciding the final
* limit. An override makes the multiplier irrelevant, so we skip fetching it
* from Autumn in that case rather than paying for a value that is discarded.
*/
async function buildAuthenticatedRateLimiter(
chunk: AuthCreditUsageChunk,
mode: RateLimiterMode,
minMultiplier?: number,
): Promise<RateLimiterRedis> {
return getAutumnRateLimiter(
mode,
authenticatedRateLimitMultiplier(chunk, mode, minMultiplier),
chunk.flags,
);
}
/**
* The multiplier an authenticated team's limiter uses for a mode: a neutral 1
* under an override, otherwise the Autumn multiplier floored at
* `minMultiplier` (trusted agent traffic passes the hobby multiplier). The
* floor never applies on top of an override, which already replaces the whole
* computation.
*/
function authenticatedRateLimitMultiplier(
chunk: AuthCreditUsageChunk,
mode: RateLimiterMode,
minMultiplier?: number,
): number {
if (
getRateLimitOverride(mode, chunk.flags?.rateLimitOverrides) !== undefined
) {
return 1;
}
return minMultiplier !== undefined
? Math.max(chunk.rate_limit_multiplier, minMultiplier)
: chunk.rate_limit_multiplier;
}
type AuthenticateOptions = {
allowKeyless?: boolean;
// Route opt-in: a trusted agent-interop request may authenticate with a
// hosted_mcp_oauth key, which agent runs started from the hosted MCP carry.
allowAgentManagedKey?: boolean;
// Route opt-in: calls the hosted MCP makes get the raised per-minute and
// concurrency minimums (see getHostedMcpRateLimiter and
// getHostedMcpConcurrencyLimit) only on routes that set this.
hostedMcpRateLimit?: boolean;
};
async function supaAuthenticateUser(
req,
res,
mode: RateLimiterMode,
options?: AuthenticateOptions,
): Promise<AuthResponse> {
const authHeader =
req.headers.authorization ??
(req.headers["sec-websocket-protocol"]
? `Bearer ${req.headers["sec-websocket-protocol"]}`
: null);
if (!authHeader) {
return handleKeylessAuth(req, mode, options?.allowKeyless);
}
const token = authHeader.split(" ")[1]; // Extract the token from "Bearer <token>"
if (!token) {
return denied(req, "malformed_authorization", {
success: false,
error: "Unauthorized: Token missing",
status: 401,
});
}
const incomingIP = (req.headers["x-preview-ip"] ||
req.headers["x-forwarded-for"] ||
req.socket.remoteAddress) as string;
const iptoken = incomingIP + token;
// An agent run fans one customer request out into ~10 sub-requests against
// the team's own bucket, so a free team (×1) gets throttled by its own agent.
// Floor trusted agent traffic at the hobby multiplier; paid plans already
// meet it and are unchanged.
const trustedAgentInterop = isTrustedAgentInteropRequest(req);
const minRateMultiplier = trustedAgentInterop
? HOBBY_RATE_LIMIT_MULTIPLIER
: undefined;
let rateLimiter: RateLimiterRedis;
let subscriptionData: { team_id: string } | null = null;
let normalizedApi: string;
let teamId: string | null = null;
let chunk: AuthCreditUsageChunk | null = null;
// Set only once a hosted MCP delegated credential has verified, so the
// client never decides it.
let hostedMcpDelegation = false;
if (token == "this_is_just_a_preview_token") {
throw new Error(
"Unauthenticated Playground calls are temporarily disabled due to abuse. Please sign up.",
);
}
if (timingSafeSecretEqual(token, config.PREVIEW_TOKEN)) {
if (mode == RateLimiterMode.CrawlStatus) {
rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus);
} else if (mode == RateLimiterMode.ExtractStatus) {
rateLimiter = getRateLimiter(RateLimiterMode.ExtractStatus);
} else {
rateLimiter = getRateLimiter(RateLimiterMode.Preview);
}
teamId = `preview_${iptoken}`;
} else if (token.startsWith("fcmcp_")) {
const delegation = verifyMcpDelegatedCredential(
token,
config.MCP_DELEGATED_CREDENTIAL_SECRET,
);
if (!delegation) {
return denied(req, "invalid_mcp_credential", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
});
}
const resolvedApi = parseApi(delegation.api_key);
chunk = await getACUC(
resolvedApi,
false,
false,
RateLimiterMode.Scrape,
"hosted_mcp_oauth",
);
if (chunk === null) {
return denied(req, "unknown_key", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
});
}
teamId = chunk.team_id;
subscriptionData = { team_id: teamId };
hostedMcpDelegation = true;
rateLimiter = await buildAuthenticatedRateLimiter(
chunk,
mode,
minRateMultiplier,
);
} else if (token.startsWith("fco_")) {
// OAuth access token — resolve via introspection endpoint
let introspection: OAuthIntrospectionResponse | null;
try {
introspection = await resolveOAuthToken(token);
} catch (error) {
if (error instanceof OAuthIntrospectionUnavailableError) {
return {
success: false,
error: "OAuth authentication is temporarily unavailable",
status: 503,
};
}
throw error;
}
if (!introspection) {
return denied(req, "invalid_oauth_token", {
success: false,
error: "Unauthorized: Invalid or expired OAuth token",
status: 401,
});
}
const credentialPurpose = introspection.credential_purpose ?? "general";
const isRestGrantKey =
credentialPurpose === "hosted_mcp_oauth" &&
introspection.aud === FIRECRAWL_REST_RESOURCE;
if (credentialPurpose !== "general" && !isRestGrantKey) {
return denied(req, "oauth_purpose_mismatch", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
});
}
const resolvedApi = parseApi(introspection.api_key);
chunk = await getACUC(
resolvedApi,
false,
true,
RateLimiterMode.Scrape,
credentialPurpose,
);
if (chunk === null) {
return denied(req, "unknown_key", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
});
}
if (chunk.team_id !== introspection.team_id) {
return denied(
req,
"oauth_team_mismatch",
{
success: false,
error: "Unauthorized: Invalid token",
status: 401,
},
chunk,
);
}
teamId = chunk.team_id;
subscriptionData = {
team_id: teamId,
};
rateLimiter = await buildAuthenticatedRateLimiter(
chunk,
mode,
minRateMultiplier,
);
} else if (
config.USE_DB_AUTHENTICATION === true &&
config.VERCEL_MARKETPLACE_INTEGRATION_ID &&
looksLikeJwt(token)
) {
// Vercel Marketplace OIDC resource token. It resolves to the API key issued
// for the resource, so per-key behavior and billing stay on the key path.
let resolution: VercelOidcResolution;
try {
resolution = await resolveVercelOidcToken(
token,
config.VERCEL_MARKETPLACE_INTEGRATION_ID,
);
} catch (error) {
if (error instanceof VercelJwksUnavailableError) {
return {
success: false,
error: "Vercel authentication is temporarily unavailable",
status: 503,
};
}
throw error;
}
if (!resolution.ok) {
return denied(req, "invalid_vercel_oidc_token", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
});
}
chunk = await getACUC(
resolution.apiKey,
false,
true,
RateLimiterMode.Scrape,
"general",
);
if (chunk === null) {
return denied(req, "unknown_key", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
});
}
if (chunk.team_id !== resolution.teamId) {
return denied(
req,
"vercel_oidc_team_mismatch",
{
success: false,
error: "Unauthorized: Invalid token",
status: 401,
},
chunk,
);
}
teamId = chunk.team_id;
subscriptionData = { team_id: teamId };
rateLimiter = await buildAuthenticatedRateLimiter(
chunk,
mode,
minRateMultiplier,
);
} else {
normalizedApi = parseApi(token);
if (!normalizedApiIsUuid(normalizedApi)) {
return denied(req, "malformed_key", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
});
}
chunk = await getACUC(normalizedApi, false, true, RateLimiterMode.Scrape);
// Agent runs from the hosted MCP hold the grant's hosted_mcp_oauth key,
// not a general one. Accept it only from the trusted agent service, only
// on opted-in routes, and only via the uncached primary-read lookup.
if (
chunk === null &&
options?.allowAgentManagedKey === true &&
trustedAgentInterop
) {
chunk = await getACUC(
normalizedApi,
false,
false,
RateLimiterMode.Scrape,
"hosted_mcp_oauth",
);
}
if (chunk === null) {
return denied(req, "unknown_key", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
});
}
teamId = chunk.team_id;
subscriptionData = {
team_id: teamId,
};
rateLimiter = await buildAuthenticatedRateLimiter(
chunk,
mode,
minRateMultiplier,
);
}
// Banned teams are rejected here, where the mcp / OAuth / API-key paths
// converge. Ban enforcement used to rely on auth_credit_usage_chunk zeroing
// the rate_limits payload, but authenticated limiting now derives from Autumn
// and never reads that field, so bans went unenforced. auth_chunk_2 surfaces
// teams.banned as is_banned and we deny it explicitly.
if (chunk?.is_banned) {
return denied(
req,
"team_banned",
{
success: false,
error:
"Unauthorized: This account has been banned. Contact [email protected] if you believe this is a mistake.",
status: 403,
},
chunk,
);
}
if (chunk?.flags?.ipRestriction) {
const ipCheck = await checkIpRestriction(
req.ip ?? req.socket?.remoteAddress,
chunk.team_id,
chunk.flags,
);
if (!ipCheck.allowed) {
return denied(
req,
"ip_restricted",
{ success: false, error: ipCheck.error, status: ipCheck.status },
chunk,
);
}
}
if (chunk?.flags?.keyRestriction) {
// Enforced here rather than in route middleware so every authenticated
// surface (v0/v1/v2, websocket status) goes through the same gate.
const endpointCheck = await checkKeyEndpointRestriction(
req.originalUrl ?? req.url ?? "",
chunk.api_key_id,
chunk.flags,
);
if (!endpointCheck.allowed) {
return denied(
req,
"endpoint_restricted",
{
success: false,
error: endpointCheck.error,
status: endpointCheck.status,
},
chunk,
);
}
}
let hostedMcpCall = false;
if (chunk !== null) {
// Calls the hosted MCP makes on routes that opt in (every v2 route) get a
// raised per-minute minimum, and scrape calls a raised concurrency
// minimum. Two signals prove the hosted MCP made the call: a verified
// delegated credential (managed OAuth sessions) or the proxy secret it
// shares with the API (sessions that send a plain API key).
// `fco_` tokens never qualify: the hosted MCP keeps those for the issuer,
// so a REST call carrying one comes from the client itself. Origin and
// integration are client-set, so they never change the limit.
hostedMcpCall =
hostedMcpDelegation ||
(!token.startsWith("fco_") &&
timingSafeSecretEqual(
req.headers["x-firecrawl-keyless-secret"],
config.KEYLESS_PROXY_SECRET,
));
const hostedMcpRaise =
options?.hostedMcpRateLimit === true && hostedMcpCall;
// The tiers follow the team's own plan multiplier, not the one trusted
// agent traffic floors at hobby, so a free team never gets the hobby tier.
const hostedMcpLimiter = hostedMcpRaise
? getHostedMcpRateLimiter(mode, chunk.rate_limit_multiplier, chunk.flags)
: undefined;
const hostedMcpConcurrency = hostedMcpRaise
? getHostedMcpConcurrencyLimit(
mode,
chunk.concurrency_limit,
chunk.rate_limit_multiplier,
chunk.flags,
)
: undefined;
if (hostedMcpConcurrency !== undefined) {
// Only this request's chunk changes; the cached team limits do not.
chunk = { ...chunk, concurrency_limit: hostedMcpConcurrency };
}
if (hostedMcpLimiter) {
rateLimiter = hostedMcpLimiter;
}
}
const team_endpoint_token = timingSafeSecretEqual(token, config.PREVIEW_TOKEN)
? iptoken
: teamId;
try {
await rateLimiter.consume(team_endpoint_token);
} catch (rateLimiterRes) {
logger.error(`Rate limit exceeded: ${rateLimiterRes}`, {
teamId,
mode,
rateLimiterRes,
});
const secs = Math.ceil(rateLimiterRes.msBeforeNext / 1000) || 1;
const retryDate = new Date(Date.now() + rateLimiterRes.msBeforeNext);
// We can only send a rate limit email every 7 days, send notification already has the date in between checking
// const startDate = new Date();
// const endDate = new Date();
// endDate.setDate(endDate.getDate() + 7);
// await sendNotification(team_id, NotificationType.RATE_LIMIT_REACHED, startDate.toISOString(), endDate.toISOString());
return {
success: false,
error: `Rate limit exceeded. Consumed (req/min): ${rateLimiterRes.consumedPoints}, Remaining (req/min): ${rateLimiterRes.remainingPoints}. Upgrade your plan at https://firecrawl.dev/pricing for increased rate limits or please retry after ${secs}s, resets at ${retryDate}`,
status: 429,
retryAfterSeconds: secs,
};
}
if (
timingSafeSecretEqual(token, config.PREVIEW_TOKEN) &&
(mode === RateLimiterMode.Scrape ||
mode === RateLimiterMode.Preview ||
mode === RateLimiterMode.Map ||
mode === RateLimiterMode.Crawl ||
mode === RateLimiterMode.CrawlStatus ||
mode === RateLimiterMode.Extract ||
mode === RateLimiterMode.Search ||
mode === RateLimiterMode.Research)
) {
return {
success: true,
team_id: `preview_${iptoken}`,
org_id: null,
chunk: null,
};
// check the origin of the request and make sure its from firecrawl.dev
// const origin = req.headers.origin;
// if (origin && origin.includes("firecrawl.dev")){
// return { success: true, team_id: "preview" };
// }
// if(config.ENV !== "production") {
// return { success: true, team_id: "preview" };
// }
// return { success: false, error: "Unauthorized: Invalid token", status: 401 };
}
// Check if this is an agent-provisioned key and attach sponsor status
if (chunk && chunk.api_key_id) {
try {
const sponsorStatus = await getAgentSponsorStatus({
apiKeyId: chunk.api_key_id,
});
if (sponsorStatus) {
chunk._agentSponsor = {
status: sponsorStatus.status,
verification_deadline: sponsorStatus.verification_deadline,
email: sponsorStatus.email,
};
}
} catch (err) {
logger.warn("Failed to check agent sponsor status", {
error: err,
api_key_id: chunk.api_key_id,
});
}
}
return {
success: true,
team_id: teamId ?? undefined,
org_id: chunk?.org_id ?? null,
chunk,
...(hostedMcpCall ? { hostedMcp: true } : {}),
};
}