Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import "dotenv/config";
import { z } from "zod";
import { getMcpActionLogConfigErrors } from "./lib/mcp-action-log-config";
/* Codecs */
const delimitedList = (separator = ",") => {
return z.codec(z.string(), z.array(z.string()), {
decode: str => (str ? str.split(separator).map(s => s.trim()) : []),
encode: arr => arr.join(separator),
});
};
const emptyStringAsUndefined = <T extends z.ZodTypeAny>(schema: T) =>
z.preprocess(value => (value === "" ? undefined : value), schema.optional());
const emptyStringAsDefault = <T extends z.ZodTypeAny>(schema: T) =>
z.preprocess(value => (value === "" ? undefined : value), schema);
const worldIdSubjectList = (name: string) =>
emptyStringAsUndefined(
z
.string()
.refine(
value =>
value
.split(",")
.every(entry => /^[A-Za-z0-9_-]{43}$/.test(entry.trim())),
`${name} must be comma-separated subject hashes`,
),
);
// Trim strings and treat blank ones as unset (optional or default schemas).
const blankStringAsUnset = <T extends z.ZodTypeAny>(schema: T) =>
z.preprocess(
value =>
typeof value === "string"
? value.trim() === ""
? undefined
: value.trim()
: value,
schema,
);
const RESEARCH_PAPER_OPERATIONS = [
"search",
"inspect",
"read",
"similar",
] as const;
export type ResearchKeylessOperation =
(typeof RESEARCH_PAPER_OPERATIONS)[number];
const researchKeylessDisabled = z.preprocess(
value => {
if (typeof value !== "string") return value;
const raw = value.trim().toLowerCase();
if (raw === "") return undefined;
if (["false", "0", "off", "no", "none"].includes(raw)) return [];
if (["true", "1", "on", "yes", "all"].includes(raw)) {
return [...RESEARCH_PAPER_OPERATIONS];
}
return raw
.split(",")
.map(operation => operation.trim())
.filter(Boolean);
},
z
.array(z.enum(RESEARCH_PAPER_OPERATIONS))
.default([...RESEARCH_PAPER_OPERATIONS]),
);
const teamDailyLimits = z.preprocess(
value =>
value === undefined
? []
: typeof value === "string"
? value
.split(",")
.map(entry => entry.trim().toLowerCase())
.filter(Boolean)
.map(entry => entry.split(":").map(part => part.trim()))
: value,
z
.array(z.tuple([z.uuid(), z.coerce.number().int().positive()]))
.refine(
pairs => new Set(pairs.map(([teamId]) => teamId)).size === pairs.length,
"each team id may appear only once",
)
.transform(pairs => Object.fromEntries(pairs)),
);
const containsLoneSurrogate = (value: string): boolean => {
for (let index = 0; index < value.length; index++) {
const codeUnit = value.charCodeAt(index);
if (codeUnit >= 0xd800 && codeUnit <= 0xdbff) {
const nextCodeUnit = value.charCodeAt(index + 1);
if (!(nextCodeUnit >= 0xdc00 && nextCodeUnit <= 0xdfff)) return true;
index++;
} else if (codeUnit >= 0xdc00 && codeUnit <= 0xdfff) {
return true;
}
}
return false;
};
/* Schema */
const configSchema = z.object({
// Application
ENV: z.string().optional(),
HOST: z.string().default("localhost"),
PORT: z.coerce.number().default(3002),
IS_PRODUCTION: z.stringbool().optional(),
IS_KUBERNETES: z.stringbool().optional(),
FIRECRAWL_APP_HOST: z.string().default("firecrawl-app-service"),
FIRECRAWL_APP_PORT: z.string().default("3002"),
FIRECRAWL_APP_SCHEME: z.string().default("http"),
LOGGING_LEVEL: z.string().optional(),
FIRECRAWL_DASHBOARD_URL: z.url().default("https://www.firecrawl.dev"),
SUPPORT_AGENT_URL: z.string().url().optional(),
SUPPORT_AGENT_VERCEL_BYPASS_SECRET: z.string().optional(),
FIREBRAIN_TRACKS_URL: z.preprocess(
v => (typeof v === "string" && v.trim() === "" ? undefined : v),
z.string().url().optional(),
),
FIREBRAIN_TRACKS_API_KEY: z.preprocess(
v => (typeof v === "string" && v.trim() === "" ? undefined : v),
z.string().trim().optional(),
),
RESEARCH_PROXY_URL: z.string().url().optional(),
SEARCH_PLATFORM_URL: z.string().url().optional(),
// When true, a plain web search that the search platform classifies as
// government is served through the Government Index. It needs
// SEARCH_PLATFORM_URL. Unset or empty, no search is routed.
GOV_ROUTING_ENABLED: emptyStringAsDefault(z.stringbool().default(false)),
// The longest a plain web search waits for the routing decision before the
// regular web search starts; a decision that takes longer routes nothing.
GOV_ROUTING_DECISION_TIMEOUT_MS: emptyStringAsDefault(
z.coerce.number().int().positive().default(100),
),
RESEARCH_KEYLESS_DISABLED: researchKeylessDisabled,
// Per-team Government Index daily limits as teamId:limit,teamId:limit. A
// listed team's limit replaces its plan limit.
GOV_INDEX_CALLS_PER_DAY_OVERRIDES: teamDailyLimits,
LABS_SEARCH_URL: z.string().url().optional(),
LABS_SEARCH_SECRET: z.string().optional(),
FRESH_FEED_URL: emptyStringAsUndefined(z.string().url()),
// Express
EXPRESS_TRUST_PROXY: z.coerce.number().optional(),
// Keyless free tier (scrape/search/interact without an API key, per-IP/day).
// Non-negative integers; 0 means "enabled but no budget", unset means "off".
KEYLESS_CREDITS_PER_DAY: z.coerce.number().int().nonnegative().optional(),
KEYLESS_REQUESTS_PER_DAY: z.coerce.number().int().nonnegative().optional(),
// Shared secret that lets a trusted proxy (e.g. the hosted MCP server)
// forward the real client IP for keyless rate-limiting via the
// `x-firecrawl-keyless-ip` header. Untrusted callers can't override their IP.
KEYLESS_PROXY_SECRET: z.string().optional(),
// Dedicated HMAC key for joining keyless quota-exhaustion events to the
// existing privacy-controlled conversion pipeline. Never use the proxy or
// credential secrets here: this value is only an analytics pseudonymizer.
KEYLESS_CONVERSION_HMAC_SECRET: emptyStringAsUndefined(z.string().min(32)),
// AES-128 keys for keyless signup link tokens (firecrawl.dev/k/<token>):
// comma-separated base64, 16 bytes each. The first encrypts; every key is
// tried to decrypt, so keep a rotated-out key listed while its links live.
// Must match firecrawl-web's KEYLESS_SIGNUP_LINK_KEYS. Unset sends the
// regular signup link.
KEYLESS_SIGNUP_LINK_KEYS: emptyStringAsUndefined(
z
.string()
.refine(
value =>
value
.split(",")
.every(key => /^[A-Za-z0-9+/]{21}[AQgw]==$/.test(key.trim())),
"KEYLESS_SIGNUP_LINK_KEYS must be comma-separated base64 16-byte keys",
),
),
// Dedicated signer/verifier secret for short-lived MCP delegated credentials.
// Keep separate from KEYLESS_PROXY_SECRET because delegated credentials can
// authorize billed requests for a managed OAuth connection.
MCP_DELEGATED_CREDENTIAL_SECRET: emptyStringAsUndefined(z.string().min(32)),
// World ID verified-human keyless bucket. The issuer defaults to World's; the
// feature is off unless the client ID, client secret, credential secret and
// subject secret are set.
WORLD_ID_ISSUER: emptyStringAsDefault(
z
.string()
.trim()
.url()
.refine(
value => value.startsWith("https://"),
"WORLD_ID_ISSUER must be an https URL",
)
.default("https://auth.world.org"),
),
WORLD_ID_CLIENT_ID: emptyStringAsUndefined(z.string().trim().min(1)),
WORLD_ID_CLIENT_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),
WORLD_ID_CREDENTIAL_SECRET: emptyStringAsUndefined(z.string().trim().min(32)),
// Keys the subject hashes (bucket ids, allow and deny lists). Kept apart
// from the credential secret so rotating that one revokes credentials
// without resetting buckets or the lists.
WORLD_ID_SUBJECT_SECRET: emptyStringAsUndefined(z.string().trim().min(32)),
// Optional comma-separated allowlist of subject hashes. When set, only these
// verified humans are issued a credential; unset issues one to anyone who
// verifies.
WORLD_ID_ALLOWED_SUBJECTS: worldIdSubjectList("WORLD_ID_ALLOWED_SUBJECTS"),
// Optional comma-separated subject hashes that are refused: no new
// credential, and their existing credentials stop working.
WORLD_ID_DENIED_SUBJECTS: worldIdSubjectList("WORLD_ID_DENIED_SUBJECTS"),
// Optional Spur Context API token (https://docs.spur.us/context-api). When
// set, keyless requests have their client IP checked against Spur and are
// refused if the IP fronts anonymizing/rotating infrastructure (VPN/proxy/
// TOR). Unset disables the check entirely (keyless behaves as before).
SPUR_API_KEY: z.string().optional(),
// Threat protection (enterprise domain risk blocking). "normal" mode uses
// Google Web Risk. An unset key disables the provider (lookups then fail
// per the org's failurePolicy).
GOOGLE_WEB_RISK_API_KEY: z.string().optional(),
GOOGLE_WEB_RISK_API_URL: z
.string()
.url()
.default("https://webrisk.googleapis.com"),
// Google Web Risk Update API sync tuning. ZDR: "normal" mode checks run
// against a locally synced hash-prefix database (threatLists:computeDiff)
// instead of sending URLs to Google, and verdicts are never persisted.
//
// Floor for how often threatLists:computeDiff may run per list. Google's
// recommendedNextDiff is respected when it is later than this floor.
THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS: z.coerce
.number()
.int()
.positive()
.default(60),
// A synced threat list older than this is treated as unavailable
// (provider-failure semantics → the org's failurePolicy decides).
THREAT_LIST_STALENESS_SECONDS: z.coerce
.number()
.int()
.positive()
.default(6 * 60 * 60),
// Zscaler ZIA provider ("zscaler" threat protection mode). Base-URL
// overrides exist for tests (mock ZIA server); production always uses the
// real endpoints derived from the org's vanity domain and cloud name.
ZSCALER_TOKEN_URL_OVERRIDE: z.string().url().optional(),
ZSCALER_API_URL_OVERRIDE: z.string().url().optional(),
// Organization SIEM logging delivery. The encryption key must decode to
// exactly 32 bytes; validation happens when a secret is encrypted/decrypted
// so self-hosted deployments that do not use this feature need no key.
SIEM_LOGGING_ENCRYPTION_KEY: z.string().optional(),
PARTNER_EGRESS_PROXY_URL: z.string().url().optional(),
// API Keys & Authentication
BULL_AUTH_KEY: z.string().optional(),
BULL_AUTH_KEY_PREVIOUS: emptyStringAsUndefined(z.string()),
BULL_AUTH_KEY_PATH_ROUTES: emptyStringAsDefault(z.stringbool().default(true)),
ADMIN_INTERNAL_PORT: emptyStringAsUndefined(
z.coerce.number().int().positive().max(65535),
),
ADMIN_INTERNAL_HOST: emptyStringAsDefault(z.string().default("localhost")),
S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY: emptyStringAsUndefined(
z.string().trim().min(1),
),
OPENAI_API_KEY: z.string().optional(),
OPENAI_BASE_URL: z.string().optional(),
// Key for US-only teams; OPENAI_API_KEY is never used for them.
OPENAI_US_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),
OPENROUTER_API_KEY: z.string().optional(),
LLAMAPARSE_API_KEY: z.string().optional(),
STRIPE_SECRET_KEY: z.string().optional(),
AUTUMN_SECRET_KEY: z.string().optional(),
RESEND_API_KEY: z.string().optional(),
PREVIEW_TOKEN: z.string().optional(),
SEARCH_PREVIEW_TOKEN: z.string().optional(),
SEARCH_SERVICE_API_SECRET: z.string().optional(),
SEARCH_FEEDBACK_MAX_AGE_SEC: z.coerce.number().int().positive().default(300),
SEARCH_FEEDBACK_DAILY_CAP_CREDITS: z.coerce
.number()
.int()
.nonnegative()
.default(100),
FEEDBACK_MAX_AGE_SEC: z.coerce.number().int().positive().default(120),
FEEDBACK_DAILY_CAP_CREDITS: z.coerce.number().int().nonnegative().default(50),
FEEDBACK_REFUND_ENABLED: z.stringbool().default(true),
ALEXANDRIA_FEEDBACK_WINDOW_SEC: z.coerce
.number()
.int()
.positive()
.default(1200),
ALEXANDRIA_FEEDBACK_DAILY_CAP_CREDITS: z.coerce
.number()
.int()
.nonnegative()
.default(100),
ALEXANDRIA_FEEDBACK_WEBSITE_DAILY_CAP_CREDITS: z.coerce
.number()
.int()
.nonnegative()
.default(10),
// Vercel Marketplace integration id (oac_...). Vercel OIDC resource tokens
// (iss https://integrations.vercel.com/<id>) are accepted as bearer
// credentials when DB authentication is on. Unset uses Firecrawl's
// integration; an empty value disables the branch. Not a secret.
VERCEL_MARKETPLACE_INTEGRATION_ID: z.preprocess(
value => (value === "" ? null : value),
z
.string()
.regex(/^oac_[A-Za-z0-9]+$/)
.nullable()
.default("oac_jkJx3wOqOMTpj4TqJJcwA4l8")
.transform(value => value ?? undefined),
),
// OAuth token introspection
OAUTH_INTROSPECT_URL: z.string().optional(),
OAUTH_INTROSPECT_SECRET: z.string().optional(),
MCP_ACTION_LOG_SECRET: z.string().optional(),
MCP_ACTION_LOG_STORAGE_ENABLED: z.stringbool().default(false),
MCP_ACTION_LOG_WRITES_ENABLED: z.stringbool().default(false),
// Autumn feature id of the balance hosted MCP calls spend first (see
// AutumnService.trackHostedMcpCredits).
MCP_CREDITS_FEATURE_ID: emptyStringAsDefault(
z.string().trim().min(1).default("MCP_CREDITS"),
),
// Agent auth discovery (RFC 9728 WWW-Authenticate on 401)
AGENT_AUTH_RESOURCE_METADATA_URL: z
.url()
.default("https://www.firecrawl.dev/.well-known/oauth-protected-resource"),
// Database & Storage
POSTGRES_HOST: z.string().default("localhost"),
POSTGRES_PORT: z.string().default("5432"),
POSTGRES_DB: z.string().default("postgres"),
POSTGRES_USER: z.string().default("postgres"),
POSTGRES_PASSWORD: z.string().default("postgres"),
DATABASE_URL: z.string().optional(),
DATABASE_REPLICA_URL: z.string().optional(),
INDEX_DATABASE_URL: z.string().optional(),
// Pool sizing preset for this process (see db/pool-profiles.ts). Unset keeps
// the historical pool settings; deployments opt into `api`, `worker` or
// `utility` to keep connections warm within the pooler's client budget.
DB_POOL_PROFILE: emptyStringAsUndefined(z.enum(["api", "worker", "utility"])),
INDEX_CACHE_REDIS_URL: z.string().optional(),
// Negative (miss) caching TTL for index URL->id lookups, in ms. 0 disables
// it; the cache then only shields lookups that find data. A positive value
// (e.g. 600000 = 10min) also short-circuits repeat lookups for URLs with no
// index entry. Kept short so any missed cache-clear self-heals quickly.
INDEX_CACHE_NEGATIVE_TTL_MS: z.coerce.number().default(0),
// `parsers: [{ type: "pdf", refresh: true }]` skips URL index reads, so the
// request reaches the PDF engine and its content-cache refresh budget
// instead of being answered with the indexed copy of the old parse.
INDEX_SKIP_ON_PDF_REFRESH: z.stringbool().default(false),
// On a URL index hit for a PDF whose entry recorded its fire-pdf content
// key, ask the fire-pdf cache service whether it holds a newer parse; if it
// does, the index copy is treated as a miss so the request is answered from
// the newer entry (no parse) and re-indexed. Needs FIRE_PDF_CACHE_BASE_URL.
INDEX_PDF_SUPERSEDE_CHECK: z.stringbool().default(false),
REDIS_URL: z.string().optional(),
REDIS_EVICT_URL: z.string().optional(),
REDIS_RATE_LIMIT_URL: z.string().optional(),
SPUR_REDIS_URL: z.string().optional(),
NUQ_DATABASE_URL: z.string().optional(),
NUQ_DATABASE_URL_LISTEN: z.string().optional(),
NUQ_RABBITMQ_URL: z.string().optional(),
FDB_CLUSTER_FILE: emptyStringAsUndefined(z.string()),
NUQ_BACKEND: emptyStringAsUndefined(z.enum(["pg", "fdb"])),
NUQ_FDB_READY_SHARDS: emptyStringAsDefault(
z.coerce.number().int().positive().default(2048),
),
// 1 = strict (priority, FIFO) promotion order per team; raise for teams with
// extreme finish rates at the cost of approximate cross-shard ordering
NUQ_FDB_TEAM_PENDING_SHARDS: emptyStringAsDefault(
z.coerce.number().int().positive().default(1),
),
NUQ_FDB_TIME_BUCKETS: emptyStringAsDefault(
z.coerce.number().int().positive().default(16),
),
// Google Cloud Storage
GCS_BUCKET_NAME: z.string().optional(),
GCS_CREDENTIALS: z.string().optional(),
GCS_FIRE_ENGINE_BUCKET_NAME: z.string().optional(),
GCS_INDEX_BUCKET_NAME: z.string().optional(),
GCS_MEDIA_BUCKET_NAME: z.string().optional(),
GCS_SCREENSHOT_RESIGN_BEFORE: emptyStringAsUndefined(z.string().datetime()),
GCS_PARSE_UPLOAD_BUCKET_NAME: z.string().optional(),
PARSE_UPLOAD_STORAGE_DRIVER: z.enum(["local", "gcs"]).optional(),
PARSE_UPLOAD_REF_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),
PARSE_UPLOAD_PUBLIC_BASE_URL: z.string().url().optional(),
// Google Cloud Pub/Sub
PUBSUB_CREDENTIALS: z.string().optional(),
// Publishes with ambient credentials when PUBSUB_CREDENTIALS is unset.
PUBSUB_PROJECT_ID: emptyStringAsUndefined(z.string().trim().min(1)),
// Prepended to every log topic name. Production leaves it unset and
// publishes to `<table>`; staging sets `staging-` so its rows land in the
// `staging-<table>` topics and the staging ClickHouse database instead of
// the production tables.
PUBSUB_TOPIC_PREFIX: z.string().default(""),
// Publisher backlog cap, per process. Log publishing is fire-and-forget and
// retries for up to five minutes, so during a stall the backlog is what
// grows; rows beyond the cap are dropped and counted rather than letting a
// hung channel take the process down.
PUBSUB_MAX_OUTSTANDING_MESSAGES: z.coerce
.number()
.int()
.positive()
.default(10_000),
PUBSUB_MAX_OUTSTANDING_BYTES: z.coerce
.number()
.int()
.positive()
.default(64 * 1024 * 1024),
// Cloud Bigtable operational stores. The client
// auto-detects BIGTABLE_EMULATOR_HOST, so local dev only needs the
// emulator plus these vars. BIGTABLE_CREDENTIALS mirrors
// GCS_CREDENTIALS: base64-encoded service-account JSON; unset falls
// back to Application Default Credentials.
BIGTABLE_PROJECT_ID: z.string().optional(),
BIGTABLE_INSTANCE_ID: z.string().optional(),
BIGTABLE_APP_PROFILE_ID: z.string().optional(),
BIGTABLE_CHANGE_TRACKING_TABLE: z.string().optional(),
BIGTABLE_JOB_ACCESS_TABLE: z.string().optional(),
BIGTABLE_FEEDBACK_JOBS_TABLE: z.string().optional(),
BIGTABLE_SCRAPE_STATE_TABLE: z.string().optional(),
BIGTABLE_EXTRACT_STATE_TABLE: z.string().optional(),
BIGTABLE_REQUEST_CREDITS_TABLE: z.string().optional(),
BIGTABLE_CREDENTIALS: z.string().optional(),
// ClickHouse (Search Analytics)
CLICKHOUSE_ANALYTICS_URL: z.string().optional(),
CLICKHOUSE_ANALYTICS_DATABASE: z.string().optional(),
// Search highlights: highlighter service base URL. TOKEN is optional
// bearer auth for legacy/external services; the in-cluster service omits it.
HIGHLIGHT_MODEL_URL: z.string().optional(),
HIGHLIGHT_MODEL_TOKEN: z.string().optional(),
// Stable percentage of non-MCP/CLI cohorts whose generated highlights are
// returned. The remaining eligible traffic still runs in shadow mode.
HIGHLIGHT_ROLLOUT_PERCENT: z.coerce.number().min(0).max(100).default(0),
// TypeSafe (Jev): judges search results for the `safe: true` filter.
TYPESAFE_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),
// Alexandria tool discovery: rewrite search queries with Anthropic before
// tool lookup, and add tools to default searches (also the alexandriaAutoTools
// team flag).
ANTHROPIC_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),
ALEXANDRIA_AUTO_TOOLS: z.stringbool().default(false),
// Rank rewrite-found tools together with raw-query tools by reciprocal rank
// fusion (also the alexandriaToolFusion team flag), waiting up to
// ALEXANDRIA_TOOL_FUSION_WAIT_MS past the raw lookup for them. On by default;
// set to false to append rewrite tools after the raw ranking instead.
ALEXANDRIA_TOOL_FUSION: z.stringbool().default(true),
ALEXANDRIA_TOOL_FUSION_WAIT_MS: z.coerce
.number()
.int()
.min(0)
.max(5000)
.default(1000),
// Exchange (routed data sources service)
FIRE_EXCHANGE_URL: z.url().optional(),
EXCHANGE_INTERNAL_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),
// Fire Engine
FIRE_ENGINE_BETA_URL: z.string().optional(),
FIRE_ENGINE_STAGING_URL: z.string().optional(),
FIRE_ENGINE_AB_URL: z.string().optional(),
FIRE_ENGINE_AB_RATE: z.coerce.number().optional(),
FIRE_ENGINE_AB_MODE: z.enum(["mirror", "split"]).default("mirror"),
// Indexer
INDEXER_RABBITMQ_URL: z.string().optional(),
INDEXER_TRAFFIC_SHARE: z.coerce.number().default(0.0),
// ScrapeURL
SCRAPEURL_AB_HOST: z.string().optional(),
SCRAPEURL_AB_RATE: z.coerce.number().optional(),
SCRAPEURL_AB_EXTEND_MAXAGE: z.stringbool().optional(),
SCRAPEURL_ENGINE_WATERFALL_DELAY_MS: z.coerce.number().default(0),
// Scrape Retry Limits
SCRAPE_MAX_ATTEMPTS: z.coerce.number().int().positive().default(6),
SCRAPE_MAX_FEATURE_TOGGLES: z.coerce.number().int().positive().default(3),
SCRAPE_MAX_FEATURE_REMOVALS: z.coerce.number().int().positive().default(3),
SCRAPE_MAX_PDF_PREFETCHES: z.coerce.number().int().positive().default(2),
SCRAPE_MAX_DOCUMENT_PREFETCHES: z.coerce.number().int().positive().default(2),
// Max concurrent native PDF extractions per process. Each extraction holds
// the (≤50MB) PDF plus its parsed text/markdown in memory on a tokio blocking
// thread; unbounded concurrency OOM-killed pods when several large PDFs
// landed on one process at once. 3 keeps the worst-case transient memory
// (~3 × ~0.5GB) well under the 8G limit while being far above typical demand
// (per-pod average concurrency is ~0.02).
PDF_EXTRACTION_CONCURRENCY: z.coerce.number().int().positive().default(3),
// Search Services
SEARXNG_ENDPOINT: z.string().optional(),
SEARXNG_ENGINES: z.string().optional(),
SEARXNG_CATEGORIES: z.string().optional(),
SEARCH_SERVICE_URL: z.string().optional(),
SEARCH_INDEX_SAMPLE_RATE: z.coerce.number().default(0.1),
ENABLE_SEARCH_INDEX: z.stringbool().optional(),
// Worker Configuration
WORKER_PORT: z.coerce.number().default(3005),
NUQ_WORKER_PORT: z.coerce.number().default(3000).catch(3000), // todo: investigate why .catch is needed
NUQ_WORKER_START_PORT: z.coerce.number().default(3006),
NUQ_WORKER_COUNT: z.coerce.number().default(5),
NUQ_PREFETCH_WORKER_PORT: z.coerce.number().default(3011).catch(3011), // todo: investigate why .catch is needed
NUQ_RECONCILER_WORKER_PORT: z.coerce.number().default(3012).catch(3012),
CCLOG_WORKER_PORT: z.coerce.number().default(3013).catch(3013),
EXTRACT_WORKER_PORT: z.coerce.number().default(3004),
NUQ_WAIT_MODE: z.string().optional(),
// Harness Configuration
HARNESS_STARTUP_TIMEOUT_MS: z.coerce.number().default(60000),
// Job & Lock Management
JOB_LOCK_EXTEND_INTERVAL: z.coerce.number().default(10000),
JOB_LOCK_EXTENSION_TIME: z.coerce.number().default(60000),
WORKER_LOCK_DURATION: z.coerce.number().default(60000),
WORKER_STALLED_CHECK_INTERVAL: z.coerce.number().default(30000),
CONNECTION_MONITOR_INTERVAL: z.coerce.number().default(10),
CANT_ACCEPT_CONNECTION_INTERVAL: z.coerce.number().default(2000),
// Proxy
PROXY_SERVER: z.string().optional(),
PROXY_USERNAME: z.string().optional(),
PROXY_PASSWORD: z.string().optional(),
// External Services
PLAYWRIGHT_MICROSERVICE_URL: z.string().optional(),
HTML_TO_MARKDOWN_SERVICE_URL: z.string().optional(),
SMART_SCRAPE_API_URL: z.string().optional(),
// PDF Processing
PDF_MU_V2_BASE_URL: z.string().optional(),
PDF_MU_V2_API_KEY: z.string().optional(),
PDF_MU_V2_EXPERIMENT: z.string().optional(),
PDF_MU_V2_EXPERIMENT_PERCENT: z.coerce.number().default(100),
// MinerU direct routing (bypass Rust extraction for a % of traffic)
MINERU_PERCENT: z.coerce.number().min(0).max(100).default(0),
// Fire PDF (replaces MinerU for a % of traffic)
FIRE_PDF_ENABLE: z.stringbool().optional(),
FIRE_PDF_PERCENT: z.coerce.number().min(0).max(100).default(10),
FIRE_PDF_BASE_URL: z.string().optional(),
FIRE_PDF_API_KEY: z.string().optional(),
// Cached fire-pdf results are looked up through this service when set
// (POST /cache/lookup, same key as FIRE_PDF_API_KEY), and fire-pdf writes
// them; without it the bucket is read and written from here.
FIRE_PDF_CACHE_BASE_URL: z.string().optional(),
// `parsers: [{ type: "pdf", refresh: true }]` skips the content cache and
// forces a fresh parse. Per team, per minute, budgeted here or by the
// cache service when one is configured; beyond the budget the request is
// served normally. 0 disables the option.
FIRE_PDF_CACHE_REFRESH_PER_MINUTE: z.coerce.number().int().min(0).default(10),
// Opt in to background refresh: lookups carry `background_refresh: true`,
// so the cache service can answer a refresh-campaign match with the stale
// entry instead of a miss. When it grants a refresh (`refresh_requested`),
// the stale result is served and the bytes are submitted to fire-pdf's bulk
// queue for a fresh parse, which fire-pdf writes back to the cache. Needs
// FIRE_PDF_CACHE_BASE_URL and FIRE_PDF_BASE_URL.
FIRE_PDF_CACHE_BACKGROUND_REFRESH: z.stringbool().default(false),
// /v2/parse sends every PDF (auto/ocr mode) to FirePDF: no direct
// text-layer extraction, no FIRE_PDF_PERCENT sampling, no MinerU
// diversion. Explicit `mode: "fast"` keeps direct extraction. FirePDF
// failures still fall back to MinerU / pdf-parse. Needs FIRE_PDF_ENABLE.
PARSE_PDF_ALWAYS_FIRE_PDF: z.stringbool().default(true),
// Table rendering FirePDF applies to /v2/parse documents. `dynamic` keeps
// pipe tables where they are lossless and HTML where structure needs it
// (merged cells, large tables); `off` sends no format (pipe tables).
// Any explicit format puts the request outside the FirePDF content cache.
PARSE_FIRE_PDF_TABLE_FORMAT: z
.enum(["off", "markdown", "html", "dynamic"])
.default("dynamic"),
// Raster image OCR of image URLs and parse uploads through FirePDF (see
// lib/image-ocr-gate.ts). Needs FIRE_PDF_BASE_URL.
IMAGE_OCR_ENABLED: z.stringbool().default(false),
// Async /jobs rollout is a separate, server-controlled cohort inside
// traffic already selected for FirePDF. It is disabled by default.
FIRE_PDF_ASYNC_PERCENT: z.coerce.number().min(0).max(100).default(0),
// Separate cohort for crawl/batch-originated scrapes (any scrape
// carrying a crawlId): no caller waits on one specific document, so
// these can ramp onto the async lane ahead of interactive traffic.
FIRE_PDF_ASYNC_BULK_ORIGIN_PERCENT: z.coerce
.number()
.min(0)
.max(100)
.default(0),
FIRE_PDF_ASYNC_FORCE_TEAM_IDS: z.string().optional(),
FIRE_PDF_ASYNC_DISABLE_TEAM_IDS: z.string().optional(),
FIRE_PDF_ASYNC_ALLOW_REQUEST_OVERRIDE: z.stringbool().default(false),
// Long-poll wait sent as `wait_ms` on GET /jobs/:id: fire-pdf holds the
// request until the job is terminal or the wait elapses, so completion is
// seen as it happens instead of at the next scheduled poll. 0 disables
// it; values above fire-pdf's 25s cap are clamped to it.
FIRE_PDF_ASYNC_WAIT_MS: z.coerce.number().int().min(0).default(0),
// Large-PDF by-reference submits (30-256MB files uploaded to GCS and
// handed to fire-pdf via `input_gcs_uri`). This is an explicit on/off
// switch, not a percentage: no alternative engine exists at this size,
// so there is no cohort to sample "out" — only text-only degradation.
// FIRE_PDF_ENABLE remains the master switch for both paths.
FIRE_PDF_BY_REFERENCE_ENABLE: z.stringbool().default(true),
// Bucket that receives large-PDF inputs for by-reference async submits
// (fire-pdf reads them back via `input_gcs_uri`). fire-pdf only accepts
// URIs inside its own configured bucket + `inputs/` prefix, so this must
// match fire-pdf's FIRE_PDF_GCS_BUCKET. Upload failures (e.g. missing
// IAM grant) fall back to the pre-by-reference behavior for oversized
// files rather than failing the scrape.
FIRE_PDF_GCS_INPUT_BUCKET: z
.string()
.trim()
.min(1)
.default("firecrawl-pdf-pipeline"),
// Bucket fire-engine uses for its large-PDF handoff (files too big to
// inline as base64 in its response). Acts as the allowlist for inbound
// `file.gcs_uri` references — objects outside it are never fetched or
// copied. Deliberately no default: this is a security-sensitive inbound
// allowlist, so consuming references requires explicit opt-in (set to
// fire-engine's GCS_PDF_BUCKET_NAME); unset disables the path.
FIRE_ENGINE_PDF_GCS_BUCKET: emptyStringAsUndefined(z.string().trim().min(1)),
// Large-PDF size policy, applied per team on every acquisition path
// (direct download, fire-engine handoff, by-reference submit) and sent to
// fire-engine as the per-request pdfMaxSize. The default applies to every
// team; ids on the allowlist get the privileged cap. Both are clamped to
// the 256MB architectural ceiling.
PDF_BY_REFERENCE_MAX_BYTES_DEFAULT: z.coerce
.number()
.int()
.positive()
.default(50 * 1024 * 1024),
PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED: z.coerce
.number()
.int()
.positive()
.default(256 * 1024 * 1024),
// Comma-separated team ids granted the privileged cap. Prefer the
// `largePdfs` team flag, which grants the same cap without a deploy.
PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS: z.string().optional(),
// RunPod
RUNPOD_MU_API_KEY: z.string().optional(),
RUNPOD_MU_POD_ID: z.string().optional(),
// PDF Rust Extraction (pdf-inspector)
PDF_RUST_EXTRACT_ENABLE: z.stringbool().optional(),
PDF_SHADOW_COMPARISON_ENABLE: z.stringbool().optional(),
// Webhooks
SELF_HOSTED_WEBHOOK_URL: z.string().optional(),
SELF_HOSTED_WEBHOOK_HMAC_SECRET: z.string().optional(),
SLACK_WEBHOOK_URL: z.string().optional(),
SLACK_ADMIN_WEBHOOK_URL: z.string().optional(),
DISABLE_WEBHOOK_DELIVERY: z.stringbool().optional(),
// Slack integration ("Add to Slack" for monitor notifications + /monitor
// slash command). Credentials come from the Firecrawl Slack app.
SLACK_CLIENT_ID: z.string().optional(),
SLACK_CLIENT_SECRET: z.string().optional(),
SLACK_SIGNING_SECRET: z.string().optional(),
// Bot scopes requested during install. Override only if the Slack app manifest
// changes; keep in sync with slack-app-manifest.json.
SLACK_OAUTH_SCOPES: z
.string()
.default(
"chat:write,chat:write.public,commands,channels:read,groups:read,team:read,incoming-webhook",
),
// Absolute URL Slack redirects back to after authorize. Must exactly match a
// Redirect URL configured on the Slack app (e.g.
// https://api.firecrawl.dev/v2/slack/oauth/callback).
SLACK_OAUTH_REDIRECT_URL: z.string().optional(),
// 32-byte key (hex or base64) used to AES-256-GCM encrypt stored bot tokens.
// Required when ENV=production. Elsewhere, if unset, tokens are stored with a
// `plain:` prefix.
SLACK_TOKEN_ENCRYPTION_KEY: z.string().optional(),
ALLOW_LOCAL_WEBHOOKS: z.stringbool().optional(),
WEBHOOK_USE_RABBITMQ: z.stringbool().optional(),
// Firecrawl Features
FIRECRAWL_DEBUG_FILTER_LINKS: z.stringbool().optional(),
FIRECRAWL_LOG_TO_FILE: z.stringbool().optional(),
FIRECRAWL_SAVE_MOCKS: z.stringbool().optional(),
FIRECRAWL_INDEX_WRITE_ONLY: z.stringbool().optional(),
DISABLE_BLOCKLIST: z.stringbool().optional(),
FORCED_ENGINE_DOMAINS: z.string().optional(),
DEBUG_BRANDING: z.stringbool().optional(),
// TypeSafe Jev answers branding decisions whenever TYPESAFE_API_KEY is set
// (lib/branding/jev.ts), with the LLM as fallback. BRANDING_JEV=false sends
// every branding request to the LLM instead.
BRANDING_JEV: emptyStringAsUndefined(z.stringbool()),
// Share of teams (0-100, stable per team) whose branding scrapes also
// capture the page's first screen for Jev, as if they had asked for it with
// the branding format's __experimental_screenshot. Never for ZDR scrapes.
BRANDING_JEV_SCREENSHOT_PERCENT: z.coerce.number().min(0).max(100).default(0),
BRANDING_JEV_TIMEOUT_MS: emptyStringAsUndefined(
z.coerce.number().int().positive(),
),
// Warn scrapes of search engine results pages (google.*/search, ...) that
// /search returns those results directly (lib/serp-url.ts).
// SERP_SCRAPE_WARNING turns it on for every team, SERP_SCRAPE_WARNING_TEAM_IDS
// (comma-separated) for listed teams, and SERP_SCRAPE_WARNING_ROLLOUT_PERCENT
// for a stable share of teams.
SERP_SCRAPE_WARNING: emptyStringAsUndefined(z.stringbool()),
SERP_SCRAPE_WARNING_TEAM_IDS: delimitedList(",").optional(),
SERP_SCRAPE_WARNING_ROLLOUT_PERCENT: z.coerce
.number()
.min(0)
.max(100)
.default(0),
// AI/ML
MODEL_NAME: z.string().optional(),
MODEL_EMBEDDING_NAME: z.string().optional(),
OLLAMA_BASE_URL: z.string().optional(),
VERTEX_CREDENTIALS: z.string().optional(),
// Uses Vertex with ambient credentials when VERTEX_CREDENTIALS is unset.
VERTEX_USE_ADC: z.stringbool().default(false),
// LangSmith (tracing for interact agent)
LANGSMITH_API_KEY: z.string().optional(),
LANGSMITH_PROJECT: z.string().optional(),
LANGSMITH_ENDPOINT: z.string().optional(),
LANGSMITH_TRACING: z.stringbool().optional(),
// Rate Limiting
RATE_LIMIT_TEST_API_KEY_SCRAPE: z.coerce.number().optional(),
RATE_LIMIT_TEST_API_KEY_CRAWL: z.coerce.number().optional(),
// Testing
TEST_API_KEY: z.string().optional(),
TEST_API_URL: z.string().default("http://127.0.0.1:3002"),
TEST_TEAM_ID: z.string().optional(),
TEST_SUITE_SELF_HOSTED: z.stringbool().optional(),
TEST_SUITE_WEBSITE: z.string().default("http://127.0.0.1:4321"),
USE_DB_AUTHENTICATION: z.stringbool().optional(),
// Indexing
BACKGROUND_INDEX_TEAM_ID: z.string().optional(),
PRECRAWL_TEAM_ID: z.string().optional(),
// System
MAX_CPU: z.coerce.number().default(0.8),
MAX_RAM: z.coerce.number().default(0.8),
SYS_INFO_MAX_CACHE_DURATION: z.coerce.number().default(150),
USE_GO_MARKDOWN_PARSER: z.stringbool().optional(),
SENTRY_ENVIRONMENT: z.string().default("production"),
// OpenTelemetry. Tracing is off unless an OTLP endpoint is set; spans are then
// exported over http/protobuf at 100% sampling, and the SDK honors the
// standard OTEL_EXPORTER_OTLP_* / OTEL_BSP_* / OTEL_RESOURCE_ATTRIBUTES
// variables. Zero-data-retention spans are never exported (see otel-tracer).
OTEL_EXPORTER_OTLP_ENDPOINT: emptyStringAsUndefined(z.string().url()),
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: emptyStringAsUndefined(z.string().url()),
OTEL_SERVICE_NAME: emptyStringAsUndefined(z.string()),
NUQ_POD_NAME: z.string().default("main"),
// Billing
AUTO_RECHARGE_ENABLED: z.stringbool().default(false),
// firebill — durable usage-event store that sits in front of Autumn. Every
// billing write (check, lock, track, refund, finalize) goes through it;
// without both URL and SECRET set, billing is skipped.
FIREBILL_URL: emptyStringAsUndefined(z.string().url()),
FIREBILL_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),
// How long "this team is not partner-provisioned" is trusted. Only the
// negative is bounded: provisioning is one-way, so a positive cannot go
// stale, while a negative does the moment a partner provisions an account.
// Capped at firebill's own gateway lookup TTL (300s) — the two sides answer
// the same question, and trusting a negative for longer than firebill trusts
// an answer turns this cache back into the stale allowlist it replaced. 0
// disables caching negatives entirely.
FIREBILL_GATEWAY_NEGATIVE_TTL_SECONDS: z.coerce
.number()
.int()
.min(0)
.max(300)
.default(60),
// Miscellaneous
// The commit the image was built from, set by the Dockerfile's GIT_SHA.
FIRECRAWL_BUILD_SHA: emptyStringAsUndefined(z.string().trim().min(1)),
IDMUX_URL: z.string().optional(),
GITHUB_RUN_NUMBER: z.string().optional(),
GITHUB_REF_NAME: z.string().optional(),
RESTRICTED_COUNTRIES: delimitedList(",").optional(),
DISABLE_ENGPICKER: z.stringbool().optional(),
DISABLE_MONITORING: z.stringbool().default(false),
// Stamp an inferred `integration` on unlabelled direct callers.
INFER_INTEGRATION_ENABLED: z.stringbool().default(true),
EXTRACT_V3_BETA_URL: z.string().url().optional(),
AGENT_INTEROP_SECRET: z
.string()
.refine(value => value.trim().length > 0, {
error: "AGENT_INTEROP_SECRET must not be blank",
})
.refine(value => !containsLoneSurrogate(value), {
error: "AGENT_INTEROP_SECRET must not contain lone surrogates",
})
.optional(),
// Wikipedia Enterprise API
WIKIPEDIA_ENTERPRISE_USERNAME: z.string().optional(),
WIKIPEDIA_ENTERPRISE_PASSWORD: z.string().optional(),
// Browser Service
HANGAR_URL: z.url().optional(),
// Audio (avgrab)
AVGRAB_SERVICE_URL: z.string().optional(),
// Product extraction (product-search Rust service)
PRODUCT_EXTRACTION_SERVICE_URL: z.string().optional(),
// Menu extraction (menu-search Rust service)
MENU_EXTRACTION_SERVICE_URL: z.string().optional(),
// PII Redaction (fire-privacy)
FIRE_PRIVACY_URL: z.string().optional(),
FIRE_PRIVACY_TIMEOUT_MS: z.coerce.number().int().positive().default(5000),
NUQ_PREFETCH_WORKER_HEARTBEAT_URL: z.string().optional(),
ZDRCLEANER_HEARTBEAT_URL: z.string().optional(),
CCLOG_WORKER_HEARTBEAT_URL: z.string().optional(),
// Deterministic JSON extraction (reusable-json-mode)
EXTRACT_CODEGEN_MODEL: z.string().default("gemini-3.1-flash-lite"),
EXTRACT_ANCHOR_MODEL: z.string().default("openai/gpt-oss-120b"),
EXTRACT_LIGHT_MODEL: z.string().default("openai/gpt-oss-20b"),
CODE_SANDBOX_URL: z.string().default("ws://code-sandbox:3001"),
});
const validatedConfigSchema = configSchema.superRefine((value, context) => {
if (value.GOV_ROUTING_ENABLED && !value.SEARCH_PLATFORM_URL) {
context.addIssue({
code: "custom",
path: ["GOV_ROUTING_ENABLED"],
message:
"GOV_ROUTING_ENABLED needs SEARCH_PLATFORM_URL: set it, or turn routing off",
});
}
for (const error of getMcpActionLogConfigErrors(value)) {
context.addIssue({
code: "custom",
path: [error.path],
message: error.message,
});
}
});
export const config = validatedConfigSchema.parse(process.env);