Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

chore: updates (#3787)

GergΕ‘ MΓ³ricz722d02f64c1e
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.ts
new file mode 100644
index 000000000..cbc494f28
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts
@@ -0,0 +1,312 @@
+import { config } from "../../../config";
+import {
+ describeIf,
+ itIf,
+ TEST_API_URL,
+ TEST_PRODUCTION,
+ TEST_SUITE_WEBSITE,
+ scrapeTimeout,
+} from "../lib";
+import { redisRateLimitClient } from "../../../services/rate-limiter";
+import request from "supertest";
+
+// The keyless tier is disabled unless both limits are configured. The harness
+// passes the shell env through to the server, so we read the same env here and
+// only run when it's set (and use the configured values for cap assertions).
+const KEYLESS_REQUESTS_PER_DAY = Number(process.env.KEYLESS_REQUESTS_PER_DAY);
+const KEYLESS_CREDITS_PER_DAY = Number(process.env.KEYLESS_CREDITS_PER_DAY);
+const KEYLESS_ENABLED =
+ TEST_PRODUCTION &&
+ Number.isFinite(KEYLESS_REQUESTS_PER_DAY) &&
+ KEYLESS_REQUESTS_PER_DAY > 0 &&
+ Number.isFinite(KEYLESS_CREDITS_PER_DAY) &&
+ KEYLESS_CREDITS_PER_DAY > 0;
+
+// Keyless free tier: scrape, search, and interact work without an API key from
+// the official MCP server (origin "mcp*"), CLI (integration "cli"), or SDKs
+// (origin "<lang>-sdk@..."). Gated per-IP/day by a request cap AND a credit cap.
+// These tests are the only ones that exercise keyless mode, so they exclusively
+// own the `keyless_*` rate-limit keyspace β€” we flush it before each test (and run
+// sequentially) for isolation. `req.ip` is loopback under supertest, so the
+// per-IP counters are shared across requests here; flushing keeps tests hermetic.
+
+async function flushKeylessBuckets() {
+ const keys = await redisRateLimitClient.keys("keyless_*");
+ if (keys.length > 0) {
+ await redisRateLimitClient.del(...keys);
+ }
+}
+
+// Recover the loopback IP the server keyed on, so we can seed its credit counter.
+async function currentKeylessIp(): Promise<string> {
+ const keys = await redisRateLimitClient.keys("keyless_requests:*");
+ expect(keys.length).toBeGreaterThan(0);
+ return keys[0].slice("keyless_requests:".length);
+}
+
+describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
+ beforeAll(() => {
+ config.USE_DB_AUTHENTICATION = true;
+ });
+
+ afterAll(async () => {
+ delete config.USE_DB_AUTHENTICATION;
+ await flushKeylessBuckets();
+ });
+
+ beforeEach(async () => {
+ await flushKeylessBuckets();
+ });
+
+ it(
+ "allows keyless scrape from a raw API caller (no origin gate) (200)",
+ async () => {
+ // origin "api" (and no origin at all) is now eligible β€” the API itself is
+ // free without a key on the allowlisted endpoints.
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ url: TEST_SUITE_WEBSITE, origin: "api", formats: ["markdown"] });
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "allows keyless scrape with no origin field at all (200)",
+ async () => {
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ url: TEST_SUITE_WEBSITE, formats: ["markdown"] });
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ },
+ scrapeTimeout,
+ );
+
+ it("does not grant keyless access on non-allowlisted endpoints (401)", async () => {
+ // batch/scrape shares RateLimiterMode.Scrape but is NOT allowKeyless.
+ const response = await request(TEST_API_URL)
+ .post("/v2/batch/scrape")
+ .set("Content-Type", "application/json")
+ .send({ urls: ["https://example.com"], origin: "[email protected]" });
+
+ expect(response.statusCode).toBe(401);
+ expect(response.body.success).toBe(false);
+ expect(response.body.error).toContain("not available without an API key");
+ });
+
+ it("enforces the daily request cap with the request signup message (429)", async () => {
+ // Each request consumes a slot during auth, then fails fast in the controller
+ // (no url) β€” so we exhaust the cap without real scrapes, and 0 credits.
+ for (let i = 0; i < KEYLESS_REQUESTS_PER_DAY; i++) {
+ const r = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ origin: "mcp" });
+ expect(r.statusCode).not.toBe(429);
+ }
+
+ const blocked = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ origin: "mcp" });
+
+ expect(blocked.statusCode).toBe(429);
+ expect(blocked.body.error).toContain("unauthenticated requests");
+ // Out of quota β†’ emit the OAuth-discovery header so agents find the key flow.
+ expect(blocked.headers["www-authenticate"]).toContain("resource_metadata");
+ });
+
+ it("enforces the daily credit cap with the credit signup message (429)", async () => {
+ // Materialize the loopback IP, then seed its credit counter to the cap.
+ await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ origin: "mcp" });
+ const ip = await currentKeylessIp();
+ await redisRateLimitClient.set(
+ `keyless_credits:${ip}`,
+ String(KEYLESS_CREDITS_PER_DAY),
+ );
+
+ const blocked = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ origin: "mcp" });
+
+ expect(blocked.statusCode).toBe(429);
+ expect(blocked.body.error).toContain("unauthenticated credits");
+ });
+
+ itIf(!!process.env.KEYLESS_PROXY_SECRET)(
+ "rate-limits per forwarded client IP when the proxy secret is provided",
+ async () => {
+ const fakeIp = "203.0.113.7";
+ // Seed the forwarded IP's credit counter to the cap.
+ await redisRateLimitClient.set(
+ `keyless_credits:${fakeIp}`,
+ String(KEYLESS_CREDITS_PER_DAY),
+ );
+
+ // With the secret, the cap is keyed on the forwarded IP -> blocked.
+ const blocked = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)
+ .set("x-firecrawl-keyless-ip", fakeIp)
+ .send({ origin: "mcp" });
+ expect(blocked.statusCode).toBe(429);
+ expect(blocked.body.error).toContain("unauthenticated credits");
+
+ // Without the secret, the forwarded IP is ignored (keyed on the real IP).
+ const allowed = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .set("x-firecrawl-keyless-ip", fakeIp)
+ .send({ origin: "mcp" });
+ expect(allowed.statusCode).not.toBe(429);
+ },
+ );
+
+ itIf(!!process.env.KEYLESS_PROXY_SECRET)(
+ "denies keyless access to IPv6 clients (401)",
+ async () => {
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)
+ .set("x-firecrawl-keyless-ip", "2001:db8::1")
+ .send({ url: TEST_SUITE_WEBSITE, origin: "mcp" });
+
+ // IPv6 is not eligible β†’ falls through to the normal unauthorized path.
+ expect(response.statusCode).toBe(401);
+ },
+ );
+
+ itIf(!!process.env.KEYLESS_PROXY_SECRET)(
+ "denies keyless for a malformed forwarded IP (401)",
+ async () => {
+ // A non-IP forwarded value must not be usable as a limiter bucket.
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)
+ .set("x-firecrawl-keyless-ip", "not-an-ip")
+ .send({ url: TEST_SUITE_WEBSITE, origin: "mcp" });
+
+ expect(response.statusCode).toBe(401);
+ },
+ );
+
+ itIf(!!process.env.KEYLESS_PROXY_SECRET)(
+ "keyless eligibility endpoint reflects cap state (hosted MCP probe)",
+ async () => {
+ const ip = "203.0.113.40";
+
+ // Fresh IP under cap β†’ eligible.
+ const ok = await request(TEST_API_URL)
+ .get("/v2/keyless/eligibility")
+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)
+ .set("x-firecrawl-keyless-ip", ip);
+ expect(ok.statusCode).toBe(200);
+ expect(ok.body.eligible).toBe(true);
+
+ // Seed over the credit cap β†’ ineligible (so the MCP would issue an OAuth
+ // challenge instead of serving keyless).
+ await redisRateLimitClient.set(
+ `keyless_credits:${ip}`,
+ String(KEYLESS_CREDITS_PER_DAY),
+ );
+ const capped = await request(TEST_API_URL)
+ .get("/v2/keyless/eligibility")
+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)
+ .set("x-firecrawl-keyless-ip", ip);
+ expect(capped.body.eligible).toBe(false);
+
+ // Without the secret β†’ rejected (no leaking eligibility to untrusted callers).
+ const noSecret = await request(TEST_API_URL)
+ .get("/v2/keyless/eligibility")
+ .set("x-firecrawl-keyless-ip", ip);
+ expect(noSecret.statusCode).toBe(401);
+ },
+ );
+
+ it(
+ "grants keyless interact access (past auth, not 401)",
+ async () => {
+ // A random (non-existent) job id: the request should clear keyless auth and
+ // fail downstream, proving interact is allowKeyless β€” not return 401.
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape/00000000-0000-4000-8000-000000000000/interact")
+ .set("Content-Type", "application/json")
+ .send({ code: "return 1;", origin: "mcp" });
+
+ expect(response.statusCode).not.toBe(401);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "allows keyless scrape from an mcp origin (200)",
+ async () => {
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ url: TEST_SUITE_WEBSITE, origin: "mcp" });
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(typeof response.body.data).toBe("object");
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "allows keyless scrape from a cli integration (200)",
+ async () => {
+ // The CLI identifies itself via the `integration` field, not `origin`.
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ url: TEST_SUITE_WEBSITE, integration: "cli" });
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "allows keyless scrape from an SDK origin (200)",
+ async () => {
+ // SDKs send origin like "[email protected]" / "[email protected]".
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ url: TEST_SUITE_WEBSITE, origin: "[email protected]" });
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "allows keyless search from an mcp origin (200)",
+ async () => {
+ const response = await request(TEST_API_URL)
+ .post("/v2/search")
+ .set("Content-Type", "application/json")
+ .send({ query: "firecrawl", limit: 1, origin: "mcp" });
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ },
+ scrapeTimeout,
+ );
+});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index ca83f0f68..06abdadf7 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -50,6 +50,15 @@ const configSchema = z.object({
// Express
EXPRESS_TRUST_PROXY: z.coerce.number().optional(),
+ // Keyless free tier (scrape/search/interact without an API key, per-IP/day).
+ // Non-negative integers; 0 means "enabled but no budget", unset means "off".
+ KEYLESS_CREDITS_PER_DAY: z.coerce.number().int().nonnegative().optional(),
+ KEYLESS_REQUESTS_PER_DAY: z.coerce.number().int().nonnegative().optional(),
+ // Shared secret that lets a trusted proxy (e.g. the hosted MCP server)
+ // forward the real client IP for keyless rate-limiting via the
+ // `x-firecrawl-keyless-ip` header. Untrusted callers can't override their IP.
+ KEYLESS_PROXY_SECRET: z.string().optional(),
+
// API Keys & Authentication
BULL_AUTH_KEY: z.string().optional(),
OPENAI_API_KEY: z.string().optional(),
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index 0e43c324b..0f6bdfc5e 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -8,6 +8,12 @@ import { withAuth } from "../lib/withAuth";
import { getAgentSponsorStatus } from "../services/agent-sponsor";
import { getRedisConnection } from "../services/queue-service";
import { getRateLimiter } from "../services/rate-limiter";
+import {
+ consumeKeylessRequest,
+ isKeylessConfigured,
+ isKeylessIpEligible,
+ keylessTeamId,
+} from "../lib/keyless";
import { deleteKey, getValue, setValue } from "../services/redis";
import { redlock } from "../services/redlock";
import { eq } from "drizzle-orm";
@@ -462,17 +468,151 @@ export async function clearACUCTeam(team_id: string): Promise<void> {
await getRedisConnection().sadd("billed_teams", team_id);
}
+const KEYLESS_REQUESTS_MESSAGE = `You've reached today's limit of free, unauthenticated requests to Firecrawl. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
+
+const KEYLESS_CREDITS_MESSAGE = `You've reached today's limit of free, unauthenticated credits for Firecrawl. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
+
+const KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE = `This endpoint is not available without an API key. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
+
+/**
+ * Keyless free tier: official MCP/CLI/SDK clients can call scrape, search, and
+ * interact with no API key. `origin`/`integration` are client-set and spoofable,
+ * so they're only a soft UX gate β€” the real abuse controls are the per-IP daily
+ * request + credit caps plus the `keyless/consume` canonical log emitted here.
+ * Always returns an AuthResponse β€” handles every no-API-key request.
+ */
+async function handleKeylessAuth(
+ req,
+ mode: RateLimiterMode | undefined,
+ allowKeyless: boolean | undefined,
+): Promise<AuthResponse> {
+ const unauthorized: AuthResponse = {
+ success: false,
+ error: "Unauthorized",
+ status: 401,
+ };
+
+ // The keyless tier is off unless BOTH limits are configured (even to 0). When
+ // unconfigured we behave exactly as before β€” a generic 401 β€” and don't reveal
+ // that the tier exists.
+ if (!isKeylessConfigured()) return unauthorized;
+
+ // Configured, but this endpoint isn't part of the keyless tier: tell the user
+ // they need a key (with the signup nudge) rather than a bare "Unauthorized".
+ if (!allowKeyless) {
+ return {
+ success: false,
+ error: KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE,
+ status: 401,
+ };
+ }
+
+ const origin = req.body?.origin;
+ const integration = req.body?.integration;
+ // No origin/surface gate: any request without an API key may use the free
+ // tier on the allowlisted endpoints (the API itself is free). origin and
+ // integration are still recorded below for abuse monitoring.
+
+ // Key on the real client IP. A trusted proxy (e.g. the hosted MCP) may
+ // forward the end-user's IP via x-firecrawl-keyless-ip, authenticated with a
+ // shared secret β€” without the secret the header is ignored, so direct callers
+ // can't spoof their IP to dodge the per-IP cap.
+ let ip = req.ip ?? req.socket?.remoteAddress ?? "unknown";
+ if (
+ config.KEYLESS_PROXY_SECRET &&
+ req.headers["x-firecrawl-keyless-secret"] === config.KEYLESS_PROXY_SECRET
+ ) {
+ const forwarded = req.headers["x-firecrawl-keyless-ip"];
+ if (typeof forwarded === "string" && forwarded.trim()) {
+ ip = forwarded.trim();
+ }
+ }
+
+ // Only a valid IPv4 identity gets keyless: IPv6 is too cheap to rotate for a
+ // per-IP cap to mean anything, and malformed/forwarded values must not be
+ // usable as arbitrary limiter buckets. Anything else falls through to 401.
+ if (!isKeylessIpEligible(ip)) return unauthorized;
+
+ const teamId = keylessTeamId(ip);
+ const modeLabel =
+ mode === RateLimiterMode.Search
+ ? "search"
+ : mode === RateLimiterMode.BrowserExecute
+ ? "interact"
+ : "scrape";
+
+ let result: Awaited<ReturnType<typeof consumeKeylessRequest>>;
+ try {
+ result = await consumeKeylessRequest(ip);
+ } catch (error) {
+ // Limiter store (Redis) unavailable β€” fail closed with a controlled auth
+ // response instead of surfacing a 500, and shed the free traffic while the
+ // limiter can't enforce quotas.
+ logger.warn("Keyless quota check failed", {
+ canonicalLog: "keyless/consume",
+ ip,
+ mode: modeLabel,
+ teamId,
+ error,
+ });
+ return unauthorized;
+ }
+ const baseLog = {
+ canonicalLog: "keyless/consume",
+ ip,
+ origin,
+ integration,
+ mode: modeLabel,
+ teamId,
+ requestsUsed: result.requestsUsed,
+ creditsUsed: result.creditsUsed,
+ };
+
+ if (!result.ok) {
+ logger.warn("Keyless request blocked", {
+ ...baseLog,
+ blocked: true,
+ reason: result.reason,
+ });
+ return {
+ success: false,
+ error:
+ result.reason === "credits"
+ ? KEYLESS_CREDITS_MESSAGE
+ : KEYLESS_REQUESTS_MESSAGE,
+ status: 429,
+ // Out of free quota β€” emit the OAuth-discovery header so agents can find
+ // the key/signup flow at the moment they actually need a key.
+ agentAuthDiscovery: true,
+ };
+ }
+
+ logger.debug("Keyless request consumed", { ...baseLog, blocked: false });
+
+ // Tag as a preview team so billing (autumn isPreviewTeam) and GCS persistence
+ // are skipped automatically; mockPreviewACUC supplies concurrency 2 + credits.
+ // Actual credits consumed are charged to the IP's daily budget after the
+ // request completes (see chargeKeylessCredits).
+ return {
+ success: true,
+ team_id: teamId,
+ org_id: null,
+ chunk: mockPreviewACUC(teamId, false),
+ };
+}
+
export async function authenticateUser(
req,
res,
mode?: RateLimiterMode,
+ options?: { allowKeyless?: boolean },
): Promise<AuthResponse> {
return withAuth(supaAuthenticateUser, {
success: true,
chunk: null,
team_id: "bypass",
org_id: null,
- })(req, res, mode);
+ })(req, res, mode, options);
}
/**
@@ -517,6 +657,7 @@ async function supaAuthenticateUser(
req,
res,
mode?: RateLimiterMode,
+ options?: { allowKeyless?: boolean },
): Promise<AuthResponse> {
const authHeader =
req.headers.authorization ??
@@ -524,7 +665,7 @@ async function supaAuthenticateUser(
? `Bearer ${req.headers["sec-websocket-protocol"]}`
: null);
if (!authHeader) {
- return { success: false, error: "Unauthorized", status: 401 };
+ return handleKeylessAuth(req, mode, options?.allowKeyless);
}
const token = authHeader.split(" ")[1]; // Extract the token from "Bearer <token>"
if (!token) {
diff --git a/apps/api/src/controllers/v2/keyless-eligibility.ts b/apps/api/src/controllers/v2/keyless-eligibility.ts
new file mode 100644
index 000000000..f86701a19
--- /dev/null
+++ b/apps/api/src/controllers/v2/keyless-eligibility.ts
@@ -0,0 +1,28 @@
+import { Request, Response } from "express";
+import { config } from "../../config";
+import { checkKeylessEligibility } from "../../lib/keyless";
+
+/**
+ * Internal endpoint for trusted proxies (the hosted MCP) to check, at connect
+ * time, whether a client IP can currently use the keyless tier β€” without
+ * consuming quota. Gated by the shared KEYLESS_PROXY_SECRET; the client IP is
+ * supplied via x-firecrawl-keyless-ip. Lets the MCP serve keyless when eligible
+ * and throw (β†’ OAuth 401 challenge) when the IP is out of free quota.
+ */
+export async function keylessEligibilityController(
+ req: Request,
+ res: Response,
+): Promise<void> {
+ const secret = req.headers["x-firecrawl-keyless-secret"];
+ if (!config.KEYLESS_PROXY_SECRET || secret !== config.KEYLESS_PROXY_SECRET) {
+ res.status(401).json({ eligible: false, error: "Unauthorized" });
+ return;
+ }
+
+ const ipHeader = req.headers["x-firecrawl-keyless-ip"];
+ const ip =
+ (typeof ipHeader === "string" ? ipHeader.trim() : "") || req.ip || "";
+
+ const result = await checkKeylessEligibility(ip);
+ res.status(200).json(result);
+}
diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts
index 08f46feb4..8056e9d1b 100644
--- a/apps/api/src/controllers/v2/scrape-browser.ts
+++ b/apps/api/src/controllers/v2/scrape-browser.ts
@@ -45,6 +45,7 @@ import { sanitizeUrlForTrace } from "../../lib/scrape-interact/langsmith";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { RequestWithAuth, ScrapeOptions } from "./types";
import { billTeam } from "../../services/billing/credit_billing";
+import { chargeKeylessCredits, keylessTeamUuid } from "../../lib/keyless";
import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";
import { logRequest } from "../../services/logging/log_job";
import { integrationSchema } from "../../utils/integration";
@@ -141,7 +142,12 @@ export async function scrapeInteractController(
if (!scrape) {
return res.status(404).json({ success: false, error: "Job not found." });
}
- if (scrape.team_id !== req.auth.team_id) {
+ // Keyless scrapes are persisted under a deterministic per-IP UUID (the
+ // `scrapes.team_id` column is a UUID, so the raw `preview_keyless_<ip>` string
+ // can't be stored). Compare against that derived UUID for keyless requests.
+ const expectedScrapeTeam =
+ keylessTeamUuid(req.auth.team_id) ?? req.auth.team_id;
+ if (scrape.team_id !== expectedScrapeTeam) {
return res.status(403).json({ success: false, error: "Forbidden." });
}
@@ -457,6 +463,10 @@ export async function scrapeStopInteractiveBrowserController(
});
});
+ // Charge the keyless free tier's per-IP daily credit budget (no-op for
+ // non-keyless teams).
+ chargeKeylessCredits(req.auth.team_id, creditsBilled).catch(() => {});
+
logger.info("Browser session destroyed", {
sessionDurationMs: durationMs,
creditsBilled,
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 9a79a57d8..1db7b506f 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -7,6 +7,7 @@ import {
searchRequestSchema,
} from "./types";
import { billTeam } from "../../services/billing/credit_billing";
+import { chargeKeylessCredits } from "../../lib/keyless";
import { v7 as uuidv7 } from "uuid";
import { logSearch, logRequest } from "../../services/logging/log_job";
import { logger as _logger } from "../../lib/logger";
@@ -169,6 +170,13 @@ export async function searchController(
);
}
+ // Charge the keyless free tier's per-IP daily credit budget for search
+ // credits (no-op for non-keyless teams; scrape jobs within search charge
+ // themselves via the scrape worker).
+ chargeKeylessCredits(req.auth.team_id, result.searchCredits).catch(
+ () => {},
+ );
+
const endTime = new Date().getTime();
const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
index 8dc81b804..e5d46af89 100644
--- a/apps/api/src/db/schema/public.ts
+++ b/apps/api/src/db/schema/public.ts
@@ -20,6 +20,16 @@ const ts = (name: string) =>
const bigintNum = (name: string) => bigint(name, { mode: "number" });
const num = (name: string) => numeric(name, { mode: "number" });
+// Keyless free-tier credit usage log (see keyless_credit_usage migration).
+// team_id is the deterministic per-IP keyless team UUID; ip is the raw client IP.
+export const keyless_credit_usage = pgTable("keyless_credit_usage", {
+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),
+ team_id: uuid("team_id").notNull(),
+ ip: text("ip").notNull(),
+ credits_used: integer("credits_used").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+});
+
export const agent_sponsors = pgTable("agent_sponsors", {
id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),
email: text("email").notNull(),
diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.ts
new file mode 100644
index 000000000..25f88480b
--- /dev/null
+++ b/apps/api/src/lib/keyless.ts
@@ -0,0 +1,193 @@
+import { isIPv4 } from "node:net";
+import { v5 as uuidv5 } from "uuid";
+import { config } from "../config";
+import { db } from "../db/connection";
+import * as schema from "../db/schema";
+import { redisRateLimitClient } from "../services/rate-limiter";
+
+// Keyless free tier: scrape, search, and interact can be used without an API key
+// from the official MCP server, CLI, or SDKs. It's gated per-IP/day by TWO
+// limits, both configurable via env: a request count and a credit budget.
+// `origin`/`integration` are client-set and spoofable, so they're only a soft
+// gate to keep raw API callers on the signup path β€” the per-IP daily caps plus
+// the `keyless/consume` canonical log are the real abuse controls.
+
+// No defaults: the keyless free tier is OFF unless BOTH limits are configured.
+export const KEYLESS_REQUESTS_PER_DAY = config.KEYLESS_REQUESTS_PER_DAY;
+export const KEYLESS_CREDITS_PER_DAY = config.KEYLESS_CREDITS_PER_DAY;
+
+// The tier is "configured" when BOTH limits are set β€” even to 0. Unset means the
+// feature is off (callers get a plain Unauthorized); 0 means it's on but the
+// budget is exhausted (callers get the 429 cap message).
+export function isKeylessConfigured(): boolean {
+ return (
+ typeof KEYLESS_REQUESTS_PER_DAY === "number" &&
+ typeof KEYLESS_CREDITS_PER_DAY === "number"
+ );
+}
+
+const DAY_SECONDS = 86400;
+
+// Keyless teams reuse the `preview_` prefix so billing (autumn `isPreviewTeam`)
+// and GCS persistence are skipped automatically, with a dedicated infix so the
+// IP can be recovered when charging credits after a request completes.
+export const KEYLESS_TEAM_PREFIX = "preview_keyless_";
+
+export function keylessTeamId(ip: string): string {
+ return `${KEYLESS_TEAM_PREFIX}${ip}`;
+}
+
+export function keylessIpFromTeamId(teamId: string): string | null {
+ return teamId.startsWith(KEYLESS_TEAM_PREFIX)
+ ? teamId.slice(KEYLESS_TEAM_PREFIX.length)
+ : null;
+}
+
+// Fixed namespace for deriving a stable per-keyless-team UUID. Tables like
+// `scrapes` require a UUID team_id, but keyless teams are `preview_keyless_<ip>`
+// strings. Mapping each to a deterministic UUIDv5 keeps rows per-IP-distinct
+// (so ownership checks such as interact still isolate keyless users) while
+// satisfying the UUID column β€” unlike the shared preview placeholder.
+const KEYLESS_TEAM_UUID_NAMESPACE = "9e6c8f2a-3b1d-4c7e-8a5f-2d4b6e8c0a1f";
+
+/**
+ * Deterministic UUID for a keyless team's persisted rows, or null for
+ * non-keyless teams (callers then fall back to the raw/placeholder team_id).
+ */
+export function keylessTeamUuid(
+ teamId: string | null | undefined,
+): string | null {
+ if (!teamId || !teamId.startsWith(KEYLESS_TEAM_PREFIX)) return null;
+ return uuidv5(teamId, KEYLESS_TEAM_UUID_NAMESPACE);
+}
+
+/**
+ * Keyless is allowed only for a *valid IPv4* client identity. This both:
+ * - denies IPv6 (a single client controls a huge block β€” a /64 is ~18
+ * quintillion addresses β€” so a per-IP cap is trivially bypassed), and
+ * - rejects malformed/unknown values (e.g. a forwarded `x-firecrawl-keyless-ip`
+ * that isn't a real IP), so they can't be used to mint arbitrary buckets and
+ * weaken per-IP quota enforcement.
+ * IPv4-mapped IPv6 (e.g. "::ffff:1.2.3.4", how dual-stack sockets surface IPv4)
+ * is treated as IPv4.
+ */
+export function isKeylessIpEligible(ip: string): boolean {
+ const normalized = ip.startsWith("::ffff:")
+ ? ip.slice("::ffff:".length)
+ : ip;
+ return isIPv4(normalized);
+}
+
+const requestsKey = (ip: string) => `keyless_requests:${ip}`;
+const creditsKey = (ip: string) => `keyless_credits:${ip}`;
+
+export type KeylessConsumeResult = {
+ ok: boolean;
+ reason?: "requests" | "credits";
+ requestsUsed: number;
+ creditsUsed: number;
+};
+
+/**
+ * Consume one request from the per-IP daily request budget and check the credit
+ * budget (credits are charged after the request completes, in
+ * `chargeKeylessCredits`). Returns whether the request may proceed.
+ */
+export async function consumeKeylessRequest(
+ ip: string,
+): Promise<KeylessConsumeResult> {
+ const requestLimit = KEYLESS_REQUESTS_PER_DAY ?? 0;
+ const creditLimit = KEYLESS_CREDITS_PER_DAY ?? 0;
+
+ const rKey = requestsKey(ip);
+ const requestsUsed = await redisRateLimitClient.incr(rKey);
+ if (requestsUsed === 1) {
+ await redisRateLimitClient.expire(rKey, DAY_SECONDS);
+ }
+
+ const creditsUsed = parseInt(
+ (await redisRateLimitClient.get(creditsKey(ip))) ?? "0",
+ 10,
+ );
+
+ if (requestsUsed > requestLimit) {
+ return { ok: false, reason: "requests", requestsUsed, creditsUsed };
+ }
+ if (creditsUsed >= creditLimit) {
+ return { ok: false, reason: "credits", requestsUsed, creditsUsed };
+ }
+ return { ok: true, requestsUsed, creditsUsed };
+}
+
+/**
+ * Read-only check of whether an IP could currently use the keyless tier (no
+ * consumption). Used by the hosted MCP to decide, at connect time, whether to
+ * serve keyless (eligible) or throw so FastMCP emits the OAuth challenge (not).
+ */
+export async function checkKeylessEligibility(
+ ip: string,
+): Promise<{ eligible: boolean; reason?: string }> {
+ if (!isKeylessConfigured()) return { eligible: false, reason: "disabled" };
+ if (!ip || !isKeylessIpEligible(ip)) {
+ return { eligible: false, reason: "ineligible_ip" };
+ }
+ try {
+ const requestsUsed = parseInt(
+ (await redisRateLimitClient.get(requestsKey(ip))) ?? "0",
+ 10,
+ );
+ if (requestsUsed >= (KEYLESS_REQUESTS_PER_DAY ?? 0)) {
+ return { eligible: false, reason: "requests" };
+ }
+ const creditsUsed = parseInt(
+ (await redisRateLimitClient.get(creditsKey(ip))) ?? "0",
+ 10,
+ );
+ if (creditsUsed >= (KEYLESS_CREDITS_PER_DAY ?? 0)) {
+ return { eligible: false, reason: "credits" };
+ }
+ return { eligible: true };
+ } catch {
+ // Limiter store unavailable β€” fail closed so the MCP issues an OAuth
+ // challenge rather than granting unbounded keyless.
+ return { eligible: false, reason: "error" };
+ }
+}
+
+/**
+ * Add the actual credits a completed request consumed to the IP's daily credit
+ * counter. No-op for non-keyless teams. Best-effort; never throws.
+ */
+export async function chargeKeylessCredits(
+ teamId: string,
+ credits: number,
+): Promise<void> {
+ const ip = keylessIpFromTeamId(teamId);
+ if (!ip || !Number.isFinite(credits) || credits <= 0) return;
+ const inc = Math.ceil(credits);
+ try {
+ const key = creditsKey(ip);
+ const total = await redisRateLimitClient.incrby(key, inc);
+ if (total === inc) {
+ await redisRateLimitClient.expire(key, DAY_SECONDS);
+ }
+ } catch {
+ // Counter is best-effort; a missed charge just means the IP gets a few
+ // extra free credits today.
+ }
+
+ // Log the usage to keyless_credit_usage (per-IP keyless team UUID + raw IP)
+ // for abuse monitoring. Best-effort β€” never block the request.
+ const teamUuid = keylessTeamUuid(teamId);
+ if (config.USE_DB_AUTHENTICATION === true && teamUuid) {
+ try {
+ await db.insert(schema.keyless_credit_usage).values({
+ team_id: teamUuid,
+ ip,
+ credits_used: inc,
+ });
+ } catch {
+ // Logging is best-effort.
+ }
+ }
+}
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index f63be759e..c205eb085 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -196,6 +196,7 @@ export function checkCreditsMiddleware(
export function authMiddleware(
rateLimiterMode: RateLimiterMode,
+ options: { allowKeyless?: boolean } = {},
): (req: RequestWithMaybeAuth, res: Response, next: NextFunction) => void {
return (req, res, next) => {
(async () => {
@@ -211,11 +212,18 @@ export function authMiddleware(
// currentRateLimiterMode = RateLimiterMode.ScrapeAgentPreview;
// }
- const auth = await authenticateUser(req, res, currentRateLimiterMode);
+ const auth = await authenticateUser(
+ req,
+ res,
+ currentRateLimiterMode,
+ options,
+ );
if (!auth.success) {
if (!res.headersSent) {
- if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);
+ if (auth.status === 401 || auth.agentAuthDiscovery) {
+ applyAgentAuthDiscoveryHeader(res);
+ }
return res
.status(auth.status)
.json({ success: false, error: auth.error });
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index 5e3b0d930..5ceddf19a 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -124,7 +124,7 @@ v1Router.use(requestTimingMiddleware("v1"));
v1Router.post(
"/scrape",
- authMiddleware(RateLimiterMode.Scrape),
+ authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),
countryCheck,
checkCreditsMiddleware(1),
blocklistMiddleware,
@@ -153,7 +153,7 @@ v1Router.post(
v1Router.post(
"/search",
- authMiddleware(RateLimiterMode.Search),
+ authMiddleware(RateLimiterMode.Search, { allowKeyless: true }),
countryCheck,
checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),
wrap(searchController),
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 71e98d89b..0561b0b7d 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -8,6 +8,7 @@ import { searchController } from "../controllers/v2/search";
import { searchFeedbackController } from "../controllers/v2/search-feedback";
import { x402SearchController } from "../controllers/v2/x402-search";
import { scrapeController } from "../controllers/v2/scrape";
+import { keylessEligibilityController } from "../controllers/v2/keyless-eligibility";
import {
parseController,
parseMultipartPayloadMiddleware,
@@ -234,9 +235,13 @@ v2Router.use(requestTimingMiddleware("v2"));
// ),
// );
+// Internal: trusted-proxy (hosted MCP) keyless eligibility probe. Secret-gated
+// inside the controller; no auth middleware.
+v2Router.get("/keyless/eligibility", wrap(keylessEligibilityController));
+
v2Router.post(
"/search",
- authMiddleware(RateLimiterMode.Search),
+ authMiddleware(RateLimiterMode.Search, { allowKeyless: true }),
countryCheck,
checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),
blocklistMiddleware,
@@ -262,7 +267,7 @@ v2Router.post(
v2Router.post(
"/scrape",
- authMiddleware(RateLimiterMode.Scrape),
+ authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),
countryCheck,
checkCreditsMiddleware(1),
blocklistMiddleware,
@@ -278,14 +283,14 @@ v2Router.get(
v2Router.post(
"/scrape/:jobId/interact",
- authMiddleware(RateLimiterMode.BrowserExecute),
+ authMiddleware(RateLimiterMode.BrowserExecute, { allowKeyless: true }),
validateJobIdParam,
wrap(scrapeInteractController),
);
v2Router.delete(
"/scrape/:jobId/interact",
- authMiddleware(RateLimiterMode.BrowserExecute),
+ authMiddleware(RateLimiterMode.BrowserExecute, { allowKeyless: true }),
validateJobIdParam,
wrap(scrapeStopInteractiveBrowserController),
);
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index ecc78903c..d9a3eab41 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -16,6 +16,7 @@ import {
saveSearchToGCS,
} from "../../lib/gcs-jobs";
import { hasFormatOfType } from "../../lib/format-utils";
+import { keylessTeamUuid } from "../../lib/keyless";
import type { Document, ScrapeOptions } from "../../controllers/v2/types";
import type { CostTracking } from "../../lib/cost-tracking";
import type { Logger } from "winston";
@@ -272,9 +273,10 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
error: scrape.error ?? null,
time_taken: scrape.time_taken,
team_id:
- scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")
+ keylessTeamUuid(scrape.team_id) ??
+ (scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")
? previewTeamId
- : scrape.team_id,
+ : scrape.team_id),
options: scrape.zeroDataRetention ? null : scrape.options,
cost_tracking: scrape.zeroDataRetention
? null
diff --git a/apps/api/src/services/posthog.ts b/apps/api/src/services/posthog.ts
index 79eaa1338..0a92cecee 100644
--- a/apps/api/src/services/posthog.ts
+++ b/apps/api/src/services/posthog.ts
@@ -65,7 +65,7 @@ type RequestSurface =
* practice. NOTE: confirm the exact strings sent by firecrawl-mcp and
* firecrawl-cli β€” adjust the prefixes below if they differ.
*/
-function originToSurface(origin?: string | null): RequestSurface {
+export function originToSurface(origin?: string | null): RequestSurface {
const o = (origin ?? "").toLowerCase();
if (o === "website" || o.includes("playground")) return "playground";
if (o.startsWith("mcp")) return "mcp";
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 9c1bc0cad..dcf1d7467 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -52,6 +52,7 @@ import { createWebhookSender, WebhookEvent } from "../webhook/index";
import { CustomError } from "../../lib/custom-error";
import { startWebScraperPipeline } from "../../main/runWebScraper";
import { CostTracking } from "../../lib/cost-tracking";
+import { chargeKeylessCredits } from "../../lib/keyless";
import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
@@ -138,6 +139,10 @@ async function billScrapeJob(
unsupportedFeatures,
);
+ // Charge the keyless free tier's per-IP daily credit budget (no-op for
+ // non-keyless teams).
+ await chargeKeylessCredits(job.data.team_id, creditsToBeBilled);
+
if (
job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&
config.USE_DB_AUTHENTICATION
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index 172171645..2c8c0f30d 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -171,6 +171,9 @@ export type AuthResponse =
success: false;
error: string;
status: number;
+ // When true, send the agent OAuth-discovery WWW-Authenticate header even on
+ // non-401 responses (e.g. keyless cap 429s) so agents can find the key flow.
+ agentAuthDiscovery?: boolean;
};
export enum NotificationType {
diff --git a/apps/dot-net-sdk/Firecrawl.Tests/FirecrawlClientTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/FirecrawlClientTests.cs
index d3c1ad26f..0302a66fd 100644
--- a/apps/dot-net-sdk/Firecrawl.Tests/FirecrawlClientTests.cs
+++ b/apps/dot-net-sdk/Firecrawl.Tests/FirecrawlClientTests.cs
@@ -7,22 +7,23 @@ namespace Firecrawl.Tests;
public class FirecrawlClientTests
{
[Fact]
- public void Constructor_RequiresApiKey()
+ public void Constructor_AllowsEmptyApiKey_ForKeylessFreeTier()
{
// Clear env variable in case it's set
Environment.SetEnvironmentVariable("FIRECRAWL_API_KEY", null);
- var ex = Assert.Throws<FirecrawlException>(() => new FirecrawlClient(apiKey: ""));
- Assert.Contains("API key is required", ex.Message);
+ // No key: scrape/search use the keyless free tier; no longer throws.
+ var client = new FirecrawlClient(apiKey: "");
+ Assert.NotNull(client);
}
[Fact]
- public void Constructor_RequiresApiKey_WhenNull()
+ public void Constructor_AllowsNullApiKey_ForKeylessFreeTier()
{
Environment.SetEnvironmentVariable("FIRECRAWL_API_KEY", null);
- var ex = Assert.Throws<FirecrawlException>(() => new FirecrawlClient(apiKey: null));
- Assert.Contains("API key is required", ex.Message);
+ var client = new FirecrawlClient(apiKey: null);
+ Assert.NotNull(client);
}
[Fact]
diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
index dbaeb93ec..cda256516 100644
--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
@@ -8,7 +8,7 @@
<!-- NuGet package metadata -->
<PackageId>firecrawl-sdk</PackageId>
- <Version>1.7.0</Version>
+ <Version>1.7.1</Version>
<Authors>Firecrawl</Authors>
<Company>Firecrawl</Company>
<Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>
diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
index 11c5da6a3..fde84b672 100644
--- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
@@ -607,14 +607,28 @@ public class FirecrawlClient
// INTERNAL UTILITIES
// ================================================================
+ private const string SdkOrigin = "[email protected]";
+
private static Dictionary<string, object> BuildBody(object? options)
{
+ Dictionary<string, object> body;
if (options == null)
- return new Dictionary<string, object>();
+ {
+ body = new Dictionary<string, object>();
+ }
+ else
+ {
+ var json = JsonSerializer.Serialize(options, FirecrawlHttpClient.JsonOptions);
+ body = JsonSerializer.Deserialize<Dictionary<string, object>>(json, FirecrawlHttpClient.JsonOptions)
+ ?? new Dictionary<string, object>();
+ }
+
+ // Identify the SDK so the API can grant the keyless free tier; harmless
+ // telemetry on keyed requests.
+ if (!body.ContainsKey("origin"))
+ body["origin"] = SdkOrigin;
- var json = JsonSerializer.Serialize(options, FirecrawlHttpClient.JsonOptions);
- return JsonSerializer.Deserialize<Dictionary<string, object>>(json, FirecrawlHttpClient.JsonOptions)
- ?? new Dictionary<string, object>();
+ return body;
}
private static string BuildQuery(int? limit = null, int? offset = null, string? status = null)
@@ -643,7 +657,7 @@ public class FirecrawlClient
return query.Count == 0 ? string.Empty : "?" + string.Join("&", query);
}
- private static string ResolveApiKey(string? apiKey)
+ private static string? ResolveApiKey(string? apiKey)
{
if (!string.IsNullOrWhiteSpace(apiKey))
return apiKey;
@@ -652,8 +666,9 @@ public class FirecrawlClient
if (!string.IsNullOrWhiteSpace(envKey))
return envKey;
- throw new FirecrawlException(
- "API key is required. Pass it to the constructor or set the FIRECRAWL_API_KEY environment variable.");
+ // No key: scrape and search fall back to the keyless free tier (per-IP).
+ // Other endpoints return 401 from the API until a key is provided.
+ return null;
}
private static string ResolveApiUrl(string? apiUrl)
diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs
index a7ba80dbf..883580ce0 100644
--- a/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs
+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs
@@ -14,7 +14,7 @@ namespace Firecrawl;
internal class FirecrawlHttpClient
{
private readonly HttpClient _httpClient;
- private readonly string _apiKey;
+ private readonly string? _apiKey;
private readonly string _baseUrl;
private readonly int _maxRetries;
private readonly double _backoffFactor;
@@ -27,7 +27,7 @@ internal class FirecrawlHttpClient
};
internal FirecrawlHttpClient(
- string apiKey,
+ string? apiKey,
string baseUrl,
TimeSpan timeout,
int maxRetries,
@@ -198,7 +198,10 @@ internal class FirecrawlHttpClient
private void ApplyStandardHeaders(HttpRequestMessage request)
{
- request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _apiKey);
+ // Omit the Authorization header entirely when no key is set so that
+ // scrape/search can use the keyless free tier.
+ if (!string.IsNullOrEmpty(_apiKey))
+ request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _apiKey);
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
}
diff --git a/apps/elixir-sdk/generate.exs b/apps/elixir-sdk/generate.exs
index e0327912a..ab781bdc5 100644
--- a/apps/elixir-sdk/generate.exs
+++ b/apps/elixir-sdk/generate.exs
@@ -99,7 +99,7 @@ defmodule Firecrawl.Generator do
|> Enum.join("\n")
bt = <<96>>
- auth_header_line = ~S' headers: [{"authorization", "Bearer #{api_key}"}]'
+ headers_line = ~S' headers = if api_key, do: [{"authorization", "Bearer #{api_key}"}], else: []'
"""
# This file is auto-generated by generate.exs β€” DO NOT EDIT MANUALLY.
@@ -145,6 +145,7 @@ defmodule Firecrawl.Generator do
@type response :: {:ok, Req.Response.t()} | {:error, Exception.t() | Firecrawl.Error.t()}
@base_url "#{base_url}"
+ @sdk_origin "[email protected]"
defp client(opts) do
api_key =
@@ -152,43 +153,29 @@ defmodule Firecrawl.Generator do
Application.get_env(:firecrawl, :api_key)
end)
+ # A nil/empty key is allowed: scrape, search, and interact fall back to
+ # the keyless free tier (rate-limited per IP). Other endpoints return 401
+ # from the API until a key is provided.
api_key =
case api_key do
key when is_binary(key) ->
case String.trim(key) do
- "" ->
- raise \"\"\"
- Firecrawl API key not found or empty. Set it in your config:
-
- config :firecrawl, api_key: "fc-your-api-key"
-
- Or pass it as an option:
-
- Firecrawl.scrape_and_extract_from_url([url: "..."], api_key: "fc-your-api-key")
- \"\"\"
-
- trimmed ->
- trimmed
+ "" -> nil
+ trimmed -> trimmed
end
_ ->
- raise \"\"\"
- Firecrawl API key not found or empty. Set it in your config:
-
- config :firecrawl, api_key: "fc-your-api-key"
-
- Or pass it as an option:
-
- Firecrawl.scrape_and_extract_from_url([url: "..."], api_key: "fc-your-api-key")
- \"\"\"
+ nil
end
{base_url, opts} = Keyword.pop(opts, :base_url, @base_url)
opts = Keyword.delete(opts, :api_key)
+ #{headers_line}
+
Req.new(
base_url: base_url,
- #{auth_header_line}
+ headers: headers
)
|> Req.merge(opts)
|> Req.Request.append_response_steps(firecrawl_error_handler: &handle_api_error/1)
@@ -201,10 +188,12 @@ defmodule Firecrawl.Generator do
defp handle_api_error({request, response}), do: {request, response}
defp to_body(validated_params, key_mapping) do
- Map.new(validated_params, fn {k, v} ->
+ validated_params
+ |> Map.new(fn {k, v} ->
json_key = Map.fetch!(key_mapping, k)
{json_key, to_json_value(v)}
end)
+ |> Map.put_new("origin", @sdk_origin)
end
defp to_query(validated_params, key_mapping) do
diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.ex
index e40387c3a..dc1ea6008 100644
--- a/apps/elixir-sdk/lib/firecrawl.ex
+++ b/apps/elixir-sdk/lib/firecrawl.ex
@@ -41,6 +41,7 @@ defmodule Firecrawl do
@type response :: {:ok, Req.Response.t()} | {:error, Exception.t() | Firecrawl.Error.t()}
@base_url "https://api.firecrawl.dev/v2"
+ @sdk_origin "[email protected]"
defp client(opts) do
api_key =
@@ -48,43 +49,29 @@ defmodule Firecrawl do
Application.get_env(:firecrawl, :api_key)
end)
+ # A nil/empty key is allowed: scrape, search, and interact fall back to the
+ # keyless free tier (rate-limited per IP). Other endpoints return 401 from the
+ # API until a key is provided.
api_key =
case api_key do
key when is_binary(key) ->
case String.trim(key) do
- "" ->
- raise """
- Firecrawl API key not found or empty. Set it in your config:
-
- config :firecrawl, api_key: "fc-your-api-key"
-
- Or pass it as an option:
-
- Firecrawl.scrape_and_extract_from_url([url: "..."], api_key: "fc-your-api-key")
- """
-
- trimmed ->
- trimmed
+ "" -> nil
+ trimmed -> trimmed
end
_ ->
- raise """
- Firecrawl API key not found or empty. Set it in your config:
-
- config :firecrawl, api_key: "fc-your-api-key"
-
- Or pass it as an option:
-
- Firecrawl.scrape_and_extract_from_url([url: "..."], api_key: "fc-your-api-key")
- """
+ nil
end
{base_url, opts} = Keyword.pop(opts, :base_url, @base_url)
opts = Keyword.delete(opts, :api_key)
+ headers = if api_key, do: [{"authorization", "Bearer #{api_key}"}], else: []
+
Req.new(
base_url: base_url,
- headers: [{"authorization", "Bearer #{api_key}"}]
+ headers: headers
)
|> Req.merge(opts)
|> Req.Request.append_response_steps(firecrawl_error_handler: &handle_api_error/1)
@@ -97,10 +84,14 @@ defmodule Firecrawl do
defp handle_api_error({request, response}), do: {request, response}
defp to_body(validated_params, key_mapping) do
- Map.new(validated_params, fn {k, v} ->
+ validated_params
+ |> Map.new(fn {k, v} ->
json_key = Map.fetch!(key_mapping, k)
{json_key, to_json_value(v)}
end)
+ # Identify the SDK so the API can grant the keyless free tier; harmless
+ # telemetry on keyed requests.
+ |> Map.put_new("origin", @sdk_origin)
end
defp to_query(validated_params, key_mapping) do
diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs
index 09a0b7c46..183f12340 100644
--- a/apps/elixir-sdk/mix.exs
+++ b/apps/elixir-sdk/mix.exs
@@ -1,7 +1,7 @@
defmodule Firecrawl.MixProject do
use Mix.Project
- @version "1.6.0"
+ @version "1.6.1"
@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"
def project do
diff --git a/apps/go-sdk/firecrawl.go b/apps/go-sdk/firecrawl.go
index 57f608e0c..84495c3e0 100644
--- a/apps/go-sdk/firecrawl.go
+++ b/apps/go-sdk/firecrawl.go
@@ -56,12 +56,9 @@ func NewClient(opts ...option.RequestOption) (*Client, error) {
if apiKey == "" {
apiKey = strings.TrimSpace(os.Getenv("FIRECRAWL_API_KEY"))
}
- if apiKey == "" {
- return nil, &FirecrawlError{
- Message: "API key is required. Set it via option.WithAPIKey(), " +
- "or FIRECRAWL_API_KEY environment variable.",
- }
- }
+ // An empty API key is allowed: scrape, search, and interact fall back to the
+ // keyless free tier (rate-limited per IP). Other methods return 401 from the
+ // API until a key is provided.
// Resolve API URL.
apiURL := cfg.APIURL
@@ -95,6 +92,9 @@ func (c *Client) Scrape(ctx context.Context, url string, opts *ScrapeOptions) (*
body := map[string]interface{}{"url": url}
mergeOptions(body, opts)
+ if _, ok := body["origin"]; !ok {
+ body["origin"] = "go-sdk@" + Version
+ }
raw, err := c.http.post(ctx, "/v2/scrape", body, nil)
if err != nil {
return nil, err
@@ -132,6 +132,9 @@ func (c *Client) Interact(ctx context.Context, jobID, code string, params *Inter
}
}
+ if _, ok := body["origin"]; !ok {
+ body["origin"] = "go-sdk@" + Version
+ }
raw, err := c.http.post(ctx, "/v2/scrape/"+jobID+"/interact", body, nil)
if err != nil {
return nil, err
@@ -539,6 +542,9 @@ func (c *Client) Search(ctx context.Context, query string, opts *SearchOptions)
body := map[string]interface{}{"query": query}
mergeOptions(body, opts)
+ if _, ok := body["origin"]; !ok {
+ body["origin"] = "go-sdk@" + Version
+ }
raw, err := c.http.post(ctx, "/v2/search", body, nil)
if err != nil {
return nil, err
diff --git a/apps/go-sdk/http_client.go b/apps/go-sdk/http_client.go
index e17f221d6..2acda1394 100644
--- a/apps/go-sdk/http_client.go
+++ b/apps/go-sdk/http_client.go
@@ -136,7 +136,9 @@ func (h *httpClient) postMultipart(
return nil, &FirecrawlError{Message: fmt.Sprintf("failed to create request: %v", err)}
}
- req.Header.Set("Authorization", "Bearer "+h.apiKey)
+ if h.apiKey != "" {
+ req.Header.Set("Authorization", "Bearer "+h.apiKey)
+ }
req.Header.Set("Content-Type", contentType)
req.Header.Set("User-Agent", "firecrawl-go/"+Version)
for k, v := range h.extraHeaders {
@@ -221,7 +223,9 @@ func (h *httpClient) doJSON(ctx context.Context, method, url string, body interf
return nil, &FirecrawlError{Message: fmt.Sprintf("failed to create request: %v", err)}
}
- req.Header.Set("Authorization", "Bearer "+h.apiKey)
+ if h.apiKey != "" {
+ req.Header.Set("Authorization", "Bearer "+h.apiKey)
+ }
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "firecrawl-go/"+Version)
// Apply client-level headers.
diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go
index 7dcd1faa5..296f5f6af 100644
--- a/apps/go-sdk/version.go
+++ b/apps/go-sdk/version.go
@@ -9,4 +9,4 @@ package firecrawl
// Bump this when preparing a new release. The publish-go-sdk GitHub workflow
// reads this value and creates the corresponding monorepo-prefixed tag on
// merge to main.
-const Version = "1.6.0"
+const Version = "1.6.1"
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts
index 272c86cb5..1f2affb90 100644
--- a/apps/java-sdk/build.gradle.kts
+++ b/apps/java-sdk/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
}
group = "com.firecrawl"
-version = "1.9.0"
+version = "1.9.1"
java {
sourceCompatibility = JavaVersion.VERSION_11
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
index bfca8dfc0..71688c500 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
@@ -37,6 +37,7 @@ import java.util.concurrent.ForkJoinPool;
public class FirecrawlClient {
private static final String DEFAULT_API_URL = "https://api.firecrawl.dev";
+ private static final String SDK_ORIGIN = "[email protected]";
private static final long DEFAULT_TIMEOUT_MS = 300_000; // 5 minutes
private static final int DEFAULT_MAX_RETRIES = 3;
private static final double DEFAULT_BACKOFF_FACTOR = 0.5;
@@ -101,6 +102,7 @@ public class FirecrawlClient {
if (options != null) {
mergeOptions(body, options);
}
+ body.putIfAbsent("origin", SDK_ORIGIN);
return extractData(http.post("/v2/scrape", body, Map.class), Document.class);
}
@@ -148,6 +150,7 @@ public class FirecrawlClient {
body.put("language", language != null ? language : "node");
if (timeout != null) body.put("timeout", timeout);
if (origin != null) body.put("origin", origin);
+ body.putIfAbsent("origin", SDK_ORIGIN);
return http.post("/v2/scrape/" + jobId + "/interact", body, BrowserExecuteResponse.class);
}
@@ -549,6 +552,7 @@ public class FirecrawlClient {
if (options != null) {
mergeOptions(body, options);
}
+ body.putIfAbsent("origin", SDK_ORIGIN);
return extractData(http.post("/v2/search", body, Map.class), SearchData.class);
}
@@ -1387,11 +1391,9 @@ public class FirecrawlClient {
if (resolvedKey == null || resolvedKey.isBlank()) {
resolvedKey = System.getProperty("firecrawl.apiKey");
}
- if (resolvedKey == null || resolvedKey.isBlank()) {
- throw new FirecrawlException(
- "API key is required. Set it via builder.apiKey(), " +
- "FIRECRAWL_API_KEY environment variable, or firecrawl.apiKey system property.");
- }
+ // A null/blank key is allowed: scrape, search, and interact fall back
+ // to the keyless free tier (rate-limited per IP). Other methods return
+ // 401 from the API until a key is provided.
String resolvedUrl = apiUrl;
if (resolvedUrl == null || resolvedUrl.equals(DEFAULT_API_URL)) {
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java
index 695c9e079..b6fe62d7a 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java
@@ -55,6 +55,17 @@ class FirecrawlHttpClient {
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
+ /**
+ * Adds the Authorization header only when an API key is configured. Omitting
+ * it entirely (rather than sending an empty Bearer) lets scrape/search/interact
+ * use the keyless free tier.
+ */
+ private void applyAuth(Request.Builder builder) {
+ if (apiKey != null && !apiKey.isBlank()) {
+ builder.header("Authorization", "Bearer " + apiKey);
+ }
+ }
+
/**
* Sends a POST request with JSON body.
*/
@@ -76,9 +87,9 @@ class FirecrawlHttpClient {
RequestBody requestBody = RequestBody.create(json, JSON);
Request.Builder builder = new Request.Builder()
.url(url)
- .header("Authorization", "Bearer " + apiKey)
.header("Content-Type", "application/json")
.post(requestBody);
+ applyAuth(builder);
for (Map.Entry<String, String> entry : extraHeaders.entrySet()) {
builder.header(entry.getKey(), entry.getValue());
}
@@ -98,12 +109,12 @@ class FirecrawlHttpClient {
throw new FirecrawlException("Failed to serialize request body", e);
}
RequestBody requestBody = RequestBody.create(json, JSON);
- Request request = new Request.Builder()
+ Request.Builder builder = new Request.Builder()
.url(url)
- .header("Authorization", "Bearer " + apiKey)
.header("Content-Type", "application/json")
- .patch(requestBody)
- .build();
+ .patch(requestBody);
+ applyAuth(builder);
+ Request request = builder.build();
return executeWithRetry(request, responseType);
}
@@ -140,11 +151,11 @@ class FirecrawlHttpClient {
RequestBody fileBody = RequestBody.create(fileContent, mediaType);
multipart.addFormDataPart(fileFieldName, filename, fileBody);
- Request request = new Request.Builder()
+ Request.Builder builder = new Request.Builder()
.url(url)
- .header("Authorization", "Bearer " + apiKey)
- .post(multipart.build())
- .build();
+ .post(multipart.build());
+ applyAuth(builder);
+ Request request = builder.build();
return executeWithRetry(request, responseType);
}
@@ -154,11 +165,11 @@ class FirecrawlHttpClient {
*/
<T> T get(String path, Class<T> responseType) {
String url = baseUrl + path;
- Request request = new Request.Builder()
+ Request.Builder builder = new Request.Builder()
.url(url)
- .header("Authorization", "Bearer " + apiKey)
- .get()
- .build();
+ .get();
+ applyAuth(builder);
+ Request request = builder.build();
return executeWithRetry(request, responseType);
}
@@ -166,11 +177,11 @@ class FirecrawlHttpClient {
* Sends a GET request with full URL (for following next-page cursors).
*/
<T> T getAbsolute(String absoluteUrl, Class<T> responseType) {
- Request request = new Request.Builder()
+ Request.Builder builder = new Request.Builder()
.url(absoluteUrl)
- .header("Authorization", "Bearer " + apiKey)
- .get()
- .build();
+ .get();
+ applyAuth(builder);
+ Request request = builder.build();
return executeWithRetry(request, responseType);
}
@@ -179,11 +190,11 @@ class FirecrawlHttpClient {
*/
<T> T delete(String path, Class<T> responseType) {
String url = baseUrl + path;
- Request request = new Request.Builder()
+ Request.Builder builder = new Request.Builder()
.url(url)
- .header("Authorization", "Bearer " + apiKey)
- .delete()
- .build();
+ .delete();
+ applyAuth(builder);
+ Request request = builder.build();
return executeWithRetry(request, responseType);
}
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index de67a4c01..cd4c0d462 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.25.4",
+ "version": "4.25.5",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/clientOptions.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/clientOptions.test.ts
index 770e9df62..1746007e1 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/clientOptions.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/clientOptions.test.ts
@@ -67,12 +67,11 @@ describe('Firecrawl v2 Client Options', () => {
expect(client).toBeInstanceOf(FirecrawlClient);
});
- it('should throw for empty string API key on cloud', () => {
- expect(() => new Firecrawl('')).toThrow('API key is required');
- });
-
- it('should throw for whitespace-only string API key on cloud', () => {
- expect(() => new Firecrawl(' ')).toThrow('API key is required');
+ it('should construct without an API key for the keyless free tier', () => {
+ // No key: scrape/search/interact use the keyless free tier; the SDK no
+ // longer throws at construction.
+ expect(() => new Firecrawl('')).not.toThrow();
+ expect(() => new Firecrawl(' ')).not.toThrow();
});
it('should provide v1 accessor when constructed with string', () => {
diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.ts
index 2eaa4d28f..bd75c7285 100644
--- a/apps/js-sdk/firecrawl/src/v2/client.ts
+++ b/apps/js-sdk/firecrawl/src/v2/client.ts
@@ -137,9 +137,9 @@ export class FirecrawlClient {
const apiKey = (opts.apiKey ?? process.env.FIRECRAWL_API_KEY ?? "").trim();
const apiUrl = (opts.apiUrl ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev").replace(/\/$/, "");
- if (this.isCloudService(apiUrl) && !apiKey) {
- throw new Error("API key is required for the cloud API. Set FIRECRAWL_API_KEY env or pass apiKey.");
- }
+ // No API key is allowed: scrape, search, and interact fall back to the
+ // keyless free tier (rate-limited per IP). Other methods will return 401
+ // from the API until a key is provided.
this.http = new HttpClient({
apiKey,
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
index 842a0ea25..dc0560ed1 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
@@ -34,7 +34,10 @@ export class HttpClient {
baseURL: this.apiUrl,
timeout: options.timeoutMs ?? 300000,
headers: {
- Authorization: `Bearer ${this.apiKey}`,
+ // Omit the Authorization header entirely when no API key is set so that
+ // scrape/search/interact can use the keyless free tier (the cloud only
+ // grants it when no Authorization header is present).
+ ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
},
transitional: { clarifyTimeoutError: true },
});
diff --git a/apps/php-sdk/src/Client/FirecrawlClient.php b/apps/php-sdk/src/Client/FirecrawlClient.php
index 46c199065..9758ca9fe 100644
--- a/apps/php-sdk/src/Client/FirecrawlClient.php
+++ b/apps/php-sdk/src/Client/FirecrawlClient.php
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace Firecrawl\Client;
use Firecrawl\Exceptions\FirecrawlException;
+use Firecrawl\Version;
use Firecrawl\Exceptions\JobTimeoutException;
use Firecrawl\Models\AgentOptions;
use Firecrawl\Models\AgentResponse;
@@ -63,12 +64,10 @@ final class FirecrawlClient
float $backoffFactor = self::DEFAULT_BACKOFF_FACTOR,
?ClientInterface $httpClient = null,
): self {
+ // An empty key is allowed: scrape, search, and interact fall back to the
+ // keyless free tier (rate-limited per IP). Other methods return 401 from
+ // the API until a key is provided.
$resolvedKey = trim($apiKey ?: (getenv('FIRECRAWL_API_KEY') ?: ''));
- if ($resolvedKey === '') {
- throw new FirecrawlException(
- 'API key is required. Pass it directly or set the FIRECRAWL_API_KEY environment variable.',
- );
- }
$resolvedUrl = $apiUrl ?: (getenv('FIRECRAWL_API_URL') ?: self::DEFAULT_API_URL);
@@ -111,6 +110,7 @@ final class FirecrawlClient
if ($options !== null) {
$body = array_merge($body, $options->toArray());
}
+ $body['origin'] ??= 'php-sdk@' . Version::SDK_VERSION;
$response = $this->http->post('/v2/scrape', $body);
@@ -141,6 +141,7 @@ final class FirecrawlClient
if ($prompt !== null) {
$body['prompt'] = $prompt;
}
+ $body['origin'] ??= 'php-sdk@' . Version::SDK_VERSION;
return BrowserExecuteResponse::fromArray(
$this->http->post("/v2/scrape/{$jobId}/interact", $body),
@@ -472,6 +473,7 @@ final class FirecrawlClient
if ($options !== null) {
$body = array_merge($body, $options->toArray());
}
+ $body['origin'] ??= 'php-sdk@' . Version::SDK_VERSION;
$response = $this->http->post('/v2/search', $body);
diff --git a/apps/php-sdk/src/Client/FirecrawlHttpClient.php b/apps/php-sdk/src/Client/FirecrawlHttpClient.php
index fa1d35f5d..44e7c8d20 100644
--- a/apps/php-sdk/src/Client/FirecrawlHttpClient.php
+++ b/apps/php-sdk/src/Client/FirecrawlHttpClient.php
@@ -136,10 +136,14 @@ final class FirecrawlHttpClient
?array $multipart = null,
): array {
$defaultHeaders = [
- 'Authorization' => 'Bearer ' . $this->apiKey,
'Accept' => 'application/json',
'User-Agent' => 'firecrawl-php/' . Version::SDK_VERSION,
];
+ // Omit the Authorization header entirely when no key is set so that
+ // scrape/search/interact can use the keyless free tier.
+ if ($this->apiKey !== null && $this->apiKey !== '') {
+ $defaultHeaders['Authorization'] = 'Bearer ' . $this->apiKey;
+ }
if ($multipart === null) {
$defaultHeaders['Content-Type'] = 'application/json';
diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.php
index c627fce9c..5f15f46ef 100644
--- a/apps/php-sdk/src/Version.php
+++ b/apps/php-sdk/src/Version.php
@@ -6,5 +6,5 @@ namespace Firecrawl;
final class Version
{
- public const SDK_VERSION = '1.6.0';
+ public const SDK_VERSION = '1.6.1';
}
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index fa14ddb39..aba9a1c64 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.28.2"
+__version__ = "4.28.3"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 13ea6a4ed..b34ce4a38 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -111,11 +111,10 @@ class FirecrawlClient:
if api_key is None:
api_key = os.getenv("FIRECRAWL_API_KEY")
- if self._is_cloud_service(api_url) and not api_key:
- raise ValueError(
- "API key is required for the cloud API. Set FIRECRAWL_API_KEY environment variable "
- "or pass api_key parameter."
- )
+ # No API key is allowed: scrape, search, and interact fall back to the
+ # keyless free tier (rate-limited per IP). The HTTP client only sends an
+ # Authorization header when a key is present, and other methods return
+ # 401 from the API until a key is provided.
self.config = ClientConfig(
api_key=api_key,
diff --git a/apps/ruby-sdk/lib/firecrawl/client.rb b/apps/ruby-sdk/lib/firecrawl/client.rb
index e014427e2..3bb199d66 100644
--- a/apps/ruby-sdk/lib/firecrawl/client.rb
+++ b/apps/ruby-sdk/lib/firecrawl/client.rb
@@ -39,9 +39,10 @@ module Firecrawl
backoff_factor: DEFAULT_BACKOFF_FACTOR
)
resolved_key = api_key || ENV["FIRECRAWL_API_KEY"]
- if resolved_key.nil? || resolved_key.strip.empty?
- raise FirecrawlError, "API key is required. Provide api_key: or set FIRECRAWL_API_KEY environment variable."
- end
+ # A nil/empty key is allowed: scrape, search, and interact fall back to the
+ # keyless free tier (rate-limited per IP). Other methods return 401 from the
+ # API until a key is provided.
+ resolved_key = nil if resolved_key.nil? || resolved_key.strip.empty?
resolved_url = api_url || ENV["FIRECRAWL_API_URL"] || DEFAULT_API_URL
unless resolved_url.match?(%r{\Ahttps?://}i)
@@ -78,6 +79,7 @@ module Firecrawl
body = { "url" => url }
body.merge!(options.to_h) if options
+ body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"
raw = @http.post("/v2/scrape", body)
data = raw["data"] || raw
Models::Document.new(data)
@@ -96,6 +98,7 @@ module Firecrawl
body = { "code" => code, "language" => language }
body["timeout"] = timeout if timeout
+ body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"
@http.post("/v2/scrape/#{job_id}/interact", body)
end
@@ -377,6 +380,7 @@ module Firecrawl
body = { "query" => query }
body.merge!(options.to_h) if options
+ body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"
raw = @http.post("/v2/search", body)
data = raw["data"] || raw
Models::SearchData.new(data)
diff --git a/apps/ruby-sdk/lib/firecrawl/http_client.rb b/apps/ruby-sdk/lib/firecrawl/http_client.rb
index 4d1a819e0..3fc743a5d 100644
--- a/apps/ruby-sdk/lib/firecrawl/http_client.rb
+++ b/apps/ruby-sdk/lib/firecrawl/http_client.rb
@@ -23,7 +23,7 @@ module Firecrawl
def post(path, body, extra_headers: {})
uri = URI("#{@base_url}#{path}")
request = Net::HTTP::Post.new(uri)
- request["Authorization"] = "Bearer #{@api_key}"
+ request["Authorization"] = "Bearer #{@api_key}" if @api_key
request["Content-Type"] = "application/json"
extra_headers.each { |k, v| request[k] = v }
request.body = JSON.generate(body)
@@ -34,7 +34,7 @@ module Firecrawl
def get(path)
uri = URI("#{@base_url}#{path}")
request = Net::HTTP::Get.new(uri)
- request["Authorization"] = "Bearer #{@api_key}"
+ request["Authorization"] = "Bearer #{@api_key}" if @api_key
execute_with_retry(uri, request)
end
@@ -47,7 +47,7 @@ module Firecrawl
raise FirecrawlError, "Absolute URL origin (#{uri.scheme}://#{uri.host}:#{uri.port}) does not match API base URL origin (#{base_uri.scheme}://#{base_uri.host}:#{base_uri.port}). Refusing to send credentials."
end
request = Net::HTTP::Get.new(uri)
- request["Authorization"] = "Bearer #{@api_key}"
+ request["Authorization"] = "Bearer #{@api_key}" if @api_key
execute_with_retry(uri, request)
end
@@ -55,7 +55,7 @@ module Firecrawl
def delete(path)
uri = URI("#{@base_url}#{path}")
request = Net::HTTP::Delete.new(uri)
- request["Authorization"] = "Bearer #{@api_key}"
+ request["Authorization"] = "Bearer #{@api_key}" if @api_key
execute_with_retry(uri, request)
end
@@ -63,7 +63,7 @@ module Firecrawl
def patch(path, body)
uri = URI("#{@base_url}#{path}")
request = Net::HTTP::Patch.new(uri)
- request["Authorization"] = "Bearer #{@api_key}"
+ request["Authorization"] = "Bearer #{@api_key}" if @api_key
request["Content-Type"] = "application/json"
request.body = JSON.generate(body)
execute_with_retry(uri, request)
@@ -84,7 +84,7 @@ module Firecrawl
builder = lambda do
request = Net::HTTP::Post.new(uri)
- request["Authorization"] = "Bearer #{@api_key}"
+ request["Authorization"] = "Bearer #{@api_key}" if @api_key
request["Content-Type"] = "multipart/form-data; boundary=#{boundary}"
request.body = body
request
diff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rb
index a249e3949..52751598c 100644
--- a/apps/ruby-sdk/lib/firecrawl/version.rb
+++ b/apps/ruby-sdk/lib/firecrawl/version.rb
@@ -1,5 +1,5 @@
# frozen_string_literal: true
module Firecrawl
- VERSION = "1.8.0"
+ VERSION = "1.8.1"
end
diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock
index 3d851d6a2..091dc733c 100644
--- a/apps/rust-sdk/Cargo.lock
+++ b/apps/rust-sdk/Cargo.lock
@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "firecrawl"
-version = "2.8.0"
+version = "2.8.1"
dependencies = [
"mockito",
"reqwest",
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index e02c5b8ef..a924c654b 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "firecrawl"
-version = "2.8.0"
+version = "2.8.1"
edition = "2021"
license = "MIT"
homepage = "https://www.firecrawl.dev/"
diff --git a/apps/rust-sdk/src/client.rs b/apps/rust-sdk/src/client.rs
index 17a91b482..d0d520b3f 100644
--- a/apps/rust-sdk/src/client.rs
+++ b/apps/rust-sdk/src/client.rs
@@ -4,7 +4,7 @@ use reqwest::Response;
use serde::de::DeserializeOwned;
use serde_json::Value;
-use crate::error::{FirecrawlAPIError, FirecrawlError};
+use crate::error::FirecrawlError;
pub(crate) const API_VERSION: &str = "/v2";
const CLOUD_API_URL: &str = "https://api.firecrawl.dev";
@@ -89,32 +89,9 @@ impl Client {
let url = api_url.as_ref().trim_end_matches('/').to_string();
let api_key = api_key.map(|k| k.as_ref().to_string());
- // Reject empty or missing API key for cloud service
- if url == CLOUD_API_URL {
- match &api_key {
- None => {
- return Err(FirecrawlError::APIError(
- "Configuration".to_string(),
- FirecrawlAPIError {
- success: false,
- error: "API key is required for cloud service".to_string(),
- details: None,
- },
- ));
- }
- Some(key) if key.trim().is_empty() => {
- return Err(FirecrawlError::APIError(
- "Configuration".to_string(),
- FirecrawlAPIError {
- success: false,
- error: "API key cannot be empty for cloud service".to_string(),
- details: None,
- },
- ));
- }
- _ => {}
- }
- }
+ // An empty/missing API key is allowed: scrape, search, and interact fall
+ // back to the keyless free tier (rate-limited per IP). Other methods
+ // return 401 from the API until a key is provided.
Ok(Client {
api_key,
@@ -134,10 +111,12 @@ impl Client {
// Static string is always valid ASCII
headers.insert("Content-Type", HeaderValue::from_static("application/json"));
if let Some(api_key) = self.api_key.as_ref() {
- // API key is validated at client creation, so this should always succeed.
- // Use if-let to gracefully handle edge cases without panicking.
- if let Ok(value) = format!("Bearer {}", api_key).parse() {
- headers.insert("Authorization", value);
+ // Omit the Authorization header entirely when no key is set so that
+ // scrape/search/interact can use the keyless free tier.
+ if !api_key.trim().is_empty() {
+ if let Ok(value) = format!("Bearer {}", api_key).parse() {
+ headers.insert("Authorization", value);
+ }
}
}
if let Some(key) = idempotency_key {
diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rs
index 7f8aab88b..94820b20b 100644
--- a/apps/rust-sdk/src/scrape.rs
+++ b/apps/rust-sdk/src/scrape.rs
@@ -16,6 +16,9 @@ use crate::FirecrawlError;
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeOptions {
+ /// Origin label for request attribution (e.g., "[email protected]").
+ pub origin: Option<String>,
+
/// Output formats to include in the response.
pub formats: Option<Vec<Format>>,
@@ -247,9 +250,13 @@ impl Client {
url: impl AsRef<str>,
options: impl Into<Option<ScrapeOptions>>,
) -> Result<Document, FirecrawlError> {
+ let mut options = options.into().unwrap_or_default();
+ if options.origin.is_none() {
+ options.origin = Some(format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")));
+ }
let body = ScrapeRequest {
url: url.as_ref().to_string(),
- options: options.into().unwrap_or_default(),
+ options,
};
let headers = self.prepare_headers(None);
@@ -361,6 +368,9 @@ impl Client {
if body.language.is_none() {
body.language = Some(ScrapeExecuteLanguage::Node);
}
+ if body.origin.is_none() {
+ body.origin = Some(format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")));
+ }
let response = self
.client
diff --git a/apps/rust-sdk/src/search.rs b/apps/rust-sdk/src/search.rs
index a6d58436f..945f724ea 100644
--- a/apps/rust-sdk/src/search.rs
+++ b/apps/rust-sdk/src/search.rs
@@ -46,6 +46,9 @@ pub struct SearchOptions {
/// Integration identifier for tracking.
pub integration: Option<String>,
+
+ /// Origin label for request attribution (e.g., "[email protected]").
+ pub origin: Option<String>,
}
/// Request body for search endpoint.
@@ -173,9 +176,13 @@ impl Client {
query: impl AsRef<str>,
options: impl Into<Option<SearchOptions>>,
) -> Result<SearchResponse, FirecrawlError> {
+ let mut options = options.into().unwrap_or_default();
+ if options.origin.is_none() {
+ options.origin = Some(format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")));
+ }
let body = SearchRequest {
query: query.as_ref().to_string(),
- options: options.into().unwrap_or_default(),
+ options,
};
let headers = self.prepare_headers(None);