Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore: updates (#3787)
GergΕ MΓ³ricz722d02f64c1e
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.tsnew file mode 100644index 000000000..cbc494f28--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts@@ -0,0 +1,312 @@+import { config } from "../../../config";+import {+ describeIf,+ itIf,+ TEST_API_URL,+ TEST_PRODUCTION,+ TEST_SUITE_WEBSITE,+ scrapeTimeout,+} from "../lib";+import { redisRateLimitClient } from "../../../services/rate-limiter";+import request from "supertest";++// The keyless tier is disabled unless both limits are configured. The harness+// passes the shell env through to the server, so we read the same env here and+// only run when it's set (and use the configured values for cap assertions).+const KEYLESS_REQUESTS_PER_DAY = Number(process.env.KEYLESS_REQUESTS_PER_DAY);+const KEYLESS_CREDITS_PER_DAY = Number(process.env.KEYLESS_CREDITS_PER_DAY);+const KEYLESS_ENABLED =+ TEST_PRODUCTION &&+ Number.isFinite(KEYLESS_REQUESTS_PER_DAY) &&+ KEYLESS_REQUESTS_PER_DAY > 0 &&+ Number.isFinite(KEYLESS_CREDITS_PER_DAY) &&+ KEYLESS_CREDITS_PER_DAY > 0;++// Keyless free tier: scrape, search, and interact work without an API key from+// the official MCP server (origin "mcp*"), CLI (integration "cli"), or SDKs+// (origin "<lang>-sdk@..."). Gated per-IP/day by a request cap AND a credit cap.+// These tests are the only ones that exercise keyless mode, so they exclusively+// own the `keyless_*` rate-limit keyspace β we flush it before each test (and run+// sequentially) for isolation. `req.ip` is loopback under supertest, so the+// per-IP counters are shared across requests here; flushing keeps tests hermetic.++async function flushKeylessBuckets() {+ const keys = await redisRateLimitClient.keys("keyless_*");+ if (keys.length > 0) {+ await redisRateLimitClient.del(...keys);+ }+}++// Recover the loopback IP the server keyed on, so we can seed its credit counter.+async function currentKeylessIp(): Promise<string> {+ const keys = await redisRateLimitClient.keys("keyless_requests:*");+ expect(keys.length).toBeGreaterThan(0);+ return keys[0].slice("keyless_requests:".length);+}++describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {+ beforeAll(() => {+ config.USE_DB_AUTHENTICATION = true;+ });++ afterAll(async () => {+ delete config.USE_DB_AUTHENTICATION;+ await flushKeylessBuckets();+ });++ beforeEach(async () => {+ await flushKeylessBuckets();+ });++ it(+ "allows keyless scrape from a raw API caller (no origin gate) (200)",+ async () => {+ // origin "api" (and no origin at all) is now eligible β the API itself is+ // free without a key on the allowlisted endpoints.+ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ url: TEST_SUITE_WEBSITE, origin: "api", formats: ["markdown"] });++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ },+ scrapeTimeout,+ );++ it(+ "allows keyless scrape with no origin field at all (200)",+ async () => {+ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ url: TEST_SUITE_WEBSITE, formats: ["markdown"] });++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ },+ scrapeTimeout,+ );++ it("does not grant keyless access on non-allowlisted endpoints (401)", async () => {+ // batch/scrape shares RateLimiterMode.Scrape but is NOT allowKeyless.+ const response = await request(TEST_API_URL)+ .post("/v2/batch/scrape")+ .set("Content-Type", "application/json")+ .send({ urls: ["https://example.com"], origin: "[email protected]" });++ expect(response.statusCode).toBe(401);+ expect(response.body.success).toBe(false);+ expect(response.body.error).toContain("not available without an API key");+ });++ it("enforces the daily request cap with the request signup message (429)", async () => {+ // Each request consumes a slot during auth, then fails fast in the controller+ // (no url) β so we exhaust the cap without real scrapes, and 0 credits.+ for (let i = 0; i < KEYLESS_REQUESTS_PER_DAY; i++) {+ const r = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ origin: "mcp" });+ expect(r.statusCode).not.toBe(429);+ }++ const blocked = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ origin: "mcp" });++ expect(blocked.statusCode).toBe(429);+ expect(blocked.body.error).toContain("unauthenticated requests");+ // Out of quota β emit the OAuth-discovery header so agents find the key flow.+ expect(blocked.headers["www-authenticate"]).toContain("resource_metadata");+ });++ it("enforces the daily credit cap with the credit signup message (429)", async () => {+ // Materialize the loopback IP, then seed its credit counter to the cap.+ await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ origin: "mcp" });+ const ip = await currentKeylessIp();+ await redisRateLimitClient.set(+ `keyless_credits:${ip}`,+ String(KEYLESS_CREDITS_PER_DAY),+ );++ const blocked = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ origin: "mcp" });++ expect(blocked.statusCode).toBe(429);+ expect(blocked.body.error).toContain("unauthenticated credits");+ });++ itIf(!!process.env.KEYLESS_PROXY_SECRET)(+ "rate-limits per forwarded client IP when the proxy secret is provided",+ async () => {+ const fakeIp = "203.0.113.7";+ // Seed the forwarded IP's credit counter to the cap.+ await redisRateLimitClient.set(+ `keyless_credits:${fakeIp}`,+ String(KEYLESS_CREDITS_PER_DAY),+ );++ // With the secret, the cap is keyed on the forwarded IP -> blocked.+ const blocked = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)+ .set("x-firecrawl-keyless-ip", fakeIp)+ .send({ origin: "mcp" });+ expect(blocked.statusCode).toBe(429);+ expect(blocked.body.error).toContain("unauthenticated credits");++ // Without the secret, the forwarded IP is ignored (keyed on the real IP).+ const allowed = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .set("x-firecrawl-keyless-ip", fakeIp)+ .send({ origin: "mcp" });+ expect(allowed.statusCode).not.toBe(429);+ },+ );++ itIf(!!process.env.KEYLESS_PROXY_SECRET)(+ "denies keyless access to IPv6 clients (401)",+ async () => {+ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)+ .set("x-firecrawl-keyless-ip", "2001:db8::1")+ .send({ url: TEST_SUITE_WEBSITE, origin: "mcp" });++ // IPv6 is not eligible β falls through to the normal unauthorized path.+ expect(response.statusCode).toBe(401);+ },+ );++ itIf(!!process.env.KEYLESS_PROXY_SECRET)(+ "denies keyless for a malformed forwarded IP (401)",+ async () => {+ // A non-IP forwarded value must not be usable as a limiter bucket.+ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)+ .set("x-firecrawl-keyless-ip", "not-an-ip")+ .send({ url: TEST_SUITE_WEBSITE, origin: "mcp" });++ expect(response.statusCode).toBe(401);+ },+ );++ itIf(!!process.env.KEYLESS_PROXY_SECRET)(+ "keyless eligibility endpoint reflects cap state (hosted MCP probe)",+ async () => {+ const ip = "203.0.113.40";++ // Fresh IP under cap β eligible.+ const ok = await request(TEST_API_URL)+ .get("/v2/keyless/eligibility")+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)+ .set("x-firecrawl-keyless-ip", ip);+ expect(ok.statusCode).toBe(200);+ expect(ok.body.eligible).toBe(true);++ // Seed over the credit cap β ineligible (so the MCP would issue an OAuth+ // challenge instead of serving keyless).+ await redisRateLimitClient.set(+ `keyless_credits:${ip}`,+ String(KEYLESS_CREDITS_PER_DAY),+ );+ const capped = await request(TEST_API_URL)+ .get("/v2/keyless/eligibility")+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)+ .set("x-firecrawl-keyless-ip", ip);+ expect(capped.body.eligible).toBe(false);++ // Without the secret β rejected (no leaking eligibility to untrusted callers).+ const noSecret = await request(TEST_API_URL)+ .get("/v2/keyless/eligibility")+ .set("x-firecrawl-keyless-ip", ip);+ expect(noSecret.statusCode).toBe(401);+ },+ );++ it(+ "grants keyless interact access (past auth, not 401)",+ async () => {+ // A random (non-existent) job id: the request should clear keyless auth and+ // fail downstream, proving interact is allowKeyless β not return 401.+ const response = await request(TEST_API_URL)+ .post("/v2/scrape/00000000-0000-4000-8000-000000000000/interact")+ .set("Content-Type", "application/json")+ .send({ code: "return 1;", origin: "mcp" });++ expect(response.statusCode).not.toBe(401);+ },+ scrapeTimeout,+ );++ it(+ "allows keyless scrape from an mcp origin (200)",+ async () => {+ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ url: TEST_SUITE_WEBSITE, origin: "mcp" });++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(typeof response.body.data).toBe("object");+ },+ scrapeTimeout,+ );++ it(+ "allows keyless scrape from a cli integration (200)",+ async () => {+ // The CLI identifies itself via the `integration` field, not `origin`.+ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ url: TEST_SUITE_WEBSITE, integration: "cli" });++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ },+ scrapeTimeout,+ );++ it(+ "allows keyless scrape from an SDK origin (200)",+ async () => {+ // SDKs send origin like "[email protected]" / "[email protected]".+ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ url: TEST_SUITE_WEBSITE, origin: "[email protected]" });++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ },+ scrapeTimeout,+ );++ it(+ "allows keyless search from an mcp origin (200)",+ async () => {+ const response = await request(TEST_API_URL)+ .post("/v2/search")+ .set("Content-Type", "application/json")+ .send({ query: "firecrawl", limit: 1, origin: "mcp" });++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ },+ scrapeTimeout,+ );+});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex ca83f0f68..06abdadf7 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -50,6 +50,15 @@ const configSchema = z.object({// ExpressEXPRESS_TRUST_PROXY: z.coerce.number().optional(),+ // Keyless free tier (scrape/search/interact without an API key, per-IP/day).+ // Non-negative integers; 0 means "enabled but no budget", unset means "off".+ KEYLESS_CREDITS_PER_DAY: z.coerce.number().int().nonnegative().optional(),+ KEYLESS_REQUESTS_PER_DAY: z.coerce.number().int().nonnegative().optional(),+ // Shared secret that lets a trusted proxy (e.g. the hosted MCP server)+ // forward the real client IP for keyless rate-limiting via the+ // `x-firecrawl-keyless-ip` header. Untrusted callers can't override their IP.+ KEYLESS_PROXY_SECRET: z.string().optional(),+// API Keys & AuthenticationBULL_AUTH_KEY: z.string().optional(),OPENAI_API_KEY: z.string().optional(),diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 0e43c324b..0f6bdfc5e 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -8,6 +8,12 @@ import { withAuth } from "../lib/withAuth";import { getAgentSponsorStatus } from "../services/agent-sponsor";import { getRedisConnection } from "../services/queue-service";import { getRateLimiter } from "../services/rate-limiter";+import {+ consumeKeylessRequest,+ isKeylessConfigured,+ isKeylessIpEligible,+ keylessTeamId,+} from "../lib/keyless";import { deleteKey, getValue, setValue } from "../services/redis";import { redlock } from "../services/redlock";import { eq } from "drizzle-orm";@@ -462,17 +468,151 @@ export async function clearACUCTeam(team_id: string): Promise<void> {await getRedisConnection().sadd("billed_teams", team_id);}+const KEYLESS_REQUESTS_MESSAGE = `You've reached today's limit of free, unauthenticated requests to Firecrawl. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;++const KEYLESS_CREDITS_MESSAGE = `You've reached today's limit of free, unauthenticated credits for Firecrawl. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;++const KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE = `This endpoint is not available without an API key. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;++/**+ * Keyless free tier: official MCP/CLI/SDK clients can call scrape, search, and+ * interact with no API key. `origin`/`integration` are client-set and spoofable,+ * so they're only a soft UX gate β the real abuse controls are the per-IP daily+ * request + credit caps plus the `keyless/consume` canonical log emitted here.+ * Always returns an AuthResponse β handles every no-API-key request.+ */+async function handleKeylessAuth(+ req,+ mode: RateLimiterMode | undefined,+ allowKeyless: boolean | undefined,+): Promise<AuthResponse> {+ const unauthorized: AuthResponse = {+ success: false,+ error: "Unauthorized",+ status: 401,+ };++ // The keyless tier is off unless BOTH limits are configured (even to 0). When+ // unconfigured we behave exactly as before β a generic 401 β and don't reveal+ // that the tier exists.+ if (!isKeylessConfigured()) return unauthorized;++ // Configured, but this endpoint isn't part of the keyless tier: tell the user+ // they need a key (with the signup nudge) rather than a bare "Unauthorized".+ if (!allowKeyless) {+ return {+ success: false,+ error: KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE,+ status: 401,+ };+ }++ const origin = req.body?.origin;+ const integration = req.body?.integration;+ // No origin/surface gate: any request without an API key may use the free+ // tier on the allowlisted endpoints (the API itself is free). origin and+ // integration are still recorded below for abuse monitoring.++ // Key on the real client IP. A trusted proxy (e.g. the hosted MCP) may+ // forward the end-user's IP via x-firecrawl-keyless-ip, authenticated with a+ // shared secret β without the secret the header is ignored, so direct callers+ // can't spoof their IP to dodge the per-IP cap.+ let ip = req.ip ?? req.socket?.remoteAddress ?? "unknown";+ if (+ config.KEYLESS_PROXY_SECRET &&+ req.headers["x-firecrawl-keyless-secret"] === config.KEYLESS_PROXY_SECRET+ ) {+ const forwarded = req.headers["x-firecrawl-keyless-ip"];+ if (typeof forwarded === "string" && forwarded.trim()) {+ ip = forwarded.trim();+ }+ }++ // Only a valid IPv4 identity gets keyless: IPv6 is too cheap to rotate for a+ // per-IP cap to mean anything, and malformed/forwarded values must not be+ // usable as arbitrary limiter buckets. Anything else falls through to 401.+ if (!isKeylessIpEligible(ip)) return unauthorized;++ const teamId = keylessTeamId(ip);+ const modeLabel =+ mode === RateLimiterMode.Search+ ? "search"+ : mode === RateLimiterMode.BrowserExecute+ ? "interact"+ : "scrape";++ let result: Awaited<ReturnType<typeof consumeKeylessRequest>>;+ try {+ result = await consumeKeylessRequest(ip);+ } catch (error) {+ // Limiter store (Redis) unavailable β fail closed with a controlled auth+ // response instead of surfacing a 500, and shed the free traffic while the+ // limiter can't enforce quotas.+ logger.warn("Keyless quota check failed", {+ canonicalLog: "keyless/consume",+ ip,+ mode: modeLabel,+ teamId,+ error,+ });+ return unauthorized;+ }+ const baseLog = {+ canonicalLog: "keyless/consume",+ ip,+ origin,+ integration,+ mode: modeLabel,+ teamId,+ requestsUsed: result.requestsUsed,+ creditsUsed: result.creditsUsed,+ };++ if (!result.ok) {+ logger.warn("Keyless request blocked", {+ ...baseLog,+ blocked: true,+ reason: result.reason,+ });+ return {+ success: false,+ error:+ result.reason === "credits"+ ? KEYLESS_CREDITS_MESSAGE+ : KEYLESS_REQUESTS_MESSAGE,+ status: 429,+ // Out of free quota β emit the OAuth-discovery header so agents can find+ // the key/signup flow at the moment they actually need a key.+ agentAuthDiscovery: true,+ };+ }++ logger.debug("Keyless request consumed", { ...baseLog, blocked: false });++ // Tag as a preview team so billing (autumn isPreviewTeam) and GCS persistence+ // are skipped automatically; mockPreviewACUC supplies concurrency 2 + credits.+ // Actual credits consumed are charged to the IP's daily budget after the+ // request completes (see chargeKeylessCredits).+ return {+ success: true,+ team_id: teamId,+ org_id: null,+ chunk: mockPreviewACUC(teamId, false),+ };+}+export async function authenticateUser(req,res,mode?: RateLimiterMode,+ options?: { allowKeyless?: boolean },): Promise<AuthResponse> {return withAuth(supaAuthenticateUser, {success: true,chunk: null,team_id: "bypass",org_id: null,- })(req, res, mode);+ })(req, res, mode, options);}/**@@ -517,6 +657,7 @@ async function supaAuthenticateUser(req,res,mode?: RateLimiterMode,+ options?: { allowKeyless?: boolean },): Promise<AuthResponse> {const authHeader =req.headers.authorization ??@@ -524,7 +665,7 @@ async function supaAuthenticateUser(? `Bearer ${req.headers["sec-websocket-protocol"]}`: null);if (!authHeader) {- return { success: false, error: "Unauthorized", status: 401 };+ return handleKeylessAuth(req, mode, options?.allowKeyless);}const token = authHeader.split(" ")[1]; // Extract the token from "Bearer <token>"if (!token) {diff --git a/apps/api/src/controllers/v2/keyless-eligibility.ts b/apps/api/src/controllers/v2/keyless-eligibility.tsnew file mode 100644index 000000000..f86701a19--- /dev/null+++ b/apps/api/src/controllers/v2/keyless-eligibility.ts@@ -0,0 +1,28 @@+import { Request, Response } from "express";+import { config } from "../../config";+import { checkKeylessEligibility } from "../../lib/keyless";++/**+ * Internal endpoint for trusted proxies (the hosted MCP) to check, at connect+ * time, whether a client IP can currently use the keyless tier β without+ * consuming quota. Gated by the shared KEYLESS_PROXY_SECRET; the client IP is+ * supplied via x-firecrawl-keyless-ip. Lets the MCP serve keyless when eligible+ * and throw (β OAuth 401 challenge) when the IP is out of free quota.+ */+export async function keylessEligibilityController(+ req: Request,+ res: Response,+): Promise<void> {+ const secret = req.headers["x-firecrawl-keyless-secret"];+ if (!config.KEYLESS_PROXY_SECRET || secret !== config.KEYLESS_PROXY_SECRET) {+ res.status(401).json({ eligible: false, error: "Unauthorized" });+ return;+ }++ const ipHeader = req.headers["x-firecrawl-keyless-ip"];+ const ip =+ (typeof ipHeader === "string" ? ipHeader.trim() : "") || req.ip || "";++ const result = await checkKeylessEligibility(ip);+ res.status(200).json(result);+}diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 08f46feb4..8056e9d1b 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -45,6 +45,7 @@ import { sanitizeUrlForTrace } from "../../lib/scrape-interact/langsmith";import { getScrapeZDR } from "../../lib/zdr-helpers";import { RequestWithAuth, ScrapeOptions } from "./types";import { billTeam } from "../../services/billing/credit_billing";+import { chargeKeylessCredits, keylessTeamUuid } from "../../lib/keyless";import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";import { logRequest } from "../../services/logging/log_job";import { integrationSchema } from "../../utils/integration";@@ -141,7 +142,12 @@ export async function scrapeInteractController(if (!scrape) {return res.status(404).json({ success: false, error: "Job not found." });}- if (scrape.team_id !== req.auth.team_id) {+ // Keyless scrapes are persisted under a deterministic per-IP UUID (the+ // `scrapes.team_id` column is a UUID, so the raw `preview_keyless_<ip>` string+ // can't be stored). Compare against that derived UUID for keyless requests.+ const expectedScrapeTeam =+ keylessTeamUuid(req.auth.team_id) ?? req.auth.team_id;+ if (scrape.team_id !== expectedScrapeTeam) {return res.status(403).json({ success: false, error: "Forbidden." });}@@ -457,6 +463,10 @@ export async function scrapeStopInteractiveBrowserController(});});+ // Charge the keyless free tier's per-IP daily credit budget (no-op for+ // non-keyless teams).+ chargeKeylessCredits(req.auth.team_id, creditsBilled).catch(() => {});+logger.info("Browser session destroyed", {sessionDurationMs: durationMs,creditsBilled,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 9a79a57d8..1db7b506f 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -7,6 +7,7 @@ import {searchRequestSchema,} from "./types";import { billTeam } from "../../services/billing/credit_billing";+import { chargeKeylessCredits } from "../../lib/keyless";import { v7 as uuidv7 } from "uuid";import { logSearch, logRequest } from "../../services/logging/log_job";import { logger as _logger } from "../../lib/logger";@@ -169,6 +170,13 @@ export async function searchController();}+ // Charge the keyless free tier's per-IP daily credit budget for search+ // credits (no-op for non-keyless teams; scrape jobs within search charge+ // themselves via the scrape worker).+ chargeKeylessCredits(req.auth.team_id, result.searchCredits).catch(+ () => {},+ );+const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex 8dc81b804..e5d46af89 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -20,6 +20,16 @@ const ts = (name: string) =>const bigintNum = (name: string) => bigint(name, { mode: "number" });const num = (name: string) => numeric(name, { mode: "number" });+// Keyless free-tier credit usage log (see keyless_credit_usage migration).+// team_id is the deterministic per-IP keyless team UUID; ip is the raw client IP.+export const keyless_credit_usage = pgTable("keyless_credit_usage", {+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),+ team_id: uuid("team_id").notNull(),+ ip: text("ip").notNull(),+ credits_used: integer("credits_used").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+});+export const agent_sponsors = pgTable("agent_sponsors", {id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),email: text("email").notNull(),diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsnew file mode 100644index 000000000..25f88480b--- /dev/null+++ b/apps/api/src/lib/keyless.ts@@ -0,0 +1,193 @@+import { isIPv4 } from "node:net";+import { v5 as uuidv5 } from "uuid";+import { config } from "../config";+import { db } from "../db/connection";+import * as schema from "../db/schema";+import { redisRateLimitClient } from "../services/rate-limiter";++// Keyless free tier: scrape, search, and interact can be used without an API key+// from the official MCP server, CLI, or SDKs. It's gated per-IP/day by TWO+// limits, both configurable via env: a request count and a credit budget.+// `origin`/`integration` are client-set and spoofable, so they're only a soft+// gate to keep raw API callers on the signup path β the per-IP daily caps plus+// the `keyless/consume` canonical log are the real abuse controls.++// No defaults: the keyless free tier is OFF unless BOTH limits are configured.+export const KEYLESS_REQUESTS_PER_DAY = config.KEYLESS_REQUESTS_PER_DAY;+export const KEYLESS_CREDITS_PER_DAY = config.KEYLESS_CREDITS_PER_DAY;++// The tier is "configured" when BOTH limits are set β even to 0. Unset means the+// feature is off (callers get a plain Unauthorized); 0 means it's on but the+// budget is exhausted (callers get the 429 cap message).+export function isKeylessConfigured(): boolean {+ return (+ typeof KEYLESS_REQUESTS_PER_DAY === "number" &&+ typeof KEYLESS_CREDITS_PER_DAY === "number"+ );+}++const DAY_SECONDS = 86400;++// Keyless teams reuse the `preview_` prefix so billing (autumn `isPreviewTeam`)+// and GCS persistence are skipped automatically, with a dedicated infix so the+// IP can be recovered when charging credits after a request completes.+export const KEYLESS_TEAM_PREFIX = "preview_keyless_";++export function keylessTeamId(ip: string): string {+ return `${KEYLESS_TEAM_PREFIX}${ip}`;+}++export function keylessIpFromTeamId(teamId: string): string | null {+ return teamId.startsWith(KEYLESS_TEAM_PREFIX)+ ? teamId.slice(KEYLESS_TEAM_PREFIX.length)+ : null;+}++// Fixed namespace for deriving a stable per-keyless-team UUID. Tables like+// `scrapes` require a UUID team_id, but keyless teams are `preview_keyless_<ip>`+// strings. Mapping each to a deterministic UUIDv5 keeps rows per-IP-distinct+// (so ownership checks such as interact still isolate keyless users) while+// satisfying the UUID column β unlike the shared preview placeholder.+const KEYLESS_TEAM_UUID_NAMESPACE = "9e6c8f2a-3b1d-4c7e-8a5f-2d4b6e8c0a1f";++/**+ * Deterministic UUID for a keyless team's persisted rows, or null for+ * non-keyless teams (callers then fall back to the raw/placeholder team_id).+ */+export function keylessTeamUuid(+ teamId: string | null | undefined,+): string | null {+ if (!teamId || !teamId.startsWith(KEYLESS_TEAM_PREFIX)) return null;+ return uuidv5(teamId, KEYLESS_TEAM_UUID_NAMESPACE);+}++/**+ * Keyless is allowed only for a *valid IPv4* client identity. This both:+ * - denies IPv6 (a single client controls a huge block β a /64 is ~18+ * quintillion addresses β so a per-IP cap is trivially bypassed), and+ * - rejects malformed/unknown values (e.g. a forwarded `x-firecrawl-keyless-ip`+ * that isn't a real IP), so they can't be used to mint arbitrary buckets and+ * weaken per-IP quota enforcement.+ * IPv4-mapped IPv6 (e.g. "::ffff:1.2.3.4", how dual-stack sockets surface IPv4)+ * is treated as IPv4.+ */+export function isKeylessIpEligible(ip: string): boolean {+ const normalized = ip.startsWith("::ffff:")+ ? ip.slice("::ffff:".length)+ : ip;+ return isIPv4(normalized);+}++const requestsKey = (ip: string) => `keyless_requests:${ip}`;+const creditsKey = (ip: string) => `keyless_credits:${ip}`;++export type KeylessConsumeResult = {+ ok: boolean;+ reason?: "requests" | "credits";+ requestsUsed: number;+ creditsUsed: number;+};++/**+ * Consume one request from the per-IP daily request budget and check the credit+ * budget (credits are charged after the request completes, in+ * `chargeKeylessCredits`). Returns whether the request may proceed.+ */+export async function consumeKeylessRequest(+ ip: string,+): Promise<KeylessConsumeResult> {+ const requestLimit = KEYLESS_REQUESTS_PER_DAY ?? 0;+ const creditLimit = KEYLESS_CREDITS_PER_DAY ?? 0;++ const rKey = requestsKey(ip);+ const requestsUsed = await redisRateLimitClient.incr(rKey);+ if (requestsUsed === 1) {+ await redisRateLimitClient.expire(rKey, DAY_SECONDS);+ }++ const creditsUsed = parseInt(+ (await redisRateLimitClient.get(creditsKey(ip))) ?? "0",+ 10,+ );++ if (requestsUsed > requestLimit) {+ return { ok: false, reason: "requests", requestsUsed, creditsUsed };+ }+ if (creditsUsed >= creditLimit) {+ return { ok: false, reason: "credits", requestsUsed, creditsUsed };+ }+ return { ok: true, requestsUsed, creditsUsed };+}++/**+ * Read-only check of whether an IP could currently use the keyless tier (no+ * consumption). Used by the hosted MCP to decide, at connect time, whether to+ * serve keyless (eligible) or throw so FastMCP emits the OAuth challenge (not).+ */+export async function checkKeylessEligibility(+ ip: string,+): Promise<{ eligible: boolean; reason?: string }> {+ if (!isKeylessConfigured()) return { eligible: false, reason: "disabled" };+ if (!ip || !isKeylessIpEligible(ip)) {+ return { eligible: false, reason: "ineligible_ip" };+ }+ try {+ const requestsUsed = parseInt(+ (await redisRateLimitClient.get(requestsKey(ip))) ?? "0",+ 10,+ );+ if (requestsUsed >= (KEYLESS_REQUESTS_PER_DAY ?? 0)) {+ return { eligible: false, reason: "requests" };+ }+ const creditsUsed = parseInt(+ (await redisRateLimitClient.get(creditsKey(ip))) ?? "0",+ 10,+ );+ if (creditsUsed >= (KEYLESS_CREDITS_PER_DAY ?? 0)) {+ return { eligible: false, reason: "credits" };+ }+ return { eligible: true };+ } catch {+ // Limiter store unavailable β fail closed so the MCP issues an OAuth+ // challenge rather than granting unbounded keyless.+ return { eligible: false, reason: "error" };+ }+}++/**+ * Add the actual credits a completed request consumed to the IP's daily credit+ * counter. No-op for non-keyless teams. Best-effort; never throws.+ */+export async function chargeKeylessCredits(+ teamId: string,+ credits: number,+): Promise<void> {+ const ip = keylessIpFromTeamId(teamId);+ if (!ip || !Number.isFinite(credits) || credits <= 0) return;+ const inc = Math.ceil(credits);+ try {+ const key = creditsKey(ip);+ const total = await redisRateLimitClient.incrby(key, inc);+ if (total === inc) {+ await redisRateLimitClient.expire(key, DAY_SECONDS);+ }+ } catch {+ // Counter is best-effort; a missed charge just means the IP gets a few+ // extra free credits today.+ }++ // Log the usage to keyless_credit_usage (per-IP keyless team UUID + raw IP)+ // for abuse monitoring. Best-effort β never block the request.+ const teamUuid = keylessTeamUuid(teamId);+ if (config.USE_DB_AUTHENTICATION === true && teamUuid) {+ try {+ await db.insert(schema.keyless_credit_usage).values({+ team_id: teamUuid,+ ip,+ credits_used: inc,+ });+ } catch {+ // Logging is best-effort.+ }+ }+}diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex f63be759e..c205eb085 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -196,6 +196,7 @@ export function checkCreditsMiddleware(export function authMiddleware(rateLimiterMode: RateLimiterMode,+ options: { allowKeyless?: boolean } = {},): (req: RequestWithMaybeAuth, res: Response, next: NextFunction) => void {return (req, res, next) => {(async () => {@@ -211,11 +212,18 @@ export function authMiddleware(// currentRateLimiterMode = RateLimiterMode.ScrapeAgentPreview;// }- const auth = await authenticateUser(req, res, currentRateLimiterMode);+ const auth = await authenticateUser(+ req,+ res,+ currentRateLimiterMode,+ options,+ );if (!auth.success) {if (!res.headersSent) {- if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);+ if (auth.status === 401 || auth.agentAuthDiscovery) {+ applyAgentAuthDiscoveryHeader(res);+ }return res.status(auth.status).json({ success: false, error: auth.error });diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 5e3b0d930..5ceddf19a 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -124,7 +124,7 @@ v1Router.use(requestTimingMiddleware("v1"));v1Router.post("/scrape",- authMiddleware(RateLimiterMode.Scrape),+ authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),countryCheck,checkCreditsMiddleware(1),blocklistMiddleware,@@ -153,7 +153,7 @@ v1Router.post(v1Router.post("/search",- authMiddleware(RateLimiterMode.Search),+ authMiddleware(RateLimiterMode.Search, { allowKeyless: true }),countryCheck,checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),wrap(searchController),diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 71e98d89b..0561b0b7d 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -8,6 +8,7 @@ import { searchController } from "../controllers/v2/search";import { searchFeedbackController } from "../controllers/v2/search-feedback";import { x402SearchController } from "../controllers/v2/x402-search";import { scrapeController } from "../controllers/v2/scrape";+import { keylessEligibilityController } from "../controllers/v2/keyless-eligibility";import {parseController,parseMultipartPayloadMiddleware,@@ -234,9 +235,13 @@ v2Router.use(requestTimingMiddleware("v2"));// ),// );+// Internal: trusted-proxy (hosted MCP) keyless eligibility probe. Secret-gated+// inside the controller; no auth middleware.+v2Router.get("/keyless/eligibility", wrap(keylessEligibilityController));+v2Router.post("/search",- authMiddleware(RateLimiterMode.Search),+ authMiddleware(RateLimiterMode.Search, { allowKeyless: true }),countryCheck,checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),blocklistMiddleware,@@ -262,7 +267,7 @@ v2Router.post(v2Router.post("/scrape",- authMiddleware(RateLimiterMode.Scrape),+ authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),countryCheck,checkCreditsMiddleware(1),blocklistMiddleware,@@ -278,14 +283,14 @@ v2Router.get(v2Router.post("/scrape/:jobId/interact",- authMiddleware(RateLimiterMode.BrowserExecute),+ authMiddleware(RateLimiterMode.BrowserExecute, { allowKeyless: true }),validateJobIdParam,wrap(scrapeInteractController),);v2Router.delete("/scrape/:jobId/interact",- authMiddleware(RateLimiterMode.BrowserExecute),+ authMiddleware(RateLimiterMode.BrowserExecute, { allowKeyless: true }),validateJobIdParam,wrap(scrapeStopInteractiveBrowserController),);diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex ecc78903c..d9a3eab41 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -16,6 +16,7 @@ import {saveSearchToGCS,} from "../../lib/gcs-jobs";import { hasFormatOfType } from "../../lib/format-utils";+import { keylessTeamUuid } from "../../lib/keyless";import type { Document, ScrapeOptions } from "../../controllers/v2/types";import type { CostTracking } from "../../lib/cost-tracking";import type { Logger } from "winston";@@ -272,9 +273,10 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {error: scrape.error ?? null,time_taken: scrape.time_taken,team_id:- scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")+ keylessTeamUuid(scrape.team_id) ??+ (scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")? previewTeamId- : scrape.team_id,+ : scrape.team_id),options: scrape.zeroDataRetention ? null : scrape.options,cost_tracking: scrape.zeroDataRetention? nulldiff --git a/apps/api/src/services/posthog.ts b/apps/api/src/services/posthog.tsindex 79eaa1338..0a92cecee 100644--- a/apps/api/src/services/posthog.ts+++ b/apps/api/src/services/posthog.ts@@ -65,7 +65,7 @@ type RequestSurface =* practice. NOTE: confirm the exact strings sent by firecrawl-mcp and* firecrawl-cli β adjust the prefixes below if they differ.*/-function originToSurface(origin?: string | null): RequestSurface {+export function originToSurface(origin?: string | null): RequestSurface {const o = (origin ?? "").toLowerCase();if (o === "website" || o.includes("playground")) return "playground";if (o.startsWith("mcp")) return "mcp";diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 9c1bc0cad..dcf1d7467 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -52,6 +52,7 @@ import { createWebhookSender, WebhookEvent } from "../webhook/index";import { CustomError } from "../../lib/custom-error";import { startWebScraperPipeline } from "../../main/runWebScraper";import { CostTracking } from "../../lib/cost-tracking";+import { chargeKeylessCredits } from "../../lib/keyless";import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";@@ -138,6 +139,10 @@ async function billScrapeJob(unsupportedFeatures,);+ // Charge the keyless free tier's per-IP daily credit budget (no-op for+ // non-keyless teams).+ await chargeKeylessCredits(job.data.team_id, creditsToBeBilled);+if (job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&config.USE_DB_AUTHENTICATIONdiff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 172171645..2c8c0f30d 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -171,6 +171,9 @@ export type AuthResponse =success: false;error: string;status: number;+ // When true, send the agent OAuth-discovery WWW-Authenticate header even on+ // non-401 responses (e.g. keyless cap 429s) so agents can find the key flow.+ agentAuthDiscovery?: boolean;};export enum NotificationType {diff --git a/apps/dot-net-sdk/Firecrawl.Tests/FirecrawlClientTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/FirecrawlClientTests.csindex d3c1ad26f..0302a66fd 100644--- a/apps/dot-net-sdk/Firecrawl.Tests/FirecrawlClientTests.cs+++ b/apps/dot-net-sdk/Firecrawl.Tests/FirecrawlClientTests.cs@@ -7,22 +7,23 @@ namespace Firecrawl.Tests;public class FirecrawlClientTests{[Fact]- public void Constructor_RequiresApiKey()+ public void Constructor_AllowsEmptyApiKey_ForKeylessFreeTier(){// Clear env variable in case it's setEnvironment.SetEnvironmentVariable("FIRECRAWL_API_KEY", null);- var ex = Assert.Throws<FirecrawlException>(() => new FirecrawlClient(apiKey: ""));- Assert.Contains("API key is required", ex.Message);+ // No key: scrape/search use the keyless free tier; no longer throws.+ var client = new FirecrawlClient(apiKey: "");+ Assert.NotNull(client);}[Fact]- public void Constructor_RequiresApiKey_WhenNull()+ public void Constructor_AllowsNullApiKey_ForKeylessFreeTier(){Environment.SetEnvironmentVariable("FIRECRAWL_API_KEY", null);- var ex = Assert.Throws<FirecrawlException>(() => new FirecrawlClient(apiKey: null));- Assert.Contains("API key is required", ex.Message);+ var client = new FirecrawlClient(apiKey: null);+ Assert.NotNull(client);}[Fact]diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csprojindex dbaeb93ec..cda256516 100644--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj@@ -8,7 +8,7 @@<!-- NuGet package metadata --><PackageId>firecrawl-sdk</PackageId>- <Version>1.7.0</Version>+ <Version>1.7.1</Version><Authors>Firecrawl</Authors><Company>Firecrawl</Company><Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.csindex 11c5da6a3..fde84b672 100644--- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs@@ -607,14 +607,28 @@ public class FirecrawlClient// INTERNAL UTILITIES// ================================================================+ private const string SdkOrigin = "[email protected]";+private static Dictionary<string, object> BuildBody(object? options){+ Dictionary<string, object> body;if (options == null)- return new Dictionary<string, object>();+ {+ body = new Dictionary<string, object>();+ }+ else+ {+ var json = JsonSerializer.Serialize(options, FirecrawlHttpClient.JsonOptions);+ body = JsonSerializer.Deserialize<Dictionary<string, object>>(json, FirecrawlHttpClient.JsonOptions)+ ?? new Dictionary<string, object>();+ }++ // Identify the SDK so the API can grant the keyless free tier; harmless+ // telemetry on keyed requests.+ if (!body.ContainsKey("origin"))+ body["origin"] = SdkOrigin;- var json = JsonSerializer.Serialize(options, FirecrawlHttpClient.JsonOptions);- return JsonSerializer.Deserialize<Dictionary<string, object>>(json, FirecrawlHttpClient.JsonOptions)- ?? new Dictionary<string, object>();+ return body;}private static string BuildQuery(int? limit = null, int? offset = null, string? status = null)@@ -643,7 +657,7 @@ public class FirecrawlClientreturn query.Count == 0 ? string.Empty : "?" + string.Join("&", query);}- private static string ResolveApiKey(string? apiKey)+ private static string? ResolveApiKey(string? apiKey){if (!string.IsNullOrWhiteSpace(apiKey))return apiKey;@@ -652,8 +666,9 @@ public class FirecrawlClientif (!string.IsNullOrWhiteSpace(envKey))return envKey;- throw new FirecrawlException(- "API key is required. Pass it to the constructor or set the FIRECRAWL_API_KEY environment variable.");+ // No key: scrape and search fall back to the keyless free tier (per-IP).+ // Other endpoints return 401 from the API until a key is provided.+ return null;}private static string ResolveApiUrl(string? apiUrl)diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.csindex a7ba80dbf..883580ce0 100644--- a/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs@@ -14,7 +14,7 @@ namespace Firecrawl;internal class FirecrawlHttpClient{private readonly HttpClient _httpClient;- private readonly string _apiKey;+ private readonly string? _apiKey;private readonly string _baseUrl;private readonly int _maxRetries;private readonly double _backoffFactor;@@ -27,7 +27,7 @@ internal class FirecrawlHttpClient};internal FirecrawlHttpClient(- string apiKey,+ string? apiKey,string baseUrl,TimeSpan timeout,int maxRetries,@@ -198,7 +198,10 @@ internal class FirecrawlHttpClientprivate void ApplyStandardHeaders(HttpRequestMessage request){- request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _apiKey);+ // Omit the Authorization header entirely when no key is set so that+ // scrape/search can use the keyless free tier.+ if (!string.IsNullOrEmpty(_apiKey))+ request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _apiKey);request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));}diff --git a/apps/elixir-sdk/generate.exs b/apps/elixir-sdk/generate.exsindex e0327912a..ab781bdc5 100644--- a/apps/elixir-sdk/generate.exs+++ b/apps/elixir-sdk/generate.exs@@ -99,7 +99,7 @@ defmodule Firecrawl.Generator do|> Enum.join("\n")bt = <<96>>- auth_header_line = ~S' headers: [{"authorization", "Bearer #{api_key}"}]'+ headers_line = ~S' headers = if api_key, do: [{"authorization", "Bearer #{api_key}"}], else: []'"""# This file is auto-generated by generate.exs β DO NOT EDIT MANUALLY.@@ -145,6 +145,7 @@ defmodule Firecrawl.Generator do@type response :: {:ok, Req.Response.t()} | {:error, Exception.t() | Firecrawl.Error.t()}@base_url "#{base_url}"+ @sdk_origin "[email protected]"defp client(opts) doapi_key =@@ -152,43 +153,29 @@ defmodule Firecrawl.Generator doApplication.get_env(:firecrawl, :api_key)end)+ # A nil/empty key is allowed: scrape, search, and interact fall back to+ # the keyless free tier (rate-limited per IP). Other endpoints return 401+ # from the API until a key is provided.api_key =case api_key dokey when is_binary(key) ->case String.trim(key) do- "" ->- raise \"\"\"- Firecrawl API key not found or empty. Set it in your config:-- config :firecrawl, api_key: "fc-your-api-key"-- Or pass it as an option:-- Firecrawl.scrape_and_extract_from_url([url: "..."], api_key: "fc-your-api-key")- \"\"\"-- trimmed ->- trimmed+ "" -> nil+ trimmed -> trimmedend_ ->- raise \"\"\"- Firecrawl API key not found or empty. Set it in your config:-- config :firecrawl, api_key: "fc-your-api-key"-- Or pass it as an option:-- Firecrawl.scrape_and_extract_from_url([url: "..."], api_key: "fc-your-api-key")- \"\"\"+ nilend{base_url, opts} = Keyword.pop(opts, :base_url, @base_url)opts = Keyword.delete(opts, :api_key)+ #{headers_line}+Req.new(base_url: base_url,- #{auth_header_line}+ headers: headers)|> Req.merge(opts)|> Req.Request.append_response_steps(firecrawl_error_handler: &handle_api_error/1)@@ -201,10 +188,12 @@ defmodule Firecrawl.Generator dodefp handle_api_error({request, response}), do: {request, response}defp to_body(validated_params, key_mapping) do- Map.new(validated_params, fn {k, v} ->+ validated_params+ |> Map.new(fn {k, v} ->json_key = Map.fetch!(key_mapping, k){json_key, to_json_value(v)}end)+ |> Map.put_new("origin", @sdk_origin)enddefp to_query(validated_params, key_mapping) dodiff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.exindex e40387c3a..dc1ea6008 100644--- a/apps/elixir-sdk/lib/firecrawl.ex+++ b/apps/elixir-sdk/lib/firecrawl.ex@@ -41,6 +41,7 @@ defmodule Firecrawl do@type response :: {:ok, Req.Response.t()} | {:error, Exception.t() | Firecrawl.Error.t()}@base_url "https://api.firecrawl.dev/v2"+ @sdk_origin "[email protected]"defp client(opts) doapi_key =@@ -48,43 +49,29 @@ defmodule Firecrawl doApplication.get_env(:firecrawl, :api_key)end)+ # A nil/empty key is allowed: scrape, search, and interact fall back to the+ # keyless free tier (rate-limited per IP). Other endpoints return 401 from the+ # API until a key is provided.api_key =case api_key dokey when is_binary(key) ->case String.trim(key) do- "" ->- raise """- Firecrawl API key not found or empty. Set it in your config:-- config :firecrawl, api_key: "fc-your-api-key"-- Or pass it as an option:-- Firecrawl.scrape_and_extract_from_url([url: "..."], api_key: "fc-your-api-key")- """-- trimmed ->- trimmed+ "" -> nil+ trimmed -> trimmedend_ ->- raise """- Firecrawl API key not found or empty. Set it in your config:-- config :firecrawl, api_key: "fc-your-api-key"-- Or pass it as an option:-- Firecrawl.scrape_and_extract_from_url([url: "..."], api_key: "fc-your-api-key")- """+ nilend{base_url, opts} = Keyword.pop(opts, :base_url, @base_url)opts = Keyword.delete(opts, :api_key)+ headers = if api_key, do: [{"authorization", "Bearer #{api_key}"}], else: []+Req.new(base_url: base_url,- headers: [{"authorization", "Bearer #{api_key}"}]+ headers: headers)|> Req.merge(opts)|> Req.Request.append_response_steps(firecrawl_error_handler: &handle_api_error/1)@@ -97,10 +84,14 @@ defmodule Firecrawl dodefp handle_api_error({request, response}), do: {request, response}defp to_body(validated_params, key_mapping) do- Map.new(validated_params, fn {k, v} ->+ validated_params+ |> Map.new(fn {k, v} ->json_key = Map.fetch!(key_mapping, k){json_key, to_json_value(v)}end)+ # Identify the SDK so the API can grant the keyless free tier; harmless+ # telemetry on keyed requests.+ |> Map.put_new("origin", @sdk_origin)enddefp to_query(validated_params, key_mapping) dodiff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex 09a0b7c46..183f12340 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.6.0"+ @version "1.6.1"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/go-sdk/firecrawl.go b/apps/go-sdk/firecrawl.goindex 57f608e0c..84495c3e0 100644--- a/apps/go-sdk/firecrawl.go+++ b/apps/go-sdk/firecrawl.go@@ -56,12 +56,9 @@ func NewClient(opts ...option.RequestOption) (*Client, error) {if apiKey == "" {apiKey = strings.TrimSpace(os.Getenv("FIRECRAWL_API_KEY"))}- if apiKey == "" {- return nil, &FirecrawlError{- Message: "API key is required. Set it via option.WithAPIKey(), " +- "or FIRECRAWL_API_KEY environment variable.",- }- }+ // An empty API key is allowed: scrape, search, and interact fall back to the+ // keyless free tier (rate-limited per IP). Other methods return 401 from the+ // API until a key is provided.// Resolve API URL.apiURL := cfg.APIURL@@ -95,6 +92,9 @@ func (c *Client) Scrape(ctx context.Context, url string, opts *ScrapeOptions) (*body := map[string]interface{}{"url": url}mergeOptions(body, opts)+ if _, ok := body["origin"]; !ok {+ body["origin"] = "go-sdk@" + Version+ }raw, err := c.http.post(ctx, "/v2/scrape", body, nil)if err != nil {return nil, err@@ -132,6 +132,9 @@ func (c *Client) Interact(ctx context.Context, jobID, code string, params *Inter}}+ if _, ok := body["origin"]; !ok {+ body["origin"] = "go-sdk@" + Version+ }raw, err := c.http.post(ctx, "/v2/scrape/"+jobID+"/interact", body, nil)if err != nil {return nil, err@@ -539,6 +542,9 @@ func (c *Client) Search(ctx context.Context, query string, opts *SearchOptions)body := map[string]interface{}{"query": query}mergeOptions(body, opts)+ if _, ok := body["origin"]; !ok {+ body["origin"] = "go-sdk@" + Version+ }raw, err := c.http.post(ctx, "/v2/search", body, nil)if err != nil {return nil, errdiff --git a/apps/go-sdk/http_client.go b/apps/go-sdk/http_client.goindex e17f221d6..2acda1394 100644--- a/apps/go-sdk/http_client.go+++ b/apps/go-sdk/http_client.go@@ -136,7 +136,9 @@ func (h *httpClient) postMultipart(return nil, &FirecrawlError{Message: fmt.Sprintf("failed to create request: %v", err)}}- req.Header.Set("Authorization", "Bearer "+h.apiKey)+ if h.apiKey != "" {+ req.Header.Set("Authorization", "Bearer "+h.apiKey)+ }req.Header.Set("Content-Type", contentType)req.Header.Set("User-Agent", "firecrawl-go/"+Version)for k, v := range h.extraHeaders {@@ -221,7 +223,9 @@ func (h *httpClient) doJSON(ctx context.Context, method, url string, body interfreturn nil, &FirecrawlError{Message: fmt.Sprintf("failed to create request: %v", err)}}- req.Header.Set("Authorization", "Bearer "+h.apiKey)+ if h.apiKey != "" {+ req.Header.Set("Authorization", "Bearer "+h.apiKey)+ }req.Header.Set("Content-Type", "application/json")req.Header.Set("User-Agent", "firecrawl-go/"+Version)// Apply client-level headers.diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.goindex 7dcd1faa5..296f5f6af 100644--- a/apps/go-sdk/version.go+++ b/apps/go-sdk/version.go@@ -9,4 +9,4 @@ package firecrawl// Bump this when preparing a new release. The publish-go-sdk GitHub workflow// reads this value and creates the corresponding monorepo-prefixed tag on// merge to main.-const Version = "1.6.0"+const Version = "1.6.1"diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex 272c86cb5..1f2affb90 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.9.0"+version = "1.9.1"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.javaindex bfca8dfc0..71688c500 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java@@ -37,6 +37,7 @@ import java.util.concurrent.ForkJoinPool;public class FirecrawlClient {private static final String DEFAULT_API_URL = "https://api.firecrawl.dev";+ private static final String SDK_ORIGIN = "[email protected]";private static final long DEFAULT_TIMEOUT_MS = 300_000; // 5 minutesprivate static final int DEFAULT_MAX_RETRIES = 3;private static final double DEFAULT_BACKOFF_FACTOR = 0.5;@@ -101,6 +102,7 @@ public class FirecrawlClient {if (options != null) {mergeOptions(body, options);}+ body.putIfAbsent("origin", SDK_ORIGIN);return extractData(http.post("/v2/scrape", body, Map.class), Document.class);}@@ -148,6 +150,7 @@ public class FirecrawlClient {body.put("language", language != null ? language : "node");if (timeout != null) body.put("timeout", timeout);if (origin != null) body.put("origin", origin);+ body.putIfAbsent("origin", SDK_ORIGIN);return http.post("/v2/scrape/" + jobId + "/interact", body, BrowserExecuteResponse.class);}@@ -549,6 +552,7 @@ public class FirecrawlClient {if (options != null) {mergeOptions(body, options);}+ body.putIfAbsent("origin", SDK_ORIGIN);return extractData(http.post("/v2/search", body, Map.class), SearchData.class);}@@ -1387,11 +1391,9 @@ public class FirecrawlClient {if (resolvedKey == null || resolvedKey.isBlank()) {resolvedKey = System.getProperty("firecrawl.apiKey");}- if (resolvedKey == null || resolvedKey.isBlank()) {- throw new FirecrawlException(- "API key is required. Set it via builder.apiKey(), " +- "FIRECRAWL_API_KEY environment variable, or firecrawl.apiKey system property.");- }+ // A null/blank key is allowed: scrape, search, and interact fall back+ // to the keyless free tier (rate-limited per IP). Other methods return+ // 401 from the API until a key is provided.String resolvedUrl = apiUrl;if (resolvedUrl == null || resolvedUrl.equals(DEFAULT_API_URL)) {diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.javaindex 695c9e079..b6fe62d7a 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java@@ -55,6 +55,17 @@ class FirecrawlHttpClient {.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);}+ /**+ * Adds the Authorization header only when an API key is configured. Omitting+ * it entirely (rather than sending an empty Bearer) lets scrape/search/interact+ * use the keyless free tier.+ */+ private void applyAuth(Request.Builder builder) {+ if (apiKey != null && !apiKey.isBlank()) {+ builder.header("Authorization", "Bearer " + apiKey);+ }+ }+/*** Sends a POST request with JSON body.*/@@ -76,9 +87,9 @@ class FirecrawlHttpClient {RequestBody requestBody = RequestBody.create(json, JSON);Request.Builder builder = new Request.Builder().url(url)- .header("Authorization", "Bearer " + apiKey).header("Content-Type", "application/json").post(requestBody);+ applyAuth(builder);for (Map.Entry<String, String> entry : extraHeaders.entrySet()) {builder.header(entry.getKey(), entry.getValue());}@@ -98,12 +109,12 @@ class FirecrawlHttpClient {throw new FirecrawlException("Failed to serialize request body", e);}RequestBody requestBody = RequestBody.create(json, JSON);- Request request = new Request.Builder()+ Request.Builder builder = new Request.Builder().url(url)- .header("Authorization", "Bearer " + apiKey).header("Content-Type", "application/json")- .patch(requestBody)- .build();+ .patch(requestBody);+ applyAuth(builder);+ Request request = builder.build();return executeWithRetry(request, responseType);}@@ -140,11 +151,11 @@ class FirecrawlHttpClient {RequestBody fileBody = RequestBody.create(fileContent, mediaType);multipart.addFormDataPart(fileFieldName, filename, fileBody);- Request request = new Request.Builder()+ Request.Builder builder = new Request.Builder().url(url)- .header("Authorization", "Bearer " + apiKey)- .post(multipart.build())- .build();+ .post(multipart.build());+ applyAuth(builder);+ Request request = builder.build();return executeWithRetry(request, responseType);}@@ -154,11 +165,11 @@ class FirecrawlHttpClient {*/<T> T get(String path, Class<T> responseType) {String url = baseUrl + path;- Request request = new Request.Builder()+ Request.Builder builder = new Request.Builder().url(url)- .header("Authorization", "Bearer " + apiKey)- .get()- .build();+ .get();+ applyAuth(builder);+ Request request = builder.build();return executeWithRetry(request, responseType);}@@ -166,11 +177,11 @@ class FirecrawlHttpClient {* Sends a GET request with full URL (for following next-page cursors).*/<T> T getAbsolute(String absoluteUrl, Class<T> responseType) {- Request request = new Request.Builder()+ Request.Builder builder = new Request.Builder().url(absoluteUrl)- .header("Authorization", "Bearer " + apiKey)- .get()- .build();+ .get();+ applyAuth(builder);+ Request request = builder.build();return executeWithRetry(request, responseType);}@@ -179,11 +190,11 @@ class FirecrawlHttpClient {*/<T> T delete(String path, Class<T> responseType) {String url = baseUrl + path;- Request request = new Request.Builder()+ Request.Builder builder = new Request.Builder().url(url)- .header("Authorization", "Bearer " + apiKey)- .delete()- .build();+ .delete();+ applyAuth(builder);+ Request request = builder.build();return executeWithRetry(request, responseType);}diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex de67a4c01..cd4c0d462 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.25.4",+ "version": "4.25.5","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/clientOptions.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/clientOptions.test.tsindex 770e9df62..1746007e1 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/clientOptions.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/clientOptions.test.ts@@ -67,12 +67,11 @@ describe('Firecrawl v2 Client Options', () => {expect(client).toBeInstanceOf(FirecrawlClient);});- it('should throw for empty string API key on cloud', () => {- expect(() => new Firecrawl('')).toThrow('API key is required');- });-- it('should throw for whitespace-only string API key on cloud', () => {- expect(() => new Firecrawl(' ')).toThrow('API key is required');+ it('should construct without an API key for the keyless free tier', () => {+ // No key: scrape/search/interact use the keyless free tier; the SDK no+ // longer throws at construction.+ expect(() => new Firecrawl('')).not.toThrow();+ expect(() => new Firecrawl(' ')).not.toThrow();});it('should provide v1 accessor when constructed with string', () => {diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex 2eaa4d28f..bd75c7285 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -137,9 +137,9 @@ export class FirecrawlClient {const apiKey = (opts.apiKey ?? process.env.FIRECRAWL_API_KEY ?? "").trim();const apiUrl = (opts.apiUrl ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev").replace(/\/$/, "");- if (this.isCloudService(apiUrl) && !apiKey) {- throw new Error("API key is required for the cloud API. Set FIRECRAWL_API_KEY env or pass apiKey.");- }+ // No API key is allowed: scrape, search, and interact fall back to the+ // keyless free tier (rate-limited per IP). Other methods will return 401+ // from the API until a key is provided.this.http = new HttpClient({apiKey,diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.tsindex 842a0ea25..dc0560ed1 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts@@ -34,7 +34,10 @@ export class HttpClient {baseURL: this.apiUrl,timeout: options.timeoutMs ?? 300000,headers: {- Authorization: `Bearer ${this.apiKey}`,+ // Omit the Authorization header entirely when no API key is set so that+ // scrape/search/interact can use the keyless free tier (the cloud only+ // grants it when no Authorization header is present).+ ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),},transitional: { clarifyTimeoutError: true },});diff --git a/apps/php-sdk/src/Client/FirecrawlClient.php b/apps/php-sdk/src/Client/FirecrawlClient.phpindex 46c199065..9758ca9fe 100644--- a/apps/php-sdk/src/Client/FirecrawlClient.php+++ b/apps/php-sdk/src/Client/FirecrawlClient.php@@ -5,6 +5,7 @@ declare(strict_types=1);namespace Firecrawl\Client;use Firecrawl\Exceptions\FirecrawlException;+use Firecrawl\Version;use Firecrawl\Exceptions\JobTimeoutException;use Firecrawl\Models\AgentOptions;use Firecrawl\Models\AgentResponse;@@ -63,12 +64,10 @@ final class FirecrawlClientfloat $backoffFactor = self::DEFAULT_BACKOFF_FACTOR,?ClientInterface $httpClient = null,): self {+ // An empty key is allowed: scrape, search, and interact fall back to the+ // keyless free tier (rate-limited per IP). Other methods return 401 from+ // the API until a key is provided.$resolvedKey = trim($apiKey ?: (getenv('FIRECRAWL_API_KEY') ?: ''));- if ($resolvedKey === '') {- throw new FirecrawlException(- 'API key is required. Pass it directly or set the FIRECRAWL_API_KEY environment variable.',- );- }$resolvedUrl = $apiUrl ?: (getenv('FIRECRAWL_API_URL') ?: self::DEFAULT_API_URL);@@ -111,6 +110,7 @@ final class FirecrawlClientif ($options !== null) {$body = array_merge($body, $options->toArray());}+ $body['origin'] ??= 'php-sdk@' . Version::SDK_VERSION;$response = $this->http->post('/v2/scrape', $body);@@ -141,6 +141,7 @@ final class FirecrawlClientif ($prompt !== null) {$body['prompt'] = $prompt;}+ $body['origin'] ??= 'php-sdk@' . Version::SDK_VERSION;return BrowserExecuteResponse::fromArray($this->http->post("/v2/scrape/{$jobId}/interact", $body),@@ -472,6 +473,7 @@ final class FirecrawlClientif ($options !== null) {$body = array_merge($body, $options->toArray());}+ $body['origin'] ??= 'php-sdk@' . Version::SDK_VERSION;$response = $this->http->post('/v2/search', $body);diff --git a/apps/php-sdk/src/Client/FirecrawlHttpClient.php b/apps/php-sdk/src/Client/FirecrawlHttpClient.phpindex fa1d35f5d..44e7c8d20 100644--- a/apps/php-sdk/src/Client/FirecrawlHttpClient.php+++ b/apps/php-sdk/src/Client/FirecrawlHttpClient.php@@ -136,10 +136,14 @@ final class FirecrawlHttpClient?array $multipart = null,): array {$defaultHeaders = [- 'Authorization' => 'Bearer ' . $this->apiKey,'Accept' => 'application/json','User-Agent' => 'firecrawl-php/' . Version::SDK_VERSION,];+ // Omit the Authorization header entirely when no key is set so that+ // scrape/search/interact can use the keyless free tier.+ if ($this->apiKey !== null && $this->apiKey !== '') {+ $defaultHeaders['Authorization'] = 'Bearer ' . $this->apiKey;+ }if ($multipart === null) {$defaultHeaders['Content-Type'] = 'application/json';diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.phpindex c627fce9c..5f15f46ef 100644--- a/apps/php-sdk/src/Version.php+++ b/apps/php-sdk/src/Version.php@@ -6,5 +6,5 @@ namespace Firecrawl;final class Version{- public const SDK_VERSION = '1.6.0';+ public const SDK_VERSION = '1.6.1';}diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex fa14ddb39..aba9a1c64 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.28.2"+__version__ = "4.28.3"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 13ea6a4ed..b34ce4a38 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -111,11 +111,10 @@ class FirecrawlClient:if api_key is None:api_key = os.getenv("FIRECRAWL_API_KEY")- if self._is_cloud_service(api_url) and not api_key:- raise ValueError(- "API key is required for the cloud API. Set FIRECRAWL_API_KEY environment variable "- "or pass api_key parameter."- )+ # No API key is allowed: scrape, search, and interact fall back to the+ # keyless free tier (rate-limited per IP). The HTTP client only sends an+ # Authorization header when a key is present, and other methods return+ # 401 from the API until a key is provided.self.config = ClientConfig(api_key=api_key,diff --git a/apps/ruby-sdk/lib/firecrawl/client.rb b/apps/ruby-sdk/lib/firecrawl/client.rbindex e014427e2..3bb199d66 100644--- a/apps/ruby-sdk/lib/firecrawl/client.rb+++ b/apps/ruby-sdk/lib/firecrawl/client.rb@@ -39,9 +39,10 @@ module Firecrawlbackoff_factor: DEFAULT_BACKOFF_FACTOR)resolved_key = api_key || ENV["FIRECRAWL_API_KEY"]- if resolved_key.nil? || resolved_key.strip.empty?- raise FirecrawlError, "API key is required. Provide api_key: or set FIRECRAWL_API_KEY environment variable."- end+ # A nil/empty key is allowed: scrape, search, and interact fall back to the+ # keyless free tier (rate-limited per IP). Other methods return 401 from the+ # API until a key is provided.+ resolved_key = nil if resolved_key.nil? || resolved_key.strip.empty?resolved_url = api_url || ENV["FIRECRAWL_API_URL"] || DEFAULT_API_URLunless resolved_url.match?(%r{\Ahttps?://}i)@@ -78,6 +79,7 @@ module Firecrawlbody = { "url" => url }body.merge!(options.to_h) if options+ body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"raw = @http.post("/v2/scrape", body)data = raw["data"] || rawModels::Document.new(data)@@ -96,6 +98,7 @@ module Firecrawlbody = { "code" => code, "language" => language }body["timeout"] = timeout if timeout+ body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"@http.post("/v2/scrape/#{job_id}/interact", body)end@@ -377,6 +380,7 @@ module Firecrawlbody = { "query" => query }body.merge!(options.to_h) if options+ body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"raw = @http.post("/v2/search", body)data = raw["data"] || rawModels::SearchData.new(data)diff --git a/apps/ruby-sdk/lib/firecrawl/http_client.rb b/apps/ruby-sdk/lib/firecrawl/http_client.rbindex 4d1a819e0..3fc743a5d 100644--- a/apps/ruby-sdk/lib/firecrawl/http_client.rb+++ b/apps/ruby-sdk/lib/firecrawl/http_client.rb@@ -23,7 +23,7 @@ module Firecrawldef post(path, body, extra_headers: {})uri = URI("#{@base_url}#{path}")request = Net::HTTP::Post.new(uri)- request["Authorization"] = "Bearer #{@api_key}"+ request["Authorization"] = "Bearer #{@api_key}" if @api_keyrequest["Content-Type"] = "application/json"extra_headers.each { |k, v| request[k] = v }request.body = JSON.generate(body)@@ -34,7 +34,7 @@ module Firecrawldef get(path)uri = URI("#{@base_url}#{path}")request = Net::HTTP::Get.new(uri)- request["Authorization"] = "Bearer #{@api_key}"+ request["Authorization"] = "Bearer #{@api_key}" if @api_keyexecute_with_retry(uri, request)end@@ -47,7 +47,7 @@ module Firecrawlraise FirecrawlError, "Absolute URL origin (#{uri.scheme}://#{uri.host}:#{uri.port}) does not match API base URL origin (#{base_uri.scheme}://#{base_uri.host}:#{base_uri.port}). Refusing to send credentials."endrequest = Net::HTTP::Get.new(uri)- request["Authorization"] = "Bearer #{@api_key}"+ request["Authorization"] = "Bearer #{@api_key}" if @api_keyexecute_with_retry(uri, request)end@@ -55,7 +55,7 @@ module Firecrawldef delete(path)uri = URI("#{@base_url}#{path}")request = Net::HTTP::Delete.new(uri)- request["Authorization"] = "Bearer #{@api_key}"+ request["Authorization"] = "Bearer #{@api_key}" if @api_keyexecute_with_retry(uri, request)end@@ -63,7 +63,7 @@ module Firecrawldef patch(path, body)uri = URI("#{@base_url}#{path}")request = Net::HTTP::Patch.new(uri)- request["Authorization"] = "Bearer #{@api_key}"+ request["Authorization"] = "Bearer #{@api_key}" if @api_keyrequest["Content-Type"] = "application/json"request.body = JSON.generate(body)execute_with_retry(uri, request)@@ -84,7 +84,7 @@ module Firecrawlbuilder = lambda dorequest = Net::HTTP::Post.new(uri)- request["Authorization"] = "Bearer #{@api_key}"+ request["Authorization"] = "Bearer #{@api_key}" if @api_keyrequest["Content-Type"] = "multipart/form-data; boundary=#{boundary}"request.body = bodyrequestdiff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rbindex a249e3949..52751598c 100644--- a/apps/ruby-sdk/lib/firecrawl/version.rb+++ b/apps/ruby-sdk/lib/firecrawl/version.rb@@ -1,5 +1,5 @@# frozen_string_literal: truemodule Firecrawl- VERSION = "1.8.0"+ VERSION = "1.8.1"enddiff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lockindex 3d851d6a2..091dc733c 100644--- a/apps/rust-sdk/Cargo.lock+++ b/apps/rust-sdk/Cargo.lock@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"[[package]]name = "firecrawl"-version = "2.8.0"+version = "2.8.1"dependencies = ["mockito","reqwest",diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex e02c5b8ef..a924c654b 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,6 +1,6 @@[package]name = "firecrawl"-version = "2.8.0"+version = "2.8.1"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"diff --git a/apps/rust-sdk/src/client.rs b/apps/rust-sdk/src/client.rsindex 17a91b482..d0d520b3f 100644--- a/apps/rust-sdk/src/client.rs+++ b/apps/rust-sdk/src/client.rs@@ -4,7 +4,7 @@ use reqwest::Response;use serde::de::DeserializeOwned;use serde_json::Value;-use crate::error::{FirecrawlAPIError, FirecrawlError};+use crate::error::FirecrawlError;pub(crate) const API_VERSION: &str = "/v2";const CLOUD_API_URL: &str = "https://api.firecrawl.dev";@@ -89,32 +89,9 @@ impl Client {let url = api_url.as_ref().trim_end_matches('/').to_string();let api_key = api_key.map(|k| k.as_ref().to_string());- // Reject empty or missing API key for cloud service- if url == CLOUD_API_URL {- match &api_key {- None => {- return Err(FirecrawlError::APIError(- "Configuration".to_string(),- FirecrawlAPIError {- success: false,- error: "API key is required for cloud service".to_string(),- details: None,- },- ));- }- Some(key) if key.trim().is_empty() => {- return Err(FirecrawlError::APIError(- "Configuration".to_string(),- FirecrawlAPIError {- success: false,- error: "API key cannot be empty for cloud service".to_string(),- details: None,- },- ));- }- _ => {}- }- }+ // An empty/missing API key is allowed: scrape, search, and interact fall+ // back to the keyless free tier (rate-limited per IP). Other methods+ // return 401 from the API until a key is provided.Ok(Client {api_key,@@ -134,10 +111,12 @@ impl Client {// Static string is always valid ASCIIheaders.insert("Content-Type", HeaderValue::from_static("application/json"));if let Some(api_key) = self.api_key.as_ref() {- // API key is validated at client creation, so this should always succeed.- // Use if-let to gracefully handle edge cases without panicking.- if let Ok(value) = format!("Bearer {}", api_key).parse() {- headers.insert("Authorization", value);+ // Omit the Authorization header entirely when no key is set so that+ // scrape/search/interact can use the keyless free tier.+ if !api_key.trim().is_empty() {+ if let Ok(value) = format!("Bearer {}", api_key).parse() {+ headers.insert("Authorization", value);+ }}}if let Some(key) = idempotency_key {diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rsindex 7f8aab88b..94820b20b 100644--- a/apps/rust-sdk/src/scrape.rs+++ b/apps/rust-sdk/src/scrape.rs@@ -16,6 +16,9 @@ use crate::FirecrawlError;#[derive(Deserialize, Serialize, Debug, Default, Clone)]#[serde(rename_all = "camelCase")]pub struct ScrapeOptions {+ /// Origin label for request attribution (e.g., "[email protected]").+ pub origin: Option<String>,+/// Output formats to include in the response.pub formats: Option<Vec<Format>>,@@ -247,9 +250,13 @@ impl Client {url: impl AsRef<str>,options: impl Into<Option<ScrapeOptions>>,) -> Result<Document, FirecrawlError> {+ let mut options = options.into().unwrap_or_default();+ if options.origin.is_none() {+ options.origin = Some(format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")));+ }let body = ScrapeRequest {url: url.as_ref().to_string(),- options: options.into().unwrap_or_default(),+ options,};let headers = self.prepare_headers(None);@@ -361,6 +368,9 @@ impl Client {if body.language.is_none() {body.language = Some(ScrapeExecuteLanguage::Node);}+ if body.origin.is_none() {+ body.origin = Some(format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")));+ }let response = self.clientdiff --git a/apps/rust-sdk/src/search.rs b/apps/rust-sdk/src/search.rsindex a6d58436f..945f724ea 100644--- a/apps/rust-sdk/src/search.rs+++ b/apps/rust-sdk/src/search.rs@@ -46,6 +46,9 @@ pub struct SearchOptions {/// Integration identifier for tracking.pub integration: Option<String>,++ /// Origin label for request attribution (e.g., "[email protected]").+ pub origin: Option<String>,}/// Request body for search endpoint.@@ -173,9 +176,13 @@ impl Client {query: impl AsRef<str>,options: impl Into<Option<SearchOptions>>,) -> Result<SearchResponse, FirecrawlError> {+ let mut options = options.into().unwrap_or_default();+ if options.origin.is_none() {+ options.origin = Some(format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")));+ }let body = SearchRequest {query: query.as_ref().to_string(),- options: options.into().unwrap_or_default(),+ options,};let headers = self.prepare_headers(None);