Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): clarify keyless auth errors with signup and Bearer
Max Loffgren5d024a8e1c62
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.tsindex fa8fb776e..f7a2308f6 100644--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts@@ -103,7 +103,11 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {expect(response.statusCode).toBe(401);expect(response.body.success).toBe(false);- expect(response.body.error).toContain("not available without an API key");+ expect(response.body.error).toContain(+ "not supported by the keyless free tier",+ );+ expect(response.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(response.body.error).toContain("Authorization: Bearer YOUR_API_KEY");});it(@@ -152,7 +156,9 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {.send({ origin: "mcp" });expect(blocked.statusCode).toBe(429);- expect(blocked.body.error).toContain("unauthenticated requests");+ expect(blocked.body.error).toContain("keyless free tier rate limit");+ expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY");// Out of quota β emit the OAuth-discovery header so agents find the key flow.expect(blocked.headers["www-authenticate"]).toContain("resource_metadata");});@@ -175,7 +181,9 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {.send({ origin: "mcp" });expect(blocked.statusCode).toBe(429);- expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.body.error).toContain("keyless free tier rate limit");+ expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY");});it("enforces the daily credit cap on parse (429)", async () => {@@ -202,7 +210,9 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {});expect(blocked.statusCode).toBe(429);- expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.body.error).toContain("keyless free tier rate limit");+ expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY");});it(@@ -242,7 +252,11 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {});expect(blocked.statusCode).toBe(429);- expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.body.error).toContain("keyless free tier rate limit");+ expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(+ "Authorization: Bearer YOUR_API_KEY",+ );expect(blocked.headers["www-authenticate"]).toContain("resource_metadata",);@@ -282,7 +296,11 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {});expect(blocked.statusCode).toBe(429);- expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.body.error).toContain("keyless free tier rate limit");+ expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(+ "Authorization: Bearer YOUR_API_KEY",+ );expect(blocked.headers["www-authenticate"]).toContain("resource_metadata",);@@ -320,7 +338,11 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {});expect(blocked.statusCode).toBe(429);- expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.body.error).toContain("keyless free tier rate limit");+ expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(+ "Authorization: Bearer YOUR_API_KEY",+ );expect(blocked.headers["www-authenticate"]).toContain("resource_metadata",);@@ -446,7 +468,11 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {});expect(blocked.statusCode).toBe(429);- expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.body.error).toContain("keyless free tier rate limit");+ expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(+ "Authorization: Bearer YOUR_API_KEY",+ );expect(blocked.headers["www-authenticate"]).toContain("resource_metadata",);@@ -481,7 +507,11 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {});expect(blocked.statusCode).toBe(429);- expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.body.error).toContain("keyless free tier rate limit");+ expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(+ "Authorization: Bearer YOUR_API_KEY",+ );expect(blocked.headers["www-authenticate"]).toContain("resource_metadata",);@@ -528,7 +558,11 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {.set("x-firecrawl-keyless-ip", fakeIp).send({ origin: "mcp" });expect(blocked.statusCode).toBe(429);- expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.body.error).toContain("keyless free tier rate limit");+ expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(+ "Authorization: Bearer YOUR_API_KEY",+ );// Without the secret, the forwarded IP is ignored (keyed on the real IP).const allowed = await request(TEST_API_URL)@@ -647,7 +681,11 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {.send({ code: "return 1;", origin: "mcp" });expect(blocked.statusCode).toBe(429);- expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.body.error).toContain("keyless free tier rate limit");+ expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(+ "Authorization: Bearer YOUR_API_KEY",+ );expect(blocked.headers["www-authenticate"]).toContain("resource_metadata",);diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex fbf0bf7c4..5c9edad40 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -9,7 +9,7 @@ import { getAgentSponsorStatus } from "../services/agent-sponsor";import { getRedisConnection } from "../services/queue-service";import { getRateLimiter } from "../services/rate-limiter";import {- KEYLESS_CREDITS_MESSAGE,+ KEYLESS_FREE_TIER_LIMIT_MESSAGE,consumeKeylessRequest,isKeylessConfigured,isKeylessIpEligible,@@ -472,9 +472,10 @@ export async function clearACUCTeam(team_id: string): Promise<void> {await getRedisConnection().sadd("billed_teams", team_id);}-const KEYLESS_REQUESTS_MESSAGE = `You've reached today's limit of free, unauthenticated requests to Firecrawl. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;+const KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE = `This endpoint is not supported by the keyless free tier. Sign up for a free API key at https://www.firecrawl.dev/signin for more endpoints, more usage, and higher rate limits.-const KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE = `This endpoint is not available without an API key. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;+Then authenticate with:+Authorization: Bearer YOUR_API_KEY`;const KEYLESS_SUSPICIOUS_IP_MESSAGE = `Unfortunately, your IP address looks suspicious, so Firecrawl can't be used without an API key from here. Sign up for a free API key at https://firecrawl.dev for 1000 credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;@@ -604,10 +605,7 @@ async function handleKeylessAuth(});return {success: false,- error:- result.reason === "credits"- ? KEYLESS_CREDITS_MESSAGE- : KEYLESS_REQUESTS_MESSAGE,+ error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,status: 429,// Out of free quota β emit the OAuth-discovery header so agents can find// the key/signup flow at the moment they actually need a key.diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 7fd083171..512d8280b 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -29,7 +29,7 @@ import { getErrorContactMessage } from "../../lib/deployment";import { captureExceptionWithZdrCheck } from "../../services/sentry";import { getScrapeZDR } from "../../lib/zdr-helpers";import {- KEYLESS_CREDITS_MESSAGE,+ KEYLESS_FREE_TIER_LIMIT_MESSAGE,adjustKeylessCredits,logKeylessCreditUsage,reserveKeylessCredits,@@ -157,7 +157,7 @@ export async function scrapeController(applyAgentAuthDiscoveryHeader(res);return res.status(429).json({success: false,- error: KEYLESS_CREDITS_MESSAGE,+ error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,});}reservedKeylessCredits = projectedKeylessCredits;diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex bbf4a09a4..80d4e1d19 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -34,7 +34,7 @@ import {import { fromV1ScrapeOptions } from "../v2/types";import { getSearchForcedKind } from "../../lib/zdr-helpers";import {- KEYLESS_CREDITS_MESSAGE,+ KEYLESS_FREE_TIER_LIMIT_MESSAGE,adjustKeylessCredits,logKeylessCreditUsage,reserveKeylessCredits,@@ -212,7 +212,7 @@ export async function searchController(applyAgentAuthDiscoveryHeader(res);return res.status(429).json({success: false,- error: KEYLESS_CREDITS_MESSAGE,+ error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,});}reservedKeylessCredits = projectedKeylessCredits;diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex eda56ae0f..e1e07d9db 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -27,7 +27,7 @@ import { captureExceptionWithZdrCheck } from "../../services/sentry";import type { BillingMetadata } from "../../services/billing/types";import { getScrapeZDR } from "../../lib/zdr-helpers";import {- KEYLESS_CREDITS_MESSAGE,+ KEYLESS_FREE_TIER_LIMIT_MESSAGE,adjustKeylessCredits,logKeylessCreditUsage,reserveKeylessCredits,@@ -379,7 +379,7 @@ export async function parseController(applyAgentAuthDiscoveryHeader(res);return res.status(429).json({success: false,- error: KEYLESS_CREDITS_MESSAGE,+ error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,});}reservedKeylessCredits = projectedKeylessCredits;diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 0541d9bbe..867cc6c55 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -46,7 +46,7 @@ import { getScrapeZDR } from "../../lib/zdr-helpers";import { RequestWithAuth, ScrapeOptions } from "./types";import { billTeam } from "../../services/billing/credit_billing";import {- KEYLESS_CREDITS_MESSAGE,+ KEYLESS_FREE_TIER_LIMIT_MESSAGE,adjustKeylessCredits,keylessTeamUuid,logKeylessCreditUsage,@@ -219,7 +219,7 @@ export async function scrapeInteractController(if ("error" in created) {if (created.status === 429 &&- created.body.error === KEYLESS_CREDITS_MESSAGE+ created.body.error === KEYLESS_FREE_TIER_LIMIT_MESSAGE) {applyAgentAuthDiscoveryHeader(res);}@@ -564,7 +564,7 @@ async function createSessionForScrape(status: 429,body: {success: false,- error: KEYLESS_CREDITS_MESSAGE,+ error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,},error: true,};diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 4de9b9108..e94d6d3d8 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -30,7 +30,7 @@ import { captureExceptionWithZdrCheck } from "../../services/sentry";import type { BillingMetadata } from "../../services/billing/types";import { getScrapeZDR } from "../../lib/zdr-helpers";import {- KEYLESS_CREDITS_MESSAGE,+ KEYLESS_FREE_TIER_LIMIT_MESSAGE,adjustKeylessCredits,logKeylessCreditUsage,reserveKeylessCredits,@@ -187,7 +187,7 @@ export async function scrapeController(applyAgentAuthDiscoveryHeader(res);return res.status(429).json({success: false,- error: KEYLESS_CREDITS_MESSAGE,+ error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,});}reservedKeylessCredits = projectedKeylessCredits;diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex ffe8dd4b7..b4ae9968e 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -8,7 +8,7 @@ import {} from "./types";import { billTeam } from "../../services/billing/credit_billing";import {- KEYLESS_CREDITS_MESSAGE,+ KEYLESS_FREE_TIER_LIMIT_MESSAGE,adjustKeylessCredits,logKeylessCreditUsage,reserveKeylessCredits,@@ -192,7 +192,7 @@ export async function searchController(applyAgentAuthDiscoveryHeader(res);return res.status(429).json({success: false,- error: KEYLESS_CREDITS_MESSAGE,+ error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,});}reservedKeylessCredits = projectedKeylessCredits;diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex 665ef558f..014d559cc 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -17,7 +17,11 @@ import { isKeylessIpSuspicious } from "./spur";const KEYLESS_REQUESTS_PER_DAY = config.KEYLESS_REQUESTS_PER_DAY;const KEYLESS_CREDITS_PER_DAY = config.KEYLESS_CREDITS_PER_DAY;-export const KEYLESS_CREDITS_MESSAGE = `You've reached today's limit of free, unauthenticated credits for Firecrawl. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;+// Shared 429 copy for both keyless request-cap and credit-cap failures.+export const KEYLESS_FREE_TIER_LIMIT_MESSAGE = `You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at https://www.firecrawl.dev/signin.++Then authenticate with:+Authorization: Bearer YOUR_API_KEY`;// The tier is "configured" when BOTH limits are set β even to 0. Unset means the// feature is off (callers get a plain Unauthorized); 0 means it's on but the