Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): tag keyless signup links with UTM parameters (#4847)
Maxb5d187b19de3
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.tsindex 6bf149b7d..addb6b2ab 100644--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts@@ -40,6 +40,11 @@ async function flushKeylessBuckets() {}}+// Every keyless prompt links to signup with these tags so new accounts can be+// attributed to the keyless free tier.+const KEYLESS_SIGNUP_URL =+ "https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=api";+// Recover the loopback IP the server keyed on, so we can seed its credit counter.async function currentKeylessIp(): Promise<string> {const keys = await redisRateLimitClient.keys("keyless_requests:*");@@ -107,7 +112,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {expect(response.body.error).toContain("not supported by the keyless free tier",);- expect(response.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(response.body.error).toContain(KEYLESS_SIGNUP_URL);expect(response.body.error).toContain("Authorization: Bearer YOUR_API_KEY");});@@ -158,7 +163,8 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {expect(blocked.statusCode).toBe(429);expect(blocked.body.error).toContain("keyless free tier rate limit");- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);+ expect(blocked.body.error).not.toContain(`${KEYLESS_SIGNUP_URL}.`);expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY");// Out of quota β emit the OAuth-discovery header so agents find the key flow.expect(blocked.headers["www-authenticate"]).toContain("resource_metadata");@@ -183,7 +189,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {expect(blocked.statusCode).toBe(429);expect(blocked.body.error).toContain("keyless free tier rate limit");- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY");});@@ -212,7 +218,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {expect(blocked.statusCode).toBe(429);expect(blocked.body.error).toContain("keyless free tier rate limit");- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY");});@@ -255,7 +261,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {expect(blocked.statusCode).toBe(429);expect(blocked.body.reason).toBe("credits");expect(blocked.body.error).toContain("keyless free tier rate limit");- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY",);@@ -300,7 +306,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {expect(blocked.statusCode).toBe(429);expect(blocked.body.reason).toBe("credits");expect(blocked.body.error).toContain("keyless free tier rate limit");- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY",);@@ -342,7 +348,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {expect(blocked.statusCode).toBe(429);expect(blocked.body.error).toContain("keyless free tier rate limit");- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY",);@@ -472,7 +478,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {expect(blocked.statusCode).toBe(429);expect(blocked.body.error).toContain("keyless free tier rate limit");- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY",);@@ -511,7 +517,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {expect(blocked.statusCode).toBe(429);expect(blocked.body.error).toContain("keyless free tier rate limit");- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY",);@@ -562,7 +568,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {.send({ origin: "mcp" });expect(blocked.statusCode).toBe(429);expect(blocked.body.error).toContain("keyless free tier rate limit");- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY",);@@ -685,7 +691,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {expect(blocked.statusCode).toBe(429);expect(blocked.body.error).toContain("keyless free tier rate limit");- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY",);@@ -805,6 +811,7 @@ describeIf(SPUR_ENABLED)("Keyless free tier β Spur IP reputation", () => {expect(response.statusCode).toBe(403);expect(response.body.success).toBe(false);expect(response.body.error).toContain("suspicious");+ expect(response.body.error).toContain(KEYLESS_SIGNUP_URL);// Out of the keyless path β emit the OAuth-discovery header.expect(response.headers["www-authenticate"]).toContain("resource_metadata",diff --git a/apps/api/src/controllers/__tests__/auth.test.ts b/apps/api/src/controllers/__tests__/auth.test.tsindex 05b3d3bdb..647a43b84 100644--- a/apps/api/src/controllers/__tests__/auth.test.ts+++ b/apps/api/src/controllers/__tests__/auth.test.ts@@ -20,6 +20,7 @@ import {keylessConversionCohort,} from "../../lib/keyless";import { logger } from "../../lib/logger";+import { isKeylessIpSuspicious } from "../../lib/spur";import { db } from "../../db/connection";import { autumnService } from "../../services/autumn/autumn.service";@@ -213,6 +214,67 @@ describe("authenticateUser", () => {);});+ it("links every keyless signup prompt to the keyless-tagged signup URL", async () => {+ config.USE_DB_AUTHENTICATION = true;+ vi.mocked(isKeylessConfigured).mockReturnValue(true);+ vi.mocked(consumeKeylessRequest).mockResolvedValue({+ ok: false,+ reason: "credits",+ requestsUsed: 1,+ creditsUsed: 100,+ });+ vi.spyOn(logger, "warn").mockImplementation(() => logger);+ const keylessRequest = () => ({+ headers: {},+ socket: { remoteAddress: "203.0.113.8" },+ });+ const taggedSignupUrl =+ "https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=api";++ const limited = await authenticateUser(+ keylessRequest(),+ {},+ RateLimiterMode.Scrape,+ { allowKeyless: true },+ );+ const unsupported = await authenticateUser(+ keylessRequest(),+ {},+ RateLimiterMode.Scrape,+ { allowKeyless: false },+ );+ vi.mocked(isKeylessIpSuspicious).mockResolvedValueOnce(true);+ const suspicious = await authenticateUser(+ keylessRequest(),+ {},+ RateLimiterMode.Scrape,+ { allowKeyless: true },+ );++ // A period right after the URL would be copied into utm_medium.+ expect(limited).toEqual(+ expect.objectContaining({+ error: expect.not.stringContaining(`${taggedSignupUrl}.`),+ }),+ );+ for (const [auth, status] of [+ [limited, 429],+ [unsupported, 401],+ [suspicious, 403],+ ] as const) {+ expect(auth).toEqual(+ expect.objectContaining({+ success: false,+ status,+ // Nothing follows the URL's query, so a dropped utm_content stays out.+ error: expect.stringMatching(+ /https:\/\/www\.firecrawl\.dev\/signin\?utm_source=keyless&utm_medium=api(?![&\w])/,+ ),+ }),+ );+ }+ });+it("writes normal API-key ACUC entries to the general-purpose cache", async () => {config.USE_DB_AUTHENTICATION = true;vi.mocked(getValue).mockResolvedValue(null);diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 485726514..61591cfc4 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -14,6 +14,7 @@ import {import { isTrustedAgentInteropRequest } from "../lib/agent-interop";import {KEYLESS_FREE_TIER_LIMIT_MESSAGE,+ KEYLESS_SIGNUP_URL,consumeKeylessRequest,isKeylessConfigured,keylessExhaustionTelemetry,@@ -463,12 +464,12 @@ export async function clearACUCTeam(team_id: string): Promise<void> {await deleteKey(`acuc_team_${team_id}`);}-const KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE = `This endpoint is not supported by the keyless free tier. Sign up for a free API key at https://www.firecrawl.dev/signin for more endpoints, more usage, and higher rate limits.+const KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE = `This endpoint is not supported by the keyless free tier. Sign up for a free API key at ${KEYLESS_SIGNUP_URL} for more endpoints, more usage, and higher rate limits.Then authenticate with:Authorization: Bearer YOUR_API_KEY`;-const KEYLESS_SUSPICIOUS_IP_MESSAGE = `Unfortunately, your IP address looks suspicious, so Firecrawl can't be used without an API key from here. Sign up for a free API key at https://firecrawl.dev for 1000 credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;+const KEYLESS_SUSPICIOUS_IP_MESSAGE = `Unfortunately, your IP address looks suspicious, so Firecrawl can't be used without an API key from here. Sign up for a free API key at ${KEYLESS_SIGNUP_URL} for 1000 credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;/*** Keyless free tier: official MCP/CLI/SDK clients can call scrape, search, anddiff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex 04a91f0ba..bc619786a 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -23,8 +23,14 @@ import {const KEYLESS_REQUESTS_PER_DAY = config.KEYLESS_REQUESTS_PER_DAY;const KEYLESS_CREDITS_PER_DAY = config.KEYLESS_CREDITS_PER_DAY;-// Shared 429 copy for both keyless request-cap and credit-cap failures.-export const KEYLESS_FREE_TIER_LIMIT_MESSAGE = `You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at https://www.firecrawl.dev/signin.+// Signup link for every keyless prompt. The UTM tags attribute the new account+// to the keyless free tier; the CLI swaps utm_medium to `cli` before printing.+export const KEYLESS_SIGNUP_URL =+ "https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=api";++// Shared 429 copy for both keyless request-cap and credit-cap failures. The URL+// ends the sentence without a period so a copied link keeps a clean utm_medium.+export const KEYLESS_FREE_TIER_LIMIT_MESSAGE = `You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at ${KEYLESS_SIGNUP_URL}Then authenticate with:Authorization: Bearer YOUR_API_KEY`;diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.tsindex 09539e810..66ed3241a 100644--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts@@ -30,7 +30,7 @@ export type XTwitterPostUrl = {normalizedUrl: string;};-export type XTwitterUrl = XTwitterProfileUrl | XTwitterPostUrl;+type XTwitterUrl = XTwitterProfileUrl | XTwitterPostUrl;export function parseXTwitterUrl(url: string): XTwitterUrl | null {let parsed: URL;