Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): tag keyless signup links with UTM parameters (#4847)

Maxb5d187b19de3
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.ts
index 6bf149b7d..addb6b2ab 100644
--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts
+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts
@@ -40,6 +40,11 @@ async function flushKeylessBuckets() {
}
}
+// Every keyless prompt links to signup with these tags so new accounts can be
+// attributed to the keyless free tier.
+const KEYLESS_SIGNUP_URL =
+ "https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=api";
+
// Recover the loopback IP the server keyed on, so we can seed its credit counter.
async function currentKeylessIp(): Promise<string> {
const keys = await redisRateLimitClient.keys("keyless_requests:*");
@@ -107,7 +112,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
expect(response.body.error).toContain(
"not supported by the keyless free tier",
);
- expect(response.body.error).toContain("https://www.firecrawl.dev/signin");
+ expect(response.body.error).toContain(KEYLESS_SIGNUP_URL);
expect(response.body.error).toContain("Authorization: Bearer YOUR_API_KEY");
});
@@ -158,7 +163,8 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
expect(blocked.statusCode).toBe(429);
expect(blocked.body.error).toContain("keyless free tier rate limit");
- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");
+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);
+ expect(blocked.body.error).not.toContain(`${KEYLESS_SIGNUP_URL}.`);
expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY");
// Out of quota β†’ emit the OAuth-discovery header so agents find the key flow.
expect(blocked.headers["www-authenticate"]).toContain("resource_metadata");
@@ -183,7 +189,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
expect(blocked.statusCode).toBe(429);
expect(blocked.body.error).toContain("keyless free tier rate limit");
- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");
+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);
expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY");
});
@@ -212,7 +218,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
expect(blocked.statusCode).toBe(429);
expect(blocked.body.error).toContain("keyless free tier rate limit");
- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");
+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);
expect(blocked.body.error).toContain("Authorization: Bearer YOUR_API_KEY");
});
@@ -255,7 +261,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
expect(blocked.statusCode).toBe(429);
expect(blocked.body.reason).toBe("credits");
expect(blocked.body.error).toContain("keyless free tier rate limit");
- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");
+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);
expect(blocked.body.error).toContain(
"Authorization: Bearer YOUR_API_KEY",
);
@@ -300,7 +306,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
expect(blocked.statusCode).toBe(429);
expect(blocked.body.reason).toBe("credits");
expect(blocked.body.error).toContain("keyless free tier rate limit");
- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");
+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);
expect(blocked.body.error).toContain(
"Authorization: Bearer YOUR_API_KEY",
);
@@ -342,7 +348,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
expect(blocked.statusCode).toBe(429);
expect(blocked.body.error).toContain("keyless free tier rate limit");
- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");
+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);
expect(blocked.body.error).toContain(
"Authorization: Bearer YOUR_API_KEY",
);
@@ -472,7 +478,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
expect(blocked.statusCode).toBe(429);
expect(blocked.body.error).toContain("keyless free tier rate limit");
- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");
+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);
expect(blocked.body.error).toContain(
"Authorization: Bearer YOUR_API_KEY",
);
@@ -511,7 +517,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
expect(blocked.statusCode).toBe(429);
expect(blocked.body.error).toContain("keyless free tier rate limit");
- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");
+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);
expect(blocked.body.error).toContain(
"Authorization: Bearer YOUR_API_KEY",
);
@@ -562,7 +568,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
.send({ origin: "mcp" });
expect(blocked.statusCode).toBe(429);
expect(blocked.body.error).toContain("keyless free tier rate limit");
- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");
+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);
expect(blocked.body.error).toContain(
"Authorization: Bearer YOUR_API_KEY",
);
@@ -685,7 +691,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
expect(blocked.statusCode).toBe(429);
expect(blocked.body.error).toContain("keyless free tier rate limit");
- expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");
+ expect(blocked.body.error).toContain(KEYLESS_SIGNUP_URL);
expect(blocked.body.error).toContain(
"Authorization: Bearer YOUR_API_KEY",
);
@@ -805,6 +811,7 @@ describeIf(SPUR_ENABLED)("Keyless free tier β€” Spur IP reputation", () => {
expect(response.statusCode).toBe(403);
expect(response.body.success).toBe(false);
expect(response.body.error).toContain("suspicious");
+ expect(response.body.error).toContain(KEYLESS_SIGNUP_URL);
// Out of the keyless path β†’ emit the OAuth-discovery header.
expect(response.headers["www-authenticate"]).toContain(
"resource_metadata",
diff --git a/apps/api/src/controllers/__tests__/auth.test.ts b/apps/api/src/controllers/__tests__/auth.test.ts
index 05b3d3bdb..647a43b84 100644
--- a/apps/api/src/controllers/__tests__/auth.test.ts
+++ b/apps/api/src/controllers/__tests__/auth.test.ts
@@ -20,6 +20,7 @@ import {
keylessConversionCohort,
} from "../../lib/keyless";
import { logger } from "../../lib/logger";
+import { isKeylessIpSuspicious } from "../../lib/spur";
import { db } from "../../db/connection";
import { autumnService } from "../../services/autumn/autumn.service";
@@ -213,6 +214,67 @@ describe("authenticateUser", () => {
);
});
+ it("links every keyless signup prompt to the keyless-tagged signup URL", async () => {
+ config.USE_DB_AUTHENTICATION = true;
+ vi.mocked(isKeylessConfigured).mockReturnValue(true);
+ vi.mocked(consumeKeylessRequest).mockResolvedValue({
+ ok: false,
+ reason: "credits",
+ requestsUsed: 1,
+ creditsUsed: 100,
+ });
+ vi.spyOn(logger, "warn").mockImplementation(() => logger);
+ const keylessRequest = () => ({
+ headers: {},
+ socket: { remoteAddress: "203.0.113.8" },
+ });
+ const taggedSignupUrl =
+ "https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=api";
+
+ const limited = await authenticateUser(
+ keylessRequest(),
+ {},
+ RateLimiterMode.Scrape,
+ { allowKeyless: true },
+ );
+ const unsupported = await authenticateUser(
+ keylessRequest(),
+ {},
+ RateLimiterMode.Scrape,
+ { allowKeyless: false },
+ );
+ vi.mocked(isKeylessIpSuspicious).mockResolvedValueOnce(true);
+ const suspicious = await authenticateUser(
+ keylessRequest(),
+ {},
+ RateLimiterMode.Scrape,
+ { allowKeyless: true },
+ );
+
+ // A period right after the URL would be copied into utm_medium.
+ expect(limited).toEqual(
+ expect.objectContaining({
+ error: expect.not.stringContaining(`${taggedSignupUrl}.`),
+ }),
+ );
+ for (const [auth, status] of [
+ [limited, 429],
+ [unsupported, 401],
+ [suspicious, 403],
+ ] as const) {
+ expect(auth).toEqual(
+ expect.objectContaining({
+ success: false,
+ status,
+ // Nothing follows the URL's query, so a dropped utm_content stays out.
+ error: expect.stringMatching(
+ /https:\/\/www\.firecrawl\.dev\/signin\?utm_source=keyless&utm_medium=api(?![&\w])/,
+ ),
+ }),
+ );
+ }
+ });
+
it("writes normal API-key ACUC entries to the general-purpose cache", async () => {
config.USE_DB_AUTHENTICATION = true;
vi.mocked(getValue).mockResolvedValue(null);
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index 485726514..61591cfc4 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -14,6 +14,7 @@ import {
import { isTrustedAgentInteropRequest } from "../lib/agent-interop";
import {
KEYLESS_FREE_TIER_LIMIT_MESSAGE,
+ KEYLESS_SIGNUP_URL,
consumeKeylessRequest,
isKeylessConfigured,
keylessExhaustionTelemetry,
@@ -463,12 +464,12 @@ export async function clearACUCTeam(team_id: string): Promise<void> {
await deleteKey(`acuc_team_${team_id}`);
}
-const KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE = `This endpoint is not supported by the keyless free tier. Sign up for a free API key at https://www.firecrawl.dev/signin for more endpoints, more usage, and higher rate limits.
+const KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE = `This endpoint is not supported by the keyless free tier. Sign up for a free API key at ${KEYLESS_SIGNUP_URL} for more endpoints, more usage, and higher rate limits.
Then authenticate with:
Authorization: Bearer YOUR_API_KEY`;
-const KEYLESS_SUSPICIOUS_IP_MESSAGE = `Unfortunately, your IP address looks suspicious, so Firecrawl can't be used without an API key from here. Sign up for a free API key at https://firecrawl.dev for 1000 credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
+const KEYLESS_SUSPICIOUS_IP_MESSAGE = `Unfortunately, your IP address looks suspicious, so Firecrawl can't be used without an API key from here. Sign up for a free API key at ${KEYLESS_SIGNUP_URL} for 1000 credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
/**
* Keyless free tier: official MCP/CLI/SDK clients can call scrape, search, and
diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.ts
index 04a91f0ba..bc619786a 100644
--- a/apps/api/src/lib/keyless.ts
+++ b/apps/api/src/lib/keyless.ts
@@ -23,8 +23,14 @@ import {
const KEYLESS_REQUESTS_PER_DAY = config.KEYLESS_REQUESTS_PER_DAY;
const KEYLESS_CREDITS_PER_DAY = config.KEYLESS_CREDITS_PER_DAY;
-// Shared 429 copy for both keyless request-cap and credit-cap failures.
-export const KEYLESS_FREE_TIER_LIMIT_MESSAGE = `You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at https://www.firecrawl.dev/signin.
+// Signup link for every keyless prompt. The UTM tags attribute the new account
+// to the keyless free tier; the CLI swaps utm_medium to `cli` before printing.
+export const KEYLESS_SIGNUP_URL =
+ "https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=api";
+
+// Shared 429 copy for both keyless request-cap and credit-cap failures. The URL
+// ends the sentence without a period so a copied link keeps a clean utm_medium.
+export const KEYLESS_FREE_TIER_LIMIT_MESSAGE = `You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at ${KEYLESS_SIGNUP_URL}
Then authenticate with:
Authorization: Bearer YOUR_API_KEY`;
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts
index 09539e810..66ed3241a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts
@@ -30,7 +30,7 @@ export type XTwitterPostUrl = {
normalizedUrl: string;
};
-export type XTwitterUrl = XTwitterProfileUrl | XTwitterPostUrl;
+type XTwitterUrl = XTwitterProfileUrl | XTwitterPostUrl;
export function parseXTwitterUrl(url: string): XTwitterUrl | null {
let parsed: URL;