Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api/keyless): optional Spur Context IP reputation check (#3792)

Gergő Móricz9f77f3ddabff
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.ts
index cbc494f28..f68bec993 100644
--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts
+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts
@@ -66,7 +66,11 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
const response = await request(TEST_API_URL)
.post("/v2/scrape")
.set("Content-Type", "application/json")
- .send({ url: TEST_SUITE_WEBSITE, origin: "api", formats: ["markdown"] });
+ .send({
+ url: TEST_SUITE_WEBSITE,
+ origin: "api",
+ formats: ["markdown"],
+ });
expect(response.statusCode).toBe(200);
expect(response.body.success).toBe(true);
@@ -310,3 +314,89 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
scrapeTimeout,
);
});
+
+// Spur Context IP-reputation check on the keyless tier. Only runs when both the
+// keyless tier and the Spur integration (SPUR_API_KEY) are configured, plus the
+// proxy secret so we can forward a specific test IP. We pre-seed the 30-day
+// Spur cache so these are hermetic — no real Spur API call is made (the lookup
+// reads the cache first).
+const SPUR_ENABLED =
+ KEYLESS_ENABLED &&
+ !!process.env.SPUR_API_KEY &&
+ !!process.env.KEYLESS_PROXY_SECRET;
+
+describeIf(SPUR_ENABLED)("Keyless free tier — Spur IP reputation", () => {
+ const spurKey = (ip: string) => `spur_context:${ip}`;
+
+ beforeAll(() => {
+ config.USE_DB_AUTHENTICATION = true;
+ });
+
+ afterAll(async () => {
+ delete config.USE_DB_AUTHENTICATION;
+ const keys = await redisRateLimitClient.keys("spur_context:*");
+ if (keys.length > 0) await redisRateLimitClient.del(...keys);
+ });
+
+ it(
+ "refuses keyless for an IP Spur flags as a VPN/proxy tunnel (403)",
+ async () => {
+ const ip = "203.0.113.66";
+ // Seed the Spur cache with a suspicious context (active VPN tunnel).
+ await redisRateLimitClient.set(
+ spurKey(ip),
+ JSON.stringify({
+ ip,
+ tunnels: [{ type: "VPN", operator: "PROTON_VPN" }],
+ }),
+ );
+
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)
+ .set("x-firecrawl-keyless-ip", ip)
+ .send({
+ url: TEST_SUITE_WEBSITE,
+ origin: "mcp",
+ formats: ["markdown"],
+ });
+
+ expect(response.statusCode).toBe(403);
+ expect(response.body.success).toBe(false);
+ expect(response.body.error).toContain("suspicious");
+ // Out of the keyless path → emit the OAuth-discovery header.
+ expect(response.headers["www-authenticate"]).toContain(
+ "resource_metadata",
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "allows keyless for a clean (non-anonymizing) IP per Spur (200)",
+ async () => {
+ const ip = "203.0.113.67";
+ // Seed a clean context: datacenter alone is not "suspicious".
+ await redisRateLimitClient.set(
+ spurKey(ip),
+ JSON.stringify({ ip, infrastructure: "DATACENTER", risks: [] }),
+ );
+
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)
+ .set("x-firecrawl-keyless-ip", ip)
+ .send({
+ url: TEST_SUITE_WEBSITE,
+ origin: "mcp",
+ formats: ["markdown"],
+ });
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ },
+ scrapeTimeout,
+ );
+});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 944ab1647..a3a88d9ac 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -58,6 +58,11 @@ const configSchema = z.object({
// forward the real client IP for keyless rate-limiting via the
// `x-firecrawl-keyless-ip` header. Untrusted callers can't override their IP.
KEYLESS_PROXY_SECRET: z.string().optional(),
+ // Optional Spur Context API token (https://docs.spur.us/context-api). When
+ // set, keyless requests have their client IP checked against Spur and are
+ // refused if the IP fronts anonymizing/rotating infrastructure (VPN/proxy/
+ // TOR). Unset disables the check entirely (keyless behaves as before).
+ SPUR_API_KEY: z.string().optional(),
// API Keys & Authentication
BULL_AUTH_KEY: z.string().optional(),
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index 0f6bdfc5e..f07a6c891 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -14,6 +14,7 @@ import {
isKeylessIpEligible,
keylessTeamId,
} from "../lib/keyless";
+import { isKeylessIpSuspicious } from "../lib/spur";
import { deleteKey, getValue, setValue } from "../services/redis";
import { redlock } from "../services/redlock";
import { eq } from "drizzle-orm";
@@ -474,6 +475,8 @@ const KEYLESS_CREDITS_MESSAGE = `You've reached today's limit of free, unauthent
const KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE = `This endpoint is not available without an API key. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
+const KEYLESS_SUSPICIOUS_IP_MESSAGE = `Unfortunately, your IP address looks suspicious, so Firecrawl can't be used without an API key from here. Sign up for a free API key at https://firecrawl.dev for 1000 credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
+
/**
* Keyless free tier: official MCP/CLI/SDK clients can call scrape, search, and
* interact with no API key. `origin`/`integration` are client-set and spoofable,
@@ -533,6 +536,28 @@ async function handleKeylessAuth(
// usable as arbitrary limiter buckets. Anything else falls through to 401.
if (!isKeylessIpEligible(ip)) return unauthorized;
+ // Optional Spur Context check (only when SPUR_API_KEY is set): refuse keyless
+ // for IPs fronting anonymizing/rotating infrastructure (VPN/proxy/TOR), the
+ // main way the per-IP caps get bypassed. Fails open on any Spur error, and
+ // runs before consuming quota so a flagged IP doesn't burn a request slot.
+ if (await isKeylessIpSuspicious(ip)) {
+ logger.warn("Keyless request blocked: suspicious IP", {
+ canonicalLog: "keyless/consume",
+ ip,
+ origin: req.body?.origin,
+ integration: req.body?.integration,
+ blocked: true,
+ reason: "suspicious",
+ });
+ return {
+ success: false,
+ error: KEYLESS_SUSPICIOUS_IP_MESSAGE,
+ status: 403,
+ // Tell agents where to find the key/signup flow they now need.
+ agentAuthDiscovery: true,
+ };
+ }
+
const teamId = keylessTeamId(ip);
const modeLabel =
mode === RateLimiterMode.Search
diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.ts
index 3bdb6e2c1..a634a6a53 100644
--- a/apps/api/src/lib/keyless.ts
+++ b/apps/api/src/lib/keyless.ts
@@ -4,6 +4,7 @@ import { config } from "../config";
import { db } from "../db/connection";
import * as schema from "../db/schema";
import { redisRateLimitClient } from "../services/rate-limiter";
+import { isKeylessIpSuspicious } from "./spur";
// Keyless free tier: scrape, search, and interact can be used without an API key
// from the official MCP server, CLI, or SDKs. It's gated per-IP/day by TWO
@@ -129,6 +130,12 @@ export async function checkKeylessEligibility(
if (!ip || !isKeylessIpEligible(ip)) {
return { eligible: false, reason: "ineligible_ip" };
}
+ // Optional Spur Context check (only when SPUR_API_KEY is set): treat IPs on
+ // anonymizing/rotating infrastructure as ineligible so the hosted MCP issues
+ // an OAuth challenge instead of serving keyless that auth would then reject.
+ if (await isKeylessIpSuspicious(ip)) {
+ return { eligible: false, reason: "suspicious" };
+ }
try {
const requestsUsed = parseInt(
(await redisRateLimitClient.get(requestsKey(ip))) ?? "0",
diff --git a/apps/api/src/lib/spur.ts b/apps/api/src/lib/spur.ts
new file mode 100644
index 000000000..a61ee60cb
--- /dev/null
+++ b/apps/api/src/lib/spur.ts
@@ -0,0 +1,165 @@
+import { config } from "../config";
+import { logger } from "./logger";
+import { redisRateLimitClient } from "../services/rate-limiter";
+
+// Optional Spur Context API integration for the keyless free tier. When
+// SPUR_API_KEY is set, the IP behind every keyless request is looked up against
+// Spur's IP-context database (https://docs.spur.us/context-api). IPs fronting
+// anonymizing/rotating infrastructure — VPN/proxy/TOR tunnels or residential
+// proxy networks — are the cheapest way to defeat the per-IP keyless caps, so we
+// refuse keyless for them and steer the caller to sign up for an API key.
+//
+// Lookups are cached in Redis for 30 days so a given IP costs at most one Spur
+// API call per month. The integration is entirely optional: with no key set the
+// keyless tier behaves exactly as before, and any Spur error fails open (the
+// request is allowed) so a Spur outage can't take down the free tier.
+
+const SPUR_API_BASE = "https://api.spur.us/v2/context";
+const CACHE_TTL_SECONDS = 30 * 24 * 60 * 60; // 30 days
+const cacheKey = (ip: string) => `spur_context:${ip}`;
+
+// Subset of the Spur IP Context Object we read. See the API docs for the full
+// shape; everything here is optional because Spur omits empty fields.
+type SpurContext = {
+ ip?: string;
+ infrastructure?: string;
+ risks?: string[];
+ tunnels?: { anonymous?: boolean; operator?: string; type?: string }[];
+ client?: { behaviors?: string[]; proxies?: string[] };
+};
+
+function isSpurEnabled(): boolean {
+ return (
+ typeof config.SPUR_API_KEY === "string" && config.SPUR_API_KEY.length > 0
+ );
+}
+
+async function getCachedContext(ip: string): Promise<SpurContext | null> {
+ try {
+ const raw = await redisRateLimitClient.get(cacheKey(ip));
+ return raw ? (JSON.parse(raw) as SpurContext) : null;
+ } catch (error) {
+ // Cache read failed (store down or corrupt value) — treat as a miss.
+ logger.warn("Failed to read Spur context from cache", {
+ canonicalLog: "spur/lookup",
+ ip,
+ error,
+ });
+ return null;
+ }
+}
+
+async function cacheContext(ip: string, ctx: SpurContext): Promise<void> {
+ try {
+ await redisRateLimitClient.set(
+ cacheKey(ip),
+ JSON.stringify(ctx),
+ "EX",
+ CACHE_TTL_SECONDS,
+ );
+ } catch (error) {
+ // Best-effort: a failed cache write just means we look the IP up again.
+ logger.warn("Failed to cache Spur context", {
+ canonicalLog: "spur/lookup",
+ ip,
+ error,
+ });
+ }
+}
+
+async function fetchContext(ip: string): Promise<SpurContext | null> {
+ // Cache miss → hit the real Spur API. Logged so we can track real spend.
+ logger.info("Spur Context API request (cache miss)", {
+ canonicalLog: "spur/lookup",
+ ip,
+ });
+ const response = await fetch(`${SPUR_API_BASE}/${encodeURIComponent(ip)}`, {
+ method: "GET",
+ headers: { Token: config.SPUR_API_KEY! },
+ });
+ if (!response.ok) {
+ logger.warn("Spur Context API request failed", {
+ canonicalLog: "spur/lookup",
+ ip,
+ status: response.status,
+ });
+ return null;
+ }
+ return (await response.json()) as SpurContext;
+}
+
+/**
+ * Look up an IP's Spur context, preferring the 30-day Redis cache and only
+ * caching successful (non-error) responses. Returns null when Spur is disabled
+ * or the lookup fails — callers then fail open (treat the IP as not suspicious).
+ */
+async function getSpurContext(ip: string): Promise<SpurContext | null> {
+ if (!isSpurEnabled()) return null;
+
+ const cached = await getCachedContext(ip);
+ if (cached) return cached;
+
+ let ctx: SpurContext | null;
+ try {
+ ctx = await fetchContext(ip);
+ } catch (error) {
+ logger.warn("Spur Context API request errored", {
+ canonicalLog: "spur/lookup",
+ ip,
+ error,
+ });
+ return null;
+ }
+
+ // Only cache non-error responses.
+ if (ctx) await cacheContext(ip, ctx);
+ return ctx;
+}
+
+// Risk flags that, on their own, mark an IP as fronting proxy/tunnel
+// infrastructure. Plain DATACENTER or GEO_MISMATCH signals are intentionally
+// NOT treated as suspicious — many legitimate clients hit a free tier from
+// cloud/CGNAT, and the per-IP caps already cover those.
+const SUSPICIOUS_RISKS = new Set(["CALLBACK_PROXY", "TUNNEL"]);
+
+function isSuspiciousContext(ctx: SpurContext): boolean {
+ // A live VPN/proxy/TOR tunnel — the canonical IP-rotation vector.
+ if (Array.isArray(ctx.tunnels) && ctx.tunnels.length > 0) return true;
+ // Residential / rotating proxy networks observed exiting this IP.
+ if (Array.isArray(ctx.client?.proxies) && ctx.client.proxies.length > 0) {
+ return true;
+ }
+ // Explicit proxy/tunnel risk flags.
+ if (
+ Array.isArray(ctx.risks) &&
+ ctx.risks.some(r => SUSPICIOUS_RISKS.has(r))
+ ) {
+ return true;
+ }
+ return false;
+}
+
+/**
+ * Whether the keyless tier should refuse this IP because Spur flags it as
+ * anonymizing/rotating infrastructure. No-op (false) when Spur is disabled, and
+ * fails open (false) on any lookup error so a Spur outage never breaks keyless.
+ */
+export async function isKeylessIpSuspicious(ip: string): Promise<boolean> {
+ if (!isSpurEnabled()) return false;
+
+ const ctx = await getSpurContext(ip);
+ if (!ctx) return false;
+
+ const suspicious = isSuspiciousContext(ctx);
+ if (suspicious) {
+ logger.info("Keyless IP flagged suspicious by Spur", {
+ canonicalLog: "spur/lookup",
+ ip,
+ suspicious: true,
+ tunnels: ctx.tunnels?.map(t => t.type),
+ proxies: ctx.client?.proxies,
+ risks: ctx.risks,
+ });
+ }
+ return suspicious;
+}