Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/keyless): optional Spur Context IP reputation check (#3792)
Gergő Móricz9f77f3ddabff
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.tsindex cbc494f28..f68bec993 100644--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts@@ -66,7 +66,11 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {const response = await request(TEST_API_URL).post("/v2/scrape").set("Content-Type", "application/json")- .send({ url: TEST_SUITE_WEBSITE, origin: "api", formats: ["markdown"] });+ .send({+ url: TEST_SUITE_WEBSITE,+ origin: "api",+ formats: ["markdown"],+ });expect(response.statusCode).toBe(200);expect(response.body.success).toBe(true);@@ -310,3 +314,89 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {scrapeTimeout,);});++// Spur Context IP-reputation check on the keyless tier. Only runs when both the+// keyless tier and the Spur integration (SPUR_API_KEY) are configured, plus the+// proxy secret so we can forward a specific test IP. We pre-seed the 30-day+// Spur cache so these are hermetic — no real Spur API call is made (the lookup+// reads the cache first).+const SPUR_ENABLED =+ KEYLESS_ENABLED &&+ !!process.env.SPUR_API_KEY &&+ !!process.env.KEYLESS_PROXY_SECRET;++describeIf(SPUR_ENABLED)("Keyless free tier — Spur IP reputation", () => {+ const spurKey = (ip: string) => `spur_context:${ip}`;++ beforeAll(() => {+ config.USE_DB_AUTHENTICATION = true;+ });++ afterAll(async () => {+ delete config.USE_DB_AUTHENTICATION;+ const keys = await redisRateLimitClient.keys("spur_context:*");+ if (keys.length > 0) await redisRateLimitClient.del(...keys);+ });++ it(+ "refuses keyless for an IP Spur flags as a VPN/proxy tunnel (403)",+ async () => {+ const ip = "203.0.113.66";+ // Seed the Spur cache with a suspicious context (active VPN tunnel).+ await redisRateLimitClient.set(+ spurKey(ip),+ JSON.stringify({+ ip,+ tunnels: [{ type: "VPN", operator: "PROTON_VPN" }],+ }),+ );++ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)+ .set("x-firecrawl-keyless-ip", ip)+ .send({+ url: TEST_SUITE_WEBSITE,+ origin: "mcp",+ formats: ["markdown"],+ });++ expect(response.statusCode).toBe(403);+ expect(response.body.success).toBe(false);+ expect(response.body.error).toContain("suspicious");+ // Out of the keyless path → emit the OAuth-discovery header.+ expect(response.headers["www-authenticate"]).toContain(+ "resource_metadata",+ );+ },+ scrapeTimeout,+ );++ it(+ "allows keyless for a clean (non-anonymizing) IP per Spur (200)",+ async () => {+ const ip = "203.0.113.67";+ // Seed a clean context: datacenter alone is not "suspicious".+ await redisRateLimitClient.set(+ spurKey(ip),+ JSON.stringify({ ip, infrastructure: "DATACENTER", risks: [] }),+ );++ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .set("x-firecrawl-keyless-secret", process.env.KEYLESS_PROXY_SECRET!)+ .set("x-firecrawl-keyless-ip", ip)+ .send({+ url: TEST_SUITE_WEBSITE,+ origin: "mcp",+ formats: ["markdown"],+ });++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ },+ scrapeTimeout,+ );+});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 944ab1647..a3a88d9ac 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -58,6 +58,11 @@ const configSchema = z.object({// forward the real client IP for keyless rate-limiting via the// `x-firecrawl-keyless-ip` header. Untrusted callers can't override their IP.KEYLESS_PROXY_SECRET: z.string().optional(),+ // Optional Spur Context API token (https://docs.spur.us/context-api). When+ // set, keyless requests have their client IP checked against Spur and are+ // refused if the IP fronts anonymizing/rotating infrastructure (VPN/proxy/+ // TOR). Unset disables the check entirely (keyless behaves as before).+ SPUR_API_KEY: z.string().optional(),// API Keys & AuthenticationBULL_AUTH_KEY: z.string().optional(),diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 0f6bdfc5e..f07a6c891 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -14,6 +14,7 @@ import {isKeylessIpEligible,keylessTeamId,} from "../lib/keyless";+import { isKeylessIpSuspicious } from "../lib/spur";import { deleteKey, getValue, setValue } from "../services/redis";import { redlock } from "../services/redlock";import { eq } from "drizzle-orm";@@ -474,6 +475,8 @@ const KEYLESS_CREDITS_MESSAGE = `You've reached today's limit of free, unauthentconst KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE = `This endpoint is not available without an API key. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;+const KEYLESS_SUSPICIOUS_IP_MESSAGE = `Unfortunately, your IP address looks suspicious, so Firecrawl can't be used without an API key from here. Sign up for a free API key at https://firecrawl.dev for 1000 credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;+/*** Keyless free tier: official MCP/CLI/SDK clients can call scrape, search, and* interact with no API key. `origin`/`integration` are client-set and spoofable,@@ -533,6 +536,28 @@ async function handleKeylessAuth(// usable as arbitrary limiter buckets. Anything else falls through to 401.if (!isKeylessIpEligible(ip)) return unauthorized;+ // Optional Spur Context check (only when SPUR_API_KEY is set): refuse keyless+ // for IPs fronting anonymizing/rotating infrastructure (VPN/proxy/TOR), the+ // main way the per-IP caps get bypassed. Fails open on any Spur error, and+ // runs before consuming quota so a flagged IP doesn't burn a request slot.+ if (await isKeylessIpSuspicious(ip)) {+ logger.warn("Keyless request blocked: suspicious IP", {+ canonicalLog: "keyless/consume",+ ip,+ origin: req.body?.origin,+ integration: req.body?.integration,+ blocked: true,+ reason: "suspicious",+ });+ return {+ success: false,+ error: KEYLESS_SUSPICIOUS_IP_MESSAGE,+ status: 403,+ // Tell agents where to find the key/signup flow they now need.+ agentAuthDiscovery: true,+ };+ }+const teamId = keylessTeamId(ip);const modeLabel =mode === RateLimiterMode.Searchdiff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex 3bdb6e2c1..a634a6a53 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -4,6 +4,7 @@ import { config } from "../config";import { db } from "../db/connection";import * as schema from "../db/schema";import { redisRateLimitClient } from "../services/rate-limiter";+import { isKeylessIpSuspicious } from "./spur";// Keyless free tier: scrape, search, and interact can be used without an API key// from the official MCP server, CLI, or SDKs. It's gated per-IP/day by TWO@@ -129,6 +130,12 @@ export async function checkKeylessEligibility(if (!ip || !isKeylessIpEligible(ip)) {return { eligible: false, reason: "ineligible_ip" };}+ // Optional Spur Context check (only when SPUR_API_KEY is set): treat IPs on+ // anonymizing/rotating infrastructure as ineligible so the hosted MCP issues+ // an OAuth challenge instead of serving keyless that auth would then reject.+ if (await isKeylessIpSuspicious(ip)) {+ return { eligible: false, reason: "suspicious" };+ }try {const requestsUsed = parseInt((await redisRateLimitClient.get(requestsKey(ip))) ?? "0",diff --git a/apps/api/src/lib/spur.ts b/apps/api/src/lib/spur.tsnew file mode 100644index 000000000..a61ee60cb--- /dev/null+++ b/apps/api/src/lib/spur.ts@@ -0,0 +1,165 @@+import { config } from "../config";+import { logger } from "./logger";+import { redisRateLimitClient } from "../services/rate-limiter";++// Optional Spur Context API integration for the keyless free tier. When+// SPUR_API_KEY is set, the IP behind every keyless request is looked up against+// Spur's IP-context database (https://docs.spur.us/context-api). IPs fronting+// anonymizing/rotating infrastructure — VPN/proxy/TOR tunnels or residential+// proxy networks — are the cheapest way to defeat the per-IP keyless caps, so we+// refuse keyless for them and steer the caller to sign up for an API key.+//+// Lookups are cached in Redis for 30 days so a given IP costs at most one Spur+// API call per month. The integration is entirely optional: with no key set the+// keyless tier behaves exactly as before, and any Spur error fails open (the+// request is allowed) so a Spur outage can't take down the free tier.++const SPUR_API_BASE = "https://api.spur.us/v2/context";+const CACHE_TTL_SECONDS = 30 * 24 * 60 * 60; // 30 days+const cacheKey = (ip: string) => `spur_context:${ip}`;++// Subset of the Spur IP Context Object we read. See the API docs for the full+// shape; everything here is optional because Spur omits empty fields.+type SpurContext = {+ ip?: string;+ infrastructure?: string;+ risks?: string[];+ tunnels?: { anonymous?: boolean; operator?: string; type?: string }[];+ client?: { behaviors?: string[]; proxies?: string[] };+};++function isSpurEnabled(): boolean {+ return (+ typeof config.SPUR_API_KEY === "string" && config.SPUR_API_KEY.length > 0+ );+}++async function getCachedContext(ip: string): Promise<SpurContext | null> {+ try {+ const raw = await redisRateLimitClient.get(cacheKey(ip));+ return raw ? (JSON.parse(raw) as SpurContext) : null;+ } catch (error) {+ // Cache read failed (store down or corrupt value) — treat as a miss.+ logger.warn("Failed to read Spur context from cache", {+ canonicalLog: "spur/lookup",+ ip,+ error,+ });+ return null;+ }+}++async function cacheContext(ip: string, ctx: SpurContext): Promise<void> {+ try {+ await redisRateLimitClient.set(+ cacheKey(ip),+ JSON.stringify(ctx),+ "EX",+ CACHE_TTL_SECONDS,+ );+ } catch (error) {+ // Best-effort: a failed cache write just means we look the IP up again.+ logger.warn("Failed to cache Spur context", {+ canonicalLog: "spur/lookup",+ ip,+ error,+ });+ }+}++async function fetchContext(ip: string): Promise<SpurContext | null> {+ // Cache miss → hit the real Spur API. Logged so we can track real spend.+ logger.info("Spur Context API request (cache miss)", {+ canonicalLog: "spur/lookup",+ ip,+ });+ const response = await fetch(`${SPUR_API_BASE}/${encodeURIComponent(ip)}`, {+ method: "GET",+ headers: { Token: config.SPUR_API_KEY! },+ });+ if (!response.ok) {+ logger.warn("Spur Context API request failed", {+ canonicalLog: "spur/lookup",+ ip,+ status: response.status,+ });+ return null;+ }+ return (await response.json()) as SpurContext;+}++/**+ * Look up an IP's Spur context, preferring the 30-day Redis cache and only+ * caching successful (non-error) responses. Returns null when Spur is disabled+ * or the lookup fails — callers then fail open (treat the IP as not suspicious).+ */+async function getSpurContext(ip: string): Promise<SpurContext | null> {+ if (!isSpurEnabled()) return null;++ const cached = await getCachedContext(ip);+ if (cached) return cached;++ let ctx: SpurContext | null;+ try {+ ctx = await fetchContext(ip);+ } catch (error) {+ logger.warn("Spur Context API request errored", {+ canonicalLog: "spur/lookup",+ ip,+ error,+ });+ return null;+ }++ // Only cache non-error responses.+ if (ctx) await cacheContext(ip, ctx);+ return ctx;+}++// Risk flags that, on their own, mark an IP as fronting proxy/tunnel+// infrastructure. Plain DATACENTER or GEO_MISMATCH signals are intentionally+// NOT treated as suspicious — many legitimate clients hit a free tier from+// cloud/CGNAT, and the per-IP caps already cover those.+const SUSPICIOUS_RISKS = new Set(["CALLBACK_PROXY", "TUNNEL"]);++function isSuspiciousContext(ctx: SpurContext): boolean {+ // A live VPN/proxy/TOR tunnel — the canonical IP-rotation vector.+ if (Array.isArray(ctx.tunnels) && ctx.tunnels.length > 0) return true;+ // Residential / rotating proxy networks observed exiting this IP.+ if (Array.isArray(ctx.client?.proxies) && ctx.client.proxies.length > 0) {+ return true;+ }+ // Explicit proxy/tunnel risk flags.+ if (+ Array.isArray(ctx.risks) &&+ ctx.risks.some(r => SUSPICIOUS_RISKS.has(r))+ ) {+ return true;+ }+ return false;+}++/**+ * Whether the keyless tier should refuse this IP because Spur flags it as+ * anonymizing/rotating infrastructure. No-op (false) when Spur is disabled, and+ * fails open (false) on any lookup error so a Spur outage never breaks keyless.+ */+export async function isKeylessIpSuspicious(ip: string): Promise<boolean> {+ if (!isSpurEnabled()) return false;++ const ctx = await getSpurContext(ip);+ if (!ctx) return false;++ const suspicious = isSuspiciousContext(ctx);+ if (suspicious) {+ logger.info("Keyless IP flagged suspicious by Spur", {+ canonicalLog: "spur/lookup",+ ip,+ suspicious: true,+ tunnels: ctx.tunnels?.map(t => t.type),+ proxies: ctx.client?.proxies,+ risks: ctx.risks,+ });+ }+ return suspicious;+}