Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): single-flight Spur keyless IP lookups with bounded timeouts (#4330)
firecrawl-spring[bot]cc32c6eb6564
diff --git a/apps/api/src/lib/__tests__/spur.test.ts b/apps/api/src/lib/__tests__/spur.test.tsnew file mode 100644index 000000000..3329f1203--- /dev/null+++ b/apps/api/src/lib/__tests__/spur.test.ts@@ -0,0 +1,184 @@+import type { Mock } from "vitest";+import { isKeylessIpSuspicious } from "../spur";+import { redisRateLimitClient } from "../../services/rate-limiter";+import { config } from "../../config";+import { logger } from "../logger";++vi.mock("../../config", () => ({+ config: { SPUR_API_KEY: "test-key" },+}));++vi.mock("../logger", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },+}));++vi.mock("../../services/rate-limiter", () => ({+ redisRateLimitClient: { get: vi.fn(), set: vi.fn(), eval: vi.fn() },+}));++// Minimal in-memory Redis backing the mock: enough SET semantics (NX guard,+// ignored EX/PX) for the single-flight lock and the result cache.+function installFakeRedis(): Map<string, string> {+ const store = new Map<string, string>();+ (redisRateLimitClient.get as Mock).mockImplementation(async (k: string) =>+ store.has(k) ? store.get(k)! : null,+ );+ (redisRateLimitClient.set as Mock).mockImplementation(+ async (k: string, v: string, ...args: unknown[]) => {+ if (args.includes("NX") && store.has(k)) return null;+ store.set(k, v);+ return "OK";+ },+ );+ // Atomic compare-and-delete used by releaseLock: KEYS[1]=lock key, ARGV[1]=token.+ (redisRateLimitClient.eval as Mock).mockImplementation(+ async (_script: string, _numKeys: number, key: string, token: string) => {+ if (store.get(key) === token) {+ store.delete(key);+ return 1;+ }+ return 0;+ },+ );+ return store;+}++function mockFetch(impl: () => Promise<unknown>): Mock {+ const fn = vi.fn(async () => impl());+ vi.stubGlobal("fetch", fn);+ return fn;+}++const okResponse = (body: unknown) => ({ ok: true, json: async () => body });++describe("Spur keyless IP reputation", () => {+ let store: Map<string, string>;++ beforeEach(() => {+ vi.clearAllMocks();+ vi.unstubAllGlobals();+ config.SPUR_API_KEY = "test-key";+ store = installFakeRedis();+ });++ it("flags an IP fronting a live VPN/proxy tunnel", async () => {+ mockFetch(async () =>+ okResponse({ ip: "1.2.3.4", tunnels: [{ type: "VPN" }] }),+ );+ expect(await isKeylessIpSuspicious("1.2.3.4")).toBe(true);+ });++ it("allows a clean (datacenter-only) IP", async () => {+ mockFetch(async () =>+ okResponse({ ip: "1.2.3.5", infrastructure: "DATACENTER", risks: [] }),+ );+ expect(await isKeylessIpSuspicious("1.2.3.5")).toBe(false);+ });++ it("flags residential-proxy and explicit proxy/tunnel risk contexts", async () => {+ mockFetch(async () =>+ okResponse({ ip: "1.2.3.10", client: { proxies: ["RESIDENTIAL"] } }),+ );+ expect(await isKeylessIpSuspicious("1.2.3.10")).toBe(true);++ mockFetch(async () =>+ okResponse({ ip: "1.2.3.11", risks: ["CALLBACK_PROXY"] }),+ );+ expect(await isKeylessIpSuspicious("1.2.3.11")).toBe(true);+ });++ it("does not flag geo-mismatch / datacenter-only signals", async () => {+ mockFetch(async () =>+ okResponse({+ ip: "1.2.3.12",+ infrastructure: "DATACENTER",+ risks: ["GEO_MISMATCH"],+ }),+ );+ expect(await isKeylessIpSuspicious("1.2.3.12")).toBe(false);+ });++ it("collapses a concurrent burst from one IP into a single Spur call", async () => {+ // Winner's fetch resolves after a tick so the other callers are guaranteed+ // to miss the lock and fall into the waiter path.+ const fetchFn = mockFetch(+ () =>+ new Promise(resolve =>+ setTimeout(+ () => resolve(okResponse({ ip: "1.2.3.6", risks: [] })),+ 50,+ ),+ ),+ );++ const results = await Promise.all(+ Array.from({ length: 5 }, () => isKeylessIpSuspicious("1.2.3.6")),+ );++ expect(fetchFn).toHaveBeenCalledTimes(1);+ expect(results).toEqual([false, false, false, false, false]);+ });++ it("fails open on a non-200 Spur response and caches nothing", async () => {+ const fetchFn = mockFetch(async () => ({ ok: false, status: 429 }));+ expect(await isKeylessIpSuspicious("1.2.3.7")).toBe(false);+ expect(fetchFn).toHaveBeenCalledTimes(1);+ expect(store.has("spur_context:1.2.3.7")).toBe(false);+ });++ it("fails open when the Spur call throws, and releases the lock so a later call retries", async () => {+ const fetchFn = mockFetch(async () => {+ throw new Error("network down");+ });+ expect(await isKeylessIpSuspicious("1.2.3.8")).toBe(false);++ // Lock must be freed: a subsequent call gets to fetch again.+ fetchFn.mockImplementation(async () =>+ okResponse({ ip: "1.2.3.8", tunnels: [{ type: "TOR" }] }),+ );+ expect(await isKeylessIpSuspicious("1.2.3.8")).toBe(true);+ expect(fetchFn).toHaveBeenCalledTimes(2);+ });++ it("a lock loser fails open within the bound when no result ever lands", async () => {+ // Pre-hold the lock under a different token so this caller can never win it,+ // and never populate the cache β it must give up (bounded) and fail open+ // without ever hitting Spur.+ store.set("spur_lock:9.9.9.9", "held-by-someone-else");+ const fetchFn = mockFetch(async () =>+ okResponse({ tunnels: [{ type: "VPN" }] }),+ );++ const start = Date.now();+ const result = await isKeylessIpSuspicious("9.9.9.9");+ const elapsed = Date.now() - start;++ expect(result).toBe(false);+ expect(fetchFn).not.toHaveBeenCalled();+ // Bounded wait: it gives up around LOCK_WAIT_MS, never hangs indefinitely.+ expect(elapsed).toBeGreaterThanOrEqual(3000);+ expect(elapsed).toBeLessThan(8000);+ });++ it("fails open and flags timedOut when the Spur call times out", async () => {+ mockFetch(async () => {+ // Shape of what AbortSignal.timeout rejects with.+ const err = new Error("The operation timed out.");+ err.name = "TimeoutError";+ throw err;+ });++ expect(await isKeylessIpSuspicious("1.2.3.13")).toBe(false);+ expect(logger.warn).toHaveBeenCalledWith(+ "Spur context lookup failed; failing open",+ expect.objectContaining({ ip: "1.2.3.13", timedOut: true }),+ );+ });++ it("is a no-op (and never calls Spur) when SPUR_API_KEY is unset", async () => {+ config.SPUR_API_KEY = undefined;+ const fetchFn = mockFetch(async () => okResponse({ tunnels: [{}] }));+ expect(await isKeylessIpSuspicious("1.2.3.9")).toBe(false);+ expect(fetchFn).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/lib/spur.ts b/apps/api/src/lib/spur.tsindex a61ee60cb..6ee073f92 100644--- a/apps/api/src/lib/spur.ts+++ b/apps/api/src/lib/spur.ts@@ -1,3 +1,4 @@+import { randomUUID } from "crypto";import { config } from "../config";import { logger } from "./logger";import { redisRateLimitClient } from "../services/rate-limiter";@@ -10,14 +11,35 @@ import { redisRateLimitClient } from "../services/rate-limiter";// refuse keyless for them and steer the caller to sign up for an API key.//// Lookups are cached in Redis for 30 days so a given IP costs at most one Spur-// API call per month. The integration is entirely optional: with no key set the-// keyless tier behaves exactly as before, and any Spur error fails open (the-// request is allowed) so a Spur outage can't take down the free tier.+// API call per month. Concurrent misses for the same IP are collapsed with a+// Redis single-flight lock so a burst from one IP triggers a single upstream+// call β the loser callers briefly wait for that result instead of each hitting+// Spur. The integration is entirely optional: with no key set the keyless tier+// behaves exactly as before, and any Spur error (timeout, lock failure, outage)+// fails open β the request is allowed β so Spur can never take down the free tier.const SPUR_API_BASE = "https://api.spur.us/v2/context";const CACHE_TTL_SECONDS = 30 * 24 * 60 * 60; // 30 daysconst cacheKey = (ip: string) => `spur_context:${ip}`;+// Everything below is bounded so a request can never get stuck on the upstream+// call, on holding the lock, or on waiting for another request's result.+//+// - FETCH_TIMEOUT_MS aborts the Spur HTTP call.+// - LOCK_TTL_MS is the single-flight lock's auto-expiry; it sits ~1s above the+// fetch timeout (headroom for the surrounding Redis round-trips) so the winner+// isn't preempted mid-fetch, and it caps how long a dead winner can block+// others (Redis expires the lock even if the process crashes before releasing).+// - LOCK_WAIT_MS caps how long a loser waits for the winner's result before+// failing open; POLL_INTERVAL_MS is how often it re-checks the result cache.+const FETCH_TIMEOUT_MS = 5000;+const LOCK_TTL_MS = FETCH_TIMEOUT_MS + 1000;+const LOCK_WAIT_MS = LOCK_TTL_MS;+const POLL_INTERVAL_MS = 200;+const lockKey = (ip: string) => `spur_lock:${ip}`;++const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));+// Subset of the Spur IP Context Object we read. See the API docs for the full// shape; everything here is optional because Spur omits empty fields.type SpurContext = {@@ -67,8 +89,20 @@ async function cacheContext(ip: string, ctx: SpurContext): Promise<void> {}}+// Compact summary of a context for logs β never dump the full object.+function summarize(ctx: SpurContext) {+ return {+ infrastructure: ctx.infrastructure,+ risks: ctx.risks,+ tunnels: ctx.tunnels?.map(t => t.type),+ proxies: ctx.client?.proxies,+ };+}+async function fetchContext(ip: string): Promise<SpurContext | null> {// Cache miss β hit the real Spur API. Logged so we can track real spend.+ // The request is bounded by FETCH_TIMEOUT_MS; an abort throws and the caller+ // fails open.logger.info("Spur Context API request (cache miss)", {canonicalLog: "spur/lookup",ip,@@ -76,6 +110,7 @@ async function fetchContext(ip: string): Promise<SpurContext | null> {const response = await fetch(`${SPUR_API_BASE}/${encodeURIComponent(ip)}`, {method: "GET",headers: { Token: config.SPUR_API_KEY! },+ signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),});if (!response.ok) {logger.warn("Spur Context API request failed", {@@ -85,13 +120,75 @@ async function fetchContext(ip: string): Promise<SpurContext | null> {});return null;}- return (await response.json()) as SpurContext;+ const ctx = (await response.json()) as SpurContext;+ logger.info("Spur Context API response", {+ canonicalLog: "spur/lookup",+ ip,+ ...summarize(ctx),+ });+ return ctx;+}++// Single-flight lock: the winner (returns a token) is the only caller that hits+// Spur for this IP; everyone else waits for its cached result. `null` means we+// lost the race. Redis errors propagate so the caller fails open rather than+// waiting on a lock we never held.+async function acquireLock(ip: string): Promise<string | null> {+ const token = randomUUID();+ const result = await redisRateLimitClient.set(+ lockKey(ip),+ token,+ "PX",+ LOCK_TTL_MS,+ "NX",+ );+ return result === "OK" ? token : null;+}++// Atomic compare-and-delete so we only ever clear our own lock β a plain+// get-then-del could delete a successor's lock if our lease expired in between.+// Best-effort: failures are swallowed since the PX TTL frees the lock regardless.+const RELEASE_LOCK_SCRIPT = `+if redis.call("get", KEYS[1]) == ARGV[1] then+ return redis.call("del", KEYS[1])+end+return 0`;++async function releaseLock(ip: string, token: string): Promise<void> {+ try {+ await redisRateLimitClient.eval(RELEASE_LOCK_SCRIPT, 1, lockKey(ip), token);+ } catch (error) {+ logger.warn("Failed to release Spur single-flight lock", {+ canonicalLog: "spur/lookup",+ ip,+ error,+ });+ }+}++// Loser path: poll for the winner's cached result every POLL_INTERVAL_MS,+// bounded by LOCK_WAIT_MS. If the result never lands (winner failed or is still+// running at the deadline), give up and fail open.+async function waitForResult(ip: string): Promise<SpurContext | null> {+ const deadline = Date.now() + LOCK_WAIT_MS;+ while (Date.now() < deadline) {+ // Clamp the last sleep to the remaining budget so we never overshoot the+ // LOCK_WAIT_MS ceiling.+ await sleep(Math.min(POLL_INTERVAL_MS, deadline - Date.now()));+ const cached = await getCachedContext(ip);+ if (cached) return cached;+ }+ return null;}/*** Look up an IP's Spur context, preferring the 30-day Redis cache and only- * caching successful (non-error) responses. Returns null when Spur is disabled- * or the lookup fails β callers then fail open (treat the IP as not suspicious).+ * caching successful (non-error) responses. Concurrent misses for the same IP+ * are collapsed via a Redis single-flight lock: one caller fetches, the rest+ * wait (bounded) for its result. Returns null when Spur is disabled or anything+ * goes wrong (lock error, timeout, upstream failure) β callers then fail open+ * (treat the IP as not suspicious). Every wait here is bounded, so a request can+ * never get stuck on the lock or on another request's in-flight lookup.*/async function getSpurContext(ip: string): Promise<SpurContext | null> {if (!isSpurEnabled()) return null;@@ -99,21 +196,47 @@ async function getSpurContext(ip: string): Promise<SpurContext | null> {const cached = await getCachedContext(ip);if (cached) return cached;- let ctx: SpurContext | null;try {- ctx = await fetchContext(ip);+ const token = await acquireLock(ip);+ if (!token) {+ // Lost the race β a lookup for this IP is already in flight. Wait for its+ // result instead of making our own call. Logged so we can measure how many+ // Spur calls the single-flight coalescing saves (savedApiCall === true).+ const result = await waitForResult(ip);+ logger.info("Spur lookup coalesced by single-flight", {+ canonicalLog: "spur/lookup",+ ip,+ savedApiCall: result !== null,+ });+ return result;+ }++ try {+ // Another winner may have populated the cache between our miss and+ // winning the lock β re-check before spending a Spur call.+ const fresh = await getCachedContext(ip);+ if (fresh) return fresh;++ const ctx = await fetchContext(ip);+ // Only cache non-error responses.+ if (ctx) await cacheContext(ip, ctx);+ return ctx;+ } finally {+ await releaseLock(ip, token);+ }} catch (error) {- logger.warn("Spur Context API request errored", {+ // Lock error, fetch timeout/abort, or any upstream failure β fail open.+ // AbortSignal.timeout rejects with a TimeoutError; flag it so the timeout+ // rate is countable separately from other failures.+ const timedOut = error instanceof Error && error.name === "TimeoutError";+ logger.warn("Spur context lookup failed; failing open", {canonicalLog: "spur/lookup",ip,+ timedOut,error,});return null;}-- // Only cache non-error responses.- if (ctx) await cacheContext(ip, ctx);- return ctx;}// Risk flags that, on their own, mark an IP as fronting proxy/tunnel