Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): single-flight Spur keyless IP lookups with bounded timeouts (#4330)

firecrawl-spring[bot]cc32c6eb6564
diff --git a/apps/api/src/lib/__tests__/spur.test.ts b/apps/api/src/lib/__tests__/spur.test.ts
new file mode 100644
index 000000000..3329f1203
--- /dev/null
+++ b/apps/api/src/lib/__tests__/spur.test.ts
@@ -0,0 +1,184 @@
+import type { Mock } from "vitest";
+import { isKeylessIpSuspicious } from "../spur";
+import { redisRateLimitClient } from "../../services/rate-limiter";
+import { config } from "../../config";
+import { logger } from "../logger";
+
+vi.mock("../../config", () => ({
+ config: { SPUR_API_KEY: "test-key" },
+}));
+
+vi.mock("../logger", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
+}));
+
+vi.mock("../../services/rate-limiter", () => ({
+ redisRateLimitClient: { get: vi.fn(), set: vi.fn(), eval: vi.fn() },
+}));
+
+// Minimal in-memory Redis backing the mock: enough SET semantics (NX guard,
+// ignored EX/PX) for the single-flight lock and the result cache.
+function installFakeRedis(): Map<string, string> {
+ const store = new Map<string, string>();
+ (redisRateLimitClient.get as Mock).mockImplementation(async (k: string) =>
+ store.has(k) ? store.get(k)! : null,
+ );
+ (redisRateLimitClient.set as Mock).mockImplementation(
+ async (k: string, v: string, ...args: unknown[]) => {
+ if (args.includes("NX") && store.has(k)) return null;
+ store.set(k, v);
+ return "OK";
+ },
+ );
+ // Atomic compare-and-delete used by releaseLock: KEYS[1]=lock key, ARGV[1]=token.
+ (redisRateLimitClient.eval as Mock).mockImplementation(
+ async (_script: string, _numKeys: number, key: string, token: string) => {
+ if (store.get(key) === token) {
+ store.delete(key);
+ return 1;
+ }
+ return 0;
+ },
+ );
+ return store;
+}
+
+function mockFetch(impl: () => Promise<unknown>): Mock {
+ const fn = vi.fn(async () => impl());
+ vi.stubGlobal("fetch", fn);
+ return fn;
+}
+
+const okResponse = (body: unknown) => ({ ok: true, json: async () => body });
+
+describe("Spur keyless IP reputation", () => {
+ let store: Map<string, string>;
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ vi.unstubAllGlobals();
+ config.SPUR_API_KEY = "test-key";
+ store = installFakeRedis();
+ });
+
+ it("flags an IP fronting a live VPN/proxy tunnel", async () => {
+ mockFetch(async () =>
+ okResponse({ ip: "1.2.3.4", tunnels: [{ type: "VPN" }] }),
+ );
+ expect(await isKeylessIpSuspicious("1.2.3.4")).toBe(true);
+ });
+
+ it("allows a clean (datacenter-only) IP", async () => {
+ mockFetch(async () =>
+ okResponse({ ip: "1.2.3.5", infrastructure: "DATACENTER", risks: [] }),
+ );
+ expect(await isKeylessIpSuspicious("1.2.3.5")).toBe(false);
+ });
+
+ it("flags residential-proxy and explicit proxy/tunnel risk contexts", async () => {
+ mockFetch(async () =>
+ okResponse({ ip: "1.2.3.10", client: { proxies: ["RESIDENTIAL"] } }),
+ );
+ expect(await isKeylessIpSuspicious("1.2.3.10")).toBe(true);
+
+ mockFetch(async () =>
+ okResponse({ ip: "1.2.3.11", risks: ["CALLBACK_PROXY"] }),
+ );
+ expect(await isKeylessIpSuspicious("1.2.3.11")).toBe(true);
+ });
+
+ it("does not flag geo-mismatch / datacenter-only signals", async () => {
+ mockFetch(async () =>
+ okResponse({
+ ip: "1.2.3.12",
+ infrastructure: "DATACENTER",
+ risks: ["GEO_MISMATCH"],
+ }),
+ );
+ expect(await isKeylessIpSuspicious("1.2.3.12")).toBe(false);
+ });
+
+ it("collapses a concurrent burst from one IP into a single Spur call", async () => {
+ // Winner's fetch resolves after a tick so the other callers are guaranteed
+ // to miss the lock and fall into the waiter path.
+ const fetchFn = mockFetch(
+ () =>
+ new Promise(resolve =>
+ setTimeout(
+ () => resolve(okResponse({ ip: "1.2.3.6", risks: [] })),
+ 50,
+ ),
+ ),
+ );
+
+ const results = await Promise.all(
+ Array.from({ length: 5 }, () => isKeylessIpSuspicious("1.2.3.6")),
+ );
+
+ expect(fetchFn).toHaveBeenCalledTimes(1);
+ expect(results).toEqual([false, false, false, false, false]);
+ });
+
+ it("fails open on a non-200 Spur response and caches nothing", async () => {
+ const fetchFn = mockFetch(async () => ({ ok: false, status: 429 }));
+ expect(await isKeylessIpSuspicious("1.2.3.7")).toBe(false);
+ expect(fetchFn).toHaveBeenCalledTimes(1);
+ expect(store.has("spur_context:1.2.3.7")).toBe(false);
+ });
+
+ it("fails open when the Spur call throws, and releases the lock so a later call retries", async () => {
+ const fetchFn = mockFetch(async () => {
+ throw new Error("network down");
+ });
+ expect(await isKeylessIpSuspicious("1.2.3.8")).toBe(false);
+
+ // Lock must be freed: a subsequent call gets to fetch again.
+ fetchFn.mockImplementation(async () =>
+ okResponse({ ip: "1.2.3.8", tunnels: [{ type: "TOR" }] }),
+ );
+ expect(await isKeylessIpSuspicious("1.2.3.8")).toBe(true);
+ expect(fetchFn).toHaveBeenCalledTimes(2);
+ });
+
+ it("a lock loser fails open within the bound when no result ever lands", async () => {
+ // Pre-hold the lock under a different token so this caller can never win it,
+ // and never populate the cache β†’ it must give up (bounded) and fail open
+ // without ever hitting Spur.
+ store.set("spur_lock:9.9.9.9", "held-by-someone-else");
+ const fetchFn = mockFetch(async () =>
+ okResponse({ tunnels: [{ type: "VPN" }] }),
+ );
+
+ const start = Date.now();
+ const result = await isKeylessIpSuspicious("9.9.9.9");
+ const elapsed = Date.now() - start;
+
+ expect(result).toBe(false);
+ expect(fetchFn).not.toHaveBeenCalled();
+ // Bounded wait: it gives up around LOCK_WAIT_MS, never hangs indefinitely.
+ expect(elapsed).toBeGreaterThanOrEqual(3000);
+ expect(elapsed).toBeLessThan(8000);
+ });
+
+ it("fails open and flags timedOut when the Spur call times out", async () => {
+ mockFetch(async () => {
+ // Shape of what AbortSignal.timeout rejects with.
+ const err = new Error("The operation timed out.");
+ err.name = "TimeoutError";
+ throw err;
+ });
+
+ expect(await isKeylessIpSuspicious("1.2.3.13")).toBe(false);
+ expect(logger.warn).toHaveBeenCalledWith(
+ "Spur context lookup failed; failing open",
+ expect.objectContaining({ ip: "1.2.3.13", timedOut: true }),
+ );
+ });
+
+ it("is a no-op (and never calls Spur) when SPUR_API_KEY is unset", async () => {
+ config.SPUR_API_KEY = undefined;
+ const fetchFn = mockFetch(async () => okResponse({ tunnels: [{}] }));
+ expect(await isKeylessIpSuspicious("1.2.3.9")).toBe(false);
+ expect(fetchFn).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/lib/spur.ts b/apps/api/src/lib/spur.ts
index a61ee60cb..6ee073f92 100644
--- a/apps/api/src/lib/spur.ts
+++ b/apps/api/src/lib/spur.ts
@@ -1,3 +1,4 @@
+import { randomUUID } from "crypto";
import { config } from "../config";
import { logger } from "./logger";
import { redisRateLimitClient } from "../services/rate-limiter";
@@ -10,14 +11,35 @@ import { redisRateLimitClient } from "../services/rate-limiter";
// refuse keyless for them and steer the caller to sign up for an API key.
//
// Lookups are cached in Redis for 30 days so a given IP costs at most one Spur
-// API call per month. The integration is entirely optional: with no key set the
-// keyless tier behaves exactly as before, and any Spur error fails open (the
-// request is allowed) so a Spur outage can't take down the free tier.
+// API call per month. Concurrent misses for the same IP are collapsed with a
+// Redis single-flight lock so a burst from one IP triggers a single upstream
+// call β€” the loser callers briefly wait for that result instead of each hitting
+// Spur. The integration is entirely optional: with no key set the keyless tier
+// behaves exactly as before, and any Spur error (timeout, lock failure, outage)
+// fails open β€” the request is allowed β€” so Spur can never take down the free tier.
const SPUR_API_BASE = "https://api.spur.us/v2/context";
const CACHE_TTL_SECONDS = 30 * 24 * 60 * 60; // 30 days
const cacheKey = (ip: string) => `spur_context:${ip}`;
+// Everything below is bounded so a request can never get stuck on the upstream
+// call, on holding the lock, or on waiting for another request's result.
+//
+// - FETCH_TIMEOUT_MS aborts the Spur HTTP call.
+// - LOCK_TTL_MS is the single-flight lock's auto-expiry; it sits ~1s above the
+// fetch timeout (headroom for the surrounding Redis round-trips) so the winner
+// isn't preempted mid-fetch, and it caps how long a dead winner can block
+// others (Redis expires the lock even if the process crashes before releasing).
+// - LOCK_WAIT_MS caps how long a loser waits for the winner's result before
+// failing open; POLL_INTERVAL_MS is how often it re-checks the result cache.
+const FETCH_TIMEOUT_MS = 5000;
+const LOCK_TTL_MS = FETCH_TIMEOUT_MS + 1000;
+const LOCK_WAIT_MS = LOCK_TTL_MS;
+const POLL_INTERVAL_MS = 200;
+const lockKey = (ip: string) => `spur_lock:${ip}`;
+
+const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+
// Subset of the Spur IP Context Object we read. See the API docs for the full
// shape; everything here is optional because Spur omits empty fields.
type SpurContext = {
@@ -67,8 +89,20 @@ async function cacheContext(ip: string, ctx: SpurContext): Promise<void> {
}
}
+// Compact summary of a context for logs β€” never dump the full object.
+function summarize(ctx: SpurContext) {
+ return {
+ infrastructure: ctx.infrastructure,
+ risks: ctx.risks,
+ tunnels: ctx.tunnels?.map(t => t.type),
+ proxies: ctx.client?.proxies,
+ };
+}
+
async function fetchContext(ip: string): Promise<SpurContext | null> {
// Cache miss β†’ hit the real Spur API. Logged so we can track real spend.
+ // The request is bounded by FETCH_TIMEOUT_MS; an abort throws and the caller
+ // fails open.
logger.info("Spur Context API request (cache miss)", {
canonicalLog: "spur/lookup",
ip,
@@ -76,6 +110,7 @@ async function fetchContext(ip: string): Promise<SpurContext | null> {
const response = await fetch(`${SPUR_API_BASE}/${encodeURIComponent(ip)}`, {
method: "GET",
headers: { Token: config.SPUR_API_KEY! },
+ signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
});
if (!response.ok) {
logger.warn("Spur Context API request failed", {
@@ -85,13 +120,75 @@ async function fetchContext(ip: string): Promise<SpurContext | null> {
});
return null;
}
- return (await response.json()) as SpurContext;
+ const ctx = (await response.json()) as SpurContext;
+ logger.info("Spur Context API response", {
+ canonicalLog: "spur/lookup",
+ ip,
+ ...summarize(ctx),
+ });
+ return ctx;
+}
+
+// Single-flight lock: the winner (returns a token) is the only caller that hits
+// Spur for this IP; everyone else waits for its cached result. `null` means we
+// lost the race. Redis errors propagate so the caller fails open rather than
+// waiting on a lock we never held.
+async function acquireLock(ip: string): Promise<string | null> {
+ const token = randomUUID();
+ const result = await redisRateLimitClient.set(
+ lockKey(ip),
+ token,
+ "PX",
+ LOCK_TTL_MS,
+ "NX",
+ );
+ return result === "OK" ? token : null;
+}
+
+// Atomic compare-and-delete so we only ever clear our own lock β€” a plain
+// get-then-del could delete a successor's lock if our lease expired in between.
+// Best-effort: failures are swallowed since the PX TTL frees the lock regardless.
+const RELEASE_LOCK_SCRIPT = `
+if redis.call("get", KEYS[1]) == ARGV[1] then
+ return redis.call("del", KEYS[1])
+end
+return 0`;
+
+async function releaseLock(ip: string, token: string): Promise<void> {
+ try {
+ await redisRateLimitClient.eval(RELEASE_LOCK_SCRIPT, 1, lockKey(ip), token);
+ } catch (error) {
+ logger.warn("Failed to release Spur single-flight lock", {
+ canonicalLog: "spur/lookup",
+ ip,
+ error,
+ });
+ }
+}
+
+// Loser path: poll for the winner's cached result every POLL_INTERVAL_MS,
+// bounded by LOCK_WAIT_MS. If the result never lands (winner failed or is still
+// running at the deadline), give up and fail open.
+async function waitForResult(ip: string): Promise<SpurContext | null> {
+ const deadline = Date.now() + LOCK_WAIT_MS;
+ while (Date.now() < deadline) {
+ // Clamp the last sleep to the remaining budget so we never overshoot the
+ // LOCK_WAIT_MS ceiling.
+ await sleep(Math.min(POLL_INTERVAL_MS, deadline - Date.now()));
+ const cached = await getCachedContext(ip);
+ if (cached) return cached;
+ }
+ return null;
}
/**
* Look up an IP's Spur context, preferring the 30-day Redis cache and only
- * caching successful (non-error) responses. Returns null when Spur is disabled
- * or the lookup fails β€” callers then fail open (treat the IP as not suspicious).
+ * caching successful (non-error) responses. Concurrent misses for the same IP
+ * are collapsed via a Redis single-flight lock: one caller fetches, the rest
+ * wait (bounded) for its result. Returns null when Spur is disabled or anything
+ * goes wrong (lock error, timeout, upstream failure) β€” callers then fail open
+ * (treat the IP as not suspicious). Every wait here is bounded, so a request can
+ * never get stuck on the lock or on another request's in-flight lookup.
*/
async function getSpurContext(ip: string): Promise<SpurContext | null> {
if (!isSpurEnabled()) return null;
@@ -99,21 +196,47 @@ async function getSpurContext(ip: string): Promise<SpurContext | null> {
const cached = await getCachedContext(ip);
if (cached) return cached;
- let ctx: SpurContext | null;
try {
- ctx = await fetchContext(ip);
+ const token = await acquireLock(ip);
+ if (!token) {
+ // Lost the race β†’ a lookup for this IP is already in flight. Wait for its
+ // result instead of making our own call. Logged so we can measure how many
+ // Spur calls the single-flight coalescing saves (savedApiCall === true).
+ const result = await waitForResult(ip);
+ logger.info("Spur lookup coalesced by single-flight", {
+ canonicalLog: "spur/lookup",
+ ip,
+ savedApiCall: result !== null,
+ });
+ return result;
+ }
+
+ try {
+ // Another winner may have populated the cache between our miss and
+ // winning the lock β€” re-check before spending a Spur call.
+ const fresh = await getCachedContext(ip);
+ if (fresh) return fresh;
+
+ const ctx = await fetchContext(ip);
+ // Only cache non-error responses.
+ if (ctx) await cacheContext(ip, ctx);
+ return ctx;
+ } finally {
+ await releaseLock(ip, token);
+ }
} catch (error) {
- logger.warn("Spur Context API request errored", {
+ // Lock error, fetch timeout/abort, or any upstream failure β†’ fail open.
+ // AbortSignal.timeout rejects with a TimeoutError; flag it so the timeout
+ // rate is countable separately from other failures.
+ const timedOut = error instanceof Error && error.name === "TimeoutError";
+ logger.warn("Spur context lookup failed; failing open", {
canonicalLog: "spur/lookup",
ip,
+ timedOut,
error,
});
return null;
}
-
- // Only cache non-error responses.
- if (ctx) await cacheContext(ip, ctx);
- return ctx;
}
// Risk flags that, on their own, mark an IP as fronting proxy/tunnel